From 35fa5ab1e6630afd6a30fa4bbdd0dfc8658530a5 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Mon, 30 Dec 2024 11:36:31 +0100 Subject: [PATCH 001/117] perf: improve preprocess action perf by capturing context instead of partially resolving template strings --- core/package.json | 4 +- core/src/commands/custom.ts | 18 +- core/src/commands/workflow.ts | 14 +- core/src/config/base.ts | 34 +- core/src/config/config-template.ts | 12 +- core/src/config/constants.ts | 21 +- core/src/config/project.ts | 42 +- core/src/config/provider.ts | 3 +- core/src/config/references.ts | 227 +++++ core/src/config/render-template.ts | 54 +- core/src/config/secrets.ts | 117 +++ core/src/config/template-contexts/actions.ts | 15 +- core/src/config/template-contexts/base.ts | 37 +- core/src/config/template-contexts/module.ts | 18 +- core/src/config/validation.ts | 8 +- core/src/config/workflow.ts | 24 +- core/src/docs/template-strings.ts | 2 +- core/src/exceptions.ts | 33 - core/src/garden.ts | 19 +- core/src/graph/actions.ts | 111 +-- core/src/outputs.ts | 27 +- core/src/plugin-context.ts | 2 +- core/src/plugin/handlers/base/configure.ts | 2 +- core/src/plugins/hadolint/hadolint.ts | 2 +- core/src/resolve-module.ts | 72 +- core/src/router/base.ts | 1 - core/src/tasks/publish.ts | 3 +- core/src/tasks/resolve-action.ts | 63 +- core/src/tasks/resolve-provider.ts | 4 +- core/src/template-string/parser.d.ts | 14 - core/src/template-string/template-string.ts | 903 ------------------ core/src/template-string/types.ts | 19 - .../analysis.ts} | 43 +- core/src/{template-string => template}/ast.ts | 99 +- core/src/template/capture.ts | 60 ++ core/src/template/errors.ts | 61 ++ core/src/template/evaluate.ts | 67 ++ .../functions/date.ts} | 4 +- .../functions/index.ts} | 14 +- .../parser.pegjs | 0 core/src/template/templated-collections.ts | 465 +++++++++ core/src/template/templated-strings.ts | 227 +++++ core/src/template/types.ts | 65 ++ core/test/unit/src/config/service.ts | 3 +- .../unit/src/config/template-contexts/base.ts | 35 +- .../src/config/template-contexts/module.ts | 1 - .../src/config/template-contexts/project.ts | 2 +- core/test/unit/src/template-string.ts | 887 +++++++++-------- core/test/unit/src/types/container.ts | 4 +- plugins/pulumi/src/commands.ts | 1 - 50 files changed, 2061 insertions(+), 1902 deletions(-) create mode 100644 core/src/config/references.ts create mode 100644 core/src/config/secrets.ts delete mode 100644 core/src/template-string/parser.d.ts delete mode 100644 core/src/template-string/template-string.ts delete mode 100644 core/src/template-string/types.ts rename core/src/{template-string/static-analysis.ts => template/analysis.ts} (79%) rename core/src/{template-string => template}/ast.ts (88%) create mode 100644 core/src/template/capture.ts create mode 100644 core/src/template/errors.ts create mode 100644 core/src/template/evaluate.ts rename core/src/{template-string/date-functions.ts => template/functions/date.ts} (98%) rename core/src/{template-string/functions.ts => template/functions/index.ts} (97%) rename core/src/{template-string => template}/parser.pegjs (100%) create mode 100644 core/src/template/templated-collections.ts create mode 100644 core/src/template/templated-strings.ts create mode 100644 core/src/template/types.ts diff --git a/core/package.json b/core/package.json index 148abd4926..c1856d0b5b 100644 --- a/core/package.json +++ b/core/package.json @@ -251,11 +251,11 @@ "utility-types": "^3.11.0" }, "scripts": { - "build": "mkdir -p build/src/template-string && peggy src/template-string/parser.pegjs --output build/src/template-string/parser.js --format es", + "build": "mkdir -p build/src/template && peggy src/template/parser.pegjs --output build/src/template/parser.js --format es", "check-package-lock": "git diff-index --quiet HEAD -- package-lock.json || (echo 'package-lock.json is dirty!' && exit 1)", "check-types": "tsc -p . --noEmit", "clean": "shx rm -rf build", - "dev": "nodemon --watch src/template-string/*.pegjs --exec npm run build", + "dev": "nodemon --watch src/template/*.pegjs --exec npm run build", "fix-format": "npm run lint -- --fix --quiet", "lint": "eslint --ignore-pattern 'src/lib/**' --ext .ts,.tsx src/ test/", "migration:generate": "typeorm migration:generate --config ormconfig.js -n", diff --git a/core/src/commands/custom.ts b/core/src/commands/custom.ts index b0310ba955..86f7e678b5 100644 --- a/core/src/commands/custom.ts +++ b/core/src/commands/custom.ts @@ -21,7 +21,7 @@ import { CustomCommandContext } from "../config/template-contexts/custom-command import { validateWithPath } from "../config/validation.js" import type { GardenError } from "../exceptions.js" import { ConfigurationError, RuntimeError, InternalError, toGardenError } from "../exceptions.js" -import { resolveTemplateStrings } from "../template-string/template-string.js" +import { resolveTemplateStrings } from "../template/templated-strings.js" import { listDirectory, isConfigFilename } from "../util/fs.js" import type { CommandParams, CommandResult, PrintHeaderParams } from "./base.js" import { Command } from "./base.js" @@ -32,6 +32,7 @@ import { getBuiltinCommands } from "./commands.js" import type { Log } from "../logger/log-entry.js" import { getTracePropagationEnvVars } from "../util/open-telemetry/propagation.js" import { styles } from "../logger/styles.js" +import { deepEvaluate } from "../template/evaluate.js" function convertArgSpec(spec: CustomCommandOption) { const params = { @@ -116,10 +117,9 @@ export class CustomCommandWrapper extends Command { // Render the command variables const variablesContext = new CustomCommandContext({ ...garden, args, opts, rest }) - const commandVariables = resolveTemplateStrings({ - value: this.spec.variables, + const commandVariables = deepEvaluate(this.spec.variables, { context: variablesContext, - source: { yamlDoc, path: ["variables"] }, + opts: {}, }) const variables: any = jsonMerge(cloneDeep(garden.variables), commandVariables) @@ -134,10 +134,9 @@ export class CustomCommandWrapper extends Command { const startedAt = new Date() const exec = validateWithPath({ - config: resolveTemplateStrings({ - value: this.spec.exec, + config: deepEvaluate(this.spec.exec, { context: commandContext, - source: { yamlDoc, path: ["exec"] }, + opts: {}, }), schema: customCommandExecSchema(), path: this.spec.internal.basePath, @@ -186,10 +185,9 @@ export class CustomCommandWrapper extends Command { const startedAt = new Date() let gardenCommand = validateWithPath({ - config: resolveTemplateStrings({ - value: this.spec.gardenCommand, + config: deepEvaluate(this.spec.gardenCommand, { context: commandContext, - source: { yamlDoc, path: ["gardenCommand"] }, + opts: {}, }), schema: customCommandGardenCommandSchema(), path: this.spec.internal.basePath, diff --git a/core/src/commands/workflow.ts b/core/src/commands/workflow.ts index d03e671a1a..87e55c72b5 100644 --- a/core/src/commands/workflow.ts +++ b/core/src/commands/workflow.ts @@ -19,7 +19,7 @@ import type { GardenError } from "../exceptions.js" import { ChildProcessError, InternalError, RuntimeError, WorkflowScriptError, toGardenError } from "../exceptions.js" import type { WorkflowStepResult } from "../config/template-contexts/workflow.js" import { WorkflowConfigContext, WorkflowStepConfigContext } from "../config/template-contexts/workflow.js" -import { resolveTemplateStrings, resolveTemplateString } from "../template-string/template-string.js" +import { resolveTemplateStrings, resolveTemplateString } from "../template/templated-strings.js" import { ConfigurationError, FilesystemError } from "../exceptions.js" import { posix, join } from "path" import fsExtra from "fs-extra" @@ -34,6 +34,8 @@ import type { GardenCli } from "../cli/cli.js" import { getCustomCommands } from "./custom.js" import { getBuiltinCommands } from "./commands.js" import { styles } from "../logger/styles.js" +import { deepEvaluate } from "../template/evaluate.js" +import { ParsedTemplate } from "../template/types.js" const { ensureDir, writeFile } = fsExtra @@ -88,10 +90,9 @@ export class WorkflowCommand extends Command { garden.events.emit("workflowRunning", {}) const templateContext = new WorkflowConfigContext(garden, garden.variables) const yamlDoc = workflow.internal.yamlDoc - const files = resolveTemplateStrings({ - value: workflow.files || [], + const files = deepEvaluate((workflow.files || []) as unknown as ParsedTemplate[], { context: templateContext, - source: { yamlDoc, path: ["files"] }, + opts: {}, }) // Write all the configured files for the workflow @@ -173,10 +174,9 @@ export class WorkflowCommand extends Command { try { if (step.command) { - step.command = resolveTemplateStrings({ - value: step.command, + step.command = deepEvaluate(step.command, { context: stepTemplateContext, - source: { yamlDoc, path: ["steps", index, "command"] }, + opts: {}, }).filter((arg) => !!arg) stepResult = await runStepCommand(stepParams) diff --git a/core/src/config/base.ts b/core/src/config/base.ts index 597869e96a..e3ab769981 100644 --- a/core/src/config/base.ts +++ b/core/src/config/base.ts @@ -26,7 +26,7 @@ import { createSchema, joi } from "./common.js" import { emitNonRepeatableWarning } from "../warnings.js" import type { ActionKind, BaseActionConfig } from "../actions/types.js" import { actionKinds } from "../actions/types.js" -import { mayContainTemplateString } from "../template-string/template-string.js" +import { mayContainTemplateString } from "../template/templated-strings.js" import type { Log } from "../logger/log-entry.js" import type { Document, DocumentOptions } from "yaml" import { parseAllDocuments } from "yaml" @@ -35,6 +35,7 @@ import { makeDocsLinkStyled } from "../docs/common.js" import { profileAsync } from "../util/profiling.js" import { readFile } from "fs/promises" import { LRUCache } from "lru-cache" +import { parseTemplateCollection } from "../template/templated-collections.js" export const configTemplateKind = "ConfigTemplate" export const renderTemplateKind = "RenderTemplate" @@ -218,7 +219,15 @@ export async function validateRawConfig({ .map((s) => { const relPath = relative(projectRoot, configPath) const description = `config at ${relPath}` - return prepareResource({ log, doc: s, configFilePath: configPath, projectRoot, description, allowInvalid }) + return prepareResource({ + log, + doc: s, + spec: s.toJS(), + configFilePath: configPath, + projectRoot, + description, + allowInvalid, + }) }) .filter(isNotNull) return resources @@ -240,19 +249,21 @@ export function prepareResource({ configFilePath, projectRoot, description, + spec, allowInvalid = false, + parse = false, }: { log: Log - doc: YamlDocumentWithSource + spec: any + doc: YamlDocumentWithSource | undefined configFilePath: string projectRoot: string description: string allowInvalid?: boolean + parse?: boolean }): GardenResource | ModuleConfig | null { const relPath = relative(projectRoot, configFilePath) - const spec = doc.toJS() - if (spec === null) { return null } @@ -282,6 +293,19 @@ export function prepareResource({ } } + if (parse) { + for (const k in spec) { + // TODO: should we do this here? would be good to do it as early as possible. + spec[k] = parseTemplateCollection({ + value: spec[k], + source: { + yamlDoc: doc, + path: [k], + }, + }) + } + } + // Allow this for backwards compatibility if (kind === "ModuleTemplate") { spec.kind = kind = configTemplateKind diff --git a/core/src/config/config-template.ts b/core/src/config/config-template.ts index 82a99e682f..3cd5f85d2f 100644 --- a/core/src/config/config-template.ts +++ b/core/src/config/config-template.ts @@ -13,7 +13,6 @@ import { baseModuleSpecSchema } from "./module.js" import { dedent, naturalList } from "../util/string.js" import type { BaseGardenResource } from "./base.js" import { configTemplateKind, renderTemplateKind, baseInternalFieldsSchema } from "./base.js" -import { resolveTemplateStrings } from "../template-string/template-string.js" import { validateConfig } from "./validation.js" import type { Garden } from "../garden.js" import { ConfigurationError } from "../exceptions.js" @@ -24,6 +23,8 @@ import { ProjectConfigContext } from "./template-contexts/project.js" import type { ActionConfig } from "../actions/types.js" import { actionKinds } from "../actions/types.js" import type { WorkflowConfig } from "./workflow.js" +import { deepEvaluate } from "../template/evaluate.js" +import type { ParsedTemplate } from "../template/types.js" const inputTemplatePattern = "${inputs.*}" const parentNameTemplate = "${parent.name}" @@ -67,16 +68,15 @@ export async function resolveConfigTemplate( const loggedIn = garden.isLoggedIn() const enterpriseDomain = garden.cloudApi?.domain const context = new ProjectConfigContext({ ...garden, loggedIn, enterpriseDomain }) - const resolved = resolveTemplateStrings({ - value: partial, + const resolved = deepEvaluate(partial as unknown as ParsedTemplate, { context, - source: { yamlDoc: resource.internal.yamlDoc, path: [] }, + opts: {}, }) const configPath = resource.internal.configFilePath // Validate the partial config - const validated = validateConfig({ - config: resolved, + const validated = validateConfig({ + config: resolved as unknown as BaseGardenResource, schema: configTemplateSchema(), projectRoot: garden.projectRoot, yamlDocBasePath: [], diff --git a/core/src/config/constants.ts b/core/src/config/constants.ts index c28c49f9a5..a74be9eaf1 100644 --- a/core/src/config/constants.ts +++ b/core/src/config/constants.ts @@ -15,18 +15,17 @@ export const arrayForEachKey = "$forEach" export const arrayForEachReturnKey = "$return" export const arrayForEachFilterKey = "$filter" +const specialKeys = [ + objectSpreadKey, + conditionalKey, + conditionalThenKey, + conditionalElseKey, + arrayConcatKey, + arrayForEachKey, + arrayForEachReturnKey, + arrayForEachFilterKey, +] export function isSpecialKey(input: string): boolean { - const specialKeys = [ - objectSpreadKey, - conditionalKey, - conditionalThenKey, - conditionalElseKey, - arrayConcatKey, - arrayForEachKey, - arrayForEachReturnKey, - arrayForEachFilterKey, - ] - return specialKeys.some((key) => input === key) } diff --git a/core/src/config/project.ts b/core/src/config/project.ts index cf33c8f930..1d52acf6d7 100644 --- a/core/src/config/project.ts +++ b/core/src/config/project.ts @@ -22,7 +22,7 @@ import { joiVariablesDescription, } from "./common.js" import { validateConfig, validateWithPath } from "./validation.js" -import { resolveTemplateStrings } from "../template-string/template-string.js" +import { deepEvaluate } from "../template/evaluate.js" import { EnvironmentConfigContext, ProjectConfigContext } from "./template-contexts/project.js" import { findByName, getNames } from "../util/util.js" import { ConfigurationError, ParameterError, ValidationError } from "../exceptions.js" @@ -41,6 +41,7 @@ import { baseInternalFieldsSchema, loadVarfile, varfileDescription } from "./bas import type { Log } from "../logger/log-entry.js" import { renderDivider } from "../logger/util.js" import { styles } from "../logger/styles.js" +import type { ParsedTemplate } from "../template/types.js" export const defaultVarfilePath = "garden.env" export const defaultEnvVarfilePath = (environmentName: string) => `garden.${environmentName}.env` @@ -482,27 +483,29 @@ export function resolveProjectConfig({ let globalConfig: any try { - globalConfig = resolveTemplateStrings({ - value: { + globalConfig = deepEvaluate( + { apiVersion: config.apiVersion, varfile: config.varfile, variables: config.variables, environments: [], sources: [], }, - context: new ProjectConfigContext({ - projectName: name, - projectRoot: config.path, - artifactsPath, - vcsInfo, - username, - loggedIn, - enterpriseDomain, - secrets, - commandInfo, - }), - source: { yamlDoc: config.internal.yamlDoc, path: [] }, - }) + { + context: new ProjectConfigContext({ + projectName: name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn, + enterpriseDomain, + secrets, + commandInfo, + }), + opts: {}, + } + ) } catch (err) { log.error("Failed to resolve project configuration.") log.error(styles.bold(renderDivider())) @@ -625,8 +628,7 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ const source = { yamlDoc: projectConfig.internal.yamlDoc, path: ["environments", index] } // Resolve template strings in the environment config, except providers - environmentConfig = resolveTemplateStrings({ - value: { ...environmentConfig }, + const config = deepEvaluate(environmentConfig as unknown as ParsedTemplate, { context: new EnvironmentConfigContext({ projectName, projectRoot, @@ -639,11 +641,11 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ secrets, commandInfo, }), - source, + opts: {}, }) environmentConfig = validateWithPath({ - config: environmentConfig, + config, schema: environmentSchema(), configType: `environment ${environment}`, path: projectConfig.path, diff --git a/core/src/config/provider.ts b/core/src/config/provider.ts index 889cc10215..684093d63b 100644 --- a/core/src/config/provider.ts +++ b/core/src/config/provider.ts @@ -29,7 +29,7 @@ import type { ActionState } from "../actions/types.js" import type { ValidResultType } from "../tasks/base.js" import { uuidv4 } from "../util/random.js" import { s } from "./zod.js" -import { getContextLookupReferences, visitAll } from "../template-string/static-analysis.js" +import { getContextLookupReferences, visitAll } from "../template/analysis.js" import type { ConfigContext } from "./template-contexts/base.js" // TODO: dedupe from the joi schema below @@ -186,7 +186,6 @@ export function getProviderTemplateReferences(config: GenericProviderConfig, con const generator = getContextLookupReferences( visitAll({ value: config, - parseTemplateStrings: true, // TODO: get proper source source: { path: [], diff --git a/core/src/config/references.ts b/core/src/config/references.ts new file mode 100644 index 0000000000..9e8db1287b --- /dev/null +++ b/core/src/config/references.ts @@ -0,0 +1,227 @@ +/* + * Copyright (C) 2018-2024 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import isString from "lodash-es/isString.js" +import { + getContextLookupReferences, + isUnresolvableValue, + visitAll, + type ContextLookupReferenceFinding, + type UnresolvableValue, +} from "../template/analysis.js" +import { TemplateError } from "../template/errors.js" +import type { ActionReference } from "./common.js" +import type { ConfigContext, ContextKeySegment } from "./template-contexts/base.js" +import type { ActionConfig, ActionKind } from "../actions/types.js" +import { actionKindsLower } from "../actions/types.js" +import { titleize } from "../util/string.js" +import type { ObjectWithName } from "../util/util.js" +import type { ModuleConfig } from "./module.js" +import type { ModuleConfigContext } from "./template-contexts/module.js" + +interface ActionTemplateReference extends ActionReference { + keyPath: (ContextKeySegment | UnresolvableValue)[] +} + +export function extractActionReference(finding: ContextLookupReferenceFinding): ActionTemplateReference { + const kind = finding.keyPath[1] + if (!kind) { + throw new TemplateError({ + message: `Found invalid action reference (missing kind).`, + source: finding.yamlSource, + }) + } + + if (isUnresolvableValue(kind)) { + const err = kind.getError() + throw new TemplateError({ + message: `Found invalid action reference: ${err.message}`, + source: finding.yamlSource, + }) + } + + if (!isString(kind)) { + throw new TemplateError({ + message: `Found invalid action reference (kind is not a string).`, + source: finding.yamlSource, + }) + } + + if (!actionKindsLower.includes(kind)) { + throw new TemplateError({ + message: `Found invalid action reference (invalid kind '${kind}')`, + source: finding.yamlSource, + }) + } + + const name = finding.keyPath[2] + if (!name) { + throw new TemplateError({ + message: "Found invalid action reference (missing name)", + source: finding.yamlSource, + }) + } + + if (isUnresolvableValue(name)) { + const err = name.getError() + throw new TemplateError({ + message: `Found invalid action reference: ${err.message}`, + source: finding.yamlSource, + }) + } + + if (!isString(name)) { + throw new TemplateError({ + message: "Found invalid action reference (name is not a string)", + source: finding.yamlSource, + }) + } + + return { + kind: titleize(kind), + name, + keyPath: finding.keyPath.slice(3), + } +} + +export function extractRuntimeReference(finding: ContextLookupReferenceFinding): ActionTemplateReference { + const runtimeKind = finding.keyPath[1] + if (!runtimeKind) { + throw new TemplateError({ + message: "Found invalid runtime reference (missing kind)", + source: finding.yamlSource, + }) + } + + if (isUnresolvableValue(runtimeKind)) { + const err = runtimeKind.getError() + throw new TemplateError({ + message: `Found invalid runtime reference: ${err.message}`, + source: finding.yamlSource, + }) + } + + if (!isString(runtimeKind)) { + throw new TemplateError({ + message: "Found invalid runtime reference (kind is not a string)", + source: finding.yamlSource, + }) + } + + let kind: ActionKind + if (runtimeKind === "services") { + kind = "Deploy" + } else if (runtimeKind === "tasks") { + kind = "Run" + } else { + throw new TemplateError({ + message: `Found invalid runtime reference (invalid kind '${runtimeKind}')`, + source: finding.yamlSource, + }) + } + + const name = finding.keyPath[2] + + if (!name) { + throw new TemplateError({ + message: `Found invalid runtime reference (missing name)`, + source: finding.yamlSource, + }) + } + + if (isUnresolvableValue(name)) { + const err = name.getError() + throw new TemplateError({ + message: `Found invalid action reference: ${err.message}`, + source: finding.yamlSource, + }) + } + + if (!isString(name)) { + throw new TemplateError({ + message: "Found invalid runtime reference (name is not a string)", + source: finding.yamlSource, + }) + } + + return { + kind, + name, + keyPath: finding.keyPath.slice(3), + } +} + +/** + * Collects every reference to another action in the given config object, including translated runtime.* references. + * An error is thrown if a reference is not resolvable, i.e. if a nested template is used as a reference. + */ +export function* getActionTemplateReferences( + config: ActionConfig, + context: ConfigContext +): Generator { + const generator = getContextLookupReferences( + visitAll({ + value: config as ObjectWithName, + source: { + yamlDoc: config.internal?.yamlDoc, + path: [], + }, + }), + context + ) + + for (const finding of generator) { + const refType = finding.keyPath[0] + // ${action.*} + if (refType === "actions") { + yield extractActionReference(finding) + } + // ${runtime.*} + if (refType === "runtime") { + yield extractRuntimeReference(finding) + } + } +} + +export function getModuleTemplateReferences(config: ModuleConfig, context: ModuleConfigContext) { + const moduleNames: string[] = [] + const generator = getContextLookupReferences( + visitAll({ + value: config as ObjectWithName, + // Note: We're not implementing the YAML source mapping for modules + source: { + path: [], + }, + }), + context + ) + + for (const finding of generator) { + const keyPath = finding.keyPath + if (keyPath[0] !== "modules") { + continue + } + + const moduleName = keyPath[1] + if (isUnresolvableValue(moduleName)) { + const err = moduleName.getError() + throw new TemplateError({ + message: `Found invalid module reference: ${err.message}`, + source: finding.yamlSource, + }) + } + + if (config.name === moduleName) { + continue + } + + moduleNames.push(moduleName.toString()) + } + + return moduleNames +} diff --git a/core/src/config/render-template.ts b/core/src/config/render-template.ts index a1ff67410e..4c96eac676 100644 --- a/core/src/config/render-template.ts +++ b/core/src/config/render-template.ts @@ -17,11 +17,7 @@ import { prepareResource, renderTemplateKind, } from "./base.js" -import { - maybeTemplateString, - resolveTemplateString, - resolveTemplateStrings, -} from "../template-string/template-string.js" +import { maybeTemplateString, resolveTemplateString, resolveTemplateStrings } from "../template/templated-strings.js" import { validateWithPath } from "./validation.js" import type { Garden } from "../garden.js" import { ConfigurationError, GardenError } from "../exceptions.js" @@ -39,6 +35,10 @@ import type { DeepPrimitiveMap } from "@garden-io/platform-api-types" import { RenderTemplateConfigContext } from "./template-contexts/render.js" import type { Log } from "../logger/log-entry.js" import { GardenApiVersion } from "../constants.js" +import { capture } from "../template/capture.js" +import { deepEvaluate } from "../template/evaluate.js" +import { ParsedTemplate, TemplatePrimitive } from "../template/types.js" +import { CollectionOrValue } from "../util/objects.js" export const renderTemplateConfigSchema = createSchema({ name: renderTemplateKind, @@ -127,24 +127,13 @@ export async function renderConfigTemplate({ const enterpriseDomain = garden.cloudApi?.domain const templateContext = new EnvironmentConfigContext({ ...garden, loggedIn, enterpriseDomain }) - const yamlDoc = config.internal.yamlDoc - - const resolvedWithoutInputs = resolveTemplateStrings({ - value: { ...omit(config, "inputs") }, + const resolvedWithoutInputs = deepEvaluate(omit(config, "inputs") as unknown as ParsedTemplate, { context: templateContext, - source: { yamlDoc, path: [] }, - }) - const partiallyResolvedInputs = resolveTemplateStrings({ - value: config.inputs || {}, - context: templateContext, - contextOpts: { - allowPartial: true, - }, - source: { yamlDoc, path: ["inputs"] }, + opts: {}, }) let resolved: RenderTemplateConfig = { - ...resolvedWithoutInputs, - inputs: partiallyResolvedInputs, + ...(resolvedWithoutInputs as unknown as RenderTemplateConfig), + inputs: capture(config.inputs || {}, templateContext), } const configType = "Render " + resolved.name @@ -183,7 +172,7 @@ export async function renderConfigTemplate({ enterpriseDomain, parentName: resolved.name, templateName: template.name, - inputs: partiallyResolvedInputs, + inputs: config.inputs || {}, }) // TODO: remove in 0.14 @@ -205,17 +194,10 @@ async function renderModules({ context: RenderTemplateConfigContext renderConfig: RenderTemplateConfig }): Promise { - const yamlDoc = template.internal.yamlDoc - return Promise.all( - (template.modules || []).map(async (m, i) => { + (template.modules || []).map(async (m) => { // Run a partial template resolution with the parent+template info - const spec = resolveTemplateStrings({ - value: m, - context, - contextOpts: { allowPartial: true }, - source: { yamlDoc, path: ["modules", i] }, - }) + const spec = capture(m as unknown as ParsedTemplate, context) as unknown as ModuleConfig const renderConfigPath = renderConfig.internal.configFilePath || renderConfig.internal.basePath let moduleConfig: ModuleConfig @@ -277,15 +259,9 @@ async function renderConfigs({ const templateDescription = `${configTemplateKind} '${template.name}'` const templateConfigs = template.configs || [] - const partiallyResolvedTemplateConfigs = resolveTemplateStrings({ - value: templateConfigs, - context, - contextOpts: { allowPartial: true }, - source, - }) return Promise.all( - partiallyResolvedTemplateConfigs.map(async (m, index) => { + templateConfigs.map(async (m, index) => { // Resolve just the name, which must be immediately resolvable let resolvedName = m.name @@ -293,7 +269,6 @@ async function renderConfigs({ resolvedName = resolveTemplateString({ string: m.name, context, - contextOpts: { allowPartial: false }, source: { ...source, path: [...source.path, index, "name"] }, }) as string } catch (error) { @@ -327,7 +302,8 @@ async function renderConfigs({ try { resource = prepareResource({ log, - doc: new Document(spec) as YamlDocumentWithSource, + spec, + doc: undefined, configFilePath: renderConfigPath, projectRoot: garden.projectRoot, description: `resource in Render ${renderConfig.name}`, diff --git a/core/src/config/secrets.ts b/core/src/config/secrets.ts new file mode 100644 index 0000000000..cd12b8eadc --- /dev/null +++ b/core/src/config/secrets.ts @@ -0,0 +1,117 @@ +/* + * Copyright (C) 2018-2024 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import isString from "lodash-es/isString.js" +import type { Log } from "../logger/log-entry.js" +import { getContextLookupReferences, visitAll } from "../template/analysis.js" +import { dedent, deline } from "../util/string.js" +import type { ObjectWithName } from "../util/util.js" +import type { StringMap } from "./common.js" +import type { ConfigContext, ContextKeySegment } from "./template-contexts/base.js" +import difference from "lodash-es/difference.js" + +/** + * Gathers secret references in configs and throws an error if one or more referenced secrets isn't present (or has + * blank values) in the provided secrets map. + * + * Prefix should be e.g. "Module" or "Provider" (used when generating error messages). + */ +export function throwOnMissingSecretKeys( + configs: ObjectWithName[], + context: ConfigContext, + secrets: StringMap, + prefix: string, + log?: Log +) { + const allMissing: [string, ContextKeySegment[]][] = [] // [[key, missing keys]] + for (const config of configs) { + const missing = detectMissingSecretKeys(config, context, secrets) + if (missing.length > 0) { + allMissing.push([config.name, missing]) + } + } + + if (allMissing.length === 0) { + return + } + + const descriptions = allMissing.map(([key, missing]) => `${prefix} ${key}: ${missing.join(", ")}`) + /** + * Secret keys with empty values should have resulted in an error by this point, but we filter on keys with + * values for good measure. + */ + const loadedKeys = Object.entries(secrets) + .filter(([_key, value]) => value) + .map(([key, _value]) => key) + let footer: string + if (loadedKeys.length === 0) { + footer = deline` + Note: No secrets have been loaded. If you have defined secrets for the current project and environment in Garden + Cloud, this may indicate a problem with your configuration. + ` + } else { + footer = `Secret keys with loaded values: ${loadedKeys.join(", ")}` + } + const errMsg = dedent` + The following secret names were referenced in configuration, but are missing from the secrets loaded remotely: + + ${descriptions.join("\n\n")} + + ${footer} + ` + if (log) { + log.silly(() => errMsg) + } + // throw new ConfigurationError(errMsg, { + // loadedSecretKeys: loadedKeys, + // missingSecretKeys: uniq(flatten(allMissing.map(([_key, missing]) => missing))), + // }) +} + +/** + * Collects template references to secrets in obj, and returns an array of any secret keys referenced in it that + * aren't present (or have blank values) in the provided secrets map. + */ +export function detectMissingSecretKeys( + obj: ObjectWithName, + context: ConfigContext, + secrets: StringMap +): ContextKeySegment[] { + const referencedKeys: ContextKeySegment[] = [] + const generator = getContextLookupReferences( + visitAll({ + value: obj, + // TODO: add real yaml source + source: { + path: [], + }, + }), + context + ) + for (const finding of generator) { + const keyPath = finding.keyPath + if (keyPath[0] !== "secrets") { + continue + } + + const secretName = keyPath[1] + if (isString(secretName)) { + referencedKeys.push(secretName) + } + } + + /** + * Secret keys with empty values should have resulted in an error by this point, but we filter on keys with + * values for good measure. + */ + const keysWithValues = Object.entries(secrets) + .filter(([_key, value]) => value) + .map(([key, _value]) => key) + const missingKeys = difference(referencedKeys, keysWithValues) + return missingKeys.sort() +} diff --git a/core/src/config/template-contexts/actions.ts b/core/src/config/template-contexts/actions.ts index 30a155162f..c261abd03d 100644 --- a/core/src/config/template-contexts/actions.ts +++ b/core/src/config/template-contexts/actions.ts @@ -194,7 +194,7 @@ class ActionReferencesContext extends ConfigContext { @schema(_actionResultContextSchema.description("Alias for `run`.")) public tasks: Map - constructor(root: ConfigContext, allowPartial: boolean, actions: (ResolvedAction | ExecutedAction)[]) { + constructor(root: ConfigContext, actions: (ResolvedAction | ExecutedAction)[]) { super(root) this.build = new Map() @@ -221,10 +221,6 @@ class ActionReferencesContext extends ConfigContext { }) ) } - - // This ensures that any template string containing runtime.* references is returned unchanged when - // there is no or limited runtime context available. - this._alwaysAllowPartial = allowPartial } } @@ -232,7 +228,6 @@ export interface ActionSpecContextParams { garden: Garden resolvedProviders: ProviderMap modules: GardenModule[] - partialRuntimeResolution: boolean action: Action resolvedDependencies: ResolvedAction[] executedDependencies: ExecutedAction[] @@ -279,8 +274,7 @@ export class ActionSpecContext extends OutputConfigContext { public this: ActionReferenceContext constructor(params: ActionSpecContextParams) { - const { action, garden, partialRuntimeResolution, variables, inputs, resolvedDependencies, executedDependencies } = - params + const { action, garden, variables, inputs, resolvedDependencies, executedDependencies } = params const internal = action.getInternal() @@ -297,10 +291,7 @@ export class ActionSpecContext extends OutputConfigContext { const parentName = internal?.parentName const templateName = internal?.templateName - this.actions = new ActionReferencesContext(this, partialRuntimeResolution, [ - ...resolvedDependencies, - ...executedDependencies, - ]) + this.actions = new ActionReferencesContext(this, [...resolvedDependencies, ...executedDependencies]) // Throw specific error when attempting to resolve self this.actions[action.kind.toLowerCase()].set( diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 4c4d9da4d9..f474a93b43 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -9,32 +9,28 @@ import type Joi from "@hapi/joi" import { isString } from "lodash-es" import { ConfigurationError, GardenError } from "../../exceptions.js" -import { resolveTemplateString } from "../../template-string/template-string.js" +import { resolveTemplateString } from "../../template/templated-strings.js" import type { CustomObjectSchema } from "../common.js" import { joi, joiIdentifier } from "../common.js" import { naturalList } from "../../util/string.js" import { styles } from "../../logger/styles.js" import { Profile } from "../../util/profiling.js" import type { CollectionOrValue } from "../../util/objects.js" -import type { TemplatePrimitive } from "../../template-string/types.js" -import { isTemplatePrimitive } from "../../template-string/types.js" +import type { TemplatePrimitive } from "../../template/types.js" +import { isTemplatePrimitive, UnresolvedTemplateValue } from "../../template/types.js" export type ContextKeySegment = string | number export type ContextKey = ContextKeySegment[] export interface ContextResolveOpts { - // Allow templates to be partially resolved (used to defer runtime template resolution, for example) - allowPartial?: boolean // This is kept for backwards compatibility of rendering kubernetes manifests // TODO(0.14): Do not allow the use of template strings in kubernetes manifest files // TODO(0.14): Remove legacyAllowPartial legacyAllowPartial?: boolean - // Allow partial resolution for values that originate from a special context that always returns CONTEXT_RESOLVE_KEY_AVAILABLE_LATER. - // This is used for module resolution and can be removed whenever we remove support for modules. - allowPartialContext?: boolean // a list of previously resolved paths, used to detect circular references stack?: Set - // Unescape escaped template strings + + // TODO: remove unescape?: boolean } @@ -71,7 +67,6 @@ export class ContextResolveError extends GardenError { } export const CONTEXT_RESOLVE_KEY_NOT_FOUND: unique symbol = Symbol.for("ContextResolveKeyNotFound") -export const CONTEXT_RESOLVE_KEY_AVAILABLE_LATER: unique symbol = Symbol.for("ContextResolveKeyAvailableLater") // Note: we're using classes here to be able to use decorators to describe each context node and key @Profile() @@ -79,13 +74,9 @@ export abstract class ConfigContext { private readonly _rootContext: ConfigContext private readonly _resolvedValues: { [path: string]: any } - // This is used for special-casing e.g. runtime.* resolution - protected _alwaysAllowPartial: boolean - constructor(rootContext?: ConfigContext) { this._rootContext = rootContext || this this._resolvedValues = {} - this._alwaysAllowPartial = false } static getSchema() { @@ -188,9 +179,14 @@ export abstract class ConfigContext { } // handle templated strings in context variables - if (isString(value)) { + if (value instanceof UnresolvedTemplateValue) { opts.stack.add(getStackEntry()) - value = resolveTemplateString({ string: value, context: this._rootContext, contextOpts: opts }) + value = value.evaluate({ context: this._rootContext, opts }) + + if (typeof value === "symbol") { + value = undefined + break + } } if (value === undefined) { @@ -226,14 +222,7 @@ export abstract class ConfigContext { return { resolved: value, getUnavailableReason } } - if (this._alwaysAllowPartial || opts.allowPartial) { - return { - resolved: CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, - getUnavailableReason, - } - } else { - return { resolved: CONTEXT_RESOLVE_KEY_NOT_FOUND, getUnavailableReason } - } + return { resolved: CONTEXT_RESOLVE_KEY_NOT_FOUND, getUnavailableReason } } // Cache result, unless it is a partial resolution diff --git a/core/src/config/template-contexts/module.ts b/core/src/config/template-contexts/module.ts index e4f8275e39..2a95272f23 100644 --- a/core/src/config/template-contexts/module.ts +++ b/core/src/config/template-contexts/module.ts @@ -167,7 +167,7 @@ class RuntimeConfigContext extends ConfigContext { ) public tasks: Map - constructor(root: ConfigContext, allowPartial: boolean, graphResults?: GraphResults) { + constructor(root: ConfigContext, graphResults?: GraphResults) { super(root) this.services = new Map() @@ -187,10 +187,6 @@ class RuntimeConfigContext extends ConfigContext { } } } - - // This ensures that any template string containing runtime.* references is returned unchanged when - // there is no or limited runtime context available. - this._alwaysAllowPartial = allowPartial } } @@ -202,7 +198,6 @@ export interface OutputConfigContextParams { // We only supply this when resolving configuration in dependency order. // Otherwise we pass `${runtime.*} template strings through for later resolution. graphResults?: GraphResults - partialRuntimeResolution: boolean } /** @@ -224,21 +219,14 @@ export class OutputConfigContext extends ProviderConfigContext { ) public runtime: RuntimeConfigContext - constructor({ - garden, - resolvedProviders, - variables, - modules, - graphResults, - partialRuntimeResolution, - }: OutputConfigContextParams) { + constructor({ garden, resolvedProviders, variables, modules, graphResults }: OutputConfigContextParams) { super(garden, resolvedProviders, variables) this.modules = new Map( modules.map((config) => <[string, ModuleReferenceContext]>[config.name, new ModuleReferenceContext(this, config)]) ) - this.runtime = new RuntimeConfigContext(this, partialRuntimeResolution, graphResults) + this.runtime = new RuntimeConfigContext(this, graphResults) } } diff --git a/core/src/config/validation.ts b/core/src/config/validation.ts index 6c37b3bd32..a7b25ae4d4 100644 --- a/core/src/config/validation.ts +++ b/core/src/config/validation.ts @@ -93,15 +93,15 @@ export function validateWithPath({ return validateSchema(config, schema, validateOpts) } -export interface ValidateConfigParams { - config: T +export interface ValidateConfigParams { + config: BaseGardenResource schema: Joi.Schema projectRoot: string yamlDocBasePath: ObjectPath ErrorClass?: typeof ConfigurationError } -export function validateConfig(params: ValidateConfigParams): T { +export function validateConfig(params: ValidateConfigParams): T { const { config, schema, projectRoot, ErrorClass, yamlDocBasePath } = params const { name, kind } = config @@ -119,7 +119,7 @@ export function validateConfig(params: ValidateCon } export function validateSchema( - value: T, + value: unknown, schema: Joi.Schema, { source, context = "", ErrorClass = ConfigurationError }: ValidateOptions = {} ): T { diff --git a/core/src/config/workflow.ts b/core/src/config/workflow.ts index 856feeb2b4..2fe922603f 100644 --- a/core/src/config/workflow.ts +++ b/core/src/config/workflow.ts @@ -22,7 +22,6 @@ import { deline, dedent } from "../util/string.js" import type { ServiceLimitSpec } from "../plugins/container/moduleConfig.js" import type { Garden } from "../garden.js" import { WorkflowConfigContext } from "./template-contexts/workflow.js" -import { resolveTemplateStrings } from "../template-string/template-string.js" import { validateConfig } from "./validation.js" import { ConfigurationError, GardenError } from "../exceptions.js" import type { EnvironmentConfig } from "./project.js" @@ -31,6 +30,9 @@ import { omitUndefined } from "../util/objects.js" import type { BaseGardenResource, GardenResource } from "./base.js" import type { GardenApiVersion } from "../constants.js" import { DOCS_BASE_URL } from "../constants.js" +import { capture } from "../template/capture.js" +import { deepEvaluate } from "../template/evaluate.js" +import type { ParsedTemplate } from "../template/types.js" export const minimumWorkflowRequests = { cpu: 50, // 50 millicpu @@ -355,18 +357,16 @@ export function resolveWorkflowConfig(garden: Garden, config: WorkflowConfig) { const partialConfig = { // Don't allow templating in names and triggers ...omit(config, "name", "triggers"), - // Inputs can be partially resolved - internal: omit(config.internal, "inputs"), // Defer resolution of step commands and scripts (the dummy script will be overwritten again below) steps: config.steps.map((s) => ({ ...s, command: undefined, script: "echo" })), } let resolvedPartialConfig: WorkflowConfig = { - ...resolveTemplateStrings({ - value: partialConfig, + ...(deepEvaluate(partialConfig as unknown as Record, { context, - source: { yamlDoc: config.internal.yamlDoc, path: [] }, - }), + opts: {}, + }) as unknown as WorkflowConfig), + internal: config.internal, name: config.name, } @@ -375,15 +375,7 @@ export function resolveWorkflowConfig(garden: Garden, config: WorkflowConfig) { } if (config.internal.inputs) { - resolvedPartialConfig.internal.inputs = resolveTemplateStrings({ - value: config.internal.inputs, - context, - contextOpts: { - allowPartial: true, - }, - // TODO: Map inputs to their original YAML sources - source: undefined, - }) + resolvedPartialConfig.internal.inputs = capture(config.internal.inputs, context) } log.silly(() => `Validating config for workflow ${config.name}`) diff --git a/core/src/docs/template-strings.ts b/core/src/docs/template-strings.ts index 8f82e940db..e624431b6f 100644 --- a/core/src/docs/template-strings.ts +++ b/core/src/docs/template-strings.ts @@ -19,7 +19,7 @@ import { import { ProviderConfigContext } from "../config/template-contexts/provider.js" import { ModuleConfigContext, OutputConfigContext } from "../config/template-contexts/module.js" import { WorkflowStepConfigContext } from "../config/template-contexts/workflow.js" -import { getHelperFunctions } from "../template-string/functions.js" +import { getHelperFunctions } from "../template/functions/index.js" import { isEqual, kebabCase, sortBy } from "lodash-es" import { CustomCommandContext } from "../config/template-contexts/custom-command.js" import type Joi from "@hapi/joi" diff --git a/core/src/exceptions.ts b/core/src/exceptions.ts index 2a53bae1b0..756654e66f 100644 --- a/core/src/exceptions.ts +++ b/core/src/exceptions.ts @@ -17,8 +17,6 @@ import { constants } from "os" import dns from "node:dns" import { styles } from "./logger/styles.js" import type { ExecaError } from "execa" -import type { Location } from "./template-string/ast.js" -import { addYamlContext, type ConfigSource } from "./config/validation.js" // Unfortunately, NodeJS does not provide a list of all error codes, so we have to maintain this list manually. // See https://nodejs.org/docs/latest-v18.x/api/dns.html#error-codes @@ -307,37 +305,6 @@ export class CloudApiError extends GardenError { } } -export class TemplateStringError extends GardenError { - type = "template-string" - - loc: Location - originalMessage: string - - constructor(params: GardenErrorParams & { loc: Location; yamlSource: ConfigSource }) { - let enriched: string = params.message - try { - // TODO: Use Location information from parser to point to the specific part - enriched = addYamlContext({ source: params.yamlSource, message: params.message }) - } catch { - // ignore - } - - if (enriched === params.message) { - const { path } = params.yamlSource - - const pathDescription = path.length > 0 ? ` at path ${styles.accent(path.join("."))}` : "" - const prefix = `Invalid template string (${styles.accent( - truncate(params.loc.source.rawTemplateString, { length: 200 }).replace(/\n/g, "\\n") - )})${pathDescription}: ` - enriched = params.message.startsWith(prefix) ? params.message : prefix + params.message - } - - super({ ...params, message: enriched }) - this.loc = params.loc - this.originalMessage = params.message - } -} - interface GenericGardenErrorParams extends GardenErrorParams { type: string } diff --git a/core/src/garden.ts b/core/src/garden.ts index 1bbb4ca895..a50f6f20c3 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -98,11 +98,7 @@ import { import { dedent, deline, naturalList, wordWrap } from "./util/string.js" import { DependencyGraph } from "./graph/common.js" import { Profile, profileAsync } from "./util/profiling.js" -import { - throwOnMissingSecretKeys, - resolveTemplateString, - resolveTemplateStrings, -} from "./template-string/template-string.js" +import { resolveTemplateString } from "./template/templated-strings.js" import type { WorkflowConfig, WorkflowConfigMap } from "./config/workflow.js" import { resolveWorkflowConfig, isWorkflowConfig } from "./config/workflow.js" import type { PluginTools } from "./util/ext-tools.js" @@ -172,6 +168,9 @@ import { getGardenCloudDomain, isGardenCommunityEdition, } from "./cloud/util.js" +import { throwOnMissingSecretKeys } from "./config/secrets.js" +import { deepEvaluate } from "./template/evaluate.js" +import type { ParsedTemplate } from "./template/types.js" const defaultLocalAddress = "localhost" @@ -1018,7 +1017,6 @@ export class Garden { variables: this.variables, modules, graphResults, - partialRuntimeResolution: false, }) } @@ -1572,7 +1570,6 @@ export class Garden { const resolved = resolveTemplateString({ string: disabledFlag, context, - contextOpts: { allowPartial: false }, }) return !!resolved @@ -1685,8 +1682,8 @@ export class Garden { public getProjectSources() { const context = new RemoteSourceConfigContext(this, this.variables) const source = { yamlDoc: this.projectConfig.internal.yamlDoc, path: ["sources"] } - const resolved = validateSchema( - resolveTemplateStrings({ value: this.projectSources, context, source }), + const resolved = validateSchema( + deepEvaluate(this.projectSources as unknown as ParsedTemplate, { context, opts: {} }), projectSourcesSchema(), { context: "remote source", @@ -1842,7 +1839,7 @@ export class Garden { variables: this.variables, actionConfigs: filteredActionConfigs, moduleConfigs: moduleConfigs.map(omitInternal), - workflowConfigs: sortBy(workflowConfigs, "name"), + workflowConfigs: sortBy(workflowConfigs.map(omitInternal), "name"), projectName: this.projectName, projectRoot: this.projectRoot, projectId: this.projectId, @@ -2419,7 +2416,7 @@ export interface ConfigDump { variables: DeepPrimitiveMap actionConfigs: ActionConfigMapForDump moduleConfigs: OmitInternalConfig[] - workflowConfigs: WorkflowConfig[] + workflowConfigs: OmitInternalConfig[] projectName: string projectRoot: string projectId?: string diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index a776920e60..7a568d1783 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -6,7 +6,6 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import cloneDeep from "fast-copy" import { isEqual, isString, mapValues, memoize, omit, pick, uniq } from "lodash-es" import type { Action, @@ -27,6 +26,7 @@ import { actionIsDisabled, actionReferenceToString, addActionDependency, + baseActionConfigSchema, baseRuntimeActionConfigSchema, describeActionConfig, describeActionConfigWithPath, @@ -49,11 +49,7 @@ import type { ActionDefinitionMap } from "../plugins.js" import { getActionTypeBases } from "../plugins.js" import type { ActionRouter } from "../router/router.js" import { ResolveActionTask } from "../tasks/resolve-action.js" -import { - getActionTemplateReferences, - maybeTemplateString, - resolveTemplateStrings, -} from "../template-string/template-string.js" +import { maybeTemplateString } from "../template/templated-strings.js" import { dedent, deline, naturalList } from "../util/string.js" import { DependencyGraph, getVarfileData, mergeVariables } from "./common.js" import type { ConfigGraph } from "./config-graph.js" @@ -68,7 +64,11 @@ import { profileAsync } from "../util/profiling.js" import { uuidv4 } from "../util/random.js" import { getSourcePath } from "../vcs/vcs.js" import { styles } from "../logger/styles.js" -import { isUnresolvableValue } from "../template-string/static-analysis.js" +import { isUnresolvableValue } from "../template/analysis.js" +import { getActionTemplateReferences } from "../config/references.js" +import { capture } from "../template/capture.js" +import { deepEvaluate } from "../template/evaluate.js" +import type { ParsedTemplate } from "../template/types.js" function* sliceToBatches(dict: Record, batchSize: number) { const entries = Object.entries(dict) @@ -662,7 +662,8 @@ export async function executeAction({ const getBuiltinConfigContextKeys = memoize(() => { const keys: string[] = [] - for (const schema of [buildActionConfigSchema(), baseRuntimeActionConfigSchema()]) { + // TODO: why are type and name missing here? + for (const schema of [buildActionConfigSchema(), baseRuntimeActionConfigSchema(), baseActionConfigSchema()]) { const configKeys = schema.describe().keys for (const [k, v] of Object.entries(configKeys)) { @@ -736,9 +737,9 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi log, }) - const resolvedVariables = resolveTemplateStrings({ - value: variables, - context: new ActionConfigContext({ + const resolvedVariables = capture( + variables, + new ActionConfigContext({ garden, config: { ...config, internal: { ...config.internal, inputs: {} } }, thisContextParams: { @@ -746,17 +747,14 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi name: config.name, }, variables, - }), - contextOpts: { allowPartial: true }, - // TODO: See about mapping this to the original variable sources - source: undefined, - }) + }) + ) if (templateName) { // Partially resolve inputs - const partiallyResolvedInputs = resolveTemplateStrings({ - value: config.internal.inputs || {}, - context: new ActionConfigContext({ + const partiallyResolvedInputs = capture( + config.internal.inputs || {}, + new ActionConfigContext({ garden, config: { ...config, internal: { ...config.internal, inputs: {} } }, thisContextParams: { @@ -764,11 +762,8 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi name: config.name, }, variables: resolvedVariables, - }), - contextOpts: { allowPartial: true }, - // TODO: See about mapping this to the original inputs source - source: undefined, - }) + }) + ) const template = garden.configTemplates[templateName] @@ -782,14 +777,16 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi } // Validate inputs schema - config.internal.inputs = validateWithPath({ - config: cloneDeep(partiallyResolvedInputs), - configType: `inputs for ${description}`, - path: config.internal.basePath, - schema: template.inputsSchema, - projectRoot: garden.projectRoot, - source: undefined, - }) + // TODO: schema validation on partially resolved inputs does not make sense + // do we validate the schema on fully resolved inputs somewhere? + // config.internal.inputs = validateWithPath({ + // config: cloneDeep(partiallyResolvedInputs), + // configType: `inputs for ${description}`, + // path: config.internal.basePath, + // schema: template.inputsSchema, + // projectRoot: garden.projectRoot, + // source: undefined, + // }) } const builtinConfigKeys = getBuiltinConfigContextKeys() @@ -808,45 +805,29 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi function resolveTemplates() { // Fully resolve built-in fields that only support `ActionConfigContext`. // TODO-0.13.1: better error messages when something goes wrong here (missing inputs for example) - const resolvedBuiltin = resolveTemplateStrings({ - value: pick(config, builtinConfigKeys), - context: builtinFieldContext, - contextOpts: { - allowPartial: false, - }, - source: { yamlDoc, path: [] }, - }) - config = { ...config, ...resolvedBuiltin } + const resolvedBuiltin = deepEvaluate( + pick(config, builtinConfigKeys.concat("type", "name")) as Record, + { + context: builtinFieldContext, + opts: {}, + } + ) const { spec = {} } = config - // Validate fully resolved keys (the above + those that don't allow any templating) // TODO-0.13.1: better error messages when something goes wrong here - config = validateWithPath({ - config: { - ...config, - variables: {}, - spec: {}, - }, - schema: getActionSchema(config.kind), - configType: describeActionConfig(config), - name: config.name, - path: config.internal.basePath, - projectRoot: garden.projectRoot, - source: { yamlDoc, path: [] }, - }) - - config = { ...config, variables: resolvedVariables, spec } + config = { + ...config, + ...resolvedBuiltin, + variables: resolvedVariables, + spec, + } // Partially resolve other fields // TODO-0.13.1: better error messages when something goes wrong here (missing inputs for example) - const resolvedOther = resolveTemplateStrings({ - value: omit(config, builtinConfigKeys), - context: builtinFieldContext, - contextOpts: { - allowPartial: true, - }, - source: { yamlDoc, path: [] }, - }) + const resolvedOther = capture( + omit(config, builtinConfigKeys) as Record, + builtinFieldContext + ) config = { ...config, ...resolvedOther } } diff --git a/core/src/outputs.ts b/core/src/outputs.ts index af2e627930..3729485280 100644 --- a/core/src/outputs.ts +++ b/core/src/outputs.ts @@ -7,19 +7,17 @@ */ import type { Garden } from "./garden.js" -import { - extractActionReference, - extractRuntimeReference, - resolveTemplateStrings, -} from "./template-string/template-string.js" import { OutputConfigContext } from "./config/template-contexts/module.js" import type { Log } from "./logger/log-entry.js" import type { OutputSpec } from "./config/project.js" import type { ActionReference } from "./config/common.js" import { GraphResults } from "./graph/results.js" -import { getContextLookupReferences, visitAll } from "./template-string/static-analysis.js" +import { getContextLookupReferences, visitAll } from "./template/analysis.js" import { isString } from "lodash-es" import type { ObjectWithName } from "./util/util.js" +import { extractActionReference, extractRuntimeReference } from "./config/references.js" +import { deepEvaluate } from "./template/evaluate.js" +import type { ParsedTemplate } from "./template/types.js" /** * Resolves all declared project outputs. If necessary, this will resolve providers and modules, and ensure services @@ -38,7 +36,6 @@ export async function resolveProjectOutputs(garden: Garden, log: Log): Promise[], { context: new OutputConfigContext({ garden, resolvedProviders: {}, variables: garden.variables, modules: [], - partialRuntimeResolution: false, }), - source, - }) + opts: {}, + }) as unknown as OutputSpec[] } // Make sure all referenced services and tasks are ready and collect their outputs for the runtime context @@ -117,5 +109,8 @@ export async function resolveProjectOutputs(garden: Garden, log: Log): Promise[], { + context: configContext, + opts: {}, + }) as unknown as OutputSpec[] } diff --git a/core/src/plugin-context.ts b/core/src/plugin-context.ts index 7d37a5b08c..de44ee5587 100644 --- a/core/src/plugin-context.ts +++ b/core/src/plugin-context.ts @@ -15,7 +15,7 @@ import { deline } from "./util/string.js" import { joi, joiVariables, joiStringMap, joiIdentifier, createSchema } from "./config/common.js" import type { PluginTool } from "./util/ext-tools.js" import type { ConfigContext, ContextResolveOpts } from "./config/template-contexts/base.js" -import { resolveTemplateStrings } from "./template-string/template-string.js" +import { resolveTemplateStrings } from "./template/templated-strings.js" import type { Log } from "./logger/log-entry.js" import { logEntrySchema } from "./plugin/base.js" import { EventEmitter } from "eventemitter3" diff --git a/core/src/plugin/handlers/base/configure.ts b/core/src/plugin/handlers/base/configure.ts index ee450593a7..572efb808d 100644 --- a/core/src/plugin/handlers/base/configure.ts +++ b/core/src/plugin/handlers/base/configure.ts @@ -58,7 +58,7 @@ export class ConfigureActionConfig joi.object().keys({ - config: actionConfigSchema().required(), + config: joi.any().required(), supportedModes: joi .object() .keys({ diff --git a/core/src/plugins/hadolint/hadolint.ts b/core/src/plugins/hadolint/hadolint.ts index bf39f8f50d..04b209502c 100644 --- a/core/src/plugins/hadolint/hadolint.ts +++ b/core/src/plugins/hadolint/hadolint.ts @@ -18,7 +18,7 @@ import { defaultDockerfileName } from "../container/config.js" import { baseBuildSpecSchema } from "../../config/module.js" import { getGitHubUrl } from "../../docs/common.js" import type { TestAction, TestActionConfig } from "../../actions/test.js" -import { mayContainTemplateString } from "../../template-string/template-string.js" +import { mayContainTemplateString } from "../../template/templated-strings.js" import type { BaseAction } from "../../actions/base.js" import type { BuildAction } from "../../actions/build.js" import { sdk } from "../../plugin/sdk.js" diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index 422af69cba..b5c020102c 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -10,11 +10,10 @@ import cloneDeep from "fast-copy" import { isArray, isString, keyBy, keys, partition, pick, union, uniq } from "lodash-es" import { validateWithPath } from "./config/validation.js" import { - getModuleTemplateReferences, mayContainTemplateString, resolveTemplateString, resolveTemplateStrings, -} from "./template-string/template-string.js" +} from "./template/templated-strings.js" import { GenericContext } from "./config/template-contexts/base.js" import { dirname, posix, relative, resolve } from "path" import type { Garden } from "./garden.js" @@ -63,6 +62,10 @@ import { styles } from "./logger/styles.js" import { actionReferenceToString } from "./actions/base.js" import type { DepGraph } from "dependency-graph" import { minimatch } from "minimatch" +import { getModuleTemplateReferences } from "./config/references.js" +import { capture } from "./template/capture.js" +import type { ParsedTemplate } from "./template/types.js" +import { deepEvaluate } from "./template/evaluate.js" // This limit is fairly arbitrary, but we need to have some cap on concurrent processing. export const moduleResolutionConcurrencyLimit = 50 @@ -521,7 +524,6 @@ export class ModuleResolver { inputs: {}, modules: [], graphResults: this.graphResults, - partialRuntimeResolution: true, } // Template inputs are commonly used in module deps, so we need to resolve them first @@ -537,13 +539,10 @@ export class ModuleResolver { // Try resolving template strings if possible let buildDeps: string[] = [] - const resolvedDeps = resolveTemplateStrings({ - value: rawConfig.build.dependencies, - context: configContext, - contextOpts: { allowPartial: true }, - // Note: We're not implementing the YAML source mapping for modules - source: undefined, - }) + const resolvedDeps = capture( + rawConfig.build.dependencies as unknown as ParsedTemplate, + configContext + ) as unknown as BuildDependencyConfig // The build.dependencies field may not resolve at all, in which case we can't extract any deps from there if (isArray(resolvedDeps)) { @@ -578,13 +577,7 @@ export class ModuleResolver { return inputs } - return resolveTemplateStrings({ - value: inputs, - context: configContext, - contextOpts: { allowPartial: true }, - // Note: We're not implementing the YAML source mapping for modules - source: undefined, - }) + return capture(inputs, configContext) } /** @@ -608,13 +601,13 @@ export class ModuleResolver { templateName: config.templateName, inputs, graphResults: this.graphResults, - partialRuntimeResolution: true, } // Resolve and validate the inputs field, because template module inputs may not be fully resolved at this // time. // TODO: This whole complicated procedure could be much improved and simplified by implementing lazy resolution on // values... I'll be looking into that. - JE + // NOTE(steffen): On it! :) const templateName = config.templateName if (templateName) { @@ -638,15 +631,7 @@ export class ModuleResolver { const resolvedModuleVariables = await this.resolveVariables(config, templateContextParams) // Now resolve just references to inputs on the config - config = resolveTemplateStrings({ - value: cloneDeep(config), - context: new GenericContext({ inputs }), - contextOpts: { - allowPartial: true, - }, - // Note: We're not implementing the YAML source mapping for modules - source: undefined, - }) + config = capture(config as unknown as ParsedTemplate, new GenericContext({ inputs })) as unknown as typeof config // And finally fully resolve the config. // Template strings in the spec can have references to inputs, @@ -657,21 +642,10 @@ export class ModuleResolver { inputs: { ...inputs }, }) - config = resolveTemplateStrings({ - value: { ...config, inputs: {}, variables: {} }, - context: configContext, - contextOpts: { - allowPartial: false, - allowPartialContext: true, - // Modules will be converted to actions later, and the actions will be properly unescaped. - // We avoid premature un-escaping here, - // because otherwise it will strip the escaped value in the module config - // to the normal template string in the converted action config. - unescape: false, - }, - // Note: We're not implementing the YAML source mapping for modules - source: undefined, - }) + config = capture( + { ...config, inputs: {}, variables: {} } as unknown as ParsedTemplate, + configContext + ) as unknown as typeof config config.variables = resolvedModuleVariables config.inputs = inputs @@ -815,7 +789,6 @@ export class ModuleResolver { inputs: resolvedConfig.inputs, modules: dependencies, graphResults: this.graphResults, - partialRuntimeResolution: true, }) let updatedFiles = false @@ -943,7 +916,6 @@ export class ModuleResolver { ): Promise { const moduleConfigContext = new ModuleConfigContext(templateContextParams) const resolveOpts = { - allowPartial: false, // Modules will be converted to actions later, and the actions will be properly unescaped. // We avoid premature un-escaping here, // because otherwise it will strip the escaped value in the module config @@ -953,10 +925,9 @@ export class ModuleResolver { let varfileVars: DeepPrimitiveMap = {} if (config.varfile) { - const varfilePath = resolveTemplateString({ - string: config.varfile, + const varfilePath = deepEvaluate(config.varfile, { context: moduleConfigContext, - contextOpts: resolveOpts, + opts: resolveOpts, }) if (typeof varfilePath !== "string") { throw new ConfigurationError({ @@ -971,12 +942,9 @@ export class ModuleResolver { } const rawVariables = config.variables - const moduleVariables = resolveTemplateStrings({ - value: cloneDeep(rawVariables || {}), + const moduleVariables = deepEvaluate(rawVariables || {}, { context: moduleConfigContext, - contextOpts: resolveOpts, - // Note: We're not implementing the YAML source mapping for modules - source: undefined, + opts: resolveOpts, }) // only override the relevant variables diff --git a/core/src/router/base.ts b/core/src/router/base.ts index 9dfc7b2918..2d36da7c65 100644 --- a/core/src/router/base.ts +++ b/core/src/router/base.ts @@ -300,7 +300,6 @@ export abstract class BaseActionRouter extends BaseRouter garden: this.garden, resolvedProviders: providers, action, - partialRuntimeResolution: false, modules: graph.getModules(), resolvedDependencies: action.getResolvedDependencies(), executedDependencies: action.getExecutedDependencies(), diff --git a/core/src/tasks/publish.ts b/core/src/tasks/publish.ts index 1503901b2e..70c40a349e 100644 --- a/core/src/tasks/publish.ts +++ b/core/src/tasks/publish.ts @@ -9,7 +9,7 @@ import { BuildTask } from "./build.js" import type { BaseActionTaskParams, ActionTaskProcessParams } from "../tasks/base.js" import { BaseActionTask } from "../tasks/base.js" -import { resolveTemplateString } from "../template-string/template-string.js" +import { resolveTemplateString } from "../template/templated-strings.js" import { joi } from "../config/common.js" import { versionStringPrefix } from "../vcs/vcs.js" import { ConfigContext, schema } from "../config/template-contexts/base.js" @@ -107,7 +107,6 @@ export class PublishTask extends BaseActionTask extends ValidResultType { state: ActionState @@ -126,20 +127,15 @@ export class ResolveActionTask extends BaseActionTask extends BaseActionTask extends BaseActionTask { const source = { yamlDoc, path: yamlDocBasePath } - let resolvedConfig = resolveTemplateStrings({ value: this.config, context, source }) + let resolvedConfig = deepEvaluate(this.config, { context, opts: {} }) const providerName = resolvedConfig.name const providerLog = getProviderLog(providerName, this.log) providerLog.info("Configuring provider...") diff --git a/core/src/template-string/parser.d.ts b/core/src/template-string/parser.d.ts deleted file mode 100644 index 34a4bfb1dc..0000000000 --- a/core/src/template-string/parser.d.ts +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (C) 2018-2024 Garden Technologies, Inc. - * - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. - */ - -import peggy from "peggy" - -// This file is just a placeholder to make TypeScript happy -// The actual parser is generated by the build script -// It will be placed in the build outputs and then actually run from there -export const parse: peggy.Parser["parse"] = () => {} diff --git a/core/src/template-string/template-string.ts b/core/src/template-string/template-string.ts deleted file mode 100644 index e60beb5b7d..0000000000 --- a/core/src/template-string/template-string.ts +++ /dev/null @@ -1,903 +0,0 @@ -/* - * Copyright (C) 2018-2024 Garden Technologies, Inc. - * - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. - */ - -import type { GardenErrorParams } from "../exceptions.js" -import { ConfigurationError, GardenError, InternalError, TemplateStringError } from "../exceptions.js" -import type { ConfigContext, ContextKeySegment, ContextResolveOpts } from "../config/template-contexts/base.js" -import { GenericContext } from "../config/template-contexts/base.js" -import cloneDeep from "fast-copy" -import { difference, isPlainObject, isString } from "lodash-es" -import type { ActionReference, Primitive, StringMap } from "../config/common.js" -import { - arrayConcatKey, - arrayForEachFilterKey, - arrayForEachKey, - arrayForEachReturnKey, - conditionalElseKey, - conditionalKey, - conditionalThenKey, - isPrimitive, - isSpecialKey, - objectSpreadKey, -} from "../config/common.js" -import { dedent, deline, naturalList, titleize } from "../util/string.js" -import type { ObjectWithName } from "../util/util.js" -import type { Log } from "../logger/log-entry.js" -import type { ModuleConfigContext } from "../config/template-contexts/module.js" -import type { ActionConfig, ActionKind } from "../actions/types.js" -import { actionKindsLower } from "../actions/types.js" -import type { CollectionOrValue } from "../util/objects.js" -import { deepMap } from "../util/objects.js" -import type { ConfigSource } from "../config/validation.js" -import * as parser from "./parser.js" -import type { ObjectPath } from "../config/base.js" -import type { TemplatePrimitive } from "./types.js" -import * as ast from "./ast.js" -import { LRUCache } from "lru-cache" -import type { ContextLookupReferenceFinding, UnresolvableValue } from "./static-analysis.js" -import { getContextLookupReferences, isUnresolvableValue, visitAll } from "./static-analysis.js" -import type { ModuleConfig } from "../config/module.js" - -const escapePrefix = "$${" - -export class TemplateError extends GardenError { - type = "template" - - path: ObjectPath | undefined - value: any - resolved: any - - constructor(params: GardenErrorParams & { path: ObjectPath | undefined; value: any; resolved: any }) { - super(params) - this.path = params.path - this.value = params.value - this.resolved = params.resolved - } -} - -type ParseParams = Parameters - -function parseWithPegJs(params: ParseParams) { - return parser.parse(...params) -} - -const shouldUnescape = (ctxOpts: ContextResolveOpts) => { - // Explicit non-escaping takes the highest priority. - if (ctxOpts.unescape === false) { - return false - } - - return !!ctxOpts.unescape || !ctxOpts.allowPartial -} - -const parseTemplateStringCache = new LRUCache({ - max: 100000, -}) - -export function parseTemplateString({ - rawTemplateString, - unescape, - source, -}: { - rawTemplateString: string - unescape: boolean - source: ConfigSource -}): ast.TemplateExpression | string { - // Just return immediately if this is definitely not a template string - if (!maybeTemplateString(rawTemplateString)) { - return rawTemplateString - } - - const key = `u-${unescape ? "1" : "0"}-${rawTemplateString}` - const cached = parseTemplateStringCache.get(key) - - if (cached) { - return cached - } - - const templateStringSource: ast.TemplateStringSource = { - rawTemplateString, - } - - class ParserError extends TemplateStringError { - constructor(params: GardenErrorParams & { loc: ast.Location }) { - super({ - ...params, - yamlSource: source, - }) - } - } - - const parsed = parseWithPegJs([ - rawTemplateString, - { - ast, - escapePrefix, - optionalSuffix: "}?", - parseNested: (nested: string) => parseTemplateString({ rawTemplateString: nested, unescape, source }), - TemplateStringError: ParserError, - unescape, - grammarSource: templateStringSource, - }, - ]) - - parseTemplateStringCache.set(key, parsed) - - return parsed -} - -/** - * Parse and resolve a templated string, with the given context. The template format is similar to native JS templated - * strings but only supports simple lookups from the given context, e.g. "prefix-${nested.key}-suffix", and not - * arbitrary JS code. - * - * The context should be a ConfigContext instance. The optional `stack` parameter is used to detect circular - * dependencies when resolving context variables. - */ -export function resolveTemplateString({ - string, - context, - contextOpts = {}, - source, -}: { - string: string - context: ConfigContext - contextOpts?: ContextResolveOpts - source?: ConfigSource -}): CollectionOrValue { - if (source === undefined) { - source = { - path: [], - yamlDoc: undefined, - } - } - - const parsed = parseTemplateString({ - rawTemplateString: string, - // TODO: remove unescape hacks. - unescape: shouldUnescape(contextOpts), - source, - }) - - // string does not contain - if (typeof parsed === "string") { - return parsed - } - - const result = parsed.evaluate({ - context, - opts: contextOpts, - yamlSource: source, - }) - - if (typeof result !== "symbol") { - return result - } - - if (!contextOpts.allowPartial && !contextOpts.allowPartialContext) { - throw new InternalError({ - message: `allowPartial is false, but template expression returned symbol ${String(result)}. ast.ContextLookupExpression should have thrown an error.`, - }) - } - - // Requested partial evaluation and the template expression cannot be evaluated yet. We may be able to do it later. - - // TODO: Parse all template expressions after reading the YAML config and only re-evaluate ast.TemplateExpression instances in - // resolveTemplateStrings; Otherwise we'll inevitably have a bug where garden will resolve template expressions that might be - // contained in expression evaluation results e.g. if an environment variable contains template string, we don't want to - // evaluate the template string in there. - // See also https://github.com/garden-io/garden/issues/5825 - return string -} - -/** - * Recursively parses and resolves all templated strings in the given object. - */ -// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-constraint -export function resolveTemplateStrings({ - value, - context, - contextOpts = {}, - source, -}: { - value: T - context: ConfigContext - contextOpts?: ContextResolveOpts - source: ConfigSource | undefined -}): T { - if (value === null) { - return null as T - } - if (value === undefined) { - return undefined as T - } - - if (!source) { - source = { - path: [], - } - } - if (!source.path) { - source.path = [] - } - - if (typeof value === "string") { - return resolveTemplateString({ string: value, context, source, contextOpts }) - } else if (Array.isArray(value)) { - const output: unknown[] = [] - - for (let i = 0; i < value.length; i++) { - const v = value[i] - if (isPlainObject(v) && v[arrayConcatKey] !== undefined) { - if (Object.keys(v).length > 1) { - const extraKeys = naturalList( - Object.keys(v) - .filter((k) => k !== arrayConcatKey) - .map((k) => JSON.stringify(k)) - ) - throw new TemplateError({ - message: `A list item with a ${arrayConcatKey} key cannot have any other keys (found ${extraKeys})`, - path: source.path, - value, - resolved: undefined, - }) - } - - // Handle array concatenation via $concat - const resolved = resolveTemplateStrings({ - value: v[arrayConcatKey], - context, - contextOpts: { - ...contextOpts, - }, - source: { - ...source, - path: [...source.path, arrayConcatKey], - }, - }) - - if (Array.isArray(resolved)) { - output.push(...resolved) - } else if (contextOpts.allowPartial) { - output.push({ $concat: resolved }) - } else { - throw new TemplateError({ - message: `Value of ${arrayConcatKey} key must be (or resolve to) an array (got ${typeof resolved})`, - path: source.path, - value, - resolved, - }) - } - } else { - output.push( - resolveTemplateStrings({ - value: v, - context, - contextOpts, - source: { - ...source, - path: [...source.path, i], - }, - }) - ) - } - } - - return (output) - } else if (isPlainObject(value)) { - if (value[arrayForEachKey] !== undefined) { - // Handle $forEach loop - return handleForEachObject({ value, context, contextOpts, source }) - } else if (value[conditionalKey] !== undefined) { - // Handle $if conditional - return handleConditional({ value, context, contextOpts, source }) - } else { - // Resolve $merge keys, depth-first, leaves-first - let output = {} - - for (const k in value as Record) { - const v = value[k] - const resolved = resolveTemplateStrings({ - value: v, - context, - contextOpts, - source: { - ...source, - path: source.path && [...source.path, k], - }, - }) - - if (k === objectSpreadKey) { - if (isPlainObject(resolved)) { - output = { ...output, ...resolved } - } else if (contextOpts.allowPartial) { - output[k] = resolved - } else { - throw new TemplateError({ - message: `Value of ${objectSpreadKey} key must be (or resolve to) a mapping object (got ${typeof resolved})`, - path: [...source.path, k], - value, - resolved, - }) - } - } else { - output[k] = resolved - } - } - - return output - } - } else { - return value - } -} - -const expectedForEachKeys = [arrayForEachKey, arrayForEachReturnKey, arrayForEachFilterKey] - -function handleForEachObject({ - value, - context, - contextOpts, - source, -}: { - value: any - context: ConfigContext - contextOpts: ContextResolveOpts - source: ConfigSource -}) { - // Validate input object - if (value[arrayForEachReturnKey] === undefined) { - throw new TemplateError({ - message: `Missing ${arrayForEachReturnKey} field next to ${arrayForEachKey} field. Got ${naturalList( - Object.keys(value) - )}`, - path: source.path && [...source.path, arrayForEachKey], - value, - resolved: undefined, - }) - } - - const unexpectedKeys = Object.keys(value).filter((k) => !expectedForEachKeys.includes(k)) - - if (unexpectedKeys.length > 0) { - const extraKeys = naturalList(unexpectedKeys.map((k) => JSON.stringify(k))) - - throw new TemplateError({ - message: `Found one or more unexpected keys on ${arrayForEachKey} object: ${extraKeys}. Expected keys: ${naturalList( - expectedForEachKeys - )}`, - path: source.path, - value, - resolved: undefined, - }) - } - - // Try resolving the value of the $forEach key - let resolvedInput = resolveTemplateStrings({ - value: value[arrayForEachKey], - context, - contextOpts, - source: { - ...source, - path: source.path && [...source.path, arrayForEachKey], - }, - }) - const isObject = isPlainObject(resolvedInput) - - if (!Array.isArray(resolvedInput) && !isObject) { - if (contextOpts.allowPartial) { - return value - } else { - throw new TemplateError({ - message: `Value of ${arrayForEachKey} key must be (or resolve to) an array or mapping object (got ${typeof resolvedInput})`, - path: source.path && [...source.path, arrayForEachKey], - value, - resolved: resolvedInput, - }) - } - } - - if (isObject) { - const keys = Object.keys(resolvedInput) - const inputContainsSpecialKeys = keys.some((key) => isSpecialKey(key)) - - if (inputContainsSpecialKeys) { - // If partial application is enabled - // we cannot be sure if the object can be evaluated correctly. - // There could be an expression in there that goes `{foo || bar}` - // and `foo` is only to be filled in at a later time, so resolving now would force it to be `bar`. - // Thus we return the entire object - // - // If partial application is disabled - // then we need to make sure that the resulting expression is evaluated again - // since the magic keys only get resolved via `resolveTemplateStrings` - if (contextOpts.allowPartial) { - return value - } - - resolvedInput = resolveTemplateStrings({ value: resolvedInput, context, contextOpts, source: undefined }) - } - } - - const filterExpression = value[arrayForEachFilterKey] - - // TODO: maybe there's a more efficient way to do the cloning/extending? - const loopContext = cloneDeep(context) - - const output: unknown[] = [] - - for (const i of Object.keys(resolvedInput)) { - const itemValue = resolvedInput[i] - - loopContext["item"] = new GenericContext({ key: i, value: itemValue }) - - // Have to override the cache in the parent context here - // TODO: make this a little less hacky :P - const resolvedValues = loopContext["_resolvedValues"] - delete resolvedValues["item.key"] - delete resolvedValues["item.value"] - const subValues = Object.keys(resolvedValues).filter((k) => k.match(/item\.value\.*/)) - subValues.forEach((v) => delete resolvedValues[v]) - - // Check $filter clause output, if applicable - if (filterExpression !== undefined) { - const filterResult = resolveTemplateStrings({ - value: value[arrayForEachFilterKey], - context: loopContext, - contextOpts: { - ...contextOpts, - // filter expression must be completely resolvable - // TODO: In a future iteration of this code, we should leave the entire $forEach expression unresolved if filter cannot be resolved yet and allowPartial=true. - allowPartial: false, - }, - source: { - ...source, - path: source.path && [...source.path, arrayForEachFilterKey], - }, - }) - - if (filterResult === false) { - continue - } else if (filterResult !== true) { - throw new TemplateError({ - message: `${arrayForEachFilterKey} clause in ${arrayForEachKey} loop must resolve to a boolean value (got ${typeof filterResult})`, - path: source.path && [...source.path, arrayForEachFilterKey], - value, - resolved: undefined, - }) - } - } - - output.push( - resolveTemplateStrings({ - value: value[arrayForEachReturnKey], - context: loopContext, - contextOpts, - source: { - ...source, - path: source.path && [...source.path, arrayForEachKey, i], - }, - }) - ) - } - - // Need to resolve once more to handle e.g. $concat expressions - return resolveTemplateStrings({ value: output, context, contextOpts, source }) -} - -const expectedConditionalKeys = [conditionalKey, conditionalThenKey, conditionalElseKey] - -function handleConditional({ - value, - context, - contextOpts, - source, -}: { - value: any - context: ConfigContext - contextOpts: ContextResolveOpts - source: ConfigSource -}) { - // Validate input object - const thenExpression = value[conditionalThenKey] - const elseExpression = value[conditionalElseKey] - - if (thenExpression === undefined) { - throw new TemplateError({ - message: `Missing ${conditionalThenKey} field next to ${conditionalKey} field. Got: ${naturalList( - Object.keys(value) - )}`, - path: source.path, - value, - resolved: undefined, - }) - } - - const unexpectedKeys = Object.keys(value).filter((k) => !expectedConditionalKeys.includes(k)) - - if (unexpectedKeys.length > 0) { - const extraKeys = naturalList(unexpectedKeys.map((k) => JSON.stringify(k))) - - throw new TemplateError({ - message: `Found one or more unexpected keys on ${conditionalKey} object: ${extraKeys}. Expected: ${naturalList( - expectedConditionalKeys - )}`, - path: source.path, - value, - resolved: undefined, - }) - } - - // Try resolving the value of the $if key - const resolvedConditional = resolveTemplateStrings({ - value: value[conditionalKey], - context, - contextOpts, - source: { - ...source, - path: source.path && [...source.path, conditionalKey], - }, - }) - - if (typeof resolvedConditional !== "boolean") { - if (contextOpts.allowPartial) { - return value - } else { - throw new TemplateError({ - message: `Value of ${conditionalKey} key must be (or resolve to) a boolean (got ${typeof resolvedConditional})`, - path: source.path && [...source.path, conditionalKey], - value, - resolved: resolvedConditional, - }) - } - } - - // Note: We implicitly default the $else value to undefined - - const resolvedThen = resolveTemplateStrings({ - value: thenExpression, - context, - contextOpts, - source: { - ...source, - path: source.path && [...source.path, conditionalThenKey], - }, - }) - const resolvedElse = resolveTemplateStrings({ - value: elseExpression, - context, - contextOpts, - source: { - ...source, - path: source.path && [...source.path, conditionalElseKey], - }, - }) - - if (!!resolvedConditional) { - return resolvedThen - } else { - return resolvedElse - } -} - -/** - * Returns `true` if the given value is a string and looks to contain a template string. - */ -export function maybeTemplateString(value: Primitive) { - return !!value && typeof value === "string" && value.includes("${") -} - -/** - * Returns `true` if the given value or any value in a given object or array seems to contain a template string. - */ -export function mayContainTemplateString(obj: any): boolean { - let out = false - - if (isPrimitive(obj)) { - return maybeTemplateString(obj) - } - - deepMap(obj, (v) => { - if (maybeTemplateString(v)) { - out = true - } - }) - - return out -} - -interface ActionTemplateReference extends ActionReference { - keyPath: (ContextKeySegment | UnresolvableValue)[] -} - -export function extractActionReference(finding: ContextLookupReferenceFinding): ActionTemplateReference { - const kind = finding.keyPath[1] - if (!kind) { - throw new ConfigurationError({ - message: `Found invalid action reference (missing kind).`, - }) - } - - if (isUnresolvableValue(kind)) { - const err = kind.getError() - throw new ConfigurationError({ - message: `Found invalid action reference: ${err.message}`, - }) - } - - if (!isString(kind)) { - throw new ConfigurationError({ - message: `Found invalid action reference (kind is not a string).`, - }) - } - - if (!actionKindsLower.includes(kind)) { - throw new ConfigurationError({ - message: `Found invalid action reference (invalid kind '${kind}')`, - }) - } - - const name = finding.keyPath[2] - if (!name) { - throw new ConfigurationError({ - message: "Found invalid action reference (missing name)", - }) - } - - if (isUnresolvableValue(name)) { - const err = name.getError() - throw new ConfigurationError({ - message: `Found invalid action reference: ${err.message}`, - }) - } - - if (!isString(name)) { - throw new ConfigurationError({ - message: "Found invalid action reference (name is not a string)", - }) - } - - return { - kind: titleize(kind), - name, - keyPath: finding.keyPath.slice(3), - } -} - -export function extractRuntimeReference(finding: ContextLookupReferenceFinding): ActionTemplateReference { - const runtimeKind = finding.keyPath[1] - if (!runtimeKind) { - throw new ConfigurationError({ - message: "Found invalid runtime reference (missing kind)", - }) - } - - if (isUnresolvableValue(runtimeKind)) { - const err = runtimeKind.getError() - throw new ConfigurationError({ - message: `Found invalid runtime reference: ${err.message}`, - }) - } - - if (!isString(runtimeKind)) { - throw new ConfigurationError({ - message: "Found invalid runtime reference (kind is not a string)", - }) - } - - let kind: ActionKind - if (runtimeKind === "services") { - kind = "Deploy" - } else if (runtimeKind === "tasks") { - kind = "Run" - } else { - throw new ConfigurationError({ - message: `Found invalid runtime reference (invalid kind '${runtimeKind}')`, - }) - } - - const name = finding.keyPath[2] - - if (!name) { - throw new ConfigurationError({ - message: `Found invalid runtime reference (missing name)`, - }) - } - - if (isUnresolvableValue(name)) { - const err = name.getError() - throw new ConfigurationError({ - message: `Found invalid action reference: ${err.message}`, - }) - } - - if (!isString(name)) { - throw new ConfigurationError({ - message: "Found invalid runtime reference (name is not a string)", - }) - } - - return { - kind, - name, - keyPath: finding.keyPath.slice(3), - } -} - -/** - * Collects every reference to another action in the given config object, including translated runtime.* references. - * An error is thrown if a reference is not resolvable, i.e. if a nested template is used as a reference. - */ -export function* getActionTemplateReferences( - config: ActionConfig, - context: ConfigContext -): Generator { - const generator = getContextLookupReferences( - visitAll({ - value: config as ObjectWithName, - parseTemplateStrings: true, - source: { - yamlDoc: config.internal?.yamlDoc, - path: [], - }, - }), - context - ) - - for (const finding of generator) { - const refType = finding.keyPath[0] - // ${action.*} - if (refType === "actions") { - yield extractActionReference(finding) - } - // ${runtime.*} - if (refType === "runtime") { - yield extractRuntimeReference(finding) - } - } -} - -export function getModuleTemplateReferences(config: ModuleConfig, context: ModuleConfigContext) { - const moduleNames: string[] = [] - const generator = getContextLookupReferences( - visitAll({ - value: config as ObjectWithName, - parseTemplateStrings: true, - // Note: We're not implementing the YAML source mapping for modules - source: { - path: [], - }, - }), - context - ) - - for (const finding of generator) { - const keyPath = finding.keyPath - if (keyPath[0] !== "modules") { - continue - } - - const moduleName = keyPath[1] - if (isUnresolvableValue(moduleName)) { - const err = moduleName.getError() - throw new ConfigurationError({ - message: `Found invalid module reference: ${err.message}`, - }) - } - - if (config.name === moduleName) { - continue - } - - moduleNames.push(moduleName.toString()) - } - - return moduleNames -} - -/** - * Gathers secret references in configs and throws an error if one or more referenced secrets isn't present (or has - * blank values) in the provided secrets map. - * - * Prefix should be e.g. "Module" or "Provider" (used when generating error messages). - */ -export function throwOnMissingSecretKeys( - configs: ObjectWithName[], - context: ConfigContext, - secrets: StringMap, - prefix: string, - log?: Log -) { - const allMissing: [string, ContextKeySegment[]][] = [] // [[key, missing keys]] - for (const config of configs) { - const missing = detectMissingSecretKeys(config, context, secrets) - if (missing.length > 0) { - allMissing.push([config.name, missing]) - } - } - - if (allMissing.length === 0) { - return - } - - const descriptions = allMissing.map(([key, missing]) => `${prefix} ${key}: ${missing.join(", ")}`) - /** - * Secret keys with empty values should have resulted in an error by this point, but we filter on keys with - * values for good measure. - */ - const loadedKeys = Object.entries(secrets) - .filter(([_key, value]) => value) - .map(([key, _value]) => key) - let footer: string - if (loadedKeys.length === 0) { - footer = deline` - Note: No secrets have been loaded. If you have defined secrets for the current project and environment in Garden - Cloud, this may indicate a problem with your configuration. - ` - } else { - footer = `Secret keys with loaded values: ${loadedKeys.join(", ")}` - } - const errMsg = dedent` - The following secret names were referenced in configuration, but are missing from the secrets loaded remotely: - - ${descriptions.join("\n\n")} - - ${footer} - ` - if (log) { - log.silly(() => errMsg) - } - // throw new ConfigurationError(errMsg, { - // loadedSecretKeys: loadedKeys, - // missingSecretKeys: uniq(flatten(allMissing.map(([_key, missing]) => missing))), - // }) -} - -/** - * Collects template references to secrets in obj, and returns an array of any secret keys referenced in it that - * aren't present (or have blank values) in the provided secrets map. - */ -export function detectMissingSecretKeys( - obj: ObjectWithName, - context: ConfigContext, - secrets: StringMap -): ContextKeySegment[] { - const referencedKeys: ContextKeySegment[] = [] - const generator = getContextLookupReferences( - visitAll({ - value: obj, - parseTemplateStrings: true, - // TODO: add real yaml source - source: { - path: [], - }, - }), - context - ) - for (const finding of generator) { - const keyPath = finding.keyPath - if (keyPath[0] !== "secrets") { - continue - } - - const secretName = keyPath[1] - if (isString(secretName)) { - referencedKeys.push(secretName) - } - } - - /** - * Secret keys with empty values should have resulted in an error by this point, but we filter on keys with - * values for good measure. - */ - const keysWithValues = Object.entries(secrets) - .filter(([_key, value]) => value) - .map(([key, _value]) => key) - const missingKeys = difference(referencedKeys, keysWithValues) - return missingKeys.sort() -} diff --git a/core/src/template-string/types.ts b/core/src/template-string/types.ts deleted file mode 100644 index 480fa9af39..0000000000 --- a/core/src/template-string/types.ts +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright (C) 2018-2024 Garden Technologies, Inc. - * - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. - */ - -import type { Primitive } from "utility-types" -import { isPrimitive } from "utility-types" - -export function isTemplatePrimitive(value: unknown): value is TemplatePrimitive { - return isPrimitive(value) && typeof value !== "symbol" -} - -export type EmptyArray = never[] -export type EmptyObject = { [key: string]: never } - -export type TemplatePrimitive = Exclude diff --git a/core/src/template-string/static-analysis.ts b/core/src/template/analysis.ts similarity index 79% rename from core/src/template-string/static-analysis.ts rename to core/src/template/analysis.ts index 691ea075a9..cbeaffd2b3 100644 --- a/core/src/template-string/static-analysis.ts +++ b/core/src/template/analysis.ts @@ -10,14 +10,14 @@ import type { CollectionOrValue } from "../util/objects.js" import { isArray, isPlainObject } from "../util/objects.js" import { ContextLookupExpression, TemplateExpression } from "./ast.js" import type { TemplatePrimitive } from "./types.js" -import { parseTemplateString } from "./template-string.js" +import { UnresolvedTemplateValue } from "./types.js" import { type ConfigContext } from "../config/template-contexts/base.js" import { GardenError, InternalError } from "../exceptions.js" import { type ConfigSource } from "../config/validation.js" export type TemplateExpressionGenerator = Generator< { - value: TemplatePrimitive | TemplateExpression + value: TemplatePrimitive | UnresolvedTemplateValue | TemplateExpression yamlSource: ConfigSource }, void, @@ -26,18 +26,15 @@ export type TemplateExpressionGenerator = Generator< export function* visitAll({ value, - parseTemplateStrings = false, source, }: { - value: CollectionOrValue - parseTemplateStrings?: boolean + value: CollectionOrValue source: ConfigSource }): TemplateExpressionGenerator { if (isArray(value)) { for (const [k, v] of value.entries()) { yield* visitAll({ value: v, - parseTemplateStrings, source: { ...source, path: [...source.path, k], @@ -48,36 +45,20 @@ export function* visitAll({ for (const k of Object.keys(value)) { yield* visitAll({ value: value[k], - parseTemplateStrings, source: { ...source, path: [...source.path, k], }, }) } + } else if (value instanceof UnresolvedTemplateValue) { + yield* value.visitAll() + } else if (value instanceof TemplateExpression) { + yield* value.visitAll(source) } else { - if (parseTemplateStrings && typeof value === "string") { - const parsed = parseTemplateString({ - rawTemplateString: value, - unescape: false, - source, - }) - - if (typeof parsed === "string") { - yield { - value: parsed, - yamlSource: source, - } - } else { - yield* parsed.visitAll(source) - } - } else if (value instanceof TemplateExpression) { - yield* value.visitAll(source) - } else { - yield { - value, - yamlSource: source, - } + yield { + value, + yamlSource: source, } } } @@ -130,7 +111,7 @@ export function* getContextLookupReferences( const keyPath = value.keyPath.map((keyPathExpression) => { const key = keyPathExpression.evaluate({ context, - opts: { allowPartial: false }, + opts: {}, optional: true, yamlSource, }) @@ -141,7 +122,7 @@ export function* getContextLookupReferences( // this will throw an error, because the key could not be resolved keyPathExpression.evaluate({ context, - opts: { allowPartial: false }, + opts: {}, optional: false, yamlSource, }) diff --git a/core/src/template-string/ast.ts b/core/src/template/ast.ts similarity index 88% rename from core/src/template-string/ast.ts rename to core/src/template/ast.ts index 54abec49fe..659512af22 100644 --- a/core/src/template-string/ast.ts +++ b/core/src/template/ast.ts @@ -8,31 +8,33 @@ import { isArray, isNumber, isString } from "lodash-es" import { - CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, CONTEXT_RESOLVE_KEY_NOT_FOUND, renderKeyPath, type ConfigContext, type ContextResolveOpts, } from "../config/template-contexts/base.js" -import { GardenError, InternalError, TemplateStringError } from "../exceptions.js" -import { getHelperFunctions } from "./functions.js" +import { GardenError, InternalError } from "../exceptions.js" +import { getHelperFunctions } from "./functions/index.js" +import type { EvaluateTemplateArgs } from "./types.js" import { isTemplatePrimitive, type TemplatePrimitive } from "./types.js" import type { Collection, CollectionOrValue } from "../util/objects.js" import { type ConfigSource, validateSchema } from "../config/validation.js" -import type { TemplateExpressionGenerator } from "./static-analysis.js" +import type { TemplateExpressionGenerator } from "./analysis.js" +import { TemplateStringError } from "./errors.js" -type EvaluateArgs = { - context: ConfigContext - opts: ContextResolveOpts +type ASTEvaluateArgs = EvaluateTemplateArgs & { yamlSource: ConfigSource /** * Whether or not to throw an error if ContextLookupExpression fails to resolve variable. * The FormatStringExpression will set this parameter based on wether the OptionalSuffix (?) is present or not. */ - optional?: boolean + readonly optional?: boolean } +export type ASTEvaluationResult = T | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND +// | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER + export type TemplateStringSource = { rawTemplateString: string } @@ -54,11 +56,6 @@ export type Location = { source: TemplateStringSource } -export type TemplateEvaluationResult = - | T - | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND - | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER - function* astVisitAll(e: TemplateExpression, source: ConfigSource): TemplateExpressionGenerator { for (const key in e) { if (key === "loc") { @@ -93,7 +90,7 @@ export abstract class TemplateExpression { yield* astVisitAll(this, source) } - abstract evaluate(args: EvaluateArgs): TemplateEvaluationResult> + abstract evaluate(args: ASTEvaluateArgs): ASTEvaluationResult> } export class IdentifierExpression extends TemplateExpression { @@ -141,7 +138,7 @@ export class ArrayLiteralExpression extends TemplateExpression { super() } - override evaluate(args: EvaluateArgs): TemplateEvaluationResult> { + override evaluate(args: ASTEvaluateArgs): ASTEvaluationResult> { const result: CollectionOrValue = [] for (const e of this.literal) { const res = e.evaluate(args) @@ -164,7 +161,7 @@ export abstract class UnaryExpression extends TemplateExpression { super() } - override evaluate(args: EvaluateArgs): TemplateEvaluationResult { + override evaluate(args: ASTEvaluateArgs): ASTEvaluationResult { const inner = this.innerExpression.evaluate({ ...args, // For backwards compatibility with older versions of Garden, unary expressions do not throw errors if context lookup expressions fail. @@ -173,9 +170,9 @@ export abstract class UnaryExpression extends TemplateExpression { optional: true, }) - if (inner === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { - return inner - } + // if (inner === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { + // return inner + // } return this.transform(inner) } @@ -239,16 +236,16 @@ export function isTruthy(v: CollectionOrValue): boolean { } export class LogicalOrExpression extends LogicalExpression { - override evaluate(args: EvaluateArgs): TemplateEvaluationResult> { + override evaluate(args: ASTEvaluateArgs): ASTEvaluationResult> { const left = this.left.evaluate({ ...args, optional: true, }) - if (left === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { - // If key might be available later, we can't decide which branch to take in the logical expression yet. - return left - } + // if (left === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { + // // If key might be available later, we can't decide which branch to take in the logical expression yet. + // return left + // } if (!isNotFound(left) && isTruthy(left)) { return left @@ -259,16 +256,16 @@ export class LogicalOrExpression extends LogicalExpression { } export class LogicalAndExpression extends LogicalExpression { - override evaluate(args: EvaluateArgs): TemplateEvaluationResult> { + override evaluate(args: ASTEvaluateArgs): ASTEvaluationResult> { const left = this.left.evaluate({ ...args, optional: true, }) - if (left === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { - // If key might be available later, we can't decide which branch to take in the logical expression yet. - return left - } + // if (left === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { + // // If key might be available later, we can't decide which branch to take in the logical expression yet. + // return left + // } // We return false in case the variable could not be resolved. This is a quirk of Garden's template language that we want to keep for backwards compatibility. if (isNotFound(left)) { @@ -284,10 +281,10 @@ export class LogicalAndExpression extends LogicalExpression { optional: true, }) - if (right === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { - // If key might be available later, we can't decide on a final value yet and the logical expression needs to be reevaluated later. - return right - } + // if (right === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { + // // If key might be available later, we can't decide on a final value yet and the logical expression needs to be reevaluated later. + // return right + // } if (isNotFound(right)) { return false @@ -308,7 +305,7 @@ export abstract class BinaryExpression extends TemplateExpression { super() } - override evaluate(args: EvaluateArgs): TemplateEvaluationResult> { + override evaluate(args: ASTEvaluateArgs): ASTEvaluationResult> { const left = this.left.evaluate(args) if (typeof left === "symbol") { @@ -327,7 +324,7 @@ export abstract class BinaryExpression extends TemplateExpression { abstract transform( left: CollectionOrValue, right: CollectionOrValue, - params: EvaluateArgs + params: ASTEvaluateArgs ): CollectionOrValue } @@ -347,7 +344,7 @@ export class AddExpression extends BinaryExpression { override transform( left: CollectionOrValue, right: CollectionOrValue, - { yamlSource }: EvaluateArgs + { yamlSource }: ASTEvaluateArgs ): CollectionOrValue { if (isNumber(left) && isNumber(right)) { return left + right @@ -371,7 +368,7 @@ export class ContainsExpression extends BinaryExpression { override transform( collection: CollectionOrValue, element: CollectionOrValue, - { yamlSource }: EvaluateArgs + { yamlSource }: ASTEvaluateArgs ): boolean { if (!isTemplatePrimitive(element)) { throw new TemplateStringError({ @@ -405,7 +402,7 @@ export abstract class BinaryExpressionOnNumbers extends BinaryExpression { override transform( left: CollectionOrValue, right: CollectionOrValue, - { yamlSource }: EvaluateArgs + { yamlSource }: ASTEvaluateArgs ): CollectionOrValue { // All other operators require numbers to make sense (we're not gonna allow random JS weirdness) if (!isNumber(left) || !isNumber(right)) { @@ -482,7 +479,7 @@ export class FormatStringExpression extends TemplateExpression { super() } - override evaluate(args: EvaluateArgs): TemplateEvaluationResult> { + override evaluate(args: ASTEvaluateArgs): ASTEvaluationResult> { const result = this.innerExpression.evaluate({ ...args, opts: { @@ -545,7 +542,7 @@ export class BlockExpression extends AbstractBlockExpression { return this.expressions } - override evaluate(args: EvaluateArgs): TemplateEvaluationResult> { + override evaluate(args: ASTEvaluateArgs): ASTEvaluationResult> { const legacyAllowPartial = args.opts.legacyAllowPartial let result: string = "" @@ -606,7 +603,7 @@ export class IfBlockExpression extends AbstractBlockExpression { ) } - override evaluate(args: EvaluateArgs): TemplateEvaluationResult> { + override evaluate(args: ASTEvaluateArgs): ASTEvaluationResult> { const condition = this.condition.evaluate(args) if (typeof condition === "symbol") { @@ -666,7 +663,7 @@ export class MemberExpression extends TemplateExpression { super() } - override evaluate(args: EvaluateArgs): TemplateEvaluationResult { + override evaluate(args: ASTEvaluateArgs): ASTEvaluationResult { const inner = this.innerExpression.evaluate(args) if (typeof inner === "symbol") { @@ -699,7 +696,7 @@ export class ContextLookupExpression extends TemplateExpression { opts, optional, yamlSource, - }: EvaluateArgs): TemplateEvaluationResult> { + }: ASTEvaluateArgs): ASTEvaluationResult> { const keyPath: (string | number)[] = [] for (const k of this.keyPath) { const evaluated = k.evaluate({ context, opts, optional, yamlSource }) @@ -711,9 +708,9 @@ export class ContextLookupExpression extends TemplateExpression { const { resolved, getUnavailableReason } = this.resolveContext(context, keyPath, opts, yamlSource) - if ((opts.allowPartial || opts.allowPartialContext) && resolved === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { - return resolved - } + // if ((opts.allowPartial || opts.allowPartialContext) && resolved === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { + // return resolved + // } // if we encounter a key not found symbol, it's an error unless the optional flag is true, which is used by // logical operators and expressions, as well as the optional suffix in FormatStringExpression. @@ -769,7 +766,7 @@ export class FunctionCallExpression extends TemplateExpression { super() } - override evaluate(args: EvaluateArgs): TemplateEvaluationResult> { + override evaluate(args: ASTEvaluateArgs): ASTEvaluationResult> { const functionArgs: CollectionOrValue[] = [] for (const functionArg of this.args) { const result = functionArg.evaluate(args) @@ -867,15 +864,15 @@ export class TernaryExpression extends TemplateExpression { super() } - override evaluate(args: EvaluateArgs): TemplateEvaluationResult> { + override evaluate(args: ASTEvaluateArgs): ASTEvaluationResult> { const conditionResult = this.condition.evaluate({ ...args, optional: true, }) - if (conditionResult === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { - return conditionResult - } + // if (conditionResult === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { + // return conditionResult + // } // evaluate ternary expression const evaluationResult = diff --git a/core/src/template/capture.ts b/core/src/template/capture.ts new file mode 100644 index 0000000000..81edaabebd --- /dev/null +++ b/core/src/template/capture.ts @@ -0,0 +1,60 @@ +/* + * Copyright (C) 2018-2024 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import type { ContextResolveOutput, ContextResolveParams } from "../config/template-contexts/base.js" +import { ConfigContext } from "../config/template-contexts/base.js" +import { NotImplementedError } from "../exceptions.js" +import { deepMap } from "../util/objects.js" +import type { TemplateExpressionGenerator } from "./analysis.js" +import type { EvaluateTemplateArgs, ParsedTemplate, ResolvedTemplate } from "./types.js" +import { UnresolvedTemplateValue } from "./types.js" + +export function capture(template: T, context: ConfigContext): T { + return deepMap(template, (v) => { + if (v instanceof UnresolvedTemplateValue) { + return new CapturedContextTemplateValue(v, context) + } + return v + }) as T +} + +export class LayeredContext extends ConfigContext { + readonly #contexts: ConfigContext[] + constructor(...contexts: ConfigContext[]) { + super() + this.#contexts = contexts + } + override resolve(_args: ContextResolveParams): ContextResolveOutput { + throw new NotImplementedError({ message: "TODO" }) + } +} + +export class CapturedContextTemplateValue extends UnresolvedTemplateValue { + readonly #wrapped: UnresolvedTemplateValue + readonly #context: ConfigContext + + constructor(wrapped: UnresolvedTemplateValue, context: ConfigContext) { + super() + this.#wrapped = wrapped + this.#context = context + } + + override evaluate(args: EvaluateTemplateArgs): ResolvedTemplate { + const context = new LayeredContext(this.#context, args.context) + + return this.#wrapped.evaluate({ ...args, context }) + } + + override toJSON(): ResolvedTemplate { + return this.#wrapped.toJSON() + } + + override *visitAll(): TemplateExpressionGenerator { + yield* this.#wrapped.visitAll() + } +} diff --git a/core/src/template/errors.ts b/core/src/template/errors.ts new file mode 100644 index 0000000000..e9ef3529c5 --- /dev/null +++ b/core/src/template/errors.ts @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2018-2024 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import truncate from "lodash-es/truncate.js" +import type { ConfigSource } from "../config/validation.js" +import { addYamlContext } from "../config/validation.js" +import type { GardenErrorParams } from "../exceptions.js" +import { GardenError } from "../exceptions.js" +import { styles } from "../logger/styles.js" +import type { Location } from "./ast.js" + +export class TemplateError extends GardenError { + type = "template" + + constructor(params: GardenErrorParams & { source: ConfigSource }) { + let enriched: string = params.message + try { + enriched = addYamlContext({ source: params.source, message: params.message }) + } catch { + // ignore + } + + super({ ...params, message: enriched }) + } +} + +export class TemplateStringError extends GardenError { + type = "template-string" + + loc: Location + originalMessage: string + + constructor(params: GardenErrorParams & { loc: Location; yamlSource: ConfigSource }) { + let enriched: string = params.message + try { + // TODO: Use Location information from parser to point to the specific part + enriched = addYamlContext({ source: params.yamlSource, message: params.message }) + } catch { + // ignore + } + + if (enriched === params.message) { + const { path } = params.yamlSource + + const pathDescription = path.length > 0 ? ` at path ${styles.accent(path.join("."))}` : "" + const prefix = `Invalid template string (${styles.accent( + truncate(params.loc.source.rawTemplateString, { length: 200 }).replace(/\n/g, "\\n") + )})${pathDescription}: ` + enriched = params.message.startsWith(prefix) ? params.message : prefix + params.message + } + + super({ ...params, message: enriched }) + this.loc = params.loc + this.originalMessage = params.message + } +} diff --git a/core/src/template/evaluate.ts b/core/src/template/evaluate.ts new file mode 100644 index 0000000000..e68f06035b --- /dev/null +++ b/core/src/template/evaluate.ts @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2018-2024 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import set from "lodash-es/set.js" +import { InternalError } from "../exceptions.js" +import { isArray, isPlainObject } from "../util/objects.js" +import { visitAll } from "./analysis.js" +import type { EvaluateTemplateArgs, ParsedTemplateValue, ResolvedTemplate, TemplatePrimitive } from "./types.js" +import { isTemplatePrimitive, UnresolvedTemplateValue, type ParsedTemplate } from "./types.js" + +type Evaluate = T extends UnresolvedTemplateValue + ? ResolvedTemplate + : T extends Array + ? V extends ParsedTemplate + ? Array> + : ResolvedTemplate + : T extends { [k: string]: unknown } + ? { [P in keyof T]: T[P] extends ParsedTemplate ? Evaluate : ResolvedTemplate } + : T extends TemplatePrimitive + ? T + : ResolvedTemplate + +type _test1 = Evaluate<{ foo: UnresolvedTemplateValue }> +type _test2 = Evaluate<{ foo: "foo" }> +type _test3 = Evaluate +export function deepEvaluate( + collection: Input, + args: EvaluateTemplateArgs +): Evaluate { + if (!isArray(collection) && !isPlainObject(collection)) { + return evaluate(collection, args) as Evaluate + } + const result = isArray(collection) ? [] : {} + + for (const { value, yamlSource } of visitAll({ value: collection, source: { path: [] } })) { + if (isTemplatePrimitive(value) || value instanceof UnresolvedTemplateValue) { + const evaluated = evaluate(value, args) + set(result, yamlSource.path, evaluated) + } + } + + return result as Evaluate +} + +export function evaluate( + value: Input, + args: Args +): Evaluate { + if (!(value instanceof UnresolvedTemplateValue)) { + return value as Evaluate + } + + const result = value.evaluate(args) + + if (typeof result === "symbol") { + throw new InternalError({ + message: `Evaluation was non-optional, but template expression returned symbol ${String(result)}. ast.ContextLookupExpression should have thrown an error.`, + }) + } + + return result as Evaluate +} diff --git a/core/src/template-string/date-functions.ts b/core/src/template/functions/date.ts similarity index 98% rename from core/src/template-string/date-functions.ts rename to core/src/template/functions/date.ts index 6ddf122969..870af60b44 100644 --- a/core/src/template-string/date-functions.ts +++ b/core/src/template/functions/date.ts @@ -6,8 +6,8 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import type { TemplateHelperFunction } from "./functions.js" -import { joi } from "../config/common.js" +import type { TemplateHelperFunction } from "./index.js" +import { joi } from "../../config/common.js" import { format as formatFns, add, type Duration } from "date-fns" import { UTCDateMini } from "@date-fns/utc" diff --git a/core/src/template-string/functions.ts b/core/src/template/functions/index.ts similarity index 97% rename from core/src/template-string/functions.ts rename to core/src/template/functions/index.ts index 10253b6229..56f870e10a 100644 --- a/core/src/template-string/functions.ts +++ b/core/src/template/functions/index.ts @@ -8,17 +8,17 @@ import { v4 as uuidv4 } from "uuid" import { createHash } from "node:crypto" -import { GardenError } from "../exceptions.js" +import { GardenError } from "../../exceptions.js" import { camelCase, escapeRegExp, isArrayLike, isEmpty, isString, kebabCase, keyBy, mapValues, trim } from "lodash-es" -import type { JoiDescription, Primitive } from "../config/common.js" -import { joi, joiPrimitive } from "../config/common.js" +import type { JoiDescription, Primitive } from "../../config/common.js" +import { joi, joiPrimitive } from "../../config/common.js" import type Joi from "@hapi/joi" import { load, loadAll } from "js-yaml" -import { safeDumpYaml } from "../util/serialization.js" +import { safeDumpYaml } from "../../util/serialization.js" import indentString from "indent-string" -import { dateHelperFunctionSpecs } from "./date-functions.js" -import type { CollectionOrValue } from "../util/objects.js" -import type { TemplatePrimitive } from "./types.js" +import { dateHelperFunctionSpecs } from "./date.js" +import type { CollectionOrValue } from "../../util/objects.js" +import type { TemplatePrimitive } from "../types.js" export class TemplateFunctionCallError extends GardenError { type = "template-function-call" diff --git a/core/src/template-string/parser.pegjs b/core/src/template/parser.pegjs similarity index 100% rename from core/src/template-string/parser.pegjs rename to core/src/template/parser.pegjs diff --git a/core/src/template/templated-collections.ts b/core/src/template/templated-collections.ts new file mode 100644 index 0000000000..684a8ba5a8 --- /dev/null +++ b/core/src/template/templated-collections.ts @@ -0,0 +1,465 @@ +/* + * Copyright (C) 2018-2024 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import type { ObjectPath } from "../config/base.js" +import { GenericContext } from "../config/template-contexts/base.js" +import type { ConfigSource } from "../config/validation.js" +import { InternalError } from "../exceptions.js" +import { deepMap, isArray, isPlainObject, type CollectionOrValue } from "../util/objects.js" +import { naturalList } from "../util/string.js" +import { isTruthy } from "./ast.js" +import type { EvaluateTemplateArgs, ParsedTemplate, ResolvedTemplate } from "./types.js" +import { isTemplatePrimitive, UnresolvedTemplateValue, type TemplatePrimitive } from "./types.js" +import isBoolean from "lodash-es/isBoolean.js" +import { + arrayConcatKey, + arrayForEachFilterKey, + arrayForEachKey, + arrayForEachReturnKey, + conditionalElseKey, + conditionalKey, + conditionalThenKey, + objectSpreadKey, +} from "../config/constants.js" +import mapValues from "lodash-es/mapValues.js" +import { deepEvaluate } from "./evaluate.js" +import { LayeredContext } from "./capture.js" +import { parseTemplateString } from "./templated-strings.js" +import { TemplateError } from "./errors.js" +import { visitAll, type TemplateExpressionGenerator } from "./analysis.js" + +export function pushYamlPath(part: ObjectPath[0], configSource: ConfigSource): ConfigSource { + return { + ...configSource, + path: [...configSource.path, part], + } +} +type MaybeTplString = `${string}\${${string}` +type Parse> = T extends MaybeTplString + ? ParsedTemplate + : T extends Array + ? V extends CollectionOrValue + ? Array> + : V + : T extends { [k: string]: unknown } + ? { [P in keyof T]: T[P] extends CollectionOrValue ? Parse : T[P] } + : T extends TemplatePrimitive + ? T + : T + +/** + * Recursively parses and resolves all templated strings in the given object. + * + * @argument value The result of the YAML parser. + */ +export function parseTemplateCollection>({ + value, + source, + untemplatableKeys = [], +}: { + value: Input + source: ConfigSource + untemplatableKeys?: string[] +}): Parse { + if (!source) { + throw new InternalError({ + message: "Source parameter is required for parseTemplateCollection.", + }) + } + if (typeof value === "string") { + return parseTemplateString({ + rawTemplateString: value, + source, + }) as Parse + } else if (isTemplatePrimitive(value)) { + return value as Parse + } else if (isArray(value)) { + const parsed = value.map((v, i) => parseTemplateCollection({ value: v, source: pushYamlPath(i, source) })) + + if (value.some((v) => isPlainObject(v) && v[arrayConcatKey] !== undefined)) { + return new ConcatLazyValue(source, parsed) as Parse + } else { + return parsed as Parse + } + } else if (isPlainObject(value)) { + if (value[arrayForEachKey] !== undefined) { + const unexpectedKeys = Object.keys(value).filter((k) => !ForEachLazyValue.allowedForEachKeys.includes(k)) + + if (unexpectedKeys.length > 0) { + const extraKeys = naturalList(unexpectedKeys.map((k) => JSON.stringify(k))) + + throw new TemplateError({ + message: `Found one or more unexpected keys on ${arrayForEachKey} object: ${extraKeys}. Allowed keys: ${naturalList( + ForEachLazyValue.allowedForEachKeys + )}`, + source: pushYamlPath(extraKeys[0], source), + }) + } + + if (value[arrayForEachReturnKey] === undefined) { + throw new TemplateError({ + message: `Missing ${arrayForEachReturnKey} field next to ${arrayForEachKey} field. Got ${naturalList( + Object.keys(value) + )}`, + source: pushYamlPath(arrayForEachReturnKey, source), + }) + } + + const parsedCollectionExpression = parseTemplateCollection({ + value: value[arrayForEachKey], + source: pushYamlPath(arrayForEachKey, source), + }) + + const parsedReturnExpression = parseTemplateCollection({ + value: value[arrayForEachReturnKey], + source: pushYamlPath(arrayForEachReturnKey, source), + }) + + const parsedFilterExpression = + value[arrayForEachFilterKey] === undefined + ? undefined + : parseTemplateCollection({ + value: value[arrayForEachFilterKey], + source: pushYamlPath(arrayForEachFilterKey, source), + }) + + const forEach = new ForEachLazyValue(source, { + [arrayForEachKey]: parsedCollectionExpression, + [arrayForEachReturnKey]: parsedReturnExpression, + [arrayForEachFilterKey]: parsedFilterExpression, + }) + + if (parsedReturnExpression?.[arrayConcatKey] !== undefined) { + return new ConcatLazyValue(source, forEach) as Parse + } else { + return forEach as Parse + } + } else if (value[conditionalKey] !== undefined) { + const ifExpression = value[conditionalKey] + const thenExpression = value[conditionalThenKey] + const elseExpression = value[conditionalElseKey] + + if (thenExpression === undefined) { + throw new TemplateError({ + message: `Missing ${conditionalThenKey} field next to ${conditionalKey} field. Got: ${naturalList( + Object.keys(value) + )}`, + source, + }) + } + + const unexpectedKeys = Object.keys(value).filter((k) => !ConditionalLazyValue.allowedConditionalKeys.includes(k)) + + if (unexpectedKeys.length > 0) { + const extraKeys = naturalList(unexpectedKeys.map((k) => JSON.stringify(k))) + + throw new TemplateError({ + message: `Found one or more unexpected keys on ${conditionalKey} object: ${extraKeys}. Allowed: ${naturalList( + ConditionalLazyValue.allowedConditionalKeys + )}`, + source, + }) + } + + return new ConditionalLazyValue(source, { + [conditionalKey]: parseTemplateCollection({ + value: ifExpression, + source: pushYamlPath(conditionalKey, source), + }), + [conditionalThenKey]: parseTemplateCollection({ + value: thenExpression, + source: pushYamlPath(conditionalThenKey, source), + }), + [conditionalElseKey]: + elseExpression === undefined + ? undefined + : parseTemplateCollection({ + value: elseExpression, + source: pushYamlPath(conditionalElseKey, source), + }), + }) as Parse + } else { + const resolved = mapValues(value, (v, k) => { + // if this key is untemplatable, skip parsing this branch of the template tree. + if (untemplatableKeys.includes(k)) { + return v + } + + return parseTemplateCollection({ value: v, source: pushYamlPath(k, source) }) as ParsedTemplate + }) + if (Object.keys(value).some((k) => k === objectSpreadKey)) { + return new ObjectSpreadLazyValue(source, resolved as ObjectSpreadOperation) as Parse + } else { + return resolved as Parse + } + } + } else { + throw new InternalError({ + message: `Got unexpected value type: ${typeof value}`, + }) + } +} + +abstract class StructuralTemplateOperator extends UnresolvedTemplateValue { + #template: ParsedTemplate + constructor( + protected readonly source: ConfigSource, + template: ParsedTemplate + ) { + super() + this.#template = template + } + + override *visitAll(): TemplateExpressionGenerator { + yield* visitAll({ value: this.#template, source: this.source }) + } + + override toJSON(): ResolvedTemplate { + return deepMap(this.#template, (v) => { + if (!(v instanceof UnresolvedTemplateValue)) { + return v + } + return v.toJSON() + }) + } +} + +type ConcatOperator = { [arrayConcatKey]: CollectionOrValue } + +export class ConcatLazyValue extends StructuralTemplateOperator { + constructor( + source: ConfigSource, + private readonly yaml: (ConcatOperator | ParsedTemplate)[] | ForEachLazyValue + ) { + super(source, yaml) + } + + override evaluate(args: EvaluateTemplateArgs): ResolvedTemplate[] { + const output: ResolvedTemplate[] = [] + + let concatYaml: (ConcatOperator | ParsedTemplate)[] + + // NOTE(steffen): We need to support a construct where $concat inside a $forEach expression results in a flat list. + if (this.yaml instanceof ForEachLazyValue) { + const res = this.yaml.evaluate(args) + + // if (typeof res === "symbol") { + // return res + // } + + concatYaml = res + } else { + concatYaml = this.yaml + } + + for (const v of concatYaml) { + if (!this.isConcatOperator(v)) { + // it's not a concat operator, it's a list element. + const evaluated = deepEvaluate(v, args) + + // if (typeof evaluated === "symbol") { + // return evaluated + // } + + output.push(evaluated) + continue + } + + // handle concat operator + const toConcatenate = deepEvaluate(v[arrayConcatKey], args) + + // if (typeof toConcatenate === "symbol") { + // return toConcatenate + // } + + if (isArray(toConcatenate)) { + output.push(...toConcatenate) + } else { + throw new TemplateError({ + message: `Value of ${arrayConcatKey} key must be (or resolve to) an array (got ${typeof toConcatenate})`, + source: pushYamlPath(arrayConcatKey, this.source), + }) + } + } + + // input tracking is already being taken care of as we just concatenate arrays + return output + } + + isConcatOperator(v: ConcatOperator | ParsedTemplate): v is ConcatOperator { + if (isPlainObject(v) && v[arrayConcatKey] !== undefined) { + if (Object.keys(v).length > 1) { + const extraKeys = naturalList( + Object.keys(v) + .filter((k) => k !== arrayConcatKey) + .map((k) => JSON.stringify(k)) + ) + throw new TemplateError({ + message: `A list item with a ${arrayConcatKey} key cannot have any other keys (found ${extraKeys})`, + source: pushYamlPath(arrayConcatKey, this.source), + }) + } + return true + } + return false + } +} + +type ForEachClause = { + [arrayForEachKey]: ParsedTemplate // must resolve to an array or plain object, but might be a lazy value + [arrayForEachFilterKey]: ParsedTemplate | undefined // must resolve to boolean, but might be lazy value + [arrayForEachReturnKey]: ParsedTemplate +} + +export class ForEachLazyValue extends StructuralTemplateOperator { + static allowedForEachKeys = [arrayForEachKey, arrayForEachReturnKey, arrayForEachFilterKey] + constructor( + source: ConfigSource, + private readonly yaml: ForEachClause + ) { + super(source, yaml) + } + + override evaluate(args: EvaluateTemplateArgs): ResolvedTemplate[] { + const collectionValue = deepEvaluate(this.yaml[arrayForEachKey], args) + + // if (typeof collectionValue === "symbol") { + // return collectionValue + // } + + if (!isArray(collectionValue) && !isPlainObject(collectionValue)) { + throw new TemplateError({ + message: `Value of ${arrayForEachKey} key must be (or resolve to) an array or mapping object (got ${typeof collectionValue})`, + source: pushYamlPath(arrayForEachKey, this.source), + }) + } + + const filterExpression = this.yaml[arrayForEachFilterKey] + + const resolveOutput: ResolvedTemplate[] = [] + + for (const i of Object.keys(collectionValue)) { + // put the TemplateValue in the context, not the primitive value, so we have input tracking + const contextForIndex = new GenericContext({ + item: { key: i, value: collectionValue[i] }, + }) + const loopContext = new LayeredContext(contextForIndex, args.context) + + // Check $filter clause output, if applicable + if (filterExpression !== undefined) { + const filterResult = deepEvaluate(filterExpression, { ...args, context: loopContext }) + + // if (typeof filterResult === "symbol") { + // return filterResult + // } + + if (isBoolean(filterResult)) { + if (!filterResult) { + continue + } + } else { + throw new TemplateError({ + message: `${arrayForEachFilterKey} clause in ${arrayForEachKey} loop must resolve to a boolean value (got ${typeof filterResult})`, + source: pushYamlPath(arrayForEachFilterKey, this.source), + }) + } + } + + const returnResult = deepEvaluate(this.yaml[arrayForEachReturnKey], { ...args, context: loopContext }) + + // if (typeof returnResult === "symbol") { + // return returnResult + // } + + resolveOutput.push(returnResult) + } + + return resolveOutput + } +} + +export type ObjectSpreadOperation = { + [objectSpreadKey]: ParsedTemplate + [staticKeys: string]: ParsedTemplate +} +export class ObjectSpreadLazyValue extends StructuralTemplateOperator { + constructor( + source: ConfigSource, + private readonly yaml: ObjectSpreadOperation + ) { + super(source, yaml) + } + + override evaluate(args: EvaluateTemplateArgs): Record { + let output: Record = {} + + // Resolve $merge keys, depth-first, leaves-first + for (const [k, v] of Object.entries(this.yaml)) { + const resolved = deepEvaluate(v, args) + + // if (typeof resolved === "symbol") { + // return resolved + // } + + if (k !== objectSpreadKey) { + output[k] = resolved + continue + } + + k satisfies typeof objectSpreadKey + + if (!isPlainObject(resolved)) { + throw new TemplateError({ + message: `Value of ${objectSpreadKey} key must be (or resolve to) a mapping object (got ${typeof resolved})`, + source: pushYamlPath(k, this.source), + }) + } + + output = { ...output, ...resolved } + } + + return output + } +} + +export type ConditionalClause = { + [conditionalKey]: ParsedTemplate // must resolve to a boolean, but might be a lazy value + [conditionalThenKey]: ParsedTemplate + [conditionalElseKey]?: ParsedTemplate +} +export class ConditionalLazyValue extends StructuralTemplateOperator { + static allowedConditionalKeys = [conditionalKey, conditionalThenKey, conditionalElseKey] + + constructor( + source: ConfigSource, + private readonly yaml: ConditionalClause + ) { + super(source, yaml) + } + + override evaluate(args: EvaluateTemplateArgs): ResolvedTemplate { + const conditionalValue = deepEvaluate(this.yaml[conditionalKey], args) + + // if (typeof conditionalValue === "symbol") { + // return conditionalValue + // } + + if (typeof conditionalValue !== "boolean") { + throw new TemplateError({ + message: `Value of ${conditionalKey} key must be (or resolve to) a boolean (got ${typeof conditionalValue})`, + source: pushYamlPath(conditionalKey, this.source), + }) + } + + const branch = isTruthy(conditionalValue) ? this.yaml[conditionalThenKey] : this.yaml[conditionalElseKey] + + const evaluated = deepEvaluate(branch, args) + + return evaluated + } +} diff --git a/core/src/template/templated-strings.ts b/core/src/template/templated-strings.ts new file mode 100644 index 0000000000..a805803b78 --- /dev/null +++ b/core/src/template/templated-strings.ts @@ -0,0 +1,227 @@ +/* + * Copyright (C) 2018-2024 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import type { GardenErrorParams } from "../exceptions.js" +import { InternalError, NotImplementedError } from "../exceptions.js" +import type { ConfigContext, ContextResolveOpts } from "../config/template-contexts/base.js" +import type { Primitive } from "../config/common.js" +import { isPrimitive } from "../config/common.js" +import type { CollectionOrValue } from "../util/objects.js" +import { deepMap } from "../util/objects.js" +import type { ConfigSource } from "../config/validation.js" +import * as parser from "./parser.js" +import type { EvaluateTemplateArgs, ResolvedTemplate } from "./types.js" +import { UnresolvedTemplateValue, type TemplatePrimitive } from "./types.js" +import * as ast from "./ast.js" +import { LRUCache } from "lru-cache" +import type { TemplateExpressionGenerator } from "./analysis.js" +import { TemplateStringError } from "./errors.js" + +const escapePrefix = "$${" + +type ParseParams = Parameters + +function parseWithPegJs(params: ParseParams) { + return parser.parse(...params) +} + +const shouldUnescape = (ctxOpts: ContextResolveOpts) => { + // Explicit non-escaping takes the highest priority. + if (ctxOpts.unescape === false) { + return false + } + + return !!ctxOpts.unescape +} + +const parseTemplateStringCache = new LRUCache({ + max: 100000, +}) + +class ParsedTemplateString extends UnresolvedTemplateValue { + constructor( + private readonly source: ConfigSource, + private readonly rootNode: ast.TemplateExpression + ) { + super() + } + + override evaluate(args: EvaluateTemplateArgs): ResolvedTemplate { + const res = this.rootNode.evaluate({ ...args, yamlSource: this.source }) + if (typeof res === "symbol") { + throw new InternalError({ + message: + "ParsedTemplateString: template expression evaluated to symbol. ContextLookupExpression should have thrown.", + }) + } + return res + } + + public override toJSON(): string { + return this.rootNode.rawText + } + + public override *visitAll(): TemplateExpressionGenerator { + yield* this.rootNode.visitAll(this.source) + } +} + +export function parseTemplateString({ + rawTemplateString, + // TODO: remove unescape hacks. + unescape = false, + source, +}: { + rawTemplateString: string + unescape?: boolean + source: ConfigSource +}): ParsedTemplateString | string { + // Just return immediately if this is definitely not a template string + if (!maybeTemplateString(rawTemplateString)) { + return rawTemplateString + } + + const key = `u-${unescape ? "1" : "0"}-${rawTemplateString}` + const cached = parseTemplateStringCache.get(key) + + if (cached instanceof ast.TemplateExpression) { + return new ParsedTemplateString(source, cached) + } else if (cached) { + return cached + } + + const templateStringSource: ast.TemplateStringSource = { + rawTemplateString, + } + + class ParserError extends TemplateStringError { + constructor(params: GardenErrorParams & { loc: ast.Location }) { + super({ + ...params, + yamlSource: source, + }) + } + } + + const parsed: ast.TemplateExpression = parseWithPegJs([ + rawTemplateString, + { + ast, + escapePrefix, + optionalSuffix: "}?", + parseNested: (nested: string) => parseTemplateString({ rawTemplateString: nested, unescape, source }), + TemplateStringError: ParserError, + unescape, + grammarSource: templateStringSource, + }, + ]) + + parseTemplateStringCache.set(key, parsed) + + return new ParsedTemplateString(source, parsed) +} + +/** + * Parse and resolve a templated string, with the given context. The template format is similar to native JS templated + * strings but only supports simple lookups from the given context, e.g. "prefix-${nested.key}-suffix", and not + * arbitrary JS code. + * + * The context should be a ConfigContext instance. The optional `stack` parameter is used to detect circular + * dependencies when resolving context variables. + */ +export function resolveTemplateString({ + string, + context, + contextOpts = {}, + source, +}: { + string: string + context: ConfigContext + contextOpts?: ContextResolveOpts + source?: ConfigSource +}): CollectionOrValue { + if (source === undefined) { + source = { + path: [], + yamlDoc: undefined, + } + } + + const parsed = parseTemplateString({ + rawTemplateString: string, + // TODO: remove unescape hacks. + unescape: shouldUnescape(contextOpts), + source, + }) + + // string does not contain + if (typeof parsed === "string") { + return parsed + } + + const result = parsed.evaluate({ + context, + opts: contextOpts, + }) + + if (typeof result !== "symbol") { + return result + } + + throw new InternalError({ + message: `template expression returned symbol ${String(result)}. ast.ContextLookupExpression should have thrown an error.`, + }) + + // Requested partial evaluation and the template expression cannot be evaluated yet. We may be able to do it later. + + // TODO: Parse all template expressions after reading the YAML config and only re-evaluate ast.TemplateExpression instances in + // resolveTemplateStrings; Otherwise we'll inevitably have a bug where garden will resolve template expressions that might be + // contained in expression evaluation results e.g. if an environment variable contains template string, we don't want to + // evaluate the template string in there. + // See also https://github.com/garden-io/garden/issues/5825 +} + +/** + * Recursively parses and resolves all templated strings in the given object. + */ +// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-constraint +export function resolveTemplateStrings(_args: { + value: T + context: ConfigContext + contextOpts?: ContextResolveOpts + source: ConfigSource | undefined +}): T { + throw new NotImplementedError({ message: "TODO" }) +} + +/** + * Returns `true` if the given value is a string and looks to contain a template string. + */ +export function maybeTemplateString(value: Primitive) { + return !!value && typeof value === "string" && value.includes("${") +} + +/** + * Returns `true` if the given value or any value in a given object or array seems to contain a template string. + */ +export function mayContainTemplateString(obj: any): boolean { + let out = false + + if (isPrimitive(obj)) { + return maybeTemplateString(obj) + } + + // TODO: use visitAll instead. + deepMap(obj, (v) => { + if (maybeTemplateString(v)) { + out = true + } + }) + + return out +} diff --git a/core/src/template/types.ts b/core/src/template/types.ts new file mode 100644 index 0000000000..3711af4e87 --- /dev/null +++ b/core/src/template/types.ts @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2018-2024 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import type { Primitive } from "utility-types" +import { isPrimitive } from "utility-types" +import type { CollectionOrValue } from "../util/objects.js" +import type { ConfigContext, ContextResolveOpts } from "../config/template-contexts/base.js" +import type { TemplateExpressionGenerator } from "./analysis.js" + +export function isTemplatePrimitive(value: unknown): value is TemplatePrimitive { + return isPrimitive(value) && typeof value !== "symbol" +} + +/** + * Primitive types that can be used in the template language. + * + * Symbols have special meanings in the template language implementation and are reserved as an indicator for + * failures to look up variables in the context. See @type TemplateEvaluationResult + */ +export type TemplatePrimitive = Exclude + +/** + * Parsed template values are either: + * - primitives, if the key path did not contain template strings or structural operations + * - an instance of `UnresolvedTemplateValue`, if the key path contained template strings or structural operations + */ +export type ParsedTemplateValue = TemplatePrimitive | UnresolvedTemplateValue + +/** + * Resolved template values are just primitives, strings, numbers, booleans etc. + */ +export type ResolvedTemplateValue = TemplatePrimitive + +/** + * Parsed templates are either: + * - a parsed template value (See @type ParsedTemplateValue) + * - a Record of parsed templates + * - an array of parsed templates + */ +export type ParsedTemplate = CollectionOrValue + +/** + * Resolved templates are either: + * - primitives (See @type TemplatePrimitive) + * - a record of resolved templates + * - an array of resolved templates + */ +export type ResolvedTemplate = CollectionOrValue + +export type EvaluateTemplateArgs = { + readonly context: ConfigContext + readonly opts: Readonly +} + +export abstract class UnresolvedTemplateValue { + public abstract evaluate(args: EvaluateTemplateArgs): ResolvedTemplate + public abstract toJSON(): ResolvedTemplate + + public abstract visitAll(): TemplateExpressionGenerator +} diff --git a/core/test/unit/src/config/service.ts b/core/test/unit/src/config/service.ts index 30256479e2..b3f8fc0e68 100644 --- a/core/test/unit/src/config/service.ts +++ b/core/test/unit/src/config/service.ts @@ -7,6 +7,7 @@ */ import { expect } from "chai" +import type { CommonServiceSpec } from "../../../../src/config/service.js" import { baseServiceSpecSchema } from "../../../../src/config/service.js" import { validateSchema } from "../../../../src/config/validation.js" @@ -16,7 +17,7 @@ describe("baseServiceSpecSchema", () => { name: "foo", dependencies: ["service-a", undefined, "service-b", null, "service-c"], } - const output = validateSchema(input, baseServiceSpecSchema()) + const output = validateSchema(input, baseServiceSpecSchema()) expect(output.dependencies).to.eql(["service-a", "service-b", "service-c"]) }) }) diff --git a/core/test/unit/src/config/template-contexts/base.ts b/core/test/unit/src/config/template-contexts/base.ts index 0e5998dc09..aa446f9b3c 100644 --- a/core/test/unit/src/config/template-contexts/base.ts +++ b/core/test/unit/src/config/template-contexts/base.ts @@ -9,10 +9,7 @@ import { expect } from "chai" import stripAnsi from "strip-ansi" import type { ContextKey, ContextResolveParams } from "../../../../../src/config/template-contexts/base.js" -import { - CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, - CONTEXT_RESOLVE_KEY_NOT_FOUND, -} from "../../../../../src/config/template-contexts/base.js" +import { CONTEXT_RESOLVE_KEY_NOT_FOUND } from "../../../../../src/config/template-contexts/base.js" import { ConfigContext, schema } from "../../../../../src/config/template-contexts/base.js" import { expectError } from "../../../../helpers.js" import { joi } from "../../../../../src/config/common.js" @@ -54,21 +51,21 @@ describe("ConfigContext", () => { expect(stripAnsi(message!())).to.include("Could not find key basic") }) - context("allowPartial=true", () => { - it("should return CONTEXT_RESOLVE_KEY_AVAILABLE_LATER symbol on missing key", async () => { - const c = new GenericContext({}) - const result = resolveKey(c, ["basic"], { allowPartial: true }) - expect(result.resolved).to.eql(CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) - }) - - it("should return CONTEXT_RESOLVE_KEY_AVAILABLE_LATER symbol on missing key on nested context", async () => { - const c = new GenericContext({ - nested: new GenericContext({ key: "value" }), - }) - const result = resolveKey(c, ["nested", "bla"], { allowPartial: true }) - expect(result.resolved).to.eql(CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) - }) - }) + // context("allowPartial=true", () => { + // it("should return CONTEXT_RESOLVE_KEY_AVAILABLE_LATER symbol on missing key", async () => { + // const c = new GenericContext({}) + // const result = resolveKey(c, ["basic"], { allowPartial: true }) + // expect(result.resolved).to.eql(CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) + // }) + + // it("should return CONTEXT_RESOLVE_KEY_AVAILABLE_LATER symbol on missing key on nested context", async () => { + // const c = new GenericContext({ + // nested: new GenericContext({ key: "value" }), + // }) + // const result = resolveKey(c, ["nested", "bla"], { allowPartial: true }) + // expect(result.resolved).to.eql(CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) + // }) + // }) it("should throw when looking for nested value on primitive", async () => { const c = new GenericContext({ basic: "value" }) diff --git a/core/test/unit/src/config/template-contexts/module.ts b/core/test/unit/src/config/template-contexts/module.ts index 631047293c..258f2304e0 100644 --- a/core/test/unit/src/config/template-contexts/module.ts +++ b/core/test/unit/src/config/template-contexts/module.ts @@ -43,7 +43,6 @@ describe("ModuleConfigContext", () => { variables: garden.variables, modules, buildPath: module.buildPath, - partialRuntimeResolution: false, name: module.name, path: module.path, parentName: module.parentName, diff --git a/core/test/unit/src/config/template-contexts/project.ts b/core/test/unit/src/config/template-contexts/project.ts index 64fc541b96..e9708f827c 100644 --- a/core/test/unit/src/config/template-contexts/project.ts +++ b/core/test/unit/src/config/template-contexts/project.ts @@ -10,7 +10,7 @@ import { expect } from "chai" import stripAnsi from "strip-ansi" import type { ConfigContext } from "../../../../../src/config/template-contexts/base.js" import { DefaultEnvironmentContext, ProjectConfigContext } from "../../../../../src/config/template-contexts/project.js" -import { resolveTemplateString } from "../../../../../src/template-string/template-string.js" +import { resolveTemplateString } from "../../../../../src/template/templated-strings.js" import { deline } from "../../../../../src/util/string.js" import type { TestGarden } from "../../../../helpers.js" import { freezeTime, makeTestGardenA } from "../../../../helpers.js" diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index daaab373d5..990228e919 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -10,23 +10,17 @@ import { expect } from "chai" import repeat from "lodash-es/repeat.js" import stripAnsi from "strip-ansi" import { loadAndValidateYaml } from "../../../src/config/base.js" -import { CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, GenericContext } from "../../../src/config/template-contexts/base.js" -import { TemplateStringError } from "../../../src/exceptions.js" -import type { ContextLookupReferenceFinding } from "../../../src/template-string/static-analysis.js" -import { - getContextLookupReferences, - UnresolvableValue, - visitAll, -} from "../../../src/template-string/static-analysis.js" -import { - getActionTemplateReferences, - resolveTemplateString, - resolveTemplateStrings, - throwOnMissingSecretKeys, -} from "../../../src/template-string/template-string.js" +import { CONTEXT_RESOLVE_KEY_NOT_FOUND, GenericContext } from "../../../src/config/template-contexts/base.js" +import type { ContextLookupReferenceFinding } from "../../../src/template/analysis.js" +import { getContextLookupReferences, UnresolvableValue, visitAll } from "../../../src/template/analysis.js" +import { resolveTemplateString, resolveTemplateStrings } from "../../../src/template/templated-strings.js" import { dedent } from "../../../src/util/string.js" import type { TestGarden } from "../../helpers.js" import { expectError, expectFuzzyMatch, getDataDir, makeTestGarden } from "../../helpers.js" +import { TemplateStringError } from "../../../src/template/errors.js" +import { getActionTemplateReferences } from "../../../src/config/references.js" +import { throwOnMissingSecretKeys } from "../../../src/config/secrets.js" +import { parseTemplateCollection } from "../../../src/template/templated-collections.js" describe("resolveTemplateString", () => { it("should return a non-templated string unchanged", () => { @@ -57,37 +51,37 @@ describe("resolveTemplateString", () => { expect(res).to.equal(undefined) }) - it("should pass optional string through if allowPartial=true", () => { - const res = resolveTemplateString({ - string: "${foo}?", - context: new GenericContext({}), - contextOpts: { allowPartial: true }, - }) - expect(res).to.equal("${foo}?") - }) - - it("should not crash when variable in a member expression cannot be resolved with allowPartial=true", () => { - const inputs = [ - '${actions.run["${inputs.deployableTarget}-dummy"].var}', - '${actions.build["${parent.name}"].outputs.deployment-image-id}', - '${actions.build["${parent.name}?"]}', - ] - for (const input of inputs) { - const res = resolveTemplateString({ - string: input, - context: new GenericContext({ - actions: { - run: {}, - build: {}, - }, - }), - contextOpts: { allowPartial: true }, - }) - expect(res).to.equal(input) - } - }) - - it("should fail if optional expression in member expression cannot be resolved with allowPartial=false", async () => { + // it("should pass optional string through if allowPartial=true", () => { + // const res = resolveTemplateString({ + // string: "${foo}?", + // context: new GenericContext({}), + // contextOpts: { allowPartial: true }, + // }) + // expect(res).to.equal("${foo}?") + // }) + + // it("should not crash when variable in a member expression cannot be resolved with allowPartial=true", () => { + // const inputs = [ + // '${actions.run["${inputs.deployableTarget}-dummy"].var}', + // '${actions.build["${parent.name}"].outputs.deployment-image-id}', + // '${actions.build["${parent.name}?"]}', + // ] + // for (const input of inputs) { + // const res = resolveTemplateString({ + // string: input, + // context: new GenericContext({ + // actions: { + // run: {}, + // build: {}, + // }, + // }), + // contextOpts: { allowPartial: true }, + // }) + // expect(res).to.equal(input) + // } + // }) + + it("should fail if optional expression in member expression cannot be resolved", async () => { await expectError( () => resolveTemplateString({ @@ -97,7 +91,7 @@ describe("resolveTemplateString", () => { build: {}, }, }), - contextOpts: { allowPartial: false }, + contextOpts: {}, }), { contains: @@ -111,14 +105,14 @@ describe("resolveTemplateString", () => { expect(res).to.equal("${bar}") }) - it("should pass through a template string with a double $$ prefix if allowPartial=true", () => { - const res = resolveTemplateString({ - string: "$${bar}", - context: new GenericContext({}), - contextOpts: { allowPartial: true }, - }) - expect(res).to.equal("$${bar}") - }) + // it("should pass through a template string with a double $$ prefix if allowPartial=true", () => { + // const res = resolveTemplateString({ + // string: "$${bar}", + // context: new GenericContext({}), + // contextOpts: { allowPartial: true }, + // }) + // expect(res).to.equal("$${bar}") + // }) it("should allow unescaping a template string with a double $$ prefix", () => { const res = resolveTemplateString({ @@ -187,7 +181,7 @@ describe("resolveTemplateString", () => { const res = resolveTemplateString({ string: "$${bar}", context: new GenericContext({}), - contextOpts: { allowPartial: false }, + contextOpts: {}, }) expect(res).to.equal("${bar}") }) @@ -603,78 +597,78 @@ describe("resolveTemplateString", () => { expect(res).to.equal(false) }) - context("partial resolution", () => { - it("a missing reference as the first clause returns the original template", () => { - const res = resolveTemplateString({ - string: "${var.foo && 'a'}", - context: new GenericContext({ var: {} }), - contextOpts: { allowPartial: true }, - }) - expect(res).to.equal("${var.foo && 'a'}") - }) - - it("a missing reference as the second clause returns the original template", () => { - const res = resolveTemplateString({ - string: "${'a' && var.foo}", - context: new GenericContext({ var: {} }), - contextOpts: { allowPartial: true }, - }) - expect(res).to.equal("${'a' && var.foo}") - }) - }) - }) - - context("partial resolution in binary operators", () => { - context("arithmetic operators", () => { - const arithmeticOperators = ["-", "*", "/", "%", ">", ">=", "<", "<="] - for (const operator of arithmeticOperators) { - describe(`with ${operator} operator`, () => { - it("a missing reference as the first clause returns the original template", () => { - const res = resolveTemplateString({ - string: `$\{var.foo ${operator} 2}`, - context: new GenericContext({ var: {} }), - contextOpts: { allowPartial: true }, - }) - expect(res).to.equal(`$\{var.foo ${operator} 2}`) - }) - - it("a missing reference as the second clause returns the original template", () => { - const res = resolveTemplateString({ - string: `$\{2 ${operator} var.foo}`, - context: new GenericContext({ var: {} }), - contextOpts: { allowPartial: true }, - }) - expect(res).to.equal(`$\{2 ${operator} var.foo}`) - }) - }) - } - }) - - context("overloaded operators", () => { - const overLoadedOperators = ["+"] - for (const operator of overLoadedOperators) { - describe(`with ${operator} operator`, () => { - it(`a missing reference as the first clause returns the original template`, () => { - const res = resolveTemplateString({ - string: `$\{var.foo ${operator} '2'}`, - context: new GenericContext({ var: {} }), - contextOpts: { allowPartial: true }, - }) - expect(res).to.equal(`$\{var.foo ${operator} '2'}`) - }) - - it("a missing reference as the second clause returns the original template", () => { - const res = resolveTemplateString({ - string: `$\{2 ${operator} var.foo}`, - context: new GenericContext({ var: {} }), - contextOpts: { allowPartial: true }, - }) - expect(res).to.equal(`$\{2 ${operator} var.foo}`) - }) - }) - } - }) - }) + // context("partial resolution", () => { + // it("a missing reference as the first clause returns the original template", () => { + // const res = resolveTemplateString({ + // string: "${var.foo && 'a'}", + // context: new GenericContext({ var: {} }), + // contextOpts: { allowPartial: true }, + // }) + // expect(res).to.equal("${var.foo && 'a'}") + // }) + + // it("a missing reference as the second clause returns the original template", () => { + // const res = resolveTemplateString({ + // string: "${'a' && var.foo}", + // context: new GenericContext({ var: {} }), + // contextOpts: { allowPartial: true }, + // }) + // expect(res).to.equal("${'a' && var.foo}") + // }) + // }) + }) + + // context("partial resolution in binary operators", () => { + // context("arithmetic operators", () => { + // const arithmeticOperators = ["-", "*", "/", "%", ">", ">=", "<", "<="] + // for (const operator of arithmeticOperators) { + // describe(`with ${operator} operator`, () => { + // it("a missing reference as the first clause returns the original template", () => { + // const res = resolveTemplateString({ + // string: `$\{var.foo ${operator} 2}`, + // context: new GenericContext({ var: {} }), + // contextOpts: { allowPartial: true }, + // }) + // expect(res).to.equal(`$\{var.foo ${operator} 2}`) + // }) + + // it("a missing reference as the second clause returns the original template", () => { + // const res = resolveTemplateString({ + // string: `$\{2 ${operator} var.foo}`, + // context: new GenericContext({ var: {} }), + // contextOpts: { allowPartial: true }, + // }) + // expect(res).to.equal(`$\{2 ${operator} var.foo}`) + // }) + // }) + // } + // }) + + // context("overloaded operators", () => { + // const overLoadedOperators = ["+"] + // for (const operator of overLoadedOperators) { + // describe(`with ${operator} operator`, () => { + // it(`a missing reference as the first clause returns the original template`, () => { + // const res = resolveTemplateString({ + // string: `$\{var.foo ${operator} '2'}`, + // context: new GenericContext({ var: {} }), + // contextOpts: { allowPartial: true }, + // }) + // expect(res).to.equal(`$\{var.foo ${operator} '2'}`) + // }) + + // it("a missing reference as the second clause returns the original template", () => { + // const res = resolveTemplateString({ + // string: `$\{2 ${operator} var.foo}`, + // context: new GenericContext({ var: {} }), + // contextOpts: { allowPartial: true }, + // }) + // expect(res).to.equal(`$\{2 ${operator} var.foo}`) + // }) + // }) + // } + // }) + // }) it("should handle a positive equality comparison between equal resolved values", () => { const res = resolveTemplateString({ string: "${a == b}", context: new GenericContext({ a: "a", b: "a" }) }) @@ -1067,9 +1061,7 @@ describe("resolveTemplateString", () => { for (const [template, expectation] of Object.entries(testCases)) { const result = resolveTemplateString({ string: template, - contextOpts: { - allowPartial: false, - }, + contextOpts: {}, context: new GenericContext({ var: {} }), }) expect(result).to.eq( @@ -1079,34 +1071,34 @@ describe("resolveTemplateString", () => { } }) - context("allowPartial=true", () => { - it("passes through template strings with missing key", () => { - const res = resolveTemplateString({ - string: "${a}", - context: new GenericContext({}), - contextOpts: { allowPartial: true }, - }) - expect(res).to.equal("${a}") - }) - - it("passes through a template string with a missing key in an optional clause", () => { - const res = resolveTemplateString({ - string: "${a || b}", - context: new GenericContext({ b: 123 }), - contextOpts: { allowPartial: true }, - }) - expect(res).to.equal("${a || b}") - }) - - it("passes through a template string with a missing key in a ternary", () => { - const res = resolveTemplateString({ - string: "${a ? b : 123}", - context: new GenericContext({ b: 123 }), - contextOpts: { allowPartial: true }, - }) - expect(res).to.equal("${a ? b : 123}") - }) - }) + // context("allowPartial=true", () => { + // it("passes through template strings with missing key", () => { + // const res = resolveTemplateString({ + // string: "${a}", + // context: new GenericContext({}), + // contextOpts: { allowPartial: true }, + // }) + // expect(res).to.equal("${a}") + // }) + + // it("passes through a template string with a missing key in an optional clause", () => { + // const res = resolveTemplateString({ + // string: "${a || b}", + // context: new GenericContext({ b: 123 }), + // contextOpts: { allowPartial: true }, + // }) + // expect(res).to.equal("${a || b}") + // }) + + // it("passes through a template string with a missing key in a ternary", () => { + // const res = resolveTemplateString({ + // string: "${a ? b : 123}", + // context: new GenericContext({ b: 123 }), + // contextOpts: { allowPartial: true }, + // }) + // expect(res).to.equal("${a ? b : 123}") + // }) + // }) context("when the template string is the full input string", () => { it("should return a resolved number directly", () => { @@ -1161,45 +1153,45 @@ describe("resolveTemplateString", () => { expect(res).to.equal("foo-null") }) - context("allowPartial=true", () => { - it("does not resolve template expressions when 'b' is missing in the context", () => { - const res = resolveTemplateString({ - string: "${a}-${b}", - context: new GenericContext({ a: "foo" }), - contextOpts: { allowPartial: true }, - }) - expect(res).to.equal("${a}-${b}") - }) - - it("does not resolve template expressions when 'a' is missing in the context", () => { - const res = resolveTemplateString({ - string: "${a}-${b}", - context: new GenericContext({ b: "foo" }), - contextOpts: { allowPartial: true }, - }) - expect(res).to.equal("${a}-${b}") - }) - - it("does not resolve template expressions when 'a' is missing in the context when evaluating a conditional expression", () => { - const res = resolveTemplateString({ - string: "${a || b}-${c}", - context: new GenericContext({ b: 123, c: "foo" }), - contextOpts: { - allowPartial: true, - }, - }) - expect(res).to.equal("${a || b}-${c}") - }) - - it("resolves template expressions when the context is fully available", () => { - const res = resolveTemplateString({ - string: "${a}-${b}", - context: new GenericContext({ a: "foo", b: "bar" }), - contextOpts: { allowPartial: true }, - }) - expect(res).to.equal("foo-bar") - }) - }) + // context("allowPartial=true", () => { + // it("does not resolve template expressions when 'b' is missing in the context", () => { + // const res = resolveTemplateString({ + // string: "${a}-${b}", + // context: new GenericContext({ a: "foo" }), + // contextOpts: { allowPartial: true }, + // }) + // expect(res).to.equal("${a}-${b}") + // }) + + // it("does not resolve template expressions when 'a' is missing in the context", () => { + // const res = resolveTemplateString({ + // string: "${a}-${b}", + // context: new GenericContext({ b: "foo" }), + // contextOpts: { allowPartial: true }, + // }) + // expect(res).to.equal("${a}-${b}") + // }) + + // it("does not resolve template expressions when 'a' is missing in the context when evaluating a conditional expression", () => { + // const res = resolveTemplateString({ + // string: "${a || b}-${c}", + // context: new GenericContext({ b: 123, c: "foo" }), + // contextOpts: { + // allowPartial: true, + // }, + // }) + // expect(res).to.equal("${a || b}-${c}") + // }) + + // it("resolves template expressions when the context is fully available", () => { + // const res = resolveTemplateString({ + // string: "${a}-${b}", + // context: new GenericContext({ a: "foo", b: "bar" }), + // contextOpts: { allowPartial: true }, + // }) + // expect(res).to.equal("foo-bar") + // }) + // }) context("legacyAllowPartial=true", () => { it("partially resolves template expressions when 'b' is missing in the context", () => { const res = resolveTemplateString({ @@ -1707,27 +1699,27 @@ describe("resolveTemplateString", () => { ) }) - it("does not apply helper function on unresolved template string and returns string as-is, when allowPartial=true", () => { - const res = resolveTemplateString({ - string: "${base64Encode('${environment.namespace}')}", - context: new GenericContext({}), - contextOpts: { - allowPartial: true, - }, - }) - expect(res).to.equal("${base64Encode('${environment.namespace}')}") - }) - - it("does not apply helper function on unresolved template object and returns string as-is, when allowPartial=true", () => { - const res = resolveTemplateString({ - string: "${base64Encode(var.foo)}", - context: new GenericContext({ foo: { $forEach: ["a", "b"], $return: "${item.value}" } }), - contextOpts: { - allowPartial: true, - }, - }) - expect(res).to.equal("${base64Encode(var.foo)}") - }) + // it("does not apply helper function on unresolved template string and returns string as-is, when allowPartial=true", () => { + // const res = resolveTemplateString({ + // string: "${base64Encode('${environment.namespace}')}", + // context: new GenericContext({}), + // contextOpts: { + // allowPartial: true, + // }, + // }) + // expect(res).to.equal("${base64Encode('${environment.namespace}')}") + // }) + + // it("does not apply helper function on unresolved template object and returns string as-is, when allowPartial=true", () => { + // const res = resolveTemplateString({ + // string: "${base64Encode(var.foo)}", + // context: new GenericContext({ foo: { $forEach: ["a", "b"], $return: "${item.value}" } }), + // contextOpts: { + // allowPartial: true, + // }, + // }) + // expect(res).to.equal("${base64Encode(var.foo)}") + // }) context("concat", () => { it("allows empty strings", () => { @@ -2020,175 +2012,169 @@ describe("resolveTemplateStrings", () => { }) }) - it("should partially resolve $merge keys if a dependency cannot be resolved yet in partial mode", () => { - const obj = { - "key-value-array": { - $forEach: "${inputs.merged-object || []}", - $return: { - name: "${item.key}", - value: "${item.value}", - }, + // it("should partially resolve $merge keys if a dependency cannot be resolved yet in partial mode", () => { + // const obj = { + // "key-value-array": { + // $forEach: "${inputs.merged-object || []}", + // $return: { + // name: "${item.key}", + // value: "${item.value}", + // }, + // }, + // } + // const templateContext = new GenericContext({ + // inputs: { + // "merged-object": { + // $merge: "${var.empty || var.input-object}", + // INTERNAL_VAR_1: "INTERNAL_VAR_1", + // }, + // }, + // var: { + // "input-object": { + // EXTERNAL_VAR_1: "EXTERNAL_VAR_1", + // }, + // }, + // }) + + // const result = resolveTemplateStrings({ + // value: obj, + // context: templateContext, + // contextOpts: { allowPartial: true }, + // source: undefined, + // }) + + // expect(result).to.eql({ + // "key-value-array": { + // $forEach: "${inputs.merged-object || []}", + // $return: { + // name: "${item.key}", + // value: "${item.value}", + // }, + // }, + // }) +}) + +it("should resolve $merge keys if a dependency cannot be resolved but there's a fallback", () => { + const obj = { + "key-value-array": { + $forEach: "${inputs.merged-object || []}", + $return: { + name: "${item.key}", + value: "${item.value}", }, - } - const templateContext = new GenericContext({ - inputs: { - "merged-object": { - $merge: "${var.empty || var.input-object}", - INTERNAL_VAR_1: "INTERNAL_VAR_1", - }, + }, + } + const templateContext = new GenericContext({ + inputs: { + "merged-object": { + $merge: "${var.empty || var.input-object}", + INTERNAL_VAR_1: "INTERNAL_VAR_1", }, - var: { - "input-object": { - EXTERNAL_VAR_1: "EXTERNAL_VAR_1", - }, + }, + var: { + "input-object": { + EXTERNAL_VAR_1: "EXTERNAL_VAR_1", }, - }) + }, + }) - const result = resolveTemplateStrings({ - value: obj, - context: templateContext, - contextOpts: { allowPartial: true }, - source: undefined, - }) + const result = resolveTemplateStrings({ value: obj, context: templateContext, source: undefined }) - expect(result).to.eql({ - "key-value-array": { - $forEach: "${inputs.merged-object || []}", - $return: { - name: "${item.key}", - value: "${item.value}", - }, - }, - }) + expect(result).to.eql({ + "key-value-array": [ + { name: "EXTERNAL_VAR_1", value: "EXTERNAL_VAR_1" }, + { name: "INTERNAL_VAR_1", value: "INTERNAL_VAR_1" }, + ], }) +}) - it("should resolve $merge keys if a dependency cannot be resolved but there's a fallback", () => { - const obj = { - "key-value-array": { - $forEach: "${inputs.merged-object || []}", - $return: { - name: "${item.key}", - value: "${item.value}", - }, - }, - } - const templateContext = new GenericContext({ - inputs: { - "merged-object": { - $merge: "${var.empty || var.input-object}", - INTERNAL_VAR_1: "INTERNAL_VAR_1", - }, - }, - var: { - "input-object": { - EXTERNAL_VAR_1: "EXTERNAL_VAR_1", - }, - }, - }) +it("should ignore $merge keys if the object to be merged is undefined", () => { + const obj = { + $merge: "${var.doesnotexist}", + c: "c", + } + const templateContext = new GenericContext({ var: { obj: { a: "a", b: "b" } } }) - const result = resolveTemplateStrings({ value: obj, context: templateContext, source: undefined }) + expect(() => resolveTemplateStrings({ value: obj, context: templateContext, source: undefined })).to.throw( + "Invalid template string" + ) +}) - expect(result).to.eql({ - "key-value-array": [ - { name: "EXTERNAL_VAR_1", value: "EXTERNAL_VAR_1" }, - { name: "INTERNAL_VAR_1", value: "INTERNAL_VAR_1" }, - ], +context("$concat", () => { + it("handles array concatenation", () => { + const obj = { + foo: ["a", { $concat: ["b", "c"] }, "d"], + } + const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }) + expect(res).to.eql({ + foo: ["a", "b", "c", "d"], }) }) - it("should ignore $merge keys if the object to be merged is undefined", () => { + it("resolves $concat value before spreading", () => { const obj = { - $merge: "${var.doesnotexist}", - c: "c", + foo: ["a", { $concat: "${foo}" }, "d"], } - const templateContext = new GenericContext({ var: { obj: { a: "a", b: "b" } } }) - - expect(() => resolveTemplateStrings({ value: obj, context: templateContext, source: undefined })).to.throw( - "Invalid template string" - ) - }) - - context("$concat", () => { - it("handles array concatenation", () => { - const obj = { - foo: ["a", { $concat: ["b", "c"] }, "d"], - } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }) - expect(res).to.eql({ - foo: ["a", "b", "c", "d"], - }) + const res = resolveTemplateStrings({ + source: undefined, + value: obj, + context: new GenericContext({ foo: ["b", "c"] }), }) - - it("resolves $concat value before spreading", () => { - const obj = { - foo: ["a", { $concat: "${foo}" }, "d"], - } - const res = resolveTemplateStrings({ - source: undefined, - value: obj, - context: new GenericContext({ foo: ["b", "c"] }), - }) - expect(res).to.eql({ - foo: ["a", "b", "c", "d"], - }) + expect(res).to.eql({ + foo: ["a", "b", "c", "d"], }) + }) - it("resolves a $forEach in the $concat clause", () => { - const obj = { - foo: ["a", { $concat: { $forEach: ["B", "C"], $return: "${lower(item.value)}" } }, "d"], - } - const res = resolveTemplateStrings({ - source: undefined, - value: obj, - context: new GenericContext({ foo: ["b", "c"] }), - }) - expect(res).to.eql({ - foo: ["a", "b", "c", "d"], - }) + it("resolves a $forEach in the $concat clause", () => { + const obj = { + foo: ["a", { $concat: { $forEach: ["B", "C"], $return: "${lower(item.value)}" } }, "d"], + } + const res = resolveTemplateStrings({ + source: undefined, + value: obj, + context: new GenericContext({ foo: ["b", "c"] }), }) - - it("throws if $concat value is not an array and allowPartial=false", () => { - const obj = { - foo: ["a", { $concat: "b" }, "d"], - } - - void expectError( - () => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }), - { - contains: "Value of $concat key must be (or resolve to) an array (got string)", - } - ) + expect(res).to.eql({ + foo: ["a", "b", "c", "d"], }) + }) - it("throws if object with $concat key contains other keys as well", () => { - const obj = { - foo: ["a", { $concat: "b", nope: "nay", oops: "derp" }, "d"], - } + it("throws if $concat value is not an array and allowPartial=false", () => { + const obj = { + foo: ["a", { $concat: "b" }, "d"], + } - void expectError( - () => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }), - { - contains: 'A list item with a $concat key cannot have any other keys (found "nope" and "oops")', - } - ) + void expectError(() => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }), { + contains: "Value of $concat key must be (or resolve to) an array (got string)", }) + }) - it("ignores if $concat value is not an array and allowPartial=true", () => { - const obj = { - foo: ["a", { $concat: "${foo}" }, "d"], - } - const res = resolveTemplateStrings({ - source: undefined, - value: obj, - context: new GenericContext({}), - contextOpts: { allowPartial: true }, - }) - expect(res).to.eql({ - foo: ["a", { $concat: "${foo}" }, "d"], - }) + it("throws if object with $concat key contains other keys as well", () => { + const obj = { + foo: ["a", { $concat: "b", nope: "nay", oops: "derp" }, "d"], + } + + void expectError(() => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }), { + contains: 'A list item with a $concat key cannot have any other keys (found "nope" and "oops")', }) }) + // it("ignores if $concat value is not an array and allowPartial=true", () => { + // const obj = { + // foo: ["a", { $concat: "${foo}" }, "d"], + // } + // const res = resolveTemplateStrings({ + // source: undefined, + // value: obj, + // context: new GenericContext({}), + // contextOpts: { allowPartial: true }, + // }) + // expect(res).to.eql({ + // foo: ["a", { $concat: "${foo}" }, "d"], + // }) + // }) + // }) + context("$if objects", () => { it("resolves to $then if $if is true", () => { const obj = { @@ -2225,22 +2211,22 @@ describe("resolveTemplateStrings", () => { expect(res).to.eql({ bar: undefined }) }) - it("returns object as-is if $if doesn't resolve to boolean and allowPartial=true", () => { - const obj = { - bar: { - $if: "${foo}", - $then: 123, - $else: 456, - }, - } - const res = resolveTemplateStrings({ - source: undefined, - value: obj, - context: new GenericContext({ foo: 2 }), - contextOpts: { allowPartial: true }, - }) - expect(res).to.eql(obj) - }) + // it("returns object as-is if $if doesn't resolve to boolean and allowPartial=true", () => { + // const obj = { + // bar: { + // $if: "${foo}", + // $then: 123, + // $else: 456, + // }, + // } + // const res = resolveTemplateStrings({ + // source: undefined, + // value: obj, + // context: new GenericContext({ foo: 2 }), + // contextOpts: { allowPartial: true }, + // }) + // expect(res).to.eql(obj) + // }) it("throws if $if doesn't resolve to boolean and allowPartial=false", () => { const obj = { @@ -2338,21 +2324,21 @@ describe("resolveTemplateStrings", () => { ) }) - it("ignores the loop if the input isn't a list or object and allowPartial=true", () => { - const obj = { - foo: { - $forEach: "${foo}", - $return: "foo", - }, - } - const res = resolveTemplateStrings({ - source: undefined, - value: obj, - context: new GenericContext({}), - contextOpts: { allowPartial: true }, - }) - expect(res).to.eql(obj) - }) + // it("ignores the loop if the input isn't a list or object and allowPartial=true", () => { + // const obj = { + // foo: { + // $forEach: "${foo}", + // $return: "foo", + // }, + // } + // const res = resolveTemplateStrings({ + // source: undefined, + // value: obj, + // context: new GenericContext({}), + // contextOpts: { allowPartial: true }, + // }) + // expect(res).to.eql(obj) + // }) it("throws if there's no $return clause", () => { const obj = { @@ -2456,31 +2442,6 @@ describe("resolveTemplateStrings", () => { ) }) - // TODO: In a future iteration of this code, we should leave the entire $forEach expression unresolved if filter cannot be resolved yet and allowPartial=true. - it("throws if $filter can't be resolved, even if allowPartial=true", () => { - const obj = { - foo: { - $forEach: ["a", "b", "c"], - $filter: "${var.doesNotExist}", - $return: "${item.value}", - }, - } - - void expectError( - () => - resolveTemplateStrings({ - source: undefined, - value: obj, - context: new GenericContext({ var: { fruit: "banana" } }), - contextOpts: { allowPartial: true }, - }), - { - contains: - "invalid template string (${var.doesnotexist}) at path foo.$filter: could not find key doesnotexist under var. available keys: fruit.", - } - ) - }) - it("handles $concat clauses in $return", () => { const obj = { foo: { @@ -2589,20 +2550,22 @@ describe("resolveTemplateStrings", () => { describe("getContextLookupReferences", () => { it("should return all template string references in an object", () => { - const obj = { - foo: "${my.reference}", - nested: { - boo: "${moo}", - foo: "lalalla${moo}${moo}", - banana: "${banana.rama.llama}", + const obj = parseTemplateCollection({ + value: { + foo: "${my.reference}", + nested: { + boo: "${moo}", + foo: "lalalla${moo}${moo}", + banana: "${banana.rama.llama}", + }, }, - } + source: { path: [] }, + }) const result = Array.from( getContextLookupReferences( visitAll({ value: obj, - parseTemplateStrings: true, source: { path: [], }, @@ -2651,26 +2614,30 @@ describe("getContextLookupReferences", () => { }) it("should handle keys with dots and unresolvable member expressions correctly", async () => { - const obj = { - a: "some ${templated['key.with.dots']}", - b: "${more.stuff}", - c: "${keyThatIs[unresolvable]}", - d: '${keyThatIs["${unresolvable}"]}', - e: "${optionalAndUnresolvable}?", - f: "${keyThatIs[availableLater]}", - g: '${keyThatIs["${availableLater}"]}', - } + const obj = parseTemplateCollection({ + value: { + a: "some ${templated['key.with.dots']}", + b: "${more.stuff}", + c: "${keyThatIs[unresolvable]}", + d: '${keyThatIs["${unresolvable}"]}', + e: "${optionalAndUnresolvable}?", + // f: "${keyThatIs[availableLater]}", + // g: '${keyThatIs["${availableLater}"]}', + }, + source: { + path: [], + }, + }) const foundKeys = Array.from( getContextLookupReferences( visitAll({ value: obj, - parseTemplateStrings: true, source: { path: [], }, }), new GenericContext({ - availableLater: CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, + // availableLater: CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, }) ) ) @@ -3006,34 +2973,40 @@ describe("getActionTemplateReferences", () => { }) }) -describe.skip("throwOnMissingSecretKeys", () => { +describe("throwOnMissingSecretKeys", () => { it("should not throw an error if no secrets are referenced", () => { - const configs = [ - { - name: "foo", - foo: "${banana.llama}", - nested: { boo: "${moo}" }, - }, - ] + const configs = parseTemplateCollection({ + value: [ + { + name: "foo", + foo: "${banana.llama}", + nested: { boo: "${moo}" }, + }, + ], + source: { path: [] }, + } as const) throwOnMissingSecretKeys(configs, new GenericContext({}), {}, "Module") throwOnMissingSecretKeys(configs, new GenericContext({}), { someSecret: "123" }, "Module") }) it("should throw an error if one or more secrets is missing", () => { - const configs = [ - { - name: "moduleA", - foo: "${secrets.a}", - nested: { boo: "${secrets.b}" }, - }, - { - name: "moduleB", - bar: "${secrets.a}", - nested: { boo: "${secrets.b}" }, - baz: "${secrets.c}", - }, - ] + const configs = parseTemplateCollection({ + value: [ + { + name: "moduleA", + foo: "${secrets.a}", + nested: { boo: "${secrets.b}" }, + }, + { + name: "moduleB", + bar: "${secrets.a}", + nested: { boo: "${secrets.b}" }, + baz: "${secrets.c}", + }, + ], + source: { path: [] }, + } as const) void expectError( () => throwOnMissingSecretKeys(configs, new GenericContext({}), { b: "123" }, "Module"), diff --git a/core/test/unit/src/types/container.ts b/core/test/unit/src/types/container.ts index 7f9207a08d..6a37b2f39d 100644 --- a/core/test/unit/src/types/container.ts +++ b/core/test/unit/src/types/container.ts @@ -15,7 +15,7 @@ describe("portSchema", () => { const containerPort = 8080 const obj = { name: "a", containerPort } - const value = validateSchema(obj, portSchema()) + const value = validateSchema(obj, portSchema()) expect(value["servicePort"]).to.equal(containerPort) }) @@ -24,7 +24,7 @@ describe("portSchema", () => { const servicePort = 9090 const obj = { name: "a", containerPort, servicePort } - const value = validateSchema(obj, portSchema()) + const value = validateSchema(obj, portSchema()) expect(value["servicePort"]).to.equal(servicePort) }) }) diff --git a/plugins/pulumi/src/commands.ts b/plugins/pulumi/src/commands.ts index b8deea4ce2..b6e0160139 100644 --- a/plugins/pulumi/src/commands.ts +++ b/plugins/pulumi/src/commands.ts @@ -202,7 +202,6 @@ const makePluginContextForDeploy = async ( garden, resolvedProviders, action, - partialRuntimeResolution: false, modules: graph.getModules(), resolvedDependencies: action.getResolvedDependencies(), executedDependencies: action.getExecutedDependencies(), From e2e94705d4d97ae4994d77284a1471900aadf95c Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Mon, 30 Dec 2024 18:11:09 +0100 Subject: [PATCH 002/117] perf: use layered context and capture context instead of eager variable merging --- core/src/actions/base.ts | 9 +- core/src/actions/types.ts | 5 +- core/src/commands/custom.ts | 2 - core/src/config/base.ts | 1 + core/src/config/project.ts | 17 +- core/src/config/template-contexts/actions.ts | 22 +- core/src/config/template-contexts/base.ts | 108 +++++--- .../template-contexts/custom-command.ts | 8 +- core/src/config/template-contexts/module.ts | 2 +- core/src/config/template-contexts/project.ts | 10 +- core/src/config/template-contexts/provider.ts | 2 +- core/src/garden.ts | 32 ++- core/src/graph/actions.ts | 19 +- core/src/graph/common.ts | 28 +- core/src/resolve-module.ts | 23 +- core/src/tasks/resolve-action.ts | 68 +++-- core/src/template/analysis.ts | 6 +- core/src/template/capture.ts | 35 +-- core/src/template/evaluate.ts | 27 +- core/src/template/templated-collections.ts | 242 +++++++++--------- core/src/template/templated-strings.ts | 11 +- core/src/util/testing.ts | 3 +- core/test/unit/src/commands/workflow.ts | 3 +- core/test/unit/src/config/render-template.ts | 2 +- .../unit/src/config/template-contexts/base.ts | 4 +- core/test/unit/src/config/workflow.ts | 3 +- core/test/unit/src/garden.ts | 2 +- core/test/unit/src/tasks/resolve-action.ts | 4 +- 28 files changed, 363 insertions(+), 335 deletions(-) diff --git a/core/src/actions/base.ts b/core/src/actions/base.ts index a0c8df90ed..cde8e38bfa 100644 --- a/core/src/actions/base.ts +++ b/core/src/actions/base.ts @@ -8,7 +8,7 @@ import titleize from "titleize" import type { ConfigGraph, GetActionOpts, PickTypeByKind, ResolvedConfigGraph } from "../graph/config-graph.js" -import type { ActionReference, DeepPrimitiveMap } from "../config/common.js" +import type { ActionReference } from "../config/common.js" import { createSchema, includeGuideLink, @@ -67,6 +67,7 @@ import type { LinkedSource } from "../config-store/local.js" import type { BaseActionTaskParams, ExecuteTask } from "../tasks/base.js" import { styles } from "../logger/styles.js" import { dirname } from "node:path" +import type { ConfigContext } from "../config/template-contexts/base.js" // TODO: split this file @@ -369,7 +370,7 @@ export abstract class BaseAction< protected readonly projectRoot: string protected readonly _supportedModes: ActionModes protected readonly _treeVersion: TreeVersion - protected readonly variables: DeepPrimitiveMap + protected readonly variables: ConfigContext constructor(protected readonly params: ActionWrapperParams) { this.kind = params.config.kind @@ -582,7 +583,7 @@ export abstract class BaseAction< } } - getVariables(): DeepPrimitiveMap { + getVariables(): ConfigContext { return this.variables } @@ -734,7 +735,7 @@ export interface ResolvedActionExtension< getOutputs(): StaticOutputs - getVariables(): DeepPrimitiveMap + getVariables(): ConfigContext } // TODO: see if we can avoid the duplication here with ResolvedBuildAction diff --git a/core/src/actions/types.ts b/core/src/actions/types.ts index 8abf47c19b..a387ff9d94 100644 --- a/core/src/actions/types.ts +++ b/core/src/actions/types.ts @@ -21,6 +21,7 @@ import type { ValidResultType } from "../tasks/base.js" import type { BaseGardenResource, GardenResourceInternalFields } from "../config/base.js" import type { LinkedSource } from "../config-store/local.js" import type { GardenApiVersion } from "../constants.js" +import type { ConfigContext } from "../config/template-contexts/base.js" // TODO: split this file @@ -170,7 +171,7 @@ export interface ActionWrapperParams { remoteSourcePath: string | null supportedModes: ActionModes treeVersion: TreeVersion - variables: DeepPrimitiveMap + variables: ConfigContext } export interface ResolveActionParams = any> { @@ -181,7 +182,7 @@ export interface ResolveActionParams `garden.${environmentName}.env` @@ -618,12 +621,16 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ }) } + // TODO: parse template strings in varfiles? const projectVarfileVars = await loadVarfile({ configRoot: projectConfig.path, path: projectConfig.varfile, defaultPath: defaultVarfilePath, }) - const projectVariables: DeepPrimitiveMap = merge(projectConfig.variables, projectVarfileVars) + const projectVariables: LayeredContext = new LayeredContext( + new GenericContext(projectVarfileVars), + new GenericContext(projectConfig.variables) + ) const source = { yamlDoc: projectConfig.internal.yamlDoc, path: ["environments", index] } @@ -678,7 +685,11 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ defaultPath: defaultEnvVarfilePath(environment), }) - const variables: DeepPrimitiveMap = merge(projectVariables, merge(environmentConfig.variables, envVarfileVars)) + const variables: ConfigContext = new LayeredContext( + new GenericContext(envVarfileVars), + new GenericContext(environmentConfig.variables), + projectVariables + ) return { environmentName: environment, diff --git a/core/src/config/template-contexts/actions.ts b/core/src/config/template-contexts/actions.ts index c261abd03d..a4bf8c2159 100644 --- a/core/src/config/template-contexts/actions.ts +++ b/core/src/config/template-contexts/actions.ts @@ -6,7 +6,6 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { merge } from "lodash-es" import type { ActionConfig, Action, ExecutedAction, ResolvedAction } from "../../actions/types.js" import type { ActionMode } from "../../actions/types.js" import type { Garden } from "../../garden.js" @@ -15,18 +14,15 @@ import { dedent, deline } from "../../util/string.js" import type { DeepPrimitiveMap, PrimitiveMap } from "../common.js" import { joi, joiIdentifier, joiIdentifierMap, joiPrimitive, joiVariables } from "../common.js" import type { ProviderMap } from "../provider.js" -import { ConfigContext, ErrorContext, ParentContext, schema, TemplateContext } from "./base.js" +import { ConfigContext, ErrorContext, GenericContext, ParentContext, schema, TemplateContext } from "./base.js" import { exampleVersion, OutputConfigContext } from "./module.js" import { TemplatableConfigContext } from "./project.js" import { DOCS_BASE_URL } from "../../constants.js" import { styles } from "../../logger/styles.js" +import { LayeredContext } from "./base.js" -function mergeVariables({ garden, variables }: { garden: Garden; variables: DeepPrimitiveMap }): DeepPrimitiveMap { - const mergedVariables: DeepPrimitiveMap = {} - merge(mergedVariables, garden.variables) - merge(mergedVariables, variables) - merge(mergedVariables, garden.variableOverrides) - return mergedVariables +function mergeVariables({ garden, variables }: { garden: Garden; variables: ConfigContext }): LayeredContext { + return new LayeredContext(new GenericContext(garden.variableOverrides), variables, garden.variables) } type ActionConfigThisContextParams = Pick @@ -65,7 +61,7 @@ interface ActionConfigContextParams { garden: Garden config: ActionConfig thisContextParams: ActionConfigThisContextParams - variables: DeepPrimitiveMap + variables: ConfigContext } /** @@ -91,7 +87,7 @@ interface ActionReferenceContextParams { buildPath: string sourcePath: string mode: ActionMode - variables: DeepPrimitiveMap + variables: ConfigContext } export class ActionReferenceContext extends ConfigContext { @@ -123,7 +119,7 @@ export class ActionReferenceContext extends ConfigContext { public mode: ActionMode @schema(joiVariables().required().description("The variables configured on the action.").example({ foo: "bar" })) - public var: DeepPrimitiveMap + public var: ConfigContext constructor({ root, name, disabled, buildPath, sourcePath, mode, variables }: ActionReferenceContextParams) { super(root) @@ -217,7 +213,7 @@ class ActionReferencesContext extends ConfigContext { buildPath: action.getBuildPath(), sourcePath: action.sourcePath(), mode: action.mode(), - variables: action.getVariables(), + variables: new GenericContext(action.getVariables()), }) ) } @@ -231,7 +227,7 @@ export interface ActionSpecContextParams { action: Action resolvedDependencies: ResolvedAction[] executedDependencies: ExecutedAction[] - variables: DeepPrimitiveMap + variables: ConfigContext inputs: DeepPrimitiveMap } diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index f474a93b43..fdb8b7634f 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -7,17 +7,17 @@ */ import type Joi from "@hapi/joi" -import { isString } from "lodash-es" -import { ConfigurationError, GardenError } from "../../exceptions.js" -import { resolveTemplateString } from "../../template/templated-strings.js" +import { ConfigurationError, GardenError, InternalError } from "../../exceptions.js" import type { CustomObjectSchema } from "../common.js" import { joi, joiIdentifier } from "../common.js" import { naturalList } from "../../util/string.js" import { styles } from "../../logger/styles.js" import { Profile } from "../../util/profiling.js" -import type { CollectionOrValue } from "../../util/objects.js" +import { deepMap, type CollectionOrValue } from "../../util/objects.js" import type { TemplatePrimitive } from "../../template/types.js" import { isTemplatePrimitive, UnresolvedTemplateValue } from "../../template/types.js" +import pick from "lodash-es/pick.js" +import { evaluate } from "../../template/evaluate.js" export type ContextKeySegment = string | number export type ContextKey = ContextKeySegment[] @@ -38,11 +38,11 @@ export interface ContextResolveParams { key: ContextKey nodePath: ContextKey opts: ContextResolveOpts + rootContext?: ConfigContext } export interface ContextResolveOutput { getUnavailableReason?: () => string - partial?: boolean resolved: any } @@ -92,7 +92,7 @@ export abstract class ConfigContext { return "" } - resolve({ key, nodePath, opts }: ContextResolveParams): ContextResolveOutput { + resolve({ key, nodePath, opts, rootContext = new GenericContext({}) }: ContextResolveParams): ContextResolveOutput { const path = key.join(".") // if the key has previously been resolved, return it directly @@ -117,12 +117,18 @@ export abstract class ConfigContext { let getAvailableKeys: (() => string[]) | undefined = undefined // eslint-disable-next-line @typescript-eslint/no-this-alias - let value: CollectionOrValue | ConfigContext | Function = this - let partial = false + let value: CollectionOrValue = this let nextKey = key[0] let nestedNodePath = nodePath let getUnavailableReason: (() => string) | undefined = undefined + if (key.length === 0) { + value = pick( + this, + Object.keys(this).filter((k) => !k.startsWith("_")) + ) as Record> + } + for (let p = 0; p < key.length; p++) { nextKey = key[p] @@ -133,7 +139,7 @@ export abstract class ConfigContext { const getStackEntry = () => renderKeyPath(capturedNestedNodePath) getAvailableKeys = undefined - const parent: CollectionOrValue | ConfigContext | Function = value + const parent: CollectionOrValue = value if (isTemplatePrimitive(parent)) { throw new ContextResolveError({ message: `Attempted to look up key ${JSON.stringify(nextKey)} on a ${typeof parent}.`, @@ -150,43 +156,22 @@ export abstract class ConfigContext { value = parent[nextKey] } - if (typeof value === "function") { - // call the function to resolve the value, then continue - const stackEntry = getStackEntry() - if (opts.stack?.has(stackEntry)) { - // Circular dependency error is critical, throwing here. - throw new ContextResolveError({ - message: `Circular reference detected when resolving key ${stackEntry} (from ${Array.from(opts.stack || []).join(" -> ")})`, - }) - } - - opts.stack.add(stackEntry) - value = value({ key: getRemainder(), nodePath: nestedNodePath, opts }) - } - // handle nested contexts if (value instanceof ConfigContext) { const remainder = getRemainder() - if (remainder.length > 0) { - const stackEntry = getStackEntry() - opts.stack.add(stackEntry) - const res = value.resolve({ key: remainder, nodePath: nestedNodePath, opts }) - value = res.resolved - getUnavailableReason = res.getUnavailableReason - partial = !!res.partial - } + const stackEntry = getStackEntry() + opts.stack.add(stackEntry) + // NOTE: we resolve even if remainder.length is zero to make sure all unresolved template values have been resolved. + const res = value.resolve({ key: remainder, nodePath: nestedNodePath, opts, rootContext }) + value = res.resolved + getUnavailableReason = res.getUnavailableReason break } // handle templated strings in context variables if (value instanceof UnresolvedTemplateValue) { opts.stack.add(getStackEntry()) - value = value.evaluate({ context: this._rootContext, opts }) - - if (typeof value === "symbol") { - value = undefined - break - } + value = evaluate(value, { context: new LayeredContext(rootContext, this._rootContext), opts }) } if (value === undefined) { @@ -225,11 +210,18 @@ export abstract class ConfigContext { return { resolved: CONTEXT_RESOLVE_KEY_NOT_FOUND, getUnavailableReason } } - // Cache result, unless it is a partial resolution - if (!partial) { - this._resolvedValues[path] = value + if (!isTemplatePrimitive(value)) { + value = deepMap(value, (v, keyPath) => { + if (v instanceof ConfigContext) { + return v.resolve({ key: [], nodePath: nodePath.concat(key, keyPath), opts }).resolved + } + return evaluate(v, { context: new LayeredContext(rootContext, this._rootContext), opts }) + }) } + // Cache result + this._resolvedValues[path] = value + return { resolved: value } } } @@ -333,3 +325,39 @@ export function renderKeyPath(key: ContextKeySegment[]): string { }, stringSegments[0]) ) } +export class CapturedContext extends ConfigContext { + constructor( + private readonly wrapped: ConfigContext, + private readonly rootContext: ConfigContext + ) { + super(rootContext) + } + + override resolve(params: ContextResolveParams): ContextResolveOutput { + return this.wrapped.resolve({ + ...params, + rootContext: params.rootContext ? new LayeredContext(this.rootContext, params.rootContext) : this.rootContext, + }) + } +} + +export class LayeredContext extends ConfigContext { + private readonly contexts: ConfigContext[] + constructor(...contexts: ConfigContext[]) { + super() + this.contexts = contexts + } + override resolve(args: ContextResolveParams): ContextResolveOutput { + // TODO: This naive algorithm must be replaced with a lazy merge implementation + // See also https://github.com/garden-io/garden/pull/6669/files + for (const [i, context] of this.contexts.entries()) { + const resolved = context.resolve(args) + if (resolved.resolved !== CONTEXT_RESOLVE_KEY_NOT_FOUND || i === this.contexts.length - 1) { + return resolved + } + } + throw new InternalError({ + message: "LayeredContext has zero contexts", + }) + } +} diff --git a/core/src/config/template-contexts/custom-command.ts b/core/src/config/template-contexts/custom-command.ts index 2e48ad902a..6ea0faa5d0 100644 --- a/core/src/config/template-contexts/custom-command.ts +++ b/core/src/config/template-contexts/custom-command.ts @@ -6,11 +6,11 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import type { DeepPrimitiveMap } from "../common.js" import { variableNameRegex, joiPrimitive, joiArray, joiVariables, joiIdentifierMap } from "../common.js" import { joi } from "../common.js" import type { DefaultEnvironmentContextParams } from "./project.js" import { DefaultEnvironmentContext } from "./project.js" +import type { ConfigContext } from "./base.js" import { schema } from "./base.js" interface ArgsSchema { @@ -30,10 +30,10 @@ export class CustomCommandContext extends DefaultEnvironmentContext { .description("A map of all variables defined in the command configuration.") .meta({ keyPlaceholder: "" }) ) - public variables: DeepPrimitiveMap + public variables: ConfigContext @schema(joiIdentifierMap(joiPrimitive()).description("Alias for the variables field.")) - public var: DeepPrimitiveMap + public var: ConfigContext @schema( joi @@ -70,7 +70,7 @@ export class CustomCommandContext extends DefaultEnvironmentContext { params: DefaultEnvironmentContextParams & { args: ArgsSchema opts: OptsSchema - variables: DeepPrimitiveMap + variables: ConfigContext rest: string[] } ) { diff --git a/core/src/config/template-contexts/module.ts b/core/src/config/template-contexts/module.ts index 2a95272f23..347d65c030 100644 --- a/core/src/config/template-contexts/module.ts +++ b/core/src/config/template-contexts/module.ts @@ -193,7 +193,7 @@ class RuntimeConfigContext extends ConfigContext { export interface OutputConfigContextParams { garden: Garden resolvedProviders: ProviderMap - variables: DeepPrimitiveMap + variables: ConfigContext modules: GardenModule[] // We only supply this when resolving configuration in dependency order. // Otherwise we pass `${runtime.*} template strings through for later resolution. diff --git a/core/src/config/template-contexts/project.ts b/core/src/config/template-contexts/project.ts index 82500c4ffb..4ac2e9327c 100644 --- a/core/src/config/template-contexts/project.ts +++ b/core/src/config/template-contexts/project.ts @@ -346,7 +346,7 @@ export class ProjectConfigContext extends DefaultEnvironmentContext { } interface EnvironmentConfigContextParams extends ProjectConfigContextParams { - variables: DeepPrimitiveMap + variables: ConfigContext } /** @@ -358,10 +358,10 @@ export class EnvironmentConfigContext extends ProjectConfigContext { .description("A map of all variables defined in the project configuration.") .meta({ keyPlaceholder: "" }) ) - public variables: DeepPrimitiveMap + public variables: ConfigContext @schema(joiIdentifierMap(joiPrimitive()).description("Alias for the variables field.")) - public var: DeepPrimitiveMap + public var: ConfigContext @schema( joiStringMap(joi.string().description("The secret's value.")) @@ -394,9 +394,9 @@ export class RemoteSourceConfigContext extends EnvironmentConfigContext { ) .meta({ keyPlaceholder: "" }) ) - public override variables: DeepPrimitiveMap + public override variables: ConfigContext - constructor(garden: Garden, variables: DeepPrimitiveMap) { + constructor(garden: Garden, variables: ConfigContext) { super({ projectName: garden.projectName, projectRoot: garden.projectRoot, diff --git a/core/src/config/template-contexts/provider.ts b/core/src/config/template-contexts/provider.ts index 726ba62a4f..1d506a5d5c 100644 --- a/core/src/config/template-contexts/provider.ts +++ b/core/src/config/template-contexts/provider.ts @@ -65,7 +65,7 @@ export class ProviderConfigContext extends WorkflowConfigContext { ) public providers: Map - constructor(garden: Garden, resolvedProviders: ProviderMap, variables: DeepPrimitiveMap) { + constructor(garden: Garden, resolvedProviders: ProviderMap, variables: ConfigContext) { super(garden, variables) this.providers = new Map(Object.entries(mapValues(resolvedProviders, (p) => new ProviderContext(this, p)))) diff --git a/core/src/garden.ts b/core/src/garden.ts index a50f6f20c3..38ac92b624 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -13,7 +13,7 @@ import { platform, arch } from "os" import { relative, resolve } from "path" import cloneDeep from "fast-copy" import AsyncLock from "async-lock" -import { flatten, groupBy, keyBy, mapValues, omit, set, sortBy } from "lodash-es" +import { flatten, groupBy, keyBy, mapValues, omit, sortBy, set as setKeyPathNested } from "lodash-es" import { username } from "username" import { TreeCache } from "./cache.js" @@ -117,7 +117,7 @@ import type { CloudProject, GardenCloudApiFactory } from "./cloud/api.js" import { GardenCloudApi, CloudApiTokenRefreshError } from "./cloud/api.js" import { OutputConfigContext } from "./config/template-contexts/module.js" import { ProviderConfigContext } from "./config/template-contexts/provider.js" -import type { ConfigContext } from "./config/template-contexts/base.js" +import { CONTEXT_RESOLVE_KEY_NOT_FOUND, GenericContext, type ConfigContext } from "./config/template-contexts/base.js" import { validateSchema, validateWithPath } from "./config/validation.js" import { pMemoizeDecorator } from "./lib/p-memoize.js" import { ModuleGraph } from "./graph/modules.js" @@ -171,6 +171,7 @@ import { import { throwOnMissingSecretKeys } from "./config/secrets.js" import { deepEvaluate } from "./template/evaluate.js" import type { ParsedTemplate } from "./template/types.js" +import { LayeredContext } from "./config/template-contexts/base.js" const defaultLocalAddress = "localhost" @@ -227,7 +228,7 @@ export interface GardenParams { projectRoot: string projectSources?: SourceConfig[] providerConfigs: GenericProviderConfig[] - variables: DeepPrimitiveMap + variables: ConfigContext variableOverrides: DeepPrimitiveMap secrets: StringMap sessionId: string @@ -307,7 +308,7 @@ export class Garden { * for the current environment but can be overwritten with the `--env` flag. */ public readonly namespace: string - public readonly variables: DeepPrimitiveMap + public readonly variables: ConfigContext // Any variables passed via the `--var` CLI option (maintained here so that they can be used during module resolution // to override module variables and module varfiles). public readonly variableOverrides: DeepPrimitiveMap @@ -1836,7 +1837,7 @@ export class Garden { allEnvironmentNames, namespace: this.namespace, providers: providers.map(omitInternal), - variables: this.variables, + variables: this.variables.resolve({ key: [], nodePath: [], opts: {} }).resolved, actionConfigs: filteredActionConfigs, moduleConfigs: moduleConfigs.map(omitInternal), workflowConfigs: sortBy(workflowConfigs.map(omitInternal), "name"), @@ -2349,18 +2350,21 @@ async function getCloudProject({ } // Override variables, also allows to override nested variables using dot notation -// eslint-disable-next-line @typescript-eslint/no-shadow -export function overrideVariables(variables: DeepPrimitiveMap, overrideVariables: DeepPrimitiveMap): DeepPrimitiveMap { - const objNew = cloneDeep(variables) - Object.keys(overrideVariables).forEach((key) => { - if (objNew.hasOwnProperty(key)) { +export function overrideVariables(variables: ConfigContext, overrides: DeepPrimitiveMap): LayeredContext { + const transformedOverrides = {} + for (const key in overrides) { + if (variables.resolve({ key: [key], nodePath: [], opts: {} }).resolved !== CONTEXT_RESOLVE_KEY_NOT_FOUND) { // if the original key itself is a string with a dot, then override that - objNew[key] = overrideVariables[key] + transformedOverrides[key] = overrides[key] } else { - set(objNew, key, overrideVariables[key]) + // Transform override paths like "foo.bar[0].baz" + // into a nested object like + // { foo: { bar: [{ baz: "foo" }] } } + // which we can then use for the layered context as overrides on the nested structure within + setKeyPathNested(transformedOverrides, key, overrides[key]) } - }) - return objNew + } + return new LayeredContext(new GenericContext(overrides), variables) } /** diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index 7a568d1783..e61da9db85 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -40,7 +40,6 @@ import type { ActionReference, JoiDescription } from "../config/common.js" import { describeSchema, parseActionReference } from "../config/common.js" import type { GroupConfig } from "../config/group.js" import { ActionConfigContext } from "../config/template-contexts/actions.js" -import { validateWithPath } from "../config/validation.js" import { ConfigurationError, GardenError, InternalError, PluginError } from "../exceptions.js" import { type Garden, overrideVariables } from "../garden.js" import type { Log } from "../logger/log-entry.js" @@ -58,6 +57,7 @@ import type { ModuleGraph } from "./modules.js" import { isTruthy, type MaybeUndefined } from "../util/util.js" import { minimatch } from "minimatch" import type { ConfigContext } from "../config/template-contexts/base.js" +import { GenericContext } from "../config/template-contexts/base.js" import type { LinkedSource, LinkedSourceMap } from "../config-store/local.js" import { relative } from "path" import { profileAsync } from "../util/profiling.js" @@ -67,6 +67,7 @@ import { styles } from "../logger/styles.js" import { isUnresolvableValue } from "../template/analysis.js" import { getActionTemplateReferences } from "../config/references.js" import { capture } from "../template/capture.js" +import { CapturedContext } from "../config/template-contexts/base.js" import { deepEvaluate } from "../template/evaluate.js" import type { ParsedTemplate } from "../template/types.js" @@ -508,7 +509,7 @@ export const processActionConfig = profileAsync(async function processActionConf let variables = await mergeVariables({ basePath: effectiveConfigFileLocation, - variables: config.variables, + variables: new GenericContext(config.variables), varfiles: config.varfiles, log, }) @@ -516,7 +517,7 @@ export const processActionConfig = profileAsync(async function processActionConf // override the variables if there's any matching variables in variable overrides // passed via --var cli flag. variables passed via --var cli flag have highest precedence const variableOverrides = garden.variableOverrides || {} - variables = overrideVariables(variables ?? {}, variableOverrides) + variables = overrideVariables(variables, variableOverrides) const params: ActionWrapperParams = { baseBuildDirectory: garden.buildStaging.buildDirPath, @@ -732,12 +733,12 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi const resolvedVarFiles = config.varfiles?.filter((f) => !maybeTemplateString(getVarfileData(f).path)) const variables = await mergeVariables({ basePath: config.internal.basePath, - variables: config.variables, + variables: new GenericContext(config.variables), varfiles: resolvedVarFiles, log, }) - const resolvedVariables = capture( + const resolvedVariables = new CapturedContext( variables, new ActionConfigContext({ garden, @@ -752,7 +753,7 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi if (templateName) { // Partially resolve inputs - const partiallyResolvedInputs = capture( + config.internal.inputs = capture( config.internal.inputs || {}, new ActionConfigContext({ garden, @@ -780,7 +781,7 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi // TODO: schema validation on partially resolved inputs does not make sense // do we validate the schema on fully resolved inputs somewhere? // config.internal.inputs = validateWithPath({ - // config: cloneDeep(partiallyResolvedInputs), + // config: partiallyResolvedInputs, // configType: `inputs for ${description}`, // path: config.internal.basePath, // schema: template.inputsSchema, @@ -800,8 +801,6 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi variables: resolvedVariables, }) - const yamlDoc = config.internal.yamlDoc - function resolveTemplates() { // Fully resolve built-in fields that only support `ActionConfigContext`. // TODO-0.13.1: better error messages when something goes wrong here (missing inputs for example) @@ -818,7 +817,7 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi config = { ...config, ...resolvedBuiltin, - variables: resolvedVariables, + variables: resolvedVariables.resolve({ key: [], nodePath: [], opts: {} }).resolved, spec, } diff --git a/core/src/graph/common.ts b/core/src/graph/common.ts index 7df4d55960..dfe41ecadf 100644 --- a/core/src/graph/common.ts +++ b/core/src/graph/common.ts @@ -7,7 +7,7 @@ */ import { DepGraph } from "dependency-graph" -import { flatten, merge, uniq } from "lodash-es" +import { flatten, uniq } from "lodash-es" import { get, isEqual, join, set, uniqWith } from "lodash-es" import { CircularDependenciesError } from "../exceptions.js" import type { GraphNodes, ConfigGraphNode } from "./config-graph.js" @@ -15,9 +15,12 @@ import { Profile, profileAsync } from "../util/profiling.js" import type { ModuleDependencyGraphNode, ModuleDependencyGraphNodeKind, ModuleGraphNodes } from "./modules.js" import type { ActionKind } from "../plugin/action-types.js" import { loadVarfile } from "../config/base.js" -import type { DeepPrimitiveMap, Varfile } from "../config/common.js" +import type { Varfile } from "../config/common.js" import type { Task } from "../tasks/base.js" import type { Log, LogMetadata, TaskLogStatus } from "../logger/log-entry.js" +import { LayeredContext } from "../config/template-contexts/base.js" +import type { ConfigContext } from "../config/template-contexts/base.js" +import { GenericContext } from "../config/template-contexts/base.js" // Shared type used by ConfigGraph and TaskGraph to facilitate circular dependency detection export type DependencyGraphNode = { @@ -153,7 +156,7 @@ export const mergeVariables = profileAsync(async function mergeVariables({ log, }: { basePath: string - variables?: DeepPrimitiveMap + variables?: ConfigContext varfiles?: Varfile[] log: Log }) { @@ -170,19 +173,12 @@ export const mergeVariables = profileAsync(async function mergeVariables({ }) ) - const output: DeepPrimitiveMap = {} - - if (variables) { - merge(output, variables) - } - - // Merge different varfiles, later files taking precedence over prior files in the list. - // TODO-0.13.0: should this be a JSON merge? - for (const vars of varsByFile) { - merge(output, vars) - } - - return output + return new LayeredContext( + // Merge different varfiles, later files taking precedence over prior files in the list. + // TODO-0.13.0: should this be a JSON merge? + ...varsByFile.reverse().map((vars) => new GenericContext(vars)), + variables || new GenericContext({}) + ) }) /** diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index b5c020102c..99bd2b7378 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -9,11 +9,7 @@ import cloneDeep from "fast-copy" import { isArray, isString, keyBy, keys, partition, pick, union, uniq } from "lodash-es" import { validateWithPath } from "./config/validation.js" -import { - mayContainTemplateString, - resolveTemplateString, - resolveTemplateStrings, -} from "./template/templated-strings.js" +import { mayContainTemplateString, resolveTemplateString } from "./template/templated-strings.js" import { GenericContext } from "./config/template-contexts/base.js" import { dirname, posix, relative, resolve } from "path" import type { Garden } from "./garden.js" @@ -45,7 +41,6 @@ import type { ModuleConfigContextParams } from "./config/template-contexts/modul import { ModuleConfigContext } from "./config/template-contexts/module.js" import { pathToCacheContext } from "./cache.js" import { loadVarfile, prepareBuildDependencies } from "./config/base.js" -import { merge } from "json-merge-patch" import type { ModuleTypeDefinition } from "./plugin/plugin.js" import { serviceFromConfig } from "./types/service.js" import { taskFromConfig } from "./types/task.js" @@ -64,6 +59,7 @@ import type { DepGraph } from "dependency-graph" import { minimatch } from "minimatch" import { getModuleTemplateReferences } from "./config/references.js" import { capture } from "./template/capture.js" +import { LayeredContext } from "./config/template-contexts/base.js" import type { ParsedTemplate } from "./template/types.js" import { deepEvaluate } from "./template/evaluate.js" @@ -638,7 +634,7 @@ export class ModuleResolver { // so we also need to pass inputs here along with the available variables. const configContext = new ModuleConfigContext({ ...templateContextParams, - variables: { ...garden.variables, ...resolvedModuleVariables }, + variables: new LayeredContext(resolvedModuleVariables, garden.variables), inputs: { ...inputs }, }) @@ -647,7 +643,7 @@ export class ModuleResolver { configContext ) as unknown as typeof config - config.variables = resolvedModuleVariables + config.variables = resolvedModuleVariables.resolve({ key: [], nodePath: [], opts: {} }).resolved config.inputs = inputs const moduleTypeDefinitions = await garden.getModuleTypes() @@ -780,7 +776,7 @@ export class ModuleResolver { const configContext = new ModuleConfigContext({ garden: this.garden, resolvedProviders: this.resolvedProviders, - variables: { ...this.garden.variables, ...resolvedConfig.variables }, + variables: new LayeredContext(new GenericContext(resolvedConfig.variables), this.garden.variables), name: resolvedConfig.name, path: resolvedConfig.path, buildPath, @@ -913,7 +909,7 @@ export class ModuleResolver { private async resolveVariables( config: ModuleConfig, templateContextParams: ModuleConfigContextParams - ): Promise { + ): Promise { const moduleConfigContext = new ModuleConfigContext(templateContextParams) const resolveOpts = { // Modules will be converted to actions later, and the actions will be properly unescaped. @@ -952,8 +948,11 @@ export class ModuleResolver { this.garden.variableOverrides, union(keys(moduleVariables), keys(varfileVars)) ) - const mergedVariables: DeepPrimitiveMap = merge(moduleVariables, merge(varfileVars, relevantVariableOverrides)) - return mergedVariables + return new LayeredContext( + new GenericContext(relevantVariableOverrides), + new GenericContext(varfileVars), + new GenericContext(moduleVariables) + ) } } diff --git a/core/src/tasks/resolve-action.ts b/core/src/tasks/resolve-action.ts index 842f4e5f47..b22c15befe 100644 --- a/core/src/tasks/resolve-action.ts +++ b/core/src/tasks/resolve-action.ts @@ -18,16 +18,17 @@ import type { ResolvedAction, } from "../actions/types.js" import { ActionSpecContext } from "../config/template-contexts/actions.js" -import { resolveTemplateStrings } from "../template/templated-strings.js" import { InternalError } from "../exceptions.js" import { validateWithPath } from "../config/validation.js" -import type { DeepPrimitiveMap } from "../config/common.js" -import { merge } from "lodash-es" import { mergeVariables } from "../graph/common.js" import { actionToResolved } from "../actions/helpers.js" import { ResolvedConfigGraph } from "../graph/config-graph.js" import { OtelTraced } from "../util/open-telemetry/decorators.js" import { deepEvaluate } from "../template/evaluate.js" +import type { ConfigContext } from "../config/template-contexts/base.js" +import { GenericContext } from "../config/template-contexts/base.js" +import { LayeredContext } from "../config/template-contexts/base.js" +import { CapturedContext } from "../config/template-contexts/base.js" export interface ResolveActionResults extends ValidResultType { state: ActionState @@ -129,7 +130,7 @@ export class ResolveActionTask extends BaseActionTask extends BaseActionTask(template: T, context: ConfigCo }) as T } -export class LayeredContext extends ConfigContext { - readonly #contexts: ConfigContext[] - constructor(...contexts: ConfigContext[]) { - super() - this.#contexts = contexts - } - override resolve(_args: ContextResolveParams): ContextResolveOutput { - throw new NotImplementedError({ message: "TODO" }) - } -} - export class CapturedContextTemplateValue extends UnresolvedTemplateValue { - readonly #wrapped: UnresolvedTemplateValue - readonly #context: ConfigContext - - constructor(wrapped: UnresolvedTemplateValue, context: ConfigContext) { + constructor( + private readonly wrapped: UnresolvedTemplateValue, + private readonly context: ConfigContext + ) { super() - this.#wrapped = wrapped - this.#context = context + this.context = context } override evaluate(args: EvaluateTemplateArgs): ResolvedTemplate { - const context = new LayeredContext(this.#context, args.context) + const context = new LayeredContext(this.context, args.context) - return this.#wrapped.evaluate({ ...args, context }) + return this.wrapped.evaluate({ ...args, context }) } override toJSON(): ResolvedTemplate { - return this.#wrapped.toJSON() + return this.wrapped.toJSON() } override *visitAll(): TemplateExpressionGenerator { - yield* this.#wrapped.visitAll() + yield* this.wrapped.visitAll() } } diff --git a/core/src/template/evaluate.ts b/core/src/template/evaluate.ts index e68f06035b..d57da93ae4 100644 --- a/core/src/template/evaluate.ts +++ b/core/src/template/evaluate.ts @@ -6,12 +6,10 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import set from "lodash-es/set.js" import { InternalError } from "../exceptions.js" -import { isArray, isPlainObject } from "../util/objects.js" -import { visitAll } from "./analysis.js" +import { deepMap } from "../util/objects.js" import type { EvaluateTemplateArgs, ParsedTemplateValue, ResolvedTemplate, TemplatePrimitive } from "./types.js" -import { isTemplatePrimitive, UnresolvedTemplateValue, type ParsedTemplate } from "./types.js" +import { UnresolvedTemplateValue, type ParsedTemplate } from "./types.js" type Evaluate = T extends UnresolvedTemplateValue ? ResolvedTemplate @@ -25,26 +23,17 @@ type Evaluate = T extends UnresolvedTemplateValue ? T : ResolvedTemplate -type _test1 = Evaluate<{ foo: UnresolvedTemplateValue }> -type _test2 = Evaluate<{ foo: "foo" }> -type _test3 = Evaluate export function deepEvaluate( collection: Input, args: EvaluateTemplateArgs ): Evaluate { - if (!isArray(collection) && !isPlainObject(collection)) { - return evaluate(collection, args) as Evaluate - } - const result = isArray(collection) ? [] : {} - - for (const { value, yamlSource } of visitAll({ value: collection, source: { path: [] } })) { - if (isTemplatePrimitive(value) || value instanceof UnresolvedTemplateValue) { - const evaluated = evaluate(value, args) - set(result, yamlSource.path, evaluated) + return deepMap(collection, (v) => { + if (v instanceof UnresolvedTemplateValue) { + const evaluated = evaluate(v, args) + return evaluated } - } - - return result as Evaluate + return v + }) as Evaluate } export function evaluate( diff --git a/core/src/template/templated-collections.ts b/core/src/template/templated-collections.ts index 684a8ba5a8..5d724daffd 100644 --- a/core/src/template/templated-collections.ts +++ b/core/src/template/templated-collections.ts @@ -28,7 +28,7 @@ import { } from "../config/constants.js" import mapValues from "lodash-es/mapValues.js" import { deepEvaluate } from "./evaluate.js" -import { LayeredContext } from "./capture.js" +import { LayeredContext } from "../config/template-contexts/base.js" import { parseTemplateString } from "./templated-strings.js" import { TemplateError } from "./errors.js" import { visitAll, type TemplateExpressionGenerator } from "./analysis.js" @@ -66,143 +66,151 @@ export function parseTemplateCollection { - if (!source) { - throw new InternalError({ - message: "Source parameter is required for parseTemplateCollection.", - }) - } - if (typeof value === "string") { - return parseTemplateString({ - rawTemplateString: value, - source, - }) as Parse - } else if (isTemplatePrimitive(value)) { - return value as Parse - } else if (isArray(value)) { - const parsed = value.map((v, i) => parseTemplateCollection({ value: v, source: pushYamlPath(i, source) })) - - if (value.some((v) => isPlainObject(v) && v[arrayConcatKey] !== undefined)) { - return new ConcatLazyValue(source, parsed) as Parse - } else { - return parsed as Parse + const inner = () => { + if (!source) { + throw new InternalError({ + message: "Source parameter is required for parseTemplateCollection.", + }) } - } else if (isPlainObject(value)) { - if (value[arrayForEachKey] !== undefined) { - const unexpectedKeys = Object.keys(value).filter((k) => !ForEachLazyValue.allowedForEachKeys.includes(k)) + if (typeof value === "string") { + return parseTemplateString({ + rawTemplateString: value, + source, + }) as Parse + } else if (isTemplatePrimitive(value)) { + return value as Parse + } else if (isArray(value)) { + const parsed = value.map((v, i) => parseTemplateCollection({ value: v, source: pushYamlPath(i, source) })) + + if (value.some((v) => isPlainObject(v) && v[arrayConcatKey] !== undefined)) { + return new ConcatLazyValue(source, parsed) as Parse + } else { + return parsed as Parse + } + } else if (isPlainObject(value)) { + if (value[arrayForEachKey] !== undefined) { + const unexpectedKeys = Object.keys(value).filter((k) => !ForEachLazyValue.allowedForEachKeys.includes(k)) - if (unexpectedKeys.length > 0) { - const extraKeys = naturalList(unexpectedKeys.map((k) => JSON.stringify(k))) + if (unexpectedKeys.length > 0) { + const extraKeys = naturalList(unexpectedKeys.map((k) => JSON.stringify(k))) - throw new TemplateError({ - message: `Found one or more unexpected keys on ${arrayForEachKey} object: ${extraKeys}. Allowed keys: ${naturalList( - ForEachLazyValue.allowedForEachKeys - )}`, - source: pushYamlPath(extraKeys[0], source), + throw new TemplateError({ + message: `Found one or more unexpected keys on ${arrayForEachKey} object: ${extraKeys}. Allowed keys: ${naturalList( + ForEachLazyValue.allowedForEachKeys + )}`, + source: pushYamlPath(extraKeys[0], source), + }) + } + + if (value[arrayForEachReturnKey] === undefined) { + throw new TemplateError({ + message: `Missing ${arrayForEachReturnKey} field next to ${arrayForEachKey} field. Got ${naturalList( + Object.keys(value) + )}`, + source: pushYamlPath(arrayForEachReturnKey, source), + }) + } + + const parsedCollectionExpression = parseTemplateCollection({ + value: value[arrayForEachKey], + source: pushYamlPath(arrayForEachKey, source), }) - } - if (value[arrayForEachReturnKey] === undefined) { - throw new TemplateError({ - message: `Missing ${arrayForEachReturnKey} field next to ${arrayForEachKey} field. Got ${naturalList( - Object.keys(value) - )}`, + const parsedReturnExpression = parseTemplateCollection({ + value: value[arrayForEachReturnKey], source: pushYamlPath(arrayForEachReturnKey, source), }) - } - const parsedCollectionExpression = parseTemplateCollection({ - value: value[arrayForEachKey], - source: pushYamlPath(arrayForEachKey, source), - }) + const parsedFilterExpression = + value[arrayForEachFilterKey] === undefined + ? undefined + : parseTemplateCollection({ + value: value[arrayForEachFilterKey], + source: pushYamlPath(arrayForEachFilterKey, source), + }) + + const forEach = new ForEachLazyValue(source, { + [arrayForEachKey]: parsedCollectionExpression, + [arrayForEachReturnKey]: parsedReturnExpression, + [arrayForEachFilterKey]: parsedFilterExpression, + }) - const parsedReturnExpression = parseTemplateCollection({ - value: value[arrayForEachReturnKey], - source: pushYamlPath(arrayForEachReturnKey, source), - }) + if (parsedReturnExpression?.[arrayConcatKey] !== undefined) { + return new ConcatLazyValue(source, forEach) as Parse + } else { + return forEach as Parse + } + } else if (value[conditionalKey] !== undefined) { + const ifExpression = value[conditionalKey] + const thenExpression = value[conditionalThenKey] + const elseExpression = value[conditionalElseKey] - const parsedFilterExpression = - value[arrayForEachFilterKey] === undefined - ? undefined - : parseTemplateCollection({ - value: value[arrayForEachFilterKey], - source: pushYamlPath(arrayForEachFilterKey, source), - }) - - const forEach = new ForEachLazyValue(source, { - [arrayForEachKey]: parsedCollectionExpression, - [arrayForEachReturnKey]: parsedReturnExpression, - [arrayForEachFilterKey]: parsedFilterExpression, - }) + if (thenExpression === undefined) { + throw new TemplateError({ + message: `Missing ${conditionalThenKey} field next to ${conditionalKey} field. Got: ${naturalList( + Object.keys(value) + )}`, + source, + }) + } - if (parsedReturnExpression?.[arrayConcatKey] !== undefined) { - return new ConcatLazyValue(source, forEach) as Parse - } else { - return forEach as Parse - } - } else if (value[conditionalKey] !== undefined) { - const ifExpression = value[conditionalKey] - const thenExpression = value[conditionalThenKey] - const elseExpression = value[conditionalElseKey] + const unexpectedKeys = Object.keys(value).filter( + (k) => !ConditionalLazyValue.allowedConditionalKeys.includes(k) + ) - if (thenExpression === undefined) { - throw new TemplateError({ - message: `Missing ${conditionalThenKey} field next to ${conditionalKey} field. Got: ${naturalList( - Object.keys(value) - )}`, - source, - }) - } + if (unexpectedKeys.length > 0) { + const extraKeys = naturalList(unexpectedKeys.map((k) => JSON.stringify(k))) - const unexpectedKeys = Object.keys(value).filter((k) => !ConditionalLazyValue.allowedConditionalKeys.includes(k)) + throw new TemplateError({ + message: `Found one or more unexpected keys on ${conditionalKey} object: ${extraKeys}. Allowed: ${naturalList( + ConditionalLazyValue.allowedConditionalKeys + )}`, + source, + }) + } - if (unexpectedKeys.length > 0) { - const extraKeys = naturalList(unexpectedKeys.map((k) => JSON.stringify(k))) + return new ConditionalLazyValue(source, { + [conditionalKey]: parseTemplateCollection({ + value: ifExpression, + source: pushYamlPath(conditionalKey, source), + }), + [conditionalThenKey]: parseTemplateCollection({ + value: thenExpression, + source: pushYamlPath(conditionalThenKey, source), + }), + [conditionalElseKey]: + elseExpression === undefined + ? undefined + : parseTemplateCollection({ + value: elseExpression, + source: pushYamlPath(conditionalElseKey, source), + }), + }) as Parse + } else { + const resolved = mapValues(value, (v, k) => { + // if this key is untemplatable, skip parsing this branch of the template tree. + if (untemplatableKeys.includes(k)) { + return v + } - throw new TemplateError({ - message: `Found one or more unexpected keys on ${conditionalKey} object: ${extraKeys}. Allowed: ${naturalList( - ConditionalLazyValue.allowedConditionalKeys - )}`, - source, + return parseTemplateCollection({ value: v, source: pushYamlPath(k, source) }) as ParsedTemplate }) + if (Object.keys(value).some((k) => k === objectSpreadKey)) { + return new ObjectSpreadLazyValue(source, resolved as ObjectSpreadOperation) as Parse + } else { + return resolved as Parse + } } - - return new ConditionalLazyValue(source, { - [conditionalKey]: parseTemplateCollection({ - value: ifExpression, - source: pushYamlPath(conditionalKey, source), - }), - [conditionalThenKey]: parseTemplateCollection({ - value: thenExpression, - source: pushYamlPath(conditionalThenKey, source), - }), - [conditionalElseKey]: - elseExpression === undefined - ? undefined - : parseTemplateCollection({ - value: elseExpression, - source: pushYamlPath(conditionalElseKey, source), - }), - }) as Parse } else { - const resolved = mapValues(value, (v, k) => { - // if this key is untemplatable, skip parsing this branch of the template tree. - if (untemplatableKeys.includes(k)) { - return v - } - - return parseTemplateCollection({ value: v, source: pushYamlPath(k, source) }) as ParsedTemplate + throw new InternalError({ + message: `Got unexpected value type: ${typeof value}`, }) - if (Object.keys(value).some((k) => k === objectSpreadKey)) { - return new ObjectSpreadLazyValue(source, resolved as ObjectSpreadOperation) as Parse - } else { - return resolved as Parse - } } - } else { - throw new InternalError({ - message: `Got unexpected value type: ${typeof value}`, - }) } + + const res = inner() + Object.freeze(res) + return res } abstract class StructuralTemplateOperator extends UnresolvedTemplateValue { diff --git a/core/src/template/templated-strings.ts b/core/src/template/templated-strings.ts index a805803b78..05f94f0ca8 100644 --- a/core/src/template/templated-strings.ts +++ b/core/src/template/templated-strings.ts @@ -114,7 +114,14 @@ export function parseTemplateString({ ast, escapePrefix, optionalSuffix: "}?", - parseNested: (nested: string) => parseTemplateString({ rawTemplateString: nested, unescape, source }), + parseNested: (nested: string) => { + const p = parseTemplateString({ rawTemplateString: nested, unescape, source }) + if (p instanceof UnresolvedTemplateValue) { + return p["rootNode"] + } else { + return p + } + }, TemplateStringError: ParserError, unescape, grammarSource: templateStringSource, @@ -196,7 +203,7 @@ export function resolveTemplateStrings(_args: { contextOpts?: ContextResolveOpts source: ConfigSource | undefined }): T { - throw new NotImplementedError({ message: "TODO" }) + throw new NotImplementedError({ message: "TODO Resolve Template Strings" }) } /** diff --git a/core/src/util/testing.ts b/core/src/util/testing.ts index 4f02534f83..bd28f5da70 100644 --- a/core/src/util/testing.ts +++ b/core/src/util/testing.ts @@ -57,6 +57,7 @@ import got from "got" import { createHash } from "node:crypto" import { pipeline } from "node:stream/promises" import type { GardenCloudApiFactory } from "../cloud/api.js" +import { ConfigContext } from "../config/template-contexts/base.js" export class TestError extends GardenError { type = "_test" @@ -188,7 +189,7 @@ export class TestGarden extends Garden { public declare configTemplates: { [name: string]: ConfigTemplateConfig } public declare vcs: VcsHandler public declare secrets: StringMap - public declare variables: DeepPrimitiveMap + public declare variables: ConfigContext private repoRoot!: string public cacheKey!: string public clearConfigsOnScan = false diff --git a/core/test/unit/src/commands/workflow.ts b/core/test/unit/src/commands/workflow.ts index 071dd7d728..dabcbd4b29 100644 --- a/core/test/unit/src/commands/workflow.ts +++ b/core/test/unit/src/commands/workflow.ts @@ -33,6 +33,7 @@ import type { WorkflowStepSpec } from "../../../../src/config/workflow.js" import { defaultWorkflowResources } from "../../../../src/config/workflow.js" import { TestGardenCli } from "../../../helpers/cli.js" import { WorkflowScriptError } from "../../../../src/exceptions.js" +import { GenericContext } from "../../../../src/config/template-contexts/base.js" describe("RunWorkflowCommand", () => { const cmd = new WorkflowCommand() @@ -76,7 +77,7 @@ describe("RunWorkflowCommand", () => { }, ]) - garden.variables = { foo: null } + garden.variables = new GenericContext({ foo: null }) const result = await cmd.action({ ...defaultParams, args: { workflow: "workflow-a" } }) diff --git a/core/test/unit/src/config/render-template.ts b/core/test/unit/src/config/render-template.ts index b52ad804ae..3df01ca784 100644 --- a/core/test/unit/src/config/render-template.ts +++ b/core/test/unit/src/config/render-template.ts @@ -422,7 +422,7 @@ describe("config templates", () => { const config: RenderTemplateConfig = cloneDeep(defaults) config.inputs = { name: "${var.test}" } - garden.variables.test = "test-value" + garden.variables["test"] = "test-value" const resolved = await renderConfigTemplate({ garden, log, config, templates: _templates }) diff --git a/core/test/unit/src/config/template-contexts/base.ts b/core/test/unit/src/config/template-contexts/base.ts index aa446f9b3c..ff1bf2f9bd 100644 --- a/core/test/unit/src/config/template-contexts/base.ts +++ b/core/test/unit/src/config/template-contexts/base.ts @@ -129,10 +129,10 @@ describe("ConfigContext", () => { it("should detect a circular reference from a nested context", async () => { class NestedContext extends ConfigContext { - override resolve({ key, nodePath, opts }: ContextResolveParams) { + override resolve({ key, nodePath, opts, rootContext }: ContextResolveParams) { const circularKey = nodePath.concat(key) opts.stack!.add(circularKey.join(".")) - return c.resolve({ key: circularKey, nodePath: [], opts }) + return c.resolve({ key: circularKey, nodePath: [], opts, rootContext }) } } diff --git a/core/test/unit/src/config/workflow.ts b/core/test/unit/src/config/workflow.ts index d0869d42a6..62cd2e81fd 100644 --- a/core/test/unit/src/config/workflow.ts +++ b/core/test/unit/src/config/workflow.ts @@ -23,6 +23,7 @@ import { defaultNamespace } from "../../../../src/config/project.js" import { join } from "path" import { GardenApiVersion } from "../../../../src/constants.js" import { omit } from "lodash-es" +import { GenericContext } from "../../../../src/config/template-contexts/base.js" describe("resolveWorkflowConfig", () => { let garden: TestGarden @@ -48,7 +49,7 @@ describe("resolveWorkflowConfig", () => { before(async () => { garden = await makeTestGardenA() garden["secrets"] = { foo: "bar", bar: "baz", baz: "banana" } - garden["variables"] = { foo: "baz", skip: false } + garden["variables"] = new GenericContext({ foo: "baz", skip: false }) }) it("should pass through a canonical workflow config", async () => { diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index 73e3dd44ee..f6b1bfb2ad 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -2911,7 +2911,7 @@ describe("Garden", () => { await exec("git", ["add", "."], { cwd: repoPath }) await exec("git", ["commit", "-m", "foo"], { cwd: repoPath }) - garden.variables.sourceBranch = "main" + garden.variables["sourceBranch"] = "main" // eslint-disable-next-line @typescript-eslint/no-explicit-any const _garden = garden as any diff --git a/core/test/unit/src/tasks/resolve-action.ts b/core/test/unit/src/tasks/resolve-action.ts index 6ef7fdfaaf..ff6516bd67 100644 --- a/core/test/unit/src/tasks/resolve-action.ts +++ b/core/test/unit/src/tasks/resolve-action.ts @@ -228,8 +228,8 @@ describe("ResolveActionTask", () => { }, ]) - garden.variables.a = 1 - garden.variables.b = 200 + garden.variables["a"] = 1 + garden.variables["b"] = 200 garden.variableOverrides.b = 2000 // <-- should win const task = await getTask("Build", "foo") From 25e2715432a9194caf5b47e056461a2a88c22a46 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Mon, 30 Dec 2024 18:55:38 +0100 Subject: [PATCH 003/117] chore: fix some issues --- core/src/config/template-contexts/base.ts | 43 +++++++++++++++-------- core/src/graph/actions.ts | 1 - core/src/resolve-module.ts | 2 +- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index fdb8b7634f..5bbac40327 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -7,17 +7,18 @@ */ import type Joi from "@hapi/joi" -import { ConfigurationError, GardenError, InternalError } from "../../exceptions.js" +import { ConfigurationError, GardenError } from "../../exceptions.js" import type { CustomObjectSchema } from "../common.js" import { joi, joiIdentifier } from "../common.js" import { naturalList } from "../../util/string.js" import { styles } from "../../logger/styles.js" import { Profile } from "../../util/profiling.js" -import { deepMap, type CollectionOrValue } from "../../util/objects.js" -import type { TemplatePrimitive } from "../../template/types.js" +import { deepMap, isArray, type CollectionOrValue } from "../../util/objects.js" +import type { ResolvedTemplate, TemplatePrimitive } from "../../template/types.js" import { isTemplatePrimitive, UnresolvedTemplateValue } from "../../template/types.js" import pick from "lodash-es/pick.js" import { evaluate } from "../../template/evaluate.js" +import merge from "lodash-es/merge.js" export type ContextKeySegment = string | number export type ContextKey = ContextKeySegment[] @@ -73,10 +74,12 @@ export const CONTEXT_RESOLVE_KEY_NOT_FOUND: unique symbol = Symbol.for("ContextR export abstract class ConfigContext { private readonly _rootContext: ConfigContext private readonly _resolvedValues: { [path: string]: any } + private readonly _startingPoint: string | undefined - constructor(rootContext?: ConfigContext) { + constructor(rootContext?: ConfigContext, startingPoint?: string) { this._rootContext = rootContext || this this._resolvedValues = {} + this._startingPoint = startingPoint } static getSchema() { @@ -117,7 +120,9 @@ export abstract class ConfigContext { let getAvailableKeys: (() => string[]) | undefined = undefined // eslint-disable-next-line @typescript-eslint/no-this-alias - let value: CollectionOrValue = this + let value: CollectionOrValue = this._startingPoint + ? this[this._startingPoint] + : this let nextKey = key[0] let nestedNodePath = nodePath let getUnavailableReason: (() => string) | undefined = undefined @@ -230,9 +235,8 @@ export abstract class ConfigContext { * A generic context that just wraps an object. */ export class GenericContext extends ConfigContext { - constructor(obj: any) { - super() - Object.assign(this, obj) + constructor(private readonly data: any) { + super(undefined, "data") } static override getSchema() { @@ -348,16 +352,27 @@ export class LayeredContext extends ConfigContext { this.contexts = contexts } override resolve(args: ContextResolveParams): ContextResolveOutput { - // TODO: This naive algorithm must be replaced with a lazy merge implementation - // See also https://github.com/garden-io/garden/pull/6669/files + const items: ResolvedTemplate[] = [] + for (const [i, context] of this.contexts.entries()) { const resolved = context.resolve(args) - if (resolved.resolved !== CONTEXT_RESOLVE_KEY_NOT_FOUND || i === this.contexts.length - 1) { + if (resolved.resolved === CONTEXT_RESOLVE_KEY_NOT_FOUND && i === this.contexts.length - 1) { + return resolved + } + + if (isTemplatePrimitive(resolved.resolved)) { return resolved } } - throw new InternalError({ - message: "LayeredContext has zero contexts", - }) + + const returnValue = isArray(items[0]) ? [] : {} + + for (const i of items) { + merge(returnValue, i) + } + + return { + resolved: returnValue, + } } } diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index e61da9db85..bc31559aab 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -817,7 +817,6 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi config = { ...config, ...resolvedBuiltin, - variables: resolvedVariables.resolve({ key: [], nodePath: [], opts: {} }).resolved, spec, } diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index 99bd2b7378..596e7ba2ca 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -643,7 +643,7 @@ export class ModuleResolver { configContext ) as unknown as typeof config - config.variables = resolvedModuleVariables.resolve({ key: [], nodePath: [], opts: {} }).resolved + // config.variables = resolvedModuleVariables.resolve({ key: [], nodePath: [], opts: {} }).resolved config.inputs = inputs const moduleTypeDefinitions = await garden.getModuleTypes() From cec6fc93a637cb5fd91aee36b9238865c7995f8f Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Thu, 2 Jan 2025 11:47:59 +0100 Subject: [PATCH 004/117] fix: layered context implementation and serialisation of partially resolved values --- core/src/config/template-contexts/base.ts | 24 ++++++++++++----------- core/src/util/serialization.ts | 8 ++++++-- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 5bbac40327..e6462a2f08 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -13,7 +13,8 @@ import { joi, joiIdentifier } from "../common.js" import { naturalList } from "../../util/string.js" import { styles } from "../../logger/styles.js" import { Profile } from "../../util/profiling.js" -import { deepMap, isArray, type CollectionOrValue } from "../../util/objects.js" +import type { Collection } from "../../util/objects.js" +import { deepMap, type CollectionOrValue } from "../../util/objects.js" import type { ResolvedTemplate, TemplatePrimitive } from "../../template/types.js" import { isTemplatePrimitive, UnresolvedTemplateValue } from "../../template/types.js" import pick from "lodash-es/pick.js" @@ -129,8 +130,8 @@ export abstract class ConfigContext { if (key.length === 0) { value = pick( - this, - Object.keys(this).filter((k) => !k.startsWith("_")) + value, + Object.keys(value as Collection).filter((k) => !k.startsWith("_")) ) as Record> } @@ -356,23 +357,24 @@ export class LayeredContext extends ConfigContext { for (const [i, context] of this.contexts.entries()) { const resolved = context.resolve(args) - if (resolved.resolved === CONTEXT_RESOLVE_KEY_NOT_FOUND && i === this.contexts.length - 1) { - return resolved - } - - if (isTemplatePrimitive(resolved.resolved)) { + if (resolved.resolved !== CONTEXT_RESOLVE_KEY_NOT_FOUND) { + if (isTemplatePrimitive(resolved.resolved)) { + return resolved + } + items.push(resolved.resolved) + } else if (items.length === 0 && i === this.contexts.length - 1) { return resolved } } - const returnValue = isArray(items[0]) ? [] : {} + const returnValue = {} for (const i of items) { - merge(returnValue, i) + merge(returnValue, { resolved: i }) } return { - resolved: returnValue, + resolved: returnValue["resolved"], } } } diff --git a/core/src/util/serialization.ts b/core/src/util/serialization.ts index c38e89dbe1..c5f544c733 100644 --- a/core/src/util/serialization.ts +++ b/core/src/util/serialization.ts @@ -23,8 +23,12 @@ export async function dumpYaml(yamlPath: string, data: any) { /** * Wraps safeDump and enforces that invalid values are skipped */ -export function safeDumpYaml(data: any, opts: DumpOptions = {}) { - return dump(data, { ...opts, skipInvalid: true }) +export function safeDumpYaml(data: any, opts: Omit = {}) { + return dump(data, { + ...opts, + skipInvalid: true, + replacer: (_, v) => (typeof v === "object" && typeof v?.["toJSON"] === "function" ? v["toJSON"]() : v), + }) } /** From 8115217be91cb3366d6f0c46f48bdd58b8859c29 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 3 Jan 2025 15:49:18 +0100 Subject: [PATCH 005/117] perf: make sure it resolves and improve performance further --- core/src/actions/base.ts | 5 +- core/src/config/provider.ts | 4 -- core/src/config/references.ts | 8 --- core/src/config/render-template.ts | 2 +- core/src/config/secrets.ts | 4 -- core/src/config/template-contexts/actions.ts | 2 +- core/src/config/template-contexts/base.ts | 68 ++++++++++++++----- core/src/config/workflow.ts | 3 +- core/src/graph/actions.ts | 15 ++-- core/src/outputs.ts | 4 -- core/src/resolve-module.ts | 34 +++++----- core/src/template/analysis.ts | 34 ++-------- core/src/template/ast.ts | 7 +- core/src/template/capture.ts | 30 ++++---- core/src/template/templated-collections.ts | 8 +-- .../unit/src/config/template-contexts/base.ts | 6 +- core/test/unit/src/template-string.ts | 6 -- 17 files changed, 115 insertions(+), 125 deletions(-) diff --git a/core/src/actions/base.ts b/core/src/actions/base.ts index cde8e38bfa..41ed016212 100644 --- a/core/src/actions/base.ts +++ b/core/src/actions/base.ts @@ -34,7 +34,6 @@ import { actionOutputsSchema } from "../plugin/handlers/base/base.js" import type { GraphResult, GraphResults } from "../graph/results.js" import type { RunResult } from "../plugin/base.js" import { Memoize } from "typescript-memoize" -import cloneDeep from "fast-copy" import { flatten, fromPairs, isString, memoize, omit, sortBy } from "lodash-es" import { ActionConfigContext, ActionSpecContext } from "../config/template-contexts/actions.js" import { relative } from "path" @@ -598,7 +597,7 @@ export abstract class BaseAction< getConfig(): C getConfig(key: K): C[K] getConfig(key?: keyof C["spec"]) { - return cloneDeep(key ? this._config[key] : this._config) + return key ? this._config[key] : this._config } /** @@ -806,7 +805,7 @@ export abstract class ResolvedRuntimeAction< getSpec(): Config["spec"] getSpec(key: K): Config["spec"][K] getSpec(key?: keyof Config["spec"]) { - return cloneDeep(key ? this._config.spec[key] : this._config.spec) + return key ? this._config.spec[key] : this._config.spec } getOutput(key: K): GetOutputValueType { diff --git a/core/src/config/provider.ts b/core/src/config/provider.ts index 684093d63b..57d9d20ee8 100644 --- a/core/src/config/provider.ts +++ b/core/src/config/provider.ts @@ -186,10 +186,6 @@ export function getProviderTemplateReferences(config: GenericProviderConfig, con const generator = getContextLookupReferences( visitAll({ value: config, - // TODO: get proper source - source: { - path: [], - }, }), context ) diff --git a/core/src/config/references.ts b/core/src/config/references.ts index 9e8db1287b..c7b393d66d 100644 --- a/core/src/config/references.ts +++ b/core/src/config/references.ts @@ -167,10 +167,6 @@ export function* getActionTemplateReferences( const generator = getContextLookupReferences( visitAll({ value: config as ObjectWithName, - source: { - yamlDoc: config.internal?.yamlDoc, - path: [], - }, }), context ) @@ -193,10 +189,6 @@ export function getModuleTemplateReferences(config: ModuleConfig, context: Modul const generator = getContextLookupReferences( visitAll({ value: config as ObjectWithName, - // Note: We're not implementing the YAML source mapping for modules - source: { - path: [], - }, }), context ) diff --git a/core/src/config/render-template.ts b/core/src/config/render-template.ts index 4c96eac676..850c44efad 100644 --- a/core/src/config/render-template.ts +++ b/core/src/config/render-template.ts @@ -133,7 +133,7 @@ export async function renderConfigTemplate({ }) let resolved: RenderTemplateConfig = { ...(resolvedWithoutInputs as unknown as RenderTemplateConfig), - inputs: capture(config.inputs || {}, templateContext), + inputs: capture(config.inputs || {}, templateContext) as unknown as DeepPrimitiveMap, } const configType = "Render " + resolved.name diff --git a/core/src/config/secrets.ts b/core/src/config/secrets.ts index cd12b8eadc..416010cc6d 100644 --- a/core/src/config/secrets.ts +++ b/core/src/config/secrets.ts @@ -86,10 +86,6 @@ export function detectMissingSecretKeys( const generator = getContextLookupReferences( visitAll({ value: obj, - // TODO: add real yaml source - source: { - path: [], - }, }), context ) diff --git a/core/src/config/template-contexts/actions.ts b/core/src/config/template-contexts/actions.ts index a4bf8c2159..be832ef350 100644 --- a/core/src/config/template-contexts/actions.ts +++ b/core/src/config/template-contexts/actions.ts @@ -213,7 +213,7 @@ class ActionReferencesContext extends ConfigContext { buildPath: action.getBuildPath(), sourcePath: action.sourcePath(), mode: action.mode(), - variables: new GenericContext(action.getVariables()), + variables: action.getVariables(), }) ) } diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index e6462a2f08..9723da151e 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -7,14 +7,14 @@ */ import type Joi from "@hapi/joi" -import { ConfigurationError, GardenError } from "../../exceptions.js" +import { ConfigurationError, GardenError, InternalError } from "../../exceptions.js" import type { CustomObjectSchema } from "../common.js" import { joi, joiIdentifier } from "../common.js" import { naturalList } from "../../util/string.js" import { styles } from "../../logger/styles.js" import { Profile } from "../../util/profiling.js" import type { Collection } from "../../util/objects.js" -import { deepMap, type CollectionOrValue } from "../../util/objects.js" +import { deepMap, isPlainObject, type CollectionOrValue } from "../../util/objects.js" import type { ResolvedTemplate, TemplatePrimitive } from "../../template/types.js" import { isTemplatePrimitive, UnresolvedTemplateValue } from "../../template/types.js" import pick from "lodash-es/pick.js" @@ -29,8 +29,10 @@ export interface ContextResolveOpts { // TODO(0.14): Do not allow the use of template strings in kubernetes manifest files // TODO(0.14): Remove legacyAllowPartial legacyAllowPartial?: boolean - // a list of previously resolved paths, used to detect circular references - stack?: Set + + // a list of contexts for detecting circular references + contextStack?: Set + keyStack?: Set // TODO: remove unescape?: boolean @@ -73,14 +75,18 @@ export const CONTEXT_RESOLVE_KEY_NOT_FOUND: unique symbol = Symbol.for("ContextR // Note: we're using classes here to be able to use decorators to describe each context node and key @Profile() export abstract class ConfigContext { - private readonly _rootContext: ConfigContext + private readonly _rootContext?: ConfigContext private readonly _resolvedValues: { [path: string]: any } - private readonly _startingPoint: string | undefined + private readonly _startingPoint?: string constructor(rootContext?: ConfigContext, startingPoint?: string) { - this._rootContext = rootContext || this + if (rootContext) { + this._rootContext = rootContext + } + if (startingPoint) { + this._startingPoint = startingPoint + } this._resolvedValues = {} - this._startingPoint = startingPoint } static getSchema() { @@ -96,7 +102,14 @@ export abstract class ConfigContext { return "" } - resolve({ key, nodePath, opts, rootContext = new GenericContext({}) }: ContextResolveParams): ContextResolveOutput { + resolve({ key, nodePath, opts, rootContext }: ContextResolveParams): ContextResolveOutput { + const getRootContext = () => { + if (rootContext && this._rootContext) { + return new LayeredContext(rootContext, this._rootContext) + } + return rootContext || this._rootContext || this + } + const path = key.join(".") // if the key has previously been resolved, return it directly @@ -107,13 +120,13 @@ export abstract class ConfigContext { } // TODO: freeze opts object instead of using shallow copy - opts.stack = new Set(opts.stack || []) + opts.keyStack = new Set(opts.keyStack || []) + opts.contextStack = new Set(opts.contextStack || []) - const fullPath = nodePath.concat(key).join(".") - if (opts.stack.has(fullPath)) { + if (opts.contextStack.has(this)) { // Circular dependency error is critical, throwing here. throw new ContextResolveError({ - message: `Circular reference detected when resolving key ${path} (${Array.from(opts.stack || []).join(" -> ")})`, + message: `Circular reference detected when resolving key ${path} (${Array.from(opts.keyStack || []).join(" -> ")})`, }) } @@ -124,6 +137,19 @@ export abstract class ConfigContext { let value: CollectionOrValue = this._startingPoint ? this[this._startingPoint] : this + + if (!isPlainObject(value) && !(value instanceof ConfigContext) && !(value instanceof UnresolvedTemplateValue)) { + throw new InternalError({ + message: `Invalid config context root: ${typeof value}`, + }) + } + + if (value instanceof UnresolvedTemplateValue) { + opts.keyStack.add(nodePath.join(".")) + opts.contextStack.add(this) + value = evaluate(value, { context: getRootContext(), opts }) + } + let nextKey = key[0] let nestedNodePath = nodePath let getUnavailableReason: (() => string) | undefined = undefined @@ -166,7 +192,8 @@ export abstract class ConfigContext { if (value instanceof ConfigContext) { const remainder = getRemainder() const stackEntry = getStackEntry() - opts.stack.add(stackEntry) + opts.keyStack.add(stackEntry) + opts.contextStack.add(this) // NOTE: we resolve even if remainder.length is zero to make sure all unresolved template values have been resolved. const res = value.resolve({ key: remainder, nodePath: nestedNodePath, opts, rootContext }) value = res.resolved @@ -176,8 +203,9 @@ export abstract class ConfigContext { // handle templated strings in context variables if (value instanceof UnresolvedTemplateValue) { - opts.stack.add(getStackEntry()) - value = evaluate(value, { context: new LayeredContext(rootContext, this._rootContext), opts }) + opts.keyStack.add(getStackEntry()) + opts.contextStack.add(this) + value = evaluate(value, { context: getRootContext(), opts }) } if (value === undefined) { @@ -221,7 +249,7 @@ export abstract class ConfigContext { if (v instanceof ConfigContext) { return v.resolve({ key: [], nodePath: nodePath.concat(key, keyPath), opts }).resolved } - return evaluate(v, { context: new LayeredContext(rootContext, this._rootContext), opts }) + return evaluate(v, { context: getRootContext(), opts }) }) } @@ -237,6 +265,12 @@ export abstract class ConfigContext { */ export class GenericContext extends ConfigContext { constructor(private readonly data: any) { + if (data instanceof ConfigContext) { + throw new InternalError({ + message: + "Generic context is useless when instantiated with just another context as parameter. Use the other context directly instead.", + }) + } super(undefined, "data") } diff --git a/core/src/config/workflow.ts b/core/src/config/workflow.ts index 2fe922603f..aca3ca6f3c 100644 --- a/core/src/config/workflow.ts +++ b/core/src/config/workflow.ts @@ -33,6 +33,7 @@ import { DOCS_BASE_URL } from "../constants.js" import { capture } from "../template/capture.js" import { deepEvaluate } from "../template/evaluate.js" import type { ParsedTemplate } from "../template/types.js" +import type { DeepPrimitiveMap } from "@garden-io/platform-api-types" export const minimumWorkflowRequests = { cpu: 50, // 50 millicpu @@ -375,7 +376,7 @@ export function resolveWorkflowConfig(garden: Garden, config: WorkflowConfig) { } if (config.internal.inputs) { - resolvedPartialConfig.internal.inputs = capture(config.internal.inputs, context) + resolvedPartialConfig.internal.inputs = capture(config.internal.inputs, context) as unknown as DeepPrimitiveMap } log.silly(() => `Validating config for workflow ${config.name}`) diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index bc31559aab..9bf1acc815 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -70,6 +70,7 @@ import { capture } from "../template/capture.js" import { CapturedContext } from "../config/template-contexts/base.js" import { deepEvaluate } from "../template/evaluate.js" import type { ParsedTemplate } from "../template/types.js" +import type { DeepPrimitiveMap } from "@garden-io/platform-api-types" function* sliceToBatches(dict: Record, batchSize: number) { const entries = Object.entries(dict) @@ -764,7 +765,7 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi }, variables: resolvedVariables, }) - ) + ) as unknown as DeepPrimitiveMap const template = garden.configTemplates[templateName] @@ -820,13 +821,11 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi spec, } - // Partially resolve other fields - // TODO-0.13.1: better error messages when something goes wrong here (missing inputs for example) - const resolvedOther = capture( - omit(config, builtinConfigKeys) as Record, - builtinFieldContext - ) - config = { ...config, ...resolvedOther } + // Apparently no need to capture, as later contexts will be a superset + // TODO: verify + // for (const k in omit(config, builtinConfigKeys.concat("internal")) as Record) { + // config[k] = capture(config[k], builtinFieldContext) + // } } resolveTemplates() diff --git a/core/src/outputs.ts b/core/src/outputs.ts index 3729485280..1592bf4b7a 100644 --- a/core/src/outputs.ts +++ b/core/src/outputs.ts @@ -36,10 +36,6 @@ export async function resolveProjectOutputs(garden: Garden, log: Log): Promise { const garden = this.garden - let inputs = cloneDeep(config.inputs || {}) + let inputs = config.inputs || {} const buildPath = this.garden.buildStaging.getBuildPath(config) @@ -607,18 +609,18 @@ export class ModuleResolver { const templateName = config.templateName if (templateName) { - const template = this.garden.configTemplates[templateName] - - inputs = this.resolveInputs(config, new ModuleConfigContext(templateContextParams)) - - inputs = validateWithPath({ - config: inputs, - configType: `inputs for module ${config.name}`, - path: config.configPath || config.path, - schema: template.inputsSchema, - projectRoot: garden.projectRoot, - source: undefined, - }) + // const template = this.garden.configTemplates[templateName] + + inputs = this.resolveInputs(config, new ModuleConfigContext(templateContextParams)) as unknown as DeepPrimitiveMap + + // inputs = validateWithPath({ + // config: inputs, + // configType: `inputs for module ${config.name}`, + // path: config.configPath || config.path, + // schema: template.inputsSchema, + // projectRoot: garden.projectRoot, + // source: undefined, + // }) config.inputs = inputs } diff --git a/core/src/template/analysis.ts b/core/src/template/analysis.ts index 402e010cd8..ab33660d10 100644 --- a/core/src/template/analysis.ts +++ b/core/src/template/analysis.ts @@ -8,8 +8,9 @@ import type { CollectionOrValue } from "../util/objects.js" import { isArray, isPlainObject } from "../util/objects.js" -import { ContextLookupExpression, TemplateExpression } from "./ast.js" -import type { TemplatePrimitive } from "./types.js" +import type { TemplateExpression } from "./ast.js" +import { ContextLookupExpression } from "./ast.js" +import type { ParsedTemplate, TemplatePrimitive } from "./types.js" import { UnresolvedTemplateValue } from "./types.js" import { type ConfigContext } from "../config/template-contexts/base.js" import { GardenError, InternalError } from "../exceptions.js" @@ -17,51 +18,28 @@ import { type ConfigSource } from "../config/validation.js" export type TemplateExpressionGenerator = Generator< { - value: TemplatePrimitive | UnresolvedTemplateValue | TemplateExpression + value: TemplateExpression yamlSource: ConfigSource }, void, undefined > -export function* visitAll({ - value, - source, -}: { - value: CollectionOrValue - source: ConfigSource -}): TemplateExpressionGenerator { +export function* visitAll({ value }: { value: ParsedTemplate }): TemplateExpressionGenerator { if (isArray(value)) { - for (const [k, v] of value.entries()) { + for (const v of value) { yield* visitAll({ value: v, - source: { - ...source, - path: [...source.path, k], - }, }) } } else if (isPlainObject(value)) { for (const k of Object.keys(value)) { yield* visitAll({ value: value[k], - source: { - ...source, - path: [...source.path, k], - }, }) } } else if (value instanceof UnresolvedTemplateValue) { - yield { - value, - yamlSource: source, - } yield* value.visitAll() - } else { - yield { - value, - yamlSource: source, - } } } diff --git a/core/src/template/ast.ts b/core/src/template/ast.ts index 659512af22..01521fd5a0 100644 --- a/core/src/template/ast.ts +++ b/core/src/template/ast.ts @@ -745,9 +745,10 @@ export class ContextLookupExpression extends TemplateExpression { }, }) } catch (e) { - if (e instanceof TemplateStringError) { - throw new TemplateStringError({ message: e.originalMessage, loc: this.loc, yamlSource }) - } + // TODO: improve error handling for template strings nested inside contexts + // if (e instanceof TemplateStringError) { + // throw e + // } if (e instanceof GardenError) { throw new TemplateStringError({ message: e.message, loc: this.loc, yamlSource }) } diff --git a/core/src/template/capture.ts b/core/src/template/capture.ts index 334a29f8d2..24c993bc16 100644 --- a/core/src/template/capture.ts +++ b/core/src/template/capture.ts @@ -9,22 +9,21 @@ import type { ConfigContext } from "../config/template-contexts/base.js" import { LayeredContext } from "../config/template-contexts/base.js" import { deepMap } from "../util/objects.js" -import type { TemplateExpressionGenerator } from "./analysis.js" +import { visitAll, type TemplateExpressionGenerator } from "./analysis.js" +import { deepEvaluate } from "./evaluate.js" import type { EvaluateTemplateArgs, ParsedTemplate, ResolvedTemplate } from "./types.js" -import { UnresolvedTemplateValue } from "./types.js" +import { isTemplatePrimitive, UnresolvedTemplateValue } from "./types.js" -export function capture(template: T, context: ConfigContext): T { - return deepMap(template, (v) => { - if (v instanceof UnresolvedTemplateValue) { - return new CapturedContextTemplateValue(v, context) - } - return v - }) as T +export function capture(template: ParsedTemplate, context: ConfigContext): ParsedTemplate { + if (isTemplatePrimitive(template)) { + return template + } + return new CapturedContextTemplateValue(template, context) } export class CapturedContextTemplateValue extends UnresolvedTemplateValue { constructor( - private readonly wrapped: UnresolvedTemplateValue, + private readonly wrapped: ParsedTemplate, private readonly context: ConfigContext ) { super() @@ -34,14 +33,19 @@ export class CapturedContextTemplateValue extends UnresolvedTemplateValue { override evaluate(args: EvaluateTemplateArgs): ResolvedTemplate { const context = new LayeredContext(this.context, args.context) - return this.wrapped.evaluate({ ...args, context }) + return deepEvaluate(this.wrapped, { ...args, context }) } override toJSON(): ResolvedTemplate { - return this.wrapped.toJSON() + return deepMap(this.wrapped, (v) => { + if (v instanceof UnresolvedTemplateValue) { + return v.toJSON() + } + return v + }) } override *visitAll(): TemplateExpressionGenerator { - yield* this.wrapped.visitAll() + yield* visitAll({ value: this.wrapped }) } } diff --git a/core/src/template/templated-collections.ts b/core/src/template/templated-collections.ts index 5d724daffd..08fc93efc6 100644 --- a/core/src/template/templated-collections.ts +++ b/core/src/template/templated-collections.ts @@ -214,21 +214,19 @@ export function parseTemplateCollection { + return deepMap(this.template, (v) => { if (!(v instanceof UnresolvedTemplateValue)) { return v } diff --git a/core/test/unit/src/config/template-contexts/base.ts b/core/test/unit/src/config/template-contexts/base.ts index ff1bf2f9bd..7baafd9f29 100644 --- a/core/test/unit/src/config/template-contexts/base.ts +++ b/core/test/unit/src/config/template-contexts/base.ts @@ -123,15 +123,15 @@ describe("ConfigContext", () => { nested: new GenericContext({ key: "value" }), }) const key = ["nested", "key"] - const stack = new Set([key.join(".")]) - await expectError(() => c.resolve({ key, nodePath: [], opts: { stack } }), "context-resolve") + const keyStack = new Set([key.join(".")]) + await expectError(() => c.resolve({ key, nodePath: [], opts: { keyStack } }), "context-resolve") }) it("should detect a circular reference from a nested context", async () => { class NestedContext extends ConfigContext { override resolve({ key, nodePath, opts, rootContext }: ContextResolveParams) { const circularKey = nodePath.concat(key) - opts.stack!.add(circularKey.join(".")) + opts.keyStack!.add(circularKey.join(".")) return c.resolve({ key: circularKey, nodePath: [], opts, rootContext }) } } diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index 990228e919..9eefe60876 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -2566,9 +2566,6 @@ describe("getContextLookupReferences", () => { getContextLookupReferences( visitAll({ value: obj, - source: { - path: [], - }, }), new GenericContext({}) ) @@ -2632,9 +2629,6 @@ describe("getContextLookupReferences", () => { getContextLookupReferences( visitAll({ value: obj, - source: { - path: [], - }, }), new GenericContext({ // availableLater: CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, From 4e2a9855504f7095ed506025f8743528e028c578 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 3 Jan 2025 17:56:13 +0100 Subject: [PATCH 006/117] chore: remove yaml highlighting The yaml highlighting did not work in production builds anyway and removing this functionality made it easier to compare performance between production builds and the dev build for commands like `get graph`. --- core/package.json | 5 +- core/src/commands/get/get-modules.ts | 4 +- core/src/logger/renderers.ts | 4 +- core/src/util/serialization.ts | 21 -- core/test/unit/src/logger/renderers.ts | 6 +- package-lock.json | 300 ------------------------- 6 files changed, 9 insertions(+), 331 deletions(-) diff --git a/core/package.json b/core/package.json index c1856d0b5b..f3e0b0f64d 100644 --- a/core/package.json +++ b/core/package.json @@ -39,10 +39,10 @@ "@opentelemetry/sdk-node": "^0.57.0", "@opentelemetry/sdk-trace-base": "^1.27.0", "@opentelemetry/semantic-conventions": "^1.21.0", - "@trpc/client": "^11.0.0-next-beta.308", - "@trpc/server": "^11.0.0-next-beta.308", "@scg82/exit-hook": "^3.4.1", "@segment/analytics-node": "^2.2.0", + "@trpc/client": "^11.0.0-next-beta.308", + "@trpc/server": "^11.0.0-next-beta.308", "@types/ws": "^8.5.10", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", @@ -52,7 +52,6 @@ "chalk": "^5.3.0", "chokidar": "^3.6.0", "ci-info": "^4.1.0", - "cli-highlight": "^2.1.11", "cli-table3": "^0.6.5", "cli-truncate": "^4.0.0", "cpy": "^11.1.0", diff --git a/core/src/commands/get/get-modules.ts b/core/src/commands/get/get-modules.ts index 26743b712b..cc8cdb4ca5 100644 --- a/core/src/commands/get/get-modules.ts +++ b/core/src/commands/get/get-modules.ts @@ -20,7 +20,7 @@ import { renderTable, dedent, deline } from "../../util/string.js" import { relative, sep } from "path" import type { Garden } from "../../index.js" import type { Log } from "../../logger/log-entry.js" -import { highlightYaml, safeDumpYaml } from "../../util/serialization.js" +import { safeDumpYaml } from "../../util/serialization.js" import { deepMap } from "../../util/objects.js" import { styles } from "../../logger/styles.js" @@ -137,7 +137,7 @@ function logFull(garden: Garden, modules: GardenModule[], log: Log) { ${printEmoji("🌱", log)} Module: ${styles.success(module.name)} ${divider}\n `) - log.info(highlightYaml(yaml)) + log.info(yaml) } } diff --git a/core/src/logger/renderers.ts b/core/src/logger/renderers.ts index e9e72a41a8..715f2321fa 100644 --- a/core/src/logger/renderers.ts +++ b/core/src/logger/renderers.ts @@ -14,7 +14,7 @@ import stringWidth from "string-width" import { format } from "date-fns" import { resolveMsg, type LogEntry } from "./log-entry.js" import type { JsonLogEntry } from "./writers/json-terminal-writer.js" -import { highlightYaml, safeDumpYaml } from "../util/serialization.js" +import { safeDumpYaml } from "../util/serialization.js" import type { Logger } from "./logger.js" import { logLevelMap, LogLevel } from "./logger.js" import { toGardenError } from "../exceptions.js" @@ -131,7 +131,7 @@ export function renderData(entry: LogEntry): string { } if (!dataFormat || dataFormat === "yaml") { const asYaml = safeDumpYaml(data, { noRefs: true }) - return highlightYaml(asYaml) + return asYaml } return stringify(data, null, 2) } diff --git a/core/src/util/serialization.ts b/core/src/util/serialization.ts index c5f544c733..8a5a798967 100644 --- a/core/src/util/serialization.ts +++ b/core/src/util/serialization.ts @@ -10,11 +10,8 @@ import { mapValues } from "lodash-es" import fsExtra from "fs-extra" import type { DumpOptions } from "js-yaml" import { dump, load } from "js-yaml" -import highlightModule from "cli-highlight" -import { styles } from "../logger/styles.js" const { readFile, writeFile } = fsExtra -const highlight = highlightModule.default export async function dumpYaml(yamlPath: string, data: any) { return writeFile(yamlPath, safeDumpYaml(data, { noRefs: true })) @@ -38,24 +35,6 @@ export function encodeYamlMulti(objects: object[]) { return objects.map((s) => safeDumpYaml(s, { noRefs: true }) + "---\n").join("") } -export function highlightYaml(s: string) { - try { - return highlight(s, { - language: "yaml", - theme: { - keyword: styles.accent.italic, - literal: styles.accent.italic, - string: styles.accent, - }, - }) - } catch (err) { - // FIXME: this is a quickfix for https://github.com/garden-io/garden/issues/5442 - // The issue needs to be fixed properly, by fixing Garden single app binary construction. - // Fallback to non-highlighted yaml if an error occurs. - return s - } -} - /** * Encode and write multiple objects as a multi-doc YAML file */ diff --git a/core/test/unit/src/logger/renderers.ts b/core/test/unit/src/logger/renderers.ts index b0fb2d699f..d3d33282a1 100644 --- a/core/test/unit/src/logger/renderers.ts +++ b/core/test/unit/src/logger/renderers.ts @@ -25,7 +25,7 @@ import type { TaskMetadata } from "../../../../src/logger/log-entry.js" import { createActionLog } from "../../../../src/logger/log-entry.js" import logSymbols from "log-symbols" import stripAnsi from "strip-ansi" -import { highlightYaml, safeDumpYaml } from "../../../../src/util/serialization.js" +import { safeDumpYaml } from "../../../../src/util/serialization.js" import { freezeTime } from "../../../helpers.js" import { format } from "date-fns" import { styles } from "../../../../src/logger/styles.js" @@ -212,12 +212,12 @@ describe("renderers", () => { it("should render yaml by default if data is passed", () => { const entry = logger.createLog().info({ data: sampleData }).getLatestEntry() const dataAsYaml = safeDumpYaml(sampleData, { noRefs: true }) - expect(renderData(entry)).to.eql(highlightYaml(dataAsYaml)) + expect(renderData(entry)).to.eql(dataAsYaml) }) it('should render yaml if dataFormat is "yaml"', () => { const entry = logger.createLog().info({ data: sampleData, dataFormat: "yaml" }).getLatestEntry() const dataAsYaml = safeDumpYaml(sampleData, { noRefs: true }) - expect(renderData(entry)).to.eql(highlightYaml(dataAsYaml)) + expect(renderData(entry)).to.eql(dataAsYaml) }) it('should render json if dataFormat is "json"', () => { const entry = logger.createLog().info({ data: sampleData, dataFormat: "json" }).getLatestEntry() diff --git a/package-lock.json b/package-lock.json index c716cd406f..23f5846c80 100644 --- a/package-lock.json +++ b/package-lock.json @@ -850,7 +850,6 @@ "chalk": "^5.3.0", "chokidar": "^3.6.0", "ci-info": "^4.1.0", - "cli-highlight": "^2.1.11", "cli-table3": "^0.6.5", "cli-truncate": "^4.0.0", "cpy": "^11.1.0", @@ -2025,305 +2024,6 @@ "@types/node": "*" } }, - "core/node_modules/cli-highlight": { - "version": "2.1.11", - "license": "ISC", - "dependencies": { - "chalk": "^4.0.0", - "highlight.js": "^10.7.1", - "mz": "^2.4.0", - "parse5": "^5.1.1", - "parse5-htmlparser2-tree-adapter": "^6.0.0", - "yargs": "^16.0.0" - }, - "bin": { - "highlight": "bin/highlight" - }, - "engines": { - "node": ">=8.0.0", - "npm": ">=5.0.0" - } - }, - "core/node_modules/cli-highlight/node_modules/chalk": { - "version": "4.1.2", - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "core/node_modules/cli-highlight/node_modules/chalk/node_modules/ansi-styles": { - "version": "4.3.0", - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "core/node_modules/cli-highlight/node_modules/chalk/node_modules/ansi-styles/node_modules/color-convert": { - "version": "2.0.1", - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "core/node_modules/cli-highlight/node_modules/chalk/node_modules/ansi-styles/node_modules/color-convert/node_modules/color-name": { - "version": "1.1.4", - "license": "MIT" - }, - "core/node_modules/cli-highlight/node_modules/chalk/node_modules/supports-color": { - "version": "7.2.0", - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "core/node_modules/cli-highlight/node_modules/chalk/node_modules/supports-color/node_modules/has-flag": { - "version": "4.0.0", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "core/node_modules/cli-highlight/node_modules/highlight.js": { - "version": "10.7.3", - "license": "BSD-3-Clause", - "engines": { - "node": "*" - } - }, - "core/node_modules/cli-highlight/node_modules/mz": { - "version": "2.7.0", - "license": "MIT", - "dependencies": { - "any-promise": "^1.0.0", - "object-assign": "^4.0.1", - "thenify-all": "^1.0.0" - } - }, - "core/node_modules/cli-highlight/node_modules/mz/node_modules/any-promise": { - "version": "1.3.0", - "license": "MIT" - }, - "core/node_modules/cli-highlight/node_modules/mz/node_modules/object-assign": { - "version": "4.1.1", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "core/node_modules/cli-highlight/node_modules/mz/node_modules/thenify-all": { - "version": "1.6.0", - "license": "MIT", - "dependencies": { - "thenify": ">= 3.1.0 < 4" - }, - "engines": { - "node": ">=0.8" - } - }, - "core/node_modules/cli-highlight/node_modules/mz/node_modules/thenify-all/node_modules/thenify": { - "version": "3.3.1", - "license": "MIT", - "dependencies": { - "any-promise": "^1.0.0" - } - }, - "core/node_modules/cli-highlight/node_modules/parse5": { - "version": "5.1.1", - "license": "MIT" - }, - "core/node_modules/cli-highlight/node_modules/parse5-htmlparser2-tree-adapter": { - "version": "6.0.1", - "license": "MIT", - "dependencies": { - "parse5": "^6.0.1" - } - }, - "core/node_modules/cli-highlight/node_modules/parse5-htmlparser2-tree-adapter/node_modules/parse5": { - "version": "6.0.1", - "license": "MIT" - }, - "core/node_modules/cli-highlight/node_modules/yargs": { - "version": "16.2.0", - "license": "MIT", - "dependencies": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - }, - "engines": { - "node": ">=10" - } - }, - "core/node_modules/cli-highlight/node_modules/yargs/node_modules/cliui": { - "version": "7.0.4", - "license": "ISC", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^7.0.0" - } - }, - "core/node_modules/cli-highlight/node_modules/yargs/node_modules/cliui/node_modules/strip-ansi": { - "version": "6.0.1", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "core/node_modules/cli-highlight/node_modules/yargs/node_modules/cliui/node_modules/strip-ansi/node_modules/ansi-regex": { - "version": "5.0.1", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "core/node_modules/cli-highlight/node_modules/yargs/node_modules/cliui/node_modules/wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "core/node_modules/cli-highlight/node_modules/yargs/node_modules/cliui/node_modules/wrap-ansi/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "core/node_modules/cli-highlight/node_modules/yargs/node_modules/cliui/node_modules/wrap-ansi/node_modules/ansi-styles/node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "core/node_modules/cli-highlight/node_modules/yargs/node_modules/cliui/node_modules/wrap-ansi/node_modules/ansi-styles/node_modules/color-convert/node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" - }, - "core/node_modules/cli-highlight/node_modules/yargs/node_modules/escalade": { - "version": "3.1.1", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "core/node_modules/cli-highlight/node_modules/yargs/node_modules/get-caller-file": { - "version": "2.0.5", - "license": "ISC", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, - "core/node_modules/cli-highlight/node_modules/yargs/node_modules/require-directory": { - "version": "2.1.1", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "core/node_modules/cli-highlight/node_modules/yargs/node_modules/string-width": { - "version": "4.2.3", - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "core/node_modules/cli-highlight/node_modules/yargs/node_modules/string-width/node_modules/emoji-regex": { - "version": "8.0.0", - "license": "MIT" - }, - "core/node_modules/cli-highlight/node_modules/yargs/node_modules/string-width/node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "core/node_modules/cli-highlight/node_modules/yargs/node_modules/string-width/node_modules/strip-ansi": { - "version": "6.0.1", - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "core/node_modules/cli-highlight/node_modules/yargs/node_modules/string-width/node_modules/strip-ansi/node_modules/ansi-regex": { - "version": "5.0.1", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "core/node_modules/cli-highlight/node_modules/yargs/node_modules/y18n": { - "version": "5.0.8", - "license": "ISC", - "engines": { - "node": ">=10" - } - }, - "core/node_modules/cli-highlight/node_modules/yargs/node_modules/yargs-parser": { - "version": "20.2.9", - "license": "ISC", - "engines": { - "node": ">=10" - } - }, "core/node_modules/cli-truncate": { "version": "4.0.0", "license": "MIT", From 35dbc4da4063fa3dc50b5af2faad17cef2045448 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 3 Jan 2025 17:58:32 +0100 Subject: [PATCH 007/117] perf: do not call `resolveTemplateString` anymore, use `deepEvaluate` instead. --- core/src/commands/workflow.ts | 2 +- core/src/config/render-template.ts | 9 +++------ core/src/garden.ts | 10 +++------- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/core/src/commands/workflow.ts b/core/src/commands/workflow.ts index 87e55c72b5..1bb8c4f844 100644 --- a/core/src/commands/workflow.ts +++ b/core/src/commands/workflow.ts @@ -181,7 +181,7 @@ export class WorkflowCommand extends Command { stepResult = await runStepCommand(stepParams) } else if (step.script) { - step.script = resolveTemplateString({ string: step.script, context: stepTemplateContext }) as string + step.script = deepEvaluate(step.script, { context: stepTemplateContext, opts: {} }) as string stepResult = await runStepScript(stepParams) } else { stepResult = undefined diff --git a/core/src/config/render-template.ts b/core/src/config/render-template.ts index 850c44efad..3c53e9e2d8 100644 --- a/core/src/config/render-template.ts +++ b/core/src/config/render-template.ts @@ -255,21 +255,18 @@ async function renderConfigs({ context: RenderTemplateConfigContext renderConfig: RenderTemplateConfig }): Promise { - const source = { yamlDoc: template.internal.yamlDoc, path: ["configs"] } - const templateDescription = `${configTemplateKind} '${template.name}'` const templateConfigs = template.configs || [] return Promise.all( - templateConfigs.map(async (m, index) => { + templateConfigs.map(async (m) => { // Resolve just the name, which must be immediately resolvable let resolvedName = m.name try { - resolvedName = resolveTemplateString({ - string: m.name, + resolvedName = deepEvaluate(m.name, { context, - source: { ...source, path: [...source.path, index, "name"] }, + opts: {}, }) as string } catch (error) { throw new ConfigurationError({ diff --git a/core/src/garden.ts b/core/src/garden.ts index 38ac92b624..4bb831a003 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -98,7 +98,6 @@ import { import { dedent, deline, naturalList, wordWrap } from "./util/string.js" import { DependencyGraph } from "./graph/common.js" import { Profile, profileAsync } from "./util/profiling.js" -import { resolveTemplateString } from "./template/templated-strings.js" import type { WorkflowConfig, WorkflowConfigMap } from "./config/workflow.js" import { resolveWorkflowConfig, isWorkflowConfig } from "./config/workflow.js" import type { PluginTools } from "./util/ext-tools.js" @@ -1568,10 +1567,7 @@ export class Garden { }) } - const resolved = resolveTemplateString({ - string: disabledFlag, - context, - }) + const resolved = deepEvaluate(disabledFlag, { context, opts: {} }) return !!resolved } @@ -1915,8 +1911,7 @@ export async function resolveGardenParamsPartial(currentDirectory: string, opts: source: { yamlDoc: config.internal.yamlDoc, path: ["environments"] }, }) - const configDefaultEnvironment = resolveTemplateString({ - string: config.defaultEnvironment || "", + const configDefaultEnvironment = deepEvaluate(config.defaultEnvironment || "", { context: new DefaultEnvironmentContext({ projectName, projectRoot, @@ -1925,6 +1920,7 @@ export async function resolveGardenParamsPartial(currentDirectory: string, opts: username: _username, commandInfo, }), + opts: {}, }) as string const localConfigStore = new LocalConfigStore(gardenDirPath) From 5fa739f1f0ad1c0997a04460ece8876bc38cc49d Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 3 Jan 2025 17:59:15 +0100 Subject: [PATCH 008/117] chore: do not validate providers if not fully resolved yet. --- core/src/config/project.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/config/project.ts b/core/src/config/project.ts index 9bab4e03b3..fa9fba55d0 100644 --- a/core/src/config/project.ts +++ b/core/src/config/project.ts @@ -522,8 +522,9 @@ export function resolveProjectConfig({ ...globalConfig, name, defaultEnvironment: defaultEnvironmentName, - // environments are validated later + // environments, providers and sources are validated later environments: [{ defaultNamespace: null, name: "fake-env-only-here-for-inital-load", variables: {} }], + providers: [], sources: [], }, schema: projectSchema(), From 81e68194c99aa56e6d953bbf7fd0f65164b388e7 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 3 Jan 2025 19:06:22 +0100 Subject: [PATCH 009/117] chore: remove `resolveTemplateStrings` and make sure that the template tests pass --- core/src/commands/custom.ts | 1 - core/src/commands/workflow.ts | 3 +- core/src/config/common.ts | 12 - core/src/config/render-template.ts | 8 +- core/src/config/template-contexts/base.ts | 33 +- core/src/plugin-context.ts | 19 +- core/src/plugins/hadolint/hadolint.ts | 2 +- .../kubernetes/kubernetes-type/common.ts | 8 +- .../kubernetes/kubernetes-type/deploy.ts | 2 +- core/src/template/templated-strings.ts | 24 +- core/test/unit/src/router/_helpers.ts | 4 +- core/test/unit/src/router/base.ts | 2 +- core/test/unit/src/template-string.ts | 323 ++++++++++-------- plugins/conftest/src/index.ts | 2 +- plugins/pulumi/src/helpers.ts | 9 +- 15 files changed, 232 insertions(+), 220 deletions(-) diff --git a/core/src/commands/custom.ts b/core/src/commands/custom.ts index ebbebdaca4..8bfe44b69a 100644 --- a/core/src/commands/custom.ts +++ b/core/src/commands/custom.ts @@ -21,7 +21,6 @@ import { CustomCommandContext } from "../config/template-contexts/custom-command import { validateWithPath } from "../config/validation.js" import type { GardenError } from "../exceptions.js" import { ConfigurationError, RuntimeError, InternalError, toGardenError } from "../exceptions.js" -import { resolveTemplateStrings } from "../template/templated-strings.js" import { listDirectory, isConfigFilename } from "../util/fs.js" import type { CommandParams, CommandResult, PrintHeaderParams } from "./base.js" import { Command } from "./base.js" diff --git a/core/src/commands/workflow.ts b/core/src/commands/workflow.ts index 1bb8c4f844..dee434f178 100644 --- a/core/src/commands/workflow.ts +++ b/core/src/commands/workflow.ts @@ -19,7 +19,6 @@ import type { GardenError } from "../exceptions.js" import { ChildProcessError, InternalError, RuntimeError, WorkflowScriptError, toGardenError } from "../exceptions.js" import type { WorkflowStepResult } from "../config/template-contexts/workflow.js" import { WorkflowConfigContext, WorkflowStepConfigContext } from "../config/template-contexts/workflow.js" -import { resolveTemplateStrings, resolveTemplateString } from "../template/templated-strings.js" import { ConfigurationError, FilesystemError } from "../exceptions.js" import { posix, join } from "path" import fsExtra from "fs-extra" @@ -35,7 +34,7 @@ import { getCustomCommands } from "./custom.js" import { getBuiltinCommands } from "./commands.js" import { styles } from "../logger/styles.js" import { deepEvaluate } from "../template/evaluate.js" -import { ParsedTemplate } from "../template/types.js" +import type { ParsedTemplate } from "../template/types.js" const { ensureDir, writeFile } = fsExtra diff --git a/core/src/config/common.ts b/core/src/config/common.ts index 9538470886..fca69061f4 100644 --- a/core/src/config/common.ts +++ b/core/src/config/common.ts @@ -392,18 +392,6 @@ joi = joi.extend({ // validate(value: string, { error }) { // return { value } // }, - args(schema: any, keys: any) { - // Always allow the special $merge, $forEach etc. keys, which we resolve and collapse in resolveTemplateStrings() - // Note: we allow both the expected schema and strings, since they may be templates resolving to the expected type. - return schema.keys({ - [objectSpreadKey]: joi.alternatives(joi.object(), joi.string()), - [arrayConcatKey]: joi.alternatives(joi.array(), joi.string()), - [arrayForEachKey]: joi.alternatives(joi.array(), joi.string()), - [arrayForEachFilterKey]: joi.any(), - [arrayForEachReturnKey]: joi.any(), - ...(keys || {}), - }) - }, rules: { jsonSchema: { method(jsonSchema: object) { diff --git a/core/src/config/render-template.ts b/core/src/config/render-template.ts index 3c53e9e2d8..513112d4ed 100644 --- a/core/src/config/render-template.ts +++ b/core/src/config/render-template.ts @@ -6,10 +6,9 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { Document } from "yaml" import type { ModuleConfig } from "./module.js" import { dedent, deline, naturalList } from "../util/string.js" -import type { BaseGardenResource, RenderTemplateKind, YamlDocumentWithSource } from "./base.js" +import type { BaseGardenResource, RenderTemplateKind } from "./base.js" import { baseInternalFieldsSchema, configTemplateKind, @@ -17,7 +16,7 @@ import { prepareResource, renderTemplateKind, } from "./base.js" -import { maybeTemplateString, resolveTemplateString, resolveTemplateStrings } from "../template/templated-strings.js" +import { maybeTemplateString } from "../template/templated-strings.js" import { validateWithPath } from "./validation.js" import type { Garden } from "../garden.js" import { ConfigurationError, GardenError } from "../exceptions.js" @@ -37,8 +36,7 @@ import type { Log } from "../logger/log-entry.js" import { GardenApiVersion } from "../constants.js" import { capture } from "../template/capture.js" import { deepEvaluate } from "../template/evaluate.js" -import { ParsedTemplate, TemplatePrimitive } from "../template/types.js" -import { CollectionOrValue } from "../util/objects.js" +import type { ParsedTemplate } from "../template/types.js" export const renderTemplateConfigSchema = createSchema({ name: renderTemplateKind, diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 9723da151e..589f70d340 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -30,8 +30,8 @@ export interface ContextResolveOpts { // TODO(0.14): Remove legacyAllowPartial legacyAllowPartial?: boolean - // a list of contexts for detecting circular references - contextStack?: Set + // a list of values for detecting circular references + contextStack?: Set keyStack?: Set // TODO: remove @@ -119,17 +119,6 @@ export abstract class ConfigContext { return { resolved } } - // TODO: freeze opts object instead of using shallow copy - opts.keyStack = new Set(opts.keyStack || []) - opts.contextStack = new Set(opts.contextStack || []) - - if (opts.contextStack.has(this)) { - // Circular dependency error is critical, throwing here. - throw new ContextResolveError({ - message: `Circular reference detected when resolving key ${path} (${Array.from(opts.keyStack || []).join(" -> ")})`, - }) - } - // keep track of which resolvers have been called, in order to detect circular references let getAvailableKeys: (() => string[]) | undefined = undefined @@ -144,9 +133,21 @@ export abstract class ConfigContext { }) } + // TODO: freeze opts object instead of using shallow copy + opts.keyStack = new Set(opts.keyStack || []) + opts.contextStack = new Set(opts.contextStack || []) + + if (opts.contextStack.has(value)) { + // TODO: fix circular ref detection + // Circular dependency error is critical, throwing here. + // throw new ContextResolveError({ + // message: `Circular reference detected when resolving key ${path} (${Array.from(opts.keyStack || []).join(" -> ")})`, + // }) + } + if (value instanceof UnresolvedTemplateValue) { opts.keyStack.add(nodePath.join(".")) - opts.contextStack.add(this) + opts.contextStack.add(value) value = evaluate(value, { context: getRootContext(), opts }) } @@ -193,7 +194,7 @@ export abstract class ConfigContext { const remainder = getRemainder() const stackEntry = getStackEntry() opts.keyStack.add(stackEntry) - opts.contextStack.add(this) + opts.contextStack.add(value) // NOTE: we resolve even if remainder.length is zero to make sure all unresolved template values have been resolved. const res = value.resolve({ key: remainder, nodePath: nestedNodePath, opts, rootContext }) value = res.resolved @@ -204,7 +205,7 @@ export abstract class ConfigContext { // handle templated strings in context variables if (value instanceof UnresolvedTemplateValue) { opts.keyStack.add(getStackEntry()) - opts.contextStack.add(this) + opts.contextStack.add(value) value = evaluate(value, { context: getRootContext(), opts }) } diff --git a/core/src/plugin-context.ts b/core/src/plugin-context.ts index de44ee5587..72ce61dc1e 100644 --- a/core/src/plugin-context.ts +++ b/core/src/plugin-context.ts @@ -15,7 +15,7 @@ import { deline } from "./util/string.js" import { joi, joiVariables, joiStringMap, joiIdentifier, createSchema } from "./config/common.js" import type { PluginTool } from "./util/ext-tools.js" import type { ConfigContext, ContextResolveOpts } from "./config/template-contexts/base.js" -import { resolveTemplateStrings } from "./template/templated-strings.js" +import { resolveTemplateString } from "./template/templated-strings.js" import type { Log } from "./logger/log-entry.js" import { logEntrySchema } from "./plugin/base.js" import { EventEmitter } from "eventemitter3" @@ -24,6 +24,8 @@ import { EventLogger, LogLevel } from "./logger/logger.js" import { Memoize } from "typescript-memoize" import type { ParameterObject, ParameterValues } from "./cli/params.js" import type { NamespaceStatus } from "./types/namespace.js" +import type { ParsedTemplate, ResolvedTemplate } from "./template/types.js" +import { deepEvaluate } from "./template/evaluate.js" export type WrappedFromGarden = Pick< Garden, @@ -46,7 +48,7 @@ export interface CommandInfo { opts: ParameterValues } -type ResolveTemplateStringsOpts = Omit +type ResolveTemplateStringsOpts = Omit export interface PluginContext extends WrappedFromGarden { command: CommandInfo @@ -54,7 +56,8 @@ export interface PluginContext - resolveTemplateStrings: (o: T, opts?: ResolveTemplateStringsOpts) => T + legacyResolveTemplateString: (value: string, opts?: ResolveTemplateStringsOpts) => ResolvedTemplate + deepEvaluate: (value: ParsedTemplate) => ResolvedTemplate tools: { [key: string]: PluginTool } } @@ -219,8 +222,14 @@ export async function createPluginContext({ projectSources: garden.getProjectSources(), provider, production: garden.production, - resolveTemplateStrings: (o: T, opts?: ResolveTemplateStringsOpts) => { - return resolveTemplateStrings({ value: o, context: templateContext, contextOpts: opts || {}, source: undefined }) + deepEvaluate: (o: ParsedTemplate): ResolvedTemplate => { + return deepEvaluate(o, { + context: templateContext, + opts: {}, + }) + }, + legacyResolveTemplateString: (string: string, opts?: ResolveTemplateStringsOpts) => { + return resolveTemplateString({ string, context: templateContext, contextOpts: opts || {}, source: undefined }) }, sessionId: garden.sessionId, tools: await garden.getTools(), diff --git a/core/src/plugins/hadolint/hadolint.ts b/core/src/plugins/hadolint/hadolint.ts index 04b209502c..6a58d9fd67 100644 --- a/core/src/plugins/hadolint/hadolint.ts +++ b/core/src/plugins/hadolint/hadolint.ts @@ -270,7 +270,7 @@ hadolintTest.addHandler("configure", async ({ ctx, config }) => { if (!config.include.includes(dockerfilePath)) { try { - dockerfilePath = ctx.resolveTemplateStrings(dockerfilePath) + dockerfilePath = ctx.deepEvaluate(dockerfilePath) as unknown as typeof dockerfilePath } catch (error) { if (!(error instanceof GardenError)) { throw error diff --git a/core/src/plugins/kubernetes/kubernetes-type/common.ts b/core/src/plugins/kubernetes/kubernetes-type/common.ts index 08a47035de..9c6f07577e 100644 --- a/core/src/plugins/kubernetes/kubernetes-type/common.ts +++ b/core/src/plugins/kubernetes/kubernetes-type/common.ts @@ -434,11 +434,17 @@ async function readFileManifests( // template expressions will be interpreted by the YAML parser later. // Then also, the use of `legacyAllowPartial: true` is quite unfortunate here, because users will not notice // if they reference variables that do not exist. - const resolved = ctx.resolveTemplateStrings(str, { + const resolved = ctx.legacyResolveTemplateString(str, { legacyAllowPartial: true, unescape: true, }) + if (typeof resolved !== "string") { + throw new ConfigurationError({ + message: `Expected manifest template expression in file at path ${absPath} to resolve to string; Actually got ${typeof resolved}`, + }) + } + const manifests = await parseKubernetesManifests( resolved, `${basename(absPath)} in directory ${dirname(absPath)} (specified in ${action.longDescription()})`, diff --git a/core/src/plugins/kubernetes/kubernetes-type/deploy.ts b/core/src/plugins/kubernetes/kubernetes-type/deploy.ts index 991afc524a..fd8830c245 100644 --- a/core/src/plugins/kubernetes/kubernetes-type/deploy.ts +++ b/core/src/plugins/kubernetes/kubernetes-type/deploy.ts @@ -55,7 +55,7 @@ export const kubernetesDeployDefinition = (): DeployActionDefinition { - // Explicit non-escaping takes the highest priority. - if (ctxOpts.unescape === false) { - return false - } - - return !!ctxOpts.unescape +const shouldUnescape = ({ unescape = true }: ContextResolveOpts) => { + return unescape } const parseTemplateStringCache = new LRUCache({ @@ -193,19 +188,6 @@ export function resolveTemplateString({ // See also https://github.com/garden-io/garden/issues/5825 } -/** - * Recursively parses and resolves all templated strings in the given object. - */ -// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-constraint -export function resolveTemplateStrings(_args: { - value: T - context: ConfigContext - contextOpts?: ContextResolveOpts - source: ConfigSource | undefined -}): T { - throw new NotImplementedError({ message: "TODO Resolve Template Strings" }) -} - /** * Returns `true` if the given value is a string and looks to contain a template string. */ diff --git a/core/test/unit/src/router/_helpers.ts b/core/test/unit/src/router/_helpers.ts index 63a1dffaf7..230bacc9e2 100644 --- a/core/test/unit/src/router/_helpers.ts +++ b/core/test/unit/src/router/_helpers.ts @@ -400,9 +400,9 @@ function getRouterUnitTestPlugins() { outputs: { foo: "bar", plugin: "test-plugin-a", - resolvedEnvName: _params.ctx.resolveTemplateStrings("${environment.name}"), + resolvedEnvName: _params.ctx.legacyResolveTemplateString("${environment.name}"), resolvedActionVersion: "TODO-G2 (see one line below)", - // resolvedActionVersion: _params.ctx.resolveTemplateStrings("${runtime.build.module-a.version}"), + // resolvedActionVersion: _params.ctx.legacyResolveTemplateString("${runtime.build.module-a.version}"), }, } }, diff --git a/core/test/unit/src/router/base.ts b/core/test/unit/src/router/base.ts index b6e2c62df6..1c2b532a62 100644 --- a/core/test/unit/src/router/base.ts +++ b/core/test/unit/src/router/base.ts @@ -32,7 +32,7 @@ describe("BaseActionRouter", () => { base: { pluginName: params.base?.pluginName, }, - projectName: params.ctx.resolveTemplateStrings("${project.name}"), + projectName: params.ctx.legacyResolveTemplateString("${project.name}"), }, state: "ready" as const, } diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index 9eefe60876..776199dcf1 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -10,10 +10,10 @@ import { expect } from "chai" import repeat from "lodash-es/repeat.js" import stripAnsi from "strip-ansi" import { loadAndValidateYaml } from "../../../src/config/base.js" -import { CONTEXT_RESOLVE_KEY_NOT_FOUND, GenericContext } from "../../../src/config/template-contexts/base.js" +import { GenericContext } from "../../../src/config/template-contexts/base.js" import type { ContextLookupReferenceFinding } from "../../../src/template/analysis.js" import { getContextLookupReferences, UnresolvableValue, visitAll } from "../../../src/template/analysis.js" -import { resolveTemplateString, resolveTemplateStrings } from "../../../src/template/templated-strings.js" +import { resolveTemplateString } from "../../../src/template/templated-strings.js" import { dedent } from "../../../src/util/string.js" import type { TestGarden } from "../../helpers.js" import { expectError, expectFuzzyMatch, getDataDir, makeTestGarden } from "../../helpers.js" @@ -21,8 +21,9 @@ import { TemplateStringError } from "../../../src/template/errors.js" import { getActionTemplateReferences } from "../../../src/config/references.js" import { throwOnMissingSecretKeys } from "../../../src/config/secrets.js" import { parseTemplateCollection } from "../../../src/template/templated-collections.js" +import { deepEvaluate } from "../../../src/template/evaluate.js" -describe("resolveTemplateString", () => { +describe("parse and evaluate template strings", () => { it("should return a non-templated string unchanged", () => { const res = resolveTemplateString({ string: "somestring", context: new GenericContext({}) }) expect(res).to.equal("somestring") @@ -1883,7 +1884,7 @@ describe("resolveTemplateString", () => { }) }) -describe("resolveTemplateStrings", () => { +describe("parse and evaluate template collections", () => { it("should resolve all template strings in an object with the given context", () => { const obj = { some: "${key}", @@ -1892,12 +1893,13 @@ describe("resolveTemplateStrings", () => { noTemplate: "at-all", }, } - const templateContext = new GenericContext({ + const context = new GenericContext({ key: "value", something: "else", }) - const result = resolveTemplateStrings({ source: undefined, value: obj, context: templateContext }) + const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) + const result = deepEvaluate(parsed, { context, opts: {} }) expect(result).to.eql({ some: "value", @@ -1913,11 +1915,12 @@ describe("resolveTemplateStrings", () => { some: "${key}?", other: "${missing}?", } - const templateContext = new GenericContext({ + const context = new GenericContext({ key: "value", }) - const result = resolveTemplateStrings({ source: undefined, value: obj, context: templateContext }) + const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) + const result = deepEvaluate(parsed, { context, opts: {} }) expect(result).to.eql({ some: "value", @@ -1931,9 +1934,10 @@ describe("resolveTemplateStrings", () => { b: "B", c: "c", } - const templateContext = new GenericContext({}) + const context = new GenericContext({}) - const result = resolveTemplateStrings({ source: undefined, value: obj, context: templateContext }) + const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) + const result = deepEvaluate(parsed, { context, opts: {} }) expect(result).to.eql({ a: "a", @@ -1948,9 +1952,10 @@ describe("resolveTemplateStrings", () => { c: "c", $merge: { a: "a", b: "b" }, } - const templateContext = new GenericContext({}) + const context = new GenericContext({}) - const result = resolveTemplateStrings({ source: undefined, value: obj, context: templateContext }) + const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) + const result = deepEvaluate(parsed, { context, opts: {} }) expect(result).to.eql({ a: "a", @@ -1965,9 +1970,10 @@ describe("resolveTemplateStrings", () => { b: "B", c: "c", } - const templateContext = new GenericContext({ obj: { a: "a", b: "b" } }) + const context = new GenericContext({ obj: { a: "a", b: "b" } }) - const result = resolveTemplateStrings({ source: undefined, value: obj, context: templateContext }) + const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) + const result = deepEvaluate(parsed, { context, opts: {} }) expect(result).to.eql({ a: "a", @@ -1985,9 +1991,10 @@ describe("resolveTemplateStrings", () => { a: "a", }, } - const templateContext = new GenericContext({ obj: { b: "b" } }) + const context = new GenericContext({ obj: { b: "b" } }) - const result = resolveTemplateStrings({ source: undefined, value: obj, context: templateContext }) + const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) + const result = deepEvaluate(parsed, { context, opts: {} }) expect(result).to.eql({ a: "a", @@ -2001,9 +2008,10 @@ describe("resolveTemplateStrings", () => { $merge: "${var.doesnotexist || var.obj}", c: "c", } - const templateContext = new GenericContext({ var: { obj: { a: "a", b: "b" } } }) + const context = new GenericContext({ var: { obj: { a: "a", b: "b" } } }) - const result = resolveTemplateStrings({ value: obj, context: templateContext, source: undefined }) + const parsed = parseTemplateCollection({ value: obj, source: { path: [] } }) + const result = deepEvaluate(parsed, { context, opts: {} }) expect(result).to.eql({ a: "a", @@ -2022,7 +2030,7 @@ describe("resolveTemplateStrings", () => { // }, // }, // } - // const templateContext = new GenericContext({ + // const context = new GenericContext({ // inputs: { // "merged-object": { // $merge: "${var.empty || var.input-object}", @@ -2036,11 +2044,12 @@ describe("resolveTemplateStrings", () => { // }, // }) - // const result = resolveTemplateStrings({ + // const parsed = parseTemplateCollection({ + // const result = deepEvaluate(parsed, { context, opts: {} }) // value: obj, - // context: templateContext, + // , // contextOpts: { allowPartial: true }, - // source: undefined, + // source: { path: [] }, // }) // expect(result).to.eql({ @@ -2064,21 +2073,27 @@ it("should resolve $merge keys if a dependency cannot be resolved but there's a }, }, } - const templateContext = new GenericContext({ - inputs: { - "merged-object": { - $merge: "${var.empty || var.input-object}", - INTERNAL_VAR_1: "INTERNAL_VAR_1", - }, - }, - var: { - "input-object": { - EXTERNAL_VAR_1: "EXTERNAL_VAR_1", + const context = new GenericContext( + parseTemplateCollection({ + source: { path: [] }, + value: { + inputs: { + "merged-object": { + $merge: "${var.empty || var.input-object}", + INTERNAL_VAR_1: "INTERNAL_VAR_1", + }, + }, + var: { + "input-object": { + EXTERNAL_VAR_1: "EXTERNAL_VAR_1", + }, + }, }, - }, - }) + }) + ) - const result = resolveTemplateStrings({ value: obj, context: templateContext, source: undefined }) + const parsed = parseTemplateCollection({ value: obj, source: { path: [] } }) + const result = deepEvaluate(parsed, { context, opts: {} }) expect(result).to.eql({ "key-value-array": [ @@ -2089,15 +2104,17 @@ it("should resolve $merge keys if a dependency cannot be resolved but there's a }) it("should ignore $merge keys if the object to be merged is undefined", () => { - const obj = { - $merge: "${var.doesnotexist}", - c: "c", - } - const templateContext = new GenericContext({ var: { obj: { a: "a", b: "b" } } }) + const context = new GenericContext({ var: { obj: { a: "a", b: "b" } } }) - expect(() => resolveTemplateStrings({ value: obj, context: templateContext, source: undefined })).to.throw( - "Invalid template string" - ) + const parsed = parseTemplateCollection({ + value: { + $merge: "${var.doesnotexist}", + c: "c", + }, + source: { path: [] }, + }) + + expect(() => deepEvaluate(parsed, { context, opts: {} })).to.throw("Invalid template string") }) context("$concat", () => { @@ -2105,7 +2122,9 @@ context("$concat", () => { const obj = { foo: ["a", { $concat: ["b", "c"] }, "d"], } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }) + const context = new GenericContext({}) + const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) + const res = deepEvaluate(parsed, { context, opts: {} }) expect(res).to.eql({ foo: ["a", "b", "c", "d"], }) @@ -2115,11 +2134,12 @@ context("$concat", () => { const obj = { foo: ["a", { $concat: "${foo}" }, "d"], } - const res = resolveTemplateStrings({ - source: undefined, + const context = new GenericContext({ foo: ["b", "c"] }) + const parsed = parseTemplateCollection({ + source: { path: [] }, value: obj, - context: new GenericContext({ foo: ["b", "c"] }), }) + const res = deepEvaluate(parsed, { context, opts: {} }) expect(res).to.eql({ foo: ["a", "b", "c", "d"], }) @@ -2129,11 +2149,13 @@ context("$concat", () => { const obj = { foo: ["a", { $concat: { $forEach: ["B", "C"], $return: "${lower(item.value)}" } }, "d"], } - const res = resolveTemplateStrings({ - source: undefined, + const context = new GenericContext({ foo: ["b", "c"] }) + const parsed = parseTemplateCollection({ + source: { path: [] }, value: obj, - context: new GenericContext({ foo: ["b", "c"] }), }) + const res = deepEvaluate(parsed, { context, opts: {} }) + expect(res).to.eql({ foo: ["a", "b", "c", "d"], }) @@ -2144,7 +2166,10 @@ context("$concat", () => { foo: ["a", { $concat: "b" }, "d"], } - void expectError(() => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }), { + const context = new GenericContext({}) + const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) + + void expectError(() => deepEvaluate(parsed, { context, opts: {} }), { contains: "Value of $concat key must be (or resolve to) an array (got string)", }) }) @@ -2154,7 +2179,10 @@ context("$concat", () => { foo: ["a", { $concat: "b", nope: "nay", oops: "derp" }, "d"], } - void expectError(() => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }), { + const context = new GenericContext({}) + const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) + + void expectError(() => deepEvaluate(parsed, { context, opts: {} }), { contains: 'A list item with a $concat key cannot have any other keys (found "nope" and "oops")', }) }) @@ -2163,8 +2191,8 @@ context("$concat", () => { // const obj = { // foo: ["a", { $concat: "${foo}" }, "d"], // } - // const res = resolveTemplateStrings({ - // source: undefined, + // const parsed = parseTemplateCollection({ + // source: { path: [] }, // value: obj, // context: new GenericContext({}), // contextOpts: { allowPartial: true }, @@ -2184,7 +2212,12 @@ context("$concat", () => { $else: 456, }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({ foo: 1 }) }) + const context = new GenericContext({ foo: 1 }) + const parsed = parseTemplateCollection({ + source: { path: [] }, + value: obj, + }) + const res = deepEvaluate(parsed, { context, opts: {} }) expect(res).to.eql({ bar: 123 }) }) @@ -2196,7 +2229,12 @@ context("$concat", () => { $else: 456, }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({ foo: 2 }) }) + const context = new GenericContext({ foo: 2 }) + const parsed = parseTemplateCollection({ + source: { path: [] }, + value: obj, + }) + const res = deepEvaluate(parsed, { context, opts: {} }) expect(res).to.eql({ bar: 456 }) }) @@ -2207,27 +2245,15 @@ context("$concat", () => { $then: 123, }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({ foo: 2 }) }) + const context = new GenericContext({ foo: 2 }) + const parsed = parseTemplateCollection({ + source: { path: [] }, + value: obj, + }) + const res = deepEvaluate(parsed, { context, opts: {} }) expect(res).to.eql({ bar: undefined }) }) - // it("returns object as-is if $if doesn't resolve to boolean and allowPartial=true", () => { - // const obj = { - // bar: { - // $if: "${foo}", - // $then: 123, - // $else: 456, - // }, - // } - // const res = resolveTemplateStrings({ - // source: undefined, - // value: obj, - // context: new GenericContext({ foo: 2 }), - // contextOpts: { allowPartial: true }, - // }) - // expect(res).to.eql(obj) - // }) - it("throws if $if doesn't resolve to boolean and allowPartial=false", () => { const obj = { bar: { @@ -2236,12 +2262,12 @@ context("$concat", () => { }, } - void expectError( - () => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({ foo: "bla" }) }), - { - contains: "Value of $if key must be (or resolve to) a boolean (got string)", - } - ) + const context = new GenericContext({ foo: "bla" }) + const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) + + void expectError(() => deepEvaluate(parsed, { context, opts: {} }), { + contains: "Value of $if key must be (or resolve to) a boolean (got string)", + }) }) it("throws if $then key is missing", () => { @@ -2251,12 +2277,12 @@ context("$concat", () => { }, } - void expectError( - () => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({ foo: 1 }) }), - { - contains: "Missing $then field next to $if field", - } - ) + const context = new GenericContext({ foo: 1 }) + const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) + + void expectError(() => deepEvaluate(parsed, { context, opts: {} }), { + contains: "Missing $then field next to $if field", + }) }) it("throws if extra keys are found", () => { @@ -2268,12 +2294,12 @@ context("$concat", () => { }, } - void expectError( - () => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({ foo: 1 }) }), - { - contains: 'Found one or more unexpected keys on $if object: "foo"', - } - ) + const context = new GenericContext({ foo: 1 }) + const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) + + void expectError(() => deepEvaluate(parsed, { context, opts: {} }), { + contains: 'Found one or more unexpected keys on $if object: "foo"', + }) }) }) @@ -2285,7 +2311,9 @@ context("$concat", () => { $return: "foo", }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }) + const context = new GenericContext({}) + const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) + const res = deepEvaluate(parsed, { context, opts: {} }) expect(res).to.eql({ foo: ["foo", "foo", "foo"], }) @@ -2302,7 +2330,9 @@ context("$concat", () => { $return: "${item.key}: ${item.value}", }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }) + const context = new GenericContext({}) + const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) + const res = deepEvaluate(parsed, { context, opts: {} }) expect(res).to.eql({ foo: ["a: 1", "b: 2", "c: 3"], }) @@ -2315,31 +2345,13 @@ context("$concat", () => { $return: "foo", }, } - - void expectError( - () => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }), - { - contains: "Value of $forEach key must be (or resolve to) an array or mapping object (got string)", - } - ) + const context = new GenericContext({}) + const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) + void expectError(() => deepEvaluate(parsed, { context, opts: {} }), { + contains: "Value of $forEach key must be (or resolve to) an array or mapping object (got string)", + }) }) - // it("ignores the loop if the input isn't a list or object and allowPartial=true", () => { - // const obj = { - // foo: { - // $forEach: "${foo}", - // $return: "foo", - // }, - // } - // const res = resolveTemplateStrings({ - // source: undefined, - // value: obj, - // context: new GenericContext({}), - // contextOpts: { allowPartial: true }, - // }) - // expect(res).to.eql(obj) - // }) - it("throws if there's no $return clause", () => { const obj = { foo: { @@ -2347,12 +2359,9 @@ context("$concat", () => { }, } - void expectError( - () => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }), - { - contains: "Missing $return field next to $forEach field.", - } - ) + void expectError(() => parseTemplateCollection({ source: { path: [] }, value: obj }), { + contains: "Missing $return field next to $forEach field.", + }) }) it("throws if there are superfluous keys on the object", () => { @@ -2365,12 +2374,9 @@ context("$concat", () => { }, } - void expectError( - () => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }), - { - contains: 'Found one or more unexpected keys on $forEach object: "$concat" and "foo"', - } - ) + void expectError(() => parseTemplateCollection({ source: { path: [] }, value: obj }), { + contains: 'Found one or more unexpected keys on $forEach object: "$concat" and "foo"', + }) }) it("exposes item.value and item.key when resolving the $return clause", () => { @@ -2380,11 +2386,13 @@ context("$concat", () => { $return: "${item.key}: ${item.value}", }, } - const res = resolveTemplateStrings({ - source: undefined, + const context = new GenericContext({ foo: ["a", "b", "c"] }) + const parsed = parseTemplateCollection({ + source: { path: [] }, value: obj, - context: new GenericContext({ foo: ["a", "b", "c"] }), }) + const res = deepEvaluate(parsed, { context, opts: {} }) + expect(res).to.eql({ foo: ["0: a", "1: b", "2: c"], }) @@ -2397,11 +2405,13 @@ context("$concat", () => { $return: "${item.value}", }, } - const res = resolveTemplateStrings({ - source: undefined, + const context = new GenericContext({ foo: ["a", "b", "c"] }) + const parsed = parseTemplateCollection({ + source: { path: [] }, value: obj, - context: new GenericContext({ foo: ["a", "b", "c"] }), }) + const res = deepEvaluate(parsed, { context, opts: {} }) + expect(res).to.eql({ foo: ["a", "b", "c"], }) @@ -2415,11 +2425,13 @@ context("$concat", () => { $return: "${item.value}", }, } - const res = resolveTemplateStrings({ - source: undefined, + const context = new GenericContext({ foo: ["a", "b", "c"] }) + const parsed = parseTemplateCollection({ + source: { path: [] }, value: obj, - context: new GenericContext({ foo: ["a", "b", "c"] }), }) + const res = deepEvaluate(parsed, { context, opts: {} }) + expect(res).to.eql({ foo: ["a", "c"], }) @@ -2433,13 +2445,11 @@ context("$concat", () => { $return: "${item.value}", }, } - - void expectError( - () => resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }), - { - contains: "$filter clause in $forEach loop must resolve to a boolean value (got string)", - } - ) + const context = new GenericContext({}) + const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) + void expectError(() => deepEvaluate(parsed, { context, opts: {} }), { + contains: "$filter clause in $forEach loop must resolve to a boolean value (got string)", + }) }) it("handles $concat clauses in $return", () => { @@ -2451,7 +2461,10 @@ context("$concat", () => { }, }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }) + const context = new GenericContext({}) + const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) + const res = deepEvaluate(parsed, { context, opts: {} }) + expect(res).to.eql({ foo: ["a-1", "a-2", "b-1", "b-2", "c-1", "c-2"], }) @@ -2470,7 +2483,10 @@ context("$concat", () => { }, }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }) + const context = new GenericContext({}) + const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) + const res = deepEvaluate(parsed, { context, opts: {} }) + expect(res).to.eql({ foo: [ ["A1", "A2"], @@ -2486,7 +2502,10 @@ context("$concat", () => { $return: "foo", }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({}) }) + const context = new GenericContext({}) + const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) + const res = deepEvaluate(parsed, { context, opts: {} }) + expect(res).to.eql({ foo: [], }) @@ -2524,7 +2543,13 @@ context("$concat", () => { }, } - const res = resolveTemplateStrings({ source: undefined, value: obj, context: new GenericContext({ services }) }) + const context = new GenericContext({ services }) + const parsed = parseTemplateCollection({ + source: { path: [] }, + value: obj, + }) + const res = deepEvaluate(parsed, { context, opts: {} }) + expect(res).to.eql({ services: [ { diff --git a/plugins/conftest/src/index.ts b/plugins/conftest/src/index.ts index 9222d1ce0b..314f2115c4 100644 --- a/plugins/conftest/src/index.ts +++ b/plugins/conftest/src/index.ts @@ -289,7 +289,7 @@ export const gardenPlugin = () => } try { - files = ctx.resolveTemplateStrings(files) + files = ctx.deepEvaluate(files) as unknown as typeof files } catch (error) { if (!(error instanceof GardenError)) { throw error diff --git a/plugins/pulumi/src/helpers.ts b/plugins/pulumi/src/helpers.ts index 28714c3d86..8832a47f5f 100644 --- a/plugins/pulumi/src/helpers.ts +++ b/plugins/pulumi/src/helpers.ts @@ -315,7 +315,7 @@ export async function applyConfig(params: PulumiParams & { previewDirPath?: stri }) } else { log.warn(dedent` - The old Pulumi varfile schema is deprecated and will be removed in a future version of Garden. + The old Pulumi varfile schema is deprecated and will be removed in a future version of Garden. For more information see: https://docs.garden.io/pulumi-plugin/about#pulumi-varfile-schema `) for (const varfileVars of varfileContents) { @@ -577,7 +577,12 @@ async function loadPulumiVarfile({ try { const str = (await readFile(resolvedPath)).toString() - const resolved = ctx.resolveTemplateStrings(str) + const resolved = ctx.legacyResolveTemplateString(str) + if (typeof resolved !== "string") { + throw new ConfigurationError({ + message: `Expected template expression to resolve to string; Actually got: ${typeof resolved}`, + }) + } const parsed = load(resolved) return parsed as DeepPrimitiveMap } catch (error) { From 7719c181f548dd9f6e454e595515c6dfa369f4fc Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 3 Jan 2025 19:37:02 +0100 Subject: [PATCH 010/117] chore: fix docs generation --- core/src/config/project.ts | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/core/src/config/project.ts b/core/src/config/project.ts index fa9fba55d0..2806f21308 100644 --- a/core/src/config/project.ts +++ b/core/src/config/project.ts @@ -482,7 +482,7 @@ export function resolveProjectConfig({ commandInfo: CommandInfo }): ProjectConfig { // Resolve template strings for non-environment-specific fields (apart from `sources`). - const { environments = [], name, sources = [] } = config + const { environments = [], name, sources = [], providers = [] } = config let globalConfig: any try { @@ -532,14 +532,9 @@ export function resolveProjectConfig({ yamlDocBasePath: [], }) - const providers = config.providers - - // This will be validated separately, after resolving templates - config.environments = environments - config = { ...config, - environments: config.environments, + environments, providers, sources, } From 9a209a4012ae7c634fae8083cefbf74743819b7c Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 3 Jan 2025 19:52:55 +0100 Subject: [PATCH 011/117] chore: fix build by not ignoring parser.d.ts --- core/.gitignore | 1 - core/src/template/.gitignore | 1 + core/src/template/parser.d.ts | 14 ++++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 core/src/template/.gitignore create mode 100644 core/src/template/parser.d.ts diff --git a/core/.gitignore b/core/.gitignore index 0e8def88b8..e69a683d4b 100644 --- a/core/.gitignore +++ b/core/.gitignore @@ -4,7 +4,6 @@ dist/ src/**/*.js src/**/*.map src/**/*.d.ts -!src/template-string/parser.d.ts support/**/*.js support/**/*.map support/**/*.d.ts diff --git a/core/src/template/.gitignore b/core/src/template/.gitignore new file mode 100644 index 0000000000..a73b07a521 --- /dev/null +++ b/core/src/template/.gitignore @@ -0,0 +1 @@ +!parser.d.ts diff --git a/core/src/template/parser.d.ts b/core/src/template/parser.d.ts new file mode 100644 index 0000000000..34a4bfb1dc --- /dev/null +++ b/core/src/template/parser.d.ts @@ -0,0 +1,14 @@ +/* + * Copyright (C) 2018-2024 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import peggy from "peggy" + +// This file is just a placeholder to make TypeScript happy +// The actual parser is generated by the build script +// It will be placed in the build outputs and then actually run from there +export const parse: peggy.Parser["parse"] = () => {} From 8ad9cdd75d3b151924bf8ac8d71b6cea115f6f79 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Tue, 7 Jan 2025 16:03:45 +0100 Subject: [PATCH 012/117] chore: fix vote example --- core/src/config/template-contexts/base.ts | 5 +++++ core/src/garden.ts | 2 +- core/src/graph/actions.ts | 4 ++-- core/src/resolve-module.ts | 2 +- core/src/tasks/resolve-action.ts | 2 +- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 589f70d340..ed42fb6b3b 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -266,6 +266,11 @@ export abstract class ConfigContext { */ export class GenericContext extends ConfigContext { constructor(private readonly data: any) { + if (data === undefined) { + throw new InternalError({ + message: "Generic context may not be undefined.", + }) + } if (data instanceof ConfigContext) { throw new InternalError({ message: diff --git a/core/src/garden.ts b/core/src/garden.ts index 4bb831a003..9eba8bf88f 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -2360,7 +2360,7 @@ export function overrideVariables(variables: ConfigContext, overrides: DeepPrimi setKeyPathNested(transformedOverrides, key, overrides[key]) } } - return new LayeredContext(new GenericContext(overrides), variables) + return new LayeredContext(new GenericContext(transformedOverrides), variables) } /** diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index 9bf1acc815..7e49426319 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -510,7 +510,7 @@ export const processActionConfig = profileAsync(async function processActionConf let variables = await mergeVariables({ basePath: effectiveConfigFileLocation, - variables: new GenericContext(config.variables), + variables: new GenericContext(config.variables || {}), varfiles: config.varfiles, log, }) @@ -734,7 +734,7 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi const resolvedVarFiles = config.varfiles?.filter((f) => !maybeTemplateString(getVarfileData(f).path)) const variables = await mergeVariables({ basePath: config.internal.basePath, - variables: new GenericContext(config.variables), + variables: new GenericContext(config.variables || {}), varfiles: resolvedVarFiles, log, }) diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index 4928fc4f9c..cf73d9e484 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -778,7 +778,7 @@ export class ModuleResolver { const configContext = new ModuleConfigContext({ garden: this.garden, resolvedProviders: this.resolvedProviders, - variables: new LayeredContext(new GenericContext(resolvedConfig.variables), this.garden.variables), + variables: new LayeredContext(new GenericContext(resolvedConfig.variables || {}), this.garden.variables), name: resolvedConfig.name, path: resolvedConfig.path, buildPath, diff --git a/core/src/tasks/resolve-action.ts b/core/src/tasks/resolve-action.ts index b22c15befe..c6aac6cc7f 100644 --- a/core/src/tasks/resolve-action.ts +++ b/core/src/tasks/resolve-action.ts @@ -161,7 +161,7 @@ export class ResolveActionTask extends BaseActionTask Date: Tue, 7 Jan 2025 17:10:09 +0100 Subject: [PATCH 013/117] chore: fix bunch of tests --- core/src/config/project.ts | 8 +- core/src/config/template-contexts/actions.ts | 2 +- core/src/garden.ts | 2 +- core/src/graph/common.ts | 4 +- core/src/resolve-module.ts | 8 +- core/src/tasks/resolve-action.ts | 4 +- core/src/template/templated-collections.ts | 2 +- .../src/actions/action-configs-to-graph.ts | 195 ++++++++++-------- 8 files changed, 126 insertions(+), 99 deletions(-) diff --git a/core/src/config/project.ts b/core/src/config/project.ts index 2806f21308..666160fd38 100644 --- a/core/src/config/project.ts +++ b/core/src/config/project.ts @@ -624,8 +624,8 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ defaultPath: defaultVarfilePath, }) const projectVariables: LayeredContext = new LayeredContext( - new GenericContext(projectVarfileVars), - new GenericContext(projectConfig.variables) + new GenericContext(projectConfig.variables), + new GenericContext(projectVarfileVars) ) const source = { yamlDoc: projectConfig.internal.yamlDoc, path: ["environments", index] } @@ -682,9 +682,9 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ }) const variables: ConfigContext = new LayeredContext( - new GenericContext(envVarfileVars), + projectVariables, new GenericContext(environmentConfig.variables), - projectVariables + new GenericContext(envVarfileVars) ) return { diff --git a/core/src/config/template-contexts/actions.ts b/core/src/config/template-contexts/actions.ts index be832ef350..f0927d2104 100644 --- a/core/src/config/template-contexts/actions.ts +++ b/core/src/config/template-contexts/actions.ts @@ -22,7 +22,7 @@ import { styles } from "../../logger/styles.js" import { LayeredContext } from "./base.js" function mergeVariables({ garden, variables }: { garden: Garden; variables: ConfigContext }): LayeredContext { - return new LayeredContext(new GenericContext(garden.variableOverrides), variables, garden.variables) + return new LayeredContext(garden.variables, variables, new GenericContext(garden.variableOverrides)) } type ActionConfigThisContextParams = Pick diff --git a/core/src/garden.ts b/core/src/garden.ts index 9eba8bf88f..ba810710b1 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -2360,7 +2360,7 @@ export function overrideVariables(variables: ConfigContext, overrides: DeepPrimi setKeyPathNested(transformedOverrides, key, overrides[key]) } } - return new LayeredContext(new GenericContext(transformedOverrides), variables) + return new LayeredContext(variables, new GenericContext(transformedOverrides)) } /** diff --git a/core/src/graph/common.ts b/core/src/graph/common.ts index dfe41ecadf..23bbc5bbfd 100644 --- a/core/src/graph/common.ts +++ b/core/src/graph/common.ts @@ -174,10 +174,10 @@ export const mergeVariables = profileAsync(async function mergeVariables({ ) return new LayeredContext( + variables || new GenericContext({}), // Merge different varfiles, later files taking precedence over prior files in the list. // TODO-0.13.0: should this be a JSON merge? - ...varsByFile.reverse().map((vars) => new GenericContext(vars)), - variables || new GenericContext({}) + ...varsByFile.map((vars) => new GenericContext(vars)) ) }) diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index cf73d9e484..e2201726bd 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -636,7 +636,7 @@ export class ModuleResolver { // so we also need to pass inputs here along with the available variables. const configContext = new ModuleConfigContext({ ...templateContextParams, - variables: new LayeredContext(resolvedModuleVariables, garden.variables), + variables: new LayeredContext(garden.variables, resolvedModuleVariables), inputs: { ...inputs }, }) @@ -778,7 +778,7 @@ export class ModuleResolver { const configContext = new ModuleConfigContext({ garden: this.garden, resolvedProviders: this.resolvedProviders, - variables: new LayeredContext(new GenericContext(resolvedConfig.variables || {}), this.garden.variables), + variables: new LayeredContext(this.garden.variables, new GenericContext(resolvedConfig.variables || {})), name: resolvedConfig.name, path: resolvedConfig.path, buildPath, @@ -951,9 +951,9 @@ export class ModuleResolver { union(keys(moduleVariables), keys(varfileVars)) ) return new LayeredContext( - new GenericContext(relevantVariableOverrides), + new GenericContext(moduleVariables), new GenericContext(varfileVars), - new GenericContext(moduleVariables) + new GenericContext(relevantVariableOverrides) ) } } diff --git a/core/src/tasks/resolve-action.ts b/core/src/tasks/resolve-action.ts index c6aac6cc7f..074c33e83a 100644 --- a/core/src/tasks/resolve-action.ts +++ b/core/src/tasks/resolve-action.ts @@ -181,9 +181,9 @@ export class ResolveActionTask extends BaseActionTask { let tmpDir: TempDirectory @@ -303,30 +304,35 @@ describe("actionConfigsToGraph", () => { garden, log, groupConfigs: [], - configs: [ - { - kind: "Build", - type: "test", - name: "foo", - timeout: DEFAULT_BUILD_TIMEOUT_SEC, - internal: { - basePath: tmpDir.path, - }, - spec: {}, - }, - { - kind: "Build", - type: "test", - name: "bar", - timeout: DEFAULT_BUILD_TIMEOUT_SEC, - internal: { - basePath: tmpDir.path, + configs: parseTemplateCollection({ + value: [ + { + kind: "Build", + type: "test", + name: "foo", + timeout: DEFAULT_BUILD_TIMEOUT_SEC, + internal: { + basePath: tmpDir.path, + }, + spec: {}, }, - spec: { - command: ["echo", "${actions.build.foo.version}"], + { + kind: "Build", + type: "test", + name: "bar", + timeout: DEFAULT_BUILD_TIMEOUT_SEC, + internal: { + basePath: tmpDir.path, + }, + spec: { + command: ["echo", "${actions.build.foo.version}"], + }, }, + ] as const, + source: { + path: [], }, - ], + }), moduleGraph: new ModuleGraph({ modules: [], moduleTypes: {} }), actionModes: {}, linkedSources: {}, @@ -352,30 +358,33 @@ describe("actionConfigsToGraph", () => { garden, log, groupConfigs: [], - configs: [ - { - kind: "Build", - type: "test", - name: "foo", - timeout: DEFAULT_BUILD_TIMEOUT_SEC, - internal: { - basePath: tmpDir.path, - }, - spec: {}, - }, - { - kind: "Build", - type: "test", - name: "bar", - timeout: DEFAULT_BUILD_TIMEOUT_SEC, - internal: { - basePath: tmpDir.path, + configs: parseTemplateCollection({ + value: [ + { + kind: "Build", + type: "test", + name: "foo", + timeout: DEFAULT_BUILD_TIMEOUT_SEC, + internal: { + basePath: tmpDir.path, + }, + spec: {}, }, - spec: { - command: ["echo", "${actions.build.foo.outputs.bar}"], + { + kind: "Build", + type: "test", + name: "bar", + timeout: DEFAULT_BUILD_TIMEOUT_SEC, + internal: { + basePath: tmpDir.path, + }, + spec: { + command: ["echo", "${actions.build.foo.outputs.bar}"], + }, }, - }, - ], + ] as const, + source: { path: [] }, + }), moduleGraph: new ModuleGraph({ modules: [], moduleTypes: {} }), actionModes: {}, linkedSources: {}, @@ -401,30 +410,33 @@ describe("actionConfigsToGraph", () => { garden, log, groupConfigs: [], - configs: [ - { - kind: "Build", - type: "container", - name: "foo", - timeout: DEFAULT_BUILD_TIMEOUT_SEC, - internal: { - basePath: tmpDir.path, - }, - spec: {}, - }, - { - kind: "Deploy", - type: "test", - name: "bar", - timeout: DEFAULT_BUILD_TIMEOUT_SEC, - internal: { - basePath: tmpDir.path, + configs: parseTemplateCollection({ + value: [ + { + kind: "Build", + type: "container", + name: "foo", + timeout: DEFAULT_BUILD_TIMEOUT_SEC, + internal: { + basePath: tmpDir.path, + }, + spec: {}, }, - spec: { - command: ["echo", "${actions.build.foo.outputs.deploymentImageName}"], + { + kind: "Deploy", + type: "test", + name: "bar", + timeout: DEFAULT_BUILD_TIMEOUT_SEC, + internal: { + basePath: tmpDir.path, + }, + spec: { + command: ["echo", "${actions.build.foo.outputs.deploymentImageName}"], + }, }, - }, - ], + ] as const, + source: { path: [] }, + }), moduleGraph: new ModuleGraph({ modules: [], moduleTypes: {} }), actionModes: {}, linkedSources: {}, @@ -505,21 +517,24 @@ describe("actionConfigsToGraph", () => { garden, log, groupConfigs: [], - configs: [ - { - kind: "Build", - type: "test", - name: "foo", - timeout: DEFAULT_BUILD_TIMEOUT_SEC, - variables: { - projectName: "${project.name}", - }, - internal: { - basePath: tmpDir.path, + configs: parseTemplateCollection({ + value: [ + { + kind: "Build", + type: "test", + name: "foo", + timeout: DEFAULT_BUILD_TIMEOUT_SEC, + variables: { + projectName: "${project.name}" as string, + }, + internal: { + basePath: tmpDir.path, + }, + spec: {}, }, - spec: {}, - }, - ], + ] as const, + source: { path: [] }, + }), moduleGraph: new ModuleGraph({ modules: [], moduleTypes: {} }), actionModes: {}, linkedSources: {}, @@ -528,7 +543,11 @@ describe("actionConfigsToGraph", () => { const action = graph.getBuild("foo") const vars = action["variables"] - expect(vars).to.eql({ projectName: garden.projectName }) + expect( + vars.resolve({ key: [], nodePath: [], opts: {}, rootContext: garden.getProjectConfigContext() }).resolved + ).to.eql({ + projectName: garden.projectName, + }) }) it("loads varfiles for the action", async () => { @@ -562,7 +581,9 @@ describe("actionConfigsToGraph", () => { const action = graph.getBuild("foo") const vars = action["variables"] - expect(vars).to.eql({ projectName: "${project.name}" }) + expect( + vars.resolve({ key: [], nodePath: [], opts: {}, rootContext: garden.getProjectConfigContext() }).resolved + ).to.eql({ projectName: "${project.name}" }) }) it("loads optional varfiles for the action", async () => { @@ -596,7 +617,9 @@ describe("actionConfigsToGraph", () => { const action = graph.getBuild("foo") const vars = action["variables"] - expect(vars).to.eql({ projectName: "${project.name}" }) + expect( + vars.resolve({ key: [], nodePath: [], opts: {}, rootContext: garden.getProjectConfigContext() }).resolved + ).to.eql({ projectName: "${project.name}" }) }) it("correctly merges varfile with variables", async () => { @@ -635,7 +658,9 @@ describe("actionConfigsToGraph", () => { const action = graph.getBuild("foo") const vars = action["variables"] - expect(vars).to.eql({ foo: "FOO", bar: "BAR", baz: "baz" }) + expect( + vars.resolve({ key: [], nodePath: [], opts: {}, rootContext: garden.getProjectConfigContext() }).resolved + ).to.eql({ foo: "FOO", bar: "BAR", baz: "baz" }) }) it("correctly merges varfile with variables when some variables are overridden with --var cli flag", async () => { @@ -690,7 +715,9 @@ describe("actionConfigsToGraph", () => { const action = graph.getBuild("foo") const vars = action["variables"] - expect(vars).to.eql({ + expect( + vars.resolve({ key: [], nodePath: [], opts: {}, rootContext: garden.getProjectConfigContext() }).resolved + ).to.eql({ foo: "NEW_FOO", bar: "BAR", baz: "baz", From 5e0735379e486555b6cb00873810ea0a413134fb Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Wed, 8 Jan 2025 11:09:03 +0100 Subject: [PATCH 014/117] fix: print correct wrapped error stack in tests --- core/src/exceptions.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/core/src/exceptions.ts b/core/src/exceptions.ts index 756654e66f..ff8c8efae0 100644 --- a/core/src/exceptions.ts +++ b/core/src/exceptions.ts @@ -123,7 +123,7 @@ export abstract class GardenError extends Error { /** * Necessary to make testFlags.expandErrors work. */ - private unexpandedStack: string | undefined + protected unexpandedStack: string | undefined constructor({ message, wrappedErrors, taskType, code }: GardenErrorParams) { super(message.trim()) @@ -374,6 +374,10 @@ export class InternalError extends GardenError { // we want it to be obvious in amplitude data that this is not a normal error condition type = "crash" + public overrideStack(newStack: string | undefined) { + this.stack = this.unexpandedStack = newStack + } + // not using object destructuring here on purpose, because errors are of type any and then the error might be passed as the params object accidentally. static wrapError(error: Error | string | any, prefix?: string): InternalError { let message: string | undefined @@ -381,11 +385,11 @@ export class InternalError extends GardenError { let code: NodeJSErrnoException["code"] | undefined if (isErrnoException(error)) { - message = error.message + message = error.toString() stack = error.stack code = error.code } else if (error instanceof Error) { - message = error.message + message = error.toString() stack = error.stack } else if (isString(error)) { message = error @@ -397,7 +401,7 @@ export class InternalError extends GardenError { message = message ? stripAnsi(message) : "" const err = new InternalError({ message: prefix ? `${stripAnsi(prefix)}: ${message}` : message, code }) - err.stack = stack + err.overrideStack(stack) return err } From 4a7fa791a6e30f95ace792a31def1e54a0a4c227 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Wed, 8 Jan 2025 11:49:14 +0100 Subject: [PATCH 015/117] chore: detect invalid keyed access of unresolved template values fail early if we access an unresolved template value --- core/src/template/types.ts | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/core/src/template/types.ts b/core/src/template/types.ts index 3711af4e87..d14acc3c99 100644 --- a/core/src/template/types.ts +++ b/core/src/template/types.ts @@ -11,6 +11,7 @@ import { isPrimitive } from "utility-types" import type { CollectionOrValue } from "../util/objects.js" import type { ConfigContext, ContextResolveOpts } from "../config/template-contexts/base.js" import type { TemplateExpressionGenerator } from "./analysis.js" +import { InternalError } from "../exceptions.js" export function isTemplatePrimitive(value: unknown): value is TemplatePrimitive { return isPrimitive(value) && typeof value !== "symbol" @@ -58,6 +59,30 @@ export type EvaluateTemplateArgs = { } export abstract class UnresolvedTemplateValue { + private objectSpreadTrap: true = true + + constructor() { + if (!this.objectSpreadTrap) { + throw new InternalError({ + message: + "Do not remove the spread trap, it exists to make our code more robust by detecting spreading unresolved template values.", + }) + } + + const proxy = new Proxy(this, { + get: (target, key) => { + const value = target[key] + if (typeof key !== "symbol" && value === undefined) { + throw new InternalError({ + message: `Unpermitted indexed access (key: '${key}') of unresolved template value. Consider evaluating template values first.`, + }) + } + return value + }, + }) + return proxy + } + public abstract evaluate(args: EvaluateTemplateArgs): ResolvedTemplate public abstract toJSON(): ResolvedTemplate From 391f08cbc1af8b62603517e3def94a940f3ef523 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 10 Jan 2025 12:12:16 +0100 Subject: [PATCH 016/117] chore: change structural operator implementation to allow partial evaluation --- core/src/config/template-contexts/base.ts | 25 ++--- core/src/template/capture.ts | 20 +++- core/src/template/evaluate.ts | 42 +++++---- core/src/template/templated-collections.ts | 101 +++++++++------------ core/src/template/templated-strings.ts | 29 ++---- core/src/template/types.ts | 58 +++++++----- 6 files changed, 141 insertions(+), 134 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index ed42fb6b3b..1b876656c4 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -15,10 +15,10 @@ import { styles } from "../../logger/styles.js" import { Profile } from "../../util/profiling.js" import type { Collection } from "../../util/objects.js" import { deepMap, isPlainObject, type CollectionOrValue } from "../../util/objects.js" -import type { ResolvedTemplate, TemplatePrimitive } from "../../template/types.js" +import type { ParsedTemplate, ResolvedTemplate, TemplatePrimitive } from "../../template/types.js" import { isTemplatePrimitive, UnresolvedTemplateValue } from "../../template/types.js" import pick from "lodash-es/pick.js" -import { evaluate } from "../../template/evaluate.js" +import { deepEvaluate, evaluate } from "../../template/evaluate.js" import merge from "lodash-es/merge.js" export type ContextKeySegment = string | number @@ -113,17 +113,17 @@ export abstract class ConfigContext { const path = key.join(".") // if the key has previously been resolved, return it directly - const resolved = this._resolvedValues[path] + const alreadyResolved = this._resolvedValues[path] - if (resolved) { - return { resolved } + if (alreadyResolved) { + return { resolved: alreadyResolved } } // keep track of which resolvers have been called, in order to detect circular references let getAvailableKeys: (() => string[]) | undefined = undefined // eslint-disable-next-line @typescript-eslint/no-this-alias - let value: CollectionOrValue = this._startingPoint + let value: CollectionOrValue = this._startingPoint ? this[this._startingPoint] : this @@ -145,12 +145,6 @@ export abstract class ConfigContext { // }) } - if (value instanceof UnresolvedTemplateValue) { - opts.keyStack.add(nodePath.join(".")) - opts.contextStack.add(value) - value = evaluate(value, { context: getRootContext(), opts }) - } - let nextKey = key[0] let nestedNodePath = nodePath let getUnavailableReason: (() => string) | undefined = undefined @@ -172,7 +166,7 @@ export abstract class ConfigContext { const getStackEntry = () => renderKeyPath(capturedNestedNodePath) getAvailableKeys = undefined - const parent: CollectionOrValue = value + const parent: CollectionOrValue = value if (isTemplatePrimitive(parent)) { throw new ContextResolveError({ message: `Attempted to look up key ${JSON.stringify(nextKey)} on a ${typeof parent}.`, @@ -206,7 +200,8 @@ export abstract class ConfigContext { if (value instanceof UnresolvedTemplateValue) { opts.keyStack.add(getStackEntry()) opts.contextStack.add(value) - value = evaluate(value, { context: getRootContext(), opts }) + const { resolved } = evaluate(value, { context: getRootContext(), opts }) + value = resolved } if (value === undefined) { @@ -250,7 +245,7 @@ export abstract class ConfigContext { if (v instanceof ConfigContext) { return v.resolve({ key: [], nodePath: nodePath.concat(key, keyPath), opts }).resolved } - return evaluate(v, { context: getRootContext(), opts }) + return deepEvaluate(v, { context: getRootContext(), opts }) }) } diff --git a/core/src/template/capture.ts b/core/src/template/capture.ts index 24c993bc16..7febcf8c50 100644 --- a/core/src/template/capture.ts +++ b/core/src/template/capture.ts @@ -10,8 +10,8 @@ import type { ConfigContext } from "../config/template-contexts/base.js" import { LayeredContext } from "../config/template-contexts/base.js" import { deepMap } from "../util/objects.js" import { visitAll, type TemplateExpressionGenerator } from "./analysis.js" -import { deepEvaluate } from "./evaluate.js" -import type { EvaluateTemplateArgs, ParsedTemplate, ResolvedTemplate } from "./types.js" +import { evaluate } from "./evaluate.js" +import type { EvaluateTemplateArgs, ParsedTemplate, ResolvedTemplate, TemplateEvaluationResult } from "./types.js" import { isTemplatePrimitive, UnresolvedTemplateValue } from "./types.js" export function capture(template: ParsedTemplate, context: ConfigContext): ParsedTemplate { @@ -30,10 +30,22 @@ export class CapturedContextTemplateValue extends UnresolvedTemplateValue { this.context = context } - override evaluate(args: EvaluateTemplateArgs): ResolvedTemplate { + override evaluate(args: EvaluateTemplateArgs): TemplateEvaluationResult { const context = new LayeredContext(this.context, args.context) - return deepEvaluate(this.wrapped, { ...args, context }) + const { resolved, partial } = evaluate(this.wrapped, { ...args, context }) + + if (partial) { + return { + partial: true, + resolved: deepMap(resolved, (v) => capture(v, context)), + } + } + + return { + partial: false, + resolved, + } } override toJSON(): ResolvedTemplate { diff --git a/core/src/template/evaluate.ts b/core/src/template/evaluate.ts index d57da93ae4..c156bf92ae 100644 --- a/core/src/template/evaluate.ts +++ b/core/src/template/evaluate.ts @@ -6,10 +6,9 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { InternalError } from "../exceptions.js" -import { deepMap } from "../util/objects.js" -import type { EvaluateTemplateArgs, ParsedTemplateValue, ResolvedTemplate, TemplatePrimitive } from "./types.js" -import { UnresolvedTemplateValue, type ParsedTemplate } from "./types.js" +import { deepMap, isArray, isPlainObject } from "../util/objects.js" +import type { EvaluateTemplateArgs, ResolvedTemplate, TemplateEvaluationResult, TemplatePrimitive } from "./types.js" +import { isTemplatePrimitive, UnresolvedTemplateValue, type ParsedTemplate } from "./types.js" type Evaluate = T extends UnresolvedTemplateValue ? ResolvedTemplate @@ -30,27 +29,34 @@ export function deepEvaluate( return deepMap(collection, (v) => { if (v instanceof UnresolvedTemplateValue) { const evaluated = evaluate(v, args) - return evaluated + if (evaluated.partial) { + return deepEvaluate(evaluated.resolved, args) + } + return evaluated.resolved } return v }) as Evaluate } -export function evaluate( - value: Input, - args: Args -): Evaluate { - if (!(value instanceof UnresolvedTemplateValue)) { - return value as Evaluate +export function evaluate(value: ParsedTemplate, args: EvaluateTemplateArgs): TemplateEvaluationResult { + if (value instanceof UnresolvedTemplateValue) { + return value.evaluate(args) } - const result = value.evaluate(args) - - if (typeof result === "symbol") { - throw new InternalError({ - message: `Evaluation was non-optional, but template expression returned symbol ${String(result)}. ast.ContextLookupExpression should have thrown an error.`, - }) + if ( + isTemplatePrimitive(value) || + (isArray(value) && value.length === 0) || + (isPlainObject(value) && Object.keys(value).length === 0) + ) { + return { + partial: false, + // template primitives, empty array or empty object do not need to be resolved + resolved: value as ResolvedTemplate, + } } - return result as Evaluate + return { + partial: true, + resolved: value, + } } diff --git a/core/src/template/templated-collections.ts b/core/src/template/templated-collections.ts index deaffa7746..3b1864d36f 100644 --- a/core/src/template/templated-collections.ts +++ b/core/src/template/templated-collections.ts @@ -13,7 +13,7 @@ import { InternalError } from "../exceptions.js" import { deepMap, isArray, isPlainObject, type CollectionOrValue } from "../util/objects.js" import { naturalList } from "../util/string.js" import { isTruthy } from "./ast.js" -import type { EvaluateTemplateArgs, ParsedTemplate, ResolvedTemplate } from "./types.js" +import type { EvaluateTemplateArgs, ParsedTemplate, ResolvedTemplate, TemplateEvaluationResult } from "./types.js" import { isTemplatePrimitive, UnresolvedTemplateValue, type TemplatePrimitive } from "./types.js" import isBoolean from "lodash-es/isBoolean.js" import { @@ -27,11 +27,12 @@ import { objectSpreadKey, } from "../config/constants.js" import mapValues from "lodash-es/mapValues.js" -import { deepEvaluate } from "./evaluate.js" +import { deepEvaluate, evaluate } from "./evaluate.js" import { LayeredContext } from "../config/template-contexts/base.js" import { parseTemplateString } from "./templated-strings.js" import { TemplateError } from "./errors.js" import { visitAll, type TemplateExpressionGenerator } from "./analysis.js" +import { capture } from "./capture.js" export function pushYamlPath(part: ObjectPath[0], configSource: ConfigSource): ConfigSource { return { @@ -245,20 +246,19 @@ export class ConcatLazyValue extends StructuralTemplateOperator { super(source, yaml) } - override evaluate(args: EvaluateTemplateArgs): ResolvedTemplate[] { - const output: ResolvedTemplate[] = [] + override evaluate(args: EvaluateTemplateArgs): { + partial: true + resolved: ParsedTemplate[] + } { + const output: ParsedTemplate[] = [] let concatYaml: (ConcatOperator | ParsedTemplate)[] // NOTE(steffen): We need to support a construct where $concat inside a $forEach expression results in a flat list. if (this.yaml instanceof ForEachLazyValue) { - const res = this.yaml.evaluate(args) + const { resolved } = this.yaml.evaluate(args) - // if (typeof res === "symbol") { - // return res - // } - - concatYaml = res + concatYaml = resolved } else { concatYaml = this.yaml } @@ -266,22 +266,12 @@ export class ConcatLazyValue extends StructuralTemplateOperator { for (const v of concatYaml) { if (!this.isConcatOperator(v)) { // it's not a concat operator, it's a list element. - const evaluated = deepEvaluate(v, args) - - // if (typeof evaluated === "symbol") { - // return evaluated - // } - - output.push(evaluated) + output.push(v) continue } // handle concat operator - const toConcatenate = deepEvaluate(v[arrayConcatKey], args) - - // if (typeof toConcatenate === "symbol") { - // return toConcatenate - // } + const { resolved: toConcatenate } = evaluate(v[arrayConcatKey], args) if (isArray(toConcatenate)) { output.push(...toConcatenate) @@ -293,8 +283,10 @@ export class ConcatLazyValue extends StructuralTemplateOperator { } } - // input tracking is already being taken care of as we just concatenate arrays - return output + return { + partial: true, + resolved: output, + } } isConcatOperator(v: ConcatOperator | ParsedTemplate): v is ConcatOperator { @@ -331,12 +323,11 @@ export class ForEachLazyValue extends StructuralTemplateOperator { super(source, yaml) } - override evaluate(args: EvaluateTemplateArgs): ResolvedTemplate[] { - const collectionValue = deepEvaluate(this.yaml[arrayForEachKey], args) - - // if (typeof collectionValue === "symbol") { - // return collectionValue - // } + override evaluate(args: EvaluateTemplateArgs): { + partial: true + resolved: ParsedTemplate[] + } { + const { resolved: collectionValue } = evaluate(this.yaml[arrayForEachKey], args) if (!isArray(collectionValue) && !isPlainObject(collectionValue)) { throw new TemplateError({ @@ -347,7 +338,7 @@ export class ForEachLazyValue extends StructuralTemplateOperator { const filterExpression = this.yaml[arrayForEachFilterKey] - const resolveOutput: ResolvedTemplate[] = [] + const resolveOutput: ParsedTemplate[] = [] for (const i of Object.keys(collectionValue)) { // put the TemplateValue in the context, not the primitive value, so we have input tracking @@ -358,11 +349,7 @@ export class ForEachLazyValue extends StructuralTemplateOperator { // Check $filter clause output, if applicable if (filterExpression !== undefined) { - const filterResult = deepEvaluate(filterExpression, { ...args, context: loopContext }) - - // if (typeof filterResult === "symbol") { - // return filterResult - // } + const { resolved: filterResult } = evaluate(filterExpression, { ...args, context: loopContext }) if (isBoolean(filterResult)) { if (!filterResult) { @@ -376,16 +363,15 @@ export class ForEachLazyValue extends StructuralTemplateOperator { } } - const returnResult = deepEvaluate(this.yaml[arrayForEachReturnKey], { ...args, context: loopContext }) - - // if (typeof returnResult === "symbol") { - // return returnResult - // } + const returnResult = capture(this.yaml[arrayForEachReturnKey], loopContext) resolveOutput.push(returnResult) } - return resolveOutput + return { + partial: true, + resolved: resolveOutput, + } } } @@ -401,24 +387,23 @@ export class ObjectSpreadLazyValue extends StructuralTemplateOperator { super(source, yaml) } - override evaluate(args: EvaluateTemplateArgs): Record { - let output: Record = {} + override evaluate(args: EvaluateTemplateArgs): { + partial: true + resolved: Record + } { + let output: Record = {} // Resolve $merge keys, depth-first, leaves-first for (const [k, v] of Object.entries(this.yaml)) { - const resolved = deepEvaluate(v, args) - - // if (typeof resolved === "symbol") { - // return resolved - // } - if (k !== objectSpreadKey) { - output[k] = resolved + output[k] = v continue } k satisfies typeof objectSpreadKey + const { resolved } = evaluate(v, args) + if (!isPlainObject(resolved)) { throw new TemplateError({ message: `Value of ${objectSpreadKey} key must be (or resolve to) a mapping object (got ${typeof resolved})`, @@ -429,7 +414,10 @@ export class ObjectSpreadLazyValue extends StructuralTemplateOperator { output = { ...output, ...resolved } } - return output + return { + partial: true, + resolved: output, + } } } @@ -448,7 +436,7 @@ export class ConditionalLazyValue extends StructuralTemplateOperator { super(source, yaml) } - override evaluate(args: EvaluateTemplateArgs): ResolvedTemplate { + override evaluate(args: EvaluateTemplateArgs): TemplateEvaluationResult { const conditionalValue = deepEvaluate(this.yaml[conditionalKey], args) // if (typeof conditionalValue === "symbol") { @@ -464,8 +452,9 @@ export class ConditionalLazyValue extends StructuralTemplateOperator { const branch = isTruthy(conditionalValue) ? this.yaml[conditionalThenKey] : this.yaml[conditionalElseKey] - const evaluated = deepEvaluate(branch, args) - - return evaluated + return { + partial: true, + resolved: branch, + } } } diff --git a/core/src/template/templated-strings.ts b/core/src/template/templated-strings.ts index 9fd44f1f62..424446c3b2 100644 --- a/core/src/template/templated-strings.ts +++ b/core/src/template/templated-strings.ts @@ -46,7 +46,10 @@ class ParsedTemplateString extends UnresolvedTemplateValue { super() } - override evaluate(args: EvaluateTemplateArgs): ResolvedTemplate { + override evaluate(args: EvaluateTemplateArgs): { + partial: false + resolved: ResolvedTemplate + } { const res = this.rootNode.evaluate({ ...args, yamlSource: this.source }) if (typeof res === "symbol") { throw new InternalError({ @@ -54,7 +57,10 @@ class ParsedTemplateString extends UnresolvedTemplateValue { "ParsedTemplateString: template expression evaluated to symbol. ContextLookupExpression should have thrown.", }) } - return res + return { + partial: false, + resolved: res, + } } public override toJSON(): string { @@ -161,31 +167,16 @@ export function resolveTemplateString({ source, }) - // string does not contain if (typeof parsed === "string") { return parsed } - const result = parsed.evaluate({ + const { resolved } = parsed.evaluate({ context, opts: contextOpts, }) - if (typeof result !== "symbol") { - return result - } - - throw new InternalError({ - message: `template expression returned symbol ${String(result)}. ast.ContextLookupExpression should have thrown an error.`, - }) - - // Requested partial evaluation and the template expression cannot be evaluated yet. We may be able to do it later. - - // TODO: Parse all template expressions after reading the YAML config and only re-evaluate ast.TemplateExpression instances in - // resolveTemplateStrings; Otherwise we'll inevitably have a bug where garden will resolve template expressions that might be - // contained in expression evaluation results e.g. if an environment variable contains template string, we don't want to - // evaluate the template string in there. - // See also https://github.com/garden-io/garden/issues/5825 + return resolved } /** diff --git a/core/src/template/types.ts b/core/src/template/types.ts index d14acc3c99..4b3fb735cc 100644 --- a/core/src/template/types.ts +++ b/core/src/template/types.ts @@ -58,33 +58,47 @@ export type EvaluateTemplateArgs = { readonly opts: Readonly } -export abstract class UnresolvedTemplateValue { - private objectSpreadTrap: true = true - - constructor() { - if (!this.objectSpreadTrap) { - throw new InternalError({ - message: - "Do not remove the spread trap, it exists to make our code more robust by detecting spreading unresolved template values.", - }) +export type TemplateEvaluationResult = + | { + partial: false + resolved: ResolvedTemplate + } + | { + partial: true + resolved: ParsedTemplate } - const proxy = new Proxy(this, { - get: (target, key) => { - const value = target[key] - if (typeof key !== "symbol" && value === undefined) { - throw new InternalError({ - message: `Unpermitted indexed access (key: '${key}') of unresolved template value. Consider evaluating template values first.`, - }) - } - return value - }, +const accessDetector = new Proxy( + {}, + { + get: (target, key) => { + if (typeof key !== "symbol") { + throw new InternalError({ + message: `Unpermitted indexed access (key: '${key}') of unresolved template value. Consider evaluating template values first.`, + }) + } + return target[key] + }, + } +) + +export abstract class UnresolvedTemplateValue { + constructor() { + // The spread trap exists to make our code more robust by detecting spreading unresolved template values. + Object.defineProperty(this, "objectSpreadTrap", { + enumerable: true, + configurable: false, + get: () => + // trigger "unpermitted indexed access" error + accessDetector["objectSpreadTrap"], }) - return proxy } - public abstract evaluate(args: EvaluateTemplateArgs): ResolvedTemplate - public abstract toJSON(): ResolvedTemplate + public abstract evaluate(args: EvaluateTemplateArgs): TemplateEvaluationResult + public abstract toJSON(): CollectionOrValue public abstract visitAll(): TemplateExpressionGenerator } + +// NOTE: this will make sure we throw an error if this value is accidentally treated as resolved. +Object.setPrototypeOf(UnresolvedTemplateValue.prototype, accessDetector) From cfa7eba0f3a7c60ebe92574a8b4d9580362fc1f5 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Sat, 11 Jan 2025 15:05:53 +0100 Subject: [PATCH 017/117] chore: also detect unpermitted write access to unresolved template value --- core/src/template/types.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/src/template/types.ts b/core/src/template/types.ts index 4b3fb735cc..e013eece93 100644 --- a/core/src/template/types.ts +++ b/core/src/template/types.ts @@ -74,7 +74,15 @@ const accessDetector = new Proxy( get: (target, key) => { if (typeof key !== "symbol") { throw new InternalError({ - message: `Unpermitted indexed access (key: '${key}') of unresolved template value. Consider evaluating template values first.`, + message: `Unpermitted indexed access (get key: '${key}') of unresolved template value. Consider evaluating template values first.`, + }) + } + return target[key] + }, + set: (target, key) => { + if (typeof key !== "symbol") { + throw new InternalError({ + message: `Unpermitted indexed access (set key: '${key}') of unresolved template value. Consider evaluating template values first.`, }) } return target[key] From 49635df52c598de6a0056233d6e161249282c6b8 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 15 Jan 2025 14:37:43 +0100 Subject: [PATCH 018/117] fix: fix early provider resolution when resolving Garden params --- core/src/config/project.ts | 66 +++++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 26 deletions(-) diff --git a/core/src/config/project.ts b/core/src/config/project.ts index 666160fd38..203cbcf6af 100644 --- a/core/src/config/project.ts +++ b/core/src/config/project.ts @@ -22,7 +22,7 @@ import { joiVariablesDescription, } from "./common.js" import { validateConfig, validateWithPath } from "./validation.js" -import { deepEvaluate } from "../template/evaluate.js" +import { deepEvaluate, evaluate } from "../template/evaluate.js" import { EnvironmentConfigContext, ProjectConfigContext } from "./template-contexts/project.js" import { findByName, getNames } from "../util/util.js" import { ConfigurationError, ParameterError, ValidationError } from "../exceptions.js" @@ -485,6 +485,18 @@ export function resolveProjectConfig({ const { environments = [], name, sources = [], providers = [] } = config let globalConfig: any + const context = new ProjectConfigContext({ + projectName: name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn, + enterpriseDomain, + secrets, + commandInfo, + }) + try { globalConfig = deepEvaluate( { @@ -495,17 +507,7 @@ export function resolveProjectConfig({ sources: [], }, { - context: new ProjectConfigContext({ - projectName: name, - projectRoot: config.path, - artifactsPath, - vcsInfo, - username, - loggedIn, - enterpriseDomain, - secrets, - commandInfo, - }), + context, opts: {}, } ) @@ -523,7 +525,7 @@ export function resolveProjectConfig({ name, defaultEnvironment: defaultEnvironmentName, // environments, providers and sources are validated later - environments: [{ defaultNamespace: null, name: "fake-env-only-here-for-inital-load", variables: {} }], + environments: [{ defaultNamespace: null, name: "fake-env-only-here-for-initial-load", variables: {} }], providers: [], sources: [], }, @@ -631,19 +633,20 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ const source = { yamlDoc: projectConfig.internal.yamlDoc, path: ["environments", index] } // Resolve template strings in the environment config, except providers + const context = new EnvironmentConfigContext({ + projectName, + projectRoot, + artifactsPath, + vcsInfo, + username, + variables: projectVariables, + loggedIn, + enterpriseDomain, + secrets, + commandInfo, + }) const config = deepEvaluate(environmentConfig as unknown as ParsedTemplate, { - context: new EnvironmentConfigContext({ - projectName, - projectRoot, - artifactsPath, - vcsInfo, - username, - variables: projectVariables, - loggedIn, - enterpriseDomain, - secrets, - commandInfo, - }), + context, opts: {}, }) @@ -659,9 +662,20 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ namespace = getNamespace(environmentConfig, namespace) const fixedProviders = fixedPlugins.map((name) => ({ name })) + // Resolve the necessary data in providers + const previewProviders = projectConfig.providers.map((p) => { + const { resolved } = evaluate(p, { context, opts: {} }) + const config = resolved as GenericProviderConfig + // We need these values to create a provider graph, + // the remaining provider configs will be evaluated in the ResolveProviderTask. + const name = deepEvaluate(config.name, { context, opts: {} }) + const dependencies = deepEvaluate(config.dependencies, { context, opts: {} }) + const environments = deepEvaluate(config.environments, { context, opts: {} }) + return { ...config, name, dependencies, environments } + }) const allProviders = [ ...fixedProviders, - ...projectConfig.providers.filter((p) => !p.environments || p.environments.includes(environment)), + ...previewProviders.filter((p) => !p.environments || p.environments.includes(environment)), ] const mergedProviders: { [name: string]: GenericProviderConfig } = {} From ee9c21470ea87b82131690394041aa101c3cd52e Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 15 Jan 2025 15:59:02 +0100 Subject: [PATCH 019/117] refactor: introduce type `UnresolvedProviderConfig` --- core/src/config/project.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/core/src/config/project.ts b/core/src/config/project.ts index 203cbcf6af..db82d99fcd 100644 --- a/core/src/config/project.ts +++ b/core/src/config/project.ts @@ -546,6 +546,15 @@ export function resolveProjectConfig({ return config } +export class UnresolvedProviderConfig { + constructor( + public readonly name: string, + public readonly dependencies: string[], + public readonly environments: string[], + public readonly unresolvedConfig: ParsedTemplate + ) {} +} + /** * Given an environment name, pulls the relevant environment-specific configuration from the specified project * config, and merges values appropriately. Also resolves template strings in the picked environment. @@ -661,9 +670,8 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ namespace = getNamespace(environmentConfig, namespace) - const fixedProviders = fixedPlugins.map((name) => ({ name })) // Resolve the necessary data in providers - const previewProviders = projectConfig.providers.map((p) => { + const unresolvedProviders = projectConfig.providers.map((p) => { const { resolved } = evaluate(p, { context, opts: {} }) const config = resolved as GenericProviderConfig // We need these values to create a provider graph, @@ -671,11 +679,15 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ const name = deepEvaluate(config.name, { context, opts: {} }) const dependencies = deepEvaluate(config.dependencies, { context, opts: {} }) const environments = deepEvaluate(config.environments, { context, opts: {} }) - return { ...config, name, dependencies, environments } + return new UnresolvedProviderConfig(name, dependencies || [], environments || [], p) + }) + + const fixedProviders = fixedPlugins.map((name) => { + return new UnresolvedProviderConfig(name, [], [], { name }) }) const allProviders = [ ...fixedProviders, - ...previewProviders.filter((p) => !p.environments || p.environments.includes(environment)), + ...unresolvedProviders.filter((p) => !p.environments || p.environments.includes(environment)), ] const mergedProviders: { [name: string]: GenericProviderConfig } = {} From 93d4fb93f950c7f128693be8ab52246cb4f74822 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Wed, 15 Jan 2025 20:19:38 +0100 Subject: [PATCH 020/117] chore: fix provider config Co-authored-by: Vladimir Vagaytsev --- core/src/commands/plugins.ts | 6 +- core/src/commands/util/fetch-tools.ts | 17 +++- core/src/config/common.ts | 5 - core/src/config/project.ts | 91 +++++++++++++------ core/src/config/provider.ts | 17 ++-- core/src/config/template-contexts/provider.ts | 7 +- core/src/garden.ts | 47 ++++++---- core/src/plugin-context.ts | 4 +- .../handlers/Provider/prepareEnvironment.ts | 4 +- core/src/plugin/handlers/base/configure.ts | 1 - core/src/plugins.ts | 12 +-- core/src/plugins/exec/exec.ts | 8 +- .../plugins/otel-collector/otel-collector.ts | 4 +- core/src/router/base.ts | 2 +- core/src/router/module.ts | 2 +- core/src/tasks/resolve-provider.ts | 53 ++++------- core/src/template/capture.ts | 31 ++++--- core/src/template/lazy-merge.ts | 78 ++++++++++++++++ core/src/template/templated-collections.ts | 9 +- core/src/template/types.ts | 4 +- core/src/util/testing.ts | 4 +- core/test/unit/src/commands/get/get-config.ts | 2 +- core/test/unit/src/config/provider.ts | 20 ++-- core/test/unit/src/garden.ts | 4 +- core/test/unit/src/plugins.ts | 71 ++++++++++----- .../unit/src/plugins/kubernetes/kubernetes.ts | 8 +- core/test/unit/src/tasks/resolve-provider.ts | 2 +- plugins/conftest/src/index.ts | 4 +- plugins/jib/src/index.ts | 4 +- plugins/pulumi/src/provider.ts | 5 +- plugins/terraform/src/index.ts | 2 +- plugins/terraform/src/provider.ts | 4 +- 32 files changed, 338 insertions(+), 194 deletions(-) create mode 100644 core/src/template/lazy-merge.ts diff --git a/core/src/commands/plugins.ts b/core/src/commands/plugins.ts index acf1026c31..129e45d9a2 100644 --- a/core/src/commands/plugins.ts +++ b/core/src/commands/plugins.ts @@ -7,7 +7,7 @@ */ import { max, fromPairs, zip } from "lodash-es" -import { findByName, getNames } from "../util/util.js" +import { findByName } from "../util/util.js" import { dedent, naturalList, renderTable, tablePresets } from "../util/string.js" import { ParameterError, toGardenError } from "../exceptions.js" import type { Log } from "../logger/log-entry.js" @@ -25,7 +25,7 @@ const pluginArgs = { help: "The name of the plugin, whose command you wish to run.", required: false, getSuggestions: ({ configDump }) => { - return getNames(configDump.providers) + return configDump.allProviderNames }, }), command: new StringOption({ @@ -65,7 +65,7 @@ export class PluginsCommand extends Command { } async action({ garden, log, args }: CommandParams): Promise { - const providerConfigs = garden.getRawProviderConfigs() + const providerConfigs = garden.getUnresolvedProviderConfigs() const configuredPlugins = providerConfigs.map((p) => p.name) if (!args.command || !args.plugin) { diff --git a/core/src/commands/util/fetch-tools.ts b/core/src/commands/util/fetch-tools.ts index c4ec0fc528..e09a0cb1ff 100644 --- a/core/src/commands/util/fetch-tools.ts +++ b/core/src/commands/util/fetch-tools.ts @@ -17,7 +17,11 @@ import { PluginTool } from "../../util/ext-tools.js" import { fromPairs, omit, uniqBy } from "lodash-es" import { printHeader, printFooter } from "../../logger/util.js" import { BooleanParameter } from "../../cli/params.js" +import type { BaseProviderConfig, Provider } from "../../config/provider.js" +interface PluginConfigWithToolVersion extends BaseProviderConfig { + version?: string +} const fetchToolsOpts = { "all": new BooleanParameter({ help: "Fetch all tools for registered plugins, instead of just ones in the current env/project.", @@ -93,12 +97,21 @@ export class FetchToolsCommand extends Command<{}, FetchToolsOpts> { // No need to fetch the same tools multiple times, if they're used in multiple providers const deduplicated = uniqBy(tools, ({ tool }) => tool["versionPath"]) - const versionedConfigs = garden.getRawProviderConfigs({ names: ["pulumi", "terraform"], allowMissing: true }) + const configuredProviders = garden + .getUnresolvedProviderConfigs({ names: ["pulumi", "terraform"], allowMissing: true }) + .map((c) => c.name) + + const resolvedProviderConfigs: Provider[] = [] + for (const providerName of configuredProviders) { + resolvedProviderConfigs.push( + await garden.resolveProvider({ name: providerName, log }) + ) + } // If the version of the tool is configured on the provider, // download only that version of the tool. const toolsNeeded = deduplicated.filter((tool) => { - const pluginToolVersion = versionedConfigs.find((p) => p.name === tool.plugin.name)?.version + const pluginToolVersion = resolvedProviderConfigs.find((p) => p.name === tool.plugin.name)?.config.version const pluginHasVersionConfigured = !!pluginToolVersion if (!pluginHasVersionConfigured) { return true diff --git a/core/src/config/common.ts b/core/src/config/common.ts index fca69061f4..3dade31485 100644 --- a/core/src/config/common.ts +++ b/core/src/config/common.ts @@ -26,11 +26,6 @@ import type { ConfigContextType } from "./template-contexts/base.js" import { z } from "zod" import { gitUrlRegex, - objectSpreadKey, - arrayConcatKey, - arrayForEachKey, - arrayForEachFilterKey, - arrayForEachReturnKey, identifierRegex, joiIdentifierDescription, userIdentifierRegex, diff --git a/core/src/config/project.ts b/core/src/config/project.ts index db82d99fcd..9644c9033d 100644 --- a/core/src/config/project.ts +++ b/core/src/config/project.ts @@ -6,7 +6,6 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { apply } from "json-merge-patch" import { dedent, deline, naturalList } from "../util/string.js" import type { DeepPrimitiveMap, Primitive, PrimitiveMap } from "./common.js" import { @@ -21,14 +20,13 @@ import { joiVariables, joiVariablesDescription, } from "./common.js" +import type { ConfigSource } from "./validation.js" import { validateConfig, validateWithPath } from "./validation.js" import { deepEvaluate, evaluate } from "../template/evaluate.js" import { EnvironmentConfigContext, ProjectConfigContext } from "./template-contexts/project.js" import { findByName, getNames } from "../util/util.js" -import { ConfigurationError, ParameterError, ValidationError } from "../exceptions.js" -import cloneDeep from "fast-copy" +import { ConfigurationError, InternalError, ParameterError, ValidationError } from "../exceptions.js" import { memoize } from "lodash-es" -import type { GenericProviderConfig } from "./provider.js" import { providerConfigBaseSchema } from "./provider.js" import type { GitScanMode } from "../constants.js" import { DOCS_BASE_URL, GardenApiVersion, defaultGitScanMode, gitScanModes } from "../constants.js" @@ -45,6 +43,8 @@ import type { ParsedTemplate } from "../template/types.js" import { LayeredContext } from "./template-contexts/base.js" import type { ConfigContext } from "./template-contexts/base.js" import { GenericContext } from "./template-contexts/base.js" +import { LazyMergePatch } from "../template/lazy-merge.js" +import { isArray, isPlainObject } from "../util/objects.js" export const defaultVarfilePath = "garden.env" export const defaultEnvVarfilePath = (environmentName: string) => `garden.${environmentName}.env` @@ -220,7 +220,7 @@ export interface ProjectConfig extends BaseGardenResource { environments: EnvironmentConfig[] scan?: ProjectScan outputs?: OutputSpec[] - providers: GenericProviderConfig[] + providers: ParsedTemplate[] sources?: SourceConfig[] varfile?: string variables: DeepPrimitiveMap @@ -550,8 +550,9 @@ export class UnresolvedProviderConfig { constructor( public readonly name: string, public readonly dependencies: string[], - public readonly environments: string[], - public readonly unresolvedConfig: ParsedTemplate + public readonly unresolvedConfig: ParsedTemplate, + // TODO: source mapping for better error messages + public readonly source?: ConfigSource ) {} } @@ -670,37 +671,69 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ namespace = getNamespace(environmentConfig, namespace) - // Resolve the necessary data in providers - const unresolvedProviders = projectConfig.providers.map((p) => { - const { resolved } = evaluate(p, { context, opts: {} }) - const config = resolved as GenericProviderConfig - // We need these values to create a provider graph, - // the remaining provider configs will be evaluated in the ResolveProviderTask. - const name = deepEvaluate(config.name, { context, opts: {} }) - const dependencies = deepEvaluate(config.dependencies, { context, opts: {} }) - const environments = deepEvaluate(config.environments, { context, opts: {} }) - return new UnresolvedProviderConfig(name, dependencies || [], environments || [], p) - }) - - const fixedProviders = fixedPlugins.map((name) => { - return new UnresolvedProviderConfig(name, [], [], { name }) - }) + const fixedProviders = fixedPlugins.map((name) => ({ name })) const allProviders = [ ...fixedProviders, - ...unresolvedProviders.filter((p) => !p.environments || p.environments.includes(environment)), + ...projectConfig.providers.filter((p) => { + const { resolved } = evaluate(p, { context, opts: {} }) + if (!isPlainObject(resolved)) { + throw new ConfigurationError({ + message: `expected provider config to be an object, actually got ${typeof resolved}`, + }) + } + const envs = deepEvaluate(resolved.environments, { context, opts: {} }) as string[] | undefined + + return !envs || envs.includes(environment) + }), ] - const mergedProviders: { [name: string]: GenericProviderConfig } = {} + const rawProviderConfigs: { [name: string]: ParsedTemplate[] } = {} - for (const provider of allProviders) { - if (!!mergedProviders[provider.name]) { - // Merge using a JSON Merge Patch (see https://tools.ietf.org/html/rfc7396) - apply(mergedProviders[provider.name], provider) + for (const p of allProviders) { + const { resolved } = evaluate(p, { context, opts: {} }) + if (!isPlainObject(resolved)) { + throw new ConfigurationError({ + message: `expected provider config to be an object, actually got ${typeof resolved}`, + }) + } + + const name = deepEvaluate(resolved.name, { context, opts: {} }) + + if (typeof name !== "string") { + throw new ConfigurationError({ + message: `expected provider name to be a string, actually got ${typeof resolved}`, + }) + } + + if (!!rawProviderConfigs[name]) { + rawProviderConfigs[name].push(p as ParsedTemplate) } else { - mergedProviders[provider.name] = cloneDeep(provider) + rawProviderConfigs[name] = [p as ParsedTemplate] } } + const mergedProviders: { [name: string]: UnresolvedProviderConfig } = {} + + for (const name in rawProviderConfigs) { + const unresolvedConfig = new LazyMergePatch(rawProviderConfigs[name]) + const { resolved: preview } = evaluate(unresolvedConfig, { context, opts: {} }) + + if (!isPlainObject(preview)) { + throw new InternalError({ + message: `Provider config evaluated to ${typeof preview}, expected object.`, + }) + } + + const dependencies = deepEvaluate(preview["dependencies"], { context, opts: {} }) as string[] | undefined + if (!(dependencies === undefined || (isArray(dependencies) && dependencies.every((d) => typeof d === "string")))) { + throw new InternalError({ + message: `Dependencies in provider config to ${typeof dependencies}, expected string array.`, + }) + } + + mergedProviders[name] = new UnresolvedProviderConfig(name, dependencies || [], unresolvedConfig) + } + const envVarfileVars = await loadVarfile({ configRoot: projectConfig.path, path: environmentConfig.varfile, diff --git a/core/src/config/provider.ts b/core/src/config/provider.ts index 57d9d20ee8..5686308dd9 100644 --- a/core/src/config/provider.ts +++ b/core/src/config/provider.ts @@ -31,6 +31,8 @@ import { uuidv4 } from "../util/random.js" import { s } from "./zod.js" import { getContextLookupReferences, visitAll } from "../template/analysis.js" import type { ConfigContext } from "./template-contexts/base.js" +import type { ParsedTemplate } from "../template/types.js" +import type { UnresolvedProviderConfig } from "./project.js" // TODO: dedupe from the joi schema below export const baseProviderConfigSchemaZod = s.object({ @@ -56,10 +58,7 @@ export interface BaseProviderConfig { name: string dependencies?: string[] environments?: string[] -} - -export interface GenericProviderConfig extends BaseProviderConfig { - [key: string]: any + path?: string } const providerFixedFieldsSchema = memoize(() => @@ -117,7 +116,7 @@ export const providerSchema = createSchema({ }) export interface ProviderMap { - [name: string]: Provider + [name: string]: Provider } export const defaultProviders = [{ name: "container" }] @@ -143,7 +142,7 @@ export function providerFromConfig({ status, }: { plugin: GardenPluginSpec - config: GenericProviderConfig + config: BaseProviderConfig dependencies: ProviderMap moduleConfigs: ModuleConfig[] status: EnvironmentStatus @@ -167,7 +166,7 @@ export function providerFromConfig({ */ export function getAllProviderDependencyNames( plugin: GardenPluginSpec, - config: GenericProviderConfig, + config: UnresolvedProviderConfig, context: ConfigContext ) { return uniq([ @@ -180,12 +179,12 @@ export function getAllProviderDependencyNames( /** * Given a provider config, return implicit dependencies based on template strings. */ -export function getProviderTemplateReferences(config: GenericProviderConfig, context: ConfigContext) { +export function getProviderTemplateReferences(config: UnresolvedProviderConfig, context: ConfigContext) { const deps: string[] = [] const generator = getContextLookupReferences( visitAll({ - value: config, + value: config.unresolvedConfig, }), context ) diff --git a/core/src/config/template-contexts/provider.ts b/core/src/config/template-contexts/provider.ts index 1d506a5d5c..412d429f6d 100644 --- a/core/src/config/template-contexts/provider.ts +++ b/core/src/config/template-contexts/provider.ts @@ -7,10 +7,9 @@ */ import { mapValues } from "lodash-es" -import type { DeepPrimitiveMap, PrimitiveMap } from "../common.js" +import type { PrimitiveMap } from "../common.js" import { joiIdentifierMap, joiPrimitive } from "../common.js" -import type { Provider, ProviderMap } from "../provider.js" -import type { GenericProviderConfig } from "../provider.js" +import type { BaseProviderConfig, Provider, ProviderMap } from "../provider.js" import type { Garden } from "../../garden.js" import { joi } from "../common.js" import { deline } from "../../util/string.js" @@ -34,7 +33,7 @@ class ProviderContext extends ConfigContext { .example({ clusterHostname: "my-cluster.example.com" }) .meta({ keyPlaceholder: "" }) ) - public config: GenericProviderConfig + public config: BaseProviderConfig @schema( joiIdentifierMap( diff --git a/core/src/garden.ts b/core/src/garden.ts index ba810710b1..e5d0448497 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -20,7 +20,13 @@ import { TreeCache } from "./cache.js" import { getBuiltinPlugins } from "./plugins/plugins.js" import type { GardenModule, ModuleConfigMap, ModuleTypeMap } from "./types/module.js" import { getModuleCacheContext } from "./types/module.js" -import type { SourceConfig, ProjectConfig, OutputSpec, ProxyConfig } from "./config/project.js" +import type { + SourceConfig, + ProjectConfig, + OutputSpec, + ProxyConfig, + UnresolvedProviderConfig, +} from "./config/project.js" import { resolveProjectConfig, pickEnvironment, @@ -82,7 +88,7 @@ import type { Log } from "./logger/log-entry.js" import { EventBus } from "./events/events.js" import { Watcher } from "./watch.js" import { findConfigPathsInPath, getWorkingCopyId, fixedProjectExcludes, defaultDotIgnoreFile } from "./util/fs.js" -import type { Provider, GenericProviderConfig, ProviderMap } from "./config/provider.js" +import type { Provider, ProviderMap, BaseProviderConfig } from "./config/provider.js" import { getAllProviderDependencyNames, defaultProvider } from "./config/provider.js" import { ResolveProviderTask } from "./tasks/resolve-provider.js" import { ActionRouter } from "./router/router.js" @@ -226,7 +232,7 @@ export interface GardenParams { projectName: string projectRoot: string projectSources?: SourceConfig[] - providerConfigs: GenericProviderConfig[] + providerConfigs: UnresolvedProviderConfig[] variables: ConfigContext variableOverrides: DeepPrimitiveMap secrets: StringMap @@ -319,7 +325,7 @@ export class Garden { public readonly vcsInfo: VcsInfo public readonly opts: GardenOpts private readonly projectConfig: ProjectConfig - private readonly providerConfigs: GenericProviderConfig[] + private readonly providerConfigs: UnresolvedProviderConfig[] public readonly workingCopyId: string public readonly dotIgnoreFile: string public readonly proxy: ProxyConfig @@ -467,7 +473,7 @@ export class Garden { // Since we don't have the ability to hook into the post provider init stage from within the provider plugin // especially because it's the absence of said provider that needs to trigger this case, // there isn't really a cleaner way around this for now. - const providerConfigs = this.getRawProviderConfigs() + const providerConfigs = this.getUnresolvedProviderConfigs() const hasOtelCollectorProvider = providerConfigs.some((providerConfig) => { return providerConfig.name === "otel-collector" @@ -691,7 +697,7 @@ export class Garden { } this.log.silly(() => `Loading plugins`) - const rawConfigs = this.getRawProviderConfigs() + const rawConfigs = this.getUnresolvedProviderConfigs() this.loadedPlugins = await loadAndResolvePlugins(this.log, this.projectRoot, this.registeredPlugins, rawConfigs) @@ -707,7 +713,7 @@ export class Garden { @pMemoizeDecorator() async getConfiguredPlugins() { const plugins = await this.getAllPlugins() - const configNames = keyBy(this.getRawProviderConfigs(), "name") + const configNames = keyBy(this.getUnresolvedProviderConfigs(), "name") return plugins.filter((p) => configNames[p.name]) } @@ -747,20 +753,20 @@ export class Garden { return this.actionTypeBases[kind][type] || [] } - getRawProviderConfigs({ names, allowMissing = false }: { names?: string[]; allowMissing?: boolean } = {}) { + getUnresolvedProviderConfigs({ names, allowMissing = false }: { names?: string[]; allowMissing?: boolean } = {}) { return names ? findByNames({ names, entries: this.providerConfigs, description: "provider", allowMissing }) : this.providerConfigs } - async resolveProvider(params: ResolveProviderParams): Promise { + async resolveProvider(params: ResolveProviderParams): Promise> { const { name, log, statusOnly } = params if (name === "_default") { - return defaultProvider + return defaultProvider as Provider } if (this.resolvedProviders[name]) { - return cloneDeep(this.resolvedProviders[name]) + return cloneDeep(this.resolvedProviders[name]) as Provider } this.log.silly(() => `Resolving provider ${name}`) @@ -778,7 +784,7 @@ export class Garden { }) } - return provider + return provider as Provider } @OtelTraced({ @@ -791,7 +797,7 @@ export class Garden { let providerNames = names await this.asyncLock.acquire("resolve-providers", async () => { - const rawConfigs = this.getRawProviderConfigs({ names }) + const rawConfigs = this.getUnresolvedProviderConfigs({ names }) if (!providerNames) { providerNames = getNames(rawConfigs) } @@ -840,7 +846,7 @@ export class Garden { for (const dep of getAllProviderDependencyNames( plugin!, - config!, + config, new RemoteSourceConfigContext(this, this.variables) )) { validationGraph.addNode(dep) @@ -980,7 +986,7 @@ export class Garden { const plugins = keyBy(loadedPlugins, "name") // We only pass configured plugins to the router (others won't have the required configuration to call handlers) - const configuredPlugins = this.getRawProviderConfigs().map((c) => plugins[c.name]) + const configuredPlugins = this.getUnresolvedProviderConfigs().map((c) => plugins[c.name]) return new ActionRouter(this, configuredPlugins, loadedPlugins, moduleTypes) } @@ -1779,7 +1785,7 @@ export class Garden { resolveProviders?: boolean resolveWorkflows?: boolean }): Promise { - let providers: (Provider | GenericProviderConfig)[] = [] + let providers: (ParsedTemplate | Provider)[] = [] let moduleConfigs: ModuleConfig[] let workflowConfigs: WorkflowConfig[] let actionConfigs: ActionConfigMap = { @@ -1794,7 +1800,7 @@ export class Garden { if (resolveProviders) { providers = Object.values(await this.resolveProviders({ log })) } else { - providers = this.getRawProviderConfigs() + providers = this.getUnresolvedProviderConfigs().map((p) => p.unresolvedConfig) } if (!graph && resolveGraph) { @@ -1816,7 +1822,6 @@ export class Garden { workflowConfigs = await this.getRawWorkflowConfigs() } } else { - providers = this.getRawProviderConfigs() moduleConfigs = await this.getRawModuleConfigs() workflowConfigs = await this.getRawWorkflowConfigs() actionConfigs = this.actionConfigs @@ -1830,9 +1835,10 @@ export class Garden { return { environmentName: this.environmentName, + allProviderNames: this.getUnresolvedProviderConfigs().map((p) => p.name), allEnvironmentNames, namespace: this.namespace, - providers: providers.map(omitInternal), + providers: providers.map((p) => (typeof p === "object" && p !== null ? omitInternal(p) : p)), variables: this.variables.resolve({ key: [], nodePath: [], opts: {} }).resolved, actionConfigs: filteredActionConfigs, moduleConfigs: moduleConfigs.map(omitInternal), @@ -2411,8 +2417,9 @@ export async function makeDummyGarden(root: string, gardenOpts: GardenOpts) { export interface ConfigDump { environmentName: string // TODO: Remove this? allEnvironmentNames: string[] + allProviderNames: string[] namespace: string - providers: OmitInternalConfig[] + providers: (OmitInternalConfig | ParsedTemplate)[] variables: DeepPrimitiveMap actionConfigs: ActionConfigMapForDump moduleConfigs: OmitInternalConfig[] diff --git a/core/src/plugin-context.ts b/core/src/plugin-context.ts index 72ce61dc1e..d757ddc5f4 100644 --- a/core/src/plugin-context.ts +++ b/core/src/plugin-context.ts @@ -9,7 +9,7 @@ import type { Garden } from "./garden.js" import type { SourceConfig } from "./config/project.js" import { projectNameSchema, projectSourcesSchema, environmentNameSchema } from "./config/project.js" -import type { Provider, GenericProviderConfig } from "./config/provider.js" +import type { Provider, BaseProviderConfig } from "./config/provider.js" import { providerSchema } from "./config/provider.js" import { deline } from "./util/string.js" import { joi, joiVariables, joiStringMap, joiIdentifier, createSchema } from "./config/common.js" @@ -50,7 +50,7 @@ export interface CommandInfo { type ResolveTemplateStringsOpts = Omit -export interface PluginContext extends WrappedFromGarden { +export interface PluginContext extends WrappedFromGarden { command: CommandInfo log: Log events: PluginEventBroker diff --git a/core/src/plugin/handlers/Provider/prepareEnvironment.ts b/core/src/plugin/handlers/Provider/prepareEnvironment.ts index 2b851a6ff5..a4535f2cb5 100644 --- a/core/src/plugin/handlers/Provider/prepareEnvironment.ts +++ b/core/src/plugin/handlers/Provider/prepareEnvironment.ts @@ -12,10 +12,10 @@ import type { EnvironmentStatus } from "./getEnvironmentStatus.js" import { dedent } from "../../../util/string.js" import { joi } from "../../../config/common.js" import { environmentStatusSchema } from "../../../config/status.js" -import type { GenericProviderConfig } from "../../../config/provider.js" +import type { BaseProviderConfig } from "../../../config/provider.js" export interface PrepareEnvironmentParams< - C extends GenericProviderConfig = any, + C extends BaseProviderConfig = any, T extends EnvironmentStatus = EnvironmentStatus, > extends PluginActionParamsBase { status: T diff --git a/core/src/plugin/handlers/base/configure.ts b/core/src/plugin/handlers/base/configure.ts index 572efb808d..8bc2b832b5 100644 --- a/core/src/plugin/handlers/base/configure.ts +++ b/core/src/plugin/handlers/base/configure.ts @@ -16,7 +16,6 @@ import { baseActionConfigSchema } from "../../../actions/base.js" import { ActionTypeHandlerSpec } from "./base.js" import { pluginContextSchema } from "../../../plugin-context.js" import { noTemplateFields } from "../../../config/base.js" -import { actionConfigSchema } from "../../../actions/helpers.js" interface ConfigureActionConfigParams extends PluginActionContextParams { log: Log diff --git a/core/src/plugins.ts b/core/src/plugins.ts index 7506d3d7da..d6b3c108c8 100644 --- a/core/src/plugins.ts +++ b/core/src/plugins.ts @@ -15,7 +15,6 @@ import type { GardenPluginReference, } from "./plugin/plugin.js" import { pluginSchema, pluginNodeModuleSchema } from "./plugin/plugin.js" -import type { GenericProviderConfig } from "./config/provider.js" import { CircularDependenciesError, ConfigurationError, PluginError, RuntimeError } from "./exceptions.js" import { uniq, mapValues, fromPairs, flatten, keyBy, some, isString, sortBy } from "lodash-es" import type { Dictionary, MaybeUndefined } from "./util/util.js" @@ -38,12 +37,13 @@ import type { } from "./plugin/action-types.js" import type { ObjectSchema } from "@hapi/joi" import { GardenSdkPlugin } from "./plugin/sdk.js" +import type { UnresolvedProviderConfig } from "./config/project.js" export async function loadAndResolvePlugins( log: Log, projectRoot: string, registeredPlugins: RegisterPluginParam[], - configs: GenericProviderConfig[] + configs: UnresolvedProviderConfig[] ) { const loadedPlugins = await Promise.all(registeredPlugins.map((p) => loadPlugin(log, projectRoot, p))) const pluginsByName = keyBy(loadedPlugins, "name") @@ -54,7 +54,7 @@ export async function loadAndResolvePlugins( export function resolvePlugins( log: Log, loadedPlugins: Dictionary, - configs: GenericProviderConfig[] + configs: UnresolvedProviderConfig[] ): GardenPluginSpec[] { const initializedPlugins: PluginMap = {} const validatePlugin = (name: string) => { @@ -266,7 +266,7 @@ export function getDependencyOrder(loadedPlugins: PluginMap): string[] { function resolvePlugin( plugin: GardenPluginSpec, loadedPlugins: PluginMap, - configs: GenericProviderConfig[] + configs: UnresolvedProviderConfig[] ): GardenPluginSpec { if (!plugin.base) { return plugin @@ -539,7 +539,7 @@ function resolveActionTypeDefinitions({ kind, }: { resolvedPlugins: PluginMap - configs: GenericProviderConfig[] + configs: UnresolvedProviderConfig[] kind: K }): PluginMap { // Collect module type declarations @@ -776,7 +776,7 @@ interface ModuleDefinitionMap { } // TODO: deduplicate from action resolution above -function resolveModuleDefinitions(resolvedPlugins: PluginMap, configs: GenericProviderConfig[]): PluginMap { +function resolveModuleDefinitions(resolvedPlugins: PluginMap, configs: UnresolvedProviderConfig[]): PluginMap { // Collect module type declarations const graph = new DependencyGraph() const moduleDefinitionMap: { [moduleType: string]: { plugin: GardenPluginSpec; spec: ModuleTypeDefinition }[] } = {} diff --git a/core/src/plugins/exec/exec.ts b/core/src/plugins/exec/exec.ts index 4cd4aa7240..657cb5d327 100644 --- a/core/src/plugins/exec/exec.ts +++ b/core/src/plugins/exec/exec.ts @@ -10,15 +10,13 @@ import { joi } from "../../config/common.js" import { dedent } from "../../util/string.js" import { runScript } from "../../util/util.js" import { ChildProcessError, RuntimeError } from "../../exceptions.js" -import type { GenericProviderConfig, Provider } from "../../config/provider.js" +import type { BaseProviderConfig, Provider } from "../../config/provider.js" import { configureExecModule, execModuleSpecSchema } from "./moduleConfig.js" import { convertExecModule } from "./convert.js" import { sdk } from "../../plugin/sdk.js" -export type ExecProviderConfig = GenericProviderConfig - -export type ExecProvider = Provider -export interface ExecProviderOutputs { +export type ExecProvider = Provider +export interface ExecProviderOutputs extends BaseProviderConfig { initScript: { log: string } diff --git a/core/src/plugins/otel-collector/otel-collector.ts b/core/src/plugins/otel-collector/otel-collector.ts index 65c0364879..86469ba5f6 100644 --- a/core/src/plugins/otel-collector/otel-collector.ts +++ b/core/src/plugins/otel-collector/otel-collector.ts @@ -7,7 +7,7 @@ */ import { join } from "path" -import type { GenericProviderConfig, Provider } from "../../config/provider.js" +import type { BaseProviderConfig, Provider } from "../../config/provider.js" import { dedent } from "../../util/string.js" import { sdk } from "../../plugin/sdk.js" import { registerCleanupFunction } from "../../util/util.js" @@ -31,7 +31,7 @@ import { toGardenError } from "../../exceptions.js" const OTEL_CONFIG_NAME = "otel-config.yaml" -export type OtelCollectorProviderConfig = GenericProviderConfig +export type OtelCollectorProviderConfig = BaseProviderConfig export type OtelCollectorProvider = Provider export const gardenPlugin = sdk.createGardenPlugin({ diff --git a/core/src/router/base.ts b/core/src/router/base.ts index 2d36da7c65..076f93e350 100644 --- a/core/src/router/base.ts +++ b/core/src/router/base.ts @@ -452,7 +452,7 @@ export abstract class BaseActionRouter extends BaseRouter if (filtered.length > 1) { // If we still end up with multiple handlers with no obvious best candidate, we use the order of configuration // as a tie-breaker. - const configs = this.garden.getRawProviderConfigs() + const configs = this.garden.getUnresolvedProviderConfigs() for (const config of configs.reverse()) { for (const handler of filtered) { diff --git a/core/src/router/module.ts b/core/src/router/module.ts index 6ad5fbef6d..1d0d758e07 100644 --- a/core/src/router/module.ts +++ b/core/src/router/module.ts @@ -275,7 +275,7 @@ export class ModuleRouter extends BaseRouter { if (filtered.length > 1) { // If we still end up with multiple handlers with no obvious best candidate, we use the order of configuration // as a tie-breaker. - const configs = this.garden.getRawProviderConfigs() + const configs = this.garden.getUnresolvedProviderConfigs() for (const config of configs.reverse()) { for (const handler of filtered) { diff --git a/core/src/tasks/resolve-provider.ts b/core/src/tasks/resolve-provider.ts index 17cfd4c503..d5f6d26a36 100644 --- a/core/src/tasks/resolve-provider.ts +++ b/core/src/tasks/resolve-provider.ts @@ -8,7 +8,7 @@ import type { CommonTaskParams, ResolveProcessDependenciesParams, TaskProcessParams } from "./base.js" import { BaseTask } from "./base.js" -import type { GenericProviderConfig, Provider, ProviderMap } from "../config/provider.js" +import type { BaseProviderConfig, Provider, ProviderMap } from "../config/provider.js" import { providerFromConfig, getProviderTemplateReferences } from "../config/provider.js" import { ConfigurationError, PluginError } from "../exceptions.js" import { keyBy, omit, flatten, uniq } from "lodash-es" @@ -31,10 +31,10 @@ import { OtelTraced } from "../util/open-telemetry/decorators.js" import { LogLevel } from "../logger/logger.js" import type { Log } from "../logger/log-entry.js" import { styles } from "../logger/styles.js" -import type { ObjectPath } from "../config/base.js" import fsExtra from "fs-extra" import { RemoteSourceConfigContext } from "../config/template-contexts/project.js" import { deepEvaluate } from "../template/evaluate.js" +import type { UnresolvedProviderConfig } from "../config/project.js" const { readFile, writeFile, ensureDir } = fsExtra @@ -53,7 +53,7 @@ function getProviderLog(providerName: string, log: Log) { interface Params extends CommonTaskParams { plugin: GardenPluginSpec allPlugins: GardenPluginSpec[] - config: GenericProviderConfig + config: UnresolvedProviderConfig forceRefresh: boolean forceInit: boolean } @@ -79,7 +79,7 @@ export class ResolveProviderTask extends BaseTask { override readonly statusConcurrencyLimit = 20 override readonly executeConcurrencyLimit = 20 - private config: GenericProviderConfig + private config: UnresolvedProviderConfig private plugin: GardenPluginSpec private forceRefresh: boolean private forceInit: boolean @@ -123,7 +123,7 @@ export class ResolveProviderTask extends BaseTask { ).map((name) => ({ name })) const allDeps = uniq([...pluginDeps, ...explicitDeps, ...implicitDeps]) - const rawProviderConfigs = this.garden.getRawProviderConfigs() + const rawProviderConfigs = this.garden.getUnresolvedProviderConfigs() const plugins = keyBy(this.allPlugins, "name") const matchDependencies = (depName: string) => { @@ -194,43 +194,26 @@ export class ResolveProviderTask extends BaseTask { this.log.silly(() => `Resolving template strings for provider ${this.config.name}`) - const projectConfig = this.garden.getProjectConfig() - const yamlDoc = projectConfig.internal.yamlDoc - let yamlDocBasePath: ObjectPath = [] - - if (yamlDoc) { - projectConfig.providers.forEach((p, i) => { - if (p.name === this.config.name) { - yamlDocBasePath = ["providers", i] - return false - } - return true - }) - } - - const source = { yamlDoc, path: yamlDocBasePath } - - let resolvedConfig = deepEvaluate(this.config, { context, opts: {} }) - const providerName = resolvedConfig.name + const evaluatedConfig = deepEvaluate(this.config.unresolvedConfig, { context, opts: {} }) + const providerName = this.config.name const providerLog = getProviderLog(providerName, this.log) providerLog.info("Configuring provider...") this.log.silly(() => `Validating ${providerName} config`) - const validateConfig = (config: GenericProviderConfig) => { - return validateWithPath({ - config: omit(config, "path"), + const validateConfig = (config: unknown) => { + return validateWithPath({ + config, schema: this.plugin.configSchema || joi.object(), path: this.garden.projectRoot, projectRoot: this.garden.projectRoot, configType: "provider configuration", ErrorClass: ConfigurationError, - source, + source: undefined, }) } - resolvedConfig = validateConfig(resolvedConfig) - resolvedConfig.path = this.garden.projectRoot + let resolvedConfig = validateConfig(evaluatedConfig) let moduleConfigs: ModuleConfig[] = [] @@ -286,14 +269,14 @@ export class ResolveProviderTask extends BaseTask { this.log.silly(() => `Validating '${providerName}' config against '${base.name}' schema`) - resolvedConfig = validateWithPath({ - config: omit(resolvedConfig, "path"), + resolvedConfig = validateWithPath({ + config: resolvedConfig, schema: base.configSchema.unknown(true), path: this.garden.projectRoot, projectRoot: this.garden.projectRoot, configType: `provider configuration (base schema from '${base.name}' plugin)`, ErrorClass: ConfigurationError, - source: { yamlDoc, path: yamlDocBasePath }, + source: undefined, }) } @@ -326,11 +309,11 @@ export class ResolveProviderTask extends BaseTask { }) } - private hashConfig(config: GenericProviderConfig) { + private hashConfig(config: BaseProviderConfig) { return hashString(stableStringify(config)) } - private async getCachedStatus(config: GenericProviderConfig): Promise { + private async getCachedStatus(config: BaseProviderConfig): Promise { const cachePath = this.getCachePath() this.log.silly(() => `Checking provider status cache for ${this.plugin.name} at ${cachePath}`) @@ -369,7 +352,7 @@ export class ResolveProviderTask extends BaseTask { return omit(cachedStatus, ["configHash", "resolvedAt"]) } - private async setCachedStatus(config: GenericProviderConfig, status: EnvironmentStatus) { + private async setCachedStatus(config: BaseProviderConfig, status: EnvironmentStatus) { const cachePath = this.getCachePath() this.log.silly(() => `Caching provider status for ${this.plugin.name} at ${cachePath}`) diff --git a/core/src/template/capture.ts b/core/src/template/capture.ts index 7febcf8c50..38d4ebc310 100644 --- a/core/src/template/capture.ts +++ b/core/src/template/capture.ts @@ -8,17 +8,27 @@ import type { ConfigContext } from "../config/template-contexts/base.js" import { LayeredContext } from "../config/template-contexts/base.js" +import type { Collection } from "../util/objects.js" import { deepMap } from "../util/objects.js" import { visitAll, type TemplateExpressionGenerator } from "./analysis.js" import { evaluate } from "./evaluate.js" -import type { EvaluateTemplateArgs, ParsedTemplate, ResolvedTemplate, TemplateEvaluationResult } from "./types.js" +import type { + EvaluateTemplateArgs, + ParsedTemplate, + ResolvedTemplate, + TemplateEvaluationResult, + TemplatePrimitive, +} from "./types.js" import { isTemplatePrimitive, UnresolvedTemplateValue } from "./types.js" -export function capture(template: ParsedTemplate, context: ConfigContext): ParsedTemplate { +type CaptureResult = Input extends TemplatePrimitive + ? Input + : CapturedContextTemplateValue +export function capture(template: Input, context: ConfigContext): CaptureResult { if (isTemplatePrimitive(template)) { - return template + return template as CaptureResult } - return new CapturedContextTemplateValue(template, context) + return new CapturedContextTemplateValue(template, context) as CaptureResult } export class CapturedContextTemplateValue extends UnresolvedTemplateValue { @@ -33,18 +43,15 @@ export class CapturedContextTemplateValue extends UnresolvedTemplateValue { override evaluate(args: EvaluateTemplateArgs): TemplateEvaluationResult { const context = new LayeredContext(this.context, args.context) - const { resolved, partial } = evaluate(this.wrapped, { ...args, context }) + const result = evaluate(this.wrapped, { ...args, context }) - if (partial) { - return { - partial: true, - resolved: deepMap(resolved, (v) => capture(v, context)), - } + if (!result.partial) { + return result } return { - partial: false, - resolved, + partial: true, + resolved: deepMap(result.resolved, (v) => capture(v, context)) as Collection, } } diff --git a/core/src/template/lazy-merge.ts b/core/src/template/lazy-merge.ts new file mode 100644 index 0000000000..40c5c60f76 --- /dev/null +++ b/core/src/template/lazy-merge.ts @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2018-2024 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import { deepMap, isArray, type CollectionOrValue } from "../util/objects.js" +import { visitAll, type TemplateExpressionGenerator } from "./analysis.js" +import { evaluate } from "./evaluate.js" +import type { EvaluateTemplateArgs, ParsedTemplate, TemplateEvaluationResult, TemplatePrimitive } from "./types.js" +import { isTemplatePrimitive, UnresolvedTemplateValue } from "./types.js" + +// https://datatracker.ietf.org/doc/html/rfc7396 +export class LazyMergePatch extends UnresolvedTemplateValue { + constructor(private readonly items: ParsedTemplate[]) { + super() + } + + public override evaluate(args: EvaluateTemplateArgs): TemplateEvaluationResult { + const toBeMerged: Record[] = [] + + for (const item of this.items.toReversed()) { + const { resolved } = evaluate(item, args) + + if (isTemplatePrimitive(resolved)) { + return { + partial: false, + resolved, + } + } + if (isArray(resolved)) { + return { + partial: true, + resolved, + } + } + + toBeMerged.push(resolved) + } + + // in-place reverse toBeMerged; We traverse items in reverse above, so we can return early + // the items we pass onto `new LazyMergePatch` need to be in the original order + toBeMerged.reverse() + + const keys = new Set() + for (const value of toBeMerged) { + for (const k of Object.keys(value)) { + keys.add(k) + } + } + + const returnValue: Record = {} + + for (const k of keys) { + const items = toBeMerged.filter((o) => k in o).map((o) => o[k]) + if (items.length === 1) { + returnValue[k] = items[0] + } else { + returnValue[k] = new LazyMergePatch(items) + } + } + + return { + partial: true, + resolved: returnValue, + } + } + + public override toJSON(): CollectionOrValue { + return deepMap(this.items, (v) => (v instanceof UnresolvedTemplateValue ? v.toJSON() : v)) + } + + public override *visitAll(): TemplateExpressionGenerator { + yield* visitAll({ value: this.items }) + } +} diff --git a/core/src/template/templated-collections.ts b/core/src/template/templated-collections.ts index 3b1864d36f..de639bb288 100644 --- a/core/src/template/templated-collections.ts +++ b/core/src/template/templated-collections.ts @@ -439,10 +439,6 @@ export class ConditionalLazyValue extends StructuralTemplateOperator { override evaluate(args: EvaluateTemplateArgs): TemplateEvaluationResult { const conditionalValue = deepEvaluate(this.yaml[conditionalKey], args) - // if (typeof conditionalValue === "symbol") { - // return conditionalValue - // } - if (typeof conditionalValue !== "boolean") { throw new TemplateError({ message: `Value of ${conditionalKey} key must be (or resolve to) a boolean (got ${typeof conditionalValue})`, @@ -452,9 +448,6 @@ export class ConditionalLazyValue extends StructuralTemplateOperator { const branch = isTruthy(conditionalValue) ? this.yaml[conditionalThenKey] : this.yaml[conditionalElseKey] - return { - partial: true, - resolved: branch, - } + return evaluate(branch, args) } } diff --git a/core/src/template/types.ts b/core/src/template/types.ts index e013eece93..500c958530 100644 --- a/core/src/template/types.ts +++ b/core/src/template/types.ts @@ -8,7 +8,7 @@ import type { Primitive } from "utility-types" import { isPrimitive } from "utility-types" -import type { CollectionOrValue } from "../util/objects.js" +import type { Collection, CollectionOrValue } from "../util/objects.js" import type { ConfigContext, ContextResolveOpts } from "../config/template-contexts/base.js" import type { TemplateExpressionGenerator } from "./analysis.js" import { InternalError } from "../exceptions.js" @@ -65,7 +65,7 @@ export type TemplateEvaluationResult = } | { partial: true - resolved: ParsedTemplate + resolved: Collection } const accessDetector = new Proxy( diff --git a/core/src/util/testing.ts b/core/src/util/testing.ts index bd28f5da70..aba6624c96 100644 --- a/core/src/util/testing.ts +++ b/core/src/util/testing.ts @@ -13,7 +13,7 @@ import cloneDeep from "fast-copy" import { isEqual, keyBy, set, mapValues, round } from "lodash-es" import type { GardenOpts, GardenParams, GetConfigGraphParams } from "../garden.js" import { Garden, resolveGardenParams } from "../garden.js" -import type { DeepPrimitiveMap, StringMap } from "../config/common.js" +import type { StringMap } from "../config/common.js" import type { ModuleConfig } from "../config/module.js" import type { WorkflowConfig, WorkflowConfigMap } from "../config/workflow.js" import { resolveMsg, type Log, type LogEntry } from "../logger/log-entry.js" @@ -57,7 +57,7 @@ import got from "got" import { createHash } from "node:crypto" import { pipeline } from "node:stream/promises" import type { GardenCloudApiFactory } from "../cloud/api.js" -import { ConfigContext } from "../config/template-contexts/base.js" +import type { ConfigContext } from "../config/template-contexts/base.js" export class TestError extends GardenError { type = "_test" diff --git a/core/test/unit/src/commands/get/get-config.ts b/core/test/unit/src/commands/get/get-config.ts index 26f30055ca..8c5c36fbfd 100644 --- a/core/test/unit/src/commands/get/get-config.ts +++ b/core/test/unit/src/commands/get/get-config.ts @@ -701,7 +701,7 @@ describe("GetConfigCommand", () => { opts: withDefaultGlobalOpts({ "exclude-disabled": false, "resolve": "partial" }), }) - expect(res.result!.providers).to.eql(garden.getRawProviderConfigs()) + expect(res.result!.providers).to.eql(garden.getUnresolvedProviderConfigs()) }) it("should not resolve providers", async () => { diff --git a/core/test/unit/src/config/provider.ts b/core/test/unit/src/config/provider.ts index 80d009a391..f8334604b5 100644 --- a/core/test/unit/src/config/provider.ts +++ b/core/test/unit/src/config/provider.ts @@ -7,11 +7,13 @@ */ import { expect } from "chai" -import type { GenericProviderConfig } from "../../../../src/config/provider.js" import { getAllProviderDependencyNames } from "../../../../src/config/provider.js" import { expectError } from "../../../helpers.js" import { createGardenPlugin } from "../../../../src/plugin/plugin.js" import { GenericContext } from "../../../../src/config/template-contexts/base.js" +import { UnresolvedProviderConfig } from "../../../../src/config/project.js" +import type { ObjectWithName } from "../../../../src/util/util.js" +import { parseTemplateCollection } from "../../../../src/template/templated-collections.js" describe("getProviderDependencies", () => { const plugin = createGardenPlugin({ @@ -19,11 +21,11 @@ describe("getProviderDependencies", () => { }) it("should extract implicit provider dependencies from template strings", async () => { - const config: GenericProviderConfig = { + const config = makeUnresolvedProvider({ name: "my-provider", someKey: "${providers.other-provider.foo}", anotherKey: "foo-${providers.another-provider.bar}", - } + }) expect(getAllProviderDependencyNames(plugin, config, new GenericContext({}))).to.eql([ "another-provider", "other-provider", @@ -31,19 +33,19 @@ describe("getProviderDependencies", () => { }) it("should ignore template strings that don't reference providers", async () => { - const config: GenericProviderConfig = { + const config = makeUnresolvedProvider({ name: "my-provider", someKey: "${providers.other-provider.foo}", anotherKey: "foo-${some.other.ref}", - } + }) expect(getAllProviderDependencyNames(plugin, config, new GenericContext({}))).to.eql(["other-provider"]) }) it("should throw on provider-scoped template strings without a provider name", async () => { - const config: GenericProviderConfig = { + const config = makeUnresolvedProvider({ name: "my-provider", someKey: "${providers}", - } + }) await expectError(() => getAllProviderDependencyNames(plugin, config, new GenericContext({})), { contains: @@ -51,3 +53,7 @@ describe("getProviderDependencies", () => { }) }) }) + +function makeUnresolvedProvider(o: ObjectWithName) { + return new UnresolvedProviderConfig(o.name, [], parseTemplateCollection({ value: o, source: { path: [] } })) +} diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index f6b1bfb2ad..dff5c5825d 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -42,7 +42,7 @@ import type { ProviderActionName } from "../../../src/plugin/plugin.js" import { createGardenPlugin } from "../../../src/plugin/plugin.js" import type { ConfigureProviderParams } from "../../../src/plugin/handlers/Provider/configureProvider.js" import type { ProjectConfig } from "../../../src/config/project.js" -import { defaultNamespace } from "../../../src/config/project.js" +import { defaultNamespace, UnresolvedProviderConfig } from "../../../src/config/project.js" import type { ModuleConfig } from "../../../src/config/module.js" import { baseModuleSpecSchema } from "../../../src/config/module.js" import { @@ -1162,7 +1162,7 @@ describe("Garden", () => { } const garden = await makeTestGardenA([testPluginDupe]) - garden["providerConfigs"].push({ name: "test-plugin-dupe" }) + garden["providerConfigs"].push(new UnresolvedProviderConfig("test-plugin-dupe", [], { name: "test-plugin-dupe" })) await expectError(() => garden.getAllPlugins(), { contains: "Module type 'test' is declared in multiple plugins: test-plugin, test-plugin-dupe.", diff --git a/core/test/unit/src/plugins.ts b/core/test/unit/src/plugins.ts index aacabced46..6e7fe48535 100644 --- a/core/test/unit/src/plugins.ts +++ b/core/test/unit/src/plugins.ts @@ -15,6 +15,7 @@ import { ACTION_RUNTIME_LOCAL, createGardenPlugin } from "../../../src/plugin/pl import { resolvePlugins } from "../../../src/plugins.js" import { findByName } from "../../../src/util/util.js" import { expectError } from "../../helpers.js" +import { UnresolvedProviderConfig } from "../../../src/config/project.js" describe("resolvePlugins", () => { const log = getRootLogger().createLog() @@ -60,9 +61,12 @@ describe("resolvePlugins", () => { }, ] - await expectError(async () => resolvePlugins(log, { test: plugin }, [{ name: "test" }]), { - contains: "has overlapping keys in staticoutputsschema and runtimeoutputsschema", - }) + await expectError( + async () => resolvePlugins(log, { test: plugin }, [new UnresolvedProviderConfig("test", [], { name: "test" })]), + { + contains: "has overlapping keys in staticoutputsschema and runtimeoutputsschema", + } + ) }) it("throws if action type staticOutputsSchema allows unknown keys", async () => { @@ -82,9 +86,12 @@ describe("resolvePlugins", () => { }, ] - await expectError(async () => resolvePlugins(log, { test: plugin }, [{ name: "test" }]), { - contains: "allows unknown keys in the staticoutputsschema", - }) + await expectError( + async () => resolvePlugins(log, { test: plugin }, [new UnresolvedProviderConfig("base", [], { name: "test" })]), + { + contains: "allows unknown keys in the staticoutputsschema", + } + ) }) it("inherits created action type from base plugin", async () => { @@ -110,7 +117,9 @@ describe("resolvePlugins", () => { const dependant = createGardenPlugin({ name: "dependant", base: "base" }) - const result = resolvePlugins(log, { base, dependant }, [{ name: "test" }]) + const result = resolvePlugins(log, { base, dependant }, [ + new UnresolvedProviderConfig("test", [], { name: "test" }), + ]) const inheritedActionType = result.find((plugin) => plugin.name === "dependant")?.createActionTypes.Build[0] expect(inheritedActionType).to.exist expect(inheritedActionType?.name).to.eql("base") @@ -157,9 +166,13 @@ describe("resolvePlugins", () => { }, ] - await expectError(async () => resolvePlugins(log, { base, dependant }, [{ name: "test" }]), { - contains: "plugin 'dependant' redeclares the 'base' build type, already declared by its base", - }) + await expectError( + async () => + resolvePlugins(log, { base, dependant }, [new UnresolvedProviderConfig("test", [], { name: "test" })]), + { + contains: "plugin 'dependant' redeclares the 'base' build type, already declared by its base", + } + ) }) it("inherits action type extension from base plugin", async () => { @@ -186,7 +199,9 @@ describe("resolvePlugins", () => { const dependant = createGardenPlugin({ name: "dependant", base: "base2" }) - const plugins = resolvePlugins(log, { base1, base2, dependant }, [{ name: "dependant" }]) + const plugins = resolvePlugins(log, { base1, base2, dependant }, [ + new UnresolvedProviderConfig("dependant", [], { name: "dependant" }), + ]) const resolved = findByName(plugins, "dependant") expect(resolved).to.exist @@ -228,7 +243,10 @@ describe("resolvePlugins", () => { }, }) - const plugins = resolvePlugins(log, { base, extension }, [{ name: "base" }, { name: "extension" }]) + const plugins = resolvePlugins(log, { base, extension }, [ + new UnresolvedProviderConfig("base", [], { name: "base" }), + new UnresolvedProviderConfig("extension", [], { name: "extension" }), + ]) const resolved = findByName(plugins, "extension") expect(resolved).to.exist @@ -274,9 +292,9 @@ describe("resolvePlugins", () => { }) const plugins = resolvePlugins(log, { base1, base2, extension }, [ - { name: "base1" }, - { name: "base2" }, - { name: "extension" }, + new UnresolvedProviderConfig("base1", [], { name: "base1" }), + new UnresolvedProviderConfig("base2", [], { name: "base2" }), + new UnresolvedProviderConfig("extension", [], { name: "extension" }), ]) const resolved = findByName(plugins, "extension") @@ -323,7 +341,10 @@ describe("resolvePlugins", () => { }, }) - const plugins = resolvePlugins(log, { base, inheriting }, [{ name: "base" }, { name: "inheriting" }]) + const plugins = resolvePlugins(log, { base, inheriting }, [ + new UnresolvedProviderConfig("base", [], { name: "base" }), + new UnresolvedProviderConfig("inheriting", [], { name: "inheriting" }), + ]) const resolved = findByName(plugins, "inheriting") expect(resolved).to.exist @@ -383,9 +404,9 @@ describe("resolvePlugins", () => { }) const plugins = resolvePlugins(log, { base1, base2, inheriting }, [ - { name: "base1" }, - { name: "base2" }, - { name: "inheriting" }, + new UnresolvedProviderConfig("base1", [], { name: "base1" }), + new UnresolvedProviderConfig("base2", [], { name: "base2" }), + new UnresolvedProviderConfig("inheriting", [], { name: "inheriting" }), ]) const resolved = findByName(plugins, "inheriting") @@ -412,7 +433,9 @@ describe("resolvePlugins", () => { dependencies: [{ name: "base" }], }) - const plugins = resolvePlugins(log, { base, extension }, [{ name: "extension" }]) + const plugins = resolvePlugins(log, { base, extension }, [ + new UnresolvedProviderConfig("extension", [], { name: "extension" }), + ]) const resolved = findByName(plugins, "extension") expect(resolved).to.exist @@ -458,7 +481,9 @@ describe("resolvePlugins", () => { dependencies: [{ name: "base2" }], }) - const plugins = resolvePlugins(log, { base1, base2, extension }, [{ name: "extension" }]) + const plugins = resolvePlugins(log, { base1, base2, extension }, [ + new UnresolvedProviderConfig("extension", [], { name: "extension" }), + ]) const resolved = findByName(plugins, "extension") expect(resolved).to.exist @@ -522,7 +547,9 @@ describe("resolvePlugins", () => { }, }) - const plugins = resolvePlugins(log, { base1, base2, extension }, [{ name: "extension" }]) + const plugins = resolvePlugins(log, { base1, base2, extension }, [ + new UnresolvedProviderConfig("extension", [], { name: "extension" }), + ]) const resolved = findByName(plugins, "extension") expect(resolved).to.exist diff --git a/core/test/unit/src/plugins/kubernetes/kubernetes.ts b/core/test/unit/src/plugins/kubernetes/kubernetes.ts index 0673bf8a8f..a57f26c4fa 100644 --- a/core/test/unit/src/plugins/kubernetes/kubernetes.ts +++ b/core/test/unit/src/plugins/kubernetes/kubernetes.ts @@ -19,6 +19,8 @@ import { defaultSystemNamespace, defaultUtilImageRegistryDomain, } from "../../../../../src/plugins/kubernetes/constants.js" +import { UnresolvedProviderConfig } from "../../../../../src/config/project.js" +import type { ParsedTemplate } from "../../../../../src/template/types.js" describe("kubernetes configureProvider", () => { const basicConfig: KubernetesConfig = { @@ -62,7 +64,11 @@ describe("kubernetes configureProvider", () => { ctx: await garden.getPluginContext({ provider: providerFromConfig({ plugin: gardenPlugin(), - config, + config: new UnresolvedProviderConfig( + config.name, + config.dependencies || [], + config as unknown as ParsedTemplate + ), dependencies: {}, moduleConfigs: [], status: { ready: false, outputs: {} }, diff --git a/core/test/unit/src/tasks/resolve-provider.ts b/core/test/unit/src/tasks/resolve-provider.ts index 2488d5056c..c4e00a3b5e 100644 --- a/core/test/unit/src/tasks/resolve-provider.ts +++ b/core/test/unit/src/tasks/resolve-provider.ts @@ -48,7 +48,7 @@ describe("ResolveProviderTask", () => { }) const plugin = await garden.getPlugin("test-plugin") - const config = garden.getRawProviderConfigs({ names: ["test-plugin"] })[0] + const config = garden.getUnresolvedProviderConfigs({ names: ["test-plugin"] })[0] task = new ResolveProviderTask({ garden, diff --git a/plugins/conftest/src/index.ts b/plugins/conftest/src/index.ts index 314f2115c4..25f82cac52 100644 --- a/plugins/conftest/src/index.ts +++ b/plugins/conftest/src/index.ts @@ -16,7 +16,7 @@ import { dedent, naturalList } from "@garden-io/sdk/build/src/util/string.js" import { matchGlobs, listDirectory } from "@garden-io/sdk/build/src/util/fs.js" // TODO: gradually get rid of these core dependencies, move some to SDK etc. -import type { GenericProviderConfig, Provider } from "@garden-io/core/build/src/config/provider.js" +import type { BaseProviderConfig, Provider } from "@garden-io/core/build/src/config/provider.js" import { providerConfigBaseSchema } from "@garden-io/core/build/src/config/provider.js" import { joi, joiIdentifier, joiSparseArray } from "@garden-io/core/build/src/config/common.js" import { baseBuildSpecSchema } from "@garden-io/core/build/src/config/module.js" @@ -32,7 +32,7 @@ import { actionRefMatches } from "@garden-io/core/build/src/actions/base.js" import type { Resolved } from "@garden-io/core/build/src/actions/types.js" import { DEFAULT_TEST_TIMEOUT_SEC } from "@garden-io/core/build/src/constants.js" -export interface ConftestProviderConfig extends GenericProviderConfig { +export interface ConftestProviderConfig extends BaseProviderConfig { policyPath: string namespace?: string testFailureThreshold: "deny" | "warn" | "none" diff --git a/plugins/jib/src/index.ts b/plugins/jib/src/index.ts index 576927213c..1f33d4683c 100644 --- a/plugins/jib/src/index.ts +++ b/plugins/jib/src/index.ts @@ -16,7 +16,7 @@ import { mavendSpec, mvnd, mvndVersion } from "./mavend.js" import { gradle, gradleSpec, gradleVersion } from "./gradle.js" // TODO: gradually get rid of these core dependencies, move some to SDK etc. -import type { GenericProviderConfig, Provider } from "@garden-io/core/build/src/config/provider.js" +import type { BaseProviderConfig, Provider } from "@garden-io/core/build/src/config/provider.js" import { providerConfigBaseSchema } from "@garden-io/core/build/src/config/provider.js" import { getGitHubUrl } from "@garden-io/core/build/src/docs/common.js" import { @@ -38,7 +38,7 @@ import type { } from "@garden-io/core/build/src/plugin/handlers/Module/convert.js" import type { PluginEventLogContext } from "@garden-io/core/build/src/plugin-context.js" -export type JibProviderConfig = GenericProviderConfig +export type JibProviderConfig = BaseProviderConfig export type JibProvider = Provider diff --git a/plugins/pulumi/src/provider.ts b/plugins/pulumi/src/provider.ts index 3a29a41d15..5298da3c14 100644 --- a/plugins/pulumi/src/provider.ts +++ b/plugins/pulumi/src/provider.ts @@ -7,17 +7,18 @@ */ import { joi } from "@garden-io/core/build/src/config/common.js" -import type { GenericProviderConfig, Provider } from "@garden-io/core/build/src/config/provider.js" +import type { BaseProviderConfig, Provider } from "@garden-io/core/build/src/config/provider.js" import { providerConfigBaseSchema } from "@garden-io/core/build/src/config/provider.js" import { dedent } from "@garden-io/sdk/build/src/util/string.js" import { defaultPulumiVersion, supportedVersions } from "./cli.js" -export type PulumiProviderConfig = GenericProviderConfig & { +export type PulumiProviderConfig = BaseProviderConfig & { version: string | null previewDir: string | null orgName?: string backendURL: string pluginTaskConcurrencyLimit: number + useNewPulumiVarfileSchema?: boolean } export type PulumiProvider = Provider diff --git a/plugins/terraform/src/index.ts b/plugins/terraform/src/index.ts index 58ce933b4c..12da4833e1 100644 --- a/plugins/terraform/src/index.ts +++ b/plugins/terraform/src/index.ts @@ -90,7 +90,7 @@ export const gardenPlugin = () => handlers: { configure: async (params) => { const ctx = params.ctx as PluginContext - const config = params.config as TerraformProviderConfig + const config = params.config as TerraformDeployConfig const provider = ctx.provider as TerraformProvider // Use the provider config if no value is specified for the module diff --git a/plugins/terraform/src/provider.ts b/plugins/terraform/src/provider.ts index ea2553b8a2..bf0865666d 100644 --- a/plugins/terraform/src/provider.ts +++ b/plugins/terraform/src/provider.ts @@ -12,11 +12,11 @@ import type { TerraformBaseSpec } from "./helpers.js" import { variablesSchema } from "./helpers.js" import { docsBaseUrl } from "@garden-io/sdk/build/src/constants.js" -import type { GenericProviderConfig, Provider } from "@garden-io/core/build/src/config/provider.js" +import type { BaseProviderConfig, Provider } from "@garden-io/core/build/src/config/provider.js" import { providerConfigBaseSchema } from "@garden-io/core/build/src/config/provider.js" import { joi } from "@garden-io/core/build/src/config/common.js" -export type TerraformProviderConfig = GenericProviderConfig & +export type TerraformProviderConfig = BaseProviderConfig & TerraformBaseSpec & { initRoot?: string } From 0ab63538cc89070dddb6a19880d42ac01d41f784 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Thu, 16 Jan 2025 11:51:07 +0100 Subject: [PATCH 021/117] chore(template): primitives for partial resolution needed for module resolution --- core/src/template/analysis.ts | 2 +- core/src/template/ast.ts | 4 +- core/src/template/capture.ts | 10 +++- core/src/template/evaluate.ts | 12 ++++- core/src/template/templated-collections.ts | 55 +++++++++++++++++++--- core/src/template/types.ts | 13 ++++- 6 files changed, 82 insertions(+), 14 deletions(-) diff --git a/core/src/template/analysis.ts b/core/src/template/analysis.ts index ab33660d10..03f97a8fa3 100644 --- a/core/src/template/analysis.ts +++ b/core/src/template/analysis.ts @@ -39,7 +39,7 @@ export function* visitAll({ value }: { value: ParsedTemplate }): TemplateExpress }) } } else if (value instanceof UnresolvedTemplateValue) { - yield* value.visitAll() + yield* value.visitAll({}) } } diff --git a/core/src/template/ast.ts b/core/src/template/ast.ts index 01521fd5a0..e8373cf608 100644 --- a/core/src/template/ast.ts +++ b/core/src/template/ast.ts @@ -225,13 +225,13 @@ export function isNotFound( return v === CONTEXT_RESOLVE_KEY_NOT_FOUND } -export function isTruthy(v: CollectionOrValue): boolean { +export function isTruthy(v: TemplatePrimitive | Collection): boolean { if (isTemplatePrimitive(v)) { return !!v } // collections are truthy, regardless wether they are empty or not. - v satisfies Collection + v satisfies Collection return true } diff --git a/core/src/template/capture.ts b/core/src/template/capture.ts index 38d4ebc310..31a2e31fda 100644 --- a/core/src/template/capture.ts +++ b/core/src/template/capture.ts @@ -64,7 +64,13 @@ export class CapturedContextTemplateValue extends UnresolvedTemplateValue { }) } - override *visitAll(): TemplateExpressionGenerator { - yield* visitAll({ value: this.wrapped }) + override *visitAll({ onlyEssential = false }): TemplateExpressionGenerator { + if (this.wrapped instanceof UnresolvedTemplateValue) { + this.wrapped.visitAll({ onlyEssential }) + } else if (!onlyEssential) { + // wrapped is either a primitive or a collection. + // Thus, we only visit all if onlyEssential is false. + yield* visitAll({ value: this.wrapped }) + } } } diff --git a/core/src/template/evaluate.ts b/core/src/template/evaluate.ts index c156bf92ae..9f5646ad6c 100644 --- a/core/src/template/evaluate.ts +++ b/core/src/template/evaluate.ts @@ -26,8 +26,16 @@ export function deepEvaluate( collection: Input, args: EvaluateTemplateArgs ): Evaluate { + return conditionallyDeepEvaluate(collection, args, () => true) as Evaluate +} + +export function conditionallyDeepEvaluate( + collection: ParsedTemplate, + args: EvaluateTemplateArgs, + condition: (v: UnresolvedTemplateValue) => boolean +): ParsedTemplate { return deepMap(collection, (v) => { - if (v instanceof UnresolvedTemplateValue) { + if (v instanceof UnresolvedTemplateValue && condition(v)) { const evaluated = evaluate(v, args) if (evaluated.partial) { return deepEvaluate(evaluated.resolved, args) @@ -35,7 +43,7 @@ export function deepEvaluate( return evaluated.resolved } return v - }) as Evaluate + }) } export function evaluate(value: ParsedTemplate, args: EvaluateTemplateArgs): TemplateEvaluationResult { diff --git a/core/src/template/templated-collections.ts b/core/src/template/templated-collections.ts index de639bb288..2eab3b42fb 100644 --- a/core/src/template/templated-collections.ts +++ b/core/src/template/templated-collections.ts @@ -27,7 +27,7 @@ import { objectSpreadKey, } from "../config/constants.js" import mapValues from "lodash-es/mapValues.js" -import { deepEvaluate, evaluate } from "./evaluate.js" +import { evaluate } from "./evaluate.js" import { LayeredContext } from "../config/template-contexts/base.js" import { parseTemplateString } from "./templated-strings.js" import { TemplateError } from "./errors.js" @@ -222,10 +222,6 @@ abstract class StructuralTemplateOperator extends UnresolvedTemplateValue { super() } - override *visitAll(): TemplateExpressionGenerator { - yield* visitAll({ value: this.template }) - } - override toJSON(): ResolvedTemplate { return deepMap(this.template, (v) => { if (!(v instanceof UnresolvedTemplateValue)) { @@ -246,6 +242,14 @@ export class ConcatLazyValue extends StructuralTemplateOperator { super(source, yaml) } + override *visitAll({ onlyEssential = false }): TemplateExpressionGenerator { + if (!onlyEssential) { + yield* visitAll({ value: this.yaml }) + } else if (this.yaml[arrayConcatKey] instanceof UnresolvedTemplateValue) { + yield* this.yaml[arrayConcatKey].visitAll({ onlyEssential }) + } + } + override evaluate(args: EvaluateTemplateArgs): { partial: true resolved: ParsedTemplate[] @@ -323,6 +327,19 @@ export class ForEachLazyValue extends StructuralTemplateOperator { super(source, yaml) } + public override *visitAll({ onlyEssential = false }): TemplateExpressionGenerator { + if (!onlyEssential) { + return yield* visitAll({ value: this.yaml }) + } + + // let's assume that the array must be fully resolved for now + yield* visitAll({ value: this.yaml[arrayForEachKey] }) + + if (this.yaml[arrayForEachFilterKey] instanceof UnresolvedTemplateValue) { + yield* this.yaml[arrayForEachFilterKey].visitAll({ onlyEssential }) + } + } + override evaluate(args: EvaluateTemplateArgs): { partial: true resolved: ParsedTemplate[] @@ -387,6 +404,14 @@ export class ObjectSpreadLazyValue extends StructuralTemplateOperator { super(source, yaml) } + public override *visitAll({ onlyEssential = false }): TemplateExpressionGenerator { + if (!onlyEssential) { + yield* visitAll({ value: this.yaml }) + } else if (this.yaml[objectSpreadKey] instanceof UnresolvedTemplateValue) { + yield* this.yaml[objectSpreadKey].visitAll({ onlyEssential }) + } + } + override evaluate(args: EvaluateTemplateArgs): { partial: true resolved: Record @@ -436,8 +461,26 @@ export class ConditionalLazyValue extends StructuralTemplateOperator { super(source, yaml) } + public override *visitAll({ onlyEssential = false }): TemplateExpressionGenerator { + if (!onlyEssential) { + return yield* visitAll({ value: this.yaml }) + } + + if (this.yaml[conditionalKey] instanceof UnresolvedTemplateValue) { + yield* this.yaml[conditionalKey].visitAll({ onlyEssential }) + } + + if (this.yaml[conditionalThenKey] instanceof UnresolvedTemplateValue) { + yield* this.yaml[conditionalThenKey].visitAll({ onlyEssential }) + } + + if (this.yaml[conditionalElseKey] instanceof UnresolvedTemplateValue) { + yield* this.yaml[conditionalElseKey].visitAll({ onlyEssential }) + } + } + override evaluate(args: EvaluateTemplateArgs): TemplateEvaluationResult { - const conditionalValue = deepEvaluate(this.yaml[conditionalKey], args) + const conditionalValue = evaluate(this.yaml[conditionalKey], args) if (typeof conditionalValue !== "boolean") { throw new TemplateError({ diff --git a/core/src/template/types.ts b/core/src/template/types.ts index 500c958530..47773d3ed7 100644 --- a/core/src/template/types.ts +++ b/core/src/template/types.ts @@ -105,7 +105,18 @@ export abstract class UnresolvedTemplateValue { public abstract evaluate(args: EvaluateTemplateArgs): TemplateEvaluationResult public abstract toJSON(): CollectionOrValue - public abstract visitAll(): TemplateExpressionGenerator + public abstract visitAll(opts: { + /** + * If true, the returned template expression generator will only yield template expressions that + * will be evaluated when calling `evaluate`. + * + * If `evaluate` returns `partial: true`, and `onlyEssential` is set to true, then the unresolved + * expressions returned by evaluate will not be emitted by the returned generator. + * + * @default false + */ + onlyEssential?: boolean + }): TemplateExpressionGenerator } // NOTE: this will make sure we throw an error if this value is accidentally treated as resolved. From 54b1bdf5be98ec610874f5aeff2b91fd79977a39 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Thu, 16 Jan 2025 16:25:43 +0100 Subject: [PATCH 022/117] chore: first step making module resolver work --- core/src/config/common.ts | 17 ++ core/src/config/template-contexts/base.ts | 64 ++++--- core/src/garden.ts | 8 +- core/src/graph/actions.ts | 28 ++- core/src/plugin/handlers/base/configure.ts | 3 +- core/src/resolve-module.ts | 192 +++++++++++---------- core/src/template/analysis.ts | 32 ++++ core/src/template/ast.ts | 45 ++--- 8 files changed, 230 insertions(+), 159 deletions(-) diff --git a/core/src/config/common.ts b/core/src/config/common.ts index 3dade31485..182444dffc 100644 --- a/core/src/config/common.ts +++ b/core/src/config/common.ts @@ -31,6 +31,11 @@ import { userIdentifierRegex, variableNameRegex, envVarRegex, + arrayForEachReturnKey, + arrayForEachFilterKey, + arrayForEachKey, + arrayConcatKey, + objectSpreadKey, } from "./constants.js" import { renderZodError } from "./zod.js" import { makeDocsLinkPlain } from "../docs/common.js" @@ -387,6 +392,18 @@ joi = joi.extend({ // validate(value: string, { error }) { // return { value } // }, + args(schema: any, keys: any) { + // Always allow the special $merge, $forEach etc. keys, which we resolve and collapse in resolveTemplateStrings() + // Note: we allow both the expected schema and strings, since they may be templates resolving to the expected type. + return schema.keys({ + [objectSpreadKey]: joi.alternatives(joi.object(), joi.string()), + [arrayConcatKey]: joi.alternatives(joi.array(), joi.string()), + [arrayForEachKey]: joi.alternatives(joi.array(), joi.string()), + [arrayForEachFilterKey]: joi.any(), + [arrayForEachReturnKey]: joi.any(), + ...(keys || {}), + }) + }, rules: { jsonSchema: { method(jsonSchema: object) { diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 1b876656c4..38972bd7c9 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -45,10 +45,15 @@ export interface ContextResolveParams { rootContext?: ConfigContext } -export interface ContextResolveOutput { - getUnavailableReason?: () => string - resolved: any -} +export type ContextResolveOutput = + | { + resolved: ResolvedTemplate + getUnavailableReason?: undefined + } + | { + resolved: typeof CONTEXT_RESOLVE_KEY_NOT_FOUND + getUnavailableReason: () => string + } export function schema(joiSchema: Joi.Schema) { return (target: any, propName: string) => { @@ -149,7 +154,7 @@ export abstract class ConfigContext { let nestedNodePath = nodePath let getUnavailableReason: (() => string) | undefined = undefined - if (key.length === 0) { + if (key.length === 0 && !(value instanceof UnresolvedTemplateValue)) { value = pick( value, Object.keys(value as Collection).filter((k) => !k.startsWith("_")) @@ -166,23 +171,6 @@ export abstract class ConfigContext { const getStackEntry = () => renderKeyPath(capturedNestedNodePath) getAvailableKeys = undefined - const parent: CollectionOrValue = value - if (isTemplatePrimitive(parent)) { - throw new ContextResolveError({ - message: `Attempted to look up key ${JSON.stringify(nextKey)} on a ${typeof parent}.`, - }) - } else if (typeof nextKey === "string" && nextKey.startsWith("_")) { - value = undefined - } else if (parent instanceof Map) { - getAvailableKeys = () => Array.from(parent.keys()) - value = parent.get(nextKey) - } else { - getAvailableKeys = () => { - return Object.keys(parent).filter((k) => !k.startsWith("_")) - } - value = parent[nextKey] - } - // handle nested contexts if (value instanceof ConfigContext) { const remainder = getRemainder() @@ -191,6 +179,11 @@ export abstract class ConfigContext { opts.contextStack.add(value) // NOTE: we resolve even if remainder.length is zero to make sure all unresolved template values have been resolved. const res = value.resolve({ key: remainder, nodePath: nestedNodePath, opts, rootContext }) + if (res.resolved === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + throw new InternalError({ + message: "unhandled resolve key not found", + }) + } value = res.resolved getUnavailableReason = res.getUnavailableReason break @@ -204,6 +197,23 @@ export abstract class ConfigContext { value = resolved } + const parent: CollectionOrValue = value + if (isTemplatePrimitive(parent)) { + throw new ContextResolveError({ + message: `Attempted to look up key ${JSON.stringify(nextKey)} on a ${typeof parent}.`, + }) + } else if (typeof nextKey === "string" && nextKey.startsWith("_")) { + value = undefined + } else if (parent instanceof Map) { + getAvailableKeys = () => Array.from(parent.keys()) + value = parent.get(nextKey) + } else { + getAvailableKeys = () => { + return Object.keys(parent).filter((k) => !k.startsWith("_")) + } + value = parent[nextKey] + } + if (value === undefined) { break } @@ -243,7 +253,13 @@ export abstract class ConfigContext { if (!isTemplatePrimitive(value)) { value = deepMap(value, (v, keyPath) => { if (v instanceof ConfigContext) { - return v.resolve({ key: [], nodePath: nodePath.concat(key, keyPath), opts }).resolved + const { resolved } = v.resolve({ key: [], nodePath: nodePath.concat(key, keyPath), opts }) + if (resolved === CONTEXT_RESOLVE_KEY_NOT_FOUND) { + throw new InternalError({ + message: "Unhandled context resolve key not found", + }) + } + return resolved } return deepEvaluate(v, { context: getRootContext(), opts }) }) @@ -252,7 +268,7 @@ export abstract class ConfigContext { // Cache result this._resolvedValues[path] = value - return { resolved: value } + return { resolved: value as ResolvedTemplate } } } diff --git a/core/src/garden.ts b/core/src/garden.ts index e5d0448497..e615a7d903 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -175,7 +175,7 @@ import { } from "./cloud/util.js" import { throwOnMissingSecretKeys } from "./config/secrets.js" import { deepEvaluate } from "./template/evaluate.js" -import type { ParsedTemplate } from "./template/types.js" +import { isTemplatePrimitive, UnresolvedTemplateValue, type ParsedTemplate } from "./template/types.js" import { LayeredContext } from "./config/template-contexts/base.js" const defaultLocalAddress = "localhost" @@ -1838,8 +1838,10 @@ export class Garden { allProviderNames: this.getUnresolvedProviderConfigs().map((p) => p.name), allEnvironmentNames, namespace: this.namespace, - providers: providers.map((p) => (typeof p === "object" && p !== null ? omitInternal(p) : p)), - variables: this.variables.resolve({ key: [], nodePath: [], opts: {} }).resolved, + providers: providers.map((p) => + !(p instanceof UnresolvedTemplateValue || isTemplatePrimitive(p)) ? omitInternal(p) : p + ), + variables: this.variables.resolve({ key: [], nodePath: [], opts: {} }).resolved as any, actionConfigs: filteredActionConfigs, moduleConfigs: moduleConfigs.map(omitInternal), workflowConfigs: sortBy(workflowConfigs.map(omitInternal), "name"), diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index 7e49426319..21f2abbec9 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -71,6 +71,7 @@ import { CapturedContext } from "../config/template-contexts/base.js" import { deepEvaluate } from "../template/evaluate.js" import type { ParsedTemplate } from "../template/types.js" import type { DeepPrimitiveMap } from "@garden-io/platform-api-types" +import { validateWithPath } from "../config/validation.js" function* sliceToBatches(dict: Record, batchSize: number) { const entries = Object.entries(dict) @@ -510,7 +511,9 @@ export const processActionConfig = profileAsync(async function processActionConf let variables = await mergeVariables({ basePath: effectiveConfigFileLocation, - variables: new GenericContext(config.variables || {}), + variables: new GenericContext(capture(config.variables, + // TODO: What's the correct context here? + garden.getProjectConfigContext()) || {}), varfiles: config.varfiles, log, }) @@ -814,18 +817,25 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi ) const { spec = {} } = config - // TODO-0.13.1: better error messages when something goes wrong here config = { ...config, - ...resolvedBuiltin, + // Validate fully resolved keys (the above + those that don't allow any templating) + ...validateWithPath({ + config: { + ...resolvedBuiltin, + variables: {}, + spec: {}, + }, + schema: getActionSchema(config.kind), + configType: describeActionConfig(config), + name: config.name, + path: config.internal.basePath, + projectRoot: garden.projectRoot, + source: { yamlDoc: config.internal.yamlDoc, path: [] }, + }), spec, + variables: config.variables, } - - // Apparently no need to capture, as later contexts will be a superset - // TODO: verify - // for (const k in omit(config, builtinConfigKeys.concat("internal")) as Record) { - // config[k] = capture(config[k], builtinFieldContext) - // } } resolveTemplates() diff --git a/core/src/plugin/handlers/base/configure.ts b/core/src/plugin/handlers/base/configure.ts index 8bc2b832b5..ee450593a7 100644 --- a/core/src/plugin/handlers/base/configure.ts +++ b/core/src/plugin/handlers/base/configure.ts @@ -16,6 +16,7 @@ import { baseActionConfigSchema } from "../../../actions/base.js" import { ActionTypeHandlerSpec } from "./base.js" import { pluginContextSchema } from "../../../plugin-context.js" import { noTemplateFields } from "../../../config/base.js" +import { actionConfigSchema } from "../../../actions/helpers.js" interface ConfigureActionConfigParams extends PluginActionContextParams { log: Log @@ -57,7 +58,7 @@ export class ConfigureActionConfig joi.object().keys({ - config: joi.any().required(), + config: actionConfigSchema().required(), supportedModes: joi .object() .keys({ diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index e2201726bd..d0716cb387 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -6,7 +6,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { isArray, isString, keyBy, keys, partition, pick, union, uniq } from "lodash-es" +import { isArray, isString, keyBy, partition, uniq } from "lodash-es" import { validateWithPath } from "./config/validation.js" import { mayContainTemplateString, resolveTemplateString } from "./template/templated-strings.js" import { GenericContext } from "./config/template-contexts/base.js" @@ -59,8 +59,11 @@ import { minimatch } from "minimatch" import { getModuleTemplateReferences } from "./config/references.js" import { capture } from "./template/capture.js" import { LayeredContext } from "./config/template-contexts/base.js" -import type { ParsedTemplate } from "./template/types.js" -import { deepEvaluate } from "./template/evaluate.js" +import { UnresolvedTemplateValue, type ParsedTemplate } from "./template/types.js" +import { conditionallyDeepEvaluate, deepEvaluate } from "./template/evaluate.js" +import { someReferences } from "./template/analysis.js" +import { ForEachLazyValue } from "./template/templated-collections.js" +import { deepMap } from "./util/objects.js" // This limit is fairly arbitrary, but we need to have some cap on concurrent processing. export const moduleResolutionConcurrencyLimit = 50 @@ -581,83 +584,51 @@ export class ModuleResolver { /** * Resolves and validates a single module configuration. */ - async resolveModuleConfig(config: ModuleConfig, dependencies: GardenModule[]): Promise { + async resolveModuleConfig(unresolvedConfig: ModuleConfig, dependencies: GardenModule[]): Promise { const garden = this.garden - let inputs = config.inputs || {} - const buildPath = this.garden.buildStaging.getBuildPath(config) + const buildPath = this.garden.buildStaging.getBuildPath(unresolvedConfig) + + const inputs = unresolvedConfig.inputs || {} const templateContextParams: ModuleConfigContextParams = { garden, variables: garden.variables, resolvedProviders: this.resolvedProviders, modules: dependencies, - name: config.name, - path: config.path, + name: unresolvedConfig.name, + path: unresolvedConfig.path, buildPath, - parentName: config.parentName, - templateName: config.templateName, + parentName: unresolvedConfig.parentName, + templateName: unresolvedConfig.templateName, inputs, graphResults: this.graphResults, } - // Resolve and validate the inputs field, because template module inputs may not be fully resolved at this - // time. - // TODO: This whole complicated procedure could be much improved and simplified by implementing lazy resolution on - // values... I'll be looking into that. - JE - // NOTE(steffen): On it! :) - const templateName = config.templateName - - if (templateName) { - // const template = this.garden.configTemplates[templateName] - - inputs = this.resolveInputs(config, new ModuleConfigContext(templateContextParams)) as unknown as DeepPrimitiveMap - - // inputs = validateWithPath({ - // config: inputs, - // configType: `inputs for module ${config.name}`, - // path: config.configPath || config.path, - // schema: template.inputsSchema, - // projectRoot: garden.projectRoot, - // source: undefined, - // }) - - config.inputs = inputs - } + const contextWithoutVariables = new ModuleConfigContext(templateContextParams) // Resolve the variables field before resolving everything else (overriding with module varfiles if present) - const resolvedModuleVariables = await this.resolveVariables(config, templateContextParams) - - // Now resolve just references to inputs on the config - config = capture(config as unknown as ParsedTemplate, new GenericContext({ inputs })) as unknown as typeof config + const moduleVariables = await this.mergeVariables(unresolvedConfig, contextWithoutVariables) // And finally fully resolve the config. // Template strings in the spec can have references to inputs, // so we also need to pass inputs here along with the available variables. - const configContext = new ModuleConfigContext({ + const context = new ModuleConfigContext({ ...templateContextParams, - variables: new LayeredContext(garden.variables, resolvedModuleVariables), - inputs: { ...inputs }, + variables: new LayeredContext(garden.variables, moduleVariables), + inputs, }) - config = capture( - { ...config, inputs: {}, variables: {} } as unknown as ParsedTemplate, - configContext - ) as unknown as typeof config - - // config.variables = resolvedModuleVariables.resolve({ key: [], nodePath: [], opts: {} }).resolved - config.inputs = inputs - const moduleTypeDefinitions = await garden.getModuleTypes() - const description = moduleTypeDefinitions[config.type] + const description = moduleTypeDefinitions[unresolvedConfig.type] if (!description) { - const configPath = relative(garden.projectRoot, config.configPath || config.path) + const configPath = relative(garden.projectRoot, unresolvedConfig.configPath || unresolvedConfig.path) throw new ConfigurationError({ message: dedent` Unrecognized module type '${ - config.type + unresolvedConfig.type }' (defined at ${configPath}). Are you missing a provider configuration? Currently available module types: ${Object.keys(moduleTypeDefinitions)} @@ -665,6 +636,11 @@ export class ModuleResolver { }) } + let config: ModuleConfig = partiallyEvaluateModule( + unresolvedConfig as unknown as ParsedTemplate, + context + ) as unknown as ModuleConfig + // We allow specifying modules by name only as a shorthand: // // dependencies: @@ -683,15 +659,18 @@ export class ModuleResolver { // Validate the module-type specific spec if (description.schema) { - config.spec = validateWithPath({ - config: config.spec, - configType: "Module", - schema: description.schema, - name: config.name, - path: config.path, - projectRoot: garden.projectRoot, - source: undefined, - }) + config = { + ...config, + spec: validateWithPath({ + config: config.spec, + configType: "Module", + schema: description.schema, + name: config.name, + path: config.path, + projectRoot: garden.projectRoot, + source: undefined, + }), + } } // Validate the base config schema @@ -904,28 +883,16 @@ export class ModuleResolver { } /** - * Resolves module variables with the following precedence order: + * Merges module variables with the following precedence order: * * garden.variableOverrides > module varfile > config.variables */ - private async resolveVariables( - config: ModuleConfig, - templateContextParams: ModuleConfigContextParams - ): Promise { - const moduleConfigContext = new ModuleConfigContext(templateContextParams) - const resolveOpts = { - // Modules will be converted to actions later, and the actions will be properly unescaped. - // We avoid premature un-escaping here, - // because otherwise it will strip the escaped value in the module config - // to the normal template string in the converted action config. - unescape: false, - } - + private async mergeVariables(config: ModuleConfig, context: ModuleConfigContext): Promise { let varfileVars: DeepPrimitiveMap = {} if (config.varfile) { const varfilePath = deepEvaluate(config.varfile, { - context: moduleConfigContext, - opts: resolveOpts, + context, + opts: {}, }) if (typeof varfilePath !== "string") { throw new ConfigurationError({ @@ -939,21 +906,12 @@ export class ModuleResolver { }) } - const rawVariables = config.variables - const moduleVariables = deepEvaluate(rawVariables || {}, { - context: moduleConfigContext, - opts: resolveOpts, - }) + const moduleVariables = capture(config.variables || {}, context) - // only override the relevant variables - const relevantVariableOverrides = pick( - this.garden.variableOverrides, - union(keys(moduleVariables), keys(varfileVars)) - ) return new LayeredContext( new GenericContext(moduleVariables), new GenericContext(varfileVars), - new GenericContext(relevantVariableOverrides) + new GenericContext(this.garden.variableOverrides) ) } } @@ -1277,3 +1235,63 @@ function getTestNames(config: ModuleConfig) { } return names } + +/** + * The module resolution flow is special: + * + * 1. We will fully resolve all values that do not contain runtime references (with `unescape: false`). + * 2. We'll call `toJSON` on all `UnresolvedTemplateValue` instances. This will convert template expressions to raw template strings. + * 3. Then we'll apply the module schemas and call configure handlers with the partially resolved config. We know that unresolved values can cause validation errors here. + * 4. Once the config has been converted to actions, we call `parseTemplateCollection` again. + * + * This has a number of horrible consequences, e.g. we parse template strings in environment variables and you can't use runtime outputs on template values where the schema expects a number. + * + * If there was a way to avoid all that, that would definitely be preferred. + * + * Let's hope that the deprecated module code can be removed at some point. + * + * This function can be deleted together with `conditionallyDeepEvaluate` and the `unescape` option at that point. + */ +function partiallyEvaluateModule(config: Input, context: ModuleConfigContext) { + const partial = conditionallyDeepEvaluate( + config, + { + context, + opts: { + // Modules will be converted to actions later, and the actions will be properly unescaped. + // We avoid premature un-escaping here, + // because otherwise it will strip the escaped value in the module config + // to the normal template string in the converted action config. + unescape: false, + }, + }, + (value) => { + if (value instanceof ForEachLazyValue) { + return !someReferences({ + value, + context, + // if forEach expression has runtime references, we can't resolve it at all due to item context missing after converting the module to action + // as the captured context is lost when calling `toJSON` on the unresolved template value + onlyEssential: false, + matcher: (ref) => ref[0] === "runtime", + }) + } + + return !someReferences({ + value, + context, + // in other cases, we only skip evaluation when the runtime references is essential + // meaning, we evaluate everything we can evaluate. + onlyEssential: true, + matcher: (ref) => ref[0] === "runtime", + }) + } + ) + + return deepMap(partial, (v) => { + if (v instanceof UnresolvedTemplateValue) { + return v.toJSON() + } + return v + }) +} diff --git a/core/src/template/analysis.ts b/core/src/template/analysis.ts index 03f97a8fa3..7eaef7eb61 100644 --- a/core/src/template/analysis.ts +++ b/core/src/template/analysis.ts @@ -128,3 +128,35 @@ export function* getContextLookupReferences( } } } + +export function someReferences({ + value, + context, + onlyEssential = false, + matcher, +}: { + value: UnresolvedTemplateValue + context: ConfigContext + /** + * If true, the returned template expression generator will only yield template expressions that + * will be evaluated when calling `evaluate`. + * + * If `evaluate` returns `partial: true`, and `onlyEssential` is set to true, then the unresolved + * expressions returned by evaluate will not be emitted by the returned generator. + * + * @default false + */ + onlyEssential?: boolean + matcher: (ref: ContextLookupReferenceFinding) => boolean +}) { + const generator = getContextLookupReferences(value.visitAll({ onlyEssential }), context) + + for (const ref of generator) { + const isMatch = matcher(ref) + if (isMatch) { + return true + } + } + + return false +} diff --git a/core/src/template/ast.ts b/core/src/template/ast.ts index e8373cf608..d286beac3a 100644 --- a/core/src/template/ast.ts +++ b/core/src/template/ast.ts @@ -7,13 +7,8 @@ */ import { isArray, isNumber, isString } from "lodash-es" -import { - CONTEXT_RESOLVE_KEY_NOT_FOUND, - renderKeyPath, - type ConfigContext, - type ContextResolveOpts, -} from "../config/template-contexts/base.js" -import { GardenError, InternalError } from "../exceptions.js" +import { CONTEXT_RESOLVE_KEY_NOT_FOUND, renderKeyPath } from "../config/template-contexts/base.js" +import { InternalError } from "../exceptions.js" import { getHelperFunctions } from "./functions/index.js" import type { EvaluateTemplateArgs } from "./types.js" import { isTemplatePrimitive, type TemplatePrimitive } from "./types.js" @@ -706,7 +701,14 @@ export class ContextLookupExpression extends TemplateExpression { keyPath.push(evaluated) } - const { resolved, getUnavailableReason } = this.resolveContext(context, keyPath, opts, yamlSource) + const { resolved, getUnavailableReason } = context.resolve({ + key: keyPath, + nodePath: [], + // TODO: freeze opts object instead of using shallow copy + opts: { + ...opts, + }, + }) // if ((opts.allowPartial || opts.allowPartialContext) && resolved === CONTEXT_RESOLVE_KEY_AVAILABLE_LATER) { // return resolved @@ -728,33 +730,6 @@ export class ContextLookupExpression extends TemplateExpression { return resolved } - - private resolveContext( - context: ConfigContext, - keyPath: (string | number)[], - opts: ContextResolveOpts, - yamlSource: ConfigSource - ) { - try { - return context.resolve({ - key: keyPath, - nodePath: [], - // TODO: freeze opts object instead of using shallow copy - opts: { - ...opts, - }, - }) - } catch (e) { - // TODO: improve error handling for template strings nested inside contexts - // if (e instanceof TemplateStringError) { - // throw e - // } - if (e instanceof GardenError) { - throw new TemplateStringError({ message: e.message, loc: this.loc, yamlSource }) - } - throw e - } - } } export class FunctionCallExpression extends TemplateExpression { From dcf7c0089b3e6f22d85f9d5c1a96d5116f873a2c Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Thu, 16 Jan 2025 16:49:27 +0100 Subject: [PATCH 023/117] test: add extra test cases for template string access protection --- core/test/unit/src/template-string.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index 776199dcf1..1569b8521f 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -23,6 +23,25 @@ import { throwOnMissingSecretKeys } from "../../../src/config/secrets.js" import { parseTemplateCollection } from "../../../src/template/templated-collections.js" import { deepEvaluate } from "../../../src/template/evaluate.js" +describe("template string access protection", () => { + it("should crash when an unresolved value is accidentally treated as resolved", () => { + const parsed = parseTemplateCollection({ value: { foo: "${bar}" } as const, source: { path: [] } }) + expect(() => parsed.foo!["a"]).to.throw() + }) + + it("should not crash when an unresolved value is correctly evaluated", () => { + const parsed = parseTemplateCollection({ value: { foo: "${bar}" } as const, source: { path: [] } }) + const evaluated = deepEvaluate(parsed, { context: new GenericContext({ bar: "baz" }), opts: {} }) + expect(evaluated).to.eql({ foo: "baz" }) + }) + + it("should crash when an unresolved value is accidentally used in a spread operator", () => { + const parsed = parseTemplateCollection({ value: { foo: "${bar}" } as const, source: { path: [] } }) + const foo = parsed.foo as any + expect(() => ({ ...foo })).to.throw() + }) +}) + describe("parse and evaluate template strings", () => { it("should return a non-templated string unchanged", () => { const res = resolveTemplateString({ string: "somestring", context: new GenericContext({}) }) From b90551d34e839207b73c4b925ac9769c3fb3a378 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Mon, 20 Jan 2025 08:33:21 +0100 Subject: [PATCH 024/117] chore: fix lint errors --- core/src/commands/workflow.ts | 1 - core/src/config/provider.ts | 1 - core/src/graph/actions.ts | 10 +++++++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/core/src/commands/workflow.ts b/core/src/commands/workflow.ts index dee434f178..73c70eeea5 100644 --- a/core/src/commands/workflow.ts +++ b/core/src/commands/workflow.ts @@ -88,7 +88,6 @@ export class WorkflowCommand extends Command { await registerAndSetUid(garden, log, workflow) garden.events.emit("workflowRunning", {}) const templateContext = new WorkflowConfigContext(garden, garden.variables) - const yamlDoc = workflow.internal.yamlDoc const files = deepEvaluate((workflow.files || []) as unknown as ParsedTemplate[], { context: templateContext, opts: {}, diff --git a/core/src/config/provider.ts b/core/src/config/provider.ts index 5686308dd9..765eb3ff7c 100644 --- a/core/src/config/provider.ts +++ b/core/src/config/provider.ts @@ -31,7 +31,6 @@ import { uuidv4 } from "../util/random.js" import { s } from "./zod.js" import { getContextLookupReferences, visitAll } from "../template/analysis.js" import type { ConfigContext } from "./template-contexts/base.js" -import type { ParsedTemplate } from "../template/types.js" import type { UnresolvedProviderConfig } from "./project.js" // TODO: dedupe from the joi schema below diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index 21f2abbec9..21721196ed 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -511,9 +511,13 @@ export const processActionConfig = profileAsync(async function processActionConf let variables = await mergeVariables({ basePath: effectiveConfigFileLocation, - variables: new GenericContext(capture(config.variables, - // TODO: What's the correct context here? - garden.getProjectConfigContext()) || {}), + variables: new GenericContext( + capture( + config.variables, + // TODO: What's the correct context here? + garden.getProjectConfigContext() + ) || {} + ), varfiles: config.varfiles, log, }) From 236aced525efd5ca9a49e18577848eee22c9ce8f Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Mon, 20 Jan 2025 12:00:06 +0100 Subject: [PATCH 025/117] chore: enhance error handling --- core/src/config/render-template.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/config/render-template.ts b/core/src/config/render-template.ts index 513112d4ed..d1c4a7bdb3 100644 --- a/core/src/config/render-template.ts +++ b/core/src/config/render-template.ts @@ -267,6 +267,10 @@ async function renderConfigs({ opts: {}, }) as string } catch (error) { + if (!(error instanceof GardenError)) { + throw error + } + throw new ConfigurationError({ message: `Could not resolve the \`name\` field (${m.name}) for a config in ${templateDescription}: ${error}\n\nNote that template strings in config names in must be fully resolvable at the time of scanning. This means that e.g. references to other actions, modules or runtime outputs cannot be used.`, }) From bb4c33b151c0a2bef35922aa8f857bfa159ef110 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Mon, 20 Jan 2025 12:00:38 +0100 Subject: [PATCH 026/117] chore: add `toString()` for unresolved template values --- core/src/template/templated-strings.ts | 4 ++++ core/src/template/types.ts | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/core/src/template/templated-strings.ts b/core/src/template/templated-strings.ts index 424446c3b2..82ce890eab 100644 --- a/core/src/template/templated-strings.ts +++ b/core/src/template/templated-strings.ts @@ -67,6 +67,10 @@ class ParsedTemplateString extends UnresolvedTemplateValue { return this.rootNode.rawText } + override toString(): string { + return `UnresolvedTemplateValue(${this.rootNode.rawText})` + } + public override *visitAll(): TemplateExpressionGenerator { yield* this.rootNode.visitAll(this.source) } diff --git a/core/src/template/types.ts b/core/src/template/types.ts index 47773d3ed7..07172e0c8c 100644 --- a/core/src/template/types.ts +++ b/core/src/template/types.ts @@ -91,7 +91,7 @@ const accessDetector = new Proxy( ) export abstract class UnresolvedTemplateValue { - constructor() { + protected constructor() { // The spread trap exists to make our code more robust by detecting spreading unresolved template values. Object.defineProperty(this, "objectSpreadTrap", { enumerable: true, @@ -103,6 +103,7 @@ export abstract class UnresolvedTemplateValue { } public abstract evaluate(args: EvaluateTemplateArgs): TemplateEvaluationResult + public abstract toJSON(): CollectionOrValue public abstract visitAll(opts: { @@ -117,6 +118,10 @@ export abstract class UnresolvedTemplateValue { */ onlyEssential?: boolean }): TemplateExpressionGenerator + + public toString(): string { + return `UnresolvedTemplateValue(${this.constructor.name})` + } } // NOTE: this will make sure we throw an error if this value is accidentally treated as resolved. From c834b8a343d07fdcdee9ef3bd47c8d9f4bd92b00 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Mon, 20 Jan 2025 12:01:11 +0100 Subject: [PATCH 027/117] fix: fix the processing order in the context resolution loop --- core/src/config/template-contexts/base.ts | 45 ++++++++++++----------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 38972bd7c9..6b5e27152a 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -171,6 +171,27 @@ export abstract class ConfigContext { const getStackEntry = () => renderKeyPath(capturedNestedNodePath) getAvailableKeys = undefined + const parent: CollectionOrValue = value + if (isTemplatePrimitive(parent)) { + throw new ContextResolveError({ + message: `Attempted to look up key ${JSON.stringify(nextKey)} on a ${typeof parent}.`, + }) + } else if (typeof nextKey === "string" && nextKey.startsWith("_")) { + value = undefined + } else if (parent instanceof Map) { + getAvailableKeys = () => Array.from(parent.keys()) + value = parent.get(nextKey) + } else { + getAvailableKeys = () => { + return Object.keys(parent).filter((k) => !k.startsWith("_")) + } + value = parent[nextKey] + } + + if (value === undefined) { + break + } + // handle nested contexts if (value instanceof ConfigContext) { const remainder = getRemainder() @@ -196,27 +217,6 @@ export abstract class ConfigContext { const { resolved } = evaluate(value, { context: getRootContext(), opts }) value = resolved } - - const parent: CollectionOrValue = value - if (isTemplatePrimitive(parent)) { - throw new ContextResolveError({ - message: `Attempted to look up key ${JSON.stringify(nextKey)} on a ${typeof parent}.`, - }) - } else if (typeof nextKey === "string" && nextKey.startsWith("_")) { - value = undefined - } else if (parent instanceof Map) { - getAvailableKeys = () => Array.from(parent.keys()) - value = parent.get(nextKey) - } else { - getAvailableKeys = () => { - return Object.keys(parent).filter((k) => !k.startsWith("_")) - } - value = parent[nextKey] - } - - if (value === undefined) { - break - } } if (value === undefined || typeof value === "symbol") { @@ -381,6 +381,7 @@ export function renderKeyPath(key: ContextKeySegment[]): string { }, stringSegments[0]) ) } + export class CapturedContext extends ConfigContext { constructor( private readonly wrapped: ConfigContext, @@ -399,10 +400,12 @@ export class CapturedContext extends ConfigContext { export class LayeredContext extends ConfigContext { private readonly contexts: ConfigContext[] + constructor(...contexts: ConfigContext[]) { super() this.contexts = contexts } + override resolve(args: ContextResolveParams): ContextResolveOutput { const items: ResolvedTemplate[] = [] From 4b11d8e99d072c16ba5a54b0c9a0dc79ee6adb93 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Mon, 20 Jan 2025 14:42:09 +0100 Subject: [PATCH 028/117] chore: helper function to serialise unresolved template values in tests Co-authored-by: Steffen Neubauer --- core/src/template/types.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/core/src/template/types.ts b/core/src/template/types.ts index 07172e0c8c..58b896844d 100644 --- a/core/src/template/types.ts +++ b/core/src/template/types.ts @@ -9,6 +9,7 @@ import type { Primitive } from "utility-types" import { isPrimitive } from "utility-types" import type { Collection, CollectionOrValue } from "../util/objects.js" +import { deepMap } from "../util/objects.js" import type { ConfigContext, ContextResolveOpts } from "../config/template-contexts/base.js" import type { TemplateExpressionGenerator } from "./analysis.js" import { InternalError } from "../exceptions.js" @@ -126,3 +127,12 @@ export abstract class UnresolvedTemplateValue { // NOTE: this will make sure we throw an error if this value is accidentally treated as resolved. Object.setPrototypeOf(UnresolvedTemplateValue.prototype, accessDetector) + +export function serialiseUnresolvedTemplates(arg: unknown): unknown { + return deepMap(arg, (v) => { + if (v instanceof UnresolvedTemplateValue) { + return v.toJSON() + } + return v + }) +} From 321651257ec7b0bcbfd367228896e83714f7fa09 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Mon, 20 Jan 2025 14:42:30 +0100 Subject: [PATCH 029/117] test: fix one test failure --- core/test/unit/src/garden.ts | 61 +++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index dff5c5825d..8c141256f6 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -14,26 +14,26 @@ import nock from "nock" import { dirname, join, resolve } from "node:path" import { Garden } from "../../../src/garden.js" import { + createProjectConfig, expectError, + expectFuzzyMatch, + getDataDir, + getEmptyPluginActionDefinitions, + makeExtModuleSourcesGarden, + makeExtProjectSourcesGarden, + makeModuleConfig, + makeTempGarden, makeTestGarden, makeTestGardenA, projectRootA, - getDataDir, - testModuleVersion, - TestGarden, - testPlugin, - makeExtProjectSourcesGarden, - makeExtModuleSourcesGarden, - testGitUrlHash, resetLocalConfig, + TestGarden, testGitUrl, - expectFuzzyMatch, - createProjectConfig, - makeModuleConfig, - makeTempGarden, - getEmptyPluginActionDefinitions, + testGitUrlHash, + testModuleVersion, + testPlugin, } from "../../helpers.js" -import { getNames, findByName, exec } from "../../../src/util/util.js" +import { exec, findByName, getNames } from "../../../src/util/util.js" import type { LinkedSource } from "../../../src/config-store/local.js" import type { ModuleVersion, TreeVersion } from "../../../src/vcs/vcs.js" import { getModuleVersionString } from "../../../src/vcs/vcs.js" @@ -52,14 +52,12 @@ import { gardenEnv, } from "../../../src/constants.js" import { providerConfigBaseSchema } from "../../../src/config/provider.js" -import { keyBy, set, mapValues, omit, cloneDeep } from "lodash-es" +import { cloneDeep, keyBy, mapValues, omit, set } from "lodash-es" import { joi } from "../../../src/config/common.js" import { defaultDotIgnoreFile, makeTempDir } from "../../../src/util/fs.js" import fsExtra from "fs-extra" - -const { realpath, writeFile, readFile, remove, pathExists, mkdirp, copy } = fsExtra import { dedent, deline, randomString, wordWrap } from "../../../src/util/string.js" -import { getLinkedSources, addLinkedSources } from "../../../src/util/ext-source-util.js" +import { addLinkedSources, getLinkedSources } from "../../../src/util/ext-source-util.js" import { dump } from "js-yaml" import { TestVcsHandler } from "./vcs/vcs.js" import type { ActionRouter } from "../../../src/router/router.js" @@ -71,7 +69,7 @@ import { add } from "date-fns" import stripAnsi from "strip-ansi" import { GardenCloudApi } from "../../../src/cloud/api.js" import { GlobalConfigStore } from "../../../src/config-store/global.js" -import { LogLevel, getRootLogger } from "../../../src/logger/logger.js" +import { getRootLogger, LogLevel } from "../../../src/logger/logger.js" import { uuidv4 } from "../../../src/util/random.js" import { fileURLToPath } from "node:url" import { resolveMsg } from "../../../src/logger/log-entry.js" @@ -80,6 +78,9 @@ import type { RunActionConfig } from "../../../src/actions/run.js" import type { ProjectResult } from "@garden-io/platform-api-types" import { ProjectStatus } from "@garden-io/platform-api-types" import { getCloudDistributionName } from "../../../src/cloud/util.js" +import { serialiseUnresolvedTemplates } from "../../../src/template/types.js" + +const { realpath, writeFile, readFile, remove, pathExists, mkdirp, copy } = fsExtra const moduleDirName = dirname(fileURLToPath(import.meta.url)) @@ -3014,15 +3015,15 @@ describe("Garden", () => { const deploy = configs.Deploy["foo-test"] const test = configs.Test["foo-test"] - const internal = { + const expectedInternal = { basePath: garden.projectRoot, configFilePath: join(garden.projectRoot, "actions.garden.yml"), parentName: "foo", templateName: "combo", inputs: { name: "test", - envName: "${environment.name}", // <- should be resolved to itself - providerKey: "${providers.test-plugin.outputs.testKey}", // <- should be resolved to itself + envName: "${environment.name}", + providerKey: "${providers.test-plugin.outputs.testKey}", }, } @@ -3031,15 +3032,19 @@ describe("Garden", () => { expect(test).to.exist expect(build.type).to.equal("test") - expect(build.spec.command).to.include(internal.inputs.name) // <- should be resolved - expect(omit(build.internal, "yamlDoc")).to.eql(internal) + expect(serialiseUnresolvedTemplates(build.spec.command)).to.eql(["echo", "echo-prefix", "${inputs.name}"]) + expect(serialiseUnresolvedTemplates(omit(build.internal, "yamlDoc"))).to.eql(expectedInternal) - expect(deploy["build"]).to.equal(`${internal.parentName}-${internal.inputs.name}`) // <- should be resolved - expect(omit(deploy.internal, "yamlDoc")).to.eql(internal) + expect(serialiseUnresolvedTemplates(deploy["build"])).to.equal("${parent.name}-${inputs.name}") + expect(serialiseUnresolvedTemplates(omit(deploy.internal, "yamlDoc"))).to.eql(expectedInternal) - expect(test.dependencies).to.eql([`build.${internal.parentName}-${internal.inputs.name}`]) // <- should be resolved - expect(test.spec.command).to.eql(["echo", internal.inputs.envName, internal.inputs.providerKey]) // <- should be resolved - expect(omit(test.internal, "yamlDoc")).to.eql(internal) + expect(serialiseUnresolvedTemplates(test.dependencies)).to.eql(["build.${parent.name}-${inputs.name}"]) + expect(serialiseUnresolvedTemplates(test.spec.command)).to.eql([ + "echo", + "${inputs.envName}", + "${inputs.providerKey}", + ]) + expect(serialiseUnresolvedTemplates(omit(test.internal, "yamlDoc"))).to.eql(expectedInternal) }) it("should resolve disabled flag in actions and allow two actions with same key if one is disabled", async () => { From c814c5235218acaf07d3700d947709ed73d79eb7 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Mon, 20 Jan 2025 15:40:33 +0100 Subject: [PATCH 030/117] chore: fix more bugs --- core/src/config/base.ts | 4 +-- core/src/config/render-template.ts | 22 ++++++++++++---- core/test/unit/src/garden.ts | 41 +++++++++++++++++------------- 3 files changed, 42 insertions(+), 25 deletions(-) diff --git a/core/src/config/base.ts b/core/src/config/base.ts index 74677f4520..2df1ec1105 100644 --- a/core/src/config/base.ts +++ b/core/src/config/base.ts @@ -395,11 +395,11 @@ function handleProjectModules(log: Log, projectSpec: ProjectConfig): ProjectConf log, "Project configuration field `modules` is deprecated in 0.13 and will be removed in 0.14. Please use the `scan` field instead." ) - const scanConfig = projectSpec.scan || {} + let scanConfig = projectSpec.scan || {} for (const key of ["include", "exclude"]) { if (projectSpec["modules"][key]) { if (!scanConfig[key]) { - scanConfig[key] = projectSpec["modules"][key] + scanConfig = { ...scanConfig, [key]: projectSpec["modules"][key] } } else { log.warn( `Project-level \`${key}\` is set both in \`modules.${key}\` and \`scan.${key}\`. The value from \`scan.${key}\` will be used (and the value from \`modules.${key}\` will not have any effect).` diff --git a/core/src/config/render-template.ts b/core/src/config/render-template.ts index d1c4a7bdb3..3c985a0876 100644 --- a/core/src/config/render-template.ts +++ b/core/src/config/render-template.ts @@ -19,7 +19,7 @@ import { import { maybeTemplateString } from "../template/templated-strings.js" import { validateWithPath } from "./validation.js" import type { Garden } from "../garden.js" -import { ConfigurationError, GardenError } from "../exceptions.js" +import { ConfigurationError, GardenError, InternalError } from "../exceptions.js" import { resolve, posix } from "path" import fsExtra from "fs-extra" @@ -35,8 +35,9 @@ import { RenderTemplateConfigContext } from "./template-contexts/render.js" import type { Log } from "../logger/log-entry.js" import { GardenApiVersion } from "../constants.js" import { capture } from "../template/capture.js" -import { deepEvaluate } from "../template/evaluate.js" +import { deepEvaluate, evaluate } from "../template/evaluate.js" import type { ParsedTemplate } from "../template/types.js" +import { isArray } from "../util/objects.js" export const renderTemplateConfigSchema = createSchema({ name: renderTemplateKind, @@ -141,7 +142,6 @@ export async function renderConfigTemplate({ return { resolved, modules: [], configs: [] } } - // Validate the module spec resolved = validateWithPath({ config: resolved, configType, @@ -254,10 +254,22 @@ async function renderConfigs({ renderConfig: RenderTemplateConfig }): Promise { const templateDescription = `${configTemplateKind} '${template.name}'` - const templateConfigs = template.configs || [] + const templateConfigs = evaluate((template.configs || []) as unknown as ParsedTemplate, { + context, + opts: {}, + }).resolved + + if (!isArray(templateConfigs)) { + throw new InternalError({ message: "Expected templateConfigs to be an array" }) + } return Promise.all( - templateConfigs.map(async (m) => { + templateConfigs.map(async (c) => { + const m = evaluate(c, { + context, + opts: {}, + }).resolved as any + // Resolve just the name, which must be immediately resolvable let resolvedName = m.name diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index 8c141256f6..b5affff820 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -79,6 +79,8 @@ import type { ProjectResult } from "@garden-io/platform-api-types" import { ProjectStatus } from "@garden-io/platform-api-types" import { getCloudDistributionName } from "../../../src/cloud/util.js" import { serialiseUnresolvedTemplates } from "../../../src/template/types.js" +import { parseTemplateCollection } from "../../../src/template/templated-collections.js" +import { GenericContext, LayeredContext } from "../../../src/config/template-contexts/base.js" const { realpath, writeFile, readFile, remove, pathExists, mkdirp, copy } = fsExtra @@ -2912,17 +2914,20 @@ describe("Garden", () => { await exec("git", ["add", "."], { cwd: repoPath }) await exec("git", ["commit", "-m", "foo"], { cwd: repoPath }) - garden.variables["sourceBranch"] = "main" + garden.variables = new LayeredContext(garden.variables, new GenericContext({ var: { sourceBranch: "main" } })) // eslint-disable-next-line @typescript-eslint/no-explicit-any const _garden = garden as any - _garden["projectSources"] = [ - { - name: "source-a", - // Use a couple of template strings in the repo path - repositoryUrl: "file://" + _tmpDir.path + "/${project.name}#${var.sourceBranch}", - }, - ] + _garden["projectSources"] = parseTemplateCollection({ + value: [ + { + name: "source-a", + // Use a couple of template strings in the repo path + repositoryUrl: "file://" + _tmpDir.path + "/${project.name}#${var.sourceBranch}", + }, + ], + source: { path: [] }, + }) await garden.scanAndAddConfigs() @@ -3103,24 +3108,24 @@ describe("Garden", () => { type: "exec", name: runNameA, spec: { - command: ["echo", runNameA], + command: ["echo", "${item.value}"], }, internal, } - expect(omit(runA, "internal")).to.eql(omit(expectedRunA, "internal")) - expect(omit(runA.internal, "yamlDoc")).to.eql(expectedRunA.internal) + expect(serialiseUnresolvedTemplates(omit(runA, "internal"))).to.eql(omit(expectedRunA, "internal")) + expect(serialiseUnresolvedTemplates(omit(runA.internal, "yamlDoc"))).to.eql(expectedRunA.internal) const expectedRunB: Partial = { kind: "Run", type: "exec", name: runNameB, spec: { - command: ["echo", runNameB], + command: ["echo", "${item.value}"], }, internal, } - expect(omit(runB, "internal")).to.eql(omit(expectedRunB, "internal")) - expect(omit(runB.internal, "yamlDoc")).to.eql(expectedRunB.internal) + expect(serialiseUnresolvedTemplates(omit(runB, "internal"))).to.eql(omit(expectedRunB, "internal")) + expect(serialiseUnresolvedTemplates(omit(runB.internal, "yamlDoc"))).to.eql(expectedRunB.internal) }) it("should resolve a workflow from a template", async () => { @@ -3136,13 +3141,13 @@ describe("Garden", () => { templateName: "workflows", inputs: { name: "test", - envName: "${environment.name}", // <- should be resolved to itself + envName: "${environment.name}", }, } expect(workflow).to.exist - expect(workflow.steps).to.eql([{ script: `echo "${internal.inputs.envName}"` }]) // <- should be resolved - expect(omit(workflow.internal, "yamlDoc")).to.eql(internal) + expect(serialiseUnresolvedTemplates(workflow.steps)).to.eql([{ script: 'echo "${inputs.envName}"' }]) + expect(serialiseUnresolvedTemplates(omit(workflow.internal, "yamlDoc"))).to.eql(internal) }) it("should throw on duplicate config template names", async () => { @@ -3174,7 +3179,7 @@ describe("Garden", () => { }) // TODO-0.14: remove this and core/test/data/test-projects/project-include-exclude-old-syntax directory - it("should respect the modules.include and modules.exclude fields, if specified", async () => { + it("should respect the modules.include and modules.exclude fields, if specified (old syntax)", async () => { const projectRoot = getDataDir("test-projects", "project-include-exclude-old-syntax") const garden = await makeTestGarden(projectRoot) const modules = await garden.resolveModules({ log: garden.log }) From 2c06515c5c77e5ba4ccfc30574e6b722defaf6d6 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Mon, 20 Jan 2025 15:44:02 +0100 Subject: [PATCH 031/117] refactor: introduce named type for symbol `CONTEXT_RESOLVE_KEY_NOT_FOUND` --- core/src/config/template-contexts/base.ts | 3 ++- core/src/template/ast.ts | 19 ++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 6b5e27152a..2d20bd0615 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -51,7 +51,7 @@ export type ContextResolveOutput = getUnavailableReason?: undefined } | { - resolved: typeof CONTEXT_RESOLVE_KEY_NOT_FOUND + resolved: ContextResolveKeyNotFound getUnavailableReason: () => string } @@ -76,6 +76,7 @@ export class ContextResolveError extends GardenError { } export const CONTEXT_RESOLVE_KEY_NOT_FOUND: unique symbol = Symbol.for("ContextResolveKeyNotFound") +export type ContextResolveKeyNotFound = typeof CONTEXT_RESOLVE_KEY_NOT_FOUND // Note: we're using classes here to be able to use decorators to describe each context node and key @Profile() diff --git a/core/src/template/ast.ts b/core/src/template/ast.ts index d286beac3a..7bf6f72b79 100644 --- a/core/src/template/ast.ts +++ b/core/src/template/ast.ts @@ -7,6 +7,7 @@ */ import { isArray, isNumber, isString } from "lodash-es" +import type { ContextResolveKeyNotFound } from "../config/template-contexts/base.js" import { CONTEXT_RESOLVE_KEY_NOT_FOUND, renderKeyPath } from "../config/template-contexts/base.js" import { InternalError } from "../exceptions.js" import { getHelperFunctions } from "./functions/index.js" @@ -27,7 +28,7 @@ type ASTEvaluateArgs = EvaluateTemplateArgs & { readonly optional?: boolean } -export type ASTEvaluationResult = T | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND +export type ASTEvaluationResult = T | ContextResolveKeyNotFound // | typeof CONTEXT_RESOLVE_KEY_AVAILABLE_LATER export type TemplateStringSource = { @@ -173,14 +174,14 @@ export abstract class UnaryExpression extends TemplateExpression { } abstract transform( - value: CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND - ): TemplatePrimitive | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND + value: CollectionOrValue | ContextResolveKeyNotFound + ): TemplatePrimitive | ContextResolveKeyNotFound } export class TypeofExpression extends UnaryExpression { override transform( - value: CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND - ): string | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + value: CollectionOrValue | ContextResolveKeyNotFound + ): string | ContextResolveKeyNotFound { if (isNotFound(value)) { return "undefined" } @@ -190,8 +191,8 @@ export class TypeofExpression extends UnaryExpression { export class NotExpression extends UnaryExpression { override transform( - value: CollectionOrValue | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND - ): boolean | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + value: CollectionOrValue | ContextResolveKeyNotFound + ): boolean | ContextResolveKeyNotFound { if (isNotFound(value)) { return true } @@ -215,8 +216,8 @@ export function isNotFound( v: | CollectionOrValue // CONTEXT_RESOLVE_KEY_AVAILABLE_LATER is not included here on purpose, because it must always be handled separately by returning early. - | typeof CONTEXT_RESOLVE_KEY_NOT_FOUND -): v is typeof CONTEXT_RESOLVE_KEY_NOT_FOUND { + | ContextResolveKeyNotFound +): v is ContextResolveKeyNotFound { return v === CONTEXT_RESOLVE_KEY_NOT_FOUND } From 50e6d90f1cf81cc16d96873c716b2276ccda8352 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Mon, 20 Jan 2025 16:08:30 +0100 Subject: [PATCH 032/117] test: fix assertions in provider configuration tests --- core/test/unit/src/garden.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index b5affff820..2ac1cd8bcf 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -2524,7 +2524,7 @@ describe("Garden", () => { const providerB = await garden.resolveProvider({ log: garden.log, name: "test-b" }) - expect(providerB.config["foo"]).to.equal("bar") + expect(providerB.config["foo"]).to.equal("${providers.test-a.outputs.foo}") }) it("should allow providers to reference outputs from a disabled provider", async () => { @@ -2563,7 +2563,7 @@ describe("Garden", () => { const providerB = await garden.resolveProvider({ log: garden.log, name: "test-b" }) - expect(providerB.config["foo"]).to.equal("default") + expect(providerB.config["foo"]).to.equal("${providers.test-a.outputs.foo || 'default'}") }) it("should allow providers to reference variables", async () => { @@ -2583,7 +2583,7 @@ describe("Garden", () => { const providerB = await garden.resolveProvider({ log: garden.log, name: "test-a" }) - expect(providerB.config["foo"]).to.equal("bar") + expect(providerB.config["foo"]).to.equal("${var.my-variable}") }) it("should match a dependency to a plugin base", async () => { From 087472cb38d4c7d7fd9426683f298d633eccb9db Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Mon, 20 Jan 2025 21:13:14 +0100 Subject: [PATCH 033/117] chore: rewrite ConfigContext.resolve --- core/src/commands/workflow.ts | 1 - core/src/config/provider.ts | 1 - core/src/config/template-contexts/actions.ts | 32 +- core/src/config/template-contexts/base.ts | 522 ++++++++++-------- core/src/config/template-contexts/module.ts | 47 +- core/src/config/template-contexts/project.ts | 52 +- core/src/config/template-contexts/provider.ts | 11 +- core/src/config/template-contexts/render.ts | 4 +- core/src/config/template-contexts/workflow.ts | 10 +- core/src/exceptions.ts | 6 +- core/src/garden.ts | 17 +- core/src/graph/actions.ts | 14 +- core/src/plugin-context.ts | 4 +- core/src/router/base.ts | 4 +- core/src/tasks/publish.ts | 10 +- core/src/template/ast.ts | 18 +- .../src/actions/action-configs-to-graph.ts | 49 +- .../unit/src/config/template-contexts/base.ts | 212 +++---- .../src/config/template-contexts/module.ts | 50 +- .../src/config/template-contexts/project.ts | 55 +- .../src/config/template-contexts/provider.ts | 15 +- .../src/config/template-contexts/workflow.ts | 38 +- core/test/unit/src/template-string.ts | 2 +- 23 files changed, 620 insertions(+), 554 deletions(-) diff --git a/core/src/commands/workflow.ts b/core/src/commands/workflow.ts index dee434f178..73c70eeea5 100644 --- a/core/src/commands/workflow.ts +++ b/core/src/commands/workflow.ts @@ -88,7 +88,6 @@ export class WorkflowCommand extends Command { await registerAndSetUid(garden, log, workflow) garden.events.emit("workflowRunning", {}) const templateContext = new WorkflowConfigContext(garden, garden.variables) - const yamlDoc = workflow.internal.yamlDoc const files = deepEvaluate((workflow.files || []) as unknown as ParsedTemplate[], { context: templateContext, opts: {}, diff --git a/core/src/config/provider.ts b/core/src/config/provider.ts index 5686308dd9..765eb3ff7c 100644 --- a/core/src/config/provider.ts +++ b/core/src/config/provider.ts @@ -31,7 +31,6 @@ import { uuidv4 } from "../util/random.js" import { s } from "./zod.js" import { getContextLookupReferences, visitAll } from "../template/analysis.js" import type { ConfigContext } from "./template-contexts/base.js" -import type { ParsedTemplate } from "../template/types.js" import type { UnresolvedProviderConfig } from "./project.js" // TODO: dedupe from the joi schema below diff --git a/core/src/config/template-contexts/actions.ts b/core/src/config/template-contexts/actions.ts index f0927d2104..9c67ca7246 100644 --- a/core/src/config/template-contexts/actions.ts +++ b/core/src/config/template-contexts/actions.ts @@ -14,7 +14,8 @@ import { dedent, deline } from "../../util/string.js" import type { DeepPrimitiveMap, PrimitiveMap } from "../common.js" import { joi, joiIdentifier, joiIdentifierMap, joiPrimitive, joiVariables } from "../common.js" import type { ProviderMap } from "../provider.js" -import { ConfigContext, ErrorContext, GenericContext, ParentContext, schema, TemplateContext } from "./base.js" +import type { ConfigContext } from "./base.js" +import { ContextWithSchema, ErrorContext, GenericContext, ParentContext, schema, TemplateContext } from "./base.js" import { exampleVersion, OutputConfigContext } from "./module.js" import { TemplatableConfigContext } from "./project.js" import { DOCS_BASE_URL } from "../../constants.js" @@ -43,15 +44,15 @@ const actionModeSchema = joi ) .example("sync") -class ActionConfigThisContext extends ConfigContext { +class ActionConfigThisContext extends ContextWithSchema { @schema(actionNameSchema) public name: string @schema(actionModeSchema) public mode: ActionMode - constructor(root: ConfigContext, { name, mode }: ActionConfigThisContextParams) { - super(root) + constructor({ name, mode }: ActionConfigThisContextParams) { + super() this.name = name this.mode = mode } @@ -75,13 +76,12 @@ export class ActionConfigContext extends TemplatableConfigContext { constructor({ garden, config, thisContextParams, variables }: ActionConfigContextParams) { const mergedVariables = mergeVariables({ garden, variables }) super(garden, config) - this.this = new ActionConfigThisContext(this, thisContextParams) + this.this = new ActionConfigThisContext(thisContextParams) this.variables = this.var = mergedVariables } } interface ActionReferenceContextParams { - root: ConfigContext name: string disabled: boolean buildPath: string @@ -90,7 +90,7 @@ interface ActionReferenceContextParams { variables: ConfigContext } -export class ActionReferenceContext extends ConfigContext { +export class ActionReferenceContext extends ContextWithSchema { @schema(actionNameSchema) public name: string @@ -121,8 +121,8 @@ export class ActionReferenceContext extends ConfigContext { @schema(joiVariables().required().description("The variables configured on the action.").example({ foo: "bar" })) public var: ConfigContext - constructor({ root, name, disabled, buildPath, sourcePath, mode, variables }: ActionReferenceContextParams) { - super(root) + constructor({ name, disabled, buildPath, sourcePath, mode, variables }: ActionReferenceContextParams) { + super() this.name = name this.disabled = disabled this.buildPath = buildPath @@ -171,7 +171,7 @@ const _actionResultContextSchema = joiIdentifierMap(ActionResultContext.getSchem const actionResultContextSchema = (kind: string) => _actionResultContextSchema.description(`Information about a ${kind} action dependency, including its outputs.`) -class ActionReferencesContext extends ConfigContext { +class ActionReferencesContext extends ContextWithSchema { @schema(actionResultContextSchema("Build")) public build: Map @@ -190,8 +190,8 @@ class ActionReferencesContext extends ConfigContext { @schema(_actionResultContextSchema.description("Alias for `run`.")) public tasks: Map - constructor(root: ConfigContext, actions: (ResolvedAction | ExecutedAction)[]) { - super(root) + constructor(actions: (ResolvedAction | ExecutedAction)[]) { + super() this.build = new Map() this.deploy = new Map() @@ -205,7 +205,6 @@ class ActionReferencesContext extends ConfigContext { this[action.kind.toLowerCase()].set( action.name, new ActionResultContext({ - root: this, name: action.name, outputs: action.getOutputs(), version: action.versionString(), @@ -287,7 +286,7 @@ export class ActionSpecContext extends OutputConfigContext { const parentName = internal?.parentName const templateName = internal?.templateName - this.actions = new ActionReferencesContext(this, [...resolvedDependencies, ...executedDependencies]) + this.actions = new ActionReferencesContext([...resolvedDependencies, ...executedDependencies]) // Throw specific error when attempting to resolve self this.actions[action.kind.toLowerCase()].set( @@ -296,15 +295,14 @@ export class ActionSpecContext extends OutputConfigContext { ) if (parentName && templateName) { - this.parent = new ParentContext(this, parentName) - this.template = new TemplateContext(this, templateName) + this.parent = new ParentContext(parentName) + this.template = new TemplateContext(templateName) } this.inputs = inputs this.runtime = this.actions this.this = new ActionReferenceContext({ - root: this, disabled: action.isDisabled(), buildPath, name, diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 38972bd7c9..35eccdfa97 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -7,19 +7,18 @@ */ import type Joi from "@hapi/joi" -import { ConfigurationError, GardenError, InternalError } from "../../exceptions.js" +import { ConfigurationError, InternalError } from "../../exceptions.js" import type { CustomObjectSchema } from "../common.js" import { joi, joiIdentifier } from "../common.js" -import { naturalList } from "../../util/string.js" -import { styles } from "../../logger/styles.js" import { Profile } from "../../util/profiling.js" -import type { Collection } from "../../util/objects.js" -import { deepMap, isPlainObject, type CollectionOrValue } from "../../util/objects.js" -import type { ParsedTemplate, ResolvedTemplate, TemplatePrimitive } from "../../template/types.js" +import { deepMap, type Collection, type CollectionOrValue } from "../../util/objects.js" +import type { ParsedTemplate, ParsedTemplateValue, ResolvedTemplate, TemplatePrimitive } from "../../template/types.js" import { isTemplatePrimitive, UnresolvedTemplateValue } from "../../template/types.js" -import pick from "lodash-es/pick.js" -import { deepEvaluate, evaluate } from "../../template/evaluate.js" import merge from "lodash-es/merge.js" +import omitBy from "lodash-es/omitBy.js" +import { flatten, isEqual, uniq } from "lodash-es" +import { isMap } from "util/types" +import { deline } from "../../util/string.js" export type ContextKeySegment = string | number export type ContextKey = ContextKeySegment[] @@ -30,30 +29,55 @@ export interface ContextResolveOpts { // TODO(0.14): Remove legacyAllowPartial legacyAllowPartial?: boolean - // a list of values for detecting circular references - contextStack?: Set - keyStack?: Set + // for detecting circular references + stack?: string[] // TODO: remove unescape?: boolean } export interface ContextResolveParams { + /** + * Key path to look up in the context. + */ key: ContextKey - nodePath: ContextKey + + /** + * Context lookup options (Deprecated; These mostly affect template string evaluation) + */ opts: ContextResolveOpts + + /** + * The context to be used when evaluating encountered instances of `UnresolvedTemplateValue`. + */ rootContext?: ConfigContext } +export type ContextResolveOutputNotFound = { + found: false + /** + * @example + * { + * reason: "key_not_found", + * key: "foo" + * keyPath: ["var"], // var does not have a key foo + * } + */ + explanation: { + reason: "key_not_found" | "not_indexable" + key: string | number + keyPath: (string | number)[] + availableKeys?: (string | number)[] + } +} + export type ContextResolveOutput = | { + found: true resolved: ResolvedTemplate - getUnavailableReason?: undefined - } - | { - resolved: typeof CONTEXT_RESOLVE_KEY_NOT_FOUND - getUnavailableReason: () => string + partial?: Collection | TemplatePrimitive } + | ContextResolveOutputNotFound export function schema(joiSchema: Joi.Schema) { return (target: any, propName: string) => { @@ -62,213 +86,86 @@ export function schema(joiSchema: Joi.Schema) { } export interface ConfigContextType { - new (...params: any[]): ConfigContext + new (...params: any[]): ContextWithSchema getSchema(): CustomObjectSchema } -/** - * This error is thrown for a "final" errors, i.e. ones that cannot be ignored. - * For key not found errors that could be resolvable we still can return a special symbol. - */ -export class ContextResolveError extends GardenError { - type = "context-resolve" -} - -export const CONTEXT_RESOLVE_KEY_NOT_FOUND: unique symbol = Symbol.for("ContextResolveKeyNotFound") +let globalConfigContextCounter: number = 0 -// Note: we're using classes here to be able to use decorators to describe each context node and key -@Profile() export abstract class ConfigContext { - private readonly _rootContext?: ConfigContext - private readonly _resolvedValues: { [path: string]: any } - private readonly _startingPoint?: string - - constructor(rootContext?: ConfigContext, startingPoint?: string) { - if (rootContext) { - this._rootContext = rootContext - } - if (startingPoint) { - this._startingPoint = startingPoint - } - this._resolvedValues = {} - } - - static getSchema() { - const schemas = (this)._schemas - return joi.object().keys(schemas).required() - } + private readonly _cache: Map + private readonly _id: number - /** - * Override this method to add more context to error messages thrown in the `resolve` method when a missing key is - * referenced. - */ - getMissingKeyErrorFooter(_key: ContextKeySegment, _path: ContextKeySegment[]): string { - return "" + constructor() { + this._id = globalConfigContextCounter++ + this._cache = new Map() } - resolve({ key, nodePath, opts, rootContext }: ContextResolveParams): ContextResolveOutput { - const getRootContext = () => { - if (rootContext && this._rootContext) { - return new LayeredContext(rootContext, this._rootContext) - } - return rootContext || this._rootContext || this - } - - const path = key.join(".") - - // if the key has previously been resolved, return it directly - const alreadyResolved = this._resolvedValues[path] - - if (alreadyResolved) { - return { resolved: alreadyResolved } - } - - // keep track of which resolvers have been called, in order to detect circular references - let getAvailableKeys: (() => string[]) | undefined = undefined - - // eslint-disable-next-line @typescript-eslint/no-this-alias - let value: CollectionOrValue = this._startingPoint - ? this[this._startingPoint] - : this - - if (!isPlainObject(value) && !(value instanceof ConfigContext) && !(value instanceof UnresolvedTemplateValue)) { - throw new InternalError({ - message: `Invalid config context root: ${typeof value}`, + private detectCircularReference({ key, opts }: ContextResolveParams) { + const keyStr = `${this.constructor.name}(${this._id})-${renderKeyPath(key)}` + if (opts.stack?.includes(keyStr)) { + throw new ConfigurationError({ + message: `Circular reference detected: ${opts.stack.map((s) => s.split("-")[1]).join(" -> ")}`, }) } + return keyStr + } - // TODO: freeze opts object instead of using shallow copy - opts.keyStack = new Set(opts.keyStack || []) - opts.contextStack = new Set(opts.contextStack || []) - - if (opts.contextStack.has(value)) { - // TODO: fix circular ref detection - // Circular dependency error is critical, throwing here. - // throw new ContextResolveError({ - // message: `Circular reference detected when resolving key ${path} (${Array.from(opts.keyStack || []).join(" -> ")})`, - // }) - } - - let nextKey = key[0] - let nestedNodePath = nodePath - let getUnavailableReason: (() => string) | undefined = undefined + protected abstract resolveImpl(params: ContextResolveParams): ContextResolveOutput - if (key.length === 0 && !(value instanceof UnresolvedTemplateValue)) { - value = pick( - value, - Object.keys(value as Collection).filter((k) => !k.startsWith("_")) - ) as Record> + public resolve(params: ContextResolveParams): ContextResolveOutput { + const key = this.detectCircularReference(params) + if (!params.opts.stack) { + params.opts.stack = [key] + } else { + params.opts.stack.push(key) } - for (let p = 0; p < key.length; p++) { - nextKey = key[p] - - nestedNodePath = nodePath.concat(key.slice(0, p + 1)) - const getRemainder = () => key.slice(p + 1) - - const capturedNestedNodePath = nestedNodePath - const getStackEntry = () => renderKeyPath(capturedNestedNodePath) - getAvailableKeys = undefined - - // handle nested contexts - if (value instanceof ConfigContext) { - const remainder = getRemainder() - const stackEntry = getStackEntry() - opts.keyStack.add(stackEntry) - opts.contextStack.add(value) - // NOTE: we resolve even if remainder.length is zero to make sure all unresolved template values have been resolved. - const res = value.resolve({ key: remainder, nodePath: nestedNodePath, opts, rootContext }) - if (res.resolved === CONTEXT_RESOLVE_KEY_NOT_FOUND) { - throw new InternalError({ - message: "unhandled resolve key not found", - }) - } - value = res.resolved - getUnavailableReason = res.getUnavailableReason - break - } - - // handle templated strings in context variables - if (value instanceof UnresolvedTemplateValue) { - opts.keyStack.add(getStackEntry()) - opts.contextStack.add(value) - const { resolved } = evaluate(value, { context: getRootContext(), opts }) - value = resolved - } - - const parent: CollectionOrValue = value - if (isTemplatePrimitive(parent)) { - throw new ContextResolveError({ - message: `Attempted to look up key ${JSON.stringify(nextKey)} on a ${typeof parent}.`, - }) - } else if (typeof nextKey === "string" && nextKey.startsWith("_")) { - value = undefined - } else if (parent instanceof Map) { - getAvailableKeys = () => Array.from(parent.keys()) - value = parent.get(nextKey) - } else { - getAvailableKeys = () => { - return Object.keys(parent).filter((k) => !k.startsWith("_")) - } - value = parent[nextKey] + try { + let res = this._cache.get(key) + if (res) { + return res } - - if (value === undefined) { - break + res = this.resolveImpl(params) + if (res.found) { + this._cache.set(key, res) } + return res + } finally { + params.opts.stack.pop() } + } - if (value === undefined || typeof value === "symbol") { - if (getUnavailableReason === undefined) { - getUnavailableReason = () => { - let message = styles.error(`Could not find key ${styles.highlight(String(nextKey))}`) - if (nestedNodePath.length > 1) { - message += styles.error(" under ") + styles.highlight(renderKeyPath(nestedNodePath.slice(0, -1))) - } - message += styles.error(".") - - if (getAvailableKeys) { - const availableKeys = getAvailableKeys() - const availableStr = availableKeys.length - ? naturalList(availableKeys.sort().map((k) => styles.highlight(k))) - : "(none)" - message += styles.error(" Available keys: " + availableStr + ".") - } - const messageFooter = this.getMissingKeyErrorFooter(nextKey, nestedNodePath.slice(0, -1)) - if (messageFooter) { - message += `\n\n${messageFooter}` - } - return message - } - } - - if (typeof value === "symbol") { - return { resolved: value, getUnavailableReason } - } - - return { resolved: CONTEXT_RESOLVE_KEY_NOT_FOUND, getUnavailableReason } - } + /** + * Override this method to add more context to error messages thrown in the `resolve` method when a missing key is + * referenced. + */ + protected getMissingKeyErrorFooter(_key: ContextKeySegment, _path: ContextKeySegment[]): string { + return "" + } +} - if (!isTemplatePrimitive(value)) { - value = deepMap(value, (v, keyPath) => { - if (v instanceof ConfigContext) { - const { resolved } = v.resolve({ key: [], nodePath: nodePath.concat(key, keyPath), opts }) - if (resolved === CONTEXT_RESOLVE_KEY_NOT_FOUND) { - throw new InternalError({ - message: "Unhandled context resolve key not found", - }) - } - return resolved - } - return deepEvaluate(v, { context: getRootContext(), opts }) - }) - } +// Note: we're using classes here to be able to use decorators to describe each context node and key +@Profile() +export abstract class ContextWithSchema extends ConfigContext { + static getSchema() { + const schemas = (this)._schemas + return joi.object().keys(schemas).required() + } - // Cache result - this._resolvedValues[path] = value + private get startingPoint() { + // Make sure we filter keys that start with underscore + return + } - return { resolved: value as ResolvedTemplate } + protected override resolveImpl(params: ContextResolveParams): ContextResolveOutput { + return traverseContext( + omitBy(this, (key) => typeof key === "string" && key.startsWith("_")) as CollectionOrValue< + ParsedTemplate | ConfigContext + >, + { ...params, rootContext: params.rootContext || this } + ) } } @@ -276,27 +173,27 @@ export abstract class ConfigContext { * A generic context that just wraps an object. */ export class GenericContext extends ConfigContext { - constructor(private readonly data: any) { + constructor(protected readonly data: ParsedTemplate | Collection) { if (data === undefined) { throw new InternalError({ message: "Generic context may not be undefined.", }) } - if (data instanceof ConfigContext) { + if (data instanceof ContextWithSchema) { throw new InternalError({ message: "Generic context is useless when instantiated with just another context as parameter. Use the other context directly instead.", }) } - super(undefined, "data") + super() } - static override getSchema() { - return joi.object() + protected override resolveImpl(params: ContextResolveParams): ContextResolveOutput { + return traverseContext(this.data, { ...params, rootContext: params.rootContext || this }) } } -export class EnvironmentContext extends ConfigContext { +export class EnvironmentContext extends ContextWithSchema { @schema( joi .string() @@ -318,8 +215,8 @@ export class EnvironmentContext extends ConfigContext { @schema(joi.string().description("The currently active namespace (if any).").example("my-namespace")) public namespace: string - constructor(root: ConfigContext, name: string, fullName: string, namespace?: string) { - super(root) + constructor(name: string, fullName: string, namespace?: string) { + super() this.name = name this.fullName = fullName this.namespace = namespace || "" @@ -334,27 +231,27 @@ export class ErrorContext extends ConfigContext { super() } - override resolve({}): ContextResolveOutput { + protected override resolveImpl({}): ContextResolveOutput { throw new ConfigurationError({ message: this.message }) } } -export class ParentContext extends ConfigContext { +export class ParentContext extends ContextWithSchema { @schema(joiIdentifier().description(`The name of the parent config.`)) public name: string - constructor(root: ConfigContext, name: string) { - super(root) + constructor(name: string) { + super() this.name = name } } -export class TemplateContext extends ConfigContext { +export class TemplateContext extends ContextWithSchema { @schema(joiIdentifier().description(`The name of the template.`)) public name: string - constructor(root: ConfigContext, name: string) { - super(root) + constructor(name: string) { + super() this.name = name } } @@ -386,12 +283,17 @@ export class CapturedContext extends ConfigContext { private readonly wrapped: ConfigContext, private readonly rootContext: ConfigContext ) { - super(rootContext) + super() } - override resolve(params: ContextResolveParams): ContextResolveOutput { + override resolveImpl(params: ContextResolveParams): ContextResolveOutput { return this.wrapped.resolve({ ...params, + opts: { + ...params.opts, + // to avoid circular dep errors + stack: params.opts.stack?.slice(0, -1), + }, rootContext: params.rootContext ? new LayeredContext(this.rootContext, params.rootContext) : this.rootContext, }) } @@ -403,29 +305,191 @@ export class LayeredContext extends ConfigContext { super() this.contexts = contexts } - override resolve(args: ContextResolveParams): ContextResolveOutput { - const items: ResolvedTemplate[] = [] - - for (const [i, context] of this.contexts.entries()) { - const resolved = context.resolve(args) - if (resolved.resolved !== CONTEXT_RESOLVE_KEY_NOT_FOUND) { + override resolveImpl(args: ContextResolveParams): ContextResolveOutput { + const items: ContextResolveOutput[] = [] + + for (const context of this.contexts) { + const resolved = context.resolve({ + ...args, + opts: { + ...args.opts, + // to avoid circular dependency errors + stack: args.opts.stack?.slice(0, -1), + }, + }) + if (resolved.found) { if (isTemplatePrimitive(resolved.resolved)) { return resolved } - items.push(resolved.resolved) - } else if (items.length === 0 && i === this.contexts.length - 1) { - return resolved + } + + items.push(resolved) + } + + // if it could not be found in any context, aggregate error information from all contexts + if (items.every((res) => !res.found)) { + // find deepest key path (most specific error) + let deepestKeyPath: (number | string)[] = [] + for (const res of items) { + if (res.explanation.keyPath.length > deepestKeyPath.length) { + deepestKeyPath = res.explanation.keyPath + } + } + + // identify all errors with the same key path + const all = items.filter((res) => isEqual(res.explanation.keyPath, deepestKeyPath)) + const lastError = all[all.length - 1] + + return { + ...lastError, + explanation: { + ...lastError.explanation, + availableKeys: uniq(flatten(all.map((res) => res.explanation.availableKeys || []))), + }, } } const returnValue = {} for (const i of items) { - merge(returnValue, { resolved: i }) + if (!i.found) { + continue + } + + merge(returnValue, { resolved: i.resolved }) } return { + found: true, resolved: returnValue["resolved"], } } } + +function traverseContext( + value: CollectionOrValue, + params: ContextResolveParams & { rootContext: ConfigContext } +): ContextResolveOutput { + const rootContext = params.rootContext + if (value instanceof UnresolvedTemplateValue) { + const evaluated = value.evaluate({ context: rootContext, opts: params.opts }) + return traverseContext(evaluated.resolved, params) + } + + if (value instanceof ConfigContext) { + const evaluated = value.resolve(params) + return evaluated + } + + const keyPath = params.key + if (keyPath.length > 0) { + const nextKey = params.key[0] + + if (isTemplatePrimitive(value)) { + return { + found: false, + explanation: { + reason: "not_indexable", + key: nextKey, + keyPath: [], + }, + } + } + + const remainder = params.key.slice(1) + + let nextValue: CollectionOrValue + if (isMap(value)) { + nextValue = value.get(nextKey) as CollectionOrValue + } else { + nextValue = value[nextKey] + } + + if (nextValue === undefined) { + return { + found: false, + explanation: { + reason: "key_not_found", + key: nextKey, + keyPath: [], + availableKeys: isMap(value) ? (value.keys() as any).toArray() : Object.keys(value), + }, + } + } + + const result = traverseContext(nextValue, { + ...params, + key: remainder, + }) + + if (result.found) { + return result + } + + return prependKeyPath(result, [nextKey]) + } + + // handles the case when keyPath.length === 0 (here, we need to eagerly resolve everything) + const notFoundValues: ContextResolveOutputNotFound[] = [] + const resolved = deepMap(value, (v, _, deepMapKeyPath) => { + const innerParams = { + ...params, + // we ask nested values to be fully resolved recursively + key: [], + } + if (v instanceof UnresolvedTemplateValue || v instanceof ConfigContext) { + const res = traverseContext(v, innerParams) + + if (res.found) { + return res.resolved + } + + notFoundValues.push(prependKeyPath(res, params.key.concat(deepMapKeyPath))) + + return undefined + } + + return v + }) + + if (notFoundValues.length > 0) { + return notFoundValues[0] + } + + return { + found: true, + resolved, + } +} + +function prependKeyPath( + res: ContextResolveOutputNotFound, + keyPathToPrepend: (string | number)[] +): ContextResolveOutputNotFound { + return { + ...res, + explanation: { + ...res.explanation, + keyPath: [...keyPathToPrepend, ...res.explanation.keyPath], + }, + } +} + +export function getUnavailableReason(result: ContextResolveOutput): string { + if (result.found) { + throw new InternalError({ + message: "called getUnavailableReason on key where found=true", + }) + } + + if (result.explanation.reason === "not_indexable") { + return `Cannot lookup key ${result.explanation.key} on primitive value ${renderKeyPath(result.explanation.keyPath)}.` + } + + const available = result.explanation.availableKeys + + return deline` + Could not find key ${result.explanation.key}${result.explanation.keyPath.length > 0 ? ` under ${renderKeyPath(result.explanation.keyPath)}` : ""}. + ${available?.length ? `Available keys: ${available.join(", ")}.` : ""} + ` +} diff --git a/core/src/config/template-contexts/module.ts b/core/src/config/template-contexts/module.ts index 347d65c030..bfce30972b 100644 --- a/core/src/config/template-contexts/module.ts +++ b/core/src/config/template-contexts/module.ts @@ -14,7 +14,8 @@ import { joi } from "../common.js" import { deline } from "../../util/string.js" import { getModuleTypeUrl } from "../../docs/common.js" import type { GardenModule } from "../../types/module.js" -import { ConfigContext, schema, ErrorContext, ParentContext, TemplateContext } from "./base.js" +import type { ConfigContext } from "./base.js" +import { ContextWithSchema, schema, ErrorContext, ParentContext, TemplateContext } from "./base.js" import { ProviderConfigContext } from "./provider.js" import type { GraphResultFromTask, GraphResults } from "../../graph/results.js" import type { DeployTask } from "../../tasks/deploy.js" @@ -25,13 +26,12 @@ import { styles } from "../../logger/styles.js" export const exampleVersion = "v-17ad4cb3fd" export interface ModuleThisContextParams { - root: ConfigContext buildPath: string name: string path: string } -class ModuleThisContext extends ConfigContext { +class ModuleThisContext extends ContextWithSchema { @schema( joi .string() @@ -53,8 +53,8 @@ class ModuleThisContext extends ConfigContext { ) public path: string - constructor({ root, buildPath, name, path }: ModuleThisContextParams) { - super(root) + constructor({ buildPath, name, path }: ModuleThisContextParams) { + super() this.buildPath = buildPath this.name = name this.path = path @@ -88,15 +88,15 @@ export class ModuleReferenceContext extends ModuleThisContext { @schema(joi.string().required().description("The current version of the module.").example(exampleVersion)) public version: string - constructor(root: ConfigContext, module: GardenModule) { - super({ root, buildPath: module.buildPath, name: module.name, path: module.path }) + constructor(module: GardenModule) { + super({ buildPath: module.buildPath, name: module.name, path: module.path }) this.outputs = module.outputs this.var = module.variables this.version = module.version.versionString } } -export class ServiceRuntimeContext extends ConfigContext { +export class ServiceRuntimeContext extends ContextWithSchema { @schema( joiIdentifierMap( joiPrimitive().description( @@ -116,8 +116,8 @@ export class ServiceRuntimeContext extends ConfigContext { @schema(joi.string().required().description("The current version of the service.").example(exampleVersion)) public version: string - constructor(root: ConfigContext, outputs: PrimitiveMap, version: string) { - super(root) + constructor(outputs: PrimitiveMap, version: string) { + super() this.outputs = outputs this.version = version } @@ -143,14 +143,14 @@ export class TaskRuntimeContext extends ServiceRuntimeContext { @schema(joi.string().required().description("The current version of the task.").example(exampleVersion)) public override version: string - constructor(root: ConfigContext, outputs: PrimitiveMap, version: string) { - super(root, outputs, version) + constructor(outputs: PrimitiveMap, version: string) { + super(outputs, version) this.outputs = outputs this.version = version } } -class RuntimeConfigContext extends ConfigContext { +class RuntimeConfigContext extends ContextWithSchema { @schema( joiIdentifierMap(ServiceRuntimeContext.getSchema()) .required() @@ -167,8 +167,8 @@ class RuntimeConfigContext extends ConfigContext { ) public tasks: Map - constructor(root: ConfigContext, graphResults?: GraphResults) { - super(root) + constructor(graphResults?: GraphResults) { + super() this.services = new Map() this.tasks = new Map() @@ -177,13 +177,10 @@ class RuntimeConfigContext extends ConfigContext { for (const result of Object.values(graphResults.getMap())) { if (result?.task.type === "deploy" && result.result) { const r = (>result).result! - this.services.set( - result.name, - new ServiceRuntimeContext(this, result.outputs, r.executedAction.versionString()) - ) + this.services.set(result.name, new ServiceRuntimeContext(result.outputs, r.executedAction.versionString())) } else if (result?.task.type === "run") { const r = (>result).result! - this.tasks.set(result.name, new TaskRuntimeContext(this, result.outputs, r.executedAction.versionString())) + this.tasks.set(result.name, new TaskRuntimeContext(result.outputs, r.executedAction.versionString())) } } } @@ -223,10 +220,10 @@ export class OutputConfigContext extends ProviderConfigContext { super(garden, resolvedProviders, variables) this.modules = new Map( - modules.map((config) => <[string, ModuleReferenceContext]>[config.name, new ModuleReferenceContext(this, config)]) + modules.map((config) => <[string, ModuleReferenceContext]>[config.name, new ModuleReferenceContext(config)]) ) - this.runtime = new RuntimeConfigContext(this, graphResults) + this.runtime = new RuntimeConfigContext(graphResults) } } @@ -280,11 +277,11 @@ export class ModuleConfigContext extends OutputConfigContext { this.modules.set(name, new ErrorContext(`Config ${styles.highlight.bold(name)} cannot reference itself.`)) if (parentName && templateName) { - this.parent = new ParentContext(this, parentName) - this.template = new TemplateContext(this, templateName) + this.parent = new ParentContext(parentName) + this.template = new TemplateContext(templateName) } this.inputs = inputs || {} - this.this = new ModuleThisContext({ root: this, buildPath, name, path }) + this.this = new ModuleThisContext({ buildPath, name, path }) } } diff --git a/core/src/config/template-contexts/project.ts b/core/src/config/template-contexts/project.ts index 4ac2e9327c..e4b623a72d 100644 --- a/core/src/config/template-contexts/project.ts +++ b/core/src/config/template-contexts/project.ts @@ -11,8 +11,8 @@ import type { PrimitiveMap, DeepPrimitiveMap } from "../common.js" import { joiIdentifierMap, joiStringMap, joiPrimitive, joiVariables } from "../common.js" import { joi } from "../common.js" import { deline, dedent } from "../../util/string.js" -import type { ContextKeySegment } from "./base.js" -import { schema, ConfigContext, EnvironmentContext, ParentContext, TemplateContext } from "./base.js" +import type { ConfigContext, ContextKeySegment } from "./base.js" +import { schema, ContextWithSchema, EnvironmentContext, ParentContext, TemplateContext } from "./base.js" import type { CommandInfo } from "../../plugin-context.js" import type { Garden } from "../../garden.js" import type { VcsInfo } from "../../vcs/vcs.js" @@ -20,7 +20,7 @@ import type { ActionConfig } from "../../actions/types.js" import type { WorkflowConfig } from "../workflow.js" import { styles } from "../../logger/styles.js" -class LocalContext extends ConfigContext { +class LocalContext extends ContextWithSchema { @schema( joi .string() @@ -83,8 +83,8 @@ class LocalContext extends ConfigContext { ) public usernameLowerCase?: string - constructor(root: ConfigContext, artifactsPath: string, projectRoot: string, username?: string) { - super(root) + constructor(artifactsPath: string, projectRoot: string, username?: string) { + super() this.artifactsPath = artifactsPath this.arch = process.arch this.env = process.env @@ -95,17 +95,17 @@ class LocalContext extends ConfigContext { } } -class ProjectContext extends ConfigContext { +class ProjectContext extends ContextWithSchema { @schema(joi.string().description("The name of the Garden project.").example("my-project")) public name: string - constructor(root: ConfigContext, name: string) { - super(root) + constructor(name: string) { + super() this.name = name } } -class DatetimeContext extends ConfigContext { +class DatetimeContext extends ContextWithSchema { @schema( joi .string() @@ -130,8 +130,8 @@ class DatetimeContext extends ConfigContext { ) public timestamp: number - constructor(root: ConfigContext) { - super(root) + constructor() { + super() const now = new Date() this.now = now.toISOString() @@ -140,7 +140,7 @@ class DatetimeContext extends ConfigContext { } } -class VcsContext extends ConfigContext { +class VcsContext extends ContextWithSchema { @schema( joi .string() @@ -189,15 +189,15 @@ class VcsContext extends ConfigContext { ) public originUrl: string - constructor(root: ConfigContext, info: VcsInfo) { - super(root) + constructor(info: VcsInfo) { + super() this.branch = info.branch this.commitHash = info.commitHash this.originUrl = info.originUrl } } -class CommandContext extends ConfigContext { +class CommandContext extends ContextWithSchema { @schema( joi .string() @@ -225,8 +225,8 @@ class CommandContext extends ConfigContext { ) public params: DeepPrimitiveMap - constructor(root: ConfigContext, commandInfo: CommandInfo) { - super(root) + constructor(commandInfo: CommandInfo) { + super() this.name = commandInfo.name this.params = { ...commandInfo.args, ...commandInfo.opts } } @@ -244,7 +244,7 @@ export interface DefaultEnvironmentContextParams { /** * This context is available for template strings in the `defaultEnvironment` field in project configs. */ -export class DefaultEnvironmentContext extends ConfigContext { +export class DefaultEnvironmentContext extends ContextWithSchema { @schema( LocalContext.getSchema().description( "Context variables that are specific to the currently running environment/machine." @@ -273,11 +273,11 @@ export class DefaultEnvironmentContext extends ConfigContext { commandInfo, }: DefaultEnvironmentContextParams) { super() - this.local = new LocalContext(this, artifactsPath, projectRoot, username) - this.datetime = new DatetimeContext(this) - this.git = new VcsContext(this, vcsInfo) - this.project = new ProjectContext(this, projectName) - this.command = new CommandContext(this, commandInfo) + this.local = new LocalContext(artifactsPath, projectRoot, username) + this.datetime = new DatetimeContext() + this.git = new VcsContext(vcsInfo) + this.project = new ProjectContext(projectName) + this.command = new CommandContext(commandInfo) } } @@ -411,7 +411,7 @@ export class RemoteSourceConfigContext extends EnvironmentConfigContext { }) const fullEnvName = garden.namespace ? `${garden.namespace}.${garden.environmentName}` : garden.environmentName - this.environment = new EnvironmentContext(this, garden.environmentName, fullEnvName, garden.namespace) + this.environment = new EnvironmentContext(garden.environmentName, fullEnvName, garden.namespace) this.variables = this.var = variables } } @@ -441,7 +441,7 @@ export class TemplatableConfigContext extends RemoteSourceConfigContext { constructor(garden: Garden, config: ActionConfig | WorkflowConfig) { super(garden, garden.variables) this.inputs = config.internal.inputs || {} - this.parent = config.internal.parentName ? new ParentContext(this, config.internal.parentName) : undefined - this.template = config.internal.templateName ? new TemplateContext(this, config.internal.templateName) : undefined + this.parent = config.internal.parentName ? new ParentContext(config.internal.parentName) : undefined + this.template = config.internal.templateName ? new TemplateContext(config.internal.templateName) : undefined } } diff --git a/core/src/config/template-contexts/provider.ts b/core/src/config/template-contexts/provider.ts index 412d429f6d..089e02edfb 100644 --- a/core/src/config/template-contexts/provider.ts +++ b/core/src/config/template-contexts/provider.ts @@ -14,10 +14,11 @@ import type { Garden } from "../../garden.js" import { joi } from "../common.js" import { deline } from "../../util/string.js" import { getProviderUrl } from "../../docs/common.js" -import { ConfigContext, schema } from "./base.js" +import type { ConfigContext } from "./base.js" +import { ContextWithSchema, schema } from "./base.js" import { WorkflowConfigContext } from "./workflow.js" -class ProviderContext extends ConfigContext { +class ProviderContext extends ContextWithSchema { @schema( joi .object() @@ -49,8 +50,8 @@ class ProviderContext extends ConfigContext { ) public outputs: PrimitiveMap - constructor(root: ConfigContext, provider: Provider) { - super(root) + constructor(provider: Provider) { + super() this.config = provider.config this.outputs = provider.status.outputs } @@ -67,6 +68,6 @@ export class ProviderConfigContext extends WorkflowConfigContext { constructor(garden: Garden, resolvedProviders: ProviderMap, variables: ConfigContext) { super(garden, variables) - this.providers = new Map(Object.entries(mapValues(resolvedProviders, (p) => new ProviderContext(this, p)))) + this.providers = new Map(Object.entries(mapValues(resolvedProviders, (p) => new ProviderContext(p)))) } } diff --git a/core/src/config/template-contexts/render.ts b/core/src/config/template-contexts/render.ts index b44a2947a7..d61bd7244b 100644 --- a/core/src/config/template-contexts/render.ts +++ b/core/src/config/template-contexts/render.ts @@ -30,8 +30,8 @@ export class RenderTemplateConfigContext extends ProjectConfigContext { params: { parentName: string; templateName: string; inputs: DeepPrimitiveMap } & ProjectConfigContextParams ) { super(params) - this.parent = new ParentContext(this, params.parentName) - this.template = new TemplateContext(this, params.templateName) + this.parent = new ParentContext(params.parentName) + this.template = new TemplateContext(params.templateName) this.inputs = params.inputs } } diff --git a/core/src/config/template-contexts/workflow.ts b/core/src/config/template-contexts/workflow.ts index dfaa828e6e..d0fcb46e1f 100644 --- a/core/src/config/template-contexts/workflow.ts +++ b/core/src/config/template-contexts/workflow.ts @@ -12,7 +12,7 @@ import type { Garden } from "../../garden.js" import { joi } from "../common.js" import { dedent } from "../../util/string.js" import { RemoteSourceConfigContext, TemplatableConfigContext } from "./project.js" -import { schema, ConfigContext, ErrorContext } from "./base.js" +import { schema, ContextWithSchema, ErrorContext } from "./base.js" import type { WorkflowConfig } from "../workflow.js" /** @@ -20,7 +20,7 @@ import type { WorkflowConfig } from "../workflow.js" */ export class WorkflowConfigContext extends RemoteSourceConfigContext {} -class WorkflowStepContext extends ConfigContext { +class WorkflowStepContext extends ContextWithSchema { @schema(joi.string().description("The full output log from the step.")) public log: string @@ -39,8 +39,8 @@ class WorkflowStepContext extends ConfigContext { ) public outputs: DeepPrimitiveMap - constructor(root: ConfigContext, stepResult: WorkflowStepResult) { - super(root) + constructor(stepResult: WorkflowStepResult) { + super() this.log = stepResult.log this.outputs = stepResult.outputs } @@ -100,7 +100,7 @@ export class WorkflowStepConfigContext extends TemplatableConfigContext { ) for (const [name, result] of Object.entries(resolvedSteps)) { - this.steps.set(name, new WorkflowStepContext(this, result)) + this.steps.set(name, new WorkflowStepContext(result)) } } } diff --git a/core/src/exceptions.ts b/core/src/exceptions.ts index ff8c8efae0..b448940186 100644 --- a/core/src/exceptions.ts +++ b/core/src/exceptions.ts @@ -247,10 +247,6 @@ export class ParameterError extends GardenError { type = "parameter" } -export class NotImplementedError extends GardenError { - type = "not-implemented" -} - export class DeploymentError extends GardenError { type = "deployment" } @@ -424,6 +420,8 @@ export class InternalError extends GardenError { } } +export class NotImplementedError extends InternalError {} + export function toGardenError(err: Error | GardenError | string | any): GardenError { if (err instanceof GardenError) { return err diff --git a/core/src/garden.ts b/core/src/garden.ts index e615a7d903..3675234a6a 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -122,7 +122,8 @@ import type { CloudProject, GardenCloudApiFactory } from "./cloud/api.js" import { GardenCloudApi, CloudApiTokenRefreshError } from "./cloud/api.js" import { OutputConfigContext } from "./config/template-contexts/module.js" import { ProviderConfigContext } from "./config/template-contexts/provider.js" -import { CONTEXT_RESOLVE_KEY_NOT_FOUND, GenericContext, type ConfigContext } from "./config/template-contexts/base.js" +import type { ConfigContext } from "./config/template-contexts/base.js" +import { GenericContext, getUnavailableReason, type ContextWithSchema } from "./config/template-contexts/base.js" import { validateSchema, validateWithPath } from "./config/validation.js" import { pMemoizeDecorator } from "./lib/p-memoize.js" import { ModuleGraph } from "./graph/modules.js" @@ -575,7 +576,7 @@ export class Garden { events, }: { provider: Provider - templateContext: ConfigContext | undefined + templateContext: ContextWithSchema | undefined events: PluginEventBroker | undefined }) { return createPluginContext({ @@ -1833,6 +1834,13 @@ export class Garden { mapValues(actionConfigs, (configsForKind) => mapValues(configsForKind, omitInternal)) ) + const variableResolveResult = this.variables.resolve({ key: [], opts: {} }) + if (!variableResolveResult.found) { + throw new ConfigurationError({ + message: `Could not resolve variables: ${getUnavailableReason(variableResolveResult)}`, + }) + } + return { environmentName: this.environmentName, allProviderNames: this.getUnresolvedProviderConfigs().map((p) => p.name), @@ -1841,7 +1849,7 @@ export class Garden { providers: providers.map((p) => !(p instanceof UnresolvedTemplateValue || isTemplatePrimitive(p)) ? omitInternal(p) : p ), - variables: this.variables.resolve({ key: [], nodePath: [], opts: {} }).resolved as any, + variables: variableResolveResult.resolved as any, actionConfigs: filteredActionConfigs, moduleConfigs: moduleConfigs.map(omitInternal), workflowConfigs: sortBy(workflowConfigs.map(omitInternal), "name"), @@ -2357,7 +2365,8 @@ async function getCloudProject({ export function overrideVariables(variables: ConfigContext, overrides: DeepPrimitiveMap): LayeredContext { const transformedOverrides = {} for (const key in overrides) { - if (variables.resolve({ key: [key], nodePath: [], opts: {} }).resolved !== CONTEXT_RESOLVE_KEY_NOT_FOUND) { + const res = variables.resolve({ key: [key], opts: {} }) + if (res.found) { // if the original key itself is a string with a dot, then override that transformedOverrides[key] = overrides[key] } else { diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index 21f2abbec9..16fed10557 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -56,7 +56,7 @@ import { MutableConfigGraph } from "./config-graph.js" import type { ModuleGraph } from "./modules.js" import { isTruthy, type MaybeUndefined } from "../util/util.js" import { minimatch } from "minimatch" -import type { ConfigContext } from "../config/template-contexts/base.js" +import type { ContextWithSchema } from "../config/template-contexts/base.js" import { GenericContext } from "../config/template-contexts/base.js" import type { LinkedSource, LinkedSourceMap } from "../config-store/local.js" import { relative } from "path" @@ -511,9 +511,13 @@ export const processActionConfig = profileAsync(async function processActionConf let variables = await mergeVariables({ basePath: effectiveConfigFileLocation, - variables: new GenericContext(capture(config.variables, - // TODO: What's the correct context here? - garden.getProjectConfigContext()) || {}), + variables: new GenericContext( + capture( + config.variables, + // TODO: What's the correct context here? + garden.getProjectConfigContext() + ) || {} + ), varfiles: config.varfiles, log, }) @@ -942,7 +946,7 @@ function dependenciesFromActionConfig({ config: ActionConfig configsByKey: ActionConfigsByKey definition: MaybeUndefined> - templateContext: ConfigContext + templateContext: ContextWithSchema actionTypes: ActionDefinitionMap }) { const description = describeActionConfig(config) diff --git a/core/src/plugin-context.ts b/core/src/plugin-context.ts index d757ddc5f4..607f01cbb9 100644 --- a/core/src/plugin-context.ts +++ b/core/src/plugin-context.ts @@ -14,7 +14,7 @@ import { providerSchema } from "./config/provider.js" import { deline } from "./util/string.js" import { joi, joiVariables, joiStringMap, joiIdentifier, createSchema } from "./config/common.js" import type { PluginTool } from "./util/ext-tools.js" -import type { ConfigContext, ContextResolveOpts } from "./config/template-contexts/base.js" +import type { ContextWithSchema, ContextResolveOpts } from "./config/template-contexts/base.js" import { resolveTemplateString } from "./template/templated-strings.js" import type { Log } from "./logger/log-entry.js" import { logEntrySchema } from "./plugin/base.js" @@ -207,7 +207,7 @@ export async function createPluginContext({ garden: Garden provider: Provider command: CommandInfo - templateContext: ConfigContext + templateContext: ContextWithSchema events: PluginEventBroker | undefined }): Promise { return { diff --git a/core/src/router/base.ts b/core/src/router/base.ts index 076f93e350..7b524a5abf 100644 --- a/core/src/router/base.ts +++ b/core/src/router/base.ts @@ -18,7 +18,7 @@ import type { } from "../plugin/base.js" import type { GardenPluginSpec, ActionHandler, PluginMap } from "../plugin/plugin.js" import type { PluginContext, PluginEventBroker } from "../plugin-context.js" -import type { ConfigContext } from "../config/template-contexts/base.js" +import type { ContextWithSchema } from "../config/template-contexts/base.js" import type { BaseAction } from "../actions/base.js" import type { ActionKind, BaseActionConfig, Resolved } from "../actions/types.js" import type { @@ -70,7 +70,7 @@ export abstract class BaseRouter { protected async commonParams( handler: WrappedActionHandler | WrappedActionTypeHandler, log: Log, - templateContext: ConfigContext | undefined, + templateContext: ContextWithSchema | undefined, events: PluginEventBroker | undefined ): Promise { const provider = await this.garden.resolveProvider({ log, name: handler.pluginName }) diff --git a/core/src/tasks/publish.ts b/core/src/tasks/publish.ts index 70c40a349e..accd3e42b0 100644 --- a/core/src/tasks/publish.ts +++ b/core/src/tasks/publish.ts @@ -12,7 +12,7 @@ import { BaseActionTask } from "../tasks/base.js" import { resolveTemplateString } from "../template/templated-strings.js" import { joi } from "../config/common.js" import { versionStringPrefix } from "../vcs/vcs.js" -import { ConfigContext, schema } from "../config/template-contexts/base.js" +import { ContextWithSchema, schema } from "../config/template-contexts/base.js" import type { PublishActionResult } from "../plugin/handlers/Build/publish.js" import type { BuildAction } from "../actions/build.js" import type { ActionSpecContextParams } from "../config/template-contexts/actions.js" @@ -144,7 +144,7 @@ export class PublishTask extends BaseActionTask { const action = graph.getBuild("foo") const vars = action["variables"] - expect( - vars.resolve({ key: [], nodePath: [], opts: {}, rootContext: garden.getProjectConfigContext() }).resolved - ).to.eql({ - projectName: garden.projectName, + expect(vars.resolve({ key: [], opts: {} })).to.eql({ + found: true, + resolved: { + projectName: garden.projectName, + }, }) }) @@ -581,9 +582,12 @@ describe("actionConfigsToGraph", () => { const action = graph.getBuild("foo") const vars = action["variables"] - expect( - vars.resolve({ key: [], nodePath: [], opts: {}, rootContext: garden.getProjectConfigContext() }).resolved - ).to.eql({ projectName: "${project.name}" }) + expect(vars.resolve({ key: [], opts: {}, rootContext: garden.getProjectConfigContext() })).to.eql({ + found: true, + resolved: { + projectName: "${project.name}", + }, + }) }) it("loads optional varfiles for the action", async () => { @@ -617,9 +621,10 @@ describe("actionConfigsToGraph", () => { const action = graph.getBuild("foo") const vars = action["variables"] - expect( - vars.resolve({ key: [], nodePath: [], opts: {}, rootContext: garden.getProjectConfigContext() }).resolved - ).to.eql({ projectName: "${project.name}" }) + expect(vars.resolve({ key: [], opts: {}, rootContext: garden.getProjectConfigContext() })).to.eql({ + found: true, + resolved: { projectName: "${project.name}" }, + }) }) it("correctly merges varfile with variables", async () => { @@ -658,9 +663,10 @@ describe("actionConfigsToGraph", () => { const action = graph.getBuild("foo") const vars = action["variables"] - expect( - vars.resolve({ key: [], nodePath: [], opts: {}, rootContext: garden.getProjectConfigContext() }).resolved - ).to.eql({ foo: "FOO", bar: "BAR", baz: "baz" }) + expect(vars.resolve({ key: [], opts: {}, rootContext: garden.getProjectConfigContext() })).to.eql({ + found: true, + resolved: { foo: "FOO", bar: "BAR", baz: "baz" }, + }) }) it("correctly merges varfile with variables when some variables are overridden with --var cli flag", async () => { @@ -715,14 +721,15 @@ describe("actionConfigsToGraph", () => { const action = graph.getBuild("foo") const vars = action["variables"] - expect( - vars.resolve({ key: [], nodePath: [], opts: {}, rootContext: garden.getProjectConfigContext() }).resolved - ).to.eql({ - foo: "NEW_FOO", - bar: "BAR", - baz: "baz", - nested: { - key1: "NEW_KEY_1_VALUE", + expect(vars.resolve({ key: [], opts: {}, rootContext: garden.getProjectConfigContext() })).to.eql({ + found: true, + resolved: { + foo: "NEW_FOO", + bar: "BAR", + baz: "baz", + nested: { + key1: "NEW_KEY_1_VALUE", + }, }, }) } finally { diff --git a/core/test/unit/src/config/template-contexts/base.ts b/core/test/unit/src/config/template-contexts/base.ts index 7baafd9f29..a15404f6e0 100644 --- a/core/test/unit/src/config/template-contexts/base.ts +++ b/core/test/unit/src/config/template-contexts/base.ts @@ -8,47 +8,63 @@ import { expect } from "chai" import stripAnsi from "strip-ansi" -import type { ContextKey, ContextResolveParams } from "../../../../../src/config/template-contexts/base.js" -import { CONTEXT_RESOLVE_KEY_NOT_FOUND } from "../../../../../src/config/template-contexts/base.js" -import { ConfigContext, schema } from "../../../../../src/config/template-contexts/base.js" +import type { + ConfigContext, + ContextKey, + ContextResolveParams, +} from "../../../../../src/config/template-contexts/base.js" +import { + GenericContext, + getUnavailableReason, + LayeredContext, +} from "../../../../../src/config/template-contexts/base.js" +import { ContextWithSchema, schema } from "../../../../../src/config/template-contexts/base.js" import { expectError } from "../../../../helpers.js" import { joi } from "../../../../../src/config/common.js" +import { parseTemplateString } from "../../../../../src/template/templated-strings.js" +import { deepEvaluate } from "../../../../../src/template/evaluate.js" +import { isPlainObject } from "../../../../../src/util/objects.js" +import { InternalError } from "../../../../../src/exceptions.js" +import { parseTemplateCollection } from "../../../../../src/template/templated-collections.js" -type TestValue = string | ConfigContext | TestValues | TestValueFunction -type TestValueFunction = () => TestValue | Promise +type TestValue = string | ConfigContext | TestValues interface TestValues { [key: string]: TestValue } describe("ConfigContext", () => { - class GenericContext extends ConfigContext { - constructor(obj: TestValues, root?: ConfigContext) { - super(root) - this.addValues(obj) + class TestContext extends GenericContext { + constructor(obj: TestValues) { + super(obj) } addValues(obj: TestValues) { - Object.assign(this, obj) + if (!isPlainObject(this.data)) { + throw new InternalError({ + message: "TestContext expects data to be a plain object", + }) + } + Object.assign(this.data, obj) } } describe("resolve", () => { // just a shorthand to aid in testing function resolveKey(c: ConfigContext, key: ContextKey, opts = {}) { - return c.resolve({ key, nodePath: [], opts }) + return c.resolve({ key, opts }) } it("should resolve simple keys", async () => { const c = new GenericContext({ basic: "value" }) - expect(resolveKey(c, ["basic"])).to.eql({ resolved: "value" }) + expect(resolveKey(c, ["basic"])).to.eql({ found: true, resolved: "value" }) }) - it("should return CONTEXT_RESOLVE_KEY_NOT_FOUND for missing key", async () => { + it("should return found: false for missing key", async () => { const c = new GenericContext({}) - const { resolved, getUnavailableReason: message } = resolveKey(c, ["basic"]) - expect(resolved).to.be.equal(CONTEXT_RESOLVE_KEY_NOT_FOUND) - expect(stripAnsi(message!())).to.include("Could not find key basic") + const result = resolveKey(c, ["basic"]) + expect(result.found).to.be.equal(false) + expect(stripAnsi(getUnavailableReason(result))).to.include("Could not find key basic") }) // context("allowPartial=true", () => { @@ -69,203 +85,181 @@ describe("ConfigContext", () => { it("should throw when looking for nested value on primitive", async () => { const c = new GenericContext({ basic: "value" }) - await expectError(() => resolveKey(c, ["basic", "nested"]), "context-resolve") + const result = resolveKey(c, ["basic", "nested"]) + expect(stripAnsi(getUnavailableReason(result))).to.equal("Cannot lookup key nested on primitive value basic.") }) it("should resolve nested keys", async () => { const c = new GenericContext({ nested: { key: "value" } }) - expect(resolveKey(c, ["nested", "key"])).eql({ resolved: "value" }) + expect(resolveKey(c, ["nested", "key"])).eql({ found: true, resolved: "value" }) }) it("should resolve keys on nested contexts", async () => { const c = new GenericContext({ nested: new GenericContext({ key: "value" }), }) - expect(resolveKey(c, ["nested", "key"])).eql({ resolved: "value" }) + expect(resolveKey(c, ["nested", "key"])).eql({ found: true, resolved: "value" }) }) - it("should return CONTEXT_RESOLVE_KEY_NOT_FOUND for missing keys on nested context", async () => { + it("should return found: false for missing keys on nested context", async () => { const c = new GenericContext({ nested: new GenericContext({ key: "value" }), }) - const { resolved, getUnavailableReason: message } = resolveKey(c, ["basic", "bla"]) - expect(resolved).to.be.equal(CONTEXT_RESOLVE_KEY_NOT_FOUND) - expect(stripAnsi(message!())).to.equal("Could not find key basic. Available keys: nested.") - }) - - it("should resolve keys with value behind callable", async () => { - const c = new GenericContext({ basic: () => "value" }) - expect(resolveKey(c, ["basic"])).to.eql({ resolved: "value" }) - }) - - it("should resolve keys on nested contexts where context is behind callable", async () => { - const c = new GenericContext({ - nested: () => new GenericContext({ key: "value" }), - }) - expect(resolveKey(c, ["nested", "key"])).to.eql({ resolved: "value" }) + const result = resolveKey(c, ["basic", "bla"]) + expect(result.found).to.be.equal(false) + expect(stripAnsi(getUnavailableReason(result))).to.equal("Could not find key basic. Available keys: nested.") }) it("should cache resolved values", async () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const nested: any = new GenericContext({ key: "value" }) + const nested = new TestContext({ key: "value" }) const c = new GenericContext({ nested, }) resolveKey(c, ["nested", "key"]) - nested.key = "foo" - - expect(resolveKey(c, ["nested", "key"])).to.eql({ resolved: "value" }) - }) - - it("should throw if resolving a key that's already in the lookup stack", async () => { - const c = new GenericContext({ - nested: new GenericContext({ key: "value" }), + nested.addValues({ + key: "foo", }) - const key = ["nested", "key"] - const keyStack = new Set([key.join(".")]) - await expectError(() => c.resolve({ key, nodePath: [], opts: { keyStack } }), "context-resolve") + + expect(resolveKey(c, ["nested", "key"])).to.eql({ found: true, resolved: "value" }) }) it("should detect a circular reference from a nested context", async () => { - class NestedContext extends ConfigContext { - override resolve({ key, nodePath, opts, rootContext }: ContextResolveParams) { - const circularKey = nodePath.concat(key) - opts.keyStack!.add(circularKey.join(".")) - return c.resolve({ key: circularKey, nodePath: [], opts, rootContext }) + class NestedContext extends ContextWithSchema { + override resolve({ opts, rootContext }: ContextResolveParams) { + return c.resolve({ key: ["nested", "bla"], opts, rootContext }) } } const c = new GenericContext({ nested: new NestedContext(), }) - await expectError(() => resolveKey(c, ["nested", "bla"]), "context-resolve") + await expectError(() => resolveKey(c, ["nested", "bla"]), "crash") }) it("should return helpful message when unable to resolve nested key in map", async () => { - class Context extends ConfigContext { + class Context extends ContextWithSchema { nested: Map - constructor(parent?: ConfigContext) { - super(parent) + constructor() { + super() this.nested = new Map() } } const c = new Context() - const { getUnavailableReason: message } = resolveKey(c, ["nested", "bla"]) - expect(stripAnsi(message!())).to.include("Could not find key bla under nested.") + const result = resolveKey(c, ["nested", "bla"]) + expect(stripAnsi(getUnavailableReason(result))).to.include("Could not find key bla under nested.") }) it("should show helpful error when unable to resolve nested key in object", async () => { - class Context extends ConfigContext { + class Context extends ContextWithSchema { // eslint-disable-next-line @typescript-eslint/no-explicit-any nested: any - constructor(parent?: ConfigContext) { - super(parent) + constructor() { + super() this.nested = {} } } const c = new Context() - const { getUnavailableReason: message } = resolveKey(c, ["nested", "bla"]) - expect(stripAnsi(message!())).to.include("Could not find key bla under nested.") + const result = resolveKey(c, ["nested", "bla"]) + expect(stripAnsi(getUnavailableReason(result))).to.include("Could not find key bla under nested.") }) it("should show helpful error when unable to resolve two-level nested key in object", async () => { - class Context extends ConfigContext { + class Context extends ContextWithSchema { // eslint-disable-next-line @typescript-eslint/no-explicit-any nested: any - constructor(parent?: ConfigContext) { - super(parent) + constructor() { + super() this.nested = { deeper: {} } } } const c = new Context() - const { getUnavailableReason: message } = resolveKey(c, ["nested", "deeper", "bla"]) - expect(stripAnsi(message!())).to.include("Could not find key bla under nested.deeper.") + const result = resolveKey(c, ["nested", "deeper", "bla"]) + expect(stripAnsi(getUnavailableReason(result))).to.include("Could not find key bla under nested.deeper.") }) it("should show helpful error when unable to resolve in nested context", async () => { - class Nested extends ConfigContext {} + class Nested extends ContextWithSchema {} - class Context extends ConfigContext { - nested: ConfigContext + class Context extends ContextWithSchema { + nested: ContextWithSchema - constructor(parent?: ConfigContext) { - super(parent) - this.nested = new Nested(this) + constructor() { + super() + this.nested = new Nested() } } const c = new Context() - const { getUnavailableReason: message } = resolveKey(c, ["nested", "bla"]) - expect(stripAnsi(message!())).to.include("Could not find key bla under nested.") + const result = resolveKey(c, ["nested", "bla"]) + expect(stripAnsi(getUnavailableReason(result))).to.include("Could not find key bla under nested.") }) it("should resolve template strings", async () => { - const c = new GenericContext({ + const c = new TestContext({ foo: "value", }) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const nested: any = new GenericContext({ key: "${foo}" }, c) + const nested = new GenericContext(parseTemplateCollection({ value: { key: "${foo}" }, source: { path: [] } })) c.addValues({ nested }) - expect(resolveKey(c, ["nested", "key"])).to.eql({ resolved: "value" }) + expect(resolveKey(c, ["nested", "key"])).to.eql({ found: true, resolved: "value" }) }) it("should resolve template strings with nested context", async () => { - const c = new GenericContext({ + const c = new TestContext({ foo: "bar", }) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const nested: any = new GenericContext({ key: "${nested.foo}", foo: "value" }, c) + const nested = new GenericContext( + parseTemplateCollection({ value: { key: "${nested.foo}", foo: "value" }, source: { path: [] } }) + ) c.addValues({ nested }) - expect(resolveKey(c, ["nested", "key"])).to.eql({ resolved: "value" }) + expect(resolveKey(c, ["nested", "key"])).to.eql({ found: true, resolved: "value" }) }) it("should detect a self-reference when resolving a template string", async () => { - const c = new GenericContext({ key: "${key}" }) + const c = new GenericContext(parseTemplateCollection({ value: { key: "${key}" }, source: { path: [] } })) await expectError(() => resolveKey(c, ["key"]), "template-string") }) it("should detect a nested self-reference when resolving a template string", async () => { - const c = new GenericContext({ + const c = new TestContext({ foo: "bar", }) - const nested = new GenericContext({ key: "${nested.key}" }, c) + const nested = new GenericContext({ key: "${nested.key}" }) c.addValues({ nested }) await expectError(() => resolveKey(c, ["nested", "key"]), "template-string") }) it("should detect a circular reference when resolving a template string", async () => { - const c = new GenericContext({ + const c = new TestContext({ foo: "bar", }) // eslint-disable-next-line @typescript-eslint/no-explicit-any - const nested: any = new GenericContext({ key: "${nested.foo}", foo: "${nested.key}" }, c) + const nested: any = new GenericContext({ key: "${nested.foo}", foo: "${nested.key}" }) c.addValues({ nested }) await expectError(() => resolveKey(c, ["nested", "key"]), "template-string") }) it("should detect a circular reference when resolving a nested template string", async () => { - const c = new GenericContext({ + const c = new TestContext({ foo: "bar", }) // eslint-disable-next-line @typescript-eslint/no-explicit-any - const nested: any = new GenericContext({ key: "${nested.foo}", foo: "${'${nested.key}'}" }, c) + const nested: any = new GenericContext({ key: "${nested.foo}", foo: "${'${nested.key}'}" }) c.addValues({ nested }) await expectError(() => resolveKey(c, ["nested", "key"]), "template-string") }) it("should detect a circular reference when nested template string resolves to self", async () => { - const c = new GenericContext({ + const c = new TestContext({ foo: "bar", }) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const nested: any = new GenericContext({ key: "${'${nested.key}'}" }, c) + const nested = new GenericContext({ key: "${'${nested.key}'}" }) c.addValues({ nested }) await expectError(() => resolveKey(c, ["nested", "key"]), { contains: @@ -276,12 +270,12 @@ describe("ConfigContext", () => { describe("getSchema", () => { it("should return a Joi object schema with all described attributes", () => { - class Nested extends ConfigContext { + class Nested extends ContextWithSchema { @schema(joi.string().description("Nested description")) nestedKey?: string } - class Context extends ConfigContext { + class Context extends ContextWithSchema { @schema(joi.string().description("Some description")) key?: string @@ -310,3 +304,27 @@ describe("ConfigContext", () => { }) }) }) + +describe("LayeredContext", () => { + it("allows you to merge multiple contexts", () => { + const variables = new LayeredContext( + new GenericContext({ + foo: "foo", + }), + new GenericContext({ + bar: "bar", + }) + ) + + const tpl = parseTemplateString({ rawTemplateString: "${var.foo}-${var.bar}", source: { path: [] } }) + + const res = deepEvaluate(tpl, { + context: new GenericContext({ + var: variables, + }), + opts: {}, + }) + + expect(res).to.eq("foo-bar") + }) +}) diff --git a/core/test/unit/src/config/template-contexts/module.ts b/core/test/unit/src/config/template-contexts/module.ts index 258f2304e0..ff058f16b9 100644 --- a/core/test/unit/src/config/template-contexts/module.ts +++ b/core/test/unit/src/config/template-contexts/module.ts @@ -9,7 +9,6 @@ import { expect } from "chai" import { join } from "path" import { keyBy } from "lodash-es" -import type { ConfigContext } from "../../../../../src/config/template-contexts/base.js" import type { TestGarden } from "../../../../helpers.js" import { makeTestGardenA } from "../../../../helpers.js" import { ModuleConfigContext } from "../../../../../src/config/template-contexts/module.js" @@ -17,13 +16,6 @@ import { WorkflowConfigContext } from "../../../../../src/config/template-contex import type { GardenModule } from "../../../../../src/types/module.js" import type { ConfigGraph } from "../../../../../src/graph/config-graph.js" -type TestValue = string | ConfigContext | TestValues | TestValueFunction -type TestValueFunction = () => TestValue | Promise - -interface TestValues { - [key: string]: TestValue -} - describe("ModuleConfigContext", () => { let garden: TestGarden let graph: ConfigGraph @@ -53,83 +45,83 @@ describe("ModuleConfigContext", () => { it("should resolve local env variables", async () => { process.env.TEST_VARIABLE = "foo" - expect(c.resolve({ key: ["local", "env", "TEST_VARIABLE"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["local", "env", "TEST_VARIABLE"], opts: {} })).to.eql({ resolved: "foo", }) delete process.env.TEST_VARIABLE }) it("should resolve the local arch", async () => { - expect(c.resolve({ key: ["local", "arch"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["local", "arch"], opts: {} })).to.eql({ resolved: process.arch, }) }) it("should resolve the local platform", async () => { - expect(c.resolve({ key: ["local", "platform"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["local", "platform"], opts: {} })).to.eql({ resolved: process.platform, }) }) it("should resolve the environment config", async () => { - expect(c.resolve({ key: ["environment", "name"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["environment", "name"], opts: {} })).to.eql({ resolved: garden.environmentName, }) }) it("should resolve the current git branch", () => { - expect(c.resolve({ key: ["git", "branch"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["git", "branch"], opts: {} })).to.eql({ resolved: garden.vcsInfo.branch, }) }) it("should resolve the path of a module", async () => { const path = join(garden.projectRoot, "module-a") - expect(c.resolve({ key: ["modules", "module-a", "path"], nodePath: [], opts: {} })).to.eql({ resolved: path }) + expect(c.resolve({ key: ["modules", "module-a", "path"], opts: {} })).to.eql({ resolved: path }) }) it("should should resolve the version of a module", async () => { const { versionString } = graph.getModule("module-a").version - expect(c.resolve({ key: ["modules", "module-a", "version"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["modules", "module-a", "version"], opts: {} })).to.eql({ resolved: versionString, }) }) it("should resolve the outputs of a module", async () => { - expect(c.resolve({ key: ["modules", "module-a", "outputs", "foo"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["modules", "module-a", "outputs", "foo"], opts: {} })).to.eql({ resolved: "bar", }) }) it("should resolve this.buildPath", async () => { - expect(c.resolve({ key: ["this", "buildPath"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["this", "buildPath"], opts: {} })).to.eql({ resolved: module.buildPath, }) }) it("should resolve this.path", async () => { - expect(c.resolve({ key: ["this", "path"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["this", "path"], opts: {} })).to.eql({ resolved: module.path, }) }) it("should resolve this.name", async () => { - expect(c.resolve({ key: ["this", "name"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["this", "name"], opts: {} })).to.eql({ resolved: module.name, }) }) it("should resolve a project variable", async () => { - expect(c.resolve({ key: ["variables", "some"], nodePath: [], opts: {} })).to.eql({ resolved: "variable" }) + expect(c.resolve({ key: ["variables", "some"], opts: {} })).to.eql({ resolved: "variable" }) }) it("should resolve a project variable under the var alias", async () => { - expect(c.resolve({ key: ["var", "some"], nodePath: [], opts: {} })).to.eql({ resolved: "variable" }) + expect(c.resolve({ key: ["var", "some"], opts: {} })).to.eql({ resolved: "variable" }) }) context("secrets", () => { it("should resolve a secret", async () => { - expect(c.resolve({ key: ["secrets", "someSecret"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["secrets", "someSecret"], opts: {} })).to.eql({ resolved: "someSecretValue", }) }) @@ -148,41 +140,41 @@ describe("WorkflowConfigContext", () => { it("should resolve local env variables", async () => { process.env.TEST_VARIABLE = "foo" - expect(c.resolve({ key: ["local", "env", "TEST_VARIABLE"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["local", "env", "TEST_VARIABLE"], opts: {} })).to.eql({ resolved: "foo", }) delete process.env.TEST_VARIABLE }) it("should resolve the local platform", async () => { - expect(c.resolve({ key: ["local", "platform"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["local", "platform"], opts: {} })).to.eql({ resolved: process.platform, }) }) it("should resolve the current git branch", () => { - expect(c.resolve({ key: ["git", "branch"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["git", "branch"], opts: {} })).to.eql({ resolved: garden.vcsInfo.branch, }) }) it("should resolve the environment config", async () => { - expect(c.resolve({ key: ["environment", "name"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["environment", "name"], opts: {} })).to.eql({ resolved: garden.environmentName, }) }) it("should resolve a project variable", async () => { - expect(c.resolve({ key: ["variables", "some"], nodePath: [], opts: {} })).to.eql({ resolved: "variable" }) + expect(c.resolve({ key: ["variables", "some"], opts: {} })).to.eql({ resolved: "variable" }) }) it("should resolve a project variable under the var alias", async () => { - expect(c.resolve({ key: ["var", "some"], nodePath: [], opts: {} })).to.eql({ resolved: "variable" }) + expect(c.resolve({ key: ["var", "some"], opts: {} })).to.eql({ resolved: "variable" }) }) context("secrets", () => { it("should resolve a secret", async () => { - expect(c.resolve({ key: ["secrets", "someSecret"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["secrets", "someSecret"], opts: {} })).to.eql({ resolved: "someSecretValue", }) }) diff --git a/core/test/unit/src/config/template-contexts/project.ts b/core/test/unit/src/config/template-contexts/project.ts index e9708f827c..f926e5ee20 100644 --- a/core/test/unit/src/config/template-contexts/project.ts +++ b/core/test/unit/src/config/template-contexts/project.ts @@ -8,19 +8,13 @@ import { expect } from "chai" import stripAnsi from "strip-ansi" -import type { ConfigContext } from "../../../../../src/config/template-contexts/base.js" +import { getUnavailableReason } from "../../../../../src/config/template-contexts/base.js" import { DefaultEnvironmentContext, ProjectConfigContext } from "../../../../../src/config/template-contexts/project.js" import { resolveTemplateString } from "../../../../../src/template/templated-strings.js" import { deline } from "../../../../../src/util/string.js" import type { TestGarden } from "../../../../helpers.js" import { freezeTime, makeTestGardenA } from "../../../../helpers.js" -type TestValue = string | ConfigContext | TestValues | TestValueFunction -type TestValueFunction = () => TestValue | Promise -interface TestValues { - [key: string]: TestValue -} - const vcsInfo = { branch: "main", commitHash: "abcdefgh", @@ -43,37 +37,37 @@ describe("DefaultEnvironmentContext", () => { }) it("should resolve the current git branch", () => { - expect(c.resolve({ key: ["git", "branch"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["git", "branch"], opts: {} })).to.eql({ resolved: garden.vcsInfo.branch, }) }) it("should resolve the current git commit hash", () => { - expect(c.resolve({ key: ["git", "commitHash"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["git", "commitHash"], opts: {} })).to.eql({ resolved: garden.vcsInfo.commitHash, }) }) it("should resolve the current git origin URL", () => { - expect(c.resolve({ key: ["git", "originUrl"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["git", "originUrl"], opts: {} })).to.eql({ resolved: garden.vcsInfo.originUrl, }) }) it("should resolve datetime.now to ISO datetime string", () => { - expect(c.resolve({ key: ["datetime", "now"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["datetime", "now"], opts: {} })).to.eql({ resolved: now.toISOString(), }) }) it("should resolve datetime.today to ISO datetime string", () => { - expect(c.resolve({ key: ["datetime", "today"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["datetime", "today"], opts: {} })).to.eql({ resolved: now.toISOString().slice(0, 10), }) }) it("should resolve datetime.timestamp to Unix timestamp in seconds", () => { - expect(c.resolve({ key: ["datetime", "timestamp"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["datetime", "timestamp"], opts: {} })).to.eql({ resolved: Math.round(now.getTime() / 1000), }) }) @@ -95,7 +89,7 @@ describe("ProjectConfigContext", () => { secrets: {}, commandInfo: { name: "test", args: {}, opts: {} }, }) - expect(c.resolve({ key: ["local", "env", "TEST_VARIABLE"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["local", "env", "TEST_VARIABLE"], opts: {} })).to.eql({ resolved: "value", }) delete process.env.TEST_VARIABLE @@ -113,7 +107,7 @@ describe("ProjectConfigContext", () => { secrets: {}, commandInfo: { name: "test", args: {}, opts: {} }, }) - expect(c.resolve({ key: ["git", "branch"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["git", "branch"], opts: {} })).to.eql({ resolved: "main", }) }) @@ -130,7 +124,7 @@ describe("ProjectConfigContext", () => { secrets: { foo: "banana" }, commandInfo: { name: "test", args: {}, opts: {} }, }) - expect(c.resolve({ key: ["secrets", "foo"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["secrets", "foo"], opts: {} })).to.eql({ resolved: "banana", }) }) @@ -149,9 +143,11 @@ describe("ProjectConfigContext", () => { commandInfo: { name: "test", args: {}, opts: {} }, }) - const { getUnavailableReason: message } = c.resolve({ key: ["secrets", "bar"], nodePath: [], opts: {} }) + const result = c.resolve({ key: ["secrets", "bar"], opts: {} }) - expect(stripAnsi(message!())).to.match(/Please log in via the garden login command to use Garden with secrets/) + expect(stripAnsi(getUnavailableReason(result))).to.match( + /Please log in via the garden login command to use Garden with secrets/ + ) }) context("when logged in", () => { @@ -168,13 +164,13 @@ describe("ProjectConfigContext", () => { commandInfo: { name: "test", args: {}, opts: {} }, }) - const { getUnavailableReason: message } = c.resolve({ key: ["secrets", "bar"], nodePath: [], opts: {} }) + const result = c.resolve({ key: ["secrets", "bar"], opts: {} }) const errMsg = deline` Looks like no secrets have been created for this project and/or environment in Garden Cloud. To create secrets, please visit ${enterpriseDomain} and navigate to the secrets section for this project. ` - expect(stripAnsi(message!())).to.match(new RegExp(errMsg)) + expect(stripAnsi(getUnavailableReason(result))).to.match(new RegExp(errMsg)) }) it("if a non-empty set of secrets was returned by the backend, provide a helpful suggestion", () => { @@ -190,13 +186,13 @@ describe("ProjectConfigContext", () => { commandInfo: { name: "test", args: {}, opts: {} }, }) - const { getUnavailableReason: message } = c.resolve({ key: ["secrets", "bar"], nodePath: [], opts: {} }) + const result = c.resolve({ key: ["secrets", "bar"], opts: {} }) const errMsg = deline` Please make sure that all required secrets for this project exist in Garden Cloud, and are accessible in this environment. ` - expect(stripAnsi(message!())).to.match(new RegExp(errMsg)) + expect(stripAnsi(getUnavailableReason(result))).to.match(new RegExp(errMsg)) }) }) }) @@ -215,9 +211,8 @@ describe("ProjectConfigContext", () => { }) const key = "fiaogsyecgbsjyawecygaewbxrbxajyrgew" - const { getUnavailableReason: message } = c.resolve({ key: ["local", "env", key], nodePath: [], opts: {} }) - - expect(stripAnsi(message!())).to.match( + const result = c.resolve({ key: ["local", "env", key], opts: {} }) + expect(stripAnsi(getUnavailableReason(result))).to.match( /Could not find key fiaogsyecgbsjyawecygaewbxrbxajyrgew under local.env. Available keys: / ) }) @@ -234,7 +229,7 @@ describe("ProjectConfigContext", () => { secrets: {}, commandInfo: { name: "test", args: {}, opts: {} }, }) - expect(c.resolve({ key: ["local", "arch"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["local", "arch"], opts: {} })).to.eql({ resolved: process.arch, }) }) @@ -251,7 +246,7 @@ describe("ProjectConfigContext", () => { secrets: {}, commandInfo: { name: "test", args: {}, opts: {} }, }) - expect(c.resolve({ key: ["local", "platform"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["local", "platform"], opts: {} })).to.eql({ resolved: process.platform, }) }) @@ -268,10 +263,10 @@ describe("ProjectConfigContext", () => { secrets: {}, commandInfo: { name: "test", args: {}, opts: {} }, }) - expect(c.resolve({ key: ["local", "username"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["local", "username"], opts: {} })).to.eql({ resolved: "SomeUser", }) - expect(c.resolve({ key: ["local", "usernameLowerCase"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["local", "usernameLowerCase"], opts: {} })).to.eql({ resolved: "someuser", }) }) @@ -288,7 +283,7 @@ describe("ProjectConfigContext", () => { secrets: {}, commandInfo: { name: "test", args: {}, opts: {} }, }) - expect(c.resolve({ key: ["command", "name"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["command", "name"], opts: {} })).to.eql({ resolved: "test", }) }) diff --git a/core/test/unit/src/config/template-contexts/provider.ts b/core/test/unit/src/config/template-contexts/provider.ts index 2059c3ae62..5d1fef742b 100644 --- a/core/test/unit/src/config/template-contexts/provider.ts +++ b/core/test/unit/src/config/template-contexts/provider.ts @@ -7,30 +7,23 @@ */ import { expect } from "chai" -import type { ConfigContext } from "../../../../../src/config/template-contexts/base.js" import { projectRootA, makeTestGarden } from "../../../../helpers.js" import { ProviderConfigContext } from "../../../../../src/config/template-contexts/provider.js" -type TestValue = string | ConfigContext | TestValues | TestValueFunction -type TestValueFunction = () => TestValue | Promise -interface TestValues { - [key: string]: TestValue -} - describe("ProviderConfigContext", () => { it("should set an empty namespace and environment.fullName to environment.name if no namespace is set", async () => { const garden = await makeTestGarden(projectRootA, { environmentString: "local" }) const c = new ProviderConfigContext(garden, await garden.resolveProviders({ log: garden.log }), garden.variables) - expect(c.resolve({ key: ["environment", "name"], nodePath: [], opts: {} })).to.eql({ resolved: "local" }) + expect(c.resolve({ key: ["environment", "name"], opts: {} })).to.eql({ resolved: "local" }) }) it("should set environment.namespace and environment.fullName to properly if namespace is set", async () => { const garden = await makeTestGarden(projectRootA, { environmentString: "foo.local" }) const c = new ProviderConfigContext(garden, await garden.resolveProviders({ log: garden.log }), garden.variables) - expect(c.resolve({ key: ["environment", "name"], nodePath: [], opts: {} })).to.eql({ resolved: "local" }) - expect(c.resolve({ key: ["environment", "namespace"], nodePath: [], opts: {} })).to.eql({ resolved: "foo" }) - expect(c.resolve({ key: ["environment", "fullName"], nodePath: [], opts: {} })).to.eql({ resolved: "foo.local" }) + expect(c.resolve({ key: ["environment", "name"], opts: {} })).to.eql({ resolved: "local" }) + expect(c.resolve({ key: ["environment", "namespace"], opts: {} })).to.eql({ resolved: "foo" }) + expect(c.resolve({ key: ["environment", "fullName"], opts: {} })).to.eql({ resolved: "foo.local" }) }) }) diff --git a/core/test/unit/src/config/template-contexts/workflow.ts b/core/test/unit/src/config/template-contexts/workflow.ts index 543f55c710..56633d762c 100644 --- a/core/test/unit/src/config/template-contexts/workflow.ts +++ b/core/test/unit/src/config/template-contexts/workflow.ts @@ -7,7 +7,6 @@ */ import { expect } from "chai" -import type { ConfigContext } from "../../../../../src/config/template-contexts/base.js" import type { TestGarden } from "../../../../helpers.js" import { expectError, makeTestGardenA } from "../../../../helpers.js" import { @@ -18,13 +17,6 @@ import type { WorkflowConfig } from "../../../../../src/config/workflow.js" import { defaultWorkflowResources } from "../../../../../src/config/workflow.js" import { GardenApiVersion } from "../../../../../src/constants.js" -type TestValue = string | ConfigContext | TestValues | TestValueFunction -type TestValueFunction = () => TestValue | Promise - -interface TestValues { - [key: string]: TestValue -} - describe("WorkflowConfigContext", () => { let garden: TestGarden let c: WorkflowConfigContext @@ -37,41 +29,41 @@ describe("WorkflowConfigContext", () => { it("should resolve local env variables", async () => { process.env.TEST_VARIABLE = "foo" - expect(c.resolve({ key: ["local", "env", "TEST_VARIABLE"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["local", "env", "TEST_VARIABLE"], opts: {} })).to.eql({ resolved: "foo", }) delete process.env.TEST_VARIABLE }) it("should resolve the local arch", async () => { - expect(c.resolve({ key: ["local", "arch"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["local", "arch"], opts: {} })).to.eql({ resolved: process.arch, }) }) it("should resolve the local platform", async () => { - expect(c.resolve({ key: ["local", "platform"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["local", "platform"], opts: {} })).to.eql({ resolved: process.platform, }) }) it("should resolve the environment config", async () => { - expect(c.resolve({ key: ["environment", "name"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["environment", "name"], opts: {} })).to.eql({ resolved: garden.environmentName, }) }) it("should resolve a project variable", async () => { - expect(c.resolve({ key: ["variables", "some"], nodePath: [], opts: {} })).to.eql({ resolved: "variable" }) + expect(c.resolve({ key: ["variables", "some"], opts: {} })).to.eql({ resolved: "variable" }) }) it("should resolve a project variable under the var alias", async () => { - expect(c.resolve({ key: ["var", "some"], nodePath: [], opts: {} })).to.eql({ resolved: "variable" }) + expect(c.resolve({ key: ["var", "some"], opts: {} })).to.eql({ resolved: "variable" }) }) context("secrets", () => { it("should resolve a secret", async () => { - expect(c.resolve({ key: ["secrets", "someSecret"], nodePath: [], opts: {} })).to.eql({ + expect(c.resolve({ key: ["secrets", "someSecret"], opts: {} })).to.eql({ resolved: "someSecretValue", }) }) @@ -112,9 +104,10 @@ describe("WorkflowStepConfigContext", () => { }, stepName: "step-2", }) - expect(c.resolve({ key: ["steps", "step-1", "outputs", "some"], nodePath: [], opts: {} }).resolved).to.equal( - "value" - ) + expect(c.resolve({ key: ["steps", "step-1", "outputs", "some"], opts: {} })).to.equal({ + found: true, + resolved: "value", + }) }) it("should successfully resolve the log from a prior resolved step", () => { @@ -131,7 +124,10 @@ describe("WorkflowStepConfigContext", () => { }, stepName: "step-2", }) - expect(c.resolve({ key: ["steps", "step-1", "log"], nodePath: [], opts: {} }).resolved).to.equal("bla") + expect(c.resolve({ key: ["steps", "step-1", "log"], opts: {} })).to.equal({ + found: true, + resolved: "bla", + }) }) it("should throw error when attempting to reference a following step", () => { @@ -142,7 +138,7 @@ describe("WorkflowStepConfigContext", () => { resolvedSteps: {}, stepName: "step-1", }) - void expectError(() => c.resolve({ key: ["steps", "step-2", "log"], nodePath: [], opts: {} }), { + void expectError(() => c.resolve({ key: ["steps", "step-2", "log"], opts: {} }), { contains: "Step step-2 is referenced in a template for step step-1, but step step-2 is later in the execution order. Only previous steps in the workflow can be referenced.", }) @@ -156,7 +152,7 @@ describe("WorkflowStepConfigContext", () => { resolvedSteps: {}, stepName: "step-1", }) - void expectError(() => c.resolve({ key: ["steps", "step-1", "log"], nodePath: [], opts: {} }), { + void expectError(() => c.resolve({ key: ["steps", "step-1", "log"], opts: {} }), { contains: "Step step-1 references itself in a template. Only previous steps in the workflow can be referenced.", }) }) diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index 776199dcf1..4696ed1f52 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -1735,7 +1735,7 @@ describe("parse and evaluate template strings", () => { errorMessage, }: { template: string - GenericContextVars?: object + GenericContextVars?: GenericContext["data"] errorMessage: string }) { void expectError( From 3c2f4d65a47428ba1b322d0ed8252d9a0199573b Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Mon, 20 Jan 2025 21:14:00 +0100 Subject: [PATCH 034/117] chore: do not pass unresolved action variables to configure handler (to avoid problems joi validation issues) --- core/src/graph/actions.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index 16fed10557..cd5b75c89e 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -844,7 +844,15 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi resolveTemplates() - const configureActionResult = await router.configureAction({ config, log }) + // hack: because variables are partially resolved & that doesn't play well with joi, we do not provide them to the configure handler. + const configureActionResult = await router.configureAction({ + config: { + ...config, + variables: {}, + }, + log, + }) + configureActionResult.config.variables = config.variables const { config: updatedConfig } = configureActionResult From b48bcadd098e795d525fcc21039afbc24af3ff38 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Tue, 21 Jan 2025 13:00:12 +0100 Subject: [PATCH 035/117] chore: fix more tests and reintroduce nodePath --- core/src/config/template-contexts/base.ts | 116 ++++++++++-------- core/src/config/template-contexts/project.ts | 6 +- core/src/garden.ts | 4 +- core/src/template/ast.ts | 44 ++++--- .../src/actions/action-configs-to-graph.ts | 10 +- core/test/unit/src/cli/cli.ts | 4 +- .../unit/src/config/template-contexts/base.ts | 41 ++++--- .../src/config/template-contexts/module.ts | 73 +++++++---- .../src/config/template-contexts/project.ts | 44 ++++--- .../src/config/template-contexts/provider.ts | 8 +- .../src/config/template-contexts/workflow.ts | 22 ++-- 11 files changed, 229 insertions(+), 143 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index b7f70b481f..44ecdde7dc 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -7,7 +7,7 @@ */ import type Joi from "@hapi/joi" -import { ConfigurationError, InternalError } from "../../exceptions.js" +import { ConfigurationError, GardenError, InternalError } from "../../exceptions.js" import type { CustomObjectSchema } from "../common.js" import { joi, joiIdentifier } from "../common.js" import { Profile } from "../../util/profiling.js" @@ -19,6 +19,7 @@ import omitBy from "lodash-es/omitBy.js" import { flatten, isEqual, uniq } from "lodash-es" import { isMap } from "util/types" import { deline } from "../../util/string.js" +import { styles } from "../../logger/styles.js" export type ContextKeySegment = string | number export type ContextKey = ContextKeySegment[] @@ -42,6 +43,11 @@ export interface ContextResolveParams { */ key: ContextKey + /** + * If the context was nested in another context, the key path that led to the inner context. + */ + nodePath: ContextKey + /** * Context lookup options (Deprecated; These mostly affect template string evaluation) */ @@ -67,7 +73,8 @@ export type ContextResolveOutputNotFound = { reason: "key_not_found" | "not_indexable" key: string | number keyPath: (string | number)[] - availableKeys?: (string | number)[] + getAvailableKeys: () => (string | number)[] + getFooterMessage?: () => string } } @@ -103,11 +110,12 @@ export abstract class ConfigContext { this._cache = new Map() } - private detectCircularReference({ key, opts }: ContextResolveParams) { - const keyStr = `${this.constructor.name}(${this._id})-${renderKeyPath(key)}` + private detectCircularReference({ nodePath, key, opts }: ContextResolveParams) { + const plainKey = renderKeyPath(key) + const keyStr = `${this.constructor.name}(${this._id})-${plainKey}` if (opts.stack?.includes(keyStr)) { - throw new ConfigurationError({ - message: `Circular reference detected: ${opts.stack.map((s) => s.split("-")[1]).join(" -> ")}`, + throw new ContextCircularlyReferencesItself({ + message: `Circular reference detected when resolving key ${styles.highlight(renderKeyPath([...nodePath, ...key]))}`, }) } return keyStr @@ -131,8 +139,15 @@ export abstract class ConfigContext { res = this.resolveImpl(params) if (res.found) { this._cache.set(key, res) + return res + } + return { + found: false, + explanation: { + ...res.explanation, + getFooterMessage: () => this.getMissingKeyErrorFooter(params), + }, } - return res } finally { params.opts.stack.pop() } @@ -142,7 +157,7 @@ export abstract class ConfigContext { * Override this method to add more context to error messages thrown in the `resolve` method when a missing key is * referenced. */ - protected getMissingKeyErrorFooter(_key: ContextKeySegment, _path: ContextKeySegment[]): string { + protected getMissingKeyErrorFooter(_params: ContextResolveParams): string { return "" } } @@ -342,7 +357,7 @@ export class LayeredContext extends ConfigContext { ...lastError, explanation: { ...lastError.explanation, - availableKeys: uniq(flatten(all.map((res) => res.explanation.availableKeys || []))), + getAvailableKeys: () => uniq(flatten(all.map((res) => res.explanation.getAvailableKeys()))), }, } } @@ -364,6 +379,20 @@ export class LayeredContext extends ConfigContext { } } +export abstract class ContextResolveError extends GardenError { + type = "context-resolve" +} + +/** + * Occurs when looking up a key in a context turns out to be circular. + */ +export class ContextCircularlyReferencesItself extends ContextResolveError {} + +/** + * Occurs when attempting to look up a key on non-primitive values. + */ +export class ContextLookupNotIndexable extends ContextResolveError {} + function traverseContext( value: CollectionOrValue, params: ContextResolveParams & { rootContext: ConfigContext } @@ -376,6 +405,7 @@ function traverseContext( if (value instanceof ConfigContext) { const evaluated = value.resolve(params) + // no need to recurse, as the nested context took care of recursing if needed return evaluated } @@ -384,23 +414,21 @@ function traverseContext( const nextKey = params.key[0] if (isTemplatePrimitive(value)) { - return { - found: false, - explanation: { - reason: "not_indexable", - key: nextKey, - keyPath: [], - }, - } + throw new ContextLookupNotIndexable({ + message: `Attempting to look up key ${nextKey} on primitive value ${renderKeyPath([...params.nodePath, ...params.key])}`, + }) } const remainder = params.key.slice(1) let nextValue: CollectionOrValue + let getAvailableKeys: () => (string | number)[] if (isMap(value)) { nextValue = value.get(nextKey) as CollectionOrValue + getAvailableKeys = () => Array.from(value.keys()).filter((k) => typeof k === "string" || typeof k === "number") } else { nextValue = value[nextKey] + getAvailableKeys = () => Object.keys(value) } if (nextValue === undefined) { @@ -409,40 +437,47 @@ function traverseContext( explanation: { reason: "key_not_found", key: nextKey, - keyPath: [], - availableKeys: isMap(value) ? (value.keys() as any).toArray() : Object.keys(value), + keyPath: params.nodePath, + getAvailableKeys, }, } } - const result = traverseContext(nextValue, { + const nodePath = [...params.nodePath, nextKey] + return traverseContext(nextValue, { ...params, + nodePath, key: remainder, }) + } - if (result.found) { - return result - } + // from now on we handle the case when keyPath.length === 0 - return prependKeyPath(result, [nextKey]) + if (isTemplatePrimitive(value)) { + return { + found: true, + resolved: value, + } } - // handles the case when keyPath.length === 0 (here, we need to eagerly resolve everything) const notFoundValues: ContextResolveOutputNotFound[] = [] + + // we are handling the case here, where the user looks up a collection of context keys, e.g. ${YAMLEncode(var)} const resolved = deepMap(value, (v, _, deepMapKeyPath) => { - const innerParams = { + const innerTraverseParams = { ...params, + nodePath: [...params.nodePath, ...deepMapKeyPath], // we ask nested values to be fully resolved recursively key: [], } if (v instanceof UnresolvedTemplateValue || v instanceof ConfigContext) { - const res = traverseContext(v, innerParams) + const res = traverseContext(v, innerTraverseParams) if (res.found) { return res.resolved } - notFoundValues.push(prependKeyPath(res, params.key.concat(deepMapKeyPath))) + notFoundValues.push(res) return undefined } @@ -460,19 +495,6 @@ function traverseContext( } } -function prependKeyPath( - res: ContextResolveOutputNotFound, - keyPathToPrepend: (string | number)[] -): ContextResolveOutputNotFound { - return { - ...res, - explanation: { - ...res.explanation, - keyPath: [...keyPathToPrepend, ...res.explanation.keyPath], - }, - } -} - export function getUnavailableReason(result: ContextResolveOutput): string { if (result.found) { throw new InternalError({ @@ -480,14 +502,12 @@ export function getUnavailableReason(result: ContextResolveOutput): string { }) } - if (result.explanation.reason === "not_indexable") { - return `Cannot lookup key ${result.explanation.key} on primitive value ${renderKeyPath(result.explanation.keyPath)}.` - } - - const available = result.explanation.availableKeys + const available = result.explanation.getAvailableKeys() - return deline` + return ( + deline` Could not find key ${result.explanation.key}${result.explanation.keyPath.length > 0 ? ` under ${renderKeyPath(result.explanation.keyPath)}` : ""}. ${available?.length ? `Available keys: ${available.join(", ")}.` : ""} - ` + ` + result.explanation.getFooterMessage?.() || "" + ) } diff --git a/core/src/config/template-contexts/project.ts b/core/src/config/template-contexts/project.ts index e4b623a72d..363517c2dc 100644 --- a/core/src/config/template-contexts/project.ts +++ b/core/src/config/template-contexts/project.ts @@ -11,7 +11,7 @@ import type { PrimitiveMap, DeepPrimitiveMap } from "../common.js" import { joiIdentifierMap, joiStringMap, joiPrimitive, joiVariables } from "../common.js" import { joi } from "../common.js" import { deline, dedent } from "../../util/string.js" -import type { ConfigContext, ContextKeySegment } from "./base.js" +import type { ConfigContext, ContextKeySegment, ContextResolveParams } from "./base.js" import { schema, ContextWithSchema, EnvironmentContext, ParentContext, TemplateContext } from "./base.js" import type { CommandInfo } from "../../plugin-context.js" import type { Garden } from "../../garden.js" @@ -307,8 +307,8 @@ export class ProjectConfigContext extends DefaultEnvironmentContext { private _enterpriseDomain: string | undefined private _loggedIn: boolean - override getMissingKeyErrorFooter(_key: ContextKeySegment, path: ContextKeySegment[]): string { - if (last(path) !== "secrets") { + override getMissingKeyErrorFooter({ nodePath }: ContextResolveParams): string { + if (nodePath[0] !== "secrets") { return "" } diff --git a/core/src/garden.ts b/core/src/garden.ts index 3675234a6a..d71d78ef6c 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -1834,7 +1834,7 @@ export class Garden { mapValues(actionConfigs, (configsForKind) => mapValues(configsForKind, omitInternal)) ) - const variableResolveResult = this.variables.resolve({ key: [], opts: {} }) + const variableResolveResult = this.variables.resolve({ nodePath: [], key: [], opts: {} }) if (!variableResolveResult.found) { throw new ConfigurationError({ message: `Could not resolve variables: ${getUnavailableReason(variableResolveResult)}`, @@ -2365,7 +2365,7 @@ async function getCloudProject({ export function overrideVariables(variables: ConfigContext, overrides: DeepPrimitiveMap): LayeredContext { const transformedOverrides = {} for (const key in overrides) { - const res = variables.resolve({ key: [key], opts: {} }) + const res = variables.resolve({ nodePath: [], key: [key], opts: {} }) if (res.found) { // if the original key itself is a string with a dot, then override that transformedOverrides[key] = overrides[key] diff --git a/core/src/template/ast.ts b/core/src/template/ast.ts index d60964ec3f..6af588a979 100644 --- a/core/src/template/ast.ts +++ b/core/src/template/ast.ts @@ -7,7 +7,7 @@ */ import { isArray, isNumber, isString } from "lodash-es" -import { getUnavailableReason } from "../config/template-contexts/base.js" +import { ContextResolveError, getUnavailableReason } from "../config/template-contexts/base.js" import { InternalError } from "../exceptions.js" import { getHelperFunctions } from "./functions/index.js" import type { EvaluateTemplateArgs } from "./types.js" @@ -688,32 +688,23 @@ export class ContextLookupExpression extends TemplateExpression { super() } - override evaluate({ - context, - opts, - optional, - yamlSource, - }: ASTEvaluateArgs): ASTEvaluationResult> { + override evaluate(args: ASTEvaluateArgs): ASTEvaluationResult> { const keyPath: (string | number)[] = [] for (const k of this.keyPath) { - const evaluated = k.evaluate({ context, opts, optional, yamlSource }) + const evaluated = k.evaluate(args) if (typeof evaluated === "symbol") { return evaluated } keyPath.push(evaluated) } - const result = context.resolve({ - key: keyPath, - // TODO: freeze opts object instead of using shallow copy - opts: { - ...opts, - }, - }) + const result = this.lookup(keyPath, args) // if we encounter a key that could not be found, it's an error unless the optional flag is true, which is used by // logical operators and expressions, as well as the optional suffix in FormatStringExpression. if (!result.found) { + const { optional, yamlSource } = args + if (optional) { return CONTEXT_RESOLVE_KEY_NOT_FOUND } @@ -727,6 +718,29 @@ export class ContextLookupExpression extends TemplateExpression { return result.resolved } + + private lookup(keyPath: (string | number)[], { context, opts, yamlSource }: ASTEvaluateArgs) { + try { + return context.resolve({ + nodePath: [], + key: keyPath, + // TODO: freeze opts object instead of using shallow copy + opts: { + ...opts, + }, + }) + } catch (e) { + if (e instanceof ContextResolveError) { + throw new TemplateStringError({ + message: e.message, + loc: this.loc, + yamlSource, + wrappedErrors: [e], + }) + } + throw e + } + } } export class FunctionCallExpression extends TemplateExpression { diff --git a/core/test/unit/src/actions/action-configs-to-graph.ts b/core/test/unit/src/actions/action-configs-to-graph.ts index e09254704b..31fcb22a10 100644 --- a/core/test/unit/src/actions/action-configs-to-graph.ts +++ b/core/test/unit/src/actions/action-configs-to-graph.ts @@ -543,7 +543,7 @@ describe("actionConfigsToGraph", () => { const action = graph.getBuild("foo") const vars = action["variables"] - expect(vars.resolve({ key: [], opts: {} })).to.eql({ + expect(vars.resolve({ nodePath: [], key: [], opts: {} })).to.eql({ found: true, resolved: { projectName: garden.projectName, @@ -582,7 +582,7 @@ describe("actionConfigsToGraph", () => { const action = graph.getBuild("foo") const vars = action["variables"] - expect(vars.resolve({ key: [], opts: {}, rootContext: garden.getProjectConfigContext() })).to.eql({ + expect(vars.resolve({ nodePath: [], key: [], opts: {}, rootContext: garden.getProjectConfigContext() })).to.eql({ found: true, resolved: { projectName: "${project.name}", @@ -621,7 +621,7 @@ describe("actionConfigsToGraph", () => { const action = graph.getBuild("foo") const vars = action["variables"] - expect(vars.resolve({ key: [], opts: {}, rootContext: garden.getProjectConfigContext() })).to.eql({ + expect(vars.resolve({ nodePath: [], key: [], opts: {}, rootContext: garden.getProjectConfigContext() })).to.eql({ found: true, resolved: { projectName: "${project.name}" }, }) @@ -663,7 +663,7 @@ describe("actionConfigsToGraph", () => { const action = graph.getBuild("foo") const vars = action["variables"] - expect(vars.resolve({ key: [], opts: {}, rootContext: garden.getProjectConfigContext() })).to.eql({ + expect(vars.resolve({ nodePath: [], key: [], opts: {}, rootContext: garden.getProjectConfigContext() })).to.eql({ found: true, resolved: { foo: "FOO", bar: "BAR", baz: "baz" }, }) @@ -721,7 +721,7 @@ describe("actionConfigsToGraph", () => { const action = graph.getBuild("foo") const vars = action["variables"] - expect(vars.resolve({ key: [], opts: {}, rootContext: garden.getProjectConfigContext() })).to.eql({ + expect(vars.resolve({ nodePath: [], key: [], opts: {}, rootContext: garden.getProjectConfigContext() })).to.eql({ found: true, resolved: { foo: "NEW_FOO", diff --git a/core/test/unit/src/cli/cli.ts b/core/test/unit/src/cli/cli.ts index 93ad2ec5c7..a43f37cb9e 100644 --- a/core/test/unit/src/cli/cli.ts +++ b/core/test/unit/src/cli/cli.ts @@ -35,7 +35,7 @@ import fsExtra from "fs-extra" const { mkdirp } = fsExtra import { uuidv4 } from "../../../../src/util/random.js" -import { makeDummyGarden } from "../../../../src/garden.js" +import { Garden, makeDummyGarden } from "../../../../src/garden.js" import { TestGardenCli } from "../../../helpers/cli.js" import { NotImplementedError } from "../../../../src/exceptions.js" import dedent from "dedent" @@ -960,7 +960,7 @@ describe("cli", () => { override printHeader() {} - async action({ garden }) { + async action({ garden }: { garden: Garden }) { return { result: { variables: garden.variables } } } } diff --git a/core/test/unit/src/config/template-contexts/base.ts b/core/test/unit/src/config/template-contexts/base.ts index a15404f6e0..59329f124a 100644 --- a/core/test/unit/src/config/template-contexts/base.ts +++ b/core/test/unit/src/config/template-contexts/base.ts @@ -52,7 +52,7 @@ describe("ConfigContext", () => { describe("resolve", () => { // just a shorthand to aid in testing function resolveKey(c: ConfigContext, key: ContextKey, opts = {}) { - return c.resolve({ key, opts }) + return c.resolve({ nodePath: [], key, opts }) } it("should resolve simple keys", async () => { @@ -85,8 +85,7 @@ describe("ConfigContext", () => { it("should throw when looking for nested value on primitive", async () => { const c = new GenericContext({ basic: "value" }) - const result = resolveKey(c, ["basic", "nested"]) - expect(stripAnsi(getUnavailableReason(result))).to.equal("Cannot lookup key nested on primitive value basic.") + await expectError(() => resolveKey(c, ["basic", "nested"]), "context-resolve") }) it("should resolve nested keys", async () => { @@ -125,16 +124,24 @@ describe("ConfigContext", () => { }) it("should detect a circular reference from a nested context", async () => { - class NestedContext extends ContextWithSchema { + class NestedContextOne extends ContextWithSchema { override resolve({ opts, rootContext }: ContextResolveParams) { - return c.resolve({ key: ["nested", "bla"], opts, rootContext }) + return c.resolve({ nodePath: [], key: ["nestedOne", "bla"], opts, rootContext }) } } + const nestedTwo = new TestContext({}) + nestedTwo.addValues({ + bla: nestedTwo, + }) + const c = new GenericContext({ - nested: new NestedContext(), + nestedOne: new NestedContextOne(), + nestedTwo, }) - await expectError(() => resolveKey(c, ["nested", "bla"]), "crash") + + await expectError(() => resolveKey(c, ["nestedOne", "bla"]), "context-resolve") + await expectError(() => resolveKey(c, ["nestedTwo", "bla"]), "context-resolve") }) it("should return helpful message when unable to resolve nested key in map", async () => { @@ -230,7 +237,9 @@ describe("ConfigContext", () => { const c = new TestContext({ foo: "bar", }) - const nested = new GenericContext({ key: "${nested.key}" }) + const nested = new GenericContext( + parseTemplateCollection({ value: { key: "${nested.key}" }, source: { path: [] } }) + ) c.addValues({ nested }) await expectError(() => resolveKey(c, ["nested", "key"]), "template-string") }) @@ -239,8 +248,9 @@ describe("ConfigContext", () => { const c = new TestContext({ foo: "bar", }) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const nested: any = new GenericContext({ key: "${nested.foo}", foo: "${nested.key}" }) + const nested = new GenericContext( + parseTemplateCollection({ value: { key: "${nested.foo}", foo: "${nested.key}" }, source: { path: [] } }) + ) c.addValues({ nested }) await expectError(() => resolveKey(c, ["nested", "key"]), "template-string") }) @@ -249,8 +259,9 @@ describe("ConfigContext", () => { const c = new TestContext({ foo: "bar", }) - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const nested: any = new GenericContext({ key: "${nested.foo}", foo: "${'${nested.key}'}" }) + const nested = new GenericContext( + parseTemplateCollection({ value: { key: "${nested.foo}", foo: "${'${nested.key}'}" }, source: { path: [] } }) + ) c.addValues({ nested }) await expectError(() => resolveKey(c, ["nested", "key"]), "template-string") }) @@ -259,11 +270,13 @@ describe("ConfigContext", () => { const c = new TestContext({ foo: "bar", }) - const nested = new GenericContext({ key: "${'${nested.key}'}" }) + const nested = new GenericContext( + parseTemplateCollection({ value: { key: "${'${nested.key}'}" }, source: { path: [] } }) + ) c.addValues({ nested }) await expectError(() => resolveKey(c, ["nested", "key"]), { contains: - "Invalid template string (${nested.key}): Circular reference detected when resolving key nested.key (nested -> nested.key)", + "Invalid template string (${nested.key}) at path key: Circular reference detected when resolving key nested.key", }) }) }) diff --git a/core/test/unit/src/config/template-contexts/module.ts b/core/test/unit/src/config/template-contexts/module.ts index ff058f16b9..d76b441e99 100644 --- a/core/test/unit/src/config/template-contexts/module.ts +++ b/core/test/unit/src/config/template-contexts/module.ts @@ -45,83 +45,103 @@ describe("ModuleConfigContext", () => { it("should resolve local env variables", async () => { process.env.TEST_VARIABLE = "foo" - expect(c.resolve({ key: ["local", "env", "TEST_VARIABLE"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["local", "env", "TEST_VARIABLE"], opts: {} })).to.eql({ + found: true, resolved: "foo", }) delete process.env.TEST_VARIABLE }) it("should resolve the local arch", async () => { - expect(c.resolve({ key: ["local", "arch"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["local", "arch"], opts: {} })).to.eql({ + found: true, resolved: process.arch, }) }) it("should resolve the local platform", async () => { - expect(c.resolve({ key: ["local", "platform"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["local", "platform"], opts: {} })).to.eql({ + found: true, resolved: process.platform, }) }) it("should resolve the environment config", async () => { - expect(c.resolve({ key: ["environment", "name"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["environment", "name"], opts: {} })).to.eql({ + found: true, resolved: garden.environmentName, }) }) it("should resolve the current git branch", () => { - expect(c.resolve({ key: ["git", "branch"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["git", "branch"], opts: {} })).to.eql({ + found: true, resolved: garden.vcsInfo.branch, }) }) it("should resolve the path of a module", async () => { const path = join(garden.projectRoot, "module-a") - expect(c.resolve({ key: ["modules", "module-a", "path"], opts: {} })).to.eql({ resolved: path }) + expect(c.resolve({ nodePath: [], key: ["modules", "module-a", "path"], opts: {} })).to.eql({ + found: true, + resolved: path, + }) }) it("should should resolve the version of a module", async () => { const { versionString } = graph.getModule("module-a").version - expect(c.resolve({ key: ["modules", "module-a", "version"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["modules", "module-a", "version"], opts: {} })).to.eql({ + found: true, resolved: versionString, }) }) it("should resolve the outputs of a module", async () => { - expect(c.resolve({ key: ["modules", "module-a", "outputs", "foo"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["modules", "module-a", "outputs", "foo"], opts: {} })).to.eql({ + found: true, resolved: "bar", }) }) it("should resolve this.buildPath", async () => { - expect(c.resolve({ key: ["this", "buildPath"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["this", "buildPath"], opts: {} })).to.eql({ + found: true, resolved: module.buildPath, }) }) it("should resolve this.path", async () => { - expect(c.resolve({ key: ["this", "path"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["this", "path"], opts: {} })).to.eql({ + found: true, resolved: module.path, }) }) it("should resolve this.name", async () => { - expect(c.resolve({ key: ["this", "name"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["this", "name"], opts: {} })).to.eql({ + found: true, resolved: module.name, }) }) it("should resolve a project variable", async () => { - expect(c.resolve({ key: ["variables", "some"], opts: {} })).to.eql({ resolved: "variable" }) + expect(c.resolve({ nodePath: [], key: ["variables", "some"], opts: {} })).to.eql({ + found: true, + resolved: "variable", + }) }) it("should resolve a project variable under the var alias", async () => { - expect(c.resolve({ key: ["var", "some"], opts: {} })).to.eql({ resolved: "variable" }) + expect(c.resolve({ nodePath: [], key: ["var", "some"], opts: {} })).to.eql({ + found: true, + resolved: "variable", + }) }) context("secrets", () => { it("should resolve a secret", async () => { - expect(c.resolve({ key: ["secrets", "someSecret"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["secrets", "someSecret"], opts: {} })).to.eql({ + found: true, resolved: "someSecretValue", }) }) @@ -140,41 +160,52 @@ describe("WorkflowConfigContext", () => { it("should resolve local env variables", async () => { process.env.TEST_VARIABLE = "foo" - expect(c.resolve({ key: ["local", "env", "TEST_VARIABLE"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["local", "env", "TEST_VARIABLE"], opts: {} })).to.eql({ + found: true, resolved: "foo", }) delete process.env.TEST_VARIABLE }) it("should resolve the local platform", async () => { - expect(c.resolve({ key: ["local", "platform"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["local", "platform"], opts: {} })).to.eql({ + found: true, resolved: process.platform, }) }) it("should resolve the current git branch", () => { - expect(c.resolve({ key: ["git", "branch"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["git", "branch"], opts: {} })).to.eql({ + found: true, resolved: garden.vcsInfo.branch, }) }) it("should resolve the environment config", async () => { - expect(c.resolve({ key: ["environment", "name"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["environment", "name"], opts: {} })).to.eql({ + found: true, resolved: garden.environmentName, }) }) it("should resolve a project variable", async () => { - expect(c.resolve({ key: ["variables", "some"], opts: {} })).to.eql({ resolved: "variable" }) + expect(c.resolve({ nodePath: [], key: ["variables", "some"], opts: {} })).to.eql({ + found: true, + resolved: "variable", + }) }) it("should resolve a project variable under the var alias", async () => { - expect(c.resolve({ key: ["var", "some"], opts: {} })).to.eql({ resolved: "variable" }) + expect(c.resolve({ nodePath: [], key: ["var", "some"], opts: {} })).to.eql({ + found: true, + resolved: "variable", + }) }) context("secrets", () => { it("should resolve a secret", async () => { - expect(c.resolve({ key: ["secrets", "someSecret"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["secrets", "someSecret"], opts: {} })).to.eql({ + found: true, resolved: "someSecretValue", }) }) diff --git a/core/test/unit/src/config/template-contexts/project.ts b/core/test/unit/src/config/template-contexts/project.ts index f926e5ee20..a689bbf259 100644 --- a/core/test/unit/src/config/template-contexts/project.ts +++ b/core/test/unit/src/config/template-contexts/project.ts @@ -37,37 +37,37 @@ describe("DefaultEnvironmentContext", () => { }) it("should resolve the current git branch", () => { - expect(c.resolve({ key: ["git", "branch"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["git", "branch"], opts: {} })).to.eql({ resolved: garden.vcsInfo.branch, }) }) it("should resolve the current git commit hash", () => { - expect(c.resolve({ key: ["git", "commitHash"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["git", "commitHash"], opts: {} })).to.eql({ resolved: garden.vcsInfo.commitHash, }) }) it("should resolve the current git origin URL", () => { - expect(c.resolve({ key: ["git", "originUrl"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["git", "originUrl"], opts: {} })).to.eql({ resolved: garden.vcsInfo.originUrl, }) }) it("should resolve datetime.now to ISO datetime string", () => { - expect(c.resolve({ key: ["datetime", "now"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["datetime", "now"], opts: {} })).to.eql({ resolved: now.toISOString(), }) }) it("should resolve datetime.today to ISO datetime string", () => { - expect(c.resolve({ key: ["datetime", "today"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["datetime", "today"], opts: {} })).to.eql({ resolved: now.toISOString().slice(0, 10), }) }) it("should resolve datetime.timestamp to Unix timestamp in seconds", () => { - expect(c.resolve({ key: ["datetime", "timestamp"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["datetime", "timestamp"], opts: {} })).to.eql({ resolved: Math.round(now.getTime() / 1000), }) }) @@ -89,7 +89,8 @@ describe("ProjectConfigContext", () => { secrets: {}, commandInfo: { name: "test", args: {}, opts: {} }, }) - expect(c.resolve({ key: ["local", "env", "TEST_VARIABLE"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["local", "env", "TEST_VARIABLE"], opts: {} })).to.eql({ + found: true, resolved: "value", }) delete process.env.TEST_VARIABLE @@ -107,7 +108,8 @@ describe("ProjectConfigContext", () => { secrets: {}, commandInfo: { name: "test", args: {}, opts: {} }, }) - expect(c.resolve({ key: ["git", "branch"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["git", "branch"], opts: {} })).to.eql({ + found: true, resolved: "main", }) }) @@ -124,7 +126,8 @@ describe("ProjectConfigContext", () => { secrets: { foo: "banana" }, commandInfo: { name: "test", args: {}, opts: {} }, }) - expect(c.resolve({ key: ["secrets", "foo"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["secrets", "foo"], opts: {} })).to.eql({ + found: true, resolved: "banana", }) }) @@ -143,7 +146,7 @@ describe("ProjectConfigContext", () => { commandInfo: { name: "test", args: {}, opts: {} }, }) - const result = c.resolve({ key: ["secrets", "bar"], opts: {} }) + const result = c.resolve({ nodePath: [], key: ["secrets", "bar"], opts: {} }) expect(stripAnsi(getUnavailableReason(result))).to.match( /Please log in via the garden login command to use Garden with secrets/ @@ -164,7 +167,7 @@ describe("ProjectConfigContext", () => { commandInfo: { name: "test", args: {}, opts: {} }, }) - const result = c.resolve({ key: ["secrets", "bar"], opts: {} }) + const result = c.resolve({ nodePath: [], key: ["secrets", "bar"], opts: {} }) const errMsg = deline` Looks like no secrets have been created for this project and/or environment in Garden Cloud. @@ -186,7 +189,7 @@ describe("ProjectConfigContext", () => { commandInfo: { name: "test", args: {}, opts: {} }, }) - const result = c.resolve({ key: ["secrets", "bar"], opts: {} }) + const result = c.resolve({ nodePath: [], key: ["secrets", "bar"], opts: {} }) const errMsg = deline` Please make sure that all required secrets for this project exist in Garden Cloud, and are accessible in this @@ -211,7 +214,7 @@ describe("ProjectConfigContext", () => { }) const key = "fiaogsyecgbsjyawecygaewbxrbxajyrgew" - const result = c.resolve({ key: ["local", "env", key], opts: {} }) + const result = c.resolve({ nodePath: [], key: ["local", "env", key], opts: {} }) expect(stripAnsi(getUnavailableReason(result))).to.match( /Could not find key fiaogsyecgbsjyawecygaewbxrbxajyrgew under local.env. Available keys: / ) @@ -229,7 +232,8 @@ describe("ProjectConfigContext", () => { secrets: {}, commandInfo: { name: "test", args: {}, opts: {} }, }) - expect(c.resolve({ key: ["local", "arch"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["local", "arch"], opts: {} })).to.eql({ + found: true, resolved: process.arch, }) }) @@ -246,7 +250,8 @@ describe("ProjectConfigContext", () => { secrets: {}, commandInfo: { name: "test", args: {}, opts: {} }, }) - expect(c.resolve({ key: ["local", "platform"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["local", "platform"], opts: {} })).to.eql({ + found: true, resolved: process.platform, }) }) @@ -263,10 +268,12 @@ describe("ProjectConfigContext", () => { secrets: {}, commandInfo: { name: "test", args: {}, opts: {} }, }) - expect(c.resolve({ key: ["local", "username"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["local", "username"], opts: {} })).to.eql({ + found: true, resolved: "SomeUser", }) - expect(c.resolve({ key: ["local", "usernameLowerCase"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["local", "usernameLowerCase"], opts: {} })).to.eql({ + found: true, resolved: "someuser", }) }) @@ -283,7 +290,8 @@ describe("ProjectConfigContext", () => { secrets: {}, commandInfo: { name: "test", args: {}, opts: {} }, }) - expect(c.resolve({ key: ["command", "name"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["command", "name"], opts: {} })).to.eql({ + found: true, resolved: "test", }) }) diff --git a/core/test/unit/src/config/template-contexts/provider.ts b/core/test/unit/src/config/template-contexts/provider.ts index 5d1fef742b..b423c8a77a 100644 --- a/core/test/unit/src/config/template-contexts/provider.ts +++ b/core/test/unit/src/config/template-contexts/provider.ts @@ -15,15 +15,15 @@ describe("ProviderConfigContext", () => { const garden = await makeTestGarden(projectRootA, { environmentString: "local" }) const c = new ProviderConfigContext(garden, await garden.resolveProviders({ log: garden.log }), garden.variables) - expect(c.resolve({ key: ["environment", "name"], opts: {} })).to.eql({ resolved: "local" }) + expect(c.resolve({ nodePath: [], key: ["environment", "name"], opts: {} })).to.eql({ resolved: "local" }) }) it("should set environment.namespace and environment.fullName to properly if namespace is set", async () => { const garden = await makeTestGarden(projectRootA, { environmentString: "foo.local" }) const c = new ProviderConfigContext(garden, await garden.resolveProviders({ log: garden.log }), garden.variables) - expect(c.resolve({ key: ["environment", "name"], opts: {} })).to.eql({ resolved: "local" }) - expect(c.resolve({ key: ["environment", "namespace"], opts: {} })).to.eql({ resolved: "foo" }) - expect(c.resolve({ key: ["environment", "fullName"], opts: {} })).to.eql({ resolved: "foo.local" }) + expect(c.resolve({ nodePath: [], key: ["environment", "name"], opts: {} })).to.eql({ resolved: "local" }) + expect(c.resolve({ nodePath: [], key: ["environment", "namespace"], opts: {} })).to.eql({ resolved: "foo" }) + expect(c.resolve({ nodePath: [], key: ["environment", "fullName"], opts: {} })).to.eql({ resolved: "foo.local" }) }) }) diff --git a/core/test/unit/src/config/template-contexts/workflow.ts b/core/test/unit/src/config/template-contexts/workflow.ts index 56633d762c..e322154c3f 100644 --- a/core/test/unit/src/config/template-contexts/workflow.ts +++ b/core/test/unit/src/config/template-contexts/workflow.ts @@ -29,41 +29,41 @@ describe("WorkflowConfigContext", () => { it("should resolve local env variables", async () => { process.env.TEST_VARIABLE = "foo" - expect(c.resolve({ key: ["local", "env", "TEST_VARIABLE"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["local", "env", "TEST_VARIABLE"], opts: {} })).to.eql({ resolved: "foo", }) delete process.env.TEST_VARIABLE }) it("should resolve the local arch", async () => { - expect(c.resolve({ key: ["local", "arch"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["local", "arch"], opts: {} })).to.eql({ resolved: process.arch, }) }) it("should resolve the local platform", async () => { - expect(c.resolve({ key: ["local", "platform"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["local", "platform"], opts: {} })).to.eql({ resolved: process.platform, }) }) it("should resolve the environment config", async () => { - expect(c.resolve({ key: ["environment", "name"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["environment", "name"], opts: {} })).to.eql({ resolved: garden.environmentName, }) }) it("should resolve a project variable", async () => { - expect(c.resolve({ key: ["variables", "some"], opts: {} })).to.eql({ resolved: "variable" }) + expect(c.resolve({ nodePath: [], key: ["variables", "some"], opts: {} })).to.eql({ resolved: "variable" }) }) it("should resolve a project variable under the var alias", async () => { - expect(c.resolve({ key: ["var", "some"], opts: {} })).to.eql({ resolved: "variable" }) + expect(c.resolve({ nodePath: [], key: ["var", "some"], opts: {} })).to.eql({ resolved: "variable" }) }) context("secrets", () => { it("should resolve a secret", async () => { - expect(c.resolve({ key: ["secrets", "someSecret"], opts: {} })).to.eql({ + expect(c.resolve({ nodePath: [], key: ["secrets", "someSecret"], opts: {} })).to.eql({ resolved: "someSecretValue", }) }) @@ -104,7 +104,7 @@ describe("WorkflowStepConfigContext", () => { }, stepName: "step-2", }) - expect(c.resolve({ key: ["steps", "step-1", "outputs", "some"], opts: {} })).to.equal({ + expect(c.resolve({ nodePath: [], key: ["steps", "step-1", "outputs", "some"], opts: {} })).to.equal({ found: true, resolved: "value", }) @@ -124,7 +124,7 @@ describe("WorkflowStepConfigContext", () => { }, stepName: "step-2", }) - expect(c.resolve({ key: ["steps", "step-1", "log"], opts: {} })).to.equal({ + expect(c.resolve({ nodePath: [], key: ["steps", "step-1", "log"], opts: {} })).to.equal({ found: true, resolved: "bla", }) @@ -138,7 +138,7 @@ describe("WorkflowStepConfigContext", () => { resolvedSteps: {}, stepName: "step-1", }) - void expectError(() => c.resolve({ key: ["steps", "step-2", "log"], opts: {} }), { + void expectError(() => c.resolve({ nodePath: [], key: ["steps", "step-2", "log"], opts: {} }), { contains: "Step step-2 is referenced in a template for step step-1, but step step-2 is later in the execution order. Only previous steps in the workflow can be referenced.", }) @@ -152,7 +152,7 @@ describe("WorkflowStepConfigContext", () => { resolvedSteps: {}, stepName: "step-1", }) - void expectError(() => c.resolve({ key: ["steps", "step-1", "log"], opts: {} }), { + void expectError(() => c.resolve({ nodePath: [], key: ["steps", "step-1", "log"], opts: {} }), { contains: "Step step-1 references itself in a template. Only previous steps in the workflow can be referenced.", }) }) From 54f7cd512a217a7972f84eb8d4cbedae5fc4e0fb Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Tue, 21 Jan 2025 14:12:24 +0100 Subject: [PATCH 036/117] chore: fix all ConfigContext tests --- core/src/config/template-contexts/base.ts | 23 +++++++++++++++---- core/src/config/template-contexts/project.ts | 4 ++-- .../src/config/template-contexts/project.ts | 5 ++-- .../src/config/template-contexts/provider.ts | 20 ++++++++++++---- .../src/config/template-contexts/workflow.ts | 19 +++++++++++---- 5 files changed, 53 insertions(+), 18 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 44ecdde7dc..ecb2f7aa67 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -145,7 +145,14 @@ export abstract class ConfigContext { found: false, explanation: { ...res.explanation, - getFooterMessage: () => this.getMissingKeyErrorFooter(params), + getFooterMessage: () => { + const previousMsg = res.explanation.getFooterMessage?.() + const msg = this.getMissingKeyErrorFooter(params) + if (previousMsg) { + return `${previousMsg}\n${msg}` + } + return msg + }, }, } } finally { @@ -504,10 +511,16 @@ export function getUnavailableReason(result: ContextResolveOutput): string { const available = result.explanation.getAvailableKeys() - return ( - deline` + const message = deline` Could not find key ${result.explanation.key}${result.explanation.keyPath.length > 0 ? ` under ${renderKeyPath(result.explanation.keyPath)}` : ""}. ${available?.length ? `Available keys: ${available.join(", ")}.` : ""} - ` + result.explanation.getFooterMessage?.() || "" - ) + ` + + const footer = result.explanation.getFooterMessage?.() + + if (footer) { + return `${message}\n\n${footer}` + } + + return message } diff --git a/core/src/config/template-contexts/project.ts b/core/src/config/template-contexts/project.ts index 363517c2dc..fbe9682875 100644 --- a/core/src/config/template-contexts/project.ts +++ b/core/src/config/template-contexts/project.ts @@ -307,8 +307,8 @@ export class ProjectConfigContext extends DefaultEnvironmentContext { private _enterpriseDomain: string | undefined private _loggedIn: boolean - override getMissingKeyErrorFooter({ nodePath }: ContextResolveParams): string { - if (nodePath[0] !== "secrets") { + override getMissingKeyErrorFooter({ key }: ContextResolveParams): string { + if (key[0] !== "secrets") { return "" } diff --git a/core/test/unit/src/config/template-contexts/project.ts b/core/test/unit/src/config/template-contexts/project.ts index a689bbf259..7021223177 100644 --- a/core/test/unit/src/config/template-contexts/project.ts +++ b/core/test/unit/src/config/template-contexts/project.ts @@ -148,9 +148,8 @@ describe("ProjectConfigContext", () => { const result = c.resolve({ nodePath: [], key: ["secrets", "bar"], opts: {} }) - expect(stripAnsi(getUnavailableReason(result))).to.match( - /Please log in via the garden login command to use Garden with secrets/ - ) + const msg = getUnavailableReason(result) + expect(stripAnsi(msg)).to.match(/Please log in via the garden login command to use Garden with secrets/) }) context("when logged in", () => { diff --git a/core/test/unit/src/config/template-contexts/provider.ts b/core/test/unit/src/config/template-contexts/provider.ts index b423c8a77a..9ca11d46c1 100644 --- a/core/test/unit/src/config/template-contexts/provider.ts +++ b/core/test/unit/src/config/template-contexts/provider.ts @@ -15,15 +15,27 @@ describe("ProviderConfigContext", () => { const garden = await makeTestGarden(projectRootA, { environmentString: "local" }) const c = new ProviderConfigContext(garden, await garden.resolveProviders({ log: garden.log }), garden.variables) - expect(c.resolve({ nodePath: [], key: ["environment", "name"], opts: {} })).to.eql({ resolved: "local" }) + expect(c.resolve({ nodePath: [], key: ["environment", "name"], opts: {} })).to.eql({ + found: true, + resolved: "local", + }) }) it("should set environment.namespace and environment.fullName to properly if namespace is set", async () => { const garden = await makeTestGarden(projectRootA, { environmentString: "foo.local" }) const c = new ProviderConfigContext(garden, await garden.resolveProviders({ log: garden.log }), garden.variables) - expect(c.resolve({ nodePath: [], key: ["environment", "name"], opts: {} })).to.eql({ resolved: "local" }) - expect(c.resolve({ nodePath: [], key: ["environment", "namespace"], opts: {} })).to.eql({ resolved: "foo" }) - expect(c.resolve({ nodePath: [], key: ["environment", "fullName"], opts: {} })).to.eql({ resolved: "foo.local" }) + expect(c.resolve({ nodePath: [], key: ["environment", "name"], opts: {} })).to.eql({ + found: true, + resolved: "local", + }) + expect(c.resolve({ nodePath: [], key: ["environment", "namespace"], opts: {} })).to.eql({ + found: true, + resolved: "foo", + }) + expect(c.resolve({ nodePath: [], key: ["environment", "fullName"], opts: {} })).to.eql({ + found: true, + resolved: "foo.local", + }) }) }) diff --git a/core/test/unit/src/config/template-contexts/workflow.ts b/core/test/unit/src/config/template-contexts/workflow.ts index e322154c3f..79f3bc6ff8 100644 --- a/core/test/unit/src/config/template-contexts/workflow.ts +++ b/core/test/unit/src/config/template-contexts/workflow.ts @@ -30,6 +30,7 @@ describe("WorkflowConfigContext", () => { it("should resolve local env variables", async () => { process.env.TEST_VARIABLE = "foo" expect(c.resolve({ nodePath: [], key: ["local", "env", "TEST_VARIABLE"], opts: {} })).to.eql({ + found: true, resolved: "foo", }) delete process.env.TEST_VARIABLE @@ -37,33 +38,43 @@ describe("WorkflowConfigContext", () => { it("should resolve the local arch", async () => { expect(c.resolve({ nodePath: [], key: ["local", "arch"], opts: {} })).to.eql({ + found: true, resolved: process.arch, }) }) it("should resolve the local platform", async () => { expect(c.resolve({ nodePath: [], key: ["local", "platform"], opts: {} })).to.eql({ + found: true, resolved: process.platform, }) }) it("should resolve the environment config", async () => { expect(c.resolve({ nodePath: [], key: ["environment", "name"], opts: {} })).to.eql({ + found: true, resolved: garden.environmentName, }) }) it("should resolve a project variable", async () => { - expect(c.resolve({ nodePath: [], key: ["variables", "some"], opts: {} })).to.eql({ resolved: "variable" }) + expect(c.resolve({ nodePath: [], key: ["variables", "some"], opts: {} })).to.eql({ + found: true, + resolved: "variable", + }) }) it("should resolve a project variable under the var alias", async () => { - expect(c.resolve({ nodePath: [], key: ["var", "some"], opts: {} })).to.eql({ resolved: "variable" }) + expect(c.resolve({ nodePath: [], key: ["var", "some"], opts: {} })).to.eql({ + found: true, + resolved: "variable", + }) }) context("secrets", () => { it("should resolve a secret", async () => { expect(c.resolve({ nodePath: [], key: ["secrets", "someSecret"], opts: {} })).to.eql({ + found: true, resolved: "someSecretValue", }) }) @@ -104,7 +115,7 @@ describe("WorkflowStepConfigContext", () => { }, stepName: "step-2", }) - expect(c.resolve({ nodePath: [], key: ["steps", "step-1", "outputs", "some"], opts: {} })).to.equal({ + expect(c.resolve({ nodePath: [], key: ["steps", "step-1", "outputs", "some"], opts: {} })).to.deep.eq({ found: true, resolved: "value", }) @@ -124,7 +135,7 @@ describe("WorkflowStepConfigContext", () => { }, stepName: "step-2", }) - expect(c.resolve({ nodePath: [], key: ["steps", "step-1", "log"], opts: {} })).to.equal({ + expect(c.resolve({ nodePath: [], key: ["steps", "step-1", "log"], opts: {} })).to.deep.equal({ found: true, resolved: "bla", }) From 20e703c92d23dc37de9664878ac74198d8ff95ad Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Tue, 21 Jan 2025 15:08:01 +0100 Subject: [PATCH 037/117] chore: fix BuildCommand tests --- core/src/config/module.ts | 5 ++-- core/src/config/render-template.ts | 42 ++++++++++++++++++++---------- core/src/resolve-module.ts | 2 +- 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/core/src/config/module.ts b/core/src/config/module.ts index 1d3a69e1e0..eba51081f2 100644 --- a/core/src/config/module.ts +++ b/core/src/config/module.ts @@ -170,17 +170,18 @@ export const baseBuildSpecSchema = createSchema({ }) // These fields are validated immediately when loading the config file -const coreModuleSpecKeys = memoize(() => ({ +const coreModuleSpecSchemaKeys = memoize(() => ({ apiVersion: unusedApiVersionSchema(), kind: joi.string().default("Module").valid("Module"), type: joiIdentifier().required().description("The type of this module.").example("container"), name: joiUserIdentifier().required().description("The name of this module.").example("my-sweet-module"), })) +export const coreModuleSpecKeys = () => Object.keys(coreModuleSpecSchemaKeys()) export const coreModuleSpecSchema = createSchema({ name: "core-module-spec", description: "Configure a module whose sources are located in this directory.", - keys: coreModuleSpecKeys, + keys: coreModuleSpecSchemaKeys, allowUnknown: true, meta: { extendable: true }, }) diff --git a/core/src/config/render-template.ts b/core/src/config/render-template.ts index 3c985a0876..8ec7c4006a 100644 --- a/core/src/config/render-template.ts +++ b/core/src/config/render-template.ts @@ -6,7 +6,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import type { ModuleConfig } from "./module.js" +import { coreModuleSpecKeys, type ModuleConfig } from "./module.js" import { dedent, deline, naturalList } from "../util/string.js" import type { BaseGardenResource, RenderTemplateKind } from "./base.js" import { @@ -36,8 +36,8 @@ import type { Log } from "../logger/log-entry.js" import { GardenApiVersion } from "../constants.js" import { capture } from "../template/capture.js" import { deepEvaluate, evaluate } from "../template/evaluate.js" -import type { ParsedTemplate } from "../template/types.js" -import { isArray } from "../util/objects.js" +import { UnresolvedTemplateValue, type ParsedTemplate } from "../template/types.js" +import { isArray, isPlainObject } from "../util/objects.js" export const renderTemplateConfigSchema = createSchema({ name: renderTemplateKind, @@ -193,28 +193,42 @@ async function renderModules({ renderConfig: RenderTemplateConfig }): Promise { return Promise.all( - (template.modules || []).map(async (m) => { + (template.modules || []).map(async (m, index) => { // Run a partial template resolution with the parent+template info - const spec = capture(m as unknown as ParsedTemplate, context) as unknown as ModuleConfig + const spec = evaluate(capture(m as unknown as ParsedTemplate, context), { + context, + opts: {}, + }).resolved + + if (!isPlainObject(spec)) { + throw new ConfigurationError({ + message: `${configTemplateKind} ${template.name} returned an invalid module at index ${index}: Must be or resolve to a plain object`, + }) + } + const renderConfigPath = renderConfig.internal.configFilePath || renderConfig.internal.basePath let moduleConfig: ModuleConfig try { - moduleConfig = prepareModuleResource(spec, renderConfigPath, garden.projectRoot) + const resolvedSpec = { ...spec } + for (const key of coreModuleSpecKeys()) { + resolvedSpec[key] = deepEvaluate(resolvedSpec[key], { context, opts: {} }) + } + moduleConfig = prepareModuleResource(resolvedSpec, renderConfigPath, garden.projectRoot) } catch (error) { if (!(error instanceof GardenError)) { throw error } let msg = error.message - if (spec.name && spec.name.includes && spec.name.includes("${")) { + if (coreModuleSpecKeys().some((k) => spec[k] instanceof UnresolvedTemplateValue)) { msg += - ". Note that if a template string is used in the name of a module in a template, then the template string must be fully resolvable at the time of module scanning. This means that e.g. references to other modules or runtime outputs cannot be used." + ". Note that if a template string is used for the name, kind, type or apiVersion of a module in a template, then the template string must be fully resolvable at the time of module scanning. This means that e.g. references to other modules or runtime outputs cannot be used." } throw new ConfigurationError({ - message: `${configTemplateKind} ${template.name} returned an invalid module (named ${spec.name}) for templated module ${renderConfig.name}: ${msg}`, + message: `${configTemplateKind} ${template.name}: invalid module at index ${index} for templated module ${renderConfig.name}: ${msg}`, }) } @@ -224,11 +238,11 @@ async function renderModules({ sourcePath: f.sourcePath && resolve(template.internal.basePath, ...f.sourcePath.split(posix.sep)), })) - // If a path is set, resolve the path and ensure that directory exists - if (spec.path) { - moduleConfig.path = resolve(renderConfig.internal.basePath, ...spec.path.split(posix.sep)) - await ensureDir(moduleConfig.path) - } + // // If a path is set, resolve the path and ensure that directory exists + // if (spec.path) { + // moduleConfig.path = resolve(renderConfig.internal.basePath, ...spec.path.split(posix.sep)) + // await ensureDir(moduleConfig.path) + // } // Attach metadata moduleConfig.parentName = renderConfig.name diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index d0716cb387..6601d42e90 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -540,7 +540,7 @@ export class ModuleResolver { // Try resolving template strings if possible let buildDeps: string[] = [] - const resolvedDeps = capture( + const resolvedDeps = partiallyEvaluateModule( rawConfig.build.dependencies as unknown as ParsedTemplate, configContext ) as unknown as BuildDependencyConfig From b33380159ec629430e7952f047e1d7d6be3aec6f Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Tue, 21 Jan 2025 15:37:48 +0100 Subject: [PATCH 038/117] chore: fix a symlink test --- core/test/unit/src/build-staging/helpers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/test/unit/src/build-staging/helpers.ts b/core/test/unit/src/build-staging/helpers.ts index 8831393cc2..cbccce375d 100644 --- a/core/test/unit/src/build-staging/helpers.ts +++ b/core/test/unit/src/build-staging/helpers.ts @@ -540,7 +540,7 @@ describe("build staging helpers", () => { it("throws if a path to a non-symlink (e.g. directory) is given", async () => { return expectError(() => resolveSymlink({ path: tmpPath }), { - contains: `Error reading symlink: EINVAL: invalid argument, readlink '${tmpPath}'`, + contains: ["Error reading symlink", `EINVAL: invalid argument, readlink '${tmpPath}'`], }) }) From d829b2c5915d39b58d7c2759d55ebf7c6f70639a Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Tue, 21 Jan 2025 15:38:03 +0100 Subject: [PATCH 039/117] chore: fix CustomCommandWrapper tests --- core/src/commands/custom.ts | 9 +- core/test/unit/src/commands/custom.ts | 317 ++++++++++++++------------ 2 files changed, 181 insertions(+), 145 deletions(-) diff --git a/core/src/commands/custom.ts b/core/src/commands/custom.ts index 8bfe44b69a..5b79d8a365 100644 --- a/core/src/commands/custom.ts +++ b/core/src/commands/custom.ts @@ -32,6 +32,8 @@ import type { Log } from "../logger/log-entry.js" import { getTracePropagationEnvVars } from "../util/open-telemetry/propagation.js" import { styles } from "../logger/styles.js" import { deepEvaluate } from "../template/evaluate.js" +import { capture } from "../template/capture.js" +import { GenericContext, LayeredContext } from "../config/template-contexts/base.js" function convertArgSpec(spec: CustomCommandOption) { const params = { @@ -114,11 +116,8 @@ export class CustomCommandWrapper extends Command { // Render the command variables const variablesContext = new CustomCommandContext({ ...garden, args, opts, rest }) - const commandVariables = deepEvaluate(this.spec.variables, { - context: variablesContext, - opts: {}, - }) - const variables: any = jsonMerge(cloneDeep(garden.variables), commandVariables) + const commandVariables = new GenericContext(capture(this.spec.variables, variablesContext)) + const variables = new LayeredContext(commandVariables, garden.variables) // Make a new template context with the resolved variables const commandContext = new CustomCommandContext({ ...garden, args, opts, variables, rest }) diff --git a/core/test/unit/src/commands/custom.ts b/core/test/unit/src/commands/custom.ts index 77467b04fa..e396398b47 100644 --- a/core/test/unit/src/commands/custom.ts +++ b/core/test/unit/src/commands/custom.ts @@ -15,6 +15,8 @@ import { expectError } from "../../../../src/util/testing.js" import { makeTestGardenA, withDefaultGlobalOpts } from "../../../helpers.js" import { GardenApiVersion } from "../../../../src/constants.js" import { TestGardenCli } from "../../../helpers/cli.js" +import { parseTemplateCollection } from "../../../../src/template/templated-collections.js" +import type { CommandResource } from "../../../../src/config/command.js" describe("CustomCommandWrapper", () => { let garden: TestGarden @@ -85,21 +87,26 @@ describe("CustomCommandWrapper", () => { const short = "Test" const long = "Here's the full description" - const cmd = new CustomCommandWrapper({ - apiVersion: GardenApiVersion.v0, - kind: "Command", - name: "test", - internal: { - basePath: "/tmp", - }, - description: { - short, - long, - }, - args: [], - opts: [], - variables: {}, - }) + const cmd = new CustomCommandWrapper( + parseTemplateCollection({ + value: { + apiVersion: GardenApiVersion.v0, + kind: "Command", + name: "test", + internal: { + basePath: "/tmp", + }, + description: { + short, + long, + }, + args: [], + opts: [], + variables: {}, + }, + source: { path: [] }, + }) + ) expect(cmd.name).to.equal("test") expect(cmd.help).to.equal(short) @@ -107,29 +114,34 @@ describe("CustomCommandWrapper", () => { }) it("sets the ${args.$rest} variable correctly", async () => { - const cmd = new CustomCommandWrapper({ - apiVersion: GardenApiVersion.v0, - kind: "Command", - name: "test", - internal: { - basePath: "/tmp", - }, - description: { - short: "Test", - }, - args: [ - { type: "string", name: "a", description: "Arg A", required: true }, - { type: "integer", name: "b", description: "Arg B" }, - ], - opts: [ - { type: "string", name: "a", description: "Opt A", required: true }, - { type: "boolean", name: "b", description: "Opt B" }, - ], - variables: {}, - exec: { - command: ["echo", "${join(args.$rest, ' ')}"], - }, - }) + const cmd = new CustomCommandWrapper( + parseTemplateCollection({ + value: { + apiVersion: GardenApiVersion.v0, + kind: "Command", + name: "test", + internal: { + basePath: "/tmp", + }, + description: { + short: "Test", + }, + args: [ + { type: "string", name: "a", description: "Arg A", required: true }, + { type: "integer", name: "b", description: "Arg B" }, + ], + opts: [ + { type: "string", name: "a", description: "Opt A", required: true }, + { type: "boolean", name: "b", description: "Opt B" }, + ], + variables: {}, + exec: { + command: ["echo", "${join(args.$rest, ' ')}"], + }, + }, + source: { path: [] }, + }) as unknown as CommandResource + ) const { result } = await cmd.action({ cli, @@ -148,25 +160,30 @@ describe("CustomCommandWrapper", () => { }) it("resolves template strings in command variables", async () => { - const cmd = new CustomCommandWrapper({ - apiVersion: GardenApiVersion.v0, - kind: "Command", - name: "test", - internal: { - basePath: "/tmp", - }, - description: { - short: "Test", - }, - args: [], - opts: [], - variables: { - foo: "${project.name}", - }, - exec: { - command: ["echo", "${var.foo}"], - }, - }) + const cmd = new CustomCommandWrapper( + parseTemplateCollection({ + value: { + apiVersion: GardenApiVersion.v0, + kind: "Command", + name: "test", + internal: { + basePath: "/tmp", + }, + description: { + short: "Test", + }, + args: [], + opts: [], + variables: { + foo: "${project.name}", + }, + exec: { + command: ["echo", "${var.foo}"], + }, + }, + source: { path: [] }, + }) + ) const { result } = await cmd.action({ cli, @@ -181,25 +198,30 @@ describe("CustomCommandWrapper", () => { }) it("runs an exec command with resolved templates", async () => { - const cmd = new CustomCommandWrapper({ - apiVersion: GardenApiVersion.v0, - kind: "Command", - name: "test", - internal: { - basePath: "/tmp", - }, - description: { - short: "Test", - }, - args: [], - opts: [], - variables: { - foo: "test", - }, - exec: { - command: ["echo", "${project.name}-${var.foo}"], - }, - }) + const cmd = new CustomCommandWrapper( + parseTemplateCollection({ + value: { + apiVersion: GardenApiVersion.v0, + kind: "Command", + name: "test", + internal: { + basePath: "/tmp", + }, + description: { + short: "Test", + }, + args: [], + opts: [], + variables: { + foo: "test", + }, + exec: { + command: ["echo", "${project.name}-${var.foo}"], + }, + }, + source: { path: [] }, + }) + ) const { result } = await cmd.action({ cli, @@ -214,23 +236,28 @@ describe("CustomCommandWrapper", () => { }) it("runs a Garden command with resolved templates", async () => { - const cmd = new CustomCommandWrapper({ - apiVersion: GardenApiVersion.v0, - kind: "Command", - name: "test", - internal: { - basePath: "/tmp", - }, - description: { - short: "Test", - }, - args: [], - opts: [], - variables: { - foo: "test", - }, - gardenCommand: ["validate"], - }) + const cmd = new CustomCommandWrapper( + parseTemplateCollection({ + value: { + apiVersion: GardenApiVersion.v0, + kind: "Command", + name: "test", + internal: { + basePath: "/tmp", + }, + description: { + short: "Test", + }, + args: [], + opts: [], + variables: { + foo: "test", + }, + gardenCommand: ["validate"], + }, + source: { path: [] }, + }) + ) const { result } = await cmd.action({ cli, @@ -244,24 +271,29 @@ describe("CustomCommandWrapper", () => { }) it("runs exec command before Garden command if both are specified", async () => { - const cmd = new CustomCommandWrapper({ - apiVersion: GardenApiVersion.v0, - kind: "Command", - name: "test", - internal: { - basePath: "/tmp", - }, - description: { - short: "Test", - }, - args: [], - opts: [], - variables: {}, - exec: { - command: ["sleep", "1"], - }, - gardenCommand: ["validate"], - }) + const cmd = new CustomCommandWrapper( + parseTemplateCollection({ + value: { + apiVersion: GardenApiVersion.v0, + kind: "Command", + name: "test", + internal: { + basePath: "/tmp", + }, + description: { + short: "Test", + }, + args: [], + opts: [], + variables: {}, + exec: { + command: ["sleep", "1"], + }, + gardenCommand: ["validate"], + }, + source: { path: [] }, + }) + ) const { result } = await cmd.action({ cli, @@ -275,35 +307,40 @@ describe("CustomCommandWrapper", () => { }) it("exposes arguments and options correctly in command templates", async () => { - const cmd = new CustomCommandWrapper({ - apiVersion: GardenApiVersion.v0, - kind: "Command", - name: "test", - internal: { - basePath: "/tmp", - }, - description: { - short: "Test", - }, - args: [ - { type: "string", name: "a", description: "Arg A", required: true }, - { type: "integer", name: "b", description: "Arg B" }, - ], - opts: [ - { type: "string", name: "a", description: "Opt A", required: true }, - { type: "boolean", name: "b", description: "Opt B" }, - ], - variables: { - foo: "test", - }, - exec: { - command: [ - "sh", - "-c", - "echo ALL: ${args.$all}\necho ARG A: ${args.a}\necho ARG B: ${args.b}\necho OPT A: ${opts.a}\necho OPT B: ${opts.b}", - ], - }, - }) + const cmd = new CustomCommandWrapper( + parseTemplateCollection({ + value: { + apiVersion: GardenApiVersion.v0, + kind: "Command", + name: "test", + internal: { + basePath: "/tmp", + }, + description: { + short: "Test", + }, + args: [ + { type: "string", name: "a", description: "Arg A", required: true }, + { type: "integer", name: "b", description: "Arg B" }, + ], + opts: [ + { type: "string", name: "a", description: "Opt A", required: true }, + { type: "boolean", name: "b", description: "Opt B" }, + ], + variables: { + foo: "test", + }, + exec: { + command: [ + "sh", + "-c", + "echo ALL: ${args.$all}\necho ARG A: ${args.a}\necho ARG B: ${args.b}\necho OPT A: ${opts.a}\necho OPT B: ${opts.b}", + ], + }, + }, + source: { path: [] }, + }) as CommandResource + ) const { result } = await cmd.action({ cli, From 04a51699a9450bfc7b5548cdbe059e286130b456 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Tue, 21 Jan 2025 15:55:42 +0100 Subject: [PATCH 040/117] chore: fix e2e test templated-k8s-container --- core/src/graph/actions.ts | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index cd5b75c89e..7c7bbfda34 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -668,15 +668,17 @@ export async function executeAction({ return >(results.results.getResult(task)!.result!.executedAction) } +// Returns non-templatable keys, and keys that can be resolved using ActionConfigContext. const getBuiltinConfigContextKeys = memoize(() => { const keys: string[] = [] - // TODO: why are type and name missing here? for (const schema of [buildActionConfigSchema(), baseRuntimeActionConfigSchema(), baseActionConfigSchema()]) { const configKeys = schema.describe().keys for (const [k, v] of Object.entries(configKeys)) { - if ((v).metas?.find((m) => m.templateContext === ActionConfigContext)) { + if ( + (v).metas?.find((m) => m.templateContext === ActionConfigContext || m.templateContext === null) + ) { keys.push(k) } } @@ -812,13 +814,10 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi function resolveTemplates() { // Fully resolve built-in fields that only support `ActionConfigContext`. // TODO-0.13.1: better error messages when something goes wrong here (missing inputs for example) - const resolvedBuiltin = deepEvaluate( - pick(config, builtinConfigKeys.concat("type", "name")) as Record, - { - context: builtinFieldContext, - opts: {}, - } - ) + const resolvedBuiltin = deepEvaluate(pick(config, builtinConfigKeys) as Record, { + context: builtinFieldContext, + opts: {}, + }) const { spec = {} } = config config = { From d4937f39874c76d323a7ae1222ff720af855b709 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Tue, 21 Jan 2025 16:02:05 +0100 Subject: [PATCH 041/117] chore: make sure to forward crashes in prepareModuleResource --- core/src/config/render-template.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/config/render-template.ts b/core/src/config/render-template.ts index 8ec7c4006a..37b191ee9e 100644 --- a/core/src/config/render-template.ts +++ b/core/src/config/render-template.ts @@ -202,7 +202,7 @@ async function renderModules({ if (!isPlainObject(spec)) { throw new ConfigurationError({ - message: `${configTemplateKind} ${template.name} returned an invalid module at index ${index}: Must be or resolve to a plain object`, + message: `${configTemplateKind} ${template.name}: invalid module at index ${index}: Must be or resolve to a plain object`, }) } @@ -217,7 +217,7 @@ async function renderModules({ } moduleConfig = prepareModuleResource(resolvedSpec, renderConfigPath, garden.projectRoot) } catch (error) { - if (!(error instanceof GardenError)) { + if (!(error instanceof GardenError) || error.type === "crash") { throw error } let msg = error.message From 2869aafccd421733dfca725610effab094f14260 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Tue, 21 Jan 2025 16:12:43 +0100 Subject: [PATCH 042/117] chore: fix lint errors --- core/src/commands/custom.ts | 2 -- core/src/config/template-contexts/project.ts | 4 ++-- core/test/unit/src/cli/cli.ts | 3 ++- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/core/src/commands/custom.ts b/core/src/commands/custom.ts index 5b79d8a365..a0f1118c26 100644 --- a/core/src/commands/custom.ts +++ b/core/src/commands/custom.ts @@ -7,8 +7,6 @@ */ import { execa } from "execa" -import { apply as jsonMerge } from "json-merge-patch" -import cloneDeep from "fast-copy" import { keyBy, mapValues, flatten } from "lodash-es" import { parseCliArgs, prepareMinimistOpts } from "../cli/helpers.js" import type { Parameter, ParameterObject } from "../cli/params.js" diff --git a/core/src/config/template-contexts/project.ts b/core/src/config/template-contexts/project.ts index fbe9682875..bfb7008ef4 100644 --- a/core/src/config/template-contexts/project.ts +++ b/core/src/config/template-contexts/project.ts @@ -6,12 +6,12 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { last, isEmpty } from "lodash-es" +import { isEmpty } from "lodash-es" import type { PrimitiveMap, DeepPrimitiveMap } from "../common.js" import { joiIdentifierMap, joiStringMap, joiPrimitive, joiVariables } from "../common.js" import { joi } from "../common.js" import { deline, dedent } from "../../util/string.js" -import type { ConfigContext, ContextKeySegment, ContextResolveParams } from "./base.js" +import type { ConfigContext, ContextResolveParams } from "./base.js" import { schema, ContextWithSchema, EnvironmentContext, ParentContext, TemplateContext } from "./base.js" import type { CommandInfo } from "../../plugin-context.js" import type { Garden } from "../../garden.js" diff --git a/core/test/unit/src/cli/cli.ts b/core/test/unit/src/cli/cli.ts index a43f37cb9e..8d157036a8 100644 --- a/core/test/unit/src/cli/cli.ts +++ b/core/test/unit/src/cli/cli.ts @@ -35,7 +35,8 @@ import fsExtra from "fs-extra" const { mkdirp } = fsExtra import { uuidv4 } from "../../../../src/util/random.js" -import { Garden, makeDummyGarden } from "../../../../src/garden.js" +import type { Garden } from "../../../../src/garden.js" +import { makeDummyGarden } from "../../../../src/garden.js" import { TestGardenCli } from "../../../helpers/cli.js" import { NotImplementedError } from "../../../../src/exceptions.js" import dedent from "dedent" From a58499721527bafddad90dd2e52b1bd172cd9e91 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Tue, 21 Jan 2025 16:16:47 +0100 Subject: [PATCH 043/117] chore: fix templated-k8s-container-modules by skipping early prepareBuildDependencies --- core/src/config/base.ts | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/core/src/config/base.ts b/core/src/config/base.ts index 2df1ec1105..812fc175e3 100644 --- a/core/src/config/base.ts +++ b/core/src/config/base.ts @@ -36,6 +36,8 @@ import { profileAsync } from "../util/profiling.js" import { readFile } from "fs/promises" import { LRUCache } from "lru-cache" import { parseTemplateCollection } from "../template/templated-collections.js" +import { evaluate } from "../template/evaluate.js" +import { GenericContext } from "./template-contexts/base.js" export const configTemplateKind = "ConfigTemplate" export const renderTemplateKind = "RenderTemplate" @@ -456,18 +458,9 @@ export function prepareProjectResource(log: Log, spec: any): ProjectConfig { } export function prepareModuleResource(spec: any, configPath: string, projectRoot: string): ModuleConfig { - // We allow specifying modules by name only as a shorthand: - // dependencies: - // - foo-module - // - name: foo-module // same as the above - // Empty strings and nulls are omitted from the array. - let dependencies: BuildDependencyConfig[] = spec.build?.dependencies || [] - - if (spec.build && spec.build.dependencies && isArray(spec.build.dependencies)) { - // We call `prepareBuildDependencies` on `spec.build.dependencies` again in `resolveModuleConfig` to ensure that - // any dependency configs whose module names resolved to null get filtered out. - dependencies = prepareBuildDependencies(spec.build.dependencies) - } + spec.build = evaluate(spec.build, { context: new GenericContext({}), opts: {} }).resolved + + const dependencies: BuildDependencyConfig[] = spec.build?.dependencies || [] const cleanedSpec = { ...omit(spec, baseModuleSchemaKeys()), From 8f3c238807d316dad47117bd3b878c5e90b7160a Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Tue, 21 Jan 2025 16:23:19 +0100 Subject: [PATCH 044/117] chore: uncomment object freeze in parseTemplateCollection to fix some tests --- core/src/template/templated-collections.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/template/templated-collections.ts b/core/src/template/templated-collections.ts index 2eab3b42fb..081f77a640 100644 --- a/core/src/template/templated-collections.ts +++ b/core/src/template/templated-collections.ts @@ -210,7 +210,8 @@ export function parseTemplateCollection Date: Tue, 21 Jan 2025 16:35:00 +0100 Subject: [PATCH 045/117] chore: fix "should correctly parse" test --- core/src/config/template-contexts/base.ts | 11 +++++++++++ core/src/garden.ts | 14 ++++---------- core/test/unit/src/cli/cli.ts | 3 ++- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index ecb2f7aa67..6b378a8360 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -524,3 +524,14 @@ export function getUnavailableReason(result: ContextResolveOutput): string { return message } + +export function deepResolveContext(description: string, context: ConfigContext, rootContext?: ConfigContext) { + const res = context.resolve({ nodePath: [], key: [], opts: {}, rootContext }) + if (!res.found) { + throw new ConfigurationError({ + message: `Could not resolve ${description}: ${getUnavailableReason(res)}`, + }) + } + + return res.resolved +} diff --git a/core/src/garden.ts b/core/src/garden.ts index bdf448b4a8..8157832055 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -123,7 +123,7 @@ import { GardenCloudApi, CloudApiTokenRefreshError } from "./cloud/api.js" import { OutputConfigContext } from "./config/template-contexts/module.js" import { ProviderConfigContext } from "./config/template-contexts/provider.js" import type { ConfigContext } from "./config/template-contexts/base.js" -import { GenericContext, getUnavailableReason, type ContextWithSchema } from "./config/template-contexts/base.js" +import { deepResolveContext, GenericContext, type ContextWithSchema } from "./config/template-contexts/base.js" import { validateSchema, validateWithPath } from "./config/validation.js" import { pMemoizeDecorator } from "./lib/p-memoize.js" import { ModuleGraph } from "./graph/modules.js" @@ -178,6 +178,7 @@ import { import { GrowCloudApi } from "./cloud/grow/api.js" import { throwOnMissingSecretKeys } from "./config/secrets.js" import { deepEvaluate } from "./template/evaluate.js" +import type { ResolvedTemplate } from "./template/types.js" import { isTemplatePrimitive, UnresolvedTemplateValue, type ParsedTemplate } from "./template/types.js" import { LayeredContext } from "./config/template-contexts/base.js" @@ -1835,13 +1836,6 @@ export class Garden { mapValues(actionConfigs, (configsForKind) => mapValues(configsForKind, omitInternal)) ) - const variableResolveResult = this.variables.resolve({ nodePath: [], key: [], opts: {} }) - if (!variableResolveResult.found) { - throw new ConfigurationError({ - message: `Could not resolve variables: ${getUnavailableReason(variableResolveResult)}`, - }) - } - return { environmentName: this.environmentName, allProviderNames: this.getUnresolvedProviderConfigs().map((p) => p.name), @@ -1850,7 +1844,7 @@ export class Garden { providers: providers.map((p) => !(p instanceof UnresolvedTemplateValue || isTemplatePrimitive(p)) ? omitInternal(p) : p ), - variables: variableResolveResult.resolved as any, + variables: deepResolveContext("project variables", this.variables), actionConfigs: filteredActionConfigs, moduleConfigs: moduleConfigs.map(omitInternal), workflowConfigs: sortBy(workflowConfigs.map(omitInternal), "name"), @@ -2465,7 +2459,7 @@ export interface ConfigDump { allProviderNames: string[] namespace: string providers: (OmitInternalConfig | ParsedTemplate)[] - variables: DeepPrimitiveMap + variables: ResolvedTemplate actionConfigs: ActionConfigMapForDump moduleConfigs: OmitInternalConfig[] workflowConfigs: OmitInternalConfig[] diff --git a/core/test/unit/src/cli/cli.ts b/core/test/unit/src/cli/cli.ts index 8d157036a8..a33f5e95c8 100644 --- a/core/test/unit/src/cli/cli.ts +++ b/core/test/unit/src/cli/cli.ts @@ -40,6 +40,7 @@ import { makeDummyGarden } from "../../../../src/garden.js" import { TestGardenCli } from "../../../helpers/cli.js" import { NotImplementedError } from "../../../../src/exceptions.js" import dedent from "dedent" +import { deepResolveContext } from "../../../../src/config/template-contexts/base.js" /** * Helper functions for removing/resetting the global logger config which is set when @@ -962,7 +963,7 @@ describe("cli", () => { override printHeader() {} async action({ garden }: { garden: Garden }) { - return { result: { variables: garden.variables } } + return { result: { variables: deepResolveContext("project variables", garden.variables) } } } } From 8801d090250bf099c4a6146787dfcddb8aa04b32 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Tue, 21 Jan 2025 16:47:01 +0100 Subject: [PATCH 046/117] test: fix tests in RunWorkflowCommand --- core/test/unit/src/commands/workflow.ts | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/core/test/unit/src/commands/workflow.ts b/core/test/unit/src/commands/workflow.ts index dabcbd4b29..0df981b35f 100644 --- a/core/test/unit/src/commands/workflow.ts +++ b/core/test/unit/src/commands/workflow.ts @@ -26,6 +26,7 @@ import { joi } from "../../../../src/config/common.js" import type { ProjectConfig } from "../../../../src/config/project.js" import { join } from "path" import fsExtra from "fs-extra" + const { remove, readFile, pathExists } = fsExtra import { dedent } from "../../../../src/util/string.js" import { resolveMsg, type LogEntry } from "../../../../src/logger/log-entry.js" @@ -34,6 +35,7 @@ import { defaultWorkflowResources } from "../../../../src/config/workflow.js" import { TestGardenCli } from "../../../helpers/cli.js" import { WorkflowScriptError } from "../../../../src/exceptions.js" import { GenericContext } from "../../../../src/config/template-contexts/base.js" +import { parseTemplateCollection } from "../../../../src/template/templated-collections.js" describe("RunWorkflowCommand", () => { const cmd = new WorkflowCommand() @@ -53,6 +55,8 @@ describe("RunWorkflowCommand", () => { }) it("should run a workflow", async () => { + const parsedDeployArg = parseTemplateCollection({ value: "${var.foo}", source: { path: [] } }) as string + garden.setWorkflowConfigs([ { apiVersion: GardenApiVersion.v0, @@ -67,7 +71,7 @@ describe("RunWorkflowCommand", () => { { command: ["deploy"], description: "deploy services" }, { command: ["get", "outputs"] }, { command: ["test"] }, - { command: ["deploy", "${var.foo}"] }, // <-- the second (null) element should get filtered out + { command: ["deploy", parsedDeployArg] }, // <-- the second (null) element should get filtered out { command: ["test", "module-a-unit"] }, { command: ["run", "task-a"] }, { command: ["cleanup", "service", "service-a"] }, @@ -981,6 +985,11 @@ describe("RunWorkflowCommand", () => { }) it("should resolve references to previous steps when running a command step", async () => { + const parsedCommand = parseTemplateCollection({ + value: ["run", "${steps.step-1.outputs.taskName}"], + source: { path: [] }, + }) + garden.setWorkflowConfigs([ { apiVersion: GardenApiVersion.v0, @@ -992,7 +1001,7 @@ describe("RunWorkflowCommand", () => { files: [], envVars: {}, resources: defaultWorkflowResources, - steps: [{ command: ["get", "outputs"] }, { command: ["run", "${steps.step-1.outputs.taskName}"] }], + steps: [{ command: ["get", "outputs"] }, { command: parsedCommand }], }, ]) @@ -1007,6 +1016,11 @@ describe("RunWorkflowCommand", () => { }) it("should resolve references to previous steps when running a script step", async () => { + const parsedScript = parseTemplateCollection({ + value: "echo ${steps.step-1.outputs.taskName}", + source: { path: [] }, + }) as string + garden.setWorkflowConfigs([ { apiVersion: GardenApiVersion.v0, @@ -1018,7 +1032,7 @@ describe("RunWorkflowCommand", () => { files: [], envVars: {}, resources: defaultWorkflowResources, - steps: [{ command: ["get", "outputs"] }, { script: "echo ${steps.step-1.outputs.taskName}" }], + steps: [{ command: ["get", "outputs"] }, { script: parsedScript }], }, ]) From f281af586ecce537c6463256507bfa2961180ff0 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Tue, 21 Jan 2025 16:41:02 +0100 Subject: [PATCH 047/117] chore: fix command error handling tests --- core/test/unit/src/cli/cli.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/test/unit/src/cli/cli.ts b/core/test/unit/src/cli/cli.ts index a33f5e95c8..1b71a10c27 100644 --- a/core/test/unit/src/cli/cli.ts +++ b/core/test/unit/src/cli/cli.ts @@ -38,7 +38,7 @@ import { uuidv4 } from "../../../../src/util/random.js" import type { Garden } from "../../../../src/garden.js" import { makeDummyGarden } from "../../../../src/garden.js" import { TestGardenCli } from "../../../helpers/cli.js" -import { NotImplementedError } from "../../../../src/exceptions.js" +import { RuntimeError } from "../../../../src/exceptions.js" import dedent from "dedent" import { deepResolveContext } from "../../../../src/config/template-contexts/base.js" @@ -1084,7 +1084,7 @@ describe("cli", () => { override printHeader() {} async action({}): Promise { - throw new NotImplementedError({ message: "Error message" }) + throw new RuntimeError({ message: "Error message" }) } } @@ -1126,7 +1126,7 @@ describe("cli", () => { expect(firstEightLines).to.eql(dedent` Encountered an unexpected Garden error. This is likely a bug 🍂 - You can help by reporting this on GitHub: https://github.com/garden-io/garden/issues/new?labels=bug,crash&template=CRASH.md&title=Crash%3A%20Cannot%20read%20property%20foo%20of%20undefined. + You can help by reporting this on GitHub: https://github.com/garden-io/garden/issues/new?labels=bug,crash&template=CRASH.md&title=Crash%3A%20TypeError%3A%20Cannot%20read%20property%20foo%20of%20undefined. Please attach the following information to the bug report after making sure that the error message does not contain sensitive information: @@ -1169,7 +1169,7 @@ describe("cli", () => { errors: [ { type: "crash", - message: "Some unexpected error that leads to a crash", + message: "Error: Some unexpected error that leads to a crash", stack: "stack", }, ], From b4af168c5c838a836c2e2d20603f1422ff69eef0 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Tue, 21 Jan 2025 16:53:03 +0100 Subject: [PATCH 048/117] chore: fix further tests --- core/src/commands/get/get-config.ts | 3 +++ core/src/commands/plugins.ts | 2 +- core/src/garden.ts | 18 ++++++++---------- core/test/unit/src/commands/get/get-config.ts | 6 +++++- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/core/src/commands/get/get-config.ts b/core/src/commands/get/get-config.ts index d98e6e923e..12110a4d5c 100644 --- a/core/src/commands/get/get-config.ts +++ b/core/src/commands/get/get-config.ts @@ -42,6 +42,9 @@ export class GetConfigCommand extends Command<{}, Opts, ConfigDump> { override outputsSchema = () => joi.object().keys({ allEnvironmentNames: joiArray(environmentNameSchema()).required(), + allAvailablePlugins: joiArray(joi.string()) + .description("A list of all plugins available to be used in the provider configuration.") + .required(), environmentName: environmentNameSchema().required(), namespace: joiIdentifier().description("The namespace of the current environment (if applicable)."), providers: joiArray(joi.alternatives(providerSchema(), providerConfigBaseSchema())).description( diff --git a/core/src/commands/plugins.ts b/core/src/commands/plugins.ts index 129e45d9a2..4388833212 100644 --- a/core/src/commands/plugins.ts +++ b/core/src/commands/plugins.ts @@ -25,7 +25,7 @@ const pluginArgs = { help: "The name of the plugin, whose command you wish to run.", required: false, getSuggestions: ({ configDump }) => { - return configDump.allProviderNames + return configDump.allAvailablePlugins }, }), command: new StringOption({ diff --git a/core/src/garden.ts b/core/src/garden.ts index 8157832055..0e4509bfda 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -179,7 +179,7 @@ import { GrowCloudApi } from "./cloud/grow/api.js" import { throwOnMissingSecretKeys } from "./config/secrets.js" import { deepEvaluate } from "./template/evaluate.js" import type { ResolvedTemplate } from "./template/types.js" -import { isTemplatePrimitive, UnresolvedTemplateValue, type ParsedTemplate } from "./template/types.js" +import { serialiseUnresolvedTemplates, type ParsedTemplate } from "./template/types.js" import { LayeredContext } from "./config/template-contexts/base.js" const defaultLocalAddress = "localhost" @@ -1788,7 +1788,7 @@ export class Garden { resolveProviders?: boolean resolveWorkflows?: boolean }): Promise { - let providers: (ParsedTemplate | Provider)[] = [] + let providers: (unknown | OmitInternalConfig)[] = [] let moduleConfigs: ModuleConfig[] let workflowConfigs: WorkflowConfig[] let actionConfigs: ActionConfigMap = { @@ -1801,9 +1801,9 @@ export class Garden { await this.scanAndAddConfigs() if (resolveProviders) { - providers = Object.values(await this.resolveProviders({ log })) + providers = Object.values(await this.resolveProviders({ log })).map((c) => omitInternal(c)) } else { - providers = this.getUnresolvedProviderConfigs().map((p) => p.unresolvedConfig) + providers = this.getUnresolvedProviderConfigs().map((p) => serialiseUnresolvedTemplates(p.unresolvedConfig)) } if (!graph && resolveGraph) { @@ -1838,12 +1838,10 @@ export class Garden { return { environmentName: this.environmentName, - allProviderNames: this.getUnresolvedProviderConfigs().map((p) => p.name), + allAvailablePlugins: this.getUnresolvedProviderConfigs().map((p) => p.name), allEnvironmentNames, namespace: this.namespace, - providers: providers.map((p) => - !(p instanceof UnresolvedTemplateValue || isTemplatePrimitive(p)) ? omitInternal(p) : p - ), + providers, variables: deepResolveContext("project variables", this.variables), actionConfigs: filteredActionConfigs, moduleConfigs: moduleConfigs.map(omitInternal), @@ -2456,9 +2454,9 @@ export async function makeDummyGarden(root: string, gardenOpts: GardenOpts) { export interface ConfigDump { environmentName: string // TODO: Remove this? allEnvironmentNames: string[] - allProviderNames: string[] + allAvailablePlugins: string[] namespace: string - providers: (OmitInternalConfig | ParsedTemplate)[] + providers: (OmitInternalConfig | unknown)[] variables: ResolvedTemplate actionConfigs: ActionConfigMapForDump moduleConfigs: OmitInternalConfig[] diff --git a/core/test/unit/src/commands/get/get-config.ts b/core/test/unit/src/commands/get/get-config.ts index 8c5c36fbfd..d75e22f8f0 100644 --- a/core/test/unit/src/commands/get/get-config.ts +++ b/core/test/unit/src/commands/get/get-config.ts @@ -21,6 +21,7 @@ import type { WorkflowConfig } from "../../../../../src/config/workflow.js" import { defaultWorkflowResources } from "../../../../../src/config/workflow.js" import { defaultContainerLimits } from "../../../../../src/plugins/container/moduleConfig.js" import type { ModuleConfig } from "../../../../../src/config/module.js" +import { serialiseUnresolvedTemplates } from "../../../../../src/template/types.js" describe("GetConfigCommand", () => { const command = new GetConfigCommand() @@ -701,7 +702,10 @@ describe("GetConfigCommand", () => { opts: withDefaultGlobalOpts({ "exclude-disabled": false, "resolve": "partial" }), }) - expect(res.result!.providers).to.eql(garden.getUnresolvedProviderConfigs()) + const unresolvedProviderConfigs = garden + .getUnresolvedProviderConfigs() + .map((c) => serialiseUnresolvedTemplates(c.unresolvedConfig)) + expect(res.result!.providers).to.eql(unresolvedProviderConfigs) }) it("should not resolve providers", async () => { From 9b55bbe74e53d016d487cca75b7df5edbbb9398b Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Tue, 21 Jan 2025 17:05:40 +0100 Subject: [PATCH 049/117] chore: fix output test --- core/src/config/project.ts | 7 +++++-- core/test/unit/src/commands/get/get-outputs.ts | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/core/src/config/project.ts b/core/src/config/project.ts index 9644c9033d..5eb5c0eb7b 100644 --- a/core/src/config/project.ts +++ b/core/src/config/project.ts @@ -39,7 +39,7 @@ import { baseInternalFieldsSchema, loadVarfile, varfileDescription } from "./bas import type { Log } from "../logger/log-entry.js" import { renderDivider } from "../logger/util.js" import { styles } from "../logger/styles.js" -import type { ParsedTemplate } from "../template/types.js" +import { serialiseUnresolvedTemplates, type ParsedTemplate } from "../template/types.js" import { LayeredContext } from "./template-contexts/base.js" import type { ConfigContext } from "./template-contexts/base.js" import { GenericContext } from "./template-contexts/base.js" @@ -482,7 +482,7 @@ export function resolveProjectConfig({ commandInfo: CommandInfo }): ProjectConfig { // Resolve template strings for non-environment-specific fields (apart from `sources`). - const { environments = [], name, sources = [], providers = [] } = config + const { environments = [], name, sources = [], providers = [], outputs = [] } = config let globalConfig: any const context = new ProjectConfigContext({ @@ -528,6 +528,8 @@ export function resolveProjectConfig({ environments: [{ defaultNamespace: null, name: "fake-env-only-here-for-initial-load", variables: {} }], providers: [], sources: [], + // this makes sure that the output declaration shape is valid + outputs: serialiseUnresolvedTemplates(outputs), }, schema: projectSchema(), projectRoot: config.path, @@ -539,6 +541,7 @@ export function resolveProjectConfig({ environments, providers, sources, + outputs, } config.defaultEnvironment = getDefaultEnvironmentName(defaultEnvironmentName, config) diff --git a/core/test/unit/src/commands/get/get-outputs.ts b/core/test/unit/src/commands/get/get-outputs.ts index 0c230da91c..7ff81bce60 100644 --- a/core/test/unit/src/commands/get/get-outputs.ts +++ b/core/test/unit/src/commands/get/get-outputs.ts @@ -12,6 +12,7 @@ import { withDefaultGlobalOpts, TestGarden, createProjectConfig, makeTempDir } f import { GetOutputsCommand } from "../../../../../src/commands/get/get-outputs.js" import type { ProjectConfig } from "../../../../../src/config/project.js" import { createGardenPlugin } from "../../../../../src/plugin/plugin.js" +import { parseTemplateCollection } from "../../../../../src/template/templated-collections.js" describe("GetOutputsCommand", () => { let tmpDir: tmp.DirectoryResult @@ -40,7 +41,10 @@ describe("GetOutputsCommand", () => { }, }) - projectConfig.outputs = [{ name: "test", value: "${providers.test.outputs.test}" }] + projectConfig.outputs = parseTemplateCollection({ + value: [{ name: "test", value: "${providers.test.outputs.test}" }], + source: { path: [] }, + }) const garden = await TestGarden.factory(tmpDir.path, { plugins: [plugin], From 4fd8e049bda618808d85a80284476bb33a0735ae Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Tue, 21 Jan 2025 17:02:47 +0100 Subject: [PATCH 050/117] refactor(test): parse whole workflow configs as template values --- core/test/unit/src/commands/workflow.ts | 121 ++++++++++++------------ 1 file changed, 62 insertions(+), 59 deletions(-) diff --git a/core/test/unit/src/commands/workflow.ts b/core/test/unit/src/commands/workflow.ts index 0df981b35f..b7af2cc64f 100644 --- a/core/test/unit/src/commands/workflow.ts +++ b/core/test/unit/src/commands/workflow.ts @@ -30,7 +30,7 @@ import fsExtra from "fs-extra" const { remove, readFile, pathExists } = fsExtra import { dedent } from "../../../../src/util/string.js" import { resolveMsg, type LogEntry } from "../../../../src/logger/log-entry.js" -import type { WorkflowStepSpec } from "../../../../src/config/workflow.js" +import type { WorkflowConfig, WorkflowStepSpec } from "../../../../src/config/workflow.js" import { defaultWorkflowResources } from "../../../../src/config/workflow.js" import { TestGardenCli } from "../../../helpers/cli.js" import { WorkflowScriptError } from "../../../../src/exceptions.js" @@ -55,31 +55,34 @@ describe("RunWorkflowCommand", () => { }) it("should run a workflow", async () => { - const parsedDeployArg = parseTemplateCollection({ value: "${var.foo}", source: { path: [] } }) as string - - garden.setWorkflowConfigs([ - { - apiVersion: GardenApiVersion.v0, - name: "workflow-a", - kind: "Workflow", - envVars: {}, - resources: defaultWorkflowResources, - internal: { - basePath: garden.projectRoot, + const parsedWorkflowConfigs = parseTemplateCollection({ + value: [ + { + apiVersion: GardenApiVersion.v0, + name: "workflow-a", + kind: "Workflow", + envVars: {}, + resources: defaultWorkflowResources, + internal: { + basePath: garden.projectRoot, + }, + steps: [ + { command: ["deploy"], description: "deploy services" }, + { command: ["get", "outputs"] }, + { command: ["test"] }, + { command: ["deploy", "${var.foo}"] }, // <-- the second (null) element should get filtered out + { command: ["test", "module-a-unit"] }, + { command: ["run", "task-a"] }, + { command: ["cleanup", "service", "service-a"] }, + { command: ["cleanup", "namespace"] }, + { command: ["publish"] }, + ], }, - steps: [ - { command: ["deploy"], description: "deploy services" }, - { command: ["get", "outputs"] }, - { command: ["test"] }, - { command: ["deploy", parsedDeployArg] }, // <-- the second (null) element should get filtered out - { command: ["test", "module-a-unit"] }, - { command: ["run", "task-a"] }, - { command: ["cleanup", "service", "service-a"] }, - { command: ["cleanup", "namespace"] }, - { command: ["publish"] }, - ], - }, - ]) + ], + source: { path: [] }, + }) as WorkflowConfig[] + + garden.setWorkflowConfigs(parsedWorkflowConfigs) garden.variables = new GenericContext({ foo: null }) @@ -985,25 +988,25 @@ describe("RunWorkflowCommand", () => { }) it("should resolve references to previous steps when running a command step", async () => { - const parsedCommand = parseTemplateCollection({ - value: ["run", "${steps.step-1.outputs.taskName}"], + const parsedWorkflowConfigs = parseTemplateCollection({ + value: [ + { + apiVersion: GardenApiVersion.v0, + name: "workflow-a", + kind: "Workflow", + internal: { + basePath: garden.projectRoot, + }, + files: [], + envVars: {}, + resources: defaultWorkflowResources, + steps: [{ command: ["get", "outputs"] }, { command: ["run", "${steps.step-1.outputs.taskName}"] }], + }, + ], source: { path: [] }, - }) + }) as WorkflowConfig[] - garden.setWorkflowConfigs([ - { - apiVersion: GardenApiVersion.v0, - name: "workflow-a", - kind: "Workflow", - internal: { - basePath: garden.projectRoot, - }, - files: [], - envVars: {}, - resources: defaultWorkflowResources, - steps: [{ command: ["get", "outputs"] }, { command: parsedCommand }], - }, - ]) + garden.setWorkflowConfigs(parsedWorkflowConfigs) const { result, errors } = await cmd.action({ ...defaultParams, args: { workflow: "workflow-a" } }) @@ -1016,25 +1019,25 @@ describe("RunWorkflowCommand", () => { }) it("should resolve references to previous steps when running a script step", async () => { - const parsedScript = parseTemplateCollection({ - value: "echo ${steps.step-1.outputs.taskName}", + const parsedWorkflowConfigs = parseTemplateCollection({ + value: [ + { + apiVersion: GardenApiVersion.v0, + name: "workflow-a", + kind: "Workflow", + internal: { + basePath: garden.projectRoot, + }, + files: [], + envVars: {}, + resources: defaultWorkflowResources, + steps: [{ command: ["get", "outputs"] }, { script: "echo ${steps.step-1.outputs.taskName}" }], + }, + ], source: { path: [] }, - }) as string + }) as WorkflowConfig[] - garden.setWorkflowConfigs([ - { - apiVersion: GardenApiVersion.v0, - name: "workflow-a", - kind: "Workflow", - internal: { - basePath: garden.projectRoot, - }, - files: [], - envVars: {}, - resources: defaultWorkflowResources, - steps: [{ command: ["get", "outputs"] }, { script: parsedScript }], - }, - ]) + garden.setWorkflowConfigs(parsedWorkflowConfigs) const { result, errors } = await cmd.action({ ...defaultParams, args: { workflow: "workflow-a" } }) From 388018f195d23cb3d04cb8a6eea0fd8307a913ba Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Tue, 21 Jan 2025 17:04:26 +0100 Subject: [PATCH 051/117] chore: fix lint issues --- core/src/config/base.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/config/base.ts b/core/src/config/base.ts index 812fc175e3..673d46eaf2 100644 --- a/core/src/config/base.ts +++ b/core/src/config/base.ts @@ -10,7 +10,7 @@ import dotenv from "dotenv" import { sep, resolve, relative, basename, dirname, join } from "path" import { load } from "js-yaml" import { lint } from "yaml-lint" -import { omit, isPlainObject, isArray } from "lodash-es" +import { omit, isPlainObject } from "lodash-es" import type { BuildDependencyConfig, ModuleConfig } from "./module.js" import { coreModuleSpecSchema, baseModuleSchemaKeys } from "./module.js" import { ConfigurationError, FilesystemError, isErrnoException, ParameterError } from "../exceptions.js" From 027e97ea807656c851ae0393d15eefcc07918a53 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Tue, 21 Jan 2025 17:06:20 +0100 Subject: [PATCH 052/117] docs: re-generate docs --- docs/reference/commands.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 45ff8ec91e..d9e6cb21ec 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -1432,6 +1432,9 @@ stderr: ```yaml allEnvironmentNames: +# A list of all plugins available to be used in the provider configuration. +allAvailablePlugins: + # The name of the environment. environmentName: From 7994d6dac781fce80869764627015e5e642ef5f1 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Tue, 21 Jan 2025 17:38:01 +0100 Subject: [PATCH 053/117] test: fix test crashes --- core/src/garden.ts | 2 +- core/test/unit/src/config/base.ts | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/core/src/garden.ts b/core/src/garden.ts index 0e4509bfda..5bf1b6d346 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -1875,7 +1875,7 @@ export class Garden { } } -function omitInternal(obj: T): OmitInternalConfig { +export function omitInternal(obj: T): OmitInternalConfig { return omit(obj, "internal") } diff --git a/core/test/unit/src/config/base.ts b/core/test/unit/src/config/base.ts index 770cda5b8d..153f2f977e 100644 --- a/core/test/unit/src/config/base.ts +++ b/core/test/unit/src/config/base.ts @@ -27,6 +27,8 @@ import { ConfigurationError } from "../../../../src/exceptions.js" import { resetNonRepeatableWarningHistory } from "../../../../src/warnings.js" import { omit } from "lodash-es" import { dedent } from "../../../../src/util/string.js" +import { omitInternal } from "../../../../src/garden.js" +import { serialiseUnresolvedTemplates } from "../../../../src/template/types.js" const projectPathA = getDataDir("test-project-a") const modulePathA = resolve(projectPathA, "module-a") @@ -256,11 +258,11 @@ describe("loadConfigResources", () => { it("should load and parse a module config", async () => { const configPath = resolve(modulePathA, "garden.yml") - const parsed = await loadConfigResources(log, projectPathA, configPath) - - expect(parsed.length).to.equal(1) + const configResources = await loadConfigResources(log, projectPathA, configPath) + expect(configResources.length).to.equal(1) - expect(omit(parsed[0], "internal")).to.eql({ + const configResource = serialiseUnresolvedTemplates(omitInternal(configResources[0])) + expect(configResource).to.eql({ apiVersion: GardenApiVersion.v0, kind: "Module", name: "module-a", @@ -317,12 +319,11 @@ describe("loadConfigResources", () => { it("should load and parse a module template", async () => { const projectPath = getDataDir("test-projects", "module-templates") const configFilePath = resolve(projectPath, "templates.garden.yml") - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const parsed: any = await loadConfigResources(log, projectPath, configFilePath) - - expect(parsed.length).to.equal(1) + const configResources = await loadConfigResources(log, projectPath, configFilePath) + expect(configResources.length).to.equal(1) - expect(omit(parsed[0], "internal")).to.eql({ + const configResource = serialiseUnresolvedTemplates(omitInternal(configResources[0])) + expect(configResource).to.eql({ kind: configTemplateKind, name: "combo", From 90afdb7d3e70b50a73007b0fb364ff3f7317cb63 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Tue, 21 Jan 2025 17:38:08 +0100 Subject: [PATCH 054/117] test: fix assertions --- core/test/unit/src/config/base.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/test/unit/src/config/base.ts b/core/test/unit/src/config/base.ts index 153f2f977e..6b4e7a1f71 100644 --- a/core/test/unit/src/config/base.ts +++ b/core/test/unit/src/config/base.ts @@ -455,7 +455,7 @@ describe("loadConfigResources", () => { exclude: undefined, repositoryUrl: undefined, build: { - dependencies: [{ name: "module-from-project-config", copy: [] }], + dependencies: ["module-from-project-config"], timeout: DEFAULT_BUILD_TIMEOUT_SEC, }, local: undefined, @@ -464,7 +464,7 @@ describe("loadConfigResources", () => { spec: { build: { command: ["echo", "A1"], - dependencies: [{ name: "module-from-project-config", copy: [] }], + dependencies: ["module-from-project-config"], }, services: [{ name: "service-a1" }], tests: [{ name: "unit", command: ["echo", "OK"] }], From e44d1954fce3f846034bf6785146c9ddc3450fe7 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Tue, 21 Jan 2025 18:10:25 +0100 Subject: [PATCH 055/117] chore: fix some tests by not cloning actions --- .../kubernetes/kubernetes-type/common.ts | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/core/test/integ/src/plugins/kubernetes/kubernetes-type/common.ts b/core/test/integ/src/plugins/kubernetes/kubernetes-type/common.ts index 352501c633..c5b5d57351 100644 --- a/core/test/integ/src/plugins/kubernetes/kubernetes-type/common.ts +++ b/core/test/integ/src/plugins/kubernetes/kubernetes-type/common.ts @@ -66,7 +66,7 @@ describe("getManifests", () => { it("crashes with yaml syntax error if an if block references variable that does not exist", async () => { action = await garden.resolveAction({ - action: cloneDeep(graph.getDeploy("legacypartial-ifblock-doesnotexist")), + action: graph.getDeploy("legacypartial-ifblock-doesnotexist"), log: garden.log, graph, }) @@ -78,7 +78,7 @@ describe("getManifests", () => { it("should not crash due to indentation with if block statement", async () => { action = await garden.resolveAction({ - action: cloneDeep(graph.getDeploy("legacypartial-ifblock-indentation")), + action: graph.getDeploy("legacypartial-ifblock-indentation"), log: garden.log, graph, }) @@ -89,7 +89,7 @@ describe("getManifests", () => { it("partially resolves the consequent branch of ${if true} block", async () => { action = await garden.resolveAction({ - action: cloneDeep(graph.getDeploy("legacypartial-ifblock-true")), + action: graph.getDeploy("legacypartial-ifblock-true"), log: garden.log, graph, }) @@ -101,7 +101,7 @@ describe("getManifests", () => { it("partially resolves the alternate branch of ${if false} block", async () => { action = await garden.resolveAction({ - action: cloneDeep(graph.getDeploy("legacypartial-ifblock-false")), + action: graph.getDeploy("legacypartial-ifblock-false"), log: garden.log, graph, }) @@ -131,7 +131,7 @@ describe("getManifests", () => { it("finds duplicates in manifests declared inline", async () => { action = await garden.resolveAction({ - action: cloneDeep(graph.getDeploy("duplicates-inline")), + action: graph.getDeploy("duplicates-inline"), log: garden.log, graph, }) @@ -151,7 +151,7 @@ describe("getManifests", () => { it("finds duplicates between manifests declared both inline and using kustomize", async () => { action = await garden.resolveAction({ - action: cloneDeep(graph.getDeploy("duplicates-inline-kustomize")), + action: graph.getDeploy("duplicates-inline-kustomize"), log: garden.log, graph, }) @@ -174,7 +174,7 @@ describe("getManifests", () => { it("finds duplicates between manifests declared both inline and in files", async () => { action = await garden.resolveAction({ - action: cloneDeep(graph.getDeploy("duplicates-files-inline")), + action: graph.getDeploy("duplicates-files-inline"), log: garden.log, graph, }) @@ -197,7 +197,7 @@ describe("getManifests", () => { it("finds duplicates between manifests declared both using kustomize and in files", async () => { action = await garden.resolveAction({ - action: cloneDeep(graph.getDeploy("duplicates-files-kustomize")), + action: graph.getDeploy("duplicates-files-kustomize"), log: garden.log, graph, }) @@ -240,7 +240,7 @@ describe("getManifests", () => { beforeEach(async () => { graph = await garden.getConfigGraph({ log: garden.log, emit: false }) action = await garden.resolveAction({ - action: cloneDeep(graph.getDeploy("hello-world")), + action: graph.getDeploy("hello-world"), log: garden.log, graph, }) @@ -319,7 +319,7 @@ describe("getManifests", () => { it("should support regular files paths", async () => { const executedAction = await garden.executeAction({ - action: cloneDeep(graph.getDeploy("with-build-action")), + action: graph.getDeploy("with-build-action"), log: garden.log, graph, }) @@ -340,7 +340,7 @@ describe("getManifests", () => { }) it("should support both regular paths and glob patterns with deduplication", async () => { - const action = cloneDeep(graph.getDeploy("with-build-action")) + const action = graph.getDeploy("with-build-action") // Append a valid filename that results to the default glob pattern '*.yaml'. action["_config"]["spec"]["files"].push("deployment.yaml") const executedAction = await garden.resolveAction({ @@ -365,7 +365,7 @@ describe("getManifests", () => { }) it("should throw on missing regular path", async () => { - const action = cloneDeep(graph.getDeploy("with-build-action")) + const action = graph.getDeploy("with-build-action") action["_config"]["spec"]["files"].push("missing-file.yaml") const resolvedAction = await garden.resolveAction({ action, @@ -389,7 +389,7 @@ describe("getManifests", () => { }) it("should throw when no files found from glob pattens", async () => { - const action = cloneDeep(graph.getDeploy("with-build-action")) + const action = graph.getDeploy("with-build-action") // Rewrite the whole files array to have a glob pattern that results to an empty list of files. action["_config"]["spec"]["files"] = ["./**/manifests/*.yaml"] const resolvedAction = await garden.resolveAction({ @@ -429,7 +429,7 @@ describe("getManifests", () => { }) it("should apply patches to a manifest", async () => { - const action = cloneDeep(graph.getDeploy("deploy-action")) + const action = graph.getDeploy("deploy-action") action["_config"]["spec"]["patchResources"] = [ { name: "busybox-deployment", @@ -486,7 +486,7 @@ describe("getManifests", () => { expect(manifests[0].spec.replicas).to.eql(3) }) it("should handle multiple patches", async () => { - const action = cloneDeep(graph.getDeploy("deploy-action")) + const action = graph.getDeploy("deploy-action") action["_config"]["spec"]["patchResources"] = [ { name: "busybox-deployment", @@ -519,7 +519,7 @@ describe("getManifests", () => { expect(manifests[1].data.hello).to.eql("patched-world") }) it("should store patched version in metadata ConfigMap", async () => { - const action = cloneDeep(graph.getDeploy("deploy-action")) + const action = graph.getDeploy("deploy-action") action["_config"]["spec"]["patchResources"] = [ { name: "busybox-deployment", @@ -567,7 +567,7 @@ describe("getManifests", () => { }) }) it("should apply patches to file and inline manifests", async () => { - const action = cloneDeep(graph.getDeploy("deploy-action")) + const action = graph.getDeploy("deploy-action") action["_config"]["spec"]["manifests"] = [ { apiVersion: "v1", @@ -622,7 +622,7 @@ describe("getManifests", () => { expect(manifests[2].data.hello).to.eql("patched-world") }) it("should apply patches BEFORE post processing manifests", async () => { - const action = cloneDeep(graph.getDeploy("deploy-action")) + const action = graph.getDeploy("deploy-action") action["_config"]["spec"]["patchResources"] = [ { name: "busybox-deployment", @@ -657,7 +657,7 @@ describe("getManifests", () => { }) }) it("should allow the user to configure the merge patch strategy", async () => { - const action = cloneDeep(graph.getDeploy("deploy-action")) + const action = graph.getDeploy("deploy-action") action["_config"]["spec"]["patchResources"] = [ { name: "busybox-deployment", @@ -705,7 +705,7 @@ describe("getManifests", () => { }) it("should log a warning if patches don't match manifests", async () => { garden.log.root["entries"].length = 0 - const action = cloneDeep(graph.getDeploy("deploy-action")) + const action = graph.getDeploy("deploy-action") action["_config"]["spec"]["patchResources"] = [ { name: "non-existent-resource", From c3a4813f08002428ed28d7027b6f9b0940b4dfde Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Tue, 21 Jan 2025 18:10:49 +0100 Subject: [PATCH 056/117] test: fix crash in pickEnvironment tests --- core/test/unit/src/config/project.ts | 48 ++++++++++++++++------------ 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/core/test/unit/src/config/project.ts b/core/test/unit/src/config/project.ts index 82d2f6afe8..11bb2553f3 100644 --- a/core/test/unit/src/config/project.ts +++ b/core/test/unit/src/config/project.ts @@ -22,10 +22,14 @@ import { } from "../../../../src/config/project.js" import { createProjectConfig, expectError } from "../../../helpers.js" import fsExtra from "fs-extra" -const { realpath, writeFile } = fsExtra import { dedent } from "../../../../src/util/string.js" import { resolve, join } from "path" import { getRootLogger } from "../../../../src/logger/logger.js" +import { deepEvaluate } from "../../../../src/template/evaluate.js" +import { GenericContext } from "../../../../src/config/template-contexts/base.js" +import { omit } from "lodash-es" + +const { realpath, writeFile } = fsExtra const enterpriseDomain = "https://garden.mydomain.com" const commandInfo = { name: "test", args: {}, opts: {} } @@ -538,31 +542,35 @@ describe("pickEnvironment", () => { ], }) - expect( - await pickEnvironment({ - projectConfig: config, - envString: "default", - artifactsPath, - vcsInfo, - username, - loggedIn: true, - enterpriseDomain, - secrets: {}, - commandInfo, - }) - ).to.eql({ + const env = await pickEnvironment({ + projectConfig: config, + envString: "default", + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain, + secrets: {}, + commandInfo, + }) + + expect(omit(env, "providers")).to.eql({ environmentName: "default", defaultNamespace: "default", namespace: "default", - providers: [ - { name: "exec" }, - { name: "container", newKey: "foo", dependencies: [] }, - { name: "templated" }, - { name: "my-provider", b: "b" }, - ], production: false, variables: {}, }) + + const resolvedProviders = env.providers.map((p) => + deepEvaluate(p.unresolvedConfig, { context: new GenericContext({}), opts: {} }) + ) + expect(resolvedProviders).to.eql([ + { name: "exec" }, + { name: "container", newKey: "foo", dependencies: [] }, + { name: "templated" }, + { name: "my-provider", b: "b" }, + ]) }) it("should correctly merge project and environment variables", async () => { From 00b4e223abd7be1c7b4ce6866580a16b2b32253a Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 22 Jan 2025 09:00:39 +0100 Subject: [PATCH 057/117] chore: fix lint issues --- core/test/integ/src/plugins/kubernetes/kubernetes-type/common.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/core/test/integ/src/plugins/kubernetes/kubernetes-type/common.ts b/core/test/integ/src/plugins/kubernetes/kubernetes-type/common.ts index c5b5d57351..bb00a5fce5 100644 --- a/core/test/integ/src/plugins/kubernetes/kubernetes-type/common.ts +++ b/core/test/integ/src/plugins/kubernetes/kubernetes-type/common.ts @@ -7,7 +7,6 @@ */ import { expect } from "chai" -import cloneDeep from "fast-copy" import type { ConfigGraph } from "../../../../../../src/graph/config-graph.js" import type { PluginContext } from "../../../../../../src/plugin-context.js" From 628bbf578c555498a59e10f03637ebb99f1ebabd Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 22 Jan 2025 09:01:04 +0100 Subject: [PATCH 058/117] test: fix some test crashes in `scanAndAddConfigs` --- core/test/unit/src/garden.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index 51533427a0..3881688f7e 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -81,7 +81,8 @@ import { getCloudDistributionName } from "../../../src/cloud/util.js" import { resolveAction } from "../../../src/graph/actions.js" import { serialiseUnresolvedTemplates } from "../../../src/template/types.js" import { parseTemplateCollection } from "../../../src/template/templated-collections.js" -import { GenericContext, LayeredContext } from "../../../src/config/template-contexts/base.js" +import { deepResolveContext, GenericContext, LayeredContext } from "../../../src/config/template-contexts/base.js" +import { ResolvedBuildAction } from "../../../src/actions/build.js" const { realpath, writeFile, readFile, remove, pathExists, mkdirp, copy } = fsExtra @@ -2949,7 +2950,7 @@ describe("Garden", () => { const configB = (await garden.getRawModuleConfigs(["foo-test-b"]))[0] // note that module config versions should default to v0 (previous version) - expect(omitUndefined(configA)).to.eql({ + expect(serialiseUnresolvedTemplates(omitUndefined(configA))).to.eql({ apiVersion: GardenApiVersion.v0, kind: "Module", build: { dependencies: [], timeout: DEFAULT_BUILD_TIMEOUT_SEC }, @@ -2981,7 +2982,7 @@ describe("Garden", () => { value: "${providers.test-plugin.outputs.testKey}", }, }) - expect(omitUndefined(configB)).to.eql({ + expect(serialiseUnresolvedTemplates(omitUndefined(configB))).to.eql({ apiVersion: GardenApiVersion.v0, kind: "Module", build: { dependencies: [{ name: "foo-test-a", copy: [] }], timeout: DEFAULT_BUILD_TIMEOUT_SEC }, @@ -3170,9 +3171,10 @@ describe("Garden", () => { name: "foo-test-dt", }), }) - expect(resolved).to.exist - expect(resolved.getVariables()).to.deep.eq({ + + const variables = deepResolveContext("resolved action variables", resolved.getVariables()) + expect(variables).to.deep.eq({ myDir: "../../../test", syncTargets: [ { From 8fcbec365aac42997818500196c7b2c33eaf6d07 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 22 Jan 2025 09:06:56 +0100 Subject: [PATCH 059/117] test: fix some test crashes in `getConfigGraph` --- core/test/unit/src/garden.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index 3881688f7e..d0be591feb 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -4381,10 +4381,10 @@ describe("Garden", () => { } expect(build.type).to.equal("test") - expect(omit(build.getInternal(), "yamlDoc")).to.eql(internal) + expect(serialiseUnresolvedTemplates(omit(build.getInternal(), "yamlDoc"))).to.eql(internal) expect(deploy.getBuildAction()?.name).to.equal("foo-test") // <- should be resolved - expect(omit(deploy.getInternal(), "yamlDoc")).to.eql(internal) + expect(serialiseUnresolvedTemplates(omit(deploy.getInternal(), "yamlDoc"))).to.eql(internal) expect(test.getDependencies().map((a) => a.key())).to.eql(["build.foo-test"]) // <- should be resolved expect(omit(test.getInternal(), "yamlDoc")).to.eql(internal) From 253ffee8f53afe28c258b1661b6b0b57cd6c94f3 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 22 Jan 2025 09:33:30 +0100 Subject: [PATCH 060/117] test: fix template string assertion --- core/test/unit/src/garden.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index d0be591feb..581e2cf83c 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -2961,7 +2961,7 @@ describe("Garden", () => { serviceConfigs: [], spec: { build: { - command: ["${providers.test-plugin.outputs.testKey}"], + command: ["${inputs.value}"], dependencies: [], }, }, From 4b642d0503471476f55e477a4eb96c607bc6107d Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 22 Jan 2025 09:37:16 +0100 Subject: [PATCH 061/117] test: fix assertions for variables --- core/test/unit/src/garden.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index 581e2cf83c..3ff55e391f 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -204,7 +204,8 @@ describe("Garden", () => { }, }) - expect(garden.variables).to.eql({ + const variables = deepResolveContext("Garden variables", garden.variables) + expect(variables).to.eql({ some: "variable", }) }) @@ -252,7 +253,8 @@ describe("Garden", () => { }, }) - expect(garden.variables).to.eql({ + const variables = deepResolveContext("Garden variables", garden.variables) + expect(variables).to.eql({ "some": "banana", "service-a-build-command": "OK", }) @@ -375,7 +377,8 @@ describe("Garden", () => { it("should load default varfiles if they exist", async () => { const projectRoot = getDataDir("test-projects", "varfiles") const garden = await makeTestGarden(projectRoot, {}) - expect(garden.variables).to.eql({ + const variables = deepResolveContext("Garden variables", garden.variables) + expect(variables).to.eql({ a: "a", b: "B", c: "c", @@ -385,7 +388,8 @@ describe("Garden", () => { it("should load custom varfiles if specified", async () => { const projectRoot = getDataDir("test-projects", "varfiles-custom") const garden = await makeTestGarden(projectRoot, {}) - expect(garden.variables).to.eql({ + const variables = deepResolveContext("Garden variables", garden.variables) + expect(variables).to.eql({ a: "a", b: "B", c: "c", @@ -414,7 +418,8 @@ describe("Garden", () => { const garden = await makeTestGarden(projectRoot) const graph = await garden.getConfigGraph({ log: garden.log, emit: false }) const runAction = graph.getRun("run-a") - expect(runAction.getVariables()).to.eql({}) + const runActionVariables = deepResolveContext("Run action variables", runAction.getVariables()) + expect(runActionVariables).to.eql({}) }) it("should throw if project root is not in a git repo root", async () => { @@ -495,7 +500,8 @@ describe("Garden", () => { variableOverrides: { "foo": "override", "nested.nestedKey2": "somevalue2new", "key.withdot": "somevalue3new" }, }) - expect(garden.variables).to.eql({ + const variables = deepResolveContext("Garden variables", garden.variables) + expect(variables).to.eql({ "foo": "override", "bar": "something", "nested": { From 032a05860e274449f838adc2b7c8eccc3d07329d Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 22 Jan 2025 10:06:37 +0100 Subject: [PATCH 062/117] test: fix some assertions in template-string.ts --- core/src/config/template-contexts/base.ts | 2 +- core/test/unit/src/template-string.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 6b378a8360..3be39480c0 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -422,7 +422,7 @@ function traverseContext( if (isTemplatePrimitive(value)) { throw new ContextLookupNotIndexable({ - message: `Attempting to look up key ${nextKey} on primitive value ${renderKeyPath([...params.nodePath, ...params.key])}`, + message: `Attempted to look up key ${nextKey} on primitive value ${renderKeyPath([...params.nodePath, ...params.key])}.`, }) } diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index da2c6809c0..a38177e759 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -940,7 +940,7 @@ describe("parse and evaluate template strings", () => { void expectError( () => resolveTemplateString({ string: "${foo[bar]}", context: new GenericContext({ foo: 123, bar: "baz" }) }), { - contains: 'Invalid template string (${foo[bar]}): Attempted to look up key "baz" on a number.', + contains: 'Invalid template string (${foo[bar]}): Attempted to look up key baz on primitive value foo.baz.', } ) }) @@ -1027,7 +1027,7 @@ describe("parse and evaluate template strings", () => { }), { contains: - "Invalid template string (${nested.missing}): Could not find key missing under nested. Available keys: bar, baz and foo.", + "Invalid template string (${nested.missing}): Could not find key missing under nested. Available keys: foo, bar, baz.", } ) }) @@ -1041,7 +1041,7 @@ describe("parse and evaluate template strings", () => { }), { contains: - "Invalid template string (${nested.missing}): Could not find key missing under nested. Available keys: bar and foo.", + "Invalid template string (${nested.missing}): Could not find key missing under nested. Available keys: foo, bar.", } ) }) From b10cbb03ef9fb60c01fb289c9537e4e4c1714915 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 22 Jan 2025 12:50:01 +0100 Subject: [PATCH 063/117] test: fix some assertions in template-string.ts --- core/test/unit/src/template-string.ts | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index a38177e759..fef2edc86e 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -940,7 +940,7 @@ describe("parse and evaluate template strings", () => { void expectError( () => resolveTemplateString({ string: "${foo[bar]}", context: new GenericContext({ foo: 123, bar: "baz" }) }), { - contains: 'Invalid template string (${foo[bar]}): Attempted to look up key baz on primitive value foo.baz.', + contains: "Invalid template string (${foo[bar]}): Attempted to look up key baz on primitive value foo.baz.", } ) }) @@ -2296,10 +2296,7 @@ context("$concat", () => { }, } - const context = new GenericContext({ foo: 1 }) - const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) - - void expectError(() => deepEvaluate(parsed, { context, opts: {} }), { + void expectError(() => parseTemplateCollection({ source: { path: [] }, value: obj }), { contains: "Missing $then field next to $if field", }) }) @@ -2313,10 +2310,7 @@ context("$concat", () => { }, } - const context = new GenericContext({ foo: 1 }) - const parsed = parseTemplateCollection({ source: { path: [] }, value: obj }) - - void expectError(() => deepEvaluate(parsed, { context, opts: {} }), { + void expectError(() => parseTemplateCollection({ source: { path: [] }, value: obj }), { contains: 'Found one or more unexpected keys on $if object: "foo"', }) }) From eae37919a26542c5270536b02263319a1f72c08e Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 22 Jan 2025 12:50:40 +0100 Subject: [PATCH 064/117] fix: process `$concat` correctly in `$forEach` --- core/src/template/templated-collections.ts | 54 ++++++++++++++-------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/core/src/template/templated-collections.ts b/core/src/template/templated-collections.ts index 081f77a640..61f94552e3 100644 --- a/core/src/template/templated-collections.ts +++ b/core/src/template/templated-collections.ts @@ -16,6 +16,7 @@ import { isTruthy } from "./ast.js" import type { EvaluateTemplateArgs, ParsedTemplate, ResolvedTemplate, TemplateEvaluationResult } from "./types.js" import { isTemplatePrimitive, UnresolvedTemplateValue, type TemplatePrimitive } from "./types.js" import isBoolean from "lodash-es/isBoolean.js" +import mapValues from "lodash-es/mapValues.js" import { arrayConcatKey, arrayForEachFilterKey, @@ -26,7 +27,6 @@ import { conditionalThenKey, objectSpreadKey, } from "../config/constants.js" -import mapValues from "lodash-es/mapValues.js" import { evaluate } from "./evaluate.js" import { LayeredContext } from "../config/template-contexts/base.js" import { parseTemplateString } from "./templated-strings.js" @@ -40,6 +40,7 @@ export function pushYamlPath(part: ObjectPath[0], configSource: ConfigSource): C path: [...configSource.path, part], } } + type MaybeTplString = `${string}\${${string}` type Parse> = T extends MaybeTplString ? ParsedTemplate @@ -117,10 +118,25 @@ export function parseTemplateCollection } else { return forEach as Parse @@ -255,20 +271,17 @@ export class ConcatLazyValue extends StructuralTemplateOperator { partial: true resolved: ParsedTemplate[] } { - const output: ParsedTemplate[] = [] - - let concatYaml: (ConcatOperator | ParsedTemplate)[] - - // NOTE(steffen): We need to support a construct where $concat inside a $forEach expression results in a flat list. + let collectionValue: (ConcatOperator | ParsedTemplate)[] if (this.yaml instanceof ForEachLazyValue) { const { resolved } = this.yaml.evaluate(args) - - concatYaml = resolved + // This is to handle the special case when forEach lazy value returns $concat + collectionValue = resolved.map((element) => ({ [arrayConcatKey]: element })) } else { - concatYaml = this.yaml + collectionValue = this.yaml } - for (const v of concatYaml) { + const output: ParsedTemplate[] = [] + for (const v of collectionValue) { if (!this.isConcatOperator(v)) { // it's not a concat operator, it's a list element. output.push(v) @@ -321,6 +334,7 @@ type ForEachClause = { export class ForEachLazyValue extends StructuralTemplateOperator { static allowedForEachKeys = [arrayForEachKey, arrayForEachReturnKey, arrayForEachFilterKey] + constructor( source: ConfigSource, private readonly yaml: ForEachClause @@ -397,6 +411,7 @@ export type ObjectSpreadOperation = { [objectSpreadKey]: ParsedTemplate [staticKeys: string]: ParsedTemplate } + export class ObjectSpreadLazyValue extends StructuralTemplateOperator { constructor( source: ConfigSource, @@ -452,6 +467,7 @@ export type ConditionalClause = { [conditionalThenKey]: ParsedTemplate [conditionalElseKey]?: ParsedTemplate } + export class ConditionalLazyValue extends StructuralTemplateOperator { static allowedConditionalKeys = [conditionalKey, conditionalThenKey, conditionalElseKey] @@ -481,16 +497,16 @@ export class ConditionalLazyValue extends StructuralTemplateOperator { } override evaluate(args: EvaluateTemplateArgs): TemplateEvaluationResult { - const conditionalValue = evaluate(this.yaml[conditionalKey], args) + const { resolved } = evaluate(this.yaml[conditionalKey], args) - if (typeof conditionalValue !== "boolean") { + if (typeof resolved !== "boolean") { throw new TemplateError({ - message: `Value of ${conditionalKey} key must be (or resolve to) a boolean (got ${typeof conditionalValue})`, + message: `Value of ${conditionalKey} key must be (or resolve to) a boolean (got ${typeof resolved})`, source: pushYamlPath(conditionalKey, this.source), }) } - const branch = isTruthy(conditionalValue) ? this.yaml[conditionalThenKey] : this.yaml[conditionalElseKey] + const branch = isTruthy(resolved) ? this.yaml[conditionalThenKey] : this.yaml[conditionalElseKey] return evaluate(branch, args) } From 331fe0b99c4f67fb4e4a615eb52903b2d2cb75ff Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Wed, 22 Jan 2025 13:44:52 +0100 Subject: [PATCH 065/117] fix: add work-around for nodejs exit 0 crash --- core/test/setup.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/core/test/setup.ts b/core/test/setup.ts index 4e89edd49b..407708007c 100644 --- a/core/test/setup.ts +++ b/core/test/setup.ts @@ -13,12 +13,24 @@ import { getDefaultProfiler } from "../src/util/profiling.js" import { gardenEnv } from "../src/constants.js" import { testFlags } from "../src/util/util.js" import { initTestLogger, testProjectTempDirs } from "./helpers.js" - +import mocha from "mocha" import sourceMapSupport from "source-map-support" +import { UnresolvedTemplateValue } from "../src/template/types.js" sourceMapSupport.install() initTestLogger() +// Work-around for nodejs crash with exit code 0 +// This happens when unresolved template values are involved in expect(), it tries to render a diff which triggers the +// "objectSpreadTrap" error and that causes nodejs to exit with code 0 for a mysterious reason. +const origCanonicalize = mocha.utils.canonicalize +mocha.utils.canonicalize = function (value, stack, typeHint) { + if (value instanceof UnresolvedTemplateValue) { + return `[${value.toString()}]` + } + return origCanonicalize(value, stack, typeHint) +} + // Global hooks export const mochaHooks = { async beforeAll() { From 2f2bdca42092374f803ac6efac2c2bc761f74f11 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Wed, 22 Jan 2025 13:45:26 +0100 Subject: [PATCH 066/117] fix: parse template strings in project config tests --- core/test/helpers.ts | 8 +++++++- core/test/unit/src/config/project.ts | 27 +++++++++++++++------------ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/core/test/helpers.ts b/core/test/helpers.ts index f6ccbfb703..e68ec7fee5 100644 --- a/core/test/helpers.ts +++ b/core/test/helpers.ts @@ -58,6 +58,9 @@ import { dumpYaml } from "../src/util/serialization.js" import { testPlugins } from "./helpers/test-plugin.js" import { testDataDir, testGitUrl } from "./helpers/constants.js" import { exec } from "../src/util/util.js" +import { parseTemplateCollection } from "../src/template/templated-collections.js" +import type { TemplatePrimitive } from "../src/template/types.js" +import type { CollectionOrValue } from "../src/util/objects.js" export { TempDirectory, makeTempDir } from "../src/util/fs.js" export { TestGarden, TestError, TestEventBus, expectError, expectFuzzyMatch } from "../src/util/testing.js" @@ -119,7 +122,10 @@ export const getDefaultProjectConfig = (): ProjectConfig => export const createProjectConfig = (partialCustomConfig: Partial): ProjectConfig => { const baseConfig = getDefaultProjectConfig() - return merge(baseConfig, partialCustomConfig) + return parseTemplateCollection({ + value: merge(baseConfig, partialCustomConfig) as unknown as CollectionOrValue, + source: { path: [] }, + }) as unknown as ProjectConfig } export const defaultModuleConfig: ModuleConfig = { diff --git a/core/test/unit/src/config/project.ts b/core/test/unit/src/config/project.ts index 11bb2553f3..6fc363697b 100644 --- a/core/test/unit/src/config/project.ts +++ b/core/test/unit/src/config/project.ts @@ -28,6 +28,7 @@ import { getRootLogger } from "../../../../src/logger/logger.js" import { deepEvaluate } from "../../../../src/template/evaluate.js" import { GenericContext } from "../../../../src/config/template-contexts/base.js" import { omit } from "lodash-es" +import { serialiseUnresolvedTemplates } from "../../../../src/template/types.js" const { realpath, writeFile } = fsExtra @@ -113,18 +114,20 @@ describe("resolveProjectConfig", () => { process.env.TEST_ENV_VAR = "foo" expect( - resolveProjectConfig({ - log, - defaultEnvironmentName: defaultEnvironment, - config, - artifactsPath: "/tmp", - vcsInfo, - username: "some-user", - loggedIn: true, - enterpriseDomain, - secrets: { foo: "banana" }, - commandInfo, - }) + serialiseUnresolvedTemplates( + resolveProjectConfig({ + log, + defaultEnvironmentName: defaultEnvironment, + config, + artifactsPath: "/tmp", + vcsInfo, + username: "some-user", + loggedIn: true, + enterpriseDomain, + secrets: { foo: "banana" }, + commandInfo, + }) + ) ).to.eql({ ...config, dotIgnoreFiles: [], From 5e095f41481d77e44a824db68c2368be85103dcb Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Wed, 22 Jan 2025 14:04:08 +0100 Subject: [PATCH 067/117] fix: resolveProjectConfig test --- core/test/unit/src/config/project.ts | 29 +++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/core/test/unit/src/config/project.ts b/core/test/unit/src/config/project.ts index 6fc363697b..8eaa6de90a 100644 --- a/core/test/unit/src/config/project.ts +++ b/core/test/unit/src/config/project.ts @@ -29,6 +29,7 @@ import { deepEvaluate } from "../../../../src/template/evaluate.js" import { GenericContext } from "../../../../src/config/template-contexts/base.js" import { omit } from "lodash-es" import { serialiseUnresolvedTemplates } from "../../../../src/template/types.js" +import type { DeepPrimitiveMap } from "@garden-io/platform-api-types" const { realpath, writeFile } = fsExtra @@ -190,20 +191,22 @@ describe("resolveProjectConfig", () => { process.env.TEST_ENV_VAR_B = "boo" expect( - resolveProjectConfig({ - log, - defaultEnvironmentName: defaultEnvironment, - config, - artifactsPath: "/tmp", - vcsInfo, - username: "some-user", - loggedIn: true, - enterpriseDomain, - secrets: {}, - commandInfo, - }) + serialiseUnresolvedTemplates( + resolveProjectConfig({ + log, + defaultEnvironmentName: defaultEnvironment, + config, + artifactsPath: "/tmp", + vcsInfo, + username: "some-user", + loggedIn: true, + enterpriseDomain, + secrets: {}, + commandInfo, + }) + ) ).to.eql({ - ...config, + ...(serialiseUnresolvedTemplates(config) as DeepPrimitiveMap), dotIgnoreFiles: [], environments: [ { From e2cecae499a60995d5178901654ccb58efe3d54c Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 22 Jan 2025 14:03:44 +0100 Subject: [PATCH 068/117] fix: throw on missing secret keys --- core/src/config/secrets.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/core/src/config/secrets.ts b/core/src/config/secrets.ts index 416010cc6d..313c0c49d6 100644 --- a/core/src/config/secrets.ts +++ b/core/src/config/secrets.ts @@ -14,6 +14,8 @@ import type { ObjectWithName } from "../util/util.js" import type { StringMap } from "./common.js" import type { ConfigContext, ContextKeySegment } from "./template-contexts/base.js" import difference from "lodash-es/difference.js" +import { ConfigurationError } from "../exceptions.js" +import { flatten, uniq } from "lodash-es" /** * Gathers secret references in configs and throws an error if one or more referenced secrets isn't present (or has @@ -67,10 +69,7 @@ export function throwOnMissingSecretKeys( if (log) { log.silly(() => errMsg) } - // throw new ConfigurationError(errMsg, { - // loadedSecretKeys: loadedKeys, - // missingSecretKeys: uniq(flatten(allMissing.map(([_key, missing]) => missing))), - // }) + throw new ConfigurationError({ message: errMsg }) } /** From 6f14b1c33c128217a300ced691383c9f953c4f3c Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 22 Jan 2025 14:14:21 +0100 Subject: [PATCH 069/117] test: fix tests for `getActionTemplateReferences` --- core/test/unit/src/template-string.ts | 40 ++++++++++++++++----------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index fef2edc86e..823398d328 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -2784,6 +2784,14 @@ describe("getContextLookupReferences", () => { }) describe("getActionTemplateReferences", () => { + function prepareActionTemplateReferences(config) { + const parsedConfig = parseTemplateCollection({ + value: config, + source: { path: [] }, + }) + return Array.from(getActionTemplateReferences(parsedConfig as any, new GenericContext({}))) + } + context("actions.*", () => { it("returns valid action references", () => { const config = { @@ -2796,7 +2804,7 @@ describe("getActionTemplateReferences", () => { test: '${actions["test"].test-a}', testFoo: '${actions["test"].test-a.outputs.foo}', } - const actionTemplateReferences = Array.from(getActionTemplateReferences(config as any, new GenericContext({}))) + const actionTemplateReferences = prepareActionTemplateReferences(config) expect(actionTemplateReferences).to.eql([ { kind: "Build", @@ -2845,7 +2853,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${actions}", } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { + void expectError(() => prepareActionTemplateReferences(config), { contains: "Found invalid action reference (missing kind)", }) }) @@ -2854,7 +2862,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions["badkind"].some-name}', } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { + void expectError(() => prepareActionTemplateReferences(config), { contains: "Found invalid action reference (invalid kind 'badkind')", }) }) @@ -2863,7 +2871,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${actions[123]}", } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { + void expectError(() => prepareActionTemplateReferences(config), { contains: "Found invalid action reference (kind is not a string)", }) }) @@ -2872,7 +2880,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${actions[foo.bar].some-name}", } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { + void expectError(() => prepareActionTemplateReferences(config), { contains: "found invalid action reference: invalid template string (${actions[foo.bar].some-name}) at path foo: could not find key foo. available keys: (none)", }) @@ -2882,7 +2890,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions[foo.bar || "hello"].some-name}', } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { + void expectError(() => prepareActionTemplateReferences(config), { contains: "found invalid action reference (invalid kind 'hello')", }) }) @@ -2891,7 +2899,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions["build"]}', } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { + void expectError(() => prepareActionTemplateReferences(config), { contains: "Found invalid action reference (missing name)", }) }) @@ -2900,7 +2908,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions["build"].123}', } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { + void expectError(() => prepareActionTemplateReferences(config), { contains: "Found invalid action reference (name is not a string)", }) }) @@ -2909,7 +2917,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${actions["build"][123]}', } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { + void expectError(() => prepareActionTemplateReferences(config), { contains: "Found invalid action reference (name is not a string)", }) }) @@ -2923,7 +2931,7 @@ describe("getActionTemplateReferences", () => { tasks: '${runtime["tasks"].task-a}', tasksFoo: '${runtime["tasks"].task-a.outputs.foo}', } - const actionTemplateReferences = Array.from(getActionTemplateReferences(config as any, new GenericContext({}))) + const actionTemplateReferences = prepareActionTemplateReferences(config) expect(actionTemplateReferences).to.eql([ { kind: "Deploy", @@ -2952,7 +2960,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${runtime}", } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { + void expectError(() => prepareActionTemplateReferences(config), { contains: "Found invalid runtime reference (missing kind)", }) }) @@ -2961,7 +2969,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${runtime["badkind"].some-name}', } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { + void expectError(() => prepareActionTemplateReferences(config), { contains: "Found invalid runtime reference (invalid kind 'badkind')", }) }) @@ -2970,7 +2978,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${runtime[123]}", } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { + void expectError(() => prepareActionTemplateReferences(config), { contains: "Found invalid runtime reference (kind is not a string)", }) }) @@ -2979,7 +2987,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: "${runtime[foo.bar].some-name}", } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { + void expectError(() => prepareActionTemplateReferences(config), { contains: "found invalid runtime reference: invalid template string (${runtime[foo.bar].some-name}) at path foo: could not find key foo. available keys: (none).", }) @@ -2989,7 +2997,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${runtime["tasks"]}', } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { + void expectError(() => prepareActionTemplateReferences(config), { contains: "Found invalid runtime reference (missing name)", }) }) @@ -2998,7 +3006,7 @@ describe("getActionTemplateReferences", () => { const config = { foo: '${runtime["tasks"].123}', } - void expectError(() => Array.from(getActionTemplateReferences(config as any, new GenericContext({}))), { + void expectError(() => prepareActionTemplateReferences(config), { contains: "Found invalid runtime reference (name is not a string)", }) }) From 4b1a557d26981279deceb6993fb47778d529aeee Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 22 Jan 2025 14:19:24 +0100 Subject: [PATCH 070/117] fix: more informative message if no keys are available in the context --- core/src/config/template-contexts/base.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 3be39480c0..7de1736035 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -325,6 +325,7 @@ export class LayeredContext extends ConfigContext { super() this.contexts = contexts } + override resolveImpl(args: ContextResolveParams): ContextResolveOutput { const items: ContextResolveOutput[] = [] @@ -513,7 +514,7 @@ export function getUnavailableReason(result: ContextResolveOutput): string { const message = deline` Could not find key ${result.explanation.key}${result.explanation.keyPath.length > 0 ? ` under ${renderKeyPath(result.explanation.keyPath)}` : ""}. - ${available?.length ? `Available keys: ${available.join(", ")}.` : ""} + ${`Available keys: ${available?.length ? available.join(", ") : "(none)"}.`} ` const footer = result.explanation.getFooterMessage?.() From b2750b170ad710bf0bb6cf9de9c657e111810258 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 22 Jan 2025 14:27:04 +0100 Subject: [PATCH 071/117] test: fix assertions for "should handle keys with dots and unresolvable member expressions correctly" --- core/test/unit/src/template-string.ts | 46 +-------------------------- 1 file changed, 1 insertion(+), 45 deletions(-) diff --git a/core/test/unit/src/template-string.ts b/core/test/unit/src/template-string.ts index 823398d328..74d4c283d6 100644 --- a/core/test/unit/src/template-string.ts +++ b/core/test/unit/src/template-string.ts @@ -2656,8 +2656,6 @@ describe("getContextLookupReferences", () => { c: "${keyThatIs[unresolvable]}", d: '${keyThatIs["${unresolvable}"]}', e: "${optionalAndUnresolvable}?", - // f: "${keyThatIs[availableLater]}", - // g: '${keyThatIs["${availableLater}"]}', }, source: { path: [], @@ -2668,9 +2666,7 @@ describe("getContextLookupReferences", () => { visitAll({ value: obj, }), - new GenericContext({ - // availableLater: CONTEXT_RESOLVE_KEY_AVAILABLE_LATER, - }) + new GenericContext({}) ) ) @@ -2724,32 +2720,6 @@ describe("getContextLookupReferences", () => { path: ["e"], }, }, - { - type: "resolvable", - keyPath: ["availableLater"], - yamlSource: { - path: ["f"], - }, - }, - { - type: "unresolvable", - keyPath: ["keyThatIs", foundKeys[8].keyPath[1]], - yamlSource: { - path: ["f"], - }, - }, - { - type: "resolvable", - keyPath: ["availableLater"], - yamlSource: { - path: ["g"], - }, - }, - { - type: "unresolvable", - keyPath: ["keyThatIs", foundKeys[10].keyPath[1]], - yamlSource: { path: ["g"] }, - }, ] expect(foundKeys, `Unexpected found keys. JSON: ${JSON.stringify(foundKeys)}`).to.deep.equals(expected) @@ -2766,20 +2736,6 @@ describe("getContextLookupReferences", () => { unresolvable2.getError().message, "invalid template string (${unresolvable}) at path d: could not find key unresolvable." ) - - const availableLater1 = foundKeys[8].keyPath[1] as UnresolvableValue - expect(availableLater1).to.be.instanceOf(UnresolvableValue) - expectFuzzyMatch( - availableLater1.getError().message, - "invalid template string (${keythatis[availablelater]}) at path f: could not find key availableLater." - ) - - const availableLater2 = foundKeys[10].keyPath[1] as UnresolvableValue - expect(availableLater2).to.be.instanceOf(UnresolvableValue) - expectFuzzyMatch( - availableLater2.getError().message, - "invalid template string (${availablelater}) at path g: could not find key availableLater." - ) }) }) From 08b9404b05c0f71621c147cefcb572ce48008dc5 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 22 Jan 2025 14:51:51 +0100 Subject: [PATCH 072/117] fix: update plugin context schema --- core/src/plugin-context.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/plugin-context.ts b/core/src/plugin-context.ts index 8bc22806d7..04ca486ffb 100644 --- a/core/src/plugin-context.ts +++ b/core/src/plugin-context.ts @@ -94,11 +94,12 @@ export const pluginContextSchema = createSchema({ projectRoot: joi.string().description("The absolute path of the project root."), projectSources: projectSourcesSchema(), provider: providerSchema().description("The provider being used for this context.").id("ctxProviderSchema"), - resolveTemplateStrings: joi + legacyResolveTemplateString: joi .function() .description( - "Helper function to resolve template strings, given the same templating context as was used to render the configuration before calling the handler. Accepts any data type, and returns the same data type back with all template strings resolved." + "Helper function to resolve a template string in a legacy way, given the same templating context as was used to render the configuration before calling the handler. Accepts any data type, and returns the same data type back with all template strings resolved." ), + deepEvaluate: joi.function().description("Helper function to deeply resolve parsed template strings."), sessionId: joi.string().description("The unique ID of the currently active session."), tools: joiStringMap(joi.object()), workingCopyId: joi.string().description("A unique ID assigned to the current project working copy."), From e3fc6c220207279d8d2aca04b05b0980d686d22d Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 22 Jan 2025 15:22:47 +0100 Subject: [PATCH 073/117] test: fix variable assertions in "pickEnvironment" --- core/test/unit/src/config/project.ts | 37 ++++++++++++++++++---------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/core/test/unit/src/config/project.ts b/core/test/unit/src/config/project.ts index 8eaa6de90a..f1acf096c1 100644 --- a/core/test/unit/src/config/project.ts +++ b/core/test/unit/src/config/project.ts @@ -26,7 +26,7 @@ import { dedent } from "../../../../src/util/string.js" import { resolve, join } from "path" import { getRootLogger } from "../../../../src/logger/logger.js" import { deepEvaluate } from "../../../../src/template/evaluate.js" -import { GenericContext } from "../../../../src/config/template-contexts/base.js" +import { deepResolveContext, GenericContext } from "../../../../src/config/template-contexts/base.js" import { omit } from "lodash-es" import { serialiseUnresolvedTemplates } from "../../../../src/template/types.js" import type { DeepPrimitiveMap } from "@garden-io/platform-api-types" @@ -560,13 +560,14 @@ describe("pickEnvironment", () => { commandInfo, }) - expect(omit(env, "providers")).to.eql({ + expect(omit(env, "providers", "variables")).to.eql({ environmentName: "default", defaultNamespace: "default", namespace: "default", production: false, - variables: {}, }) + const variables = deepResolveContext("resolved env variables", env.variables) + expect(variables).to.eql({}) const resolvedProviders = env.providers.map((p) => deepEvaluate(p.unresolvedConfig, { context: new GenericContext({}), opts: {} }) @@ -622,7 +623,8 @@ describe("pickEnvironment", () => { commandInfo, }) - expect(result.variables).to.eql({ + const variables = deepResolveContext("resolved env variables", result.variables) + expect(variables).to.eql({ a: "project value A", b: "env value B", c: "env value C", @@ -672,7 +674,8 @@ describe("pickEnvironment", () => { commandInfo, }) - expect(result.variables).to.eql({ + const variables = deepResolveContext("resolved env variables", result.variables) + expect(variables).to.eql({ a: "a", b: "B", c: "c", @@ -717,7 +720,8 @@ describe("pickEnvironment", () => { commandInfo, }) - expect(result.variables).to.eql({ + const variables = deepResolveContext("resolved env variables", result.variables) + expect(variables).to.eql({ a: "a", b: "B", c: "c", @@ -762,7 +766,8 @@ describe("pickEnvironment", () => { commandInfo, }) - expect(result.variables).to.eql({ + const variables = deepResolveContext("resolved env variables", result.variables) + expect(variables).to.eql({ a: "a", b: "B", c: "c", @@ -808,7 +813,8 @@ describe("pickEnvironment", () => { commandInfo, }) - expect(result.variables).to.eql({ + const variables = deepResolveContext("resolved env variables", result.variables) + expect(variables).to.eql({ a: "a", b: "B", c: "c", @@ -864,7 +870,8 @@ describe("pickEnvironment", () => { commandInfo, }) - expect(result.variables).to.eql({ + const variables = deepResolveContext("resolved env variables", result.variables) + expect(variables).to.eql({ a: "new-value", b: { some: "value", additional: "value" }, c: ["some", "values"], @@ -921,7 +928,8 @@ describe("pickEnvironment", () => { commandInfo, }) - expect(result.variables).to.eql({ + const variables = deepResolveContext("resolved env variables", result.variables) + expect(variables).to.eql({ a: "new-value", b: { some: "value", additional: "value" }, c: ["some", "values"], @@ -950,7 +958,8 @@ describe("pickEnvironment", () => { commandInfo, }) - expect(result.variables).to.eql({ + const variables = deepResolveContext("resolved env variables", result.variables) + expect(variables).to.eql({ local: username, secret: "banana", }) @@ -999,7 +1008,8 @@ describe("pickEnvironment", () => { commandInfo, }) - expect(result.variables).to.eql({ + const variables = deepResolveContext("resolved env variables", result.variables) + expect(variables).to.eql({ foo: "value", }) }) @@ -1056,7 +1066,8 @@ describe("pickEnvironment", () => { commandInfo, }) - expect(result.variables).to.eql({ + const variables = deepResolveContext("resolved env variables", result.variables) + expect(variables).to.eql({ a: "a", b: "B", c: "C", From 130e664c517653b7b48a3d4ac10abe582f542353 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Wed, 22 Jan 2025 15:28:22 +0100 Subject: [PATCH 074/117] fix: evaluate all values that do not have essential runtime references in partiallyEvaluateModule (The comparison was inverted before) --- core/src/resolve-module.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index 6601d42e90..f24f791b2f 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -1273,7 +1273,7 @@ function partiallyEvaluateModule(config: Input, co // if forEach expression has runtime references, we can't resolve it at all due to item context missing after converting the module to action // as the captured context is lost when calling `toJSON` on the unresolved template value onlyEssential: false, - matcher: (ref) => ref[0] === "runtime", + matcher: (ref) => ref[0] !== "runtime", }) } @@ -1283,7 +1283,7 @@ function partiallyEvaluateModule(config: Input, co // in other cases, we only skip evaluation when the runtime references is essential // meaning, we evaluate everything we can evaluate. onlyEssential: true, - matcher: (ref) => ref[0] === "runtime", + matcher: (ref) => ref[0] !== "runtime", }) } ) From a4007e48b3947c9acabba62a3283a4f4f0eb9fa4 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Wed, 22 Jan 2025 15:56:45 +0100 Subject: [PATCH 075/117] test: make ResolveActionTask pass --- core/src/actions/base.ts | 11 +++++++-- core/src/actions/build.ts | 5 ++++ core/src/actions/types.ts | 2 ++ core/src/config/template-contexts/actions.ts | 2 +- core/src/router/base.ts | 4 ++-- core/src/tasks/publish.ts | 2 +- core/src/tasks/resolve-action.ts | 11 ++++++++- core/src/util/testing.ts | 24 ++++++++++++++++---- core/test/unit/src/garden.ts | 11 +++++---- core/test/unit/src/tasks/resolve-action.ts | 5 ++-- plugins/pulumi/src/commands.ts | 2 +- 11 files changed, 60 insertions(+), 19 deletions(-) diff --git a/core/src/actions/base.ts b/core/src/actions/base.ts index df22f6e89e..08a49588e3 100644 --- a/core/src/actions/base.ts +++ b/core/src/actions/base.ts @@ -67,6 +67,7 @@ import type { BaseActionTaskParams, ExecuteTask } from "../tasks/base.js" import { styles } from "../logger/styles.js" import { dirname } from "node:path" import type { ConfigContext } from "../config/template-contexts/base.js" +import type { ResolvedTemplate } from "../template/types.js" // TODO: split this file @@ -583,7 +584,7 @@ export abstract class BaseAction< } } - getVariables(): ConfigContext { + getVariablesContext(): ConfigContext { return this.variables } @@ -735,7 +736,9 @@ export interface ResolvedActionExtension< getOutputs(): StaticOutputs - getVariables(): ConfigContext + getVariablesContext(): ConfigContext + + getResolvedVariables(): Record } // TODO: see if we can avoid the duplication here with ResolvedBuildAction @@ -816,6 +819,10 @@ export abstract class ResolvedRuntimeAction< getOutputs() { return this._staticOutputs } + + getResolvedVariables(): Record { + return this.params.resolvedVariables + } } export interface ExecutedActionExtension< diff --git a/core/src/actions/build.ts b/core/src/actions/build.ts index cc587d2407..a15ec47385 100644 --- a/core/src/actions/build.ts +++ b/core/src/actions/build.ts @@ -31,6 +31,7 @@ import { DEFAULT_BUILD_TIMEOUT_SEC } from "../constants.js" import { createBuildTask } from "../tasks/build.js" import type { BaseActionTaskParams, ExecuteTask } from "../tasks/base.js" import { ResolveActionTask } from "../tasks/resolve-action.js" +import { ResolvedTemplate } from "../template/types.js" export interface BuildCopyFrom { build: string @@ -247,6 +248,10 @@ export class ResolvedBuildAction< getOutputs() { return this._staticOutputs } + + getResolvedVariables(): Record { + return this.params.resolvedVariables + } } export class ExecutedBuildAction< diff --git a/core/src/actions/types.ts b/core/src/actions/types.ts index a387ff9d94..db2752053b 100644 --- a/core/src/actions/types.ts +++ b/core/src/actions/types.ts @@ -22,6 +22,7 @@ import type { BaseGardenResource, GardenResourceInternalFields } from "../config import type { LinkedSource } from "../config-store/local.js" import type { GardenApiVersion } from "../constants.js" import type { ConfigContext } from "../config/template-contexts/base.js" +import { ResolvedTemplate } from "../template/types.js" // TODO: split this file @@ -183,6 +184,7 @@ export interface ResolveActionParams } export type ResolvedActionWrapperParams< diff --git a/core/src/config/template-contexts/actions.ts b/core/src/config/template-contexts/actions.ts index 9c67ca7246..ce54728641 100644 --- a/core/src/config/template-contexts/actions.ts +++ b/core/src/config/template-contexts/actions.ts @@ -212,7 +212,7 @@ class ActionReferencesContext extends ContextWithSchema { buildPath: action.getBuildPath(), sourcePath: action.sourcePath(), mode: action.mode(), - variables: action.getVariables(), + variables: action.getVariablesContext(), }) ) } diff --git a/core/src/router/base.ts b/core/src/router/base.ts index 7f04ec926a..973c928406 100644 --- a/core/src/router/base.ts +++ b/core/src/router/base.ts @@ -299,7 +299,7 @@ export abstract class BaseActionRouter extends BaseRouter resolvedDependencies: action.getResolvedDependencies(), executedDependencies: action.getExecutedDependencies(), inputs: action.getInternal().inputs || {}, - variables: action.getVariables(), + variables: action.getVariablesContext(), }) : new ActionConfigContext({ garden: this.garden, @@ -308,7 +308,7 @@ export abstract class BaseActionRouter extends BaseRouter mode: action.mode(), name: action.name, }, - variables: action.getVariables(), + variables: action.getVariablesContext(), }) const handlerParams = { diff --git a/core/src/tasks/publish.ts b/core/src/tasks/publish.ts index accd3e42b0..783cca8424 100644 --- a/core/src/tasks/publish.ts +++ b/core/src/tasks/publish.ts @@ -110,7 +110,7 @@ export class PublishTask extends BaseActionTask extends ValidResultType { state: ActionState @@ -186,6 +187,13 @@ export class ResolveActionTask extends BaseActionTask extends BaseActionTask, + source: { path: [] }, + }) as unknown as ModuleConfig } /** @@ -329,7 +337,7 @@ export class TestGarden extends Garden { Test: {}, } actionConfigs.forEach((ac) => { - this.addActionConfig({ + const merged: BaseActionConfig = { spec: {}, ...ac, // TODO: consider making `timeout` mandatory in `PartialActionConfig`. @@ -339,7 +347,13 @@ export class TestGarden extends Garden { basePath: this.projectRoot, ...ac.internal, }, - }) + } + this.addActionConfig( + parseTemplateCollection({ + value: merged as unknown as CollectionOrValue, + source: { path: [] }, + }) as unknown as ActionConfig + ) }) } diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index 3ff55e391f..fd30ebc33b 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -82,7 +82,6 @@ import { resolveAction } from "../../../src/graph/actions.js" import { serialiseUnresolvedTemplates } from "../../../src/template/types.js" import { parseTemplateCollection } from "../../../src/template/templated-collections.js" import { deepResolveContext, GenericContext, LayeredContext } from "../../../src/config/template-contexts/base.js" -import { ResolvedBuildAction } from "../../../src/actions/build.js" const { realpath, writeFile, readFile, remove, pathExists, mkdirp, copy } = fsExtra @@ -404,7 +403,11 @@ describe("Garden", () => { garden.variableOverrides["d"] = "from-cli-var" const graph = await garden.getConfigGraph({ log: garden.log, emit: false }) const runAction = graph.getRun("run-a") - expect({ ...garden.variables, ...runAction.getVariables() }).to.eql({ + const resolvedVariables = deepResolveContext( + "Garden and run-a action variables", + new LayeredContext(garden.variables, runAction.getVariablesContext()) + ) + expect(resolvedVariables).to.eql({ a: "from-project-varfile", b: "from-action-vars", c: "from-action-varfile", @@ -418,7 +421,7 @@ describe("Garden", () => { const garden = await makeTestGarden(projectRoot) const graph = await garden.getConfigGraph({ log: garden.log, emit: false }) const runAction = graph.getRun("run-a") - const runActionVariables = deepResolveContext("Run action variables", runAction.getVariables()) + const runActionVariables = deepResolveContext("Run action variables", runAction.getVariablesContext()) expect(runActionVariables).to.eql({}) }) @@ -3179,7 +3182,7 @@ describe("Garden", () => { }) expect(resolved).to.exist - const variables = deepResolveContext("resolved action variables", resolved.getVariables()) + const variables = deepResolveContext("resolved action variables", resolved.getVariablesContext()) expect(variables).to.deep.eq({ myDir: "../../../test", syncTargets: [ diff --git a/core/test/unit/src/tasks/resolve-action.ts b/core/test/unit/src/tasks/resolve-action.ts index ff6516bd67..8256309954 100644 --- a/core/test/unit/src/tasks/resolve-action.ts +++ b/core/test/unit/src/tasks/resolve-action.ts @@ -22,6 +22,7 @@ import { getAllTaskResults, getDefaultProjectConfig, } from "../../../helpers.js" +import { parseTemplateCollection } from "../../../../src/template/templated-collections.js" describe("ResolveActionTask", () => { let garden: TestGarden @@ -188,7 +189,7 @@ describe("ResolveActionTask", () => { const result = await garden.processTask(task, { throwOnError: true }) const resolved = result!.outputs.resolvedAction - const variables = resolved.getVariables() + const variables = resolved.getResolvedVariables() expect(variables).to.eql({ foo: garden.projectName }) }) @@ -236,7 +237,7 @@ describe("ResolveActionTask", () => { const result = await garden.processTask(task, { throwOnError: true }) const resolved = result!.outputs.resolvedAction - const variables = resolved.getVariables() + const variables = resolved.getResolvedVariables() expect(variables).to.eql({ a: 100, diff --git a/plugins/pulumi/src/commands.ts b/plugins/pulumi/src/commands.ts index b6e0160139..0845dfa12f 100644 --- a/plugins/pulumi/src/commands.ts +++ b/plugins/pulumi/src/commands.ts @@ -206,7 +206,7 @@ const makePluginContextForDeploy = async ( resolvedDependencies: action.getResolvedDependencies(), executedDependencies: action.getExecutedDependencies(), inputs: action.getInternal().inputs || {}, - variables: action.getVariables(), + variables: action.getVariablesContext(), }) const ctxForDeploy = await garden.getPluginContext({ provider, templateContext, events: ctx.events }) return ctxForDeploy From bf295055b8962bb1b4a6189febcd65c9daeff289 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Wed, 22 Jan 2025 16:05:43 +0100 Subject: [PATCH 076/117] test: fix default providers definition in test helpers --- core/test/helpers.ts | 2 +- core/test/unit/src/config/project.ts | 34 ++++++++++++---------------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/core/test/helpers.ts b/core/test/helpers.ts index e68ec7fee5..85f922bda4 100644 --- a/core/test/helpers.ts +++ b/core/test/helpers.ts @@ -116,7 +116,7 @@ export const getDefaultProjectConfig = (): ProjectConfig => defaultEnvironment, dotIgnoreFile: defaultDotIgnoreFile, environments: [{ name: "default", defaultNamespace, variables: {} }], - providers: [{ name: "test-plugin", dependencies: [] }], + providers: [{ name: "test-plugin" }], variables: {}, }) diff --git a/core/test/unit/src/config/project.ts b/core/test/unit/src/config/project.ts index f1acf096c1..fccb670466 100644 --- a/core/test/unit/src/config/project.ts +++ b/core/test/unit/src/config/project.ts @@ -221,12 +221,10 @@ describe("resolveProjectConfig", () => { providers: [ { name: "provider-a", - dependencies: [], someKey: "${local.env.TEST_ENV_VAR_A}", }, { name: "provider-b", - dependencies: [], environments: ["default"], someKey: "${local.env.TEST_ENV_VAR_B}", }, @@ -331,7 +329,7 @@ describe("resolveProjectConfig", () => { defaultEnvironment: defaultEnvironmentName, environments: [{ defaultNamespace: null, name: "first-env", variables: {} }], outputs: [], - providers: [{ name: "some-provider", dependencies: [] }], + providers: [{ name: "some-provider" }], variables: {}, }) @@ -422,20 +420,19 @@ describe("resolveProjectConfig", () => { variables: {}, }) - expect( - resolveProjectConfig({ - log, - defaultEnvironmentName: defaultEnvironment, - config, - artifactsPath: "/tmp", - vcsInfo, - username: "some-user", - loggedIn: true, - enterpriseDomain, - secrets: {}, - commandInfo, - }) - ).to.eql({ + const resolvedConfig = resolveProjectConfig({ + log, + defaultEnvironmentName: defaultEnvironment, + config, + artifactsPath: "/tmp", + vcsInfo, + username: "some-user", + loggedIn: true, + enterpriseDomain, + secrets: {}, + commandInfo, + }) + expect(resolvedConfig).to.eql({ ...config, internal: { basePath: "/foo", @@ -454,16 +451,13 @@ describe("resolveProjectConfig", () => { providers: [ { name: "provider-a", - dependencies: [], }, { name: "provider-b", environments: ["default"], - dependencies: [], }, { name: "provider-c", - dependencies: [], }, ], sources: [], From d41105dda2a870e38b003d0d2653e69c959f3b24 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Wed, 22 Jan 2025 16:18:11 +0100 Subject: [PATCH 077/117] chore: fix resolveWorkflowConfig tests --- core/test/unit/src/config/workflow.ts | 88 +++++++++++++++------------ 1 file changed, 48 insertions(+), 40 deletions(-) diff --git a/core/test/unit/src/config/workflow.ts b/core/test/unit/src/config/workflow.ts index 62cd2e81fd..c8e59c9293 100644 --- a/core/test/unit/src/config/workflow.ts +++ b/core/test/unit/src/config/workflow.ts @@ -24,6 +24,8 @@ import { join } from "path" import { GardenApiVersion } from "../../../../src/constants.js" import { omit } from "lodash-es" import { GenericContext } from "../../../../src/config/template-contexts/base.js" +import { parseTemplateCollection } from "../../../../src/template/templated-collections.js" +import { serialiseUnresolvedTemplates } from "../../../../src/template/types.js" describe("resolveWorkflowConfig", () => { let garden: TestGarden @@ -130,24 +132,27 @@ describe("resolveWorkflowConfig", () => { }) it("should resolve template strings", async () => { - const config: WorkflowConfig = { - ...defaults, - apiVersion: GardenApiVersion.v0, - kind: "Workflow", - name: "workflow-a", - - description: "Secret: ${secrets.foo}, var: ${variables.foo}", - envVars: {}, - steps: [ - { - description: "Deploy the stack", - command: ["deploy"], - skip: "${var.skip}" as unknown as boolean, - when: "onSuccess", - envVars: {}, - }, - ], - } + const config: WorkflowConfig = parseTemplateCollection({ + value: { + ...defaults, + apiVersion: GardenApiVersion.v0, + kind: "Workflow" as const, + name: "workflow-a", + + description: "Secret: ${secrets.foo}, var: ${variables.foo}" as string, + envVars: {}, + steps: [ + { + description: "Deploy the stack", + command: ["deploy"], + skip: "${var.skip}" as unknown as boolean, + when: "onSuccess", + envVars: {}, + }, + ], + } as const, + source: { path: [] }, + }) const resolved = resolveWorkflowConfig(garden, config) @@ -156,25 +161,28 @@ describe("resolveWorkflowConfig", () => { }) it("should not resolve template strings in step commands and scripts", async () => { - const config: WorkflowConfig = { - ...defaults, - apiVersion: GardenApiVersion.v0, - kind: "Workflow", - name: "workflow-a", - - description: "foo", - envVars: {}, - steps: [ - { - ...defaultWorkflowStep, - description: "Deploy the stack", - command: ["deploy", "${var.foo}"], - skip: false, - when: "onSuccess", - }, - { ...defaultWorkflowStep, script: "echo ${var.foo}", skip: false, when: "onSuccess" }, - ], - } + const config: WorkflowConfig = parseTemplateCollection({ + value: { + ...defaults, + apiVersion: GardenApiVersion.v0, + kind: "Workflow" as const, + name: "workflow-a", + + description: "foo", + envVars: {}, + steps: [ + { + ...defaultWorkflowStep, + description: "Deploy the stack", + command: ["deploy", "${var.foo}" as string], + skip: false, + when: "onSuccess", + }, + { ...defaultWorkflowStep, script: "echo ${var.foo}" as string, skip: false, when: "onSuccess" }, + ], + } as const, + source: { path: [] }, + }) const resolved = resolveWorkflowConfig(garden, config) @@ -298,13 +306,13 @@ describe("resolveWorkflowConfig", () => { templateName: "workflows", inputs: { name: "test", - envName: "local", // <- should be resolved + envName: "${environment.name}", // unresolved }, } expect(workflow).to.exist - expect(workflow.steps[0].script).to.equal('echo "${environment.name}"') // <- resolved later - expect(omit(workflow.internal, "yamlDoc")).to.eql(internal) // <- `inputs.envName` should be resolved + expect(serialiseUnresolvedTemplates(workflow.steps[0].script)).to.equal('echo "${inputs.envName}"') // unresolved + expect(serialiseUnresolvedTemplates(omit(workflow.internal, "yamlDoc"))).to.eql(internal) }) describe("populateNamespaceForTriggers", () => { From c926b1aec2798a1c98ed172d699bc6adaba517b8 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Wed, 22 Jan 2025 16:40:48 +0100 Subject: [PATCH 078/117] fix: partiallyEvaluateModule accidentally compared the wrong thing --- core/src/resolve-module.ts | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index f24f791b2f..d07cac0740 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -1266,25 +1266,32 @@ function partiallyEvaluateModule(config: Input, co }, }, (value) => { - if (value instanceof ForEachLazyValue) { - return !someReferences({ + if ( + value instanceof ForEachLazyValue && + someReferences({ value, context, // if forEach expression has runtime references, we can't resolve it at all due to item context missing after converting the module to action // as the captured context is lost when calling `toJSON` on the unresolved template value onlyEssential: false, - matcher: (ref) => ref[0] !== "runtime", + matcher: (ref) => ref.keyPath[0] === "runtime", }) + ) { + return false // do not evaluate runtime references + } else if ( + someReferences({ + value, + context, + // in other cases, we only skip evaluation when the runtime references is essential + // meaning, we evaluate everything we can evaluate. + onlyEssential: true, + matcher: (ref) => ref.keyPath[0] === "runtime", + }) + ) { + return false // do not evaluate runtime references } - return !someReferences({ - value, - context, - // in other cases, we only skip evaluation when the runtime references is essential - // meaning, we evaluate everything we can evaluate. - onlyEssential: true, - matcher: (ref) => ref[0] !== "runtime", - }) + return true } ) From b42e6f4a15b43e90f05337dcdd5e60396728d31d Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Wed, 22 Jan 2025 17:52:22 +0100 Subject: [PATCH 079/117] fix: further test cases "ModuleResolver" --- core/src/resolve-module.ts | 33 ++++++++++++++-------------- core/src/template/types.ts | 1 + core/test/unit/src/resolve-module.ts | 23 ++++++++++++++----- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index d07cac0740..d8de7758c6 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -8,7 +8,7 @@ import { isArray, isString, keyBy, partition, uniq } from "lodash-es" import { validateWithPath } from "./config/validation.js" -import { mayContainTemplateString, resolveTemplateString } from "./template/templated-strings.js" +import { resolveTemplateString } from "./template/templated-strings.js" import { GenericContext } from "./config/template-contexts/base.js" import { dirname, posix, relative, resolve } from "path" import type { Garden } from "./garden.js" @@ -60,10 +60,11 @@ import { getModuleTemplateReferences } from "./config/references.js" import { capture } from "./template/capture.js" import { LayeredContext } from "./config/template-contexts/base.js" import { UnresolvedTemplateValue, type ParsedTemplate } from "./template/types.js" -import { conditionallyDeepEvaluate, deepEvaluate } from "./template/evaluate.js" +import { conditionallyDeepEvaluate, deepEvaluate, evaluate } from "./template/evaluate.js" import { someReferences } from "./template/analysis.js" import { ForEachLazyValue } from "./template/templated-collections.js" -import { deepMap } from "./util/objects.js" +import { deepMap, isPlainObject } from "./util/objects.js" +import { LazyMergePatch } from "./template/lazy-merge.js" // This limit is fairly arbitrary, but we need to have some cap on concurrent processing. export const moduleResolutionConcurrencyLimit = 50 @@ -540,16 +541,15 @@ export class ModuleResolver { // Try resolving template strings if possible let buildDeps: string[] = [] - const resolvedDeps = partiallyEvaluateModule( - rawConfig.build.dependencies as unknown as ParsedTemplate, - configContext - ) as unknown as BuildDependencyConfig - + const resolvedDeps = evaluate(rawConfig.build.dependencies as unknown as ParsedTemplate, { + context: configContext, + opts: {}, + }) // The build.dependencies field may not resolve at all, in which case we can't extract any deps from there if (isArray(resolvedDeps)) { buildDeps = resolvedDeps // We only collect fully-resolved references here - .filter((d) => !mayContainTemplateString(d) && (isString(d) || d.name)) + .filter((d) => isString(d) || (isPlainObject(d) && isString(d.name))) .map((d) => (isString(d) ? d : d.name)) } @@ -615,7 +615,7 @@ export class ModuleResolver { // so we also need to pass inputs here along with the available variables. const context = new ModuleConfigContext({ ...templateContextParams, - variables: new LayeredContext(garden.variables, moduleVariables), + variables: new LayeredContext(garden.variables, new GenericContext(moduleVariables)), inputs, }) @@ -637,7 +637,10 @@ export class ModuleResolver { } let config: ModuleConfig = partiallyEvaluateModule( - unresolvedConfig as unknown as ParsedTemplate, + { + ...unresolvedConfig, + variables: moduleVariables, + } as unknown as ParsedTemplate, context ) as unknown as ModuleConfig @@ -887,7 +890,7 @@ export class ModuleResolver { * * garden.variableOverrides > module varfile > config.variables */ - private async mergeVariables(config: ModuleConfig, context: ModuleConfigContext): Promise { + private async mergeVariables(config: ModuleConfig, context: ModuleConfigContext): Promise { let varfileVars: DeepPrimitiveMap = {} if (config.varfile) { const varfilePath = deepEvaluate(config.varfile, { @@ -908,11 +911,7 @@ export class ModuleResolver { const moduleVariables = capture(config.variables || {}, context) - return new LayeredContext( - new GenericContext(moduleVariables), - new GenericContext(varfileVars), - new GenericContext(this.garden.variableOverrides) - ) + return new LazyMergePatch([moduleVariables, varfileVars, this.garden.variableOverrides]) } } diff --git a/core/src/template/types.ts b/core/src/template/types.ts index 58b896844d..ded5d4747e 100644 --- a/core/src/template/types.ts +++ b/core/src/template/types.ts @@ -92,6 +92,7 @@ const accessDetector = new Proxy( ) export abstract class UnresolvedTemplateValue { + private readonly then: undefined // the js engine sometimes accesses this key protected constructor() { // The spread trap exists to make our code more robust by detecting spreading unresolved template values. Object.defineProperty(this, "objectSpreadTrap", { diff --git a/core/test/unit/src/resolve-module.ts b/core/test/unit/src/resolve-module.ts index 831b778b6a..dc4dfd67f3 100644 --- a/core/test/unit/src/resolve-module.ts +++ b/core/test/unit/src/resolve-module.ts @@ -22,9 +22,11 @@ import type { ConfigGraph } from "../../../src/graph/config-graph.js" import { loadYamlFile } from "../../../src/util/serialization.js" import type { DeployActionConfig } from "../../../src/actions/deploy.js" import type { BaseActionConfig } from "../../../src/actions/types.js" +import type { Log } from "../../../src/logger/log-entry.js" import { resolveMsg } from "../../../src/logger/log-entry.js" import { deline } from "../../../src/util/string.js" import stripAnsi from "strip-ansi" +import { resolveAction } from "../../../src/graph/actions.js" describe("ModuleResolver", () => { // Note: We test the ModuleResolver via the TestGarden.resolveModule method, for convenience. @@ -72,7 +74,10 @@ describe("ModuleResolver", () => { ]) const module = await garden.resolveModule("test-project-a") - expect(module.variables).to.eql({ foo: "override" }) + expect(module.variables).to.eql({ + bar: "no-override", // irrelevant overrides will appear in the variables of every module, but that doesn't hurt. + foo: "override", + }) }) it("handles a module template reference in a build dependency name", async () => { @@ -102,39 +107,45 @@ describe("ModuleResolver", () => { let dataDir: string let garden: TestGarden let graph: ConfigGraph + let log: Log before(async () => { dataDir = getDataDir("test-projects", "template-configs") garden = await makeTestGarden(dataDir) - graph = await garden.getConfigGraph({ log: garden.log, emit: false }) + log = garden.log + graph = await garden.getConfigGraph({ log, emit: false }) }) const expectedExtraFlags = "-Dbuilder=docker" context("should resolve vars and inputs with defaults", () => { it("with RenderTemplate and ConfigTemplate.configs", async () => { - const buildAction = graph.getBuild("render-template-based-build") + const action = graph.getBuild("render-template-based-build") + const buildAction = await resolveAction({ garden, graph, action, log }) const spec = buildAction.getConfig().spec expect(spec).to.exist expect(spec.extraFlags).to.eql([expectedExtraFlags]) }) it("with RenderTemplate and ConfigTemplate.modules", async () => { - const buildAction = graph.getBuild("render-template-based-module") + const action = graph.getBuild("render-template-based-module") + const buildAction = await resolveAction({ garden, graph, action, log }) const spec = buildAction.getConfig().spec expect(spec).to.exist expect(spec.extraFlags).to.eql([expectedExtraFlags]) }) it("with ModuleTemplate and ConfigTemplate.modules", async () => { - const buildAction = graph.getBuild("templated-module-based-module") + const action = graph.getBuild("templated-module-based-module") + const buildAction = await resolveAction({ garden, graph, action, log }) const spec = buildAction.getConfig().spec expect(spec).to.exist expect(spec.extraFlags).to.eql([expectedExtraFlags]) }) it("with RenderTemplate and ConfigTemplate.configs", async () => { - const buildAction = graph.getBuild("templated-module-based-build") + const action = graph.getBuild("templated-module-based-build") + const buildAction = await resolveAction({ garden, graph, action, log }) const spec = buildAction.getConfig().spec expect(spec).to.exist expect(spec.extraFlags).to.eql([expectedExtraFlags]) From 10cc2645a6c28022341779f69e0a2ff3dd4812b6 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Thu, 23 Jan 2025 10:32:48 +0100 Subject: [PATCH 080/117] test: trivial assertion corrections in "DefaultEnvironmentContext" --- core/test/unit/src/config/template-contexts/project.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/test/unit/src/config/template-contexts/project.ts b/core/test/unit/src/config/template-contexts/project.ts index 7021223177..0b4f9d9cb7 100644 --- a/core/test/unit/src/config/template-contexts/project.ts +++ b/core/test/unit/src/config/template-contexts/project.ts @@ -38,36 +38,42 @@ describe("DefaultEnvironmentContext", () => { it("should resolve the current git branch", () => { expect(c.resolve({ nodePath: [], key: ["git", "branch"], opts: {} })).to.eql({ + found: true, resolved: garden.vcsInfo.branch, }) }) it("should resolve the current git commit hash", () => { expect(c.resolve({ nodePath: [], key: ["git", "commitHash"], opts: {} })).to.eql({ + found: true, resolved: garden.vcsInfo.commitHash, }) }) it("should resolve the current git origin URL", () => { expect(c.resolve({ nodePath: [], key: ["git", "originUrl"], opts: {} })).to.eql({ + found: true, resolved: garden.vcsInfo.originUrl, }) }) it("should resolve datetime.now to ISO datetime string", () => { expect(c.resolve({ nodePath: [], key: ["datetime", "now"], opts: {} })).to.eql({ + found: true, resolved: now.toISOString(), }) }) it("should resolve datetime.today to ISO datetime string", () => { expect(c.resolve({ nodePath: [], key: ["datetime", "today"], opts: {} })).to.eql({ + found: true, resolved: now.toISOString().slice(0, 10), }) }) it("should resolve datetime.timestamp to Unix timestamp in seconds", () => { expect(c.resolve({ nodePath: [], key: ["datetime", "timestamp"], opts: {} })).to.eql({ + found: true, resolved: Math.round(now.getTime() / 1000), }) }) From 61dce6b08402d1f3ea6ec9586e4f8a900d4b65b2 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Thu, 23 Jan 2025 11:11:15 +0100 Subject: [PATCH 081/117] test: fix assertions in provider configuration tests Reverts wrong assertion changes made in 50e6d90f. --- core/test/unit/src/garden.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index fd30ebc33b..f890bfdf6a 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -2537,7 +2537,7 @@ describe("Garden", () => { const providerB = await garden.resolveProvider({ log: garden.log, name: "test-b" }) - expect(providerB.config["foo"]).to.equal("${providers.test-a.outputs.foo}") + expect(providerB.config["foo"]).to.equal("bar") }) it("should allow providers to reference outputs from a disabled provider", async () => { @@ -2576,7 +2576,7 @@ describe("Garden", () => { const providerB = await garden.resolveProvider({ log: garden.log, name: "test-b" }) - expect(providerB.config["foo"]).to.equal("${providers.test-a.outputs.foo || 'default'}") + expect(providerB.config["foo"]).to.equal("default") }) it("should allow providers to reference variables", async () => { @@ -2596,7 +2596,7 @@ describe("Garden", () => { const providerB = await garden.resolveProvider({ log: garden.log, name: "test-a" }) - expect(providerB.config["foo"]).to.equal("${var.my-variable}") + expect(providerB.config["foo"]).to.equal("bar") }) it("should match a dependency to a plugin base", async () => { From 5e104695a92358d09adac676688b93ba72e858ce Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Thu, 23 Jan 2025 11:30:22 +0100 Subject: [PATCH 082/117] fix: set provider path before calling `configureProvider` handler This fixes the test failure in "Garden.'resolveProviders.should call a configureProvider handler if applicable'" --- core/src/tasks/resolve-provider.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/tasks/resolve-provider.ts b/core/src/tasks/resolve-provider.ts index 9feb0c1aa3..c57ad5be63 100644 --- a/core/src/tasks/resolve-provider.ts +++ b/core/src/tasks/resolve-provider.ts @@ -213,6 +213,7 @@ export class ResolveProviderTask extends BaseTask { } let resolvedConfig = validateConfig(evaluatedConfig) + resolvedConfig.path = this.garden.projectRoot let moduleConfigs: ModuleConfig[] = [] @@ -226,7 +227,7 @@ export class ResolveProviderTask extends BaseTask { const pluginsByName = keyBy(plugins, "name") const plugin = pluginsByName[providerName] - const configureOutput = await actions.provider.configureProvider({ + const configureProviderOutput = await actions.provider.configureProvider({ ctx: await this.garden.getPluginContext({ provider: providerFromConfig({ plugin, @@ -250,11 +251,10 @@ export class ResolveProviderTask extends BaseTask { }) this.log.silly(() => `Validating ${providerName} config returned from configureProvider handler`) - resolvedConfig = validateConfig(configureOutput.config) - resolvedConfig.path = this.garden.projectRoot + resolvedConfig = validateConfig(configureProviderOutput.config) - if (configureOutput.moduleConfigs) { - moduleConfigs = configureOutput.moduleConfigs + if (configureProviderOutput.moduleConfigs) { + moduleConfigs = configureProviderOutput.moduleConfigs } // Validating the output config against the base plugins. This is important to make sure base handlers are From 1c1b8b216283cf94cc82af81db89ef59b0b01544 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Thu, 23 Jan 2025 11:33:45 +0100 Subject: [PATCH 083/117] Revert "fix: set provider path before calling `configureProvider` handler" This reverts commit 5e104695a92358d09adac676688b93ba72e858ce. --- core/src/tasks/resolve-provider.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/tasks/resolve-provider.ts b/core/src/tasks/resolve-provider.ts index c57ad5be63..9feb0c1aa3 100644 --- a/core/src/tasks/resolve-provider.ts +++ b/core/src/tasks/resolve-provider.ts @@ -213,7 +213,6 @@ export class ResolveProviderTask extends BaseTask { } let resolvedConfig = validateConfig(evaluatedConfig) - resolvedConfig.path = this.garden.projectRoot let moduleConfigs: ModuleConfig[] = [] @@ -227,7 +226,7 @@ export class ResolveProviderTask extends BaseTask { const pluginsByName = keyBy(plugins, "name") const plugin = pluginsByName[providerName] - const configureProviderOutput = await actions.provider.configureProvider({ + const configureOutput = await actions.provider.configureProvider({ ctx: await this.garden.getPluginContext({ provider: providerFromConfig({ plugin, @@ -251,10 +250,11 @@ export class ResolveProviderTask extends BaseTask { }) this.log.silly(() => `Validating ${providerName} config returned from configureProvider handler`) - resolvedConfig = validateConfig(configureProviderOutput.config) + resolvedConfig = validateConfig(configureOutput.config) + resolvedConfig.path = this.garden.projectRoot - if (configureProviderOutput.moduleConfigs) { - moduleConfigs = configureProviderOutput.moduleConfigs + if (configureOutput.moduleConfigs) { + moduleConfigs = configureOutput.moduleConfigs } // Validating the output config against the base plugins. This is important to make sure base handlers are From fb14cade24a9e1911af7b016111aa2b1293693db Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Thu, 23 Jan 2025 11:36:11 +0100 Subject: [PATCH 084/117] test: fix assertion in "resolveProviders.should call a configureProvider handler if applicable" --- core/test/unit/src/garden.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index f890bfdf6a..d38415fb73 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -2199,7 +2199,6 @@ describe("Garden", () => { expect(config).to.eql({ name: "test", dependencies: [], - path: projectConfig.path, foo: "bar", }) return { config: { ...config, foo: "bla" } } From 3247a3c1f4ce92c806c4f5a873988bf8c47187e8 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Thu, 23 Jan 2025 11:56:07 +0100 Subject: [PATCH 085/117] fix: remove unnecessary hack in `action.disable` flag resolution --- core/src/garden.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/core/src/garden.ts b/core/src/garden.ts index 5bf1b6d346..06405a1c19 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -1564,18 +1564,6 @@ export class Garden { } const context = new TemplatableConfigContext(this, config) - // Hack: deny variables contexts here, because those have not been fully resolved yet. - const deniedContexts = ["var", "variables"] - for (const deniedContext of deniedContexts) { - Object.defineProperty(context, deniedContext, { - get: () => { - throw new ConfigurationError({ - message: `If you have duplicate action names, the ${styles.accent("`disabled`")} flag cannot depend on the ${styles.accent(`\`${deniedContext}\``)} context.`, - }) - }, - }) - } - const resolved = deepEvaluate(disabledFlag, { context, opts: {} }) return !!resolved From ef3e6bb8852725517b57aab30c868e7d3a647b8e Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Thu, 23 Jan 2025 12:15:02 +0100 Subject: [PATCH 086/117] chore: extract defaults from input schema (best-effort) so we can apply the schema during action resolution --- core/src/actions/base.ts | 6 +- core/src/actions/build.ts | 2 +- core/src/actions/types.ts | 2 +- core/src/config/common.ts | 13 +-- core/src/config/config-template.ts | 17 +++- core/src/config/render-template.ts | 7 +- core/src/config/secrets.ts | 1 - core/src/config/template-contexts/actions.ts | 9 +- core/src/config/template-contexts/base.ts | 28 ++---- core/src/config/template-contexts/input.ts | 61 +++++++++++++ core/src/config/template-contexts/module.ts | 7 +- core/src/config/template-contexts/project.ts | 36 +------- core/src/config/template-contexts/render.ts | 8 +- .../config/template-contexts/templatable.ts | 44 +++++++++ core/src/config/template-contexts/workflow.ts | 3 +- core/src/config/workflow.ts | 6 -- core/src/garden.ts | 2 +- core/src/graph/actions.ts | 59 +----------- core/src/resolve-module.ts | 87 ++++++++---------- core/src/router/base.ts | 5 +- core/src/tasks/publish.ts | 3 +- core/src/tasks/resolve-action.ts | 89 +++++++++++-------- .../module-templates/module-templates.json | 6 +- core/test/unit/src/config/render-template.ts | 4 + .../src/config/template-contexts/module.ts | 3 +- core/test/unit/src/tasks/resolve-action.ts | 2 +- plugins/pulumi/src/commands.ts | 5 +- 27 files changed, 274 insertions(+), 241 deletions(-) create mode 100644 core/src/config/template-contexts/input.ts create mode 100644 core/src/config/template-contexts/templatable.ts diff --git a/core/src/actions/base.ts b/core/src/actions/base.ts index 08a49588e3..049779d893 100644 --- a/core/src/actions/base.ts +++ b/core/src/actions/base.ts @@ -68,6 +68,7 @@ import { styles } from "../logger/styles.js" import { dirname } from "node:path" import type { ConfigContext } from "../config/template-contexts/base.js" import type { ResolvedTemplate } from "../template/types.js" +import type { WorkflowConfig } from "../config/workflow.js" // TODO: split this file @@ -905,7 +906,10 @@ export function getSourceAbsPath(basePath: string, sourceRelPath: string) { return joinWithPosix(basePath, sourceRelPath) } -export function describeActionConfig(config: ActionConfig) { +export function describeActionConfig(config: ActionConfig | WorkflowConfig) { + if (config.kind === "Workflow") { + return `${config.kind} ${config.name}` + } const d = `${config.type} ${config.kind} ${config.name}` if (config.internal?.moduleName) { return d + ` (from module ${config.internal?.moduleName})` diff --git a/core/src/actions/build.ts b/core/src/actions/build.ts index a15ec47385..9595918c09 100644 --- a/core/src/actions/build.ts +++ b/core/src/actions/build.ts @@ -31,7 +31,7 @@ import { DEFAULT_BUILD_TIMEOUT_SEC } from "../constants.js" import { createBuildTask } from "../tasks/build.js" import type { BaseActionTaskParams, ExecuteTask } from "../tasks/base.js" import { ResolveActionTask } from "../tasks/resolve-action.js" -import { ResolvedTemplate } from "../template/types.js" +import type { ResolvedTemplate } from "../template/types.js" export interface BuildCopyFrom { build: string diff --git a/core/src/actions/types.ts b/core/src/actions/types.ts index db2752053b..4034378e9f 100644 --- a/core/src/actions/types.ts +++ b/core/src/actions/types.ts @@ -22,7 +22,7 @@ import type { BaseGardenResource, GardenResourceInternalFields } from "../config import type { LinkedSource } from "../config-store/local.js" import type { GardenApiVersion } from "../constants.js" import type { ConfigContext } from "../config/template-contexts/base.js" -import { ResolvedTemplate } from "../template/types.js" +import type { ResolvedTemplate } from "../template/types.js" // TODO: split this file diff --git a/core/src/config/common.ts b/core/src/config/common.ts index 182444dffc..78a29214c6 100644 --- a/core/src/config/common.ts +++ b/core/src/config/common.ts @@ -8,6 +8,7 @@ import type { SchemaLike } from "@hapi/joi" import Joi from "@hapi/joi" +import type { JSONSchemaType, ValidateFunction } from "ajv" import ajvPackage from "ajv" const Ajv = ajvPackage.default @@ -167,7 +168,7 @@ declare module "@hapi/joi" { export interface CustomObjectSchema extends Joi.ObjectSchema { concat(schema: object): this - jsonSchema(schema: object): this + jsonSchema(schema: any): this zodSchema(schema: z.ZodObject): this } @@ -369,7 +370,7 @@ joi = joi.extend({ * Compiles a JSON schema and caches the result. */ const compileJsonSchema = memoize( - (schema: any) => { + (schema: JSONSchemaType>) => { return ajv.compile(schema) }, (s) => JSON.stringify(s) @@ -406,7 +407,7 @@ joi = joi.extend({ }, rules: { jsonSchema: { - method(jsonSchema: object) { + method(jsonSchema: JSONSchemaType) { // eslint-disable-next-line no-invalid-this this.$_setFlag("jsonSchema", jsonSchema) // eslint-disable-next-line no-invalid-this @@ -419,7 +420,7 @@ joi = joi.extend({ return !!value }, message: "must be a valid JSON Schema with type=object", - normalize: (value) => { + normalize: (value: JSONSchemaType): false | ValidateFunction> => { if (value.type !== "object") { return false } @@ -433,7 +434,7 @@ joi = joi.extend({ }, ], validate(originalValue, helpers, args) { - const validate = args.jsonSchema + const validate: ValidateFunction> = args.jsonSchema // Need to do this to be able to assign defaults without mutating original value const value = cloneDeep(originalValue) @@ -443,7 +444,7 @@ joi = joi.extend({ return value } else { // TODO: customize the rendering here to make it a bit nicer - const errors = [...validate.errors] + const errors = [...validate.errors!] const error = helpers.error("validation") error.message = ajv.errorsText(errors, { dataVar: `value at ${joiPathPlaceholder}` }) return error diff --git a/core/src/config/config-template.ts b/core/src/config/config-template.ts index 3cd5f85d2f..5b4919b114 100644 --- a/core/src/config/config-template.ts +++ b/core/src/config/config-template.ts @@ -25,6 +25,8 @@ import { actionKinds } from "../actions/types.js" import type { WorkflowConfig } from "./workflow.js" import { deepEvaluate } from "../template/evaluate.js" import type { ParsedTemplate } from "../template/types.js" +import type { JSONSchemaType } from "ajv" +import type { DeepPrimitiveMap } from "@garden-io/platform-api-types" const inputTemplatePattern = "${inputs.*}" const parentNameTemplate = "${parent.name}" @@ -53,6 +55,7 @@ export interface ConfigTemplateResource extends BaseGardenResource { export interface ConfigTemplateConfig extends ConfigTemplateResource { inputsSchema: CustomObjectSchema + inputsSchemaDefaults: DeepPrimitiveMap } export async function resolveConfigTemplate( @@ -84,9 +87,10 @@ export async function resolveConfigTemplate( // Read and validate the JSON schema, if specified // -> default to any object - let inputsJsonSchema = { + let inputsJsonSchema: JSONSchemaType = { type: "object", additionalProperties: true, + required: [], } const configDir = configPath ? dirname(configPath) : resource.internal.basePath @@ -110,10 +114,21 @@ export async function resolveConfigTemplate( } } + const defaultValues = {} + + // this does not cover all the edge cases, consider using something like https://www.npmjs.com/package/json-schema-default + if (inputsJsonSchema.properties) { + for (const k in inputsJsonSchema.properties) { + const d = inputsJsonSchema.properties[k].default + defaultValues[k] = d + } + } + // Add the module templates back and return return { ...validated, inputsSchema: joi.object().jsonSchema(inputsJsonSchema), + inputsSchemaDefaults: defaultValues, modules: resource.modules, configs: resource.configs, } diff --git a/core/src/config/render-template.ts b/core/src/config/render-template.ts index 37b191ee9e..7f97ddda5e 100644 --- a/core/src/config/render-template.ts +++ b/core/src/config/render-template.ts @@ -38,6 +38,7 @@ import { capture } from "../template/capture.js" import { deepEvaluate, evaluate } from "../template/evaluate.js" import { UnresolvedTemplateValue, type ParsedTemplate } from "../template/types.js" import { isArray, isPlainObject } from "../util/objects.js" +import { InputContext } from "./template-contexts/input.js" export const renderTemplateConfigSchema = createSchema({ name: renderTemplateKind, @@ -132,7 +133,7 @@ export async function renderConfigTemplate({ }) let resolved: RenderTemplateConfig = { ...(resolvedWithoutInputs as unknown as RenderTemplateConfig), - inputs: capture(config.inputs || {}, templateContext) as unknown as DeepPrimitiveMap, + inputs: config.inputs, } const configType = "Render " + resolved.name @@ -170,7 +171,7 @@ export async function renderConfigTemplate({ enterpriseDomain, parentName: resolved.name, templateName: template.name, - inputs: config.inputs || {}, + inputs: InputContext.forRenderTemplate(config, template), }) // TODO: remove in 0.14 @@ -355,8 +356,8 @@ async function renderConfigs({ // Attach metadata resource.internal.parentName = renderConfig.name resource.internal.templateName = template.name - resource.internal.inputs = renderConfig.inputs + resource.internal.inputs = renderConfig.inputs return resource }) ) diff --git a/core/src/config/secrets.ts b/core/src/config/secrets.ts index 313c0c49d6..d9b8bde6da 100644 --- a/core/src/config/secrets.ts +++ b/core/src/config/secrets.ts @@ -15,7 +15,6 @@ import type { StringMap } from "./common.js" import type { ConfigContext, ContextKeySegment } from "./template-contexts/base.js" import difference from "lodash-es/difference.js" import { ConfigurationError } from "../exceptions.js" -import { flatten, uniq } from "lodash-es" /** * Gathers secret references in configs and throws an error if one or more referenced secrets isn't present (or has diff --git a/core/src/config/template-contexts/actions.ts b/core/src/config/template-contexts/actions.ts index ce54728641..9c0277e38f 100644 --- a/core/src/config/template-contexts/actions.ts +++ b/core/src/config/template-contexts/actions.ts @@ -11,16 +11,17 @@ import type { ActionMode } from "../../actions/types.js" import type { Garden } from "../../garden.js" import type { GardenModule } from "../../types/module.js" import { dedent, deline } from "../../util/string.js" -import type { DeepPrimitiveMap, PrimitiveMap } from "../common.js" +import type { PrimitiveMap } from "../common.js" import { joi, joiIdentifier, joiIdentifierMap, joiPrimitive, joiVariables } from "../common.js" import type { ProviderMap } from "../provider.js" import type { ConfigContext } from "./base.js" import { ContextWithSchema, ErrorContext, GenericContext, ParentContext, schema, TemplateContext } from "./base.js" import { exampleVersion, OutputConfigContext } from "./module.js" -import { TemplatableConfigContext } from "./project.js" +import { TemplatableConfigContext } from "./templatable.js" import { DOCS_BASE_URL } from "../../constants.js" import { styles } from "../../logger/styles.js" import { LayeredContext } from "./base.js" +import { InputContext } from "./input.js" function mergeVariables({ garden, variables }: { garden: Garden; variables: ConfigContext }): LayeredContext { return new LayeredContext(garden.variables, variables, new GenericContext(garden.variableOverrides)) @@ -227,7 +228,7 @@ export interface ActionSpecContextParams { resolvedDependencies: ResolvedAction[] executedDependencies: ExecutedAction[] variables: ConfigContext - inputs: DeepPrimitiveMap + inputs: InputContext } /** @@ -249,7 +250,7 @@ export class ActionSpecContext extends OutputConfigContext { keyPlaceholder: "", }) ) - public inputs: DeepPrimitiveMap + public inputs: InputContext @schema( ParentContext.getSchema().description( diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 7de1736035..696f0961d5 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -16,10 +16,11 @@ import type { ParsedTemplate, ParsedTemplateValue, ResolvedTemplate } from "../. import { isTemplatePrimitive, UnresolvedTemplateValue } from "../../template/types.js" import merge from "lodash-es/merge.js" import omitBy from "lodash-es/omitBy.js" -import { flatten, isEqual, uniq } from "lodash-es" +import { flatten, isEqual, isString, uniq } from "lodash-es" import { isMap } from "util/types" import { deline } from "../../util/string.js" import { styles } from "../../logger/styles.js" +import type { ContextLookupReferenceFinding } from "../../template/analysis.js" export type ContextKeySegment = string | number export type ContextKey = ContextKeySegment[] @@ -177,6 +178,10 @@ export abstract class ContextWithSchema extends ConfigContext { return joi.object().keys(schemas).required() } + public hasReferenceRoot(ref: ContextLookupReferenceFinding): boolean { + return isString(ref.keyPath[0]) && this[ref.keyPath[0]] !== undefined + } + protected override resolveImpl(params: ContextResolveParams): ContextResolveOutput { return traverseContext( omitBy(this, (key) => typeof key === "string" && key.startsWith("_")) as CollectionOrValue< @@ -297,27 +302,6 @@ export function renderKeyPath(key: ContextKeySegment[]): string { ) } -export class CapturedContext extends ConfigContext { - constructor( - private readonly wrapped: ConfigContext, - private readonly rootContext: ConfigContext - ) { - super() - } - - override resolveImpl(params: ContextResolveParams): ContextResolveOutput { - return this.wrapped.resolve({ - ...params, - opts: { - ...params.opts, - // to avoid circular dep errors - stack: params.opts.stack?.slice(0, -1), - }, - rootContext: params.rootContext ? new LayeredContext(this.rootContext, params.rootContext) : this.rootContext, - }) - } -} - export class LayeredContext extends ConfigContext { private readonly contexts: ConfigContext[] diff --git a/core/src/config/template-contexts/input.ts b/core/src/config/template-contexts/input.ts new file mode 100644 index 0000000000..6f2b0d17e6 --- /dev/null +++ b/core/src/config/template-contexts/input.ts @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2018-2024 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +import { GenericContext, LayeredContext } from "./base.js" +import type { Garden } from "../../garden.js" +import { describeConfig } from "../../vcs/vcs.js" +import type { ActionConfig } from "../../actions/types.js" +import type { WorkflowConfig } from "../workflow.js" +import { InternalError } from "../../exceptions.js" +import { describeActionConfig } from "../../actions/base.js" +import type { ConfigTemplateConfig } from "../config-template.js" +import type { ParsedTemplate } from "../../template/types.js" +import type { RenderTemplateConfig } from "../render-template.js" +import type { ModuleConfig } from "../module.js" + +export class InputContext extends LayeredContext { + public static forAction(garden: Garden, config: ActionConfig | WorkflowConfig): InputContext { + const templateName = config.internal.templateName + if (templateName) { + const template = garden.configTemplates[templateName] + if (!template) { + throw new InternalError({ + message: `Could not find template name ${templateName} for ${describeActionConfig(config)}`, + }) + } + return new this(config.internal.inputs, template) + } + + return new this(config.internal.inputs) + } + + public static forRenderTemplate(config: RenderTemplateConfig, template: ConfigTemplateConfig) { + return new this(config.inputs, template) + } + + public static forModule(garden: Garden, module: ModuleConfig) { + if (module.templateName) { + const template = garden.configTemplates[module.templateName] + if (!template) { + throw new InternalError({ + message: `Could not find template name ${module.templateName} for ${describeConfig(module)}`, + }) + } + return new this(module.inputs, template) + } + + return new this(module.inputs) + } + + constructor(inputs: ParsedTemplate, template?: ConfigTemplateConfig) { + if (template) { + super(new GenericContext(template.inputsSchemaDefaults), new GenericContext(inputs || {})) + } else { + super(new GenericContext(inputs || {})) + } + } +} diff --git a/core/src/config/template-contexts/module.ts b/core/src/config/template-contexts/module.ts index bfce30972b..f2aae7ebd7 100644 --- a/core/src/config/template-contexts/module.ts +++ b/core/src/config/template-contexts/module.ts @@ -22,6 +22,7 @@ import type { DeployTask } from "../../tasks/deploy.js" import type { RunTask } from "../../tasks/run.js" import { DOCS_BASE_URL } from "../../constants.js" import { styles } from "../../logger/styles.js" +import { InputContext } from "./input.js" export const exampleVersion = "v-17ad4cb3fd" @@ -237,7 +238,7 @@ export interface ModuleConfigContextParams extends OutputConfigContextParams { // Template attributes parentName: string | undefined templateName: string | undefined - inputs: DeepPrimitiveMap | undefined + inputs: InputContext } /** @@ -249,7 +250,7 @@ export class ModuleConfigContext extends OutputConfigContext { keyPlaceholder: "", }) ) - public inputs: DeepPrimitiveMap + public inputs: InputContext @schema( ParentContext.getSchema().description( @@ -280,7 +281,7 @@ export class ModuleConfigContext extends OutputConfigContext { this.parent = new ParentContext(parentName) this.template = new TemplateContext(templateName) } - this.inputs = inputs || {} + this.inputs = inputs this.this = new ModuleThisContext({ buildPath, name, path }) } diff --git a/core/src/config/template-contexts/project.ts b/core/src/config/template-contexts/project.ts index bfb7008ef4..4d544694b2 100644 --- a/core/src/config/template-contexts/project.ts +++ b/core/src/config/template-contexts/project.ts @@ -12,12 +12,10 @@ import { joiIdentifierMap, joiStringMap, joiPrimitive, joiVariables } from "../c import { joi } from "../common.js" import { deline, dedent } from "../../util/string.js" import type { ConfigContext, ContextResolveParams } from "./base.js" -import { schema, ContextWithSchema, EnvironmentContext, ParentContext, TemplateContext } from "./base.js" +import { schema, ContextWithSchema, EnvironmentContext } from "./base.js" import type { CommandInfo } from "../../plugin-context.js" import type { Garden } from "../../garden.js" -import type { VcsInfo } from "../../vcs/vcs.js" -import type { ActionConfig } from "../../actions/types.js" -import type { WorkflowConfig } from "../workflow.js" +import { type VcsInfo } from "../../vcs/vcs.js" import { styles } from "../../logger/styles.js" class LocalContext extends ContextWithSchema { @@ -415,33 +413,3 @@ export class RemoteSourceConfigContext extends EnvironmentConfigContext { this.variables = this.var = variables } } - -export class TemplatableConfigContext extends RemoteSourceConfigContext { - @schema( - joiVariables().description(`The inputs provided to the config through a template, if applicable.`).meta({ - keyPlaceholder: "", - }) - ) - public inputs: DeepPrimitiveMap - - @schema( - ParentContext.getSchema().description( - `Information about the config parent, if any (usually a template, if applicable).` - ) - ) - public parent?: ParentContext - - @schema( - TemplateContext.getSchema().description( - `Information about the template used when generating the config, if applicable.` - ) - ) - public template?: TemplateContext - - constructor(garden: Garden, config: ActionConfig | WorkflowConfig) { - super(garden, garden.variables) - this.inputs = config.internal.inputs || {} - this.parent = config.internal.parentName ? new ParentContext(config.internal.parentName) : undefined - this.template = config.internal.templateName ? new TemplateContext(config.internal.templateName) : undefined - } -} diff --git a/core/src/config/template-contexts/render.ts b/core/src/config/template-contexts/render.ts index d61bd7244b..b309ef6489 100644 --- a/core/src/config/template-contexts/render.ts +++ b/core/src/config/template-contexts/render.ts @@ -6,9 +6,9 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import type { DeepPrimitiveMap } from "../common.js" import { joiVariables } from "../common.js" import { ParentContext, schema, TemplateContext } from "./base.js" +import type { InputContext } from "./input.js" import type { ProjectConfigContextParams } from "./project.js" import { ProjectConfigContext } from "./project.js" @@ -24,11 +24,9 @@ export class RenderTemplateConfigContext extends ProjectConfigContext { keyPlaceholder: "", }) ) - public inputs: DeepPrimitiveMap + public inputs: InputContext - constructor( - params: { parentName: string; templateName: string; inputs: DeepPrimitiveMap } & ProjectConfigContextParams - ) { + constructor(params: { parentName: string; templateName: string; inputs: InputContext } & ProjectConfigContextParams) { super(params) this.parent = new ParentContext(params.parentName) this.template = new TemplateContext(params.templateName) diff --git a/core/src/config/template-contexts/templatable.ts b/core/src/config/template-contexts/templatable.ts new file mode 100644 index 0000000000..b66564ac84 --- /dev/null +++ b/core/src/config/template-contexts/templatable.ts @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2018-2024 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +import type { ActionConfig } from "../../actions/types.js" +import type { Garden } from "../../index.js" +import { joiVariables } from "../common.js" +import type { WorkflowConfig } from "../workflow.js" +import { schema, ParentContext, TemplateContext } from "./base.js" +import { InputContext } from "./input.js" +import { RemoteSourceConfigContext } from "./project.js" + +export class TemplatableConfigContext extends RemoteSourceConfigContext { + @schema( + joiVariables().description(`The inputs provided to the config through a template, if applicable.`).meta({ + keyPlaceholder: "", + }) + ) + public inputs: InputContext + + @schema( + ParentContext.getSchema().description( + `Information about the config parent, if any (usually a template, if applicable).` + ) + ) + public parent?: ParentContext + + @schema( + TemplateContext.getSchema().description( + `Information about the template used when generating the config, if applicable.` + ) + ) + public template?: TemplateContext + + constructor(garden: Garden, config: ActionConfig | WorkflowConfig) { + super(garden, garden.variables) + this.inputs = InputContext.forAction(garden, config) + this.parent = config.internal.parentName ? new ParentContext(config.internal.parentName) : undefined + this.template = config.internal.templateName ? new TemplateContext(config.internal.templateName) : undefined + } +} diff --git a/core/src/config/template-contexts/workflow.ts b/core/src/config/template-contexts/workflow.ts index d0fcb46e1f..442b59b00b 100644 --- a/core/src/config/template-contexts/workflow.ts +++ b/core/src/config/template-contexts/workflow.ts @@ -11,7 +11,8 @@ import { joiIdentifierMap, joiVariables } from "../common.js" import type { Garden } from "../../garden.js" import { joi } from "../common.js" import { dedent } from "../../util/string.js" -import { RemoteSourceConfigContext, TemplatableConfigContext } from "./project.js" +import { RemoteSourceConfigContext } from "./project.js" +import { TemplatableConfigContext } from "./templatable.js" import { schema, ContextWithSchema, ErrorContext } from "./base.js" import type { WorkflowConfig } from "../workflow.js" diff --git a/core/src/config/workflow.ts b/core/src/config/workflow.ts index aca3ca6f3c..4ecd3b41ca 100644 --- a/core/src/config/workflow.ts +++ b/core/src/config/workflow.ts @@ -30,10 +30,8 @@ import { omitUndefined } from "../util/objects.js" import type { BaseGardenResource, GardenResource } from "./base.js" import type { GardenApiVersion } from "../constants.js" import { DOCS_BASE_URL } from "../constants.js" -import { capture } from "../template/capture.js" import { deepEvaluate } from "../template/evaluate.js" import type { ParsedTemplate } from "../template/types.js" -import type { DeepPrimitiveMap } from "@garden-io/platform-api-types" export const minimumWorkflowRequests = { cpu: 50, // 50 millicpu @@ -375,10 +373,6 @@ export function resolveWorkflowConfig(garden: Garden, config: WorkflowConfig) { resolvedPartialConfig.triggers = config.triggers } - if (config.internal.inputs) { - resolvedPartialConfig.internal.inputs = capture(config.internal.inputs, context) as unknown as DeepPrimitiveMap - } - log.silly(() => `Validating config for workflow ${config.name}`) resolvedPartialConfig = validateConfig({ diff --git a/core/src/garden.ts b/core/src/garden.ts index 06405a1c19..8a32dbcc72 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -116,8 +116,8 @@ import { DefaultEnvironmentContext, ProjectConfigContext, RemoteSourceConfigContext, - TemplatableConfigContext, } from "./config/template-contexts/project.js" +import { TemplatableConfigContext } from "./config/template-contexts/templatable.js" import type { GardenCloudApiFactory } from "./cloud/api.js" import { GardenCloudApi, CloudApiTokenRefreshError } from "./cloud/api.js" import { OutputConfigContext } from "./config/template-contexts/module.js" diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index 7c7bbfda34..32864b7533 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -40,7 +40,7 @@ import type { ActionReference, JoiDescription } from "../config/common.js" import { describeSchema, parseActionReference } from "../config/common.js" import type { GroupConfig } from "../config/group.js" import { ActionConfigContext } from "../config/template-contexts/actions.js" -import { ConfigurationError, GardenError, InternalError, PluginError } from "../exceptions.js" +import { ConfigurationError, GardenError, PluginError } from "../exceptions.js" import { type Garden, overrideVariables } from "../garden.js" import type { Log } from "../logger/log-entry.js" import type { ActionTypeDefinition } from "../plugin/action-types.js" @@ -67,10 +67,8 @@ import { styles } from "../logger/styles.js" import { isUnresolvableValue } from "../template/analysis.js" import { getActionTemplateReferences } from "../config/references.js" import { capture } from "../template/capture.js" -import { CapturedContext } from "../config/template-contexts/base.js" import { deepEvaluate } from "../template/evaluate.js" import type { ParsedTemplate } from "../template/types.js" -import type { DeepPrimitiveMap } from "@garden-io/platform-api-types" import { validateWithPath } from "../config/validation.js" function* sliceToBatches(dict: Record, batchSize: number) { @@ -737,7 +735,6 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi actionTypes: ActionDefinitionMap }): Promise { const description = describeActionConfig(config) - const templateName = config.internal.templateName // in pre-processing, only use varfiles that are not template strings const resolvedVarFiles = config.varfiles?.filter((f) => !maybeTemplateString(getVarfileData(f).path)) @@ -748,58 +745,6 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi log, }) - const resolvedVariables = new CapturedContext( - variables, - new ActionConfigContext({ - garden, - config: { ...config, internal: { ...config.internal, inputs: {} } }, - thisContextParams: { - mode, - name: config.name, - }, - variables, - }) - ) - - if (templateName) { - // Partially resolve inputs - config.internal.inputs = capture( - config.internal.inputs || {}, - new ActionConfigContext({ - garden, - config: { ...config, internal: { ...config.internal, inputs: {} } }, - thisContextParams: { - mode, - name: config.name, - }, - variables: resolvedVariables, - }) - ) as unknown as DeepPrimitiveMap - - const template = garden.configTemplates[templateName] - - // Note: This shouldn't happen in normal user flows - if (!template) { - throw new InternalError({ - message: `${description} references template '${templateName}' which cannot be found. Available templates: ${ - naturalList(Object.keys(garden.configTemplates)) || "(none)" - }`, - }) - } - - // Validate inputs schema - // TODO: schema validation on partially resolved inputs does not make sense - // do we validate the schema on fully resolved inputs somewhere? - // config.internal.inputs = validateWithPath({ - // config: partiallyResolvedInputs, - // configType: `inputs for ${description}`, - // path: config.internal.basePath, - // schema: template.inputsSchema, - // projectRoot: garden.projectRoot, - // source: undefined, - // }) - } - const builtinConfigKeys = getBuiltinConfigContextKeys() const builtinFieldContext = new ActionConfigContext({ garden, @@ -808,7 +753,7 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi mode, name: config.name, }, - variables: resolvedVariables, + variables, }) function resolveTemplates() { diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index d8de7758c6..d886aaffc6 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -65,6 +65,7 @@ import { someReferences } from "./template/analysis.js" import { ForEachLazyValue } from "./template/templated-collections.js" import { deepMap, isPlainObject } from "./util/objects.js" import { LazyMergePatch } from "./template/lazy-merge.js" +import { InputContext } from "./config/template-contexts/input.js" // This limit is fairly arbitrary, but we need to have some cap on concurrent processing. export const moduleResolutionConcurrencyLimit = 50 @@ -520,17 +521,11 @@ export class ModuleResolver { buildPath, parentName: rawConfig.parentName, templateName: rawConfig.templateName, - inputs: {}, + inputs: InputContext.forModule(this.garden, rawConfig), modules: [], graphResults: this.graphResults, } - // Template inputs are commonly used in module deps, so we need to resolve them first - contextParams.inputs = this.resolveInputs( - rawConfig, - new ModuleConfigContext(contextParams) - ) as unknown as DeepPrimitiveMap - const configContext = new ModuleConfigContext(contextParams) const templateDeps = getModuleTemplateReferences(rawConfig, configContext) @@ -571,16 +566,6 @@ export class ModuleResolver { return getLinkedSources(this.garden, "module") } - private resolveInputs(config: ModuleConfig, configContext: ModuleConfigContext) { - const inputs = config.inputs || {} - - if (!config.templateName) { - return inputs - } - - return capture(inputs, configContext) - } - /** * Resolves and validates a single module configuration. */ @@ -589,8 +574,6 @@ export class ModuleResolver { const buildPath = this.garden.buildStaging.getBuildPath(unresolvedConfig) - const inputs = unresolvedConfig.inputs || {} - const templateContextParams: ModuleConfigContextParams = { garden, variables: garden.variables, @@ -601,7 +584,7 @@ export class ModuleResolver { buildPath, parentName: unresolvedConfig.parentName, templateName: unresolvedConfig.templateName, - inputs, + inputs: InputContext.forModule(garden, unresolvedConfig), graphResults: this.graphResults, } @@ -616,7 +599,6 @@ export class ModuleResolver { const context = new ModuleConfigContext({ ...templateContextParams, variables: new LayeredContext(garden.variables, new GenericContext(moduleVariables)), - inputs, }) const moduleTypeDefinitions = await garden.getModuleTypes() @@ -766,7 +748,7 @@ export class ModuleResolver { buildPath, parentName: resolvedConfig.parentName, templateName: resolvedConfig.templateName, - inputs: resolvedConfig.inputs, + inputs: InputContext.forModule(this.garden, resolvedConfig), modules: dependencies, graphResults: this.graphResults, }) @@ -1252,6 +1234,38 @@ function getTestNames(config: ModuleConfig) { * This function can be deleted together with `conditionallyDeepEvaluate` and the `unescape` option at that point. */ function partiallyEvaluateModule(config: Input, context: ModuleConfigContext) { + /** + * Returns true if the unresolved template value can be resolved at this point and false otherwise. + */ + const shouldEvaluate = (value: UnresolvedTemplateValue) => { + if ( + value instanceof ForEachLazyValue && + someReferences({ + value, + context, + // if forEach expression has runtime references, we can't resolve it at all due to item context missing after converting the module to action + // as the captured context is lost when calling `toJSON` on the unresolved template value + onlyEssential: false, + matcher: (ref) => ref.keyPath[0] === "runtime", + }) + ) { + return false // do not evaluate runtime references + } else if ( + someReferences({ + value, + context, + // in other cases, we only skip evaluation when the runtime references is essential + // meaning, we evaluate everything we can evaluate. + onlyEssential: true, + matcher: (ref) => ref.keyPath[0] === "runtime", + }) + ) { + return false // do not evaluate runtime references + } + + return true + } + const partial = conditionallyDeepEvaluate( config, { @@ -1264,34 +1278,7 @@ function partiallyEvaluateModule(config: Input, co unescape: false, }, }, - (value) => { - if ( - value instanceof ForEachLazyValue && - someReferences({ - value, - context, - // if forEach expression has runtime references, we can't resolve it at all due to item context missing after converting the module to action - // as the captured context is lost when calling `toJSON` on the unresolved template value - onlyEssential: false, - matcher: (ref) => ref.keyPath[0] === "runtime", - }) - ) { - return false // do not evaluate runtime references - } else if ( - someReferences({ - value, - context, - // in other cases, we only skip evaluation when the runtime references is essential - // meaning, we evaluate everything we can evaluate. - onlyEssential: true, - matcher: (ref) => ref.keyPath[0] === "runtime", - }) - ) { - return false // do not evaluate runtime references - } - - return true - } + shouldEvaluate ) return deepMap(partial, (v) => { diff --git a/core/src/router/base.ts b/core/src/router/base.ts index 973c928406..0a729e57a4 100644 --- a/core/src/router/base.ts +++ b/core/src/router/base.ts @@ -39,9 +39,10 @@ import { getNames } from "../util/util.js" import { defaultProvider } from "../config/provider.js" import type { ConfigGraph } from "../graph/config-graph.js" import { ActionConfigContext, ActionSpecContext } from "../config/template-contexts/actions.js" -import { TemplatableConfigContext } from "../config/template-contexts/project.js" +import { TemplatableConfigContext } from "../config/template-contexts/templatable.js" import type { ParamsBase } from "../plugin/handlers/base/base.js" import { Profile } from "../util/profiling.js" +import { InputContext } from "../config/template-contexts/input.js" export type CommonParams = keyof PluginActionContextParams @@ -298,7 +299,7 @@ export abstract class BaseActionRouter extends BaseRouter modules: graph.getModules(), resolvedDependencies: action.getResolvedDependencies(), executedDependencies: action.getExecutedDependencies(), - inputs: action.getInternal().inputs || {}, + inputs: new InputContext(action.getInternal().inputs), variables: action.getVariablesContext(), }) : new ActionConfigContext({ diff --git a/core/src/tasks/publish.ts b/core/src/tasks/publish.ts index 783cca8424..ee02c5894e 100644 --- a/core/src/tasks/publish.ts +++ b/core/src/tasks/publish.ts @@ -18,6 +18,7 @@ import type { BuildAction } from "../actions/build.js" import type { ActionSpecContextParams } from "../config/template-contexts/actions.js" import { ActionSpecContext } from "../config/template-contexts/actions.js" import { OtelTraced } from "../util/open-telemetry/decorators.js" +import { InputContext } from "../config/template-contexts/input.js" export interface PublishTaskParams extends BaseActionTaskParams { /** @@ -109,7 +110,7 @@ export class PublishTask extends BaseActionTask extends ValidResultType { state: ActionState @@ -123,7 +125,45 @@ export class ResolveActionTask extends BaseActionTask({ + config: resolvedInputs, + configType: `inputs for action ${config.name}`, + path: this.action.effectiveConfigFileLocation(), + schema: template.inputsSchema, + projectRoot: this.garden.projectRoot, + source: undefined, + }) + } + + const inputs = new InputContext( + resolvedInputs, + // defaults are already applied on the inputs + undefined + ) + const inputsContext = new ActionSpecContext({ garden: this.garden, resolvedProviders: await this.garden.resolveProviders({ log: this.log }), @@ -132,12 +172,7 @@ export class ResolveActionTask extends BaseActionTask extends BaseActionTask extends BaseActionTask extends BaseActionTask extends BaseActionTask diff --git a/core/test/data/test-projects/module-templates/module-templates.json b/core/test/data/test-projects/module-templates/module-templates.json index 399c98c606..b67a2e7480 100644 --- a/core/test/data/test-projects/module-templates/module-templates.json +++ b/core/test/data/test-projects/module-templates/module-templates.json @@ -6,6 +6,10 @@ }, "value": { "type": "string" + }, + "test": { + "type": "string", + "default": "hello" } } -} \ No newline at end of file +} diff --git a/core/test/unit/src/config/render-template.ts b/core/test/unit/src/config/render-template.ts index 3df01ca784..da6335fc96 100644 --- a/core/test/unit/src/config/render-template.ts +++ b/core/test/unit/src/config/render-template.ts @@ -56,6 +56,9 @@ describe("config templates", () => { } const resolved = await resolveConfigTemplate(garden, config) expect(resolved.inputsSchemaPath).to.eql("module-templates.json") + expect(resolved.inputsSchemaDefaults).to.eql({ + test: "hello", + }) }) it("ignores template strings in modules", async () => { @@ -147,6 +150,7 @@ describe("config templates", () => { inputsSchema: joi.object().keys({ foo: joi.string(), }), + inputsSchemaDefaults: {}, modules: [], } templates.test = template diff --git a/core/test/unit/src/config/template-contexts/module.ts b/core/test/unit/src/config/template-contexts/module.ts index d76b441e99..30055f4316 100644 --- a/core/test/unit/src/config/template-contexts/module.ts +++ b/core/test/unit/src/config/template-contexts/module.ts @@ -15,6 +15,7 @@ import { ModuleConfigContext } from "../../../../../src/config/template-contexts import { WorkflowConfigContext } from "../../../../../src/config/template-contexts/workflow.js" import type { GardenModule } from "../../../../../src/types/module.js" import type { ConfigGraph } from "../../../../../src/graph/config-graph.js" +import { InputContext } from "../../../../../src/config/template-contexts/input.js" describe("ModuleConfigContext", () => { let garden: TestGarden @@ -38,7 +39,7 @@ describe("ModuleConfigContext", () => { name: module.name, path: module.path, parentName: module.parentName, - inputs: module.inputs, + inputs: InputContext.forModule(garden, module), templateName: module.templateName, }) }) diff --git a/core/test/unit/src/tasks/resolve-action.ts b/core/test/unit/src/tasks/resolve-action.ts index 8256309954..dab1f418fe 100644 --- a/core/test/unit/src/tasks/resolve-action.ts +++ b/core/test/unit/src/tasks/resolve-action.ts @@ -22,7 +22,6 @@ import { getAllTaskResults, getDefaultProjectConfig, } from "../../../helpers.js" -import { parseTemplateCollection } from "../../../../src/template/templated-collections.js" describe("ResolveActionTask", () => { let garden: TestGarden @@ -467,6 +466,7 @@ describe("ResolveActionTask", () => { kind: configTemplateKind, name: "template", inputsSchema: joi.object(), + inputsSchemaDefaults: {}, internal: { basePath: garden.projectRoot, }, diff --git a/plugins/pulumi/src/commands.ts b/plugins/pulumi/src/commands.ts index 0845dfa12f..efdabd990e 100644 --- a/plugins/pulumi/src/commands.ts +++ b/plugins/pulumi/src/commands.ts @@ -38,7 +38,6 @@ const { copy, emptyDir, writeJSON } = fsExtra import { join } from "path" import { isBuildAction } from "@garden-io/core/build/src/actions/build.js" import { isDeployAction } from "@garden-io/core/build/src/actions/deploy.js" -import { TemplatableConfigContext } from "@garden-io/core/build/src/config/template-contexts/project.js" import type { ActionTaskProcessParams, BaseTask, ValidResultType } from "@garden-io/core/build/src/tasks/base.js" import { deletePulumiDeploy } from "./handlers.js" import type { ActionLog, Log } from "@garden-io/core/build/src/logger/log-entry.js" @@ -47,6 +46,8 @@ import { ActionSpecContext } from "@garden-io/core/build/src/config/template-con import type { ProviderMap } from "@garden-io/core/build/src/config/provider.js" import { styles } from "@garden-io/core/build/src/logger/styles.js" import { isTruthy } from "@garden-io/core/build/src/util/util.js" +import { InputContext } from "@garden-io/core/src/config/template-contexts/input.js" +import { TemplatableConfigContext } from "@garden-io/core/src/config/template-contexts/templatable.js" type PulumiBaseParams = Omit @@ -205,7 +206,7 @@ const makePluginContextForDeploy = async ( modules: graph.getModules(), resolvedDependencies: action.getResolvedDependencies(), executedDependencies: action.getExecutedDependencies(), - inputs: action.getInternal().inputs || {}, + inputs: new InputContext(action.getInternal().inputs), variables: action.getVariablesContext(), }) const ctxForDeploy = await garden.getPluginContext({ provider, templateContext, events: ctx.events }) From ad06084351d441bace35949b27a28560ffff3a20 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Thu, 23 Jan 2025 12:30:23 +0100 Subject: [PATCH 087/117] fix: build --- plugins/pulumi/src/commands.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/pulumi/src/commands.ts b/plugins/pulumi/src/commands.ts index efdabd990e..af9b48f3d1 100644 --- a/plugins/pulumi/src/commands.ts +++ b/plugins/pulumi/src/commands.ts @@ -46,8 +46,8 @@ import { ActionSpecContext } from "@garden-io/core/build/src/config/template-con import type { ProviderMap } from "@garden-io/core/build/src/config/provider.js" import { styles } from "@garden-io/core/build/src/logger/styles.js" import { isTruthy } from "@garden-io/core/build/src/util/util.js" -import { InputContext } from "@garden-io/core/src/config/template-contexts/input.js" -import { TemplatableConfigContext } from "@garden-io/core/src/config/template-contexts/templatable.js" +import { InputContext } from "@garden-io/core/build/src/config/template-contexts/input.js" +import { TemplatableConfigContext } from "@garden-io/core/build/src/config/template-contexts/templatable.js" type PulumiBaseParams = Omit From 05e2045ed4c96232e5c1071fbfd100839653ec8a Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Thu, 23 Jan 2025 14:30:58 +0100 Subject: [PATCH 088/117] fix: do not use capture in `renderModules` --- core/src/config/render-template.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/config/render-template.ts b/core/src/config/render-template.ts index 7f97ddda5e..e958069968 100644 --- a/core/src/config/render-template.ts +++ b/core/src/config/render-template.ts @@ -196,7 +196,7 @@ async function renderModules({ return Promise.all( (template.modules || []).map(async (m, index) => { // Run a partial template resolution with the parent+template info - const spec = evaluate(capture(m as unknown as ParsedTemplate, context), { + const spec = evaluate(m as unknown as ParsedTemplate, { context, opts: {}, }).resolved From 7f79b9dbbf166af2e0a7da3dcf9a59463155895c Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Thu, 23 Jan 2025 14:44:42 +0100 Subject: [PATCH 089/117] fix: partial module resolution dependency detection / BuildCommand tests --- core/src/config/template-contexts/actions.ts | 2 +- core/src/plugin/handlers/base/configure.ts | 5 +++-- core/src/resolve-module.ts | 17 ++++++++++------- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/core/src/config/template-contexts/actions.ts b/core/src/config/template-contexts/actions.ts index 9c0277e38f..7f16f8e5c3 100644 --- a/core/src/config/template-contexts/actions.ts +++ b/core/src/config/template-contexts/actions.ts @@ -21,7 +21,7 @@ import { TemplatableConfigContext } from "./templatable.js" import { DOCS_BASE_URL } from "../../constants.js" import { styles } from "../../logger/styles.js" import { LayeredContext } from "./base.js" -import { InputContext } from "./input.js" +import type { InputContext } from "./input.js" function mergeVariables({ garden, variables }: { garden: Garden; variables: ConfigContext }): LayeredContext { return new LayeredContext(garden.variables, variables, new GenericContext(garden.variableOverrides)) diff --git a/core/src/plugin/handlers/base/configure.ts b/core/src/plugin/handlers/base/configure.ts index ee450593a7..9349255e92 100644 --- a/core/src/plugin/handlers/base/configure.ts +++ b/core/src/plugin/handlers/base/configure.ts @@ -49,7 +49,8 @@ export class ConfigureActionConfig joi.object().keys({ - config: actionConfigSchema().required(), + config: joi.any().required(), // We will validate the action config again in preprocess action config, this is a performance optimisation supportedModes: joi .object() .keys({ diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index d886aaffc6..6fe6656878 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -535,17 +535,20 @@ export class ModuleResolver { rawConfig["_templateDeps"] = templateDeps // Try resolving template strings if possible - let buildDeps: string[] = [] - const resolvedDeps = evaluate(rawConfig.build.dependencies as unknown as ParsedTemplate, { + const buildDeps: string[] = [] + const { resolved } = evaluate(rawConfig.build.dependencies as unknown as ParsedTemplate, { context: configContext, opts: {}, }) // The build.dependencies field may not resolve at all, in which case we can't extract any deps from there - if (isArray(resolvedDeps)) { - buildDeps = resolvedDeps - // We only collect fully-resolved references here - .filter((d) => isString(d) || (isPlainObject(d) && isString(d.name))) - .map((d) => (isString(d) ? d : d.name)) + if (isArray(resolved)) { + for (const d of resolved) { + if (isString(d)) { + buildDeps.push(d) + } else if (isPlainObject(d) && isString(d.name)) { + buildDeps.push(d.name) + } + } } const deps = uniq([...templateDeps, ...buildDeps]) From 8eb5b669bb90bbb44bd9e7f736113b125cae0764 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Thu, 23 Jan 2025 14:49:03 +0100 Subject: [PATCH 090/117] test: remove test "should normalize build dependencies" - prepareModuleResource does not normalise build deps anymore. --- core/test/unit/src/config/base.ts | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/core/test/unit/src/config/base.ts b/core/test/unit/src/config/base.ts index 6b4e7a1f71..fa5b1ce2a2 100644 --- a/core/test/unit/src/config/base.ts +++ b/core/test/unit/src/config/base.ts @@ -549,20 +549,6 @@ describe("loadConfigResources", () => { }) }) -describe("prepareModuleResource", () => { - it("should normalize build dependencies", async () => { - const moduleConfigPath = resolve(modulePathA, "garden.yml") - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const parsed: any = (await loadConfigResources(log, projectPathA, moduleConfigPath))[0] - parsed.build!.dependencies = [{ name: "apple" }, "banana", null] - const prepared = prepareModuleResource(parsed, moduleConfigPath, projectPathA) - expect(prepared.build!.dependencies).to.eql([ - { name: "apple", copy: [] }, - { name: "banana", copy: [] }, - ]) - }) -}) - describe("findProjectConfig", async () => { const customConfigPath = getDataDir("test-projects", "custom-config-names") From 28a0711ac261062a4dd344518ae8495dbf5230f4 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Thu, 23 Jan 2025 14:49:28 +0100 Subject: [PATCH 091/117] test: serialise unresolved templates before using expect --- core/test/unit/src/commands/get/get-config.ts | 2 +- core/test/unit/src/config/project.ts | 26 ++++++++++--------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/core/test/unit/src/commands/get/get-config.ts b/core/test/unit/src/commands/get/get-config.ts index d75e22f8f0..49435bcd2a 100644 --- a/core/test/unit/src/commands/get/get-config.ts +++ b/core/test/unit/src/commands/get/get-config.ts @@ -688,7 +688,7 @@ describe("GetConfigCommand", () => { opts: withDefaultGlobalOpts({ "exclude-disabled": false, "resolve": "partial" }), }) - expect(res.result?.moduleConfigs).to.deep.equal(rawConfigs) + expect(serialiseUnresolvedTemplates(res.result?.moduleConfigs)).to.deep.equal(rawConfigs) }) it("should return raw provider configs instead of fully resolved providers", async () => { diff --git a/core/test/unit/src/config/project.ts b/core/test/unit/src/config/project.ts index fccb670466..34dd263950 100644 --- a/core/test/unit/src/config/project.ts +++ b/core/test/unit/src/config/project.ts @@ -285,18 +285,20 @@ describe("resolveProjectConfig", () => { process.env.TEST_ENV_VAR = "foo" expect( - resolveProjectConfig({ - log, - defaultEnvironmentName: defaultEnvironment, - config, - artifactsPath: "/tmp", - vcsInfo, - username: "some-user", - loggedIn: true, - enterpriseDomain, - secrets: {}, - commandInfo, - }) + serialiseUnresolvedTemplates( + resolveProjectConfig({ + log, + defaultEnvironmentName: defaultEnvironment, + config, + artifactsPath: "/tmp", + vcsInfo, + username: "some-user", + loggedIn: true, + enterpriseDomain, + secrets: {}, + commandInfo, + }) + ) ).to.eql({ ...config, dotIgnoreFiles: [], From d83bc0405cfe5a71c47c058526c17f9de9e73ff1 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Thu, 23 Jan 2025 15:00:26 +0100 Subject: [PATCH 092/117] fix: test "should remove null values in provider configs" --- core/src/template/lazy-merge.ts | 12 +++++++----- core/test/unit/src/config/project.ts | 8 ++++++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/core/src/template/lazy-merge.ts b/core/src/template/lazy-merge.ts index 40c5c60f76..58a5dc8970 100644 --- a/core/src/template/lazy-merge.ts +++ b/core/src/template/lazy-merge.ts @@ -25,6 +25,12 @@ export class LazyMergePatch extends UnresolvedTemplateValue { const { resolved } = evaluate(item, args) if (isTemplatePrimitive(resolved)) { + if (resolved === null && item !== this.items[0]) { + return { + partial: false, + resolved: undefined, // null values are supposed to remove the key + } + } return { partial: false, resolved, @@ -55,11 +61,7 @@ export class LazyMergePatch extends UnresolvedTemplateValue { for (const k of keys) { const items = toBeMerged.filter((o) => k in o).map((o) => o[k]) - if (items.length === 1) { - returnValue[k] = items[0] - } else { - returnValue[k] = new LazyMergePatch(items) - } + returnValue[k] = new LazyMergePatch(items) } return { diff --git a/core/test/unit/src/config/project.ts b/core/test/unit/src/config/project.ts index 34dd263950..53f1f01941 100644 --- a/core/test/unit/src/config/project.ts +++ b/core/test/unit/src/config/project.ts @@ -570,9 +570,13 @@ describe("pickEnvironment", () => { ) expect(resolvedProviders).to.eql([ { name: "exec" }, - { name: "container", newKey: "foo", dependencies: [] }, + { name: "container", newKey: "foo" }, { name: "templated" }, - { name: "my-provider", b: "b" }, + { + name: "my-provider", + a: undefined, // setting a to undefined is semantically equivalent to removing it in this context + b: "b", + }, ]) }) From a430cca795c6b38b861ec5353f9d8c40b6b8102e Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Thu, 23 Jan 2025 15:08:23 +0100 Subject: [PATCH 093/117] fix: resolveConfigTemplate tests --- core/src/config/config-template.ts | 4 +++- core/test/unit/src/config/render-template.ts | 13 +++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/core/src/config/config-template.ts b/core/src/config/config-template.ts index 5b4919b114..c4e958f28c 100644 --- a/core/src/config/config-template.ts +++ b/core/src/config/config-template.ts @@ -120,7 +120,9 @@ export async function resolveConfigTemplate( if (inputsJsonSchema.properties) { for (const k in inputsJsonSchema.properties) { const d = inputsJsonSchema.properties[k].default - defaultValues[k] = d + if (d !== undefined) { + defaultValues[k] = d + } } } diff --git a/core/test/unit/src/config/render-template.ts b/core/test/unit/src/config/render-template.ts index da6335fc96..bc0b37d6ec 100644 --- a/core/test/unit/src/config/render-template.ts +++ b/core/test/unit/src/config/render-template.ts @@ -22,6 +22,7 @@ import { configTemplateKind, renderTemplateKind } from "../../../../src/config/b import type { RenderTemplateConfig } from "../../../../src/config/render-template.js" import { renderConfigTemplate } from "../../../../src/config/render-template.js" import type { Log } from "../../../../src/logger/log-entry.js" +import { parseTemplateCollection } from "../../../../src/template/templated-collections.js" describe("config templates", () => { let garden: TestGarden @@ -50,10 +51,13 @@ describe("config templates", () => { }) it("resolves template strings for fields other than modules and files", async () => { - const config: ConfigTemplateResource = { - ...defaults, - inputsSchemaPath: "${project.name}.json", - } + const config: ConfigTemplateResource = parseTemplateCollection({ + value: { + ...defaults, + inputsSchemaPath: "${project.name}.json", + }, + source: { path: [] }, + }) const resolved = await resolveConfigTemplate(garden, config) expect(resolved.inputsSchemaPath).to.eql("module-templates.json") expect(resolved.inputsSchemaDefaults).to.eql({ @@ -98,6 +102,7 @@ describe("config templates", () => { expect((resolved.inputsSchema)._rules[0].args.jsonSchema.schema).to.eql({ type: "object", additionalProperties: true, + required: [], }) }) From a372534b0c1a692ccea302367625f37cdc7d0137 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Thu, 23 Jan 2025 15:21:30 +0100 Subject: [PATCH 094/117] fix: make a couple of renderConfigTemplate tests work --- core/test/unit/src/config/render-template.ts | 52 +++++++++++++------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/core/test/unit/src/config/render-template.ts b/core/test/unit/src/config/render-template.ts index bc0b37d6ec..61a4777896 100644 --- a/core/test/unit/src/config/render-template.ts +++ b/core/test/unit/src/config/render-template.ts @@ -23,6 +23,8 @@ import type { RenderTemplateConfig } from "../../../../src/config/render-templat import { renderConfigTemplate } from "../../../../src/config/render-template.js" import type { Log } from "../../../../src/logger/log-entry.js" import { parseTemplateCollection } from "../../../../src/template/templated-collections.js" +import { serialiseUnresolvedTemplates, UnresolvedTemplateValue } from "../../../../src/template/types.js" +import { deepEvaluate } from "../../../../src/template/evaluate.js" describe("config templates", () => { let garden: TestGarden @@ -176,29 +178,43 @@ describe("config templates", () => { it("resolves template strings on the templated module config", async () => { const config: RenderTemplateConfig = { ...defaults, - inputs: { - foo: "${project.name}", - }, + ...parseTemplateCollection({ + value: { + inputs: { + foo: "${project.name}", + }, + }, + source: { path: [] }, + }), } const { resolved } = await renderConfigTemplate({ garden, log, config, templates }) - expect(resolved.inputs?.foo).to.equal("module-templates") + expect(resolved.inputs?.foo).to.be.instanceOf(UnresolvedTemplateValue) + const evaluated = deepEvaluate(resolved.inputs, { context: garden.getProjectConfigContext(), opts: {} }) + expect(evaluated).to.be.eql({ + foo: "module-templates", + }) }) - it("resolves all parent, template and input template strings, ignoring others", async () => { + it("resolves core fields like name, but leaves others unresolved, like dependencies and image", async () => { const _templates = { test: { ...template, - modules: [ - { - type: "test", - name: "${parent.name}-${template.name}-${inputs.foo}", - build: { - dependencies: [{ name: "${parent.name}-${template.name}-foo", copy: [] }], - timeout: DEFAULT_BUILD_TIMEOUT_SEC, - }, - image: "${modules.foo.outputs.bar || inputs.foo}", + ...parseTemplateCollection({ + value: { + modules: [ + { + type: "test", + name: "${parent.name}-${template.name}-${inputs.foo}", + build: { + dependencies: [{ name: "${parent.name}-${template.name}-foo", copy: [] }], + timeout: DEFAULT_BUILD_TIMEOUT_SEC, + }, + image: "${modules.foo.outputs.bar || inputs.foo}", + }, + ], }, - ], + source: { path: [] }, + }), }, } const config: RenderTemplateConfig = { @@ -212,8 +228,10 @@ describe("config templates", () => { const module = resolved.modules[0] expect(module.name).to.equal("test-test-bar") - expect(module.build.dependencies).to.eql([{ name: "test-test-foo", copy: [] }]) - expect(module.spec.image).to.equal("${modules.foo.outputs.bar || inputs.foo}") + expect(serialiseUnresolvedTemplates(module.build.dependencies)).to.eql([ + { name: "${parent.name}-${template.name}-foo", copy: [] }, + ]) + expect(serialiseUnresolvedTemplates(module.spec.image)).to.equal("${modules.foo.outputs.bar || inputs.foo}") }) it("throws if config is invalid", async () => { From d9cf05c2fb56bf55a0b0074319621bd5cc3733d2 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Thu, 23 Jan 2025 14:48:25 +0100 Subject: [PATCH 095/117] fix: ensure right-to-left the order of precedence in `LayeredContext` --- core/src/commands/custom.ts | 2 +- core/src/config/template-contexts/base.ts | 16 +++--- core/src/graph/common.ts | 1 - .../unit/src/config/template-contexts/base.ts | 57 +++++++++++++++++-- 4 files changed, 62 insertions(+), 14 deletions(-) diff --git a/core/src/commands/custom.ts b/core/src/commands/custom.ts index a0f1118c26..696552ae8c 100644 --- a/core/src/commands/custom.ts +++ b/core/src/commands/custom.ts @@ -115,7 +115,7 @@ export class CustomCommandWrapper extends Command { // Render the command variables const variablesContext = new CustomCommandContext({ ...garden, args, opts, rest }) const commandVariables = new GenericContext(capture(this.spec.variables, variablesContext)) - const variables = new LayeredContext(commandVariables, garden.variables) + const variables = new LayeredContext(garden.variables, commandVariables) // Make a new template context with the resolved variables const commandContext = new CustomCommandContext({ ...garden, args, opts, variables, rest }) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index 696f0961d5..ad18e6c35f 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -311,9 +311,9 @@ export class LayeredContext extends ConfigContext { } override resolveImpl(args: ContextResolveParams): ContextResolveOutput { - const items: ContextResolveOutput[] = [] + const layeredItems: ContextResolveOutput[] = [] - for (const context of this.contexts) { + for (const context of this.contexts.toReversed()) { const resolved = context.resolve({ ...args, opts: { @@ -328,21 +328,21 @@ export class LayeredContext extends ConfigContext { } } - items.push(resolved) + layeredItems.push(resolved) } // if it could not be found in any context, aggregate error information from all contexts - if (items.every((res) => !res.found)) { + if (layeredItems.every((res) => !res.found)) { // find deepest key path (most specific error) let deepestKeyPath: (number | string)[] = [] - for (const res of items) { + for (const res of layeredItems) { if (res.explanation.keyPath.length > deepestKeyPath.length) { deepestKeyPath = res.explanation.keyPath } } // identify all errors with the same key path - const all = items.filter((res) => isEqual(res.explanation.keyPath, deepestKeyPath)) + const all = layeredItems.filter((res) => isEqual(res.explanation.keyPath, deepestKeyPath)) const lastError = all[all.length - 1] return { @@ -356,7 +356,9 @@ export class LayeredContext extends ConfigContext { const returnValue = {} - for (const i of items) { + // Here we need to reverse the layers again, because we apply merge function + // that merges the right operand into the left one. + for (const i of layeredItems.toReversed()) { if (!i.found) { continue } diff --git a/core/src/graph/common.ts b/core/src/graph/common.ts index 23bbc5bbfd..0429bce3dc 100644 --- a/core/src/graph/common.ts +++ b/core/src/graph/common.ts @@ -176,7 +176,6 @@ export const mergeVariables = profileAsync(async function mergeVariables({ return new LayeredContext( variables || new GenericContext({}), // Merge different varfiles, later files taking precedence over prior files in the list. - // TODO-0.13.0: should this be a JSON merge? ...varsByFile.map((vars) => new GenericContext(vars)) ) }) diff --git a/core/test/unit/src/config/template-contexts/base.ts b/core/test/unit/src/config/template-contexts/base.ts index 59329f124a..a4fc1670fe 100644 --- a/core/test/unit/src/config/template-contexts/base.ts +++ b/core/test/unit/src/config/template-contexts/base.ts @@ -8,17 +8,17 @@ import { expect } from "chai" import stripAnsi from "strip-ansi" -import type { +import { ConfigContext, ContextKey, + ContextResolveOutputNotFound, ContextResolveParams, -} from "../../../../../src/config/template-contexts/base.js" -import { + ContextWithSchema, GenericContext, getUnavailableReason, LayeredContext, + schema, } from "../../../../../src/config/template-contexts/base.js" -import { ContextWithSchema, schema } from "../../../../../src/config/template-contexts/base.js" import { expectError } from "../../../../helpers.js" import { joi } from "../../../../../src/config/common.js" import { parseTemplateString } from "../../../../../src/template/templated-strings.js" @@ -338,6 +338,53 @@ describe("LayeredContext", () => { opts: {}, }) - expect(res).to.eq("foo-bar") + expect(res).to.eql("foo-bar") + }) + + it("takes the precedence from right to left when merging primitives", () => { + const layeredContext = new LayeredContext( + new GenericContext({ + foo: "foo", + }), + new GenericContext({ + foo: "overriddenFoo", + }) + ) + + const res = layeredContext.resolve({ key: ["foo"], nodePath: [], opts: {} }) + expect(res).to.eql({ found: true, resolved: "overriddenFoo" }) + }) + + it("takes the precedence from right to left when merging objects", () => { + const layeredContext = new LayeredContext( + new GenericContext({ + foo: "foo", + }), + new GenericContext({ + foo: "overriddenFoo", + }) + ) + + const res = layeredContext.resolve({ key: [], nodePath: [], opts: {} }) + expect(res).to.eql({ found: true, resolved: { foo: "overriddenFoo" } }) + }) + + it("show the available keys if attempt to resolve a non-existing key", () => { + const layeredContext = new LayeredContext( + new GenericContext({ + foo: "foo", + }), + new GenericContext({ + bar: "bar", + }) + ) + + const res = layeredContext.resolve({ key: ["baz"], nodePath: [], opts: {} }) + expect(res.found).to.eql(false) + + const explanation = (res as ContextResolveOutputNotFound).explanation + expect(explanation.key).to.eql("baz") + expect(explanation.reason).to.eql("key_not_found") + expect(explanation.getAvailableKeys().sort()).to.eql(["bar", "foo"]) }) }) From e862080732b7d61c97b949a83eac4068cf9c49eb Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Thu, 23 Jan 2025 17:45:29 +0100 Subject: [PATCH 096/117] fix: renderConfigTemplate tests --- core/src/commands/custom.ts | 5 +- core/src/config/render-template.ts | 24 +++---- core/src/config/template-contexts/base.ts | 9 +-- core/src/config/template-contexts/input.ts | 2 +- core/src/config/template-contexts/project.ts | 2 +- core/src/config/template-contexts/render.ts | 10 +-- core/src/graph/actions.ts | 9 +-- core/src/resolve-module.ts | 2 +- core/src/template/ast.ts | 12 +++- core/src/template/errors.ts | 7 +- core/src/util/testing.ts | 2 +- core/test/unit/src/config/render-template.ts | 69 +++++++++++++------- 12 files changed, 89 insertions(+), 64 deletions(-) diff --git a/core/src/commands/custom.ts b/core/src/commands/custom.ts index 696552ae8c..9e37b42d50 100644 --- a/core/src/commands/custom.ts +++ b/core/src/commands/custom.ts @@ -30,7 +30,6 @@ import type { Log } from "../logger/log-entry.js" import { getTracePropagationEnvVars } from "../util/open-telemetry/propagation.js" import { styles } from "../logger/styles.js" import { deepEvaluate } from "../template/evaluate.js" -import { capture } from "../template/capture.js" import { GenericContext, LayeredContext } from "../config/template-contexts/base.js" function convertArgSpec(spec: CustomCommandOption) { @@ -113,8 +112,8 @@ export class CustomCommandWrapper extends Command { const rest = removeSlice(parsed._unknown, this.getPath()).slice(Object.keys(this.arguments || {}).length) // Render the command variables - const variablesContext = new CustomCommandContext({ ...garden, args, opts, rest }) - const commandVariables = new GenericContext(capture(this.spec.variables, variablesContext)) + + const commandVariables = new GenericContext(this.spec.variables) const variables = new LayeredContext(garden.variables, commandVariables) // Make a new template context with the resolved variables diff --git a/core/src/config/render-template.ts b/core/src/config/render-template.ts index e958069968..175673aab7 100644 --- a/core/src/config/render-template.ts +++ b/core/src/config/render-template.ts @@ -25,7 +25,7 @@ import fsExtra from "fs-extra" const { ensureDir } = fsExtra import type { TemplatedModuleConfig } from "../plugins/templated.js" -import { omit } from "lodash-es" +import { isString, omit } from "lodash-es" import { EnvironmentConfigContext } from "./template-contexts/project.js" import type { ConfigTemplateConfig, TemplatableConfig } from "./config-template.js" import { templatableKinds, templateNoTemplateFields } from "./config-template.js" @@ -34,9 +34,8 @@ import type { DeepPrimitiveMap } from "@garden-io/platform-api-types" import { RenderTemplateConfigContext } from "./template-contexts/render.js" import type { Log } from "../logger/log-entry.js" import { GardenApiVersion } from "../constants.js" -import { capture } from "../template/capture.js" import { deepEvaluate, evaluate } from "../template/evaluate.js" -import { UnresolvedTemplateValue, type ParsedTemplate } from "../template/types.js" +import { serialiseUnresolvedTemplates, UnresolvedTemplateValue, type ParsedTemplate } from "../template/types.js" import { isArray, isPlainObject } from "../util/objects.js" import { InputContext } from "./template-contexts/input.js" @@ -211,8 +210,8 @@ async function renderModules({ let moduleConfig: ModuleConfig + const resolvedSpec = { ...spec } try { - const resolvedSpec = { ...spec } for (const key of coreModuleSpecKeys()) { resolvedSpec[key] = deepEvaluate(resolvedSpec[key], { context, opts: {} }) } @@ -225,11 +224,14 @@ async function renderModules({ if (coreModuleSpecKeys().some((k) => spec[k] instanceof UnresolvedTemplateValue)) { msg += - ". Note that if a template string is used for the name, kind, type or apiVersion of a module in a template, then the template string must be fully resolvable at the time of module scanning. This means that e.g. references to other modules or runtime outputs cannot be used." + "\n\nNote that if a template string is used for the name, kind, type or apiVersion of a module in a template, then the template string must be fully resolvable at the time of module scanning. This means that e.g. references to other modules or runtime outputs cannot be used." } throw new ConfigurationError({ - message: `${configTemplateKind} ${template.name}: invalid module at index ${index} for templated module ${renderConfig.name}: ${msg}`, + message: `${configTemplateKind} ${template.name} returned an invalid module (named ${ + // We use serializeUnresolvedTemplates here because the error message is clearer for the user with a plain unresolved template string + serialiseUnresolvedTemplates(resolvedSpec.name) + }) for templated module ${renderConfig.name}: ${msg}`, }) } @@ -239,11 +241,11 @@ async function renderModules({ sourcePath: f.sourcePath && resolve(template.internal.basePath, ...f.sourcePath.split(posix.sep)), })) - // // If a path is set, resolve the path and ensure that directory exists - // if (spec.path) { - // moduleConfig.path = resolve(renderConfig.internal.basePath, ...spec.path.split(posix.sep)) - // await ensureDir(moduleConfig.path) - // } + // If a path is set, resolve the path and ensure that directory exists + if (resolvedSpec.path && isString(resolvedSpec.path)) { + moduleConfig.path = resolve(renderConfig.internal.basePath, ...resolvedSpec.path.split(posix.sep)) + await ensureDir(moduleConfig.path) + } // Attach metadata moduleConfig.parentName = renderConfig.name diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index ad18e6c35f..e8b434c253 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -314,14 +314,7 @@ export class LayeredContext extends ConfigContext { const layeredItems: ContextResolveOutput[] = [] for (const context of this.contexts.toReversed()) { - const resolved = context.resolve({ - ...args, - opts: { - ...args.opts, - // to avoid circular dependency errors - stack: args.opts.stack?.slice(0, -1), - }, - }) + const resolved = context.resolve(args) if (resolved.found) { if (isTemplatePrimitive(resolved.resolved)) { return resolved diff --git a/core/src/config/template-contexts/input.ts b/core/src/config/template-contexts/input.ts index 6f2b0d17e6..15089c45de 100644 --- a/core/src/config/template-contexts/input.ts +++ b/core/src/config/template-contexts/input.ts @@ -11,11 +11,11 @@ import { describeConfig } from "../../vcs/vcs.js" import type { ActionConfig } from "../../actions/types.js" import type { WorkflowConfig } from "../workflow.js" import { InternalError } from "../../exceptions.js" -import { describeActionConfig } from "../../actions/base.js" import type { ConfigTemplateConfig } from "../config-template.js" import type { ParsedTemplate } from "../../template/types.js" import type { RenderTemplateConfig } from "../render-template.js" import type { ModuleConfig } from "../module.js" +import { describeActionConfig } from "../../actions/base.js" export class InputContext extends LayeredContext { public static forAction(garden: Garden, config: ActionConfig | WorkflowConfig): InputContext { diff --git a/core/src/config/template-contexts/project.ts b/core/src/config/template-contexts/project.ts index 4d544694b2..086eca6103 100644 --- a/core/src/config/template-contexts/project.ts +++ b/core/src/config/template-contexts/project.ts @@ -343,7 +343,7 @@ export class ProjectConfigContext extends DefaultEnvironmentContext { } } -interface EnvironmentConfigContextParams extends ProjectConfigContextParams { +export interface EnvironmentConfigContextParams extends ProjectConfigContextParams { variables: ConfigContext } diff --git a/core/src/config/template-contexts/render.ts b/core/src/config/template-contexts/render.ts index b309ef6489..12ad86e485 100644 --- a/core/src/config/template-contexts/render.ts +++ b/core/src/config/template-contexts/render.ts @@ -9,10 +9,10 @@ import { joiVariables } from "../common.js" import { ParentContext, schema, TemplateContext } from "./base.js" import type { InputContext } from "./input.js" -import type { ProjectConfigContextParams } from "./project.js" -import { ProjectConfigContext } from "./project.js" +import type { EnvironmentConfigContextParams } from "./project.js" +import { EnvironmentConfigContext } from "./project.js" -export class RenderTemplateConfigContext extends ProjectConfigContext { +export class RenderTemplateConfigContext extends EnvironmentConfigContext { @schema(ParentContext.getSchema().description(`Information about the templated config being resolved.`)) public parent: ParentContext @@ -26,7 +26,9 @@ export class RenderTemplateConfigContext extends ProjectConfigContext { ) public inputs: InputContext - constructor(params: { parentName: string; templateName: string; inputs: InputContext } & ProjectConfigContextParams) { + constructor( + params: { parentName: string; templateName: string; inputs: InputContext } & EnvironmentConfigContextParams + ) { super(params) this.parent = new ParentContext(params.parentName) this.template = new TemplateContext(params.templateName) diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index 32864b7533..93a6589695 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -66,7 +66,6 @@ import { getSourcePath } from "../vcs/vcs.js" import { styles } from "../logger/styles.js" import { isUnresolvableValue } from "../template/analysis.js" import { getActionTemplateReferences } from "../config/references.js" -import { capture } from "../template/capture.js" import { deepEvaluate } from "../template/evaluate.js" import type { ParsedTemplate } from "../template/types.js" import { validateWithPath } from "../config/validation.js" @@ -509,13 +508,7 @@ export const processActionConfig = profileAsync(async function processActionConf let variables = await mergeVariables({ basePath: effectiveConfigFileLocation, - variables: new GenericContext( - capture( - config.variables, - // TODO: What's the correct context here? - garden.getProjectConfigContext() - ) || {} - ), + variables: new GenericContext(config.variables || {}), varfiles: config.varfiles, log, }) diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index 6fe6656878..52eafeca6a 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -894,7 +894,7 @@ export class ModuleResolver { }) } - const moduleVariables = capture(config.variables || {}, context) + const moduleVariables = config.variables || {} return new LazyMergePatch([moduleVariables, varfileVars, this.garden.variableOverrides]) } diff --git a/core/src/template/ast.ts b/core/src/template/ast.ts index 6af588a979..ed52375c6c 100644 --- a/core/src/template/ast.ts +++ b/core/src/template/ast.ts @@ -7,7 +7,7 @@ */ import { isArray, isNumber, isString } from "lodash-es" -import { ContextResolveError, getUnavailableReason } from "../config/template-contexts/base.js" +import { ContextResolveError, getUnavailableReason, renderKeyPath } from "../config/template-contexts/base.js" import { InternalError } from "../exceptions.js" import { getHelperFunctions } from "./functions/index.js" import type { EvaluateTemplateArgs } from "./types.js" @@ -16,6 +16,7 @@ import type { Collection, CollectionOrValue } from "../util/objects.js" import { type ConfigSource, validateSchema } from "../config/validation.js" import type { TemplateExpressionGenerator } from "./analysis.js" import { TemplateStringError } from "./errors.js" +import { styles } from "../logger/styles.js" type ASTEvaluateArgs = EvaluateTemplateArgs & { yamlSource: ConfigSource @@ -713,6 +714,7 @@ export class ContextLookupExpression extends TemplateExpression { message: getUnavailableReason(result), loc: this.loc, yamlSource, + lookupResult: result, }) } @@ -738,6 +740,14 @@ export class ContextLookupExpression extends TemplateExpression { wrappedErrors: [e], }) } + if (e instanceof TemplateStringError) { + throw new TemplateStringError({ + message: `Failed to evaluate template expression at ${styles.highlight(renderKeyPath(keyPath))}: ${e.message}`, + loc: this.loc, + yamlSource, + wrappedErrors: [e], + }) + } throw e } } diff --git a/core/src/template/errors.ts b/core/src/template/errors.ts index e9ef3529c5..ed3d64a84f 100644 --- a/core/src/template/errors.ts +++ b/core/src/template/errors.ts @@ -13,6 +13,7 @@ import type { GardenErrorParams } from "../exceptions.js" import { GardenError } from "../exceptions.js" import { styles } from "../logger/styles.js" import type { Location } from "./ast.js" +import type { ContextResolveOutputNotFound } from "../config/template-contexts/base.js" export class TemplateError extends GardenError { type = "template" @@ -34,8 +35,11 @@ export class TemplateStringError extends GardenError { loc: Location originalMessage: string + lookupResult?: ContextResolveOutputNotFound - constructor(params: GardenErrorParams & { loc: Location; yamlSource: ConfigSource }) { + constructor( + params: GardenErrorParams & { loc: Location; yamlSource: ConfigSource; lookupResult?: ContextResolveOutputNotFound } + ) { let enriched: string = params.message try { // TODO: Use Location information from parser to point to the specific part @@ -57,5 +61,6 @@ export class TemplateStringError extends GardenError { super({ ...params, message: enriched }) this.loc = params.loc this.originalMessage = params.message + this.lookupResult = params.lookupResult } } diff --git a/core/src/util/testing.ts b/core/src/util/testing.ts index 1a589f4ace..60c453cd58 100644 --- a/core/src/util/testing.ts +++ b/core/src/util/testing.ts @@ -58,7 +58,7 @@ import got from "got" import { createHash } from "node:crypto" import { pipeline } from "node:stream/promises" import type { GardenCloudApiFactory } from "../cloud/api.js" -import type { ConfigContext } from "../config/template-contexts/base.js" +import { type ConfigContext } from "../config/template-contexts/base.js" import { parseTemplateCollection } from "../template/templated-collections.js" import type { TemplatePrimitive } from "../template/types.js" diff --git a/core/test/unit/src/config/render-template.ts b/core/test/unit/src/config/render-template.ts index 61a4777896..c842fe7a31 100644 --- a/core/test/unit/src/config/render-template.ts +++ b/core/test/unit/src/config/render-template.ts @@ -25,6 +25,7 @@ import type { Log } from "../../../../src/logger/log-entry.js" import { parseTemplateCollection } from "../../../../src/template/templated-collections.js" import { serialiseUnresolvedTemplates, UnresolvedTemplateValue } from "../../../../src/template/types.js" import { deepEvaluate } from "../../../../src/template/evaluate.js" +import { GenericContext } from "../../../../src/config/template-contexts/base.js" describe("config templates", () => { let garden: TestGarden @@ -341,12 +342,17 @@ describe("config templates", () => { const _templates = { test: { ...template, - modules: [ - { - type: "test", - name: "${inputs.foo}", + ...parseTemplateCollection({ + value: { + modules: [ + { + type: "test", + name: "${inputs.foo}", + }, + ], }, - ], + source: { path: [] }, + }), }, } const config: RenderTemplateConfig = { @@ -434,22 +440,28 @@ describe("config templates", () => { }) it("resolves project variable references in input fields", async () => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const _templates: any = { + const _templates = { test: { ...template, - modules: [ - { - type: "test", - name: "${inputs.name}-test", + ...parseTemplateCollection({ + value: { + modules: [ + { + type: "test", + name: "${inputs.name}-test", + }, + ], }, - ], + source: { path: [] }, + }), }, } const config: RenderTemplateConfig = cloneDeep(defaults) - config.inputs = { name: "${var.test}" } - garden.variables["test"] = "test-value" + config.inputs = parseTemplateCollection({ value: { name: "${var.test}" }, source: { path: [] } }) + garden.variables = new GenericContext({ + test: "test-value", + }) const resolved = await renderConfigTemplate({ garden, log, config, templates: _templates }) @@ -485,24 +497,33 @@ describe("config templates", () => { const _templates: any = { test: { ...template, - modules: [ - { - type: "test", - name: "${inputs.name}-test", + ...parseTemplateCollection({ + value: { + modules: [ + { + type: "test", + name: "${inputs.name}-module", + }, + ], }, - ], + source: { path: [] }, + }), }, } const config: RenderTemplateConfig = cloneDeep(defaults) - config.inputs = { name: "module-${modules.foo.version}" } + config.inputs = parseTemplateCollection({ + value: { name: "module-${modules.foo.version}" }, + source: { path: [] }, + }) await expectError(() => renderConfigTemplate({ garden, log, config, templates: _templates }), { contains: [ - "ConfigTemplate test returned an invalid module (named module-${modules.foo.version}-test) for templated module test", - "Error validating module (modules.garden.yml)", - 'name with value "module-${modules.foo.version}-test" fails to match the required pattern: /^(?!garden)(?=.{1,63}$)[a-z][a-z0-9]*(-[a-z0-9]+)*$/.', - "Note that if a template string is used in the name of a module in a template, then the template string must be fully resolvable at the time of module scanning. This means that e.g. references to other modules or runtime outputs cannot be used.", + "ConfigTemplate test returned an invalid module (named ${inputs.name}-module) for templated module test", + "failed to evaluate template expression at inputs.name", + "invalid template string (module-${modules.foo.version}) at path name", + "could not find key modules. available keys:", + "Note that if a template string is used for the name, kind, type or apiversion of a module in a template, then the template string must be fully resolvable at the time of module scanning. This means that e.g. references to other modules or runtime outputs cannot be used.", ], }) }) From 2ab7fd4629202de4013ca9ed81f9969236b19adf Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 24 Jan 2025 07:44:30 +0100 Subject: [PATCH 097/117] Revert "fix: remove unnecessary hack in `action.disable` flag resolution" This reverts commit 3247a3c1f4ce92c806c4f5a873988bf8c47187e8. --- core/src/garden.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/core/src/garden.ts b/core/src/garden.ts index 8a32dbcc72..4b469b343e 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -1564,6 +1564,18 @@ export class Garden { } const context = new TemplatableConfigContext(this, config) + // Hack: deny variables contexts here, because those have not been fully resolved yet. + const deniedContexts = ["var", "variables"] + for (const deniedContext of deniedContexts) { + Object.defineProperty(context, deniedContext, { + get: () => { + throw new ConfigurationError({ + message: `If you have duplicate action names, the ${styles.accent("`disabled`")} flag cannot depend on the ${styles.accent(`\`${deniedContext}\``)} context.`, + }) + }, + }) + } + const resolved = deepEvaluate(disabledFlag, { context, opts: {} }) return !!resolved From 0ecbe5b8a27e5d37f00444832b80e643670fd62f Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 24 Jan 2025 07:46:01 +0100 Subject: [PATCH 098/117] fix: test "should resolve template strings in project source definitions" --- core/test/unit/src/garden.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index d38415fb73..e042096f7b 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -2926,7 +2926,7 @@ describe("Garden", () => { await exec("git", ["add", "."], { cwd: repoPath }) await exec("git", ["commit", "-m", "foo"], { cwd: repoPath }) - garden.variables = new LayeredContext(garden.variables, new GenericContext({ var: { sourceBranch: "main" } })) + garden.variables = new GenericContext({ sourceBranch: "main" }) // eslint-disable-next-line @typescript-eslint/no-explicit-any const _garden = garden as any From 9bd74bffb1212f6955529592f94a23bc44909ac8 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 24 Jan 2025 07:53:45 +0100 Subject: [PATCH 099/117] fix: test "should return a different version for a module when a variable used by it changes" --- core/test/unit/src/vcs/vcs.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/test/unit/src/vcs/vcs.ts b/core/test/unit/src/vcs/vcs.ts index a8c7a5a2a6..a3c35a5a5d 100644 --- a/core/test/unit/src/vcs/vcs.ts +++ b/core/test/unit/src/vcs/vcs.ts @@ -39,6 +39,7 @@ import { defaultDotIgnoreFile, fixedProjectExcludes } from "../../../../src/util import type { BaseActionConfig } from "../../../../src/actions/types.js" import { TreeCache } from "../../../../src/cache.js" import { getHashedFilterParams } from "../../../../src/vcs/git-repo.js" +import { GenericContext } from "../../../../src/config/template-contexts/base.js" const { readFile, writeFile } = fsExtra @@ -273,7 +274,7 @@ describe("getModuleVersionString", () => { templateGarden["cacheKey"] = "" // Disable caching of the config graph const before = await templateGarden.resolveModule("module-a") - templateGarden.variables["echo-string"] = "something-else" + templateGarden.variables = new GenericContext({ "echo-string": "something-else" }) const after = await templateGarden.resolveModule("module-a") From b5726ae86fb0cf1e23314072c9b65e65c31db3b0 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 24 Jan 2025 07:54:17 +0100 Subject: [PATCH 100/117] chore: update assertions in test "should resolve modules from config templates and any modules referencing them" --- core/test/unit/src/garden.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index e042096f7b..39d8508ebe 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -2993,7 +2993,7 @@ describe("Garden", () => { expect(serialiseUnresolvedTemplates(omitUndefined(configB))).to.eql({ apiVersion: GardenApiVersion.v0, kind: "Module", - build: { dependencies: [{ name: "foo-test-a", copy: [] }], timeout: DEFAULT_BUILD_TIMEOUT_SEC }, + build: { dependencies: ["${parent.name}-${inputs.name}-a"], timeout: DEFAULT_BUILD_TIMEOUT_SEC }, include: [], configPath: resolve(garden.projectRoot, "modules.garden.yml"), name: "foo-test-b", @@ -3001,7 +3001,7 @@ describe("Garden", () => { serviceConfigs: [], spec: { build: { - dependencies: [{ name: "foo-test-a", copy: [] }], + dependencies: ["${parent.name}-${inputs.name}-a"], }, }, testConfigs: [], From 7d43437984e57a65b6d6bc134340fc2726591189 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 24 Jan 2025 07:57:27 +0100 Subject: [PATCH 101/117] fix: test "sets variables for the action" --- core/test/unit/src/actions/action-configs-to-graph.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/core/test/unit/src/actions/action-configs-to-graph.ts b/core/test/unit/src/actions/action-configs-to-graph.ts index 31fcb22a10..082f7d3027 100644 --- a/core/test/unit/src/actions/action-configs-to-graph.ts +++ b/core/test/unit/src/actions/action-configs-to-graph.ts @@ -23,6 +23,7 @@ import { import { getRemoteSourceLocalPath } from "../../../../src/util/ext-source-util.js" import { clearVarfileCache } from "../../../../src/config/base.js" import { parseTemplateCollection } from "../../../../src/template/templated-collections.js" +import { deepResolveContext } from "../../../../src/config/template-contexts/base.js" describe("actionConfigsToGraph", () => { let tmpDir: TempDirectory @@ -542,12 +543,10 @@ describe("actionConfigsToGraph", () => { const action = graph.getBuild("foo") const vars = action["variables"] + const resolved = deepResolveContext("action variables", vars, garden.getProjectConfigContext()) - expect(vars.resolve({ nodePath: [], key: [], opts: {} })).to.eql({ - found: true, - resolved: { - projectName: garden.projectName, - }, + expect(resolved).to.eql({ + projectName: garden.projectName, }) }) From b06d49a839a174de91e70ba373e83306d921b79b Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 24 Jan 2025 08:44:04 +0100 Subject: [PATCH 102/117] fix: test "should respect the action variables < action varfile < CLI var precedence order" --- core/src/config/base.ts | 30 +++---- core/src/config/render-template.ts | 4 +- core/src/graph/actions.ts | 8 +- core/src/plugins/hadolint/hadolint.ts | 8 +- core/src/template/analysis.ts | 105 +++++++++++++++---------- core/src/template/ast.ts | 2 + core/src/template/templated-strings.ts | 37 +++------ 7 files changed, 102 insertions(+), 92 deletions(-) diff --git a/core/src/config/base.ts b/core/src/config/base.ts index 673d46eaf2..2b0d6bec94 100644 --- a/core/src/config/base.ts +++ b/core/src/config/base.ts @@ -26,7 +26,7 @@ import { createSchema, joi } from "./common.js" import { emitNonRepeatableWarning } from "../warnings.js" import type { ActionKind, BaseActionConfig } from "../actions/types.js" import { actionKinds } from "../actions/types.js" -import { mayContainTemplateString } from "../template/templated-strings.js" +import { isUnresolved } from "../template/templated-strings.js" import type { Log } from "../logger/log-entry.js" import type { Document, DocumentOptions } from "yaml" import { parseAllDocuments } from "yaml" @@ -277,13 +277,26 @@ export function prepareResource({ }) } + if (parse) { + for (const k in spec) { + // TODO: should we do this here? would be good to do it as early as possible. + spec[k] = parseTemplateCollection({ + value: spec[k], + source: { + yamlDoc: doc, + path: [k], + }, + }) + } + } + let kind = spec.kind const basePath = dirname(configFilePath) if (!allowInvalid) { for (const field of noTemplateFields) { - if (spec[field] && mayContainTemplateString(spec[field])) { + if (spec[field] && isUnresolved(spec[field])) { throw new ConfigurationError({ message: `Resource in ${relPath} has a template string in field '${field}', which does not allow templating.`, }) @@ -296,19 +309,6 @@ export function prepareResource({ } } - if (parse) { - for (const k in spec) { - // TODO: should we do this here? would be good to do it as early as possible. - spec[k] = parseTemplateCollection({ - value: spec[k], - source: { - yamlDoc: doc, - path: [k], - }, - }) - } - } - // Allow this for backwards compatibility if (kind === "ModuleTemplate") { spec.kind = kind = configTemplateKind diff --git a/core/src/config/render-template.ts b/core/src/config/render-template.ts index 175673aab7..3fb83249ed 100644 --- a/core/src/config/render-template.ts +++ b/core/src/config/render-template.ts @@ -16,7 +16,7 @@ import { prepareResource, renderTemplateKind, } from "./base.js" -import { maybeTemplateString } from "../template/templated-strings.js" +import { isUnresolved } from "../template/templated-strings.js" import { validateWithPath } from "./validation.js" import type { Garden } from "../garden.js" import { ConfigurationError, GardenError, InternalError } from "../exceptions.js" @@ -307,7 +307,7 @@ async function renderConfigs({ // TODO: validate this before? for (const field of templateNoTemplateFields) { - if (maybeTemplateString(m[field])) { + if (isUnresolved(m[field])) { throw new ConfigurationError({ message: `${templateDescription} contains an invalid resource: Found a template string in '${field}' field (${m[field]}).`, }) diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index 93a6589695..752e3a13b1 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -48,7 +48,7 @@ import type { ActionDefinitionMap } from "../plugins.js" import { getActionTypeBases } from "../plugins.js" import type { ActionRouter } from "../router/router.js" import { ResolveActionTask } from "../tasks/resolve-action.js" -import { maybeTemplateString } from "../template/templated-strings.js" +import { isUnresolved } from "../template/templated-strings.js" import { dedent, deline, naturalList } from "../util/string.js" import { DependencyGraph, getVarfileData, mergeVariables } from "./common.js" import type { ConfigGraph } from "./config-graph.js" @@ -67,7 +67,7 @@ import { styles } from "../logger/styles.js" import { isUnresolvableValue } from "../template/analysis.js" import { getActionTemplateReferences } from "../config/references.js" import { deepEvaluate } from "../template/evaluate.js" -import type { ParsedTemplate } from "../template/types.js" +import { UnresolvedTemplateValue, type ParsedTemplate } from "../template/types.js" import { validateWithPath } from "../config/validation.js" function* sliceToBatches(dict: Record, batchSize: number) { @@ -730,7 +730,9 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi const description = describeActionConfig(config) // in pre-processing, only use varfiles that are not template strings - const resolvedVarFiles = config.varfiles?.filter((f) => !maybeTemplateString(getVarfileData(f).path)) + const resolvedVarFiles = config.varfiles?.filter( + (f) => !(f instanceof UnresolvedTemplateValue) && isUnresolved(getVarfileData(f).path) + ) const variables = await mergeVariables({ basePath: config.internal.basePath, variables: new GenericContext(config.variables || {}), diff --git a/core/src/plugins/hadolint/hadolint.ts b/core/src/plugins/hadolint/hadolint.ts index 6a58d9fd67..5217522ea8 100644 --- a/core/src/plugins/hadolint/hadolint.ts +++ b/core/src/plugins/hadolint/hadolint.ts @@ -18,7 +18,7 @@ import { defaultDockerfileName } from "../container/config.js" import { baseBuildSpecSchema } from "../../config/module.js" import { getGitHubUrl } from "../../docs/common.js" import type { TestAction, TestActionConfig } from "../../actions/test.js" -import { mayContainTemplateString } from "../../template/templated-strings.js" +import { isUnresolved } from "../../template/templated-strings.js" import type { BaseAction } from "../../actions/base.js" import type { BuildAction } from "../../actions/build.js" import { sdk } from "../../plugin/sdk.js" @@ -180,7 +180,7 @@ provider.addHandler("augmentGraph", async ({ ctx, actions }) => { const existingHadolintDockerfiles = actions .filter(isHadolintTest) // Can't really reason about templated dockerfile spec field - .filter((a) => !mayContainTemplateString(a.getConfig("spec").dockerfilePath)) + .filter((a) => !isUnresolved(a.getConfig("spec").dockerfilePath)) .map((a) => resolve(a.sourcePath(), a.getConfig("spec").dockerfilePath)) const pickCompatibleAction = (action: BaseAction): action is BuildAction | HadolintTest => { @@ -188,7 +188,7 @@ provider.addHandler("augmentGraph", async ({ ctx, actions }) => { if (isHadolintTest(action)) { const dockerfilePath = action.getConfig("spec").dockerfilePath if ( - !mayContainTemplateString(dockerfilePath) && + !isUnresolved(dockerfilePath) && existingHadolintDockerfiles.includes(resolve(action.sourcePath(), dockerfilePath)) ) { return false @@ -215,7 +215,7 @@ provider.addHandler("augmentGraph", async ({ ctx, actions }) => { (isHadolintTest(action) ? action.getConfig("spec").dockerfilePath : action.getConfig("spec").dockerfile) || defaultDockerfileName - const include = mayContainTemplateString(dockerfilePath) ? undefined : [dockerfilePath] + const include = isUnresolved(dockerfilePath) ? undefined : [dockerfilePath] return { kind: "Test", diff --git a/core/src/template/analysis.ts b/core/src/template/analysis.ts index 7eaef7eb61..ad9aca9268 100644 --- a/core/src/template/analysis.ts +++ b/core/src/template/analysis.ts @@ -17,10 +17,16 @@ import { GardenError, InternalError } from "../exceptions.js" import { type ConfigSource } from "../config/validation.js" export type TemplateExpressionGenerator = Generator< - { - value: TemplateExpression - yamlSource: ConfigSource - }, + | { + type: "template-expression" + value: TemplateExpression + yamlSource: ConfigSource + } + | { + type: "unresolved-template" + value: UnresolvedTemplateValue + yamlSource: undefined + }, void, undefined > @@ -39,6 +45,11 @@ export function* visitAll({ value }: { value: ParsedTemplate }): TemplateExpress }) } } else if (value instanceof UnresolvedTemplateValue) { + yield { + type: "unresolved-template", + value, + yamlSource: undefined, + } yield* value.visitAll({}) } } @@ -85,46 +96,60 @@ export function* getContextLookupReferences( generator: TemplateExpressionGenerator, context: ConfigContext ): Generator { - for (const { value, yamlSource } of generator) { - if (value instanceof ContextLookupExpression) { - let isResolvable: boolean = true - const keyPath = value.keyPath.map((keyPathExpression) => { - const key = keyPathExpression.evaluate({ - context, - opts: {}, - optional: true, - yamlSource, - }) - if (typeof key === "symbol") { - isResolvable = false - return new UnresolvableValue( - captureError(() => - // this will throw an error, because the key could not be resolved - keyPathExpression.evaluate({ - context, - opts: {}, - optional: false, - yamlSource, - }) - ) - ) - } - return key + for (const finding of generator) { + if (finding.type !== "template-expression") { + // we are only interested in template expressions here + continue + } + + const { value, yamlSource } = finding + + if (!(value instanceof ContextLookupExpression)) { + // we are only interested in ContextLookupExpression instances + continue + } + + let isResolvable: boolean = true + + const keyPath = value.keyPath.map((keyPathExpression) => { + const key = keyPathExpression.evaluate({ + context, + opts: {}, + optional: true, + yamlSource, }) - if (keyPath.length > 0) { - yield isResolvable - ? { - type: "resolvable", - keyPath: keyPath as (string | number)[], - yamlSource, - } - : { - type: "unresolvable", - keyPath, + if (typeof key === "symbol") { + isResolvable = false + + return new UnresolvableValue( + captureError(() => + // this will throw an error, because the key could not be resolved + keyPathExpression.evaluate({ + context, + opts: {}, + optional: false, yamlSource, - } + }) + ) + ) } + + return key + }) + + if (keyPath.length > 0) { + yield isResolvable + ? { + type: "resolvable", + keyPath: keyPath as (string | number)[], + yamlSource, + } + : { + type: "unresolvable", + keyPath, + yamlSource, + } } } } diff --git a/core/src/template/ast.ts b/core/src/template/ast.ts index ed52375c6c..b1705e8514 100644 --- a/core/src/template/ast.ts +++ b/core/src/template/ast.ts @@ -63,6 +63,7 @@ function* astVisitAll(e: TemplateExpression, source: ConfigSource): TemplateExpr if (propertyValue instanceof TemplateExpression) { yield* astVisitAll(propertyValue, source) yield { + type: "template-expression", value: propertyValue, yamlSource: source, } @@ -71,6 +72,7 @@ function* astVisitAll(e: TemplateExpression, source: ConfigSource): TemplateExpr if (item instanceof TemplateExpression) { yield* astVisitAll(item, source) yield { + type: "template-expression", value: item, yamlSource: source, } diff --git a/core/src/template/templated-strings.ts b/core/src/template/templated-strings.ts index 82ce890eab..fdfb89a158 100644 --- a/core/src/template/templated-strings.ts +++ b/core/src/template/templated-strings.ts @@ -9,17 +9,14 @@ import type { GardenErrorParams } from "../exceptions.js" import { InternalError } from "../exceptions.js" import type { ConfigContext, ContextResolveOpts } from "../config/template-contexts/base.js" -import type { Primitive } from "../config/common.js" -import { isPrimitive } from "../config/common.js" import type { CollectionOrValue } from "../util/objects.js" -import { deepMap } from "../util/objects.js" import type { ConfigSource } from "../config/validation.js" import * as parser from "./parser.js" -import type { EvaluateTemplateArgs, ResolvedTemplate } from "./types.js" +import type { EvaluateTemplateArgs, ParsedTemplate, ResolvedTemplate } from "./types.js" import { UnresolvedTemplateValue, type TemplatePrimitive } from "./types.js" import * as ast from "./ast.js" import { LRUCache } from "lru-cache" -import type { TemplateExpressionGenerator } from "./analysis.js" +import { visitAll, type TemplateExpressionGenerator } from "./analysis.js" import { TemplateStringError } from "./errors.js" const escapePrefix = "$${" @@ -87,7 +84,7 @@ export function parseTemplateString({ source: ConfigSource }): ParsedTemplateString | string { // Just return immediately if this is definitely not a template string - if (!maybeTemplateString(rawTemplateString)) { + if (!rawTemplateString.includes("${")) { return rawTemplateString } @@ -184,28 +181,12 @@ export function resolveTemplateString({ } /** - * Returns `true` if the given value is a string and looks to contain a template string. + * Returns `true` if the given value may be or contain instances of `UnresolvedTemplateValue`. */ -export function maybeTemplateString(value: Primitive) { - return !!value && typeof value === "string" && value.includes("${") -} - -/** - * Returns `true` if the given value or any value in a given object or array seems to contain a template string. - */ -export function mayContainTemplateString(obj: any): boolean { - let out = false - - if (isPrimitive(obj)) { - return maybeTemplateString(obj) +export function isUnresolved(value: ParsedTemplate) { + const generator = visitAll({ value }) + for (const _ of generator) { + return true } - - // TODO: use visitAll instead. - deepMap(obj, (v) => { - if (maybeTemplateString(v)) { - out = true - } - }) - - return out + return false } From a9011b05e606aa9b354b3c436eddcb69a0a748a2 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 24 Jan 2025 08:53:35 +0100 Subject: [PATCH 103/117] fix: test "should resolve disabled flag in actions and allow two actions with same key if one is disabled" --- core/src/garden.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/core/src/garden.ts b/core/src/garden.ts index 4b469b343e..727399e714 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -123,7 +123,12 @@ import { GardenCloudApi, CloudApiTokenRefreshError } from "./cloud/api.js" import { OutputConfigContext } from "./config/template-contexts/module.js" import { ProviderConfigContext } from "./config/template-contexts/provider.js" import type { ConfigContext } from "./config/template-contexts/base.js" -import { deepResolveContext, GenericContext, type ContextWithSchema } from "./config/template-contexts/base.js" +import { + deepResolveContext, + ErrorContext, + GenericContext, + type ContextWithSchema, +} from "./config/template-contexts/base.js" import { validateSchema, validateWithPath } from "./config/validation.js" import { pMemoizeDecorator } from "./lib/p-memoize.js" import { ModuleGraph } from "./graph/modules.js" @@ -1568,11 +1573,9 @@ export class Garden { const deniedContexts = ["var", "variables"] for (const deniedContext of deniedContexts) { Object.defineProperty(context, deniedContext, { - get: () => { - throw new ConfigurationError({ - message: `If you have duplicate action names, the ${styles.accent("`disabled`")} flag cannot depend on the ${styles.accent(`\`${deniedContext}\``)} context.`, - }) - }, + value: new ErrorContext( + `If you have duplicate action names, the ${styles.accent("`disabled`")} flag cannot depend on the ${styles.accent(`\`${deniedContext}\``)} context.` + ), }) } From 69efd6595b9b8f43eac3a60b17ac2cf3db8c4e03 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 24 Jan 2025 08:56:44 +0100 Subject: [PATCH 104/117] fix: test should resolve actions from config templates --- core/test/unit/src/garden.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index 39d8508ebe..ff6e104ff3 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -4383,8 +4383,9 @@ describe("Garden", () => { templateName: "combo", inputs: { name: "test", - envName: "local", // <- should be resolved - providerKey: "${providers.test-plugin.outputs.testKey}", // <- not resolvable now + // these need to be resolved later + envName: "${environment.name}", + providerKey: "${providers.test-plugin.outputs.testKey}", }, } @@ -4395,7 +4396,7 @@ describe("Garden", () => { expect(serialiseUnresolvedTemplates(omit(deploy.getInternal(), "yamlDoc"))).to.eql(internal) expect(test.getDependencies().map((a) => a.key())).to.eql(["build.foo-test"]) // <- should be resolved - expect(omit(test.getInternal(), "yamlDoc")).to.eql(internal) + expect(serialiseUnresolvedTemplates(omit(test.getInternal(), "yamlDoc"))).to.eql(internal) }) it("throws with helpful message if action type doesn't exist", async () => { From 8b0cba4f8284488ba83c1cb9accc0904b138db1d Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 24 Jan 2025 09:24:44 +0100 Subject: [PATCH 105/117] chore: remove unused and misleading internal flag --- core/src/actions/types.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/actions/types.ts b/core/src/actions/types.ts index 4034378e9f..2123780f6d 100644 --- a/core/src/actions/types.ts +++ b/core/src/actions/types.ts @@ -65,7 +65,6 @@ export interface BaseActionConfig No templating is allowed on these. internal: GardenResourceInternalFields & { groupName?: string - resolved?: boolean // Set to true if no resolution is required, e.g. set for actions converted from modules treeVersion?: TreeVersion // Set during module resolution to avoid duplicate scanning for Build actions // For forwards-compatibility, applied on actions returned from module conversion handlers remoteClonePath?: string From 9a750ac63900daf747ed9817c2b42b296cc74b72 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 24 Jan 2025 09:25:33 +0100 Subject: [PATCH 106/117] fix: "preprocessActionConfig" tests --- core/test/unit/src/graph/actions.ts | 51 ++++++++++++++++------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/core/test/unit/src/graph/actions.ts b/core/test/unit/src/graph/actions.ts index f58ff874a0..1949d29a0a 100644 --- a/core/test/unit/src/graph/actions.ts +++ b/core/test/unit/src/graph/actions.ts @@ -13,6 +13,7 @@ import type { RunActionConfig } from "../../../../src/actions/run.js" import { DEFAULT_RUN_TIMEOUT_SEC } from "../../../../src/constants.js" import type tmp from "tmp-promise" import { expect } from "chai" +import { parseTemplateCollection } from "../../../../src/template/templated-collections.js" // TODO: add more tests describe("preprocessActionConfig", () => { @@ -31,18 +32,21 @@ describe("preprocessActionConfig", () => { context("template strings", () => { context("include/exclude configs", () => { it("should resolve variables in 'exclude' config", async () => { - const config: RunActionConfig = { - internal: { basePath: tmpDir.path }, - timeout: DEFAULT_RUN_TIMEOUT_SEC, - kind: "Run", - type: "exec", - name: "run", - exclude: ["${var.anyVar}"], - variables: { - anyVar: "*/**", + const config: RunActionConfig = parseTemplateCollection({ + value: { + internal: { basePath: tmpDir.path }, + timeout: DEFAULT_RUN_TIMEOUT_SEC, + kind: "Run" as const, + type: "exec", + name: "run", + exclude: ["${var.anyVar}"], + variables: { + anyVar: "*/**", + }, + spec: { command: ["echo", "foo"] }, }, - spec: { command: ["echo", "foo"] }, - } + source: { path: [] }, + }) const router = await garden.getActionRouter() const actionTypes = await garden.getActionTypes() @@ -64,18 +68,21 @@ describe("preprocessActionConfig", () => { }) it("should resolve variables in 'include' config", async () => { - const config: RunActionConfig = { - internal: { basePath: tmpDir.path }, - timeout: DEFAULT_RUN_TIMEOUT_SEC, - kind: "Run", - type: "exec", - name: "run", - include: ["${var.anyVar}"], - variables: { - anyVar: "*/**", + const config: RunActionConfig = parseTemplateCollection({ + value: { + internal: { basePath: tmpDir.path }, + timeout: DEFAULT_RUN_TIMEOUT_SEC, + kind: "Run" as const, + type: "exec", + name: "run", + include: ["${var.anyVar}"], + variables: { + anyVar: "*/**", + }, + spec: { command: ["echo", "foo"] }, }, - spec: { command: ["echo", "foo"] }, - } + source: { path: [] }, + }) const router = await garden.getActionRouter() const actionTypes = await garden.getActionTypes() From 36c11ad1c7cd631ef3a9318ea1953821910bbe58 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 24 Jan 2025 09:25:46 +0100 Subject: [PATCH 107/117] fix: rehydrate template strings after converting modules to actions --- core/src/resolve-module.ts | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index 52eafeca6a..ea2ab397e4 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -57,12 +57,11 @@ import { actionReferenceToString } from "./actions/base.js" import type { DepGraph } from "dependency-graph" import { minimatch } from "minimatch" import { getModuleTemplateReferences } from "./config/references.js" -import { capture } from "./template/capture.js" import { LayeredContext } from "./config/template-contexts/base.js" import { UnresolvedTemplateValue, type ParsedTemplate } from "./template/types.js" import { conditionallyDeepEvaluate, deepEvaluate, evaluate } from "./template/evaluate.js" import { someReferences } from "./template/analysis.js" -import { ForEachLazyValue } from "./template/templated-collections.js" +import { ForEachLazyValue, parseTemplateCollection } from "./template/templated-collections.js" import { deepMap, isPlainObject } from "./util/objects.js" import { LazyMergePatch } from "./template/lazy-merge.js" import { InputContext } from "./config/template-contexts/input.js" @@ -1014,6 +1013,21 @@ export const convertModules = profileAsync(async function convertModules( const totalReturned = (result.actions?.length || 0) + (result.group?.actions.length || 0) + log.debug(`Rehydrating templates in ${module.name}...`) + for (const a of result.actions || []) { + for (const k in a) { + a[k] = parseTemplateCollection({ value: a[k], source: { path: [k] } }) + } + } + for (const a of result.group?.actions || []) { + for (const k in a) { + a[k] = parseTemplateCollection({ value: a[k], source: { path: [k] } }) + } + } + for (const k in module) { + module[k] = parseTemplateCollection({ value: module[k], source: { path: [k] } }) + } + log.debug(`Module ${module.name} converted to ${totalReturned} action(s)`) if (result.group) { @@ -1132,9 +1146,6 @@ function inheritModuleToAction(module: GardenModule, action: ActionConfig) { action.internal.basePath = module.basePath || module.path } - // Converted actions are fully resolved upfront - action.internal.resolved = true - // Enforce some inheritance from module action.internal.moduleName = module.name action.internal.moduleVersion = module.version From 41920bc9965de14d4c440536673de9222528c30a Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 24 Jan 2025 09:43:32 +0100 Subject: [PATCH 108/117] fix: do not parse template on already parsed module --- core/src/resolve-module.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index ea2ab397e4..694f80b401 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -1013,7 +1013,7 @@ export const convertModules = profileAsync(async function convertModules( const totalReturned = (result.actions?.length || 0) + (result.group?.actions.length || 0) - log.debug(`Rehydrating templates in ${module.name}...`) + log.debug(`Rehydrating templates for ${totalReturned} action(s) in ${module.name}...`) for (const a of result.actions || []) { for (const k in a) { a[k] = parseTemplateCollection({ value: a[k], source: { path: [k] } }) @@ -1024,9 +1024,6 @@ export const convertModules = profileAsync(async function convertModules( a[k] = parseTemplateCollection({ value: a[k], source: { path: [k] } }) } } - for (const k in module) { - module[k] = parseTemplateCollection({ value: module[k], source: { path: [k] } }) - } log.debug(`Module ${module.name} converted to ${totalReturned} action(s)`) From 57d968e1003324a146fadd6114b808b4b381eedf Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Fri, 24 Jan 2025 11:08:14 +0100 Subject: [PATCH 109/117] fix: do early module spec validation only if inputs are resolved --- core/src/resolve-module.ts | 10 +++++----- core/test/unit/src/garden.ts | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index 694f80b401..aa1fb98eff 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -8,7 +8,7 @@ import { isArray, isString, keyBy, partition, uniq } from "lodash-es" import { validateWithPath } from "./config/validation.js" -import { resolveTemplateString } from "./template/templated-strings.js" +import { isUnresolved, resolveTemplateString } from "./template/templated-strings.js" import { GenericContext } from "./config/template-contexts/base.js" import { dirname, posix, relative, resolve } from "path" import type { Garden } from "./garden.js" @@ -644,16 +644,16 @@ export class ModuleResolver { config.spec.build.dependencies = prepareBuildDependencies(config.spec.build.dependencies) } - // Validate the module-type specific spec - if (description.schema) { + // Validate the module-type specific spec if the inputs are already resolved + if (description.schema && !isUnresolved(config.inputs)) { config = { ...config, spec: validateWithPath({ config: config.spec, - configType: "Module", + configType: `spec for module`, schema: description.schema, name: config.name, - path: config.path, + path: config.configPath || config.path, projectRoot: garden.projectRoot, source: undefined, }), diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index ff6e104ff3..29d295bfa6 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -4529,8 +4529,8 @@ describe("Garden", () => { await expectError(() => garden.getConfigGraph({ log: garden.log, emit: false }), { contains: [ "Failed resolving one or more modules", - "foo-test-a: Error validating inputs for module foo-test-a (modules.garden.yml)", - "value at ./value must be string", + "Error validating spec for module 'foo-test-a' (modules.garden.yml)", + "build.command[0] must be a string", ], }) }) From 437b3abcc84901d5c03e8ddfa5f579ce899adbc7 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Fri, 24 Jan 2025 12:00:46 +0100 Subject: [PATCH 110/117] fix: do early module spec validation only if inputs are resolved Follow-up fix for 57d968e1. --- core/src/resolve-module.ts | 31 ++++++++++++++++++++++--------- core/test/unit/src/garden.ts | 4 ++-- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index aa1fb98eff..0d6ecc5fcb 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -9,7 +9,7 @@ import { isArray, isString, keyBy, partition, uniq } from "lodash-es" import { validateWithPath } from "./config/validation.js" import { isUnresolved, resolveTemplateString } from "./template/templated-strings.js" -import { GenericContext } from "./config/template-contexts/base.js" +import { GenericContext, LayeredContext } from "./config/template-contexts/base.js" import { dirname, posix, relative, resolve } from "path" import type { Garden } from "./garden.js" import type { GardenError } from "./exceptions.js" @@ -33,8 +33,6 @@ import { allowUnknown } from "./config/common.js" import type { ProviderMap } from "./config/provider.js" import { DependencyGraph } from "./graph/common.js" import fsExtra from "fs-extra" - -const { mkdirp, readFile } = fsExtra import type { Log } from "./logger/log-entry.js" import type { ModuleConfigContextParams } from "./config/template-contexts/module.js" import { ModuleConfigContext } from "./config/template-contexts/module.js" @@ -57,8 +55,7 @@ import { actionReferenceToString } from "./actions/base.js" import type { DepGraph } from "dependency-graph" import { minimatch } from "minimatch" import { getModuleTemplateReferences } from "./config/references.js" -import { LayeredContext } from "./config/template-contexts/base.js" -import { UnresolvedTemplateValue, type ParsedTemplate } from "./template/types.js" +import { type ParsedTemplate, UnresolvedTemplateValue } from "./template/types.js" import { conditionallyDeepEvaluate, deepEvaluate, evaluate } from "./template/evaluate.js" import { someReferences } from "./template/analysis.js" import { ForEachLazyValue, parseTemplateCollection } from "./template/templated-collections.js" @@ -66,6 +63,8 @@ import { deepMap, isPlainObject } from "./util/objects.js" import { LazyMergePatch } from "./template/lazy-merge.js" import { InputContext } from "./config/template-contexts/input.js" +const { mkdirp, readFile } = fsExtra + // This limit is fairly arbitrary, but we need to have some cap on concurrent processing. export const moduleResolutionConcurrencyLimit = 50 @@ -628,6 +627,20 @@ export class ModuleResolver { context ) as unknown as ModuleConfig + // Validate inputs if those are already resolved + const templateName = config.templateName + if (templateName && !isUnresolved(config.inputs)) { + const template = this.garden.configTemplates[templateName] + config.inputs = validateWithPath({ + config: config.inputs, + configType: `inputs for module ${config.name}`, + path: config.configPath || config.path, + schema: template.inputsSchema, + projectRoot: garden.projectRoot, + source: undefined, + }) + } + // We allow specifying modules by name only as a shorthand: // // dependencies: @@ -644,16 +657,16 @@ export class ModuleResolver { config.spec.build.dependencies = prepareBuildDependencies(config.spec.build.dependencies) } - // Validate the module-type specific spec if the inputs are already resolved - if (description.schema && !isUnresolved(config.inputs)) { + // Validate the module-type specific spec + if (description.schema) { config = { ...config, spec: validateWithPath({ config: config.spec, - configType: `spec for module`, + configType: "Module", schema: description.schema, name: config.name, - path: config.configPath || config.path, + path: config.path, projectRoot: garden.projectRoot, source: undefined, }), diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index 29d295bfa6..ff6e104ff3 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -4529,8 +4529,8 @@ describe("Garden", () => { await expectError(() => garden.getConfigGraph({ log: garden.log, emit: false }), { contains: [ "Failed resolving one or more modules", - "Error validating spec for module 'foo-test-a' (modules.garden.yml)", - "build.command[0] must be a string", + "foo-test-a: Error validating inputs for module foo-test-a (modules.garden.yml)", + "value at ./value must be string", ], }) }) From 6842286ad2bcf5e22215de5cc9dedccfb1ef9ef1 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Fri, 24 Jan 2025 12:29:42 +0100 Subject: [PATCH 111/117] fix: perform toJSON serialization only for unresolved template values --- core/src/util/serialization.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/core/src/util/serialization.ts b/core/src/util/serialization.ts index 8a5a798967..a4ae4cbaf5 100644 --- a/core/src/util/serialization.ts +++ b/core/src/util/serialization.ts @@ -10,6 +10,7 @@ import { mapValues } from "lodash-es" import fsExtra from "fs-extra" import type { DumpOptions } from "js-yaml" import { dump, load } from "js-yaml" +import { UnresolvedTemplateValue } from "../template/types.js" const { readFile, writeFile } = fsExtra @@ -24,7 +25,15 @@ export function safeDumpYaml(data: any, opts: Omit = {} return dump(data, { ...opts, skipInvalid: true, - replacer: (_, v) => (typeof v === "object" && typeof v?.["toJSON"] === "function" ? v["toJSON"]() : v), + replacer: (_, v) => { + if (v instanceof UnresolvedTemplateValue) { + // serialize unresolved template values to JSON + return v.toJSON() + } + + // return the original value otherwise + return v + }, }) } From 85a71cb2f3d2a862cbcdbc6169e99070295b5731 Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:02:36 +0100 Subject: [PATCH 112/117] test: fix tests in "resolveProjectOutputs" --- core/test/unit/src/outputs.ts | 44 +++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/core/test/unit/src/outputs.ts b/core/test/unit/src/outputs.ts index 363da9c58f..e153950e21 100644 --- a/core/test/unit/src/outputs.ts +++ b/core/test/unit/src/outputs.ts @@ -11,11 +11,13 @@ import { createProjectConfig, makeTempDir, TestGarden } from "../../helpers.js" import { resolveProjectOutputs } from "../../../src/outputs.js" import { expect } from "chai" import fsExtra from "fs-extra" -const { realpath } = fsExtra import { createGardenPlugin } from "../../../src/plugin/plugin.js" import type { ProjectConfig } from "../../../src/config/project.js" import { DEFAULT_BUILD_TIMEOUT_SEC, GardenApiVersion } from "../../../src/constants.js" import { joi } from "../../../src/config/common.js" +import { parseTemplateCollection } from "../../../src/template/templated-collections.js" + +const { realpath } = fsExtra describe("resolveProjectOutputs", () => { let tmpDir: tmp.DirectoryResult @@ -54,7 +56,15 @@ describe("resolveProjectOutputs", () => { }, }) - projectConfig.outputs = [{ name: "test", value: "${providers.test.outputs.test}" }] + projectConfig.outputs = parseTemplateCollection({ + value: [ + { + name: "test", + value: "${providers.test.outputs.test}", + }, + ], + source: { path: [] }, + }) const garden = await TestGarden.factory(tmpPath, { plugins: [plugin], @@ -88,7 +98,15 @@ describe("resolveProjectOutputs", () => { ], }) - projectConfig.outputs = [{ name: "test", value: "${modules.test.outputs.test}" }] + projectConfig.outputs = parseTemplateCollection({ + value: [ + { + name: "test", + value: "${modules.test.outputs.test}", + }, + ], + source: { path: [] }, + }) const garden = await TestGarden.factory(tmpPath, { plugins: [plugin], @@ -152,7 +170,15 @@ describe("resolveProjectOutputs", () => { }, }) - projectConfig.outputs = [{ name: "test", value: "${runtime.services.test.outputs.test}" }] + projectConfig.outputs = parseTemplateCollection({ + value: [ + { + name: "test", + value: "${runtime.services.test.outputs.test}", + }, + ], + source: { path: [] }, + }) const garden = await TestGarden.factory(tmpPath, { plugins: [plugin], @@ -220,7 +246,15 @@ describe("resolveProjectOutputs", () => { }, }) - projectConfig.outputs = [{ name: "test", value: "${runtime.tasks.test.outputs.log}" }] + projectConfig.outputs = parseTemplateCollection({ + value: [ + { + name: "test", + value: "${runtime.tasks.test.outputs.log}", + }, + ], + source: { path: [] }, + }) const garden = await TestGarden.factory(tmpPath, { plugins: [plugin], From 73e26d371a375dd2519ce9758c5659ec2aa0664a Mon Sep 17 00:00:00 2001 From: Vladimir Vagaytsev <10628074+vvagaytsev@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:06:23 +0100 Subject: [PATCH 113/117] chore: fix lint errors --- core/src/config/template-contexts/module.ts | 2 +- core/src/plugin/handlers/base/configure.ts | 2 -- core/test/unit/src/config/base.ts | 1 - core/test/unit/src/config/template-contexts/base.ts | 4 +++- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/core/src/config/template-contexts/module.ts b/core/src/config/template-contexts/module.ts index f2aae7ebd7..05364e0593 100644 --- a/core/src/config/template-contexts/module.ts +++ b/core/src/config/template-contexts/module.ts @@ -22,7 +22,7 @@ import type { DeployTask } from "../../tasks/deploy.js" import type { RunTask } from "../../tasks/run.js" import { DOCS_BASE_URL } from "../../constants.js" import { styles } from "../../logger/styles.js" -import { InputContext } from "./input.js" +import type { InputContext } from "./input.js" export const exampleVersion = "v-17ad4cb3fd" diff --git a/core/src/plugin/handlers/base/configure.ts b/core/src/plugin/handlers/base/configure.ts index 9349255e92..eca9f6e10c 100644 --- a/core/src/plugin/handlers/base/configure.ts +++ b/core/src/plugin/handlers/base/configure.ts @@ -12,11 +12,9 @@ import { logEntrySchema } from "../../../plugin/base.js" import { joi } from "../../../config/common.js" import type { Log } from "../../../logger/log-entry.js" import type { ActionModes, BaseActionConfig } from "../../../actions/types.js" -import { baseActionConfigSchema } from "../../../actions/base.js" import { ActionTypeHandlerSpec } from "./base.js" import { pluginContextSchema } from "../../../plugin-context.js" import { noTemplateFields } from "../../../config/base.js" -import { actionConfigSchema } from "../../../actions/helpers.js" interface ConfigureActionConfigParams extends PluginActionContextParams { log: Log diff --git a/core/test/unit/src/config/base.ts b/core/test/unit/src/config/base.ts index fa5b1ce2a2..b10d07c7e3 100644 --- a/core/test/unit/src/config/base.ts +++ b/core/test/unit/src/config/base.ts @@ -10,7 +10,6 @@ import { expect } from "chai" import { loadConfigResources, findProjectConfig, - prepareModuleResource, prepareProjectResource, noTemplateFields, validateRawConfig, diff --git a/core/test/unit/src/config/template-contexts/base.ts b/core/test/unit/src/config/template-contexts/base.ts index a4fc1670fe..e3a6ab7824 100644 --- a/core/test/unit/src/config/template-contexts/base.ts +++ b/core/test/unit/src/config/template-contexts/base.ts @@ -8,11 +8,13 @@ import { expect } from "chai" import stripAnsi from "strip-ansi" -import { +import type { ConfigContext, ContextKey, ContextResolveOutputNotFound, ContextResolveParams, +} from "../../../../../src/config/template-contexts/base.js" +import { ContextWithSchema, GenericContext, getUnavailableReason, From 7aa19b6ccadde4b87965f80dc0f085bef7b8d911 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 24 Jan 2025 17:56:56 +0100 Subject: [PATCH 114/117] fix: enforce correct variable precedence centrally and make sure version calculation is only affected if the variable is used. fixes #3473 --- core/src/actions/base.ts | 17 +- core/src/actions/types.ts | 6 +- core/src/commands/custom.ts | 13 +- core/src/config/project.ts | 81 ++-- core/src/config/template-contexts/actions.ts | 26 +- core/src/config/template-contexts/base.ts | 12 +- .../template-contexts/custom-command.ts | 5 +- core/src/config/template-contexts/module.ts | 4 +- core/src/config/template-contexts/project.ts | 9 +- core/src/config/template-contexts/provider.ts | 4 +- .../src/config/template-contexts/variables.ts | 251 ++++++++++ core/src/garden.ts | 59 +-- core/src/graph/actions.ts | 53 +-- core/src/graph/common.ts | 46 +- core/src/resolve-module.ts | 96 ++-- core/src/router/base.ts | 1 + core/src/tasks/resolve-action.ts | 40 +- core/src/template/capture.ts | 2 +- core/src/util/testing.ts | 4 +- core/src/vcs/vcs.ts | 5 +- core/test/unit/src/actions/base.ts | 2 + core/test/unit/src/commands/workflow.ts | 4 +- core/test/unit/src/config/project.ts | 448 +++++++++++++++--- core/test/unit/src/config/render-template.ts | 5 +- core/test/unit/src/config/workflow.ts | 6 +- core/test/unit/src/garden.ts | 41 +- core/test/unit/src/resolve-module.ts | 13 +- core/test/unit/src/vcs/vcs.ts | 6 +- 28 files changed, 865 insertions(+), 394 deletions(-) create mode 100644 core/src/config/template-contexts/variables.ts diff --git a/core/src/actions/base.ts b/core/src/actions/base.ts index 049779d893..931c201dbd 100644 --- a/core/src/actions/base.ts +++ b/core/src/actions/base.ts @@ -69,6 +69,7 @@ import { dirname } from "node:path" import type { ConfigContext } from "../config/template-contexts/base.js" import type { ResolvedTemplate } from "../template/types.js" import type { WorkflowConfig } from "../config/workflow.js" +import type { VariablesContext } from "../config/template-contexts/variables.js" // TODO: split this file @@ -372,7 +373,7 @@ export abstract class BaseAction< protected readonly projectRoot: string protected readonly _supportedModes: ActionModes protected readonly _treeVersion: TreeVersion - protected readonly variables: ConfigContext + protected readonly variables: VariablesContext constructor(protected readonly params: ActionWrapperParams) { this.kind = params.config.kind @@ -585,7 +586,7 @@ export abstract class BaseAction< } } - getVariablesContext(): ConfigContext { + getVariablesContext(): VariablesContext { return this.variables } @@ -955,8 +956,18 @@ export function actionIsDisabled(config: ActionConfig, environmentName: string): * see {@link VcsHandler.getTreeVersion} and {@link VcsHandler.getFiles}. * - The description field is just informational, shouldn't affect execution. * - The disabled flag is not relevant to the config version, since it only affects execution. + * - The variables and varfiles are only relevant if they have an effect on a relevant piece of configuration and thus can be omitted. */ -const nonVersionedActionConfigKeys = ["internal", "source", "include", "exclude", "description", "disabled"] as const +const nonVersionedActionConfigKeys = [ + "internal", + "source", + "include", + "exclude", + "description", + "disabled", + "variables", + "varfiles", +] as const export type NonVersionedActionConfigKey = keyof Pick export function getActionConfigVersion(config: C) { diff --git a/core/src/actions/types.ts b/core/src/actions/types.ts index 2123780f6d..63c4cf905d 100644 --- a/core/src/actions/types.ts +++ b/core/src/actions/types.ts @@ -21,8 +21,8 @@ import type { ValidResultType } from "../tasks/base.js" import type { BaseGardenResource, GardenResourceInternalFields } from "../config/base.js" import type { LinkedSource } from "../config-store/local.js" import type { GardenApiVersion } from "../constants.js" -import type { ConfigContext } from "../config/template-contexts/base.js" import type { ResolvedTemplate } from "../template/types.js" +import type { VariablesContext } from "../config/template-contexts/variables.js" // TODO: split this file @@ -171,7 +171,7 @@ export interface ActionWrapperParams { remoteSourcePath: string | null supportedModes: ActionModes treeVersion: TreeVersion - variables: ConfigContext + variables: VariablesContext } export interface ResolveActionParams = any> { @@ -182,7 +182,7 @@ export interface ResolveActionParams } diff --git a/core/src/commands/custom.ts b/core/src/commands/custom.ts index 9e37b42d50..acfce53a49 100644 --- a/core/src/commands/custom.ts +++ b/core/src/commands/custom.ts @@ -30,7 +30,7 @@ import type { Log } from "../logger/log-entry.js" import { getTracePropagationEnvVars } from "../util/open-telemetry/propagation.js" import { styles } from "../logger/styles.js" import { deepEvaluate } from "../template/evaluate.js" -import { GenericContext, LayeredContext } from "../config/template-contexts/base.js" +import { VariablesContext } from "../config/template-contexts/variables.js" function convertArgSpec(spec: CustomCommandOption) { const params = { @@ -113,11 +113,16 @@ export class CustomCommandWrapper extends Command { // Render the command variables - const commandVariables = new GenericContext(this.spec.variables) - const variables = new LayeredContext(garden.variables, commandVariables) + const variableContext = new CustomCommandContext({ ...garden, args, opts, variables: garden.variables, rest }) // Make a new template context with the resolved variables - const commandContext = new CustomCommandContext({ ...garden, args, opts, variables, rest }) + const commandContext = new CustomCommandContext({ + ...garden, + args, + opts, + variables: VariablesContext.forCustomCommand(garden, this.spec, variableContext), + rest, + }) const result: CustomCommandResult = {} const errors: GardenError[] = [] diff --git a/core/src/config/project.ts b/core/src/config/project.ts index 5eb5c0eb7b..d82a9867ab 100644 --- a/core/src/config/project.ts +++ b/core/src/config/project.ts @@ -23,7 +23,8 @@ import { import type { ConfigSource } from "./validation.js" import { validateConfig, validateWithPath } from "./validation.js" import { deepEvaluate, evaluate } from "../template/evaluate.js" -import { EnvironmentConfigContext, ProjectConfigContext } from "./template-contexts/project.js" +import type { ProjectConfigContext } from "./template-contexts/project.js" +import { EnvironmentConfigContext } from "./template-contexts/project.js" import { findByName, getNames } from "../util/util.js" import { ConfigurationError, InternalError, ParameterError, ValidationError } from "../exceptions.js" import { memoize } from "lodash-es" @@ -35,18 +36,17 @@ import type { CommandInfo } from "../plugin-context.js" import type { VcsInfo } from "../vcs/vcs.js" import { profileAsync } from "../util/profiling.js" import type { BaseGardenResource } from "./base.js" -import { baseInternalFieldsSchema, loadVarfile, varfileDescription } from "./base.js" +import { baseInternalFieldsSchema, varfileDescription } from "./base.js" import type { Log } from "../logger/log-entry.js" import { renderDivider } from "../logger/util.js" import { styles } from "../logger/styles.js" import { serialiseUnresolvedTemplates, type ParsedTemplate } from "../template/types.js" -import { LayeredContext } from "./template-contexts/base.js" -import type { ConfigContext } from "./template-contexts/base.js" -import { GenericContext } from "./template-contexts/base.js" +import { deepResolveContext } from "./template-contexts/base.js" import { LazyMergePatch } from "../template/lazy-merge.js" import { isArray, isPlainObject } from "../util/objects.js" +import { VariablesContext } from "./template-contexts/variables.js" -export const defaultVarfilePath = "garden.env" +export const defaultProjectVarfilePath = "garden.env" export const defaultEnvVarfilePath = (environmentName: string) => `garden.${environmentName}.env` export const defaultEnvironment = "default" @@ -410,7 +410,7 @@ export const projectSchema = createSchema({ sources: projectSourcesSchema(), varfile: joi .posixPath() - .default(defaultVarfilePath) + .default(defaultProjectVarfilePath) .description( dedent` Specify a path (relative to the project root) to a file containing variables, that we apply on top of the @@ -462,40 +462,17 @@ export function resolveProjectConfig({ log, defaultEnvironmentName, config, - artifactsPath, - vcsInfo, - username, - loggedIn, - enterpriseDomain, - secrets, - commandInfo, + context, }: { log: Log defaultEnvironmentName: string config: ProjectConfig - artifactsPath: string - vcsInfo: VcsInfo - username: string - loggedIn: boolean - enterpriseDomain: string | undefined - secrets: PrimitiveMap - commandInfo: CommandInfo + context: ProjectConfigContext }): ProjectConfig { // Resolve template strings for non-environment-specific fields (apart from `sources`). const { environments = [], name, sources = [], providers = [], outputs = [] } = config let globalConfig: any - const context = new ProjectConfigContext({ - projectName: name, - projectRoot: config.path, - artifactsPath, - vcsInfo, - username, - loggedIn, - enterpriseDomain, - secrets, - commandInfo, - }) try { globalConfig = deepEvaluate( @@ -587,6 +564,7 @@ export class UnresolvedProviderConfig { */ export const pickEnvironment = profileAsync(async function _pickEnvironment({ projectConfig, + variableOverrides, envString, artifactsPath, vcsInfo, @@ -595,7 +573,10 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ enterpriseDomain, secrets, commandInfo, + projectContext, }: { + projectContext: ProjectConfigContext + variableOverrides: DeepPrimitiveMap projectConfig: ProjectConfig envString: string artifactsPath: string @@ -632,17 +613,6 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ }) } - // TODO: parse template strings in varfiles? - const projectVarfileVars = await loadVarfile({ - configRoot: projectConfig.path, - path: projectConfig.varfile, - defaultPath: defaultVarfilePath, - }) - const projectVariables: LayeredContext = new LayeredContext( - new GenericContext(projectConfig.variables), - new GenericContext(projectVarfileVars) - ) - const source = { yamlDoc: projectConfig.internal.yamlDoc, path: ["environments", index] } // Resolve template strings in the environment config, except providers @@ -652,12 +622,16 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ artifactsPath, vcsInfo, username, - variables: projectVariables, + variables: await VariablesContext.forProject(projectConfig, variableOverrides, projectContext), loggedIn, enterpriseDomain, secrets, commandInfo, }) + + // resolve project variables incl. varfiles + deepResolveContext("project", context.variables) + const config = deepEvaluate(environmentConfig as unknown as ParsedTemplate, { context, opts: {}, @@ -737,18 +711,17 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ mergedProviders[name] = new UnresolvedProviderConfig(name, dependencies || [], unresolvedConfig) } - const envVarfileVars = await loadVarfile({ - configRoot: projectConfig.path, - path: environmentConfig.varfile, - defaultPath: defaultEnvVarfilePath(environment), - }) - - const variables: ConfigContext = new LayeredContext( - projectVariables, - new GenericContext(environmentConfig.variables), - new GenericContext(envVarfileVars) + const variables = await VariablesContext.forEnvironment( + environment, + projectConfig, + environmentConfig, + variableOverrides, + context ) + // resolve project and environment-level variables incl. varfiles + deepResolveContext("project environment", context.variables) + return { environmentName: environment, namespace, diff --git a/core/src/config/template-contexts/actions.ts b/core/src/config/template-contexts/actions.ts index 7f16f8e5c3..c71b0d7259 100644 --- a/core/src/config/template-contexts/actions.ts +++ b/core/src/config/template-contexts/actions.ts @@ -15,17 +15,13 @@ import type { PrimitiveMap } from "../common.js" import { joi, joiIdentifier, joiIdentifierMap, joiPrimitive, joiVariables } from "../common.js" import type { ProviderMap } from "../provider.js" import type { ConfigContext } from "./base.js" -import { ContextWithSchema, ErrorContext, GenericContext, ParentContext, schema, TemplateContext } from "./base.js" +import { ContextWithSchema, ErrorContext, ParentContext, schema, TemplateContext } from "./base.js" import { exampleVersion, OutputConfigContext } from "./module.js" import { TemplatableConfigContext } from "./templatable.js" import { DOCS_BASE_URL } from "../../constants.js" import { styles } from "../../logger/styles.js" -import { LayeredContext } from "./base.js" import type { InputContext } from "./input.js" - -function mergeVariables({ garden, variables }: { garden: Garden; variables: ConfigContext }): LayeredContext { - return new LayeredContext(garden.variables, variables, new GenericContext(garden.variableOverrides)) -} +import type { VariablesContext } from "./variables.js" type ActionConfigThisContextParams = Pick @@ -63,7 +59,7 @@ interface ActionConfigContextParams { garden: Garden config: ActionConfig thisContextParams: ActionConfigThisContextParams - variables: ConfigContext + variables: VariablesContext } /** @@ -75,10 +71,9 @@ export class ActionConfigContext extends TemplatableConfigContext { public this: ActionConfigThisContext constructor({ garden, config, thisContextParams, variables }: ActionConfigContextParams) { - const mergedVariables = mergeVariables({ garden, variables }) super(garden, config) this.this = new ActionConfigThisContext(thisContextParams) - this.variables = this.var = mergedVariables + this.variables = this.var = variables } } @@ -88,7 +83,7 @@ interface ActionReferenceContextParams { buildPath: string sourcePath: string mode: ActionMode - variables: ConfigContext + variables: VariablesContext } export class ActionReferenceContext extends ContextWithSchema { @@ -227,7 +222,7 @@ export interface ActionSpecContextParams { action: Action resolvedDependencies: ResolvedAction[] executedDependencies: ExecutedAction[] - variables: ConfigContext + variables: VariablesContext inputs: InputContext } @@ -270,15 +265,12 @@ export class ActionSpecContext extends OutputConfigContext { public this: ActionReferenceContext constructor(params: ActionSpecContextParams) { - const { action, garden, variables, inputs, resolvedDependencies, executedDependencies } = params + const { action, variables, inputs, resolvedDependencies, executedDependencies } = params const internal = action.getInternal() - - const mergedVariables = mergeVariables({ garden, variables }) - super({ ...params, - variables: mergedVariables, + variables, }) const name = action.name @@ -309,7 +301,7 @@ export class ActionSpecContext extends OutputConfigContext { name, sourcePath, mode: action.mode(), - variables: mergedVariables, + variables, }) } } diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index e8b434c253..cfd6a733f1 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -111,6 +111,10 @@ export abstract class ConfigContext { this._cache = new Map() } + protected clearCache() { + this._cache.clear() + } + private detectCircularReference({ nodePath, key, opts }: ContextResolveParams) { const plainKey = renderKeyPath(key) const keyStr = `${this.constructor.name}(${this._id})-${plainKey}` @@ -303,17 +307,17 @@ export function renderKeyPath(key: ContextKeySegment[]): string { } export class LayeredContext extends ConfigContext { - private readonly contexts: ConfigContext[] + protected readonly layers: ConfigContext[] - constructor(...contexts: ConfigContext[]) { + constructor(...layers: ConfigContext[]) { super() - this.contexts = contexts + this.layers = layers } override resolveImpl(args: ContextResolveParams): ContextResolveOutput { const layeredItems: ContextResolveOutput[] = [] - for (const context of this.contexts.toReversed()) { + for (const context of this.layers.toReversed()) { const resolved = context.resolve(args) if (resolved.found) { if (isTemplatePrimitive(resolved.resolved)) { diff --git a/core/src/config/template-contexts/custom-command.ts b/core/src/config/template-contexts/custom-command.ts index 6ea0faa5d0..b3f991a06b 100644 --- a/core/src/config/template-contexts/custom-command.ts +++ b/core/src/config/template-contexts/custom-command.ts @@ -12,6 +12,7 @@ import type { DefaultEnvironmentContextParams } from "./project.js" import { DefaultEnvironmentContext } from "./project.js" import type { ConfigContext } from "./base.js" import { schema } from "./base.js" +import type { VariablesContext } from "./variables.js" interface ArgsSchema { [name: string]: string | number | string[] @@ -30,7 +31,7 @@ export class CustomCommandContext extends DefaultEnvironmentContext { .description("A map of all variables defined in the command configuration.") .meta({ keyPlaceholder: "" }) ) - public variables: ConfigContext + public variables: VariablesContext @schema(joiIdentifierMap(joiPrimitive()).description("Alias for the variables field.")) public var: ConfigContext @@ -70,7 +71,7 @@ export class CustomCommandContext extends DefaultEnvironmentContext { params: DefaultEnvironmentContextParams & { args: ArgsSchema opts: OptsSchema - variables: ConfigContext + variables: VariablesContext rest: string[] } ) { diff --git a/core/src/config/template-contexts/module.ts b/core/src/config/template-contexts/module.ts index 05364e0593..c7ab47257c 100644 --- a/core/src/config/template-contexts/module.ts +++ b/core/src/config/template-contexts/module.ts @@ -14,7 +14,6 @@ import { joi } from "../common.js" import { deline } from "../../util/string.js" import { getModuleTypeUrl } from "../../docs/common.js" import type { GardenModule } from "../../types/module.js" -import type { ConfigContext } from "./base.js" import { ContextWithSchema, schema, ErrorContext, ParentContext, TemplateContext } from "./base.js" import { ProviderConfigContext } from "./provider.js" import type { GraphResultFromTask, GraphResults } from "../../graph/results.js" @@ -23,6 +22,7 @@ import type { RunTask } from "../../tasks/run.js" import { DOCS_BASE_URL } from "../../constants.js" import { styles } from "../../logger/styles.js" import type { InputContext } from "./input.js" +import type { VariablesContext } from "./variables.js" export const exampleVersion = "v-17ad4cb3fd" @@ -191,7 +191,7 @@ class RuntimeConfigContext extends ContextWithSchema { export interface OutputConfigContextParams { garden: Garden resolvedProviders: ProviderMap - variables: ConfigContext + variables: VariablesContext modules: GardenModule[] // We only supply this when resolving configuration in dependency order. // Otherwise we pass `${runtime.*} template strings through for later resolution. diff --git a/core/src/config/template-contexts/project.ts b/core/src/config/template-contexts/project.ts index 086eca6103..97072b12a0 100644 --- a/core/src/config/template-contexts/project.ts +++ b/core/src/config/template-contexts/project.ts @@ -17,6 +17,7 @@ import type { CommandInfo } from "../../plugin-context.js" import type { Garden } from "../../garden.js" import { type VcsInfo } from "../../vcs/vcs.js" import { styles } from "../../logger/styles.js" +import type { VariablesContext } from "./variables.js" class LocalContext extends ContextWithSchema { @schema( @@ -344,7 +345,7 @@ export class ProjectConfigContext extends DefaultEnvironmentContext { } export interface EnvironmentConfigContextParams extends ProjectConfigContextParams { - variables: ConfigContext + variables: VariablesContext } /** @@ -356,7 +357,7 @@ export class EnvironmentConfigContext extends ProjectConfigContext { .description("A map of all variables defined in the project configuration.") .meta({ keyPlaceholder: "" }) ) - public variables: ConfigContext + public variables: VariablesContext @schema(joiIdentifierMap(joiPrimitive()).description("Alias for the variables field.")) public var: ConfigContext @@ -392,9 +393,9 @@ export class RemoteSourceConfigContext extends EnvironmentConfigContext { ) .meta({ keyPlaceholder: "" }) ) - public override variables: ConfigContext + public override variables: VariablesContext - constructor(garden: Garden, variables: ConfigContext) { + constructor(garden: Garden, variables: VariablesContext) { super({ projectName: garden.projectName, projectRoot: garden.projectRoot, diff --git a/core/src/config/template-contexts/provider.ts b/core/src/config/template-contexts/provider.ts index 089e02edfb..55332af830 100644 --- a/core/src/config/template-contexts/provider.ts +++ b/core/src/config/template-contexts/provider.ts @@ -14,9 +14,9 @@ import type { Garden } from "../../garden.js" import { joi } from "../common.js" import { deline } from "../../util/string.js" import { getProviderUrl } from "../../docs/common.js" -import type { ConfigContext } from "./base.js" import { ContextWithSchema, schema } from "./base.js" import { WorkflowConfigContext } from "./workflow.js" +import type { VariablesContext } from "./variables.js" class ProviderContext extends ContextWithSchema { @schema( @@ -65,7 +65,7 @@ export class ProviderConfigContext extends WorkflowConfigContext { ) public providers: Map - constructor(garden: Garden, resolvedProviders: ProviderMap, variables: ConfigContext) { + constructor(garden: Garden, resolvedProviders: ProviderMap, variables: VariablesContext) { super(garden, variables) this.providers = new Map(Object.entries(mapValues(resolvedProviders, (p) => new ProviderContext(p)))) diff --git a/core/src/config/template-contexts/variables.ts b/core/src/config/template-contexts/variables.ts new file mode 100644 index 0000000000..13d22699ab --- /dev/null +++ b/core/src/config/template-contexts/variables.ts @@ -0,0 +1,251 @@ +/* + * Copyright (C) 2018-2024 Garden Technologies, Inc. + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +import type { DeepPrimitiveMap } from "@garden-io/platform-api-types" +import type { Garden } from "../../garden.js" +import { describeConfig } from "../../vcs/vcs.js" +import type { ModuleConfig } from "../module.js" +import type { ConfigContext, ContextWithSchema } from "./base.js" +import { GenericContext, LayeredContext } from "./base.js" +import { deepEvaluate } from "../../template/evaluate.js" +import type { ModuleConfigContext } from "./module.js" +import { ConfigurationError } from "../../exceptions.js" +import { getEffectiveConfigFileLocation, loadVarfile } from "../base.js" +import { capture } from "../../template/capture.js" +import { UnresolvedTemplateValue, type ParsedTemplate } from "../../template/types.js" +import type { EnvironmentConfig } from "../project.js" +import { defaultEnvVarfilePath, defaultProjectVarfilePath, type ProjectConfig } from "../project.js" +import type { EnvironmentConfigContext, ProjectConfigContext } from "./project.js" +import { isPlainObject, set as setKeyPathNested } from "lodash-es" +import type { ActionConfig } from "../../actions/types.js" +import { isUnresolved } from "../../template/templated-strings.js" +import type { Varfile } from "../common.js" +import type { ActionConfigContext } from "./actions.js" +import type { CustomCommandContext } from "./custom-command.js" +import type { CommandResource } from "../command.js" +import type { GroupConfig } from "../group.js" + +export class VariablesContext extends LayeredContext { + /** + * The constructor is private, use the static factory methods (below) instead. + */ + private constructor( + public readonly description: string, + { + context, + variablePrecedence, + variableOverrides, + }: { + context: EnvironmentConfigContext | ProjectConfigContext | CustomCommandContext + variablePrecedence: (ParsedTemplate | undefined | null)[] + variableOverrides: DeepPrimitiveMap + } + ) { + const layers: ConfigContext[] = [ + // project config context has no variables yet. Use empty context as root instead then + "variables" in context ? context.variables : new GenericContext({}), + ] + + const entries = variablePrecedence.filter((tpl) => !isEmpty(tpl)).entries() + + for (const [i, layer] of entries) { + const parent = layers[i] + layers.push( + new GenericContext( + // this ensures that a variable in any given context can refer to variables in the parent scope + capture(layer || {}, makeVariableRootContext(parent)) + ) + ) + } + + super(...layers) + + if (variableOverrides && !isEmpty(variableOverrides)) { + this.applyOverrides(variableOverrides, context) + } + } + + public static forTest(garden: Garden, ...vars: ParsedTemplate[]) { + return new this("test", { + context: garden.getProjectConfigContext(), + variablePrecedence: [...vars], + variableOverrides: garden.variableOverrides, + }) + } + + public static async forProject( + projectConfig: ProjectConfig, + variableOverrides: DeepPrimitiveMap, + context: ProjectConfigContext + ) { + // TODO: parse template strings in varfiles? + const projectVarfileVars = await loadVarfile({ + configRoot: projectConfig.path, + path: projectConfig.varfile, + defaultPath: defaultProjectVarfilePath, + }) + + return new this(`project ${projectConfig.name}`, { + context, + variablePrecedence: [projectConfig.variables, projectVarfileVars], + variableOverrides, + }) + } + + public static async forEnvironment( + environment: string, + projectConfig: ProjectConfig, + environmentConfig: EnvironmentConfig, + variableOverrides: DeepPrimitiveMap, + context: ProjectConfigContext + ) { + const envVarfileVars = await loadVarfile({ + configRoot: projectConfig.path, + path: environmentConfig.varfile, + defaultPath: defaultEnvVarfilePath(environment), + }) + + return new this(`environment ${environmentConfig.name}`, { + variablePrecedence: [environmentConfig.variables, envVarfileVars], + context, + variableOverrides, + }) + } + + /** + * Merges module variables with the following precedence order: + * + * garden.variableOverrides > module varfile > config.variables + */ + public static async forModule(garden: Garden, config: ModuleConfig, context: ModuleConfigContext) { + let varfileVars: DeepPrimitiveMap = {} + if (config.varfile) { + const varfilePath = deepEvaluate(config.varfile, { + context, + opts: {}, + }) + if (typeof varfilePath !== "string") { + throw new ConfigurationError({ + message: `Expected varfile template expression in module configuration ${config.name} to resolve to string, actually got ${typeof varfilePath}`, + }) + } + varfileVars = await loadVarfile({ + configRoot: config.path, + path: varfilePath, + defaultPath: undefined, + log: garden.log, + }) + } + + return new this(describeConfig(config), { + variablePrecedence: [config.variables, varfileVars], + variableOverrides: garden.variableOverrides, + context, + }) + } + + public static async forAction( + garden: Garden, + config: ActionConfig, + context: ActionConfigContext, + group?: GroupConfig + ) { + const effectiveConfigFileLocation = getEffectiveConfigFileLocation(config) + const actionVarfileVars = await loadVarfiles(garden, effectiveConfigFileLocation, config.varfiles || []) + const actionVariables = [config.variables, ...actionVarfileVars] + + let groupVarfileVars: ParsedTemplate[] = [] + let groupVariables: ParsedTemplate[] = [] + if (group) { + groupVarfileVars = await loadVarfiles(garden, group.path, group.varfiles || []) + groupVariables = [group.variables, ...groupVarfileVars] + } + + return new this(describeConfig(config) + (!group ? " (without group variables" : ""), { + variablePrecedence: [...groupVariables, ...actionVariables], + context, + variableOverrides: garden.variableOverrides, + }) + } + + static forCustomCommand(garden: Garden, spec: CommandResource, context: CustomCommandContext): VariablesContext { + return new this(`custom command ${spec.name}`, { + variableOverrides: garden.variableOverrides, + variablePrecedence: [spec.variables], + context, + }) + } + + /** + * Context-aware application of overrides + * + * If a context key "foo.bar" exists, and CLI option --var foo.bar=baz has been specified, + * we override { "foo.bar": "baz" }. Otherwise, we override { foo: { bar: "baz" } }. + */ + private applyOverrides(newValues: DeepPrimitiveMap, rootContext: ContextWithSchema) { + const transformedOverrides = {} + for (const key in newValues) { + const res = this.resolve({ nodePath: [], key: [key], opts: {}, rootContext }) + if (res.found) { + // If the original key itself is a string with a dot, then override that + transformedOverrides[key] = newValues[key] + } else { + // Transform override paths like "foo.bar[0].baz" + // into a nested object like + // { foo: { bar: [{ baz: "foo" }] } } + // which we can then use for the layered context as overrides on the nested structure within + setKeyPathNested(transformedOverrides, key, newValues[key]) + } + } + + this.layers.push(new GenericContext(transformedOverrides)) + this.clearCache() + } +} + +// helpers + +function makeVariableRootContext(contents: ConfigContext) { + // This makes the contents available under the keys `var` and `variables` + return new GenericContext({ + var: contents, + variables: contents, + }) +} + +const getVarfileData = (varfile: Varfile) => { + const path = typeof varfile === "string" ? varfile : varfile.path + const optional = typeof varfile === "string" ? false : varfile.optional + return { path, optional } +} + +async function loadVarfiles(garden: Garden, configRoot: string, varfiles: Varfile[]) { + // in pre-processing, only use varfiles that are not template strings + const resolvedVarFiles = varfiles.filter( + (f) => !(f instanceof UnresolvedTemplateValue) && !isUnresolved(getVarfileData(f).path) + ) + + const varsByFile = await Promise.all( + (resolvedVarFiles || []).map((varfile) => { + const { path, optional } = getVarfileData(varfile) + return loadVarfile({ + configRoot, + path, + defaultPath: undefined, + optional, + log: garden.log, + }) + }) + ) + + return varsByFile +} +function isEmpty(tpl: ParsedTemplate) { + // filter empty variable contexts for making the debugging easier + return !tpl || (isPlainObject(tpl) && Object.keys(tpl).length === 0) +} diff --git a/core/src/garden.ts b/core/src/garden.ts index 727399e714..f24754db78 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -13,7 +13,7 @@ import { platform, arch } from "os" import { relative, resolve } from "path" import cloneDeep from "fast-copy" import AsyncLock from "async-lock" -import { flatten, groupBy, keyBy, mapValues, omit, sortBy, set as setKeyPathNested } from "lodash-es" +import { flatten, groupBy, keyBy, mapValues, omit, sortBy } from "lodash-es" import { username } from "username" import { TreeCache } from "./cache.js" @@ -122,13 +122,7 @@ import type { GardenCloudApiFactory } from "./cloud/api.js" import { GardenCloudApi, CloudApiTokenRefreshError } from "./cloud/api.js" import { OutputConfigContext } from "./config/template-contexts/module.js" import { ProviderConfigContext } from "./config/template-contexts/provider.js" -import type { ConfigContext } from "./config/template-contexts/base.js" -import { - deepResolveContext, - ErrorContext, - GenericContext, - type ContextWithSchema, -} from "./config/template-contexts/base.js" +import { deepResolveContext, ErrorContext, type ContextWithSchema } from "./config/template-contexts/base.js" import { validateSchema, validateWithPath } from "./config/validation.js" import { pMemoizeDecorator } from "./lib/p-memoize.js" import { ModuleGraph } from "./graph/modules.js" @@ -185,7 +179,7 @@ import { throwOnMissingSecretKeys } from "./config/secrets.js" import { deepEvaluate } from "./template/evaluate.js" import type { ResolvedTemplate } from "./template/types.js" import { serialiseUnresolvedTemplates, type ParsedTemplate } from "./template/types.js" -import { LayeredContext } from "./config/template-contexts/base.js" +import type { VariablesContext } from "./config/template-contexts/variables.js" const defaultLocalAddress = "localhost" @@ -243,7 +237,7 @@ export interface GardenParams { projectRoot: string projectSources?: SourceConfig[] providerConfigs: UnresolvedProviderConfig[] - variables: ConfigContext + variables: VariablesContext variableOverrides: DeepPrimitiveMap secrets: StringMap sessionId: string @@ -325,7 +319,7 @@ export class Garden { * for the current environment but can be overwritten with the `--env` flag. */ public readonly namespace: string - public readonly variables: ConfigContext + public readonly variables: VariablesContext // Any variables passed via the `--var` CLI option (maintained here so that they can be used during module resolution // to override module variables and module varfiles). public readonly variableOverrides: DeepPrimitiveMap @@ -2094,10 +2088,9 @@ export const resolveGardenParams = profileAsync(async function _resolveGardenPar skipCloudConnect, }) - config = resolveProjectConfig({ - log, - defaultEnvironmentName: configDefaultEnvironment, - config, + const projectContext = new ProjectConfigContext({ + projectName, + projectRoot, artifactsPath, vcsInfo, username: _username, @@ -2107,10 +2100,21 @@ export const resolveGardenParams = profileAsync(async function _resolveGardenPar commandInfo, }) + config = resolveProjectConfig({ + log, + defaultEnvironmentName: configDefaultEnvironment, + config, + context: projectContext, + }) + + const variableOverrides = opts.variableOverrides || {} + const pickedEnv = await pickEnvironment({ projectConfig: config, + variableOverrides, envString: environmentStr, artifactsPath, + projectContext, vcsInfo, username: _username, loggedIn, @@ -2120,11 +2124,7 @@ export const resolveGardenParams = profileAsync(async function _resolveGardenPar }) const { providers, production } = pickedEnv - let { variables } = pickedEnv - - // Allow overriding variables - const variableOverrides = opts.variableOverrides || {} - variables = overrideVariables(variables, variableOverrides) + const { variables } = pickedEnv // Update the log context log.context.gardenKey = getGardenInstanceKey({ environmentName, namespace, projectRoot, variableOverrides }) @@ -2390,25 +2390,6 @@ async function getCloudProject({ } } -// Override variables, also allows to override nested variables using dot notation -export function overrideVariables(variables: ConfigContext, overrides: DeepPrimitiveMap): LayeredContext { - const transformedOverrides = {} - for (const key in overrides) { - const res = variables.resolve({ nodePath: [], key: [key], opts: {} }) - if (res.found) { - // if the original key itself is a string with a dot, then override that - transformedOverrides[key] = overrides[key] - } else { - // Transform override paths like "foo.bar[0].baz" - // into a nested object like - // { foo: { bar: [{ baz: "foo" }] } } - // which we can then use for the layered context as overrides on the nested structure within - setKeyPathNested(transformedOverrides, key, overrides[key]) - } - } - return new LayeredContext(variables, new GenericContext(transformedOverrides)) -} - /** * Dummy Garden class that doesn't scan for modules nor resolves providers. * Used by commands that have noProject=true. That is, commands that need diff --git a/core/src/graph/actions.ts b/core/src/graph/actions.ts index 752e3a13b1..d4e30a879e 100644 --- a/core/src/graph/actions.ts +++ b/core/src/graph/actions.ts @@ -35,29 +35,27 @@ import { BuildAction, buildActionConfigSchema, isBuildActionConfig } from "../ac import { DeployAction, deployActionConfigSchema, isDeployActionConfig } from "../actions/deploy.js" import { isRunActionConfig, RunAction, runActionConfigSchema } from "../actions/run.js" import { isTestActionConfig, TestAction, testActionConfigSchema } from "../actions/test.js" -import { getEffectiveConfigFileLocation, noTemplateFields } from "../config/base.js" +import { noTemplateFields } from "../config/base.js" import type { ActionReference, JoiDescription } from "../config/common.js" import { describeSchema, parseActionReference } from "../config/common.js" import type { GroupConfig } from "../config/group.js" import { ActionConfigContext } from "../config/template-contexts/actions.js" import { ConfigurationError, GardenError, PluginError } from "../exceptions.js" -import { type Garden, overrideVariables } from "../garden.js" +import { type Garden } from "../garden.js" import type { Log } from "../logger/log-entry.js" import type { ActionTypeDefinition } from "../plugin/action-types.js" import type { ActionDefinitionMap } from "../plugins.js" import { getActionTypeBases } from "../plugins.js" import type { ActionRouter } from "../router/router.js" import { ResolveActionTask } from "../tasks/resolve-action.js" -import { isUnresolved } from "../template/templated-strings.js" import { dedent, deline, naturalList } from "../util/string.js" -import { DependencyGraph, getVarfileData, mergeVariables } from "./common.js" +import { DependencyGraph } from "./common.js" import type { ConfigGraph } from "./config-graph.js" import { MutableConfigGraph } from "./config-graph.js" import type { ModuleGraph } from "./modules.js" import { isTruthy, type MaybeUndefined } from "../util/util.js" import { minimatch } from "minimatch" import type { ContextWithSchema } from "../config/template-contexts/base.js" -import { GenericContext } from "../config/template-contexts/base.js" import type { LinkedSource, LinkedSourceMap } from "../config-store/local.js" import { relative } from "path" import { profileAsync } from "../util/profiling.js" @@ -67,8 +65,9 @@ import { styles } from "../logger/styles.js" import { isUnresolvableValue } from "../template/analysis.js" import { getActionTemplateReferences } from "../config/references.js" import { deepEvaluate } from "../template/evaluate.js" -import { UnresolvedTemplateValue, type ParsedTemplate } from "../template/types.js" +import { type ParsedTemplate } from "../template/types.js" import { validateWithPath } from "../config/validation.js" +import { VariablesContext } from "../config/template-contexts/variables.js" function* sliceToBatches(dict: Record, batchSize: number) { const entries = Object.entries(dict) @@ -504,19 +503,17 @@ export const processActionConfig = profileAsync(async function processActionConf config.internal.treeVersion || (await garden.vcs.getTreeVersion({ log, projectName: garden.projectName, config, scanRoot })) - const effectiveConfigFileLocation = getEffectiveConfigFileLocation(config) - - let variables = await mergeVariables({ - basePath: effectiveConfigFileLocation, - variables: new GenericContext(config.variables || {}), - varfiles: config.varfiles, - log, + const variablesContext = new ActionConfigContext({ + garden, + config, + thisContextParams: { + mode, + name: config.name, + }, + variables: garden.variables, }) - // override the variables if there's any matching variables in variable overrides - // passed via --var cli flag. variables passed via --var cli flag have highest precedence - const variableOverrides = garden.variableOverrides || {} - variables = overrideVariables(variables, variableOverrides) + const variables = await VariablesContext.forAction(garden, config, variablesContext) const params: ActionWrapperParams = { baseBuildDirectory: garden.buildStaging.buildDirPath, @@ -729,18 +726,20 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi }): Promise { const description = describeActionConfig(config) - // in pre-processing, only use varfiles that are not template strings - const resolvedVarFiles = config.varfiles?.filter( - (f) => !(f instanceof UnresolvedTemplateValue) && isUnresolved(getVarfileData(f).path) - ) - const variables = await mergeVariables({ - basePath: config.internal.basePath, - variables: new GenericContext(config.variables || {}), - varfiles: resolvedVarFiles, - log, + // context for resolving variables (with project & environment level vars) + const variableContext = new ActionConfigContext({ + garden, + config, + thisContextParams: { + mode, + name: config.name, + }, + variables: garden.variables, }) const builtinConfigKeys = getBuiltinConfigContextKeys() + + // action context (may be missing some varfiles at this point) const builtinFieldContext = new ActionConfigContext({ garden, config, @@ -748,7 +747,7 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi mode, name: config.name, }, - variables, + variables: await VariablesContext.forAction(garden, config, variableContext), }) function resolveTemplates() { diff --git a/core/src/graph/common.ts b/core/src/graph/common.ts index 0429bce3dc..f9d523f7db 100644 --- a/core/src/graph/common.ts +++ b/core/src/graph/common.ts @@ -11,16 +11,11 @@ import { flatten, uniq } from "lodash-es" import { get, isEqual, join, set, uniqWith } from "lodash-es" import { CircularDependenciesError } from "../exceptions.js" import type { GraphNodes, ConfigGraphNode } from "./config-graph.js" -import { Profile, profileAsync } from "../util/profiling.js" +import { Profile } from "../util/profiling.js" import type { ModuleDependencyGraphNode, ModuleDependencyGraphNodeKind, ModuleGraphNodes } from "./modules.js" import type { ActionKind } from "../plugin/action-types.js" -import { loadVarfile } from "../config/base.js" -import type { Varfile } from "../config/common.js" import type { Task } from "../tasks/base.js" -import type { Log, LogMetadata, TaskLogStatus } from "../logger/log-entry.js" -import { LayeredContext } from "../config/template-contexts/base.js" -import type { ConfigContext } from "../config/template-contexts/base.js" -import { GenericContext } from "../config/template-contexts/base.js" +import type { LogMetadata, TaskLogStatus } from "../logger/log-entry.js" // Shared type used by ConfigGraph and TaskGraph to facilitate circular dependency detection export type DependencyGraphNode = { @@ -143,43 +138,6 @@ interface CycleGraph { } } -export const getVarfileData = (varfile: Varfile) => { - const path = typeof varfile === "string" ? varfile : varfile.path - const optional = typeof varfile === "string" ? false : varfile.optional - return { path, optional } -} - -export const mergeVariables = profileAsync(async function mergeVariables({ - basePath, - variables, - varfiles, - log, -}: { - basePath: string - variables?: ConfigContext - varfiles?: Varfile[] - log: Log -}) { - const varsByFile = await Promise.all( - (varfiles || []).map((varfile) => { - const { path, optional } = getVarfileData(varfile) - return loadVarfile({ - configRoot: basePath, - path, - defaultPath: undefined, - optional, - log, - }) - }) - ) - - return new LayeredContext( - variables || new GenericContext({}), - // Merge different varfiles, later files taking precedence over prior files in the list. - ...varsByFile.map((vars) => new GenericContext(vars)) - ) -}) - /** * Implements a variation on the Floyd-Warshall algorithm to compute minimal cycles. * diff --git a/core/src/resolve-module.ts b/core/src/resolve-module.ts index 0d6ecc5fcb..e4a9e5d463 100644 --- a/core/src/resolve-module.ts +++ b/core/src/resolve-module.ts @@ -9,7 +9,6 @@ import { isArray, isString, keyBy, partition, uniq } from "lodash-es" import { validateWithPath } from "./config/validation.js" import { isUnresolved, resolveTemplateString } from "./template/templated-strings.js" -import { GenericContext, LayeredContext } from "./config/template-contexts/base.js" import { dirname, posix, relative, resolve } from "path" import type { Garden } from "./garden.js" import type { GardenError } from "./exceptions.js" @@ -37,7 +36,7 @@ import type { Log } from "./logger/log-entry.js" import type { ModuleConfigContextParams } from "./config/template-contexts/module.js" import { ModuleConfigContext } from "./config/template-contexts/module.js" import { pathToCacheContext } from "./cache.js" -import { loadVarfile, prepareBuildDependencies } from "./config/base.js" +import { prepareBuildDependencies } from "./config/base.js" import type { ModuleTypeDefinition } from "./plugin/plugin.js" import { serviceFromConfig } from "./types/service.js" import { taskFromConfig } from "./types/task.js" @@ -56,12 +55,12 @@ import type { DepGraph } from "dependency-graph" import { minimatch } from "minimatch" import { getModuleTemplateReferences } from "./config/references.js" import { type ParsedTemplate, UnresolvedTemplateValue } from "./template/types.js" -import { conditionallyDeepEvaluate, deepEvaluate, evaluate } from "./template/evaluate.js" +import { conditionallyDeepEvaluate, evaluate } from "./template/evaluate.js" import { someReferences } from "./template/analysis.js" import { ForEachLazyValue, parseTemplateCollection } from "./template/templated-collections.js" import { deepMap, isPlainObject } from "./util/objects.js" -import { LazyMergePatch } from "./template/lazy-merge.js" import { InputContext } from "./config/template-contexts/input.js" +import { VariablesContext } from "./config/template-contexts/variables.js" const { mkdirp, readFile } = fsExtra @@ -122,7 +121,7 @@ export class ModuleResolver { const minimalRoots = await this.garden.vcs.getMinimalRoots(this.log, allPaths) - const resolvedConfigs: ModuleConfigMap = {} + const resolveResults: Record = {} const resolvedModules: ModuleMap = {} const errors: { [moduleName: string]: GardenError } = {} @@ -137,21 +136,21 @@ export class ModuleResolver { inFlight.add(moduleKey) // Resolve configuration, unless previously resolved. - let resolvedConfig = resolvedConfigs[moduleKey] + let resolveResult = resolveResults[moduleKey] let foundNewDependency = false const dependencyNames = fullGraph.dependenciesOf(moduleKey) const resolvedDependencies = dependencyNames.map((n) => resolvedModules[n]).filter(Boolean) try { - if (!resolvedConfig) { + if (!resolveResult) { const rawConfig = this.rawConfigsByKey[moduleKey] this.log.silly(() => `ModuleResolver: Resolve config ${moduleKey}`) - resolvedConfig = resolvedConfigs[moduleKey] = await this.resolveModuleConfig(rawConfig, resolvedDependencies) + resolveResult = resolveResults[moduleKey] = await this.resolveModuleConfig(rawConfig, resolvedDependencies) // Check if any new build dependencies were added by the configure handler - for (const dep of resolvedConfig.build.dependencies) { + for (const dep of resolveResult.config.build.dependencies) { const depKey = dep.name if (!dependencyNames.includes(depKey)) { @@ -181,15 +180,16 @@ export class ModuleResolver { // dependencies. if (!foundNewDependency) { const shouldResolve = - forceResolve || this.shouldResolveInline({ config: resolvedConfig, actionsFilter, fullGraph }) + forceResolve || this.shouldResolveInline({ config: resolveResult.config, actionsFilter, fullGraph }) if (shouldResolve) { - const buildPath = this.garden.buildStaging.getBuildPath(resolvedConfig) + const buildPath = this.garden.buildStaging.getBuildPath(resolveResult.config) resolvedModules[moduleKey] = await this.resolveModule({ - resolvedConfig, + resolvedConfig: resolveResult.config, + variables: resolveResult.context.variables, buildPath, dependencies: resolvedDependencies, - repoRoot: minimalRoots[resolvedConfig.path], + repoRoot: minimalRoots[resolveResult.config.path], }) } else { this.log.debug(() => `ModuleResolver: Module ${moduleKey} skipped`) @@ -208,6 +208,11 @@ export class ModuleResolver { const processLeaves = async (forceResolve: boolean) => { if (Object.keys(errors).length > 0) { + for (const err of Object.values(errors)) { + if (err instanceof InternalError) { + throw err + } + } const errorStr = Object.entries(errors) .map(([name, err]) => `${styles.highlight.bold(name)}: ${err.message}`) .join("\n") @@ -272,7 +277,7 @@ export class ModuleResolver { const taskNames = new Set() // Add runtime dependencies to the module dependency graph - for (const config of Object.values(resolvedConfigs)) { + for (const { config } of Object.values(resolveResults)) { for (const service of config.serviceConfigs) { const key = `deploy.${service.name}` runtimeGraph.addNode(key) @@ -301,7 +306,7 @@ export class ModuleResolver { } } - for (const config of Object.values(resolvedConfigs)) { + for (const { config } of Object.values(resolveResults)) { for (const service of config.serviceConfigs) { const key = `deploy.${service.name}` for (const dep of service.dependencies || []) { @@ -330,8 +335,8 @@ export class ModuleResolver { // Note: Module names in the graph don't have the build. prefix const moduleDepNames = deps.filter((d) => !d.includes(".")) for (const name of moduleDepNames) { - if (!resolvedModules[name] && resolvedConfigs[name]) { - needResolve[name] = resolvedConfigs[name] + if (!resolvedModules[name] && resolveResults[name]) { + needResolve[name] = resolveResults[name].config } } } @@ -348,7 +353,7 @@ export class ModuleResolver { const skipped = new Set() if (actionsFilter && mayNeedAdditionalResolution) { - for (const config of Object.values(resolvedConfigs)) { + for (const { config } of Object.values(resolveResults)) { if (!resolvedModules[config.name]) { skipped.add(`build.${config.name}`) for (const s of config.serviceConfigs) { @@ -396,7 +401,7 @@ export class ModuleResolver { delete config["_templateDeps"] } - return { skipped, resolvedModules: Object.values(resolvedModules), resolvedConfigs: Object.values(resolvedConfigs) } + return { skipped, resolvedModules: Object.values(resolvedModules), resolvedConfigs: Object.values(resolveResults) } } private addModulesToGraph(graph: DepGraph, configs: ModuleConfig[]) { @@ -570,7 +575,10 @@ export class ModuleResolver { /** * Resolves and validates a single module configuration. */ - async resolveModuleConfig(unresolvedConfig: ModuleConfig, dependencies: GardenModule[]): Promise { + async resolveModuleConfig( + unresolvedConfig: ModuleConfig, + dependencies: GardenModule[] + ): Promise<{ config: ModuleConfig; context: ModuleConfigContext }> { const garden = this.garden const buildPath = this.garden.buildStaging.getBuildPath(unresolvedConfig) @@ -591,15 +599,9 @@ export class ModuleResolver { const contextWithoutVariables = new ModuleConfigContext(templateContextParams) - // Resolve the variables field before resolving everything else (overriding with module varfiles if present) - const moduleVariables = await this.mergeVariables(unresolvedConfig, contextWithoutVariables) - - // And finally fully resolve the config. - // Template strings in the spec can have references to inputs, - // so we also need to pass inputs here along with the available variables. const context = new ModuleConfigContext({ ...templateContextParams, - variables: new LayeredContext(garden.variables, new GenericContext(moduleVariables)), + variables: await VariablesContext.forModule(this.garden, unresolvedConfig, contextWithoutVariables), }) const moduleTypeDefinitions = await garden.getModuleTypes() @@ -620,10 +622,7 @@ export class ModuleResolver { } let config: ModuleConfig = partiallyEvaluateModule( - { - ...unresolvedConfig, - variables: moduleVariables, - } as unknown as ParsedTemplate, + unresolvedConfig as unknown as ParsedTemplate, context ) as unknown as ModuleConfig @@ -724,7 +723,7 @@ export class ModuleResolver { delete config["_templateDeps"] - return config + return { config, context } } /** @@ -742,11 +741,13 @@ export class ModuleResolver { private async resolveModule({ resolvedConfig, + variables, buildPath, dependencies, repoRoot, }: { resolvedConfig: ModuleConfig + variables: VariablesContext buildPath: string dependencies: GardenModule[] repoRoot: string @@ -757,7 +758,7 @@ export class ModuleResolver { const configContext = new ModuleConfigContext({ garden: this.garden, resolvedProviders: this.resolvedProviders, - variables: new LayeredContext(this.garden.variables, new GenericContext(resolvedConfig.variables || {})), + variables, name: resolvedConfig.name, path: resolvedConfig.path, buildPath, @@ -881,35 +882,6 @@ export class ModuleResolver { return module } - - /** - * Merges module variables with the following precedence order: - * - * garden.variableOverrides > module varfile > config.variables - */ - private async mergeVariables(config: ModuleConfig, context: ModuleConfigContext): Promise { - let varfileVars: DeepPrimitiveMap = {} - if (config.varfile) { - const varfilePath = deepEvaluate(config.varfile, { - context, - opts: {}, - }) - if (typeof varfilePath !== "string") { - throw new ConfigurationError({ - message: `Expected varfile template expression in module configuration ${config.name} to resolve to string, actually got ${typeof varfilePath}`, - }) - } - varfileVars = await loadVarfile({ - configRoot: config.path, - path: varfilePath, - defaultPath: undefined, - }) - } - - const moduleVariables = config.variables || {} - - return new LazyMergePatch([moduleVariables, varfileVars, this.garden.variableOverrides]) - } } export interface ConvertModulesResult { diff --git a/core/src/router/base.ts b/core/src/router/base.ts index 0a729e57a4..ec346f83f7 100644 --- a/core/src/router/base.ts +++ b/core/src/router/base.ts @@ -299,6 +299,7 @@ export abstract class BaseActionRouter extends BaseRouter modules: graph.getModules(), resolvedDependencies: action.getResolvedDependencies(), executedDependencies: action.getExecutedDependencies(), + // inputs are fully resolved now inputs: new InputContext(action.getInternal().inputs), variables: action.getVariablesContext(), }) diff --git a/core/src/tasks/resolve-action.ts b/core/src/tasks/resolve-action.ts index 283bc7ae3c..68e8222b22 100644 --- a/core/src/tasks/resolve-action.ts +++ b/core/src/tasks/resolve-action.ts @@ -20,18 +20,17 @@ import type { import { ActionSpecContext } from "../config/template-contexts/actions.js" import { InternalError } from "../exceptions.js" import { validateWithPath } from "../config/validation.js" -import { mergeVariables } from "../graph/common.js" import { actionToResolved } from "../actions/helpers.js" import { ResolvedConfigGraph } from "../graph/config-graph.js" import { OtelTraced } from "../util/open-telemetry/decorators.js" import { deepEvaluate } from "../template/evaluate.js" -import type { ConfigContext } from "../config/template-contexts/base.js" -import { deepResolveContext, GenericContext } from "../config/template-contexts/base.js" -import { LayeredContext } from "../config/template-contexts/base.js" +import { deepResolveContext } from "../config/template-contexts/base.js" import { isPlainObject } from "../util/objects.js" import type { DeepPrimitiveMap } from "@garden-io/platform-api-types" import { describeActionConfig } from "../actions/base.js" import { InputContext } from "../config/template-contexts/input.js" +import { VariablesContext } from "../config/template-contexts/variables.js" +import type { GroupConfig } from "../config/group.js" export interface ResolveActionResults extends ValidResultType { state: ActionState @@ -133,8 +132,8 @@ export class ResolveActionTask extends BaseActionTask extends BaseActionTask extends BaseActionTask { exclude: { leftValue: ["file1"], rightValue: ["file2"] }, include: { leftValue: ["file1"], rightValue: ["file2"] }, internal: { leftValue: { basePath: "./base1" }, rightValue: { basePath: "./base2" } }, + variables: { leftValue: { foo: "bar" }, rightValue: { bar: "baz" } }, + varfiles: { leftValue: ["foo.yml"], rightValue: ["bar.yml"] }, source: { leftValue: { path: "path1" }, rightValue: { path: "path2" } }, } diff --git a/core/test/unit/src/commands/workflow.ts b/core/test/unit/src/commands/workflow.ts index b7af2cc64f..1ab861b6f0 100644 --- a/core/test/unit/src/commands/workflow.ts +++ b/core/test/unit/src/commands/workflow.ts @@ -34,8 +34,8 @@ import type { WorkflowConfig, WorkflowStepSpec } from "../../../../src/config/wo import { defaultWorkflowResources } from "../../../../src/config/workflow.js" import { TestGardenCli } from "../../../helpers/cli.js" import { WorkflowScriptError } from "../../../../src/exceptions.js" -import { GenericContext } from "../../../../src/config/template-contexts/base.js" import { parseTemplateCollection } from "../../../../src/template/templated-collections.js" +import { VariablesContext } from "../../../../src/config/template-contexts/variables.js" describe("RunWorkflowCommand", () => { const cmd = new WorkflowCommand() @@ -84,7 +84,7 @@ describe("RunWorkflowCommand", () => { garden.setWorkflowConfigs(parsedWorkflowConfigs) - garden.variables = new GenericContext({ foo: null }) + garden.variables = VariablesContext.forTest(garden, { foo: null }) const result = await cmd.action({ ...defaultParams, args: { workflow: "workflow-a" } }) diff --git a/core/test/unit/src/config/project.ts b/core/test/unit/src/config/project.ts index 53f1f01941..ce3e1d9ccd 100644 --- a/core/test/unit/src/config/project.ts +++ b/core/test/unit/src/config/project.ts @@ -13,7 +13,7 @@ import type { ProjectConfig } from "../../../../src/config/project.js" import { resolveProjectConfig, pickEnvironment, - defaultVarfilePath, + defaultProjectVarfilePath, defaultEnvVarfilePath, parseEnvironment, defaultNamespace, @@ -30,6 +30,7 @@ import { deepResolveContext, GenericContext } from "../../../../src/config/templ import { omit } from "lodash-es" import { serialiseUnresolvedTemplates } from "../../../../src/template/types.js" import type { DeepPrimitiveMap } from "@garden-io/platform-api-types" +import { ProjectConfigContext } from "../../../../src/config/template-contexts/project.js" const { realpath, writeFile } = fsExtra @@ -58,13 +59,17 @@ describe("resolveProjectConfig", () => { log, defaultEnvironmentName: "default", config, - artifactsPath: "/tmp", - vcsInfo, - username: "some-user", - loggedIn: true, - enterpriseDomain, - secrets: {}, - commandInfo, + context: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath: "/tmp", + vcsInfo, + username: "some-user", + loggedIn: true, + enterpriseDomain, + secrets: {}, + commandInfo, + }), }) ).to.eql({ ...config, @@ -77,7 +82,7 @@ describe("resolveProjectConfig", () => { }, ], sources: [], - varfile: defaultVarfilePath, + varfile: defaultProjectVarfilePath, }) }) @@ -120,13 +125,17 @@ describe("resolveProjectConfig", () => { log, defaultEnvironmentName: defaultEnvironment, config, - artifactsPath: "/tmp", - vcsInfo, - username: "some-user", - loggedIn: true, - enterpriseDomain, - secrets: { foo: "banana" }, - commandInfo, + context: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath: "/tmp", + vcsInfo, + username: "some-user", + loggedIn: true, + enterpriseDomain, + secrets: {}, + commandInfo, + }), }) ) ).to.eql({ @@ -149,7 +158,7 @@ describe("resolveProjectConfig", () => { repositoryUrl, }, ], - varfile: defaultVarfilePath, + varfile: defaultProjectVarfilePath, variables: { platform: platform(), secret: "banana", @@ -196,13 +205,17 @@ describe("resolveProjectConfig", () => { log, defaultEnvironmentName: defaultEnvironment, config, - artifactsPath: "/tmp", - vcsInfo, - username: "some-user", - loggedIn: true, - enterpriseDomain, - secrets: {}, - commandInfo, + context: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath: "/tmp", + vcsInfo, + username: "some-user", + loggedIn: true, + enterpriseDomain, + secrets: {}, + commandInfo, + }), }) ) ).to.eql({ @@ -230,7 +243,7 @@ describe("resolveProjectConfig", () => { }, ], sources: [], - varfile: defaultVarfilePath, + varfile: defaultProjectVarfilePath, }) delete process.env.TEST_ENV_VAR_A @@ -256,13 +269,17 @@ describe("resolveProjectConfig", () => { log, defaultEnvironmentName: defaultEnvironment, config, - artifactsPath: "/tmp", - vcsInfo, - username: "some-user", - loggedIn: true, - enterpriseDomain, - secrets: {}, - commandInfo, + context: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath: "/tmp", + vcsInfo, + username: "some-user", + loggedIn: true, + enterpriseDomain, + secrets: {}, + commandInfo, + }), }) expect(result.environments[0].variables).to.eql(config.environments[0].variables) @@ -290,13 +307,17 @@ describe("resolveProjectConfig", () => { log, defaultEnvironmentName: defaultEnvironment, config, - artifactsPath: "/tmp", - vcsInfo, - username: "some-user", - loggedIn: true, - enterpriseDomain, - secrets: {}, - commandInfo, + context: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath: "/tmp", + vcsInfo, + username: "some-user", + loggedIn: true, + enterpriseDomain, + secrets: {}, + commandInfo, + }), }) ) ).to.eql({ @@ -316,7 +337,7 @@ describe("resolveProjectConfig", () => { repositoryUrl, }, ], - varfile: defaultVarfilePath, + varfile: defaultProjectVarfilePath, variables: {}, }) @@ -340,13 +361,17 @@ describe("resolveProjectConfig", () => { log, defaultEnvironmentName, config, - artifactsPath: "/tmp", - vcsInfo, - username: "some-user", - loggedIn: true, - enterpriseDomain, - secrets: {}, - commandInfo, + context: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath: "/tmp", + vcsInfo, + username: "some-user", + loggedIn: true, + enterpriseDomain, + secrets: {}, + commandInfo, + }), }) ).to.eql({ ...config, @@ -354,7 +379,7 @@ describe("resolveProjectConfig", () => { defaultEnvironment: "first-env", environments: [{ defaultNamespace: null, name: "first-env", variables: {} }], sources: [], - varfile: defaultVarfilePath, + varfile: defaultProjectVarfilePath, }) }) @@ -375,13 +400,17 @@ describe("resolveProjectConfig", () => { log, defaultEnvironmentName, config, - artifactsPath: "/tmp", - vcsInfo, - username: "some-user", - loggedIn: true, - enterpriseDomain, - secrets: {}, - commandInfo, + context: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath: "/tmp", + vcsInfo, + username: "some-user", + loggedIn: true, + enterpriseDomain, + secrets: {}, + commandInfo, + }), }) ).to.eql({ ...config, @@ -389,7 +418,7 @@ describe("resolveProjectConfig", () => { defaultEnvironment: "default", environments: [{ defaultNamespace: null, name: "default", variables: {} }], sources: [], - varfile: defaultVarfilePath, + varfile: defaultProjectVarfilePath, }) }) @@ -426,13 +455,17 @@ describe("resolveProjectConfig", () => { log, defaultEnvironmentName: defaultEnvironment, config, - artifactsPath: "/tmp", - vcsInfo, - username: "some-user", - loggedIn: true, - enterpriseDomain, - secrets: {}, - commandInfo, + context: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath: "/tmp", + vcsInfo, + username: "some-user", + loggedIn: true, + enterpriseDomain, + secrets: {}, + commandInfo, + }), }) expect(resolvedConfig).to.eql({ ...config, @@ -463,7 +496,7 @@ describe("resolveProjectConfig", () => { }, ], sources: [], - varfile: defaultVarfilePath, + varfile: defaultProjectVarfilePath, }) }) }) @@ -493,7 +526,19 @@ describe("pickEnvironment", () => { await expectError( () => pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), projectConfig: config, + variableOverrides: {}, envString: "foo", artifactsPath, vcsInfo, @@ -514,7 +559,19 @@ describe("pickEnvironment", () => { }) const res = await pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), projectConfig: config, + variableOverrides: {}, envString: "default", artifactsPath, vcsInfo, @@ -545,7 +602,20 @@ describe("pickEnvironment", () => { }) const env = await pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), + projectConfig: config, + variableOverrides: {}, envString: "default", artifactsPath, vcsInfo, @@ -612,7 +682,20 @@ describe("pickEnvironment", () => { }) const result = await pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), + projectConfig: config, + variableOverrides: {}, envString: "default", artifactsPath, vcsInfo, @@ -638,7 +721,7 @@ describe("pickEnvironment", () => { }) it("should load variables from default project varfile if it exists", async () => { - const varfilePath = resolve(tmpPath, defaultVarfilePath) + const varfilePath = resolve(tmpPath, defaultProjectVarfilePath) await writeFile( varfilePath, dedent` @@ -663,7 +746,20 @@ describe("pickEnvironment", () => { }) const result = await pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), + projectConfig: config, + variableOverrides: {}, envString: "default", artifactsPath, vcsInfo, @@ -709,7 +805,20 @@ describe("pickEnvironment", () => { }) const result = await pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), + projectConfig: config, + variableOverrides: {}, envString: "default", artifactsPath, vcsInfo, @@ -755,7 +864,20 @@ describe("pickEnvironment", () => { }) const result = await pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), + projectConfig: config, + variableOverrides: {}, envString: "default", artifactsPath, vcsInfo, @@ -802,7 +924,20 @@ describe("pickEnvironment", () => { }) const result = await pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), + projectConfig: config, + variableOverrides: {}, envString: "default", artifactsPath, vcsInfo, @@ -859,7 +994,20 @@ describe("pickEnvironment", () => { }) const result = await pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), + projectConfig: config, + variableOverrides: {}, envString: "default", artifactsPath, vcsInfo, @@ -917,7 +1065,20 @@ describe("pickEnvironment", () => { }) const result = await pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), + projectConfig: config, + variableOverrides: {}, envString: "default", artifactsPath, vcsInfo, @@ -947,7 +1108,20 @@ describe("pickEnvironment", () => { }) const result = await pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), + projectConfig: config, + variableOverrides: {}, envString: "default", artifactsPath, vcsInfo, @@ -976,7 +1150,20 @@ describe("pickEnvironment", () => { }) await pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), + projectConfig: config, + variableOverrides: {}, envString: "default", artifactsPath, vcsInfo, @@ -997,7 +1184,20 @@ describe("pickEnvironment", () => { }) const result = await pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), + projectConfig: config, + variableOverrides: {}, envString: "default", artifactsPath, vcsInfo, @@ -1026,7 +1226,7 @@ describe("pickEnvironment", () => { // Precedence 3/4 await writeFile( - resolve(tmpPath, defaultVarfilePath), + resolve(tmpPath, defaultProjectVarfilePath), dedent` b=B c=c @@ -1055,7 +1255,20 @@ describe("pickEnvironment", () => { }) const result = await pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), + projectConfig: config, + variableOverrides: {}, envString: "default", artifactsPath, vcsInfo, @@ -1095,7 +1308,20 @@ describe("pickEnvironment", () => { await expectError( () => pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), + projectConfig: config, + variableOverrides: {}, envString: "default", artifactsPath, vcsInfo, @@ -1126,7 +1352,20 @@ describe("pickEnvironment", () => { await expectError( () => pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), + projectConfig: config, + variableOverrides: {}, envString: "default", artifactsPath, vcsInfo, @@ -1157,7 +1396,20 @@ describe("pickEnvironment", () => { await expectError( () => pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), + projectConfig: config, + variableOverrides: {}, envString: "default", artifactsPath, vcsInfo, @@ -1178,7 +1430,20 @@ describe("pickEnvironment", () => { }) const res = await pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), + projectConfig: config, + variableOverrides: {}, envString: "foo.default", artifactsPath, vcsInfo, @@ -1200,7 +1465,20 @@ describe("pickEnvironment", () => { }) const res = await pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), + projectConfig: config, + variableOverrides: {}, envString: "foo.default", artifactsPath, vcsInfo, @@ -1222,7 +1500,20 @@ describe("pickEnvironment", () => { }) const res = await pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), + projectConfig: config, + variableOverrides: {}, envString: "default", artifactsPath, username, @@ -1246,7 +1537,20 @@ describe("pickEnvironment", () => { await expectError( () => pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), + projectConfig: config, + variableOverrides: {}, envString: "$.%", artifactsPath, vcsInfo, @@ -1270,7 +1574,19 @@ describe("pickEnvironment", () => { await expectError( () => pickEnvironment({ + projectContext: new ProjectConfigContext({ + projectName: config.name, + projectRoot: config.path, + artifactsPath, + vcsInfo, + username, + loggedIn: true, + enterpriseDomain: config.domain, + secrets: {}, + commandInfo, + }), projectConfig: config, + variableOverrides: {}, envString: "default", artifactsPath, vcsInfo, diff --git a/core/test/unit/src/config/render-template.ts b/core/test/unit/src/config/render-template.ts index c842fe7a31..4441816f99 100644 --- a/core/test/unit/src/config/render-template.ts +++ b/core/test/unit/src/config/render-template.ts @@ -17,7 +17,6 @@ import { joi } from "../../../../src/config/common.js" import fsExtra from "fs-extra" const { pathExists, remove } = fsExtra import cloneDeep from "fast-copy" - import { configTemplateKind, renderTemplateKind } from "../../../../src/config/base.js" import type { RenderTemplateConfig } from "../../../../src/config/render-template.js" import { renderConfigTemplate } from "../../../../src/config/render-template.js" @@ -25,7 +24,7 @@ import type { Log } from "../../../../src/logger/log-entry.js" import { parseTemplateCollection } from "../../../../src/template/templated-collections.js" import { serialiseUnresolvedTemplates, UnresolvedTemplateValue } from "../../../../src/template/types.js" import { deepEvaluate } from "../../../../src/template/evaluate.js" -import { GenericContext } from "../../../../src/config/template-contexts/base.js" +import { VariablesContext } from "../../../../src/config/template-contexts/variables.js" describe("config templates", () => { let garden: TestGarden @@ -459,7 +458,7 @@ describe("config templates", () => { const config: RenderTemplateConfig = cloneDeep(defaults) config.inputs = parseTemplateCollection({ value: { name: "${var.test}" }, source: { path: [] } }) - garden.variables = new GenericContext({ + garden.variables = VariablesContext.forTest(garden, { test: "test-value", }) diff --git a/core/test/unit/src/config/workflow.ts b/core/test/unit/src/config/workflow.ts index c8e59c9293..811f107041 100644 --- a/core/test/unit/src/config/workflow.ts +++ b/core/test/unit/src/config/workflow.ts @@ -23,9 +23,9 @@ import { defaultNamespace } from "../../../../src/config/project.js" import { join } from "path" import { GardenApiVersion } from "../../../../src/constants.js" import { omit } from "lodash-es" -import { GenericContext } from "../../../../src/config/template-contexts/base.js" import { parseTemplateCollection } from "../../../../src/template/templated-collections.js" import { serialiseUnresolvedTemplates } from "../../../../src/template/types.js" +import { VariablesContext } from "../../../../src/config/template-contexts/variables.js" describe("resolveWorkflowConfig", () => { let garden: TestGarden @@ -50,8 +50,8 @@ describe("resolveWorkflowConfig", () => { before(async () => { garden = await makeTestGardenA() - garden["secrets"] = { foo: "bar", bar: "baz", baz: "banana" } - garden["variables"] = new GenericContext({ foo: "baz", skip: false }) + garden.secrets = { foo: "bar", bar: "baz", baz: "banana" } + garden.variables = VariablesContext.forTest(garden, { foo: "baz", skip: false }) }) it("should pass through a canonical workflow config", async () => { diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index ff6e104ff3..2b089c94d9 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -81,7 +81,8 @@ import { getCloudDistributionName } from "../../../src/cloud/util.js" import { resolveAction } from "../../../src/graph/actions.js" import { serialiseUnresolvedTemplates } from "../../../src/template/types.js" import { parseTemplateCollection } from "../../../src/template/templated-collections.js" -import { deepResolveContext, GenericContext, LayeredContext } from "../../../src/config/template-contexts/base.js" +import { deepResolveContext, LayeredContext } from "../../../src/config/template-contexts/base.js" +import { VariablesContext } from "../../../src/config/template-contexts/variables.js" const { realpath, writeFile, readFile, remove, pathExists, mkdirp, copy } = fsExtra @@ -2926,7 +2927,7 @@ describe("Garden", () => { await exec("git", ["add", "."], { cwd: repoPath }) await exec("git", ["commit", "-m", "foo"], { cwd: repoPath }) - garden.variables = new GenericContext({ sourceBranch: "main" }) + garden.variables = VariablesContext.forTest(garden, { sourceBranch: "main" }) // eslint-disable-next-line @typescript-eslint/no-explicit-any const _garden = garden as any @@ -5370,9 +5371,9 @@ describe("Garden", () => { }) context("test against fixed version hashes", async () => { - const moduleAVersionString = "v-55de0aac5c" - const moduleBVersionString = "v-daeabf68fe" - const moduleCVersionString = "v-5e9ddea45e" + const moduleAVersionString = "v-0caa1284cd" + const moduleBVersionString = "v-18fd5b86c0" + const moduleCVersionString = "v-c8700eabbf" it("should return the same module versions between runtimes", async () => { const projectRoot = getDataDir("test-projects", "fixed-version-hashes-1") @@ -5384,9 +5385,18 @@ describe("Garden", () => { const moduleA = graph.getModule("module-a") const moduleB = graph.getModule("module-b") const moduleC = graph.getModule("module-c") - expect(moduleA.version.versionString).to.equal(moduleAVersionString) - expect(moduleB.version.versionString).to.equal(moduleBVersionString) - expect(moduleC.version.versionString).to.equal(moduleCVersionString) + expect(moduleA.version.versionString).to.equal( + moduleAVersionString, + "Code changes have affected module version calculation of module-a." + ) + expect(moduleB.version.versionString).to.equal( + moduleBVersionString, + "Code changes have affected module version calculation of module-b." + ) + expect(moduleC.version.versionString).to.equal( + moduleCVersionString, + "Code changes have affected module version calculation of module-c." + ) delete process.env.TEST_ENV_VAR }) @@ -5401,9 +5411,18 @@ describe("Garden", () => { const moduleA = graph.getModule("module-a") const moduleB = graph.getModule("module-b") const moduleC = graph.getModule("module-c") - expect(moduleA.version.versionString).to.equal(moduleAVersionString) - expect(moduleB.version.versionString).to.equal(moduleBVersionString) - expect(moduleC.version.versionString).to.equal(moduleCVersionString) + expect(moduleA.version.versionString).to.equal( + moduleAVersionString, + "Code changes have affected module version calculation of module-a in different projects." + ) + expect(moduleB.version.versionString).to.equal( + moduleBVersionString, + "Code changes have affected module version calculation of module-b in different projects." + ) + expect(moduleC.version.versionString).to.equal( + moduleCVersionString, + "Code changes have affected module version calculation of module-c in different projects." + ) delete process.env.MODULE_A_TEST_ENV_VAR }) diff --git a/core/test/unit/src/resolve-module.ts b/core/test/unit/src/resolve-module.ts index dc4dfd67f3..17efd2519a 100644 --- a/core/test/unit/src/resolve-module.ts +++ b/core/test/unit/src/resolve-module.ts @@ -53,7 +53,7 @@ describe("ModuleResolver", () => { expect(module.build.dependencies[0].name).to.equal("test-project-a") }) - it("sets the relevant variables in module config from variable overrides", async () => { + it("variable overrides should only affect template evaluation, and not alter the module config itself", async () => { const garden = await makeTestGardenA([], { variableOverrides: { foo: "override", @@ -67,6 +67,9 @@ describe("ModuleResolver", () => { type: "test", path: join(garden.projectRoot, "module-a"), build: { dependencies: [], timeout: DEFAULT_BUILD_TIMEOUT_SEC }, + spec: { + command: ["echo", "${var.foo}"], + }, variables: { foo: "somevalue", }, @@ -75,8 +78,12 @@ describe("ModuleResolver", () => { const module = await garden.resolveModule("test-project-a") expect(module.variables).to.eql({ - bar: "no-override", // irrelevant overrides will appear in the variables of every module, but that doesn't hurt. - foo: "override", + // the variables section of the module config should not change + foo: "somevalue", + }) + expect(module.spec).to.eql({ + // --> ${var.foo} should evaluate to "override" + command: ["echo", "override"], }) }) diff --git a/core/test/unit/src/vcs/vcs.ts b/core/test/unit/src/vcs/vcs.ts index a3c35a5a5d..6bf466c152 100644 --- a/core/test/unit/src/vcs/vcs.ts +++ b/core/test/unit/src/vcs/vcs.ts @@ -39,7 +39,7 @@ import { defaultDotIgnoreFile, fixedProjectExcludes } from "../../../../src/util import type { BaseActionConfig } from "../../../../src/actions/types.js" import { TreeCache } from "../../../../src/cache.js" import { getHashedFilterParams } from "../../../../src/vcs/git-repo.js" -import { GenericContext } from "../../../../src/config/template-contexts/base.js" +import { VariablesContext } from "../../../../src/config/template-contexts/variables.js" const { readFile, writeFile } = fsExtra @@ -274,7 +274,7 @@ describe("getModuleVersionString", () => { templateGarden["cacheKey"] = "" // Disable caching of the config graph const before = await templateGarden.resolveModule("module-a") - templateGarden.variables = new GenericContext({ "echo-string": "something-else" }) + templateGarden.variables = VariablesContext.forTest(templateGarden, { "echo-string": "something-else" }) const after = await templateGarden.resolveModule("module-a") @@ -326,7 +326,7 @@ describe("getModuleVersionString", () => { const garden = await makeTestGarden(projectRoot, { noCache: true }) const module = await garden.resolveModule("module-a") - const fixedVersionString = "v-55de0aac5c" + const fixedVersionString = "v-0caa1284cd" expect(module.version.versionString).to.eql(fixedVersionString) delete process.env.TEST_ENV_VAR From e549151de9d4769ab09ead8696fd75ef6b8cf041 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 24 Jan 2025 18:06:49 +0100 Subject: [PATCH 115/117] fix: do not validate BaseAction instances in joi augmentGraph schema Otherwise joi --- core/src/plugin/handlers/Provider/augmentGraph.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/plugin/handlers/Provider/augmentGraph.ts b/core/src/plugin/handlers/Provider/augmentGraph.ts index 4e279b27f2..08a029e62c 100644 --- a/core/src/plugin/handlers/Provider/augmentGraph.ts +++ b/core/src/plugin/handlers/Provider/augmentGraph.ts @@ -47,9 +47,9 @@ export const augmentGraph = () => ({ and avoid any external I/O. `, paramsSchema: projectActionParamsSchema().keys({ - // allow unknown because BaseAction-s are passed not BaseActionConfigs - // FIXME: consider fixing this by passing the values of a correct type - actions: joiArray(baseActionConfigSchema().unknown(true)).description( + // allow any because BaseAction-s are passed not BaseActionConfigs + // we do not want joi to validate BaseAction + actions: joiArray(joi.any()).description( dedent` A list of all previously defined actions in the project, including all actions added by any \`augmentGraph\` handlers defined by other providers that this provider depends on. From 26116b7575ea1ec2a10889f18a65d770178c51da Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 24 Jan 2025 19:54:24 +0100 Subject: [PATCH 116/117] fix: additional test fixes --- core/src/config/template-contexts/base.ts | 6 +++++- core/src/template/types.ts | 7 +++++++ core/src/util/logging.ts | 5 +++++ core/test/unit/src/resolve-module.ts | 10 ++++++---- core/test/unit/src/server/server.ts | 4 ++++ 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index cfd6a733f1..e76f9fa535 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -106,11 +106,15 @@ export abstract class ConfigContext { private readonly _cache: Map private readonly _id: number - constructor() { + constructor(private readonly _description?: string) { this._id = globalConfigContextCounter++ this._cache = new Map() } + public toSanitizedValue() { + return `<${this.constructor.name}(${this._description})>` + } + protected clearCache() { this._cache.clear() } diff --git a/core/src/template/types.ts b/core/src/template/types.ts index ded5d4747e..4fe16bfddc 100644 --- a/core/src/template/types.ts +++ b/core/src/template/types.ts @@ -108,6 +108,13 @@ export abstract class UnresolvedTemplateValue { public abstract toJSON(): CollectionOrValue + /** + * @see sanitizeValue + */ + public toSanitizedValue() { + return this.toJSON() + } + public abstract visitAll(opts: { /** * If true, the returned template expression generator will only yield template expressions that diff --git a/core/src/util/logging.ts b/core/src/util/logging.ts index 0a9f69b244..1e8387e7fc 100644 --- a/core/src/util/logging.ts +++ b/core/src/util/logging.ts @@ -32,6 +32,11 @@ export function sanitizeValue(value: any, _parents?: WeakSet): any { if (value === null || value === undefined) { return value + } else if (value instanceof Error) { + return { + message: value.message, + stack: value.stack, + } } else if (Buffer.isBuffer(value)) { return "" // This is hacky but fairly reliably identifies a Joi schema object diff --git a/core/test/unit/src/resolve-module.ts b/core/test/unit/src/resolve-module.ts index 17efd2519a..bdfce6ff13 100644 --- a/core/test/unit/src/resolve-module.ts +++ b/core/test/unit/src/resolve-module.ts @@ -68,7 +68,9 @@ describe("ModuleResolver", () => { path: join(garden.projectRoot, "module-a"), build: { dependencies: [], timeout: DEFAULT_BUILD_TIMEOUT_SEC }, spec: { - command: ["echo", "${var.foo}"], + build: { + command: ["echo", "${var.foo}"], + }, }, variables: { foo: "somevalue", @@ -81,10 +83,10 @@ describe("ModuleResolver", () => { // the variables section of the module config should not change foo: "somevalue", }) - expect(module.spec).to.eql({ + expect(module.spec.build.command).to.eql( // --> ${var.foo} should evaluate to "override" - command: ["echo", "override"], - }) + ["echo", "override"] + ) }) it("handles a module template reference in a build dependency name", async () => { diff --git a/core/test/unit/src/server/server.ts b/core/test/unit/src/server/server.ts index c42a7783ce..75e845aea8 100644 --- a/core/test/unit/src/server/server.ts +++ b/core/test/unit/src/server/server.ts @@ -212,6 +212,8 @@ describe("GardenServer", () => { .set({ [authTokenHeader]: gardenServer.authKey }) .send({ command: "get config", stringArguments: [] }) .expect(200) + expect(res.body.errors).to.eq(undefined, `error response: ${res.body.errors?.[0]?.stack}`) + const config = await garden.dumpConfig({ log: garden.log }) expect(res.body.result).to.eql(deepOmitUndefined(config)) }) @@ -225,6 +227,7 @@ describe("GardenServer", () => { }) .expect(200) + expect(res.body.errors).to.eq(undefined, `error response: ${res.body.errors?.[0]?.stack}`) const result = taskResultOutputs(res.body.result) expect(result["build.module-a"]).to.exist expect(result["build.module-a"].state).to.equal("ready") @@ -236,6 +239,7 @@ describe("GardenServer", () => { .set({ [authTokenHeader]: gardenServer.authKey }) .send({ command: "get config --var foo=bar" }) .expect(200) + expect(res.body.errors).to.eq(undefined, `error response: ${res.body.errors?.[0]?.stack}`) expect(res.body.result.variables.foo).to.equal("bar") }) }) From b161b80075fcb51ebf34b084a5dfa65b29b600b0 Mon Sep 17 00:00:00 2001 From: Steffen Neubauer Date: Fri, 24 Jan 2025 20:52:16 +0100 Subject: [PATCH 117/117] refactor: use @ts-expect-error todo comments instead of as unknown casts --- core/src/actions/base.ts | 2 +- core/src/commands/workflow.ts | 4 +- core/src/config/config-template.ts | 7 +- core/src/config/project.ts | 3 +- core/src/config/render-template.ts | 13 ++-- core/src/config/template-contexts/base.ts | 9 ++- core/src/config/template-contexts/input.ts | 8 +- .../src/config/template-contexts/variables.ts | 4 +- core/src/config/workflow.ts | 5 +- core/src/garden.ts | 3 +- core/src/outputs.ts | 11 +-- core/src/plugins/hadolint/hadolint.ts | 3 +- .../kubernetes/kubernetes-type/deploy.ts | 3 +- core/src/resolve-module.ts | 74 +++++++++++++------ core/src/template/capture.ts | 2 +- core/src/template/templated-collections.ts | 2 +- core/src/util/testing.ts | 12 +-- core/test/helpers.ts | 6 +- core/test/unit/src/commands/custom.ts | 6 +- .../unit/src/config/template-contexts/base.ts | 4 + core/test/unit/src/garden.ts | 12 +-- .../unit/src/plugins/kubernetes/kubernetes.ts | 3 +- plugins/conftest/src/index.ts | 3 +- 23 files changed, 125 insertions(+), 74 deletions(-) diff --git a/core/src/actions/base.ts b/core/src/actions/base.ts index 931c201dbd..b9ffd2af67 100644 --- a/core/src/actions/base.ts +++ b/core/src/actions/base.ts @@ -738,7 +738,7 @@ export interface ResolvedActionExtension< getOutputs(): StaticOutputs - getVariablesContext(): ConfigContext + getVariablesContext(): VariablesContext getResolvedVariables(): Record } diff --git a/core/src/commands/workflow.ts b/core/src/commands/workflow.ts index 73c70eeea5..90b8b4d06d 100644 --- a/core/src/commands/workflow.ts +++ b/core/src/commands/workflow.ts @@ -88,7 +88,9 @@ export class WorkflowCommand extends Command { await registerAndSetUid(garden, log, workflow) garden.events.emit("workflowRunning", {}) const templateContext = new WorkflowConfigContext(garden, garden.variables) - const files = deepEvaluate((workflow.files || []) as unknown as ParsedTemplate[], { + + // @ts-expect-error todo: correct types for unresolved configs + const files: WorkflowFileSpec[] = deepEvaluate(workflow.files || [], { context: templateContext, opts: {}, }) diff --git a/core/src/config/config-template.ts b/core/src/config/config-template.ts index c4e958f28c..08bcc4acdd 100644 --- a/core/src/config/config-template.ts +++ b/core/src/config/config-template.ts @@ -71,15 +71,18 @@ export async function resolveConfigTemplate( const loggedIn = garden.isLoggedIn() const enterpriseDomain = garden.cloudApi?.domain const context = new ProjectConfigContext({ ...garden, loggedIn, enterpriseDomain }) - const resolved = deepEvaluate(partial as unknown as ParsedTemplate, { + + // @ts-expect-error todo: correct types for unresolved configs + const resolved: BaseGardenResource = deepEvaluate(partial, { context, opts: {}, }) + const configPath = resource.internal.configFilePath // Validate the partial config const validated = validateConfig({ - config: resolved as unknown as BaseGardenResource, + config: resolved, schema: configTemplateSchema(), projectRoot: garden.projectRoot, yamlDocBasePath: [], diff --git a/core/src/config/project.ts b/core/src/config/project.ts index d82a9867ab..8da1019048 100644 --- a/core/src/config/project.ts +++ b/core/src/config/project.ts @@ -632,7 +632,8 @@ export const pickEnvironment = profileAsync(async function _pickEnvironment({ // resolve project variables incl. varfiles deepResolveContext("project", context.variables) - const config = deepEvaluate(environmentConfig as unknown as ParsedTemplate, { + // @ts-expect-error todo: correct types for unresolved configs + const config = deepEvaluate(environmentConfig, { context, opts: {}, }) diff --git a/core/src/config/render-template.ts b/core/src/config/render-template.ts index 3fb83249ed..041d9b134d 100644 --- a/core/src/config/render-template.ts +++ b/core/src/config/render-template.ts @@ -126,12 +126,14 @@ export async function renderConfigTemplate({ const enterpriseDomain = garden.cloudApi?.domain const templateContext = new EnvironmentConfigContext({ ...garden, loggedIn, enterpriseDomain }) - const resolvedWithoutInputs = deepEvaluate(omit(config, "inputs") as unknown as ParsedTemplate, { + // @ts-expect-error todo: correct types for unresolved configs + const resolvedWithoutInputs: RenderTemplateConfig = deepEvaluate(omit(config, "inputs"), { context: templateContext, opts: {}, }) + let resolved: RenderTemplateConfig = { - ...(resolvedWithoutInputs as unknown as RenderTemplateConfig), + ...resolvedWithoutInputs, inputs: config.inputs, } @@ -194,8 +196,8 @@ async function renderModules({ }): Promise { return Promise.all( (template.modules || []).map(async (m, index) => { - // Run a partial template resolution with the parent+template info - const spec = evaluate(m as unknown as ParsedTemplate, { + // @ts-expect-error todo: correct types for unresolved configs + const spec = evaluate(m, { context, opts: {}, }).resolved @@ -271,7 +273,8 @@ async function renderConfigs({ renderConfig: RenderTemplateConfig }): Promise { const templateDescription = `${configTemplateKind} '${template.name}'` - const templateConfigs = evaluate((template.configs || []) as unknown as ParsedTemplate, { + // @ts-expect-error todo: correct types for unresolved configs + const templateConfigs = evaluate(template.configs || [], { context, opts: {}, }).resolved diff --git a/core/src/config/template-contexts/base.ts b/core/src/config/template-contexts/base.ts index e76f9fa535..3de109ecf3 100644 --- a/core/src/config/template-contexts/base.ts +++ b/core/src/config/template-contexts/base.ts @@ -106,9 +106,12 @@ export abstract class ConfigContext { private readonly _cache: Map private readonly _id: number - constructor(private readonly _description?: string) { + constructor(public readonly _description?: string) { this._id = globalConfigContextCounter++ this._cache = new Map() + if (!_description) { + this._description = "" + } } public toSanitizedValue() { @@ -313,8 +316,8 @@ export function renderKeyPath(key: ContextKeySegment[]): string { export class LayeredContext extends ConfigContext { protected readonly layers: ConfigContext[] - constructor(...layers: ConfigContext[]) { - super() + constructor(description: string, ...layers: ConfigContext[]) { + super(description) this.layers = layers } diff --git a/core/src/config/template-contexts/input.ts b/core/src/config/template-contexts/input.ts index 15089c45de..45ef5de3c2 100644 --- a/core/src/config/template-contexts/input.ts +++ b/core/src/config/template-contexts/input.ts @@ -53,9 +53,13 @@ export class InputContext extends LayeredContext { constructor(inputs: ParsedTemplate, template?: ConfigTemplateConfig) { if (template) { - super(new GenericContext(template.inputsSchemaDefaults), new GenericContext(inputs || {})) + super( + "unresolved inputs (with best-effort schema defaults)", + new GenericContext(template.inputsSchemaDefaults), + new GenericContext(inputs || {}) + ) } else { - super(new GenericContext(inputs || {})) + super("fully resolved inputs", new GenericContext(inputs || {})) } } } diff --git a/core/src/config/template-contexts/variables.ts b/core/src/config/template-contexts/variables.ts index 13d22699ab..5383d3ad60 100644 --- a/core/src/config/template-contexts/variables.ts +++ b/core/src/config/template-contexts/variables.ts @@ -35,7 +35,7 @@ export class VariablesContext extends LayeredContext { * The constructor is private, use the static factory methods (below) instead. */ private constructor( - public readonly description: string, + description: string, { context, variablePrecedence, @@ -63,7 +63,7 @@ export class VariablesContext extends LayeredContext { ) } - super(...layers) + super(description, ...layers) if (variableOverrides && !isEmpty(variableOverrides)) { this.applyOverrides(variableOverrides, context) diff --git a/core/src/config/workflow.ts b/core/src/config/workflow.ts index 4ecd3b41ca..f1ec86cb1e 100644 --- a/core/src/config/workflow.ts +++ b/core/src/config/workflow.ts @@ -361,10 +361,11 @@ export function resolveWorkflowConfig(garden: Garden, config: WorkflowConfig) { } let resolvedPartialConfig: WorkflowConfig = { - ...(deepEvaluate(partialConfig as unknown as Record, { + // @ts-expect-error todo: correct types for unresolved configs + ...deepEvaluate(partialConfig, { context, opts: {}, - }) as unknown as WorkflowConfig), + }), internal: config.internal, name: config.name, } diff --git a/core/src/garden.ts b/core/src/garden.ts index f24754db78..b0f1445bf9 100644 --- a/core/src/garden.ts +++ b/core/src/garden.ts @@ -1686,7 +1686,8 @@ export class Garden { const context = new RemoteSourceConfigContext(this, this.variables) const source = { yamlDoc: this.projectConfig.internal.yamlDoc, path: ["sources"] } const resolved = validateSchema( - deepEvaluate(this.projectSources as unknown as ParsedTemplate, { context, opts: {} }), + // @ts-expect-error todo: correct types for unresolved configs + deepEvaluate(this.projectSources, { context, opts: {} }), projectSourcesSchema(), { context: "remote source", diff --git a/core/src/outputs.ts b/core/src/outputs.ts index 1592bf4b7a..a7a0949038 100644 --- a/core/src/outputs.ts +++ b/core/src/outputs.ts @@ -68,8 +68,8 @@ export async function resolveProjectOutputs(garden: Garden, log: Log): Promise[], { + // @ts-expect-error todo: correct types for unresolved configs + return deepEvaluate(garden.rawOutputs, { context: new OutputConfigContext({ garden, resolvedProviders: {}, @@ -77,7 +77,7 @@ export async function resolveProjectOutputs(garden: Garden, log: Log): Promise[], { + // @ts-expect-error todo: correct types for unresolved configs + return deepEvaluate(garden.rawOutputs, { context: configContext, opts: {}, - }) as unknown as OutputSpec[] + }) } diff --git a/core/src/plugins/hadolint/hadolint.ts b/core/src/plugins/hadolint/hadolint.ts index 5217522ea8..07d6a95653 100644 --- a/core/src/plugins/hadolint/hadolint.ts +++ b/core/src/plugins/hadolint/hadolint.ts @@ -270,7 +270,8 @@ hadolintTest.addHandler("configure", async ({ ctx, config }) => { if (!config.include.includes(dockerfilePath)) { try { - dockerfilePath = ctx.deepEvaluate(dockerfilePath) as unknown as typeof dockerfilePath + // @ts-expect-error todo: correct types for unresolved configs + dockerfilePath = ctx.deepEvaluate(dockerfilePath) } catch (error) { if (!(error instanceof GardenError)) { throw error diff --git a/core/src/plugins/kubernetes/kubernetes-type/deploy.ts b/core/src/plugins/kubernetes/kubernetes-type/deploy.ts index fd8830c245..028e3eceb6 100644 --- a/core/src/plugins/kubernetes/kubernetes-type/deploy.ts +++ b/core/src/plugins/kubernetes/kubernetes-type/deploy.ts @@ -55,7 +55,8 @@ export const kubernetesDeployDefinition = (): DeployActionDefinition(config: Input, context: ModuleConfigContext) { /** - * Returns true if the unresolved template value can be resolved at this point and false otherwise. + * Returns false if the unresolved template value contains runtime references, in order to skip resolving it at this point. */ - const shouldEvaluate = (value: UnresolvedTemplateValue) => { + const skipRuntimeReferences = (value: UnresolvedTemplateValue) => { if ( - value instanceof ForEachLazyValue && someReferences({ value, context, - // if forEach expression has runtime references, we can't resolve it at all due to item context missing after converting the module to action - // as the captured context is lost when calling `toJSON` on the unresolved template value - onlyEssential: false, - matcher: (ref) => ref.keyPath[0] === "runtime", - }) - ) { - return false // do not evaluate runtime references - } else if ( - someReferences({ - value, - context, - // in other cases, we only skip evaluation when the runtime references is essential - // meaning, we evaluate everything we can evaluate. onlyEssential: true, matcher: (ref) => ref.keyPath[0] === "runtime", }) @@ -1262,7 +1247,7 @@ function partiallyEvaluateModule(config: Input, co return true } - const partial = conditionallyDeepEvaluate( + const interim = conditionallyDeepEvaluate( config, { context, @@ -1274,9 +1259,50 @@ function partiallyEvaluateModule(config: Input, co unescape: false, }, }, - shouldEvaluate + skipRuntimeReferences + ) + + /** + * Returns true if the unresolved template value contains "item" references + * + * If runtime references are used together with item references, the item references need to be resolved + * in a second pass using "legacyAllowPartial". + * + * We don't use "legacyAllowPartial" for the first pass so that the user receives appropriate error messages + * if a key cannot be resolved. + */ + const findItemReferences = (value: UnresolvedTemplateValue) => { + if ( + someReferences({ + value, + context, + onlyEssential: true, + matcher: (ref) => ref.keyPath[0] === "item", + }) + ) { + return true // we want to evaluate item references + } + + return false + } + + const partial = conditionallyDeepEvaluate( + interim, + { + context, + opts: { + // in the second pass, we eliminate lurking item references + // this can happen if the item context ($forEach) has been used together with runtime in the same template string + legacyAllowPartial: true, + // we also do not unescape here, as the template strings will unfortunately be parsed again later. + unescape: false, + }, + }, + findItemReferences ) + // any leftover unresolved template values are now turned back into the raw form + // this was necessary because we apply module schemas on the partially resolved config return deepMap(partial, (v) => { if (v instanceof UnresolvedTemplateValue) { return v.toJSON() diff --git a/core/src/template/capture.ts b/core/src/template/capture.ts index edb15baf27..8bda4ceb8f 100644 --- a/core/src/template/capture.ts +++ b/core/src/template/capture.ts @@ -41,7 +41,7 @@ export class CapturedContextTemplateValue extends UnresolvedTemplateValue { } override evaluate(args: EvaluateTemplateArgs): TemplateEvaluationResult { - const context = new LayeredContext(args.context, this.context) + const context = new LayeredContext(`captured ${this.context.toSanitizedValue()}`, args.context, this.context) const result = evaluate(this.wrapped, { ...args, context }) diff --git a/core/src/template/templated-collections.ts b/core/src/template/templated-collections.ts index 61f94552e3..3fc138ad4e 100644 --- a/core/src/template/templated-collections.ts +++ b/core/src/template/templated-collections.ts @@ -377,7 +377,7 @@ export class ForEachLazyValue extends StructuralTemplateOperator { const contextForIndex = new GenericContext({ item: { key: i, value: collectionValue[i] }, }) - const loopContext = new LayeredContext(args.context, contextForIndex) + const loopContext = new LayeredContext("item ($forEach)", args.context, contextForIndex) // Check $filter clause output, if applicable if (filterExpression !== undefined) { diff --git a/core/src/util/testing.ts b/core/src/util/testing.ts index 25d5244ebc..ec4fe5da20 100644 --- a/core/src/util/testing.ts +++ b/core/src/util/testing.ts @@ -128,10 +128,8 @@ export function moduleConfigWithDefaults(partial: PartialModuleConfig): ModuleCo }, } - return parseTemplateCollection({ - value: config as unknown as CollectionOrValue, - source: { path: [] }, - }) as unknown as ModuleConfig + // @ts-expect-error todo: correct types for unresolved configs + return parseTemplateCollection({ value: config, source: { path: [] } }) } /** @@ -349,10 +347,8 @@ export class TestGarden extends Garden { }, } this.addActionConfig( - parseTemplateCollection({ - value: merged as unknown as CollectionOrValue, - source: { path: [] }, - }) as unknown as ActionConfig + // @ts-expect-error todo: correct types for unresolved configs + parseTemplateCollection({ value: merged, source: { path: [] } }) ) }) } diff --git a/core/test/helpers.ts b/core/test/helpers.ts index 85f922bda4..fe5efcfaef 100644 --- a/core/test/helpers.ts +++ b/core/test/helpers.ts @@ -122,10 +122,12 @@ export const getDefaultProjectConfig = (): ProjectConfig => export const createProjectConfig = (partialCustomConfig: Partial): ProjectConfig => { const baseConfig = getDefaultProjectConfig() + // @ts-expect-error todo: correct types for unresolved configs return parseTemplateCollection({ - value: merge(baseConfig, partialCustomConfig) as unknown as CollectionOrValue, + // @ts-expect-error todo: correct types for unresolved configs + value: merge(baseConfig, partialCustomConfig), source: { path: [] }, - }) as unknown as ProjectConfig + }) } export const defaultModuleConfig: ModuleConfig = { diff --git a/core/test/unit/src/commands/custom.ts b/core/test/unit/src/commands/custom.ts index e396398b47..8c21a2aa65 100644 --- a/core/test/unit/src/commands/custom.ts +++ b/core/test/unit/src/commands/custom.ts @@ -136,11 +136,11 @@ describe("CustomCommandWrapper", () => { ], variables: {}, exec: { - command: ["echo", "${join(args.$rest, ' ')}"], + command: ["echo", "${join(args.$rest, ' ')}" as string], }, - }, + } as const, source: { path: [] }, - }) as unknown as CommandResource + }) ) const { result } = await cmd.action({ diff --git a/core/test/unit/src/config/template-contexts/base.ts b/core/test/unit/src/config/template-contexts/base.ts index e3a6ab7824..6a7a5b2812 100644 --- a/core/test/unit/src/config/template-contexts/base.ts +++ b/core/test/unit/src/config/template-contexts/base.ts @@ -323,6 +323,7 @@ describe("ConfigContext", () => { describe("LayeredContext", () => { it("allows you to merge multiple contexts", () => { const variables = new LayeredContext( + "test", new GenericContext({ foo: "foo", }), @@ -345,6 +346,7 @@ describe("LayeredContext", () => { it("takes the precedence from right to left when merging primitives", () => { const layeredContext = new LayeredContext( + "test", new GenericContext({ foo: "foo", }), @@ -359,6 +361,7 @@ describe("LayeredContext", () => { it("takes the precedence from right to left when merging objects", () => { const layeredContext = new LayeredContext( + "test", new GenericContext({ foo: "foo", }), @@ -373,6 +376,7 @@ describe("LayeredContext", () => { it("show the available keys if attempt to resolve a non-existing key", () => { const layeredContext = new LayeredContext( + "test", new GenericContext({ foo: "foo", }), diff --git a/core/test/unit/src/garden.ts b/core/test/unit/src/garden.ts index 2b089c94d9..050630de56 100644 --- a/core/test/unit/src/garden.ts +++ b/core/test/unit/src/garden.ts @@ -81,7 +81,7 @@ import { getCloudDistributionName } from "../../../src/cloud/util.js" import { resolveAction } from "../../../src/graph/actions.js" import { serialiseUnresolvedTemplates } from "../../../src/template/types.js" import { parseTemplateCollection } from "../../../src/template/templated-collections.js" -import { deepResolveContext, LayeredContext } from "../../../src/config/template-contexts/base.js" +import { deepResolveContext } from "../../../src/config/template-contexts/base.js" import { VariablesContext } from "../../../src/config/template-contexts/variables.js" const { realpath, writeFile, readFile, remove, pathExists, mkdirp, copy } = fsExtra @@ -404,10 +404,7 @@ describe("Garden", () => { garden.variableOverrides["d"] = "from-cli-var" const graph = await garden.getConfigGraph({ log: garden.log, emit: false }) const runAction = graph.getRun("run-a") - const resolvedVariables = deepResolveContext( - "Garden and run-a action variables", - new LayeredContext(garden.variables, runAction.getVariablesContext()) - ) + const resolvedVariables = deepResolveContext("Garden and run-a action variables", runAction.getVariablesContext()) expect(resolvedVariables).to.eql({ a: "from-project-varfile", b: "from-action-vars", @@ -3182,7 +3179,7 @@ describe("Garden", () => { }) expect(resolved).to.exist - const variables = deepResolveContext("resolved action variables", resolved.getVariablesContext()) + const variables = resolved.getResolvedVariables() expect(variables).to.deep.eq({ myDir: "../../../test", syncTargets: [ @@ -3193,6 +3190,9 @@ describe("Garden", () => { source: "../../../bar", }, ], + sync_targets: { + test: ["foo", "bar"], + }, }) }) diff --git a/core/test/unit/src/plugins/kubernetes/kubernetes.ts b/core/test/unit/src/plugins/kubernetes/kubernetes.ts index a57f26c4fa..2419bed3d9 100644 --- a/core/test/unit/src/plugins/kubernetes/kubernetes.ts +++ b/core/test/unit/src/plugins/kubernetes/kubernetes.ts @@ -67,7 +67,8 @@ describe("kubernetes configureProvider", () => { config: new UnresolvedProviderConfig( config.name, config.dependencies || [], - config as unknown as ParsedTemplate + // @ts-expect-error todo: correct types for unresolved configs + config ), dependencies: {}, moduleConfigs: [], diff --git a/plugins/conftest/src/index.ts b/plugins/conftest/src/index.ts index 25f82cac52..5c9340583e 100644 --- a/plugins/conftest/src/index.ts +++ b/plugins/conftest/src/index.ts @@ -289,7 +289,8 @@ export const gardenPlugin = () => } try { - files = ctx.deepEvaluate(files) as unknown as typeof files + // @ts-expect-error todo: correct types for unresolved configs + files = ctx.deepEvaluate(files) } catch (error) { if (!(error instanceof GardenError)) { throw error