From 798e5390ddf4ede4cf5af2911f8ad5e6ce19d7f1 Mon Sep 17 00:00:00 2001 From: Tiago Natel Date: Tue, 5 Dec 2023 17:15:37 +0000 Subject: [PATCH] feat: evaluator v2 Signed-off-by: Tiago Natel --- cmd/terramate/cli/cli.go | 133 +++-- cmd/terramate/cli/project.go | 2 + config/assert_test.go | 9 +- config/config.go | 13 +- config/script_test.go | 5 +- config/stack.go | 41 +- generate/generate.go | 100 ++-- generate/generate_bench_test.go | 8 +- generate/generate_hcl_test.go | 2 +- generate/generate_test.go | 5 +- generate/genfile/genfile.go | 18 +- generate/genfile/genfile_lets_map_test.go | 5 +- generate/genfile/genfile_test.go | 25 +- generate/genhcl/genhcl.go | 245 +++++---- generate/genhcl/genhcl_lets_map_test.go | 5 +- generate/genhcl/genhcl_test.go | 15 +- generate/genhcl/partial_eval_test.go | 16 +- generate/hcl_expr_func_test.go | 2 + generate/load_test.go | 22 +- generate/outdated_detection_test.go | 4 +- generate/vendor_test.go | 2 +- globals/README.md | 405 +++++++++++++++ globals/eval_report.go | 54 -- globals/globals.go | 581 ---------------------- globals/globals_map_test.go | 5 +- globals/globals_resolver.go | 195 ++++++++ globals/globals_test.go | 153 +++--- globals/globals_v2_test.go | 496 ++++++++++++++++++ globals/stack.go | 21 - go.mod | 4 + go.sum | 11 +- hcl/ast/expr.go | 25 + hcl/eval/eval.go | 449 +++++++++++++++-- hcl/eval/eval_test.go | 9 +- hcl/eval/object.go | 342 ------------- hcl/eval/object_test.go | 162 ------ hcl/eval/partial_eval.go | 7 +- hcl/eval/partial_eval_bench_test.go | 81 +-- hcl/eval/partial_eval_test.go | 54 +- hcl/eval/partial_fuzz_test.go | 59 ++- hcl/eval/ref.go | 187 +++++++ hcl/eval/ref_test.go | 189 +++++++ hcl/eval/stmt.go | 271 ++++++++++ hcl/eval/stmt_filter_test.go | 206 ++++++++ hcl/hcl.go | 24 +- lets/doc.go | 5 + lets/lets.go | 236 +++------ project/project.go | 3 + run/env.go | 72 ++- run/env_test.go | 3 +- runtime/doc.go | 17 + runtime/runtime_resolver.go | 45 ++ scripts/cpu_profile.sh | 38 ++ stack/eval.go | 41 -- stack/manager.go | 4 +- stdlib/funcs.go | 72 ++- stdlib/funcs_test.go | 21 +- stdlib/ternary.go | 18 +- test/eval.go | 59 +++ test/hcl.go | 6 +- test/hclwrite/hclwrite.go | 23 +- test/sandbox/sandbox.go | 29 +- 62 files changed, 3429 insertions(+), 1930 deletions(-) create mode 100644 globals/README.md delete mode 100644 globals/eval_report.go delete mode 100644 globals/globals.go create mode 100644 globals/globals_resolver.go create mode 100644 globals/globals_v2_test.go delete mode 100644 globals/stack.go delete mode 100644 hcl/eval/object.go delete mode 100644 hcl/eval/object_test.go create mode 100644 hcl/eval/ref.go create mode 100644 hcl/eval/ref_test.go create mode 100644 hcl/eval/stmt.go create mode 100644 hcl/eval/stmt_filter_test.go create mode 100644 lets/doc.go create mode 100644 runtime/doc.go create mode 100644 runtime/runtime_resolver.go create mode 100644 scripts/cpu_profile.sh delete mode 100644 stack/eval.go create mode 100644 test/eval.go diff --git a/cmd/terramate/cli/cli.go b/cmd/terramate/cli/cli.go index afc5749f5..c7d081131 100644 --- a/cmd/terramate/cli/cli.go +++ b/cmd/terramate/cli/cli.go @@ -11,6 +11,7 @@ import ( "os" "path" "path/filepath" + "runtime/pprof" "strings" "time" @@ -34,6 +35,7 @@ import ( "github.com/terramate-io/terramate/hcl/fmt" "github.com/terramate-io/terramate/hcl/info" "github.com/terramate-io/terramate/modvendor/download" + "github.com/terramate-io/terramate/runtime" "github.com/terramate-io/terramate/versions" "github.com/terramate-io/terramate/stack/trigger" @@ -109,6 +111,7 @@ type cliSpec struct { LogDestination string `optional:"true" default:"stderr" enum:"stderr,stdout" help:"Destination of log messages"` Quiet bool `optional:"false" help:"Disable output"` Verbose int `short:"v" optional:"true" default:"0" type:"counter" help:"Increase verboseness of output"` + CPUProfiling bool `optional:"true" default:"false" help:"Create a CPU profile file when running"` DisableCheckGitUntracked bool `optional:"true" default:"false" help:"Disable git check for untracked files"` DisableCheckGitUncommitted bool `optional:"true" default:"false" help:"Disable git check for uncommitted files"` @@ -324,6 +327,19 @@ func newCLI(version string, args []string, stdin io.Reader, stdout, stderr io.Wr fatal(err, "parsing cli args %v", args) } + if parsedArgs.CPUProfiling { + stdfmt.Println("Creating CPU profile...") + f, err := os.Create("terramate.prof") + if err != nil { + fatal(err, "can't create profile output file") + } + err = pprof.StartCPUProfile(f) + if err != nil { + fatal(err, "error when starting CPU profiling") + } + + } + configureLogging(parsedArgs.LogLevel, parsedArgs.LogFmt, parsedArgs.LogDestination, stdout, stderr) // If we don't re-create the logger after configuring we get some @@ -474,6 +490,8 @@ func newCLI(version string, args []string, stdin io.Reader, stdout, stderr io.Wr log.Fatal().Msg("flag --changed provided but no git repository found") } + globalsResolver := globals.NewResolver(&prj.root) + prj.globals = globalsResolver uimode := HumanMode if val := os.Getenv("CI"); envVarIsSet(val) { uimode = AutomationMode @@ -516,6 +534,13 @@ func (c *cli) run() { logger.Debug().Msg("Handle command.") + // We start the CPU Profiling during the flags parsing, but can't defer + // the stop there, as the CLI parsing returns far before the program is + // done running. Therefore we schedule it here. + if c.parsedArgs.CPUProfiling { + defer pprof.StopCPUProfile() + } + switch c.ctx.Command() { case "fmt": c.format() @@ -812,7 +837,7 @@ func (c *cli) gencodeWithVendor() (generate.Report, download.Report) { log.Debug().Msg("generating code") - report := generate.Do(c.cfg(), c.vendorDir(), vendorRequestEvents) + report := generate.Do(c.cfg(), c.globals(), c.vendorDir(), vendorRequestEvents) log.Debug().Msg("code generation finished, waiting for vendor requests to be handled") @@ -1496,7 +1521,7 @@ func (c *cli) generateDebug() { selectedStacks[stack.Dir()] = struct{}{} } - results, err := generate.Load(c.cfg(), c.vendorDir()) + results, err := generate.Load(c.cfg(), c.globals(), c.vendorDir()) if err != nil { fatal(err, "generate debug: loading generated code") } @@ -1536,8 +1561,17 @@ func (c *cli) printStacksGlobals() { for _, stackEntry := range c.filterStacks(report.Stacks) { stack := stackEntry.Stack - report := globals.ForStack(c.cfg(), stack) - if err := report.AsError(); err != nil { + tree := stackEntry.Stack.Tree() + evalctx := eval.New( + stack.Dir, + runtime.NewResolver(c.cfg(), stack), + c.globals(), + ) + evalctx.SetFunctions(stdlib.Functions(evalctx, tree.HostDir())) + + expr, _ := ast.ParseExpression(`global`, ``) + globals, err := evalctx.Eval(expr) + if err != nil { logger := log.With(). Stringer("stack", stack.Dir). Logger() @@ -1545,7 +1579,7 @@ func (c *cli) printStacksGlobals() { errlog.Fatal(logger, err, "listing stacks globals: loading stack") } - globalsStrRepr := report.Globals.String() + globalsStrRepr := fmt.FormatAttributes(globals.AsValueMap()) if globalsStrRepr == "" { continue } @@ -1676,6 +1710,18 @@ func (c *cli) partialEval() { } } +func (c *cli) detectEvalContext(overrideGlobals map[string]string) *eval.Context { + var st *config.Stack + if config.IsStack(c.cfg(), c.wd()) { + var err error + st, err = config.LoadStack(c.cfg(), prj.PrjAbsPath(c.rootdir(), c.wd())) + if err != nil { + fatal(err, "setup eval context: loading stack config") + } + } + return c.setupEvalContext(st, overrideGlobals) +} + func (c *cli) evalRunArgs(st *config.Stack, cmd []string) []string { ctx := c.setupEvalContext(st, map[string]string{}) var newargs []string @@ -1749,62 +1795,42 @@ func (c *cli) outputEvalResult(val cty.Value, asJSON bool) { c.output.MsgStdOut(string(data)) } -func (c *cli) detectEvalContext(overrideGlobals map[string]string) *eval.Context { - var st *config.Stack - if config.IsStack(c.cfg(), c.wd()) { - var err error - st, err = config.LoadStack(c.cfg(), prj.PrjAbsPath(c.rootdir(), c.wd())) - if err != nil { - fatal(err, "setup eval context: loading stack config") - } - } - return c.setupEvalContext(st, overrideGlobals) -} - func (c *cli) setupEvalContext(st *config.Stack, overrideGlobals map[string]string) *eval.Context { - runtime := c.cfg().Runtime() - - var tdir string + var pdir prj.Path if st != nil { - tdir = st.HostDir(c.cfg()) - runtime.Merge(st.RuntimeValues(c.cfg())) + pdir = st.Dir } else { - tdir = c.wd() - } - - ctx := eval.NewContext(stdlib.NoFS(tdir)) - ctx.SetNamespace("terramate", runtime) - - wdPath := prj.PrjAbsPath(c.rootdir(), tdir) - tree, ok := c.cfg().Lookup(wdPath) - if !ok { - fatal(errors.E("configuration at %s not found", wdPath)) - } - exprs, err := globals.LoadExprs(tree) - if err != nil { - fatal(err, "loading globals expressions") + pdir = prj.PrjAbsPath(c.rootdir(), c.wd()) } + var overrideStmts eval.Stmts for name, exprStr := range overrideGlobals { expr, err := ast.ParseExpression(exprStr, "") if err != nil { fatal(errors.E(err, "--global %s=%s is an invalid expresssion", name, exprStr)) } parts := strings.Split(name, ".") - length := len(parts) - globalPath := globals.NewGlobalAttrPath(parts[0:length-1], parts[length-1]) - exprs.SetOverride( - wdPath, - globalPath, - expr, - info.NewRange(c.rootdir(), hhcl.Range{ - Filename: "", + ref := eval.NewRef("global", parts...) + overrideStmts = append(overrideStmts, eval.Stmt{ + Origin: ref, + LHS: ref, + Info: eval.NewInfo(pdir, info.NewRange(c.rootdir(), hhcl.Range{ Start: hhcl.InitialPos, End: hhcl.InitialPos, - }), - ) + Filename: ``, + })), + RHS: eval.NewExprRHS(expr), + }) } - _ = exprs.Eval(ctx) + + tree, _ := c.cfg().Lookup(pdir) + ctx := eval.New( + tree.Dir(), + runtime.NewResolver(c.cfg(), st), + globals.NewResolver(c.cfg(), overrideStmts...), + ) + + ctx.SetFunctions(stdlib.NoFS(ctx, c.wd())) return ctx } @@ -1821,7 +1847,7 @@ func (c *cli) checkOutdatedGeneratedCode() { return } - outdatedFiles, err := generate.DetectOutdated(c.cfg(), c.vendorDir()) + outdatedFiles, err := generate.DetectOutdated(c.cfg(), c.globals(), c.vendorDir()) if err != nil { fatal(err, "failed to check outdated code on project") } @@ -1860,11 +1886,12 @@ func (c *cli) gitSafeguardRemoteEnabled() bool { return true } -func (c *cli) wd() string { return c.prj.wd } -func (c *cli) rootdir() string { return c.prj.rootdir } -func (c *cli) cfg() *config.Root { return &c.prj.root } -func (c *cli) rootNode() hcl.Config { return c.prj.root.Tree().Node } -func (c *cli) cred() credential { return c.cloud.client.Credential.(credential) } +func (c *cli) wd() string { return c.prj.wd } +func (c *cli) rootdir() string { return c.prj.rootdir } +func (c *cli) cfg() *config.Root { return &c.prj.root } +func (c *cli) globals() *globals.Resolver { return c.prj.globals } +func (c *cli) rootNode() hcl.Config { return c.prj.root.Tree().Node } +func (c *cli) cred() credential { return c.cloud.client.Credential.(credential) } func (c *cli) friendlyFmtDir(dir string) (string, bool) { return prj.FriendlyFmtDir(c.rootdir(), c.wd(), dir) diff --git a/cmd/terramate/cli/project.go b/cmd/terramate/cli/project.go index c2133d9ea..85a2d4b74 100644 --- a/cmd/terramate/cli/project.go +++ b/cmd/terramate/cli/project.go @@ -11,6 +11,7 @@ import ( "github.com/terramate-io/terramate/config" "github.com/terramate-io/terramate/errors" "github.com/terramate-io/terramate/git" + "github.com/terramate-io/terramate/globals" "github.com/terramate-io/terramate/hcl" "github.com/terramate-io/terramate/stack" ) @@ -21,6 +22,7 @@ type project struct { isRepo bool root config.Root baseRef string + globals *globals.Resolver normalizedRepo string git struct { diff --git a/config/assert_test.go b/config/assert_test.go index 489f1bf8b..6275d122f 100644 --- a/config/assert_test.go +++ b/config/assert_test.go @@ -13,6 +13,7 @@ import ( "github.com/terramate-io/terramate/errors" "github.com/terramate-io/terramate/hcl" "github.com/terramate-io/terramate/hcl/eval" + "github.com/terramate-io/terramate/project" "github.com/terramate-io/terramate/stdlib" "github.com/terramate-io/terramate/test" "github.com/zclconf/go-cty/cty" @@ -175,13 +176,15 @@ func TestAssertConfigEval(t *testing.T) { tcase := tcase t.Run(tcase.name, func(t *testing.T) { t.Parallel() - hclctx := eval.NewContext(stdlib.Functions(test.TempDir(t))) + evalctx := eval.New(project.RootPath) + funcs := stdlib.Functions(evalctx, t.TempDir()) + evalctx.SetFunctions(funcs) for k, v := range tcase.namespaces { - hclctx.SetNamespace(k, v.asCtyMap()) + evalctx.SetNamespace(k, v.asCtyMap()) } - got, err := config.EvalAssert(hclctx, tcase.assert) + got, err := config.EvalAssert(evalctx, tcase.assert) assert.IsError(t, err, tcase.wantErr) if !equalAsserts(tcase.want, got) { t.Fatalf("got %#v != want %#v", got, tcase.want) diff --git a/config/config.go b/config/config.go index 0b51cd1b7..25c5250ff 100644 --- a/config/config.go +++ b/config/config.go @@ -38,6 +38,8 @@ const ( type Root struct { tree Tree + lookupCache map[string]*Tree + runtime project.Runtime } @@ -104,7 +106,8 @@ func TryLoadConfig(fromdir string) (tree *Root, configpath string, found bool, e // NewRoot creates a new [Root] tree for the cfg tree. func NewRoot(tree *Tree) *Root { r := &Root{ - tree: *tree, + tree: *tree, + lookupCache: make(map[string]*Tree), } r.initRuntime() return r @@ -127,7 +130,13 @@ func (root *Root) HostDir() string { return root.tree.RootDir() } // Lookup a node from the root using a filesystem query path. func (root *Root) Lookup(path project.Path) (*Tree, bool) { - return root.tree.lookup(path) + tree, ok := root.lookupCache[path.String()] + if ok { + return tree, tree != nil + } + tree, ok = root.tree.lookup(path) + root.lookupCache[path.String()] = tree + return tree, ok } // StacksByPaths returns the stacks from the provided relative paths. diff --git a/config/script_test.go b/config/script_test.go index 00b45d800..ee09f16a1 100644 --- a/config/script_test.go +++ b/config/script_test.go @@ -16,6 +16,7 @@ import ( "github.com/terramate-io/terramate/hcl/ast" "github.com/terramate-io/terramate/hcl/eval" "github.com/terramate-io/terramate/hcl/info" + "github.com/terramate-io/terramate/project" "github.com/terramate-io/terramate/stdlib" "github.com/terramate-io/terramate/test" @@ -298,7 +299,9 @@ func TestScriptEval(t *testing.T) { tcase := tcase t.Run(tcase.name, func(t *testing.T) { t.Parallel() - hclctx := eval.NewContext(stdlib.Functions(test.TempDir(t))) + scopedir := test.TempDir(t) + hclctx := eval.New(project.NewPath("/")) + hclctx.SetFunctions(stdlib.Functions(hclctx, scopedir)) hclctx.SetNamespace("global", tcase.globals) got, err := config.EvalScript(hclctx, tcase.script) diff --git a/config/stack.go b/config/stack.go index 8cb70d09b..65cf5f2ee 100644 --- a/config/stack.go +++ b/config/stack.go @@ -13,7 +13,6 @@ import ( "github.com/rs/zerolog/log" "github.com/terramate-io/terramate/config/tag" "github.com/terramate-io/terramate/errors" - "github.com/terramate-io/terramate/hcl" "github.com/terramate-io/terramate/project" "github.com/zclconf/go-cty/cty" ) @@ -56,6 +55,8 @@ type ( // IsChanged tells if this is a changed stack. IsChanged bool + + tree *Tree } // SortableStack is a wrapper for the Stack which implements the [DirElem] type. @@ -85,28 +86,31 @@ const ( ) // NewStackFromHCL creates a new stack from raw configuration cfg. -func NewStackFromHCL(root string, cfg hcl.Config) (*Stack, error) { - name := cfg.Stack.Name +func NewStackFromHCL(root string, treecfg *Tree) (*Stack, error) { + stackcfg := treecfg.Node.Stack + name := stackcfg.Name if name == "" { - name = filepath.Base(cfg.AbsDir()) + name = filepath.Base(treecfg.Dir().String()) } - watchFiles, err := validateWatchPaths(root, cfg.AbsDir(), cfg.Stack.Watch) + watchFiles, err := validateWatchPaths(root, treecfg.HostDir(), stackcfg.Watch) if err != nil { return nil, errors.E(err, ErrStackInvalidWatch) } stack := &Stack{ Name: name, - ID: cfg.Stack.ID, - Description: cfg.Stack.Description, - Tags: cfg.Stack.Tags, - After: cfg.Stack.After, - Before: cfg.Stack.Before, - Wants: cfg.Stack.Wants, - WantedBy: cfg.Stack.WantedBy, + ID: stackcfg.ID, + Description: stackcfg.Description, + Tags: stackcfg.Tags, + After: stackcfg.After, + Before: stackcfg.Before, + Wants: stackcfg.Wants, + WantedBy: stackcfg.WantedBy, Watch: watchFiles, - Dir: project.PrjAbsPath(root, cfg.AbsDir()), + Dir: treecfg.Dir(), + + tree: treecfg, } err = stack.Validate() if err != nil { @@ -168,6 +172,9 @@ func (s Stack) ValidateSets() error { return errs.AsError() } +// Tree returns the stack node tree. +func (s Stack) Tree() *Tree { return s.tree } + func validateSet(field string, set []string) error { elems := map[string]struct{}{} for _, s := range set { @@ -282,7 +289,7 @@ func validateWatchPaths(rootdir string, stackpath string, paths []string) (proje func StacksFromTrees(root string, trees List[*Tree]) (List[*SortableStack], error) { var stacks List[*SortableStack] for _, tree := range trees { - s, err := NewStackFromHCL(root, tree.Node) + s, err := NewStackFromHCL(root, tree) if err != nil { return List[*SortableStack]{}, err } @@ -302,7 +309,7 @@ func LoadAllStacks(cfg *Tree) (List[*SortableStack], error) { stacksIDs := map[string]*Stack{} for _, stackNode := range cfg.Stacks() { - stack, err := NewStackFromHCL(cfg.RootDir(), stackNode.Node) + stack, err := NewStackFromHCL(cfg.RootDir(), stackNode) if err != nil { return List[*SortableStack]{}, err } @@ -339,7 +346,7 @@ func LoadStack(root *Root, dir project.Path) (*Stack, error) { if !node.IsStack() { return nil, errors.E("config at %q is not a stack", dir) } - return NewStackFromHCL(root.HostDir(), node.Node) + return NewStackFromHCL(root.HostDir(), node) } // TryLoadStack tries to load a single stack from dir. It sets found as true in case @@ -354,7 +361,7 @@ func TryLoadStack(root *Root, cfgdir project.Path) (stack *Stack, found bool, er return nil, false, nil } - s, err := NewStackFromHCL(root.HostDir(), tree.Node) + s, err := NewStackFromHCL(root.HostDir(), tree) if err != nil { return nil, true, err } diff --git a/generate/generate.go b/generate/generate.go index c01b86447..9329cf49f 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -24,7 +24,7 @@ import ( "github.com/terramate-io/terramate/hcl/eval" "github.com/terramate-io/terramate/hcl/info" "github.com/terramate-io/terramate/project" - "github.com/terramate-io/terramate/stack" + "github.com/terramate-io/terramate/runtime" "github.com/terramate-io/terramate/stdlib" ) @@ -89,7 +89,7 @@ type LoadResult struct { // If a critical error that fails the loading of all results happens it returns // a non-nil error. In this case the error is not specific to generating code // for a specific dir. -func Load(root *config.Root, vendorDir project.Path) ([]LoadResult, error) { +func Load(root *config.Root, globals *globals.Resolver, vendorDir project.Path) ([]LoadResult, error) { stacks, err := config.LoadAllStacks(root.Tree()) if err != nil { return nil, err @@ -98,14 +98,16 @@ func Load(root *config.Root, vendorDir project.Path) ([]LoadResult, error) { for i, st := range stacks { res := LoadResult{Dir: st.Dir()} - loadres := globals.ForStack(root, st.Stack) - if err := loadres.AsError(); err != nil { - res.Err = err - results[i] = res - continue - } + tree, _ := root.Lookup(st.Dir()) + evalctx := eval.New( + st.Dir(), + runtime.NewResolver(root, st.Stack), + globals, + ) - generated, err := loadStackCodeCfgs(root, st.Stack, loadres.Globals, vendorDir, nil) + evalctx.SetFunctions(stdlib.Functions(evalctx, tree.HostDir())) + + generated, err := loadStackCodeCfgs(root, evalctx, st.Stack, vendorDir, nil) if err != nil { res.Err = errors.E(err, "while loading configs of stack %s", st.Dir()) results[i] = res @@ -120,7 +122,12 @@ func Load(root *config.Root, vendorDir project.Path) ([]LoadResult, error) { continue } res := LoadResult{Dir: dircfg.Dir()} - evalctx := eval.NewContext(stdlib.Functions(dircfg.HostDir())) + evalctx := eval.New( + dircfg.Dir(), + runtime.NewResolver(root, nil), + globals, + ) + evalctx.SetFunctions(stdlib.Functions(evalctx, dircfg.HostDir())) var generated []GenFile for _, block := range dircfg.Node.Generate.Files { @@ -128,7 +135,7 @@ func Load(root *config.Root, vendorDir project.Path) ([]LoadResult, error) { continue } - file, err := genfile.Eval(block, evalctx) + file, err := genfile.Eval(block, evalctx, dircfg.Dir()) if err != nil { res.Err = errors.L(res.Err, err).AsError() results = append(results, res) @@ -174,20 +181,21 @@ func Load(root *config.Root, vendorDir project.Path) ([]LoadResult, error) { // obtained and the report needs to be inspected to check. func Do( root *config.Root, + globalsResolver *globals.Resolver, vendorDir project.Path, vendorRequests chan<- event.VendorRequest, ) Report { - stackReport := forEachStack(root, vendorDir, + stackReport := forEachStack(root, globalsResolver, vendorDir, vendorRequests, doStackGeneration) - rootReport := doRootGeneration(root) + rootReport := doRootGeneration(root, globalsResolver) report := mergeReports(stackReport, rootReport) return cleanupOrphaned(root, report) } func doStackGeneration( root *config.Root, + evalctx *eval.Context, stack *config.Stack, - globals *eval.Object, vendorDir project.Path, vendorRequests chan<- event.VendorRequest, ) dirReport { @@ -201,7 +209,7 @@ func doStackGeneration( logger.Debug().Msg("generating files") - generated, err := loadStackCodeCfgs(root, stack, globals, vendorDir, vendorRequests) + generated, err := loadStackCodeCfgs(root, evalctx, stack, vendorDir, vendorRequests) if err != nil { report.err = err return report @@ -298,14 +306,18 @@ func doStackGeneration( return report } -func doRootGeneration(root *config.Root) Report { +func doRootGeneration(root *config.Root, globalsResolver *globals.Resolver) Report { logger := log.With(). Str("action", "generate.doRootGeneration"). Logger() report := Report{} - evalctx := eval.NewContext(stdlib.Functions(root.HostDir())) - evalctx.SetNamespace("terramate", root.Runtime()) + evalctx := eval.New( + project.RootPath, + runtime.NewResolver(root, nil), + globalsResolver, + ) + evalctx.SetFunctions(stdlib.Functions(evalctx, root.HostDir())) var files []GenFile for _, cfg := range root.Tree().AsList() { @@ -345,7 +357,7 @@ func doRootGeneration(root *config.Root) Report { logger.Debug().Msg("block validated successfully") - file, err := genfile.Eval(block, evalctx) + file, err := genfile.Eval(block, evalctx, cfg.Dir()) if err != nil { report.addFailure(targetDir, err) return report @@ -488,7 +500,7 @@ processSubdirs: // DetectOutdated will verify if the given config has outdated code // and return a list of filenames that are outdated, ordered lexicographically. -func DetectOutdated(root *config.Root, vendorDir project.Path) ([]string, error) { +func DetectOutdated(root *config.Root, globals *globals.Resolver, vendorDir project.Path) ([]string, error) { logger := log.With(). Str("action", "generate.DetectOutdated()"). Logger() @@ -504,7 +516,14 @@ func DetectOutdated(root *config.Root, vendorDir project.Path) ([]string, error) logger.Debug().Msg("checking outdated code inside stacks") for _, stack := range stacks { - outdated, err := stackOutdated(root, stack.Stack, vendorDir) + tree := stack.Tree() + evalctx := eval.New( + stack.Dir(), + runtime.NewResolver(root, stack.Stack), + globals, + ) + evalctx.SetFunctions(stdlib.Functions(evalctx, tree.HostDir())) + outdated, err := stackOutdated(root, evalctx, stack.Stack, vendorDir) if err != nil { errs.Append(err) continue @@ -549,6 +568,7 @@ func DetectOutdated(root *config.Root, vendorDir project.Path) ([]string, error) // If the stack has an invalid configuration it will return an error. func stackOutdated( root *config.Root, + evalctx *eval.Context, st *config.Stack, vendorDir project.Path, ) ([]string, error) { @@ -557,13 +577,7 @@ func stackOutdated( Stringer("stack", st). Logger() - report := globals.ForStack(root, st) - if err := report.AsError(); err != nil { - return nil, errors.E(err, "checking for outdated code") - } - - globals := report.Globals - generated, err := loadStackCodeCfgs(root, st, globals, vendorDir, nil) + generated, err := loadStackCodeCfgs(root, evalctx, st, vendorDir, nil) if err != nil { return nil, err } @@ -739,14 +753,15 @@ func readFile(path string) (string, bool, error) { type forEachStackFunc func( *config.Root, + *eval.Context, *config.Stack, - *eval.Object, project.Path, chan<- event.VendorRequest, ) dirReport func forEachStack( root *config.Root, + globalsResolver *globals.Resolver, vendorDir project.Path, vendorRequests chan<- event.VendorRequest, fn forEachStackFunc, @@ -760,13 +775,15 @@ func forEachStack( } for _, elem := range stacks { - globalsReport := globals.ForStack(root, elem.Stack) - if err := globalsReport.AsError(); err != nil { - report.addFailure(elem.Dir(), errors.E(ErrLoadingGlobals, err)) - continue - } + tree := elem.Stack.Tree() + evalctx := eval.New( + tree.Dir(), + runtime.NewResolver(root, elem.Stack), + globalsResolver, + ) + evalctx.SetFunctions(stdlib.Functions(evalctx, tree.HostDir())) - stackReport := fn(root, elem.Stack, globalsReport.Globals, vendorDir, vendorRequests) + stackReport := fn(root, evalctx, elem.Stack, vendorDir, vendorRequests) report.addDirReport(elem.Dir(), stackReport) } @@ -1138,7 +1155,7 @@ func checkFileConflict(generated []GenFile) map[string]error { return errsmap } -func loadAsserts(root *config.Root, st *config.Stack, globals *eval.Object) ([]config.Assert, error) { +func loadAsserts(root *config.Root, evalctx *eval.Context, st *config.Stack) ([]config.Assert, error) { logger := log.With(). Str("action", "generate.loadAsserts"). Str("rootdir", root.HostDir()). @@ -1154,11 +1171,10 @@ func loadAsserts(root *config.Root, st *config.Stack, globals *eval.Object) ([]c Stringer("curdir", curdir). Logger() - evalctx := stack.NewEvalCtx(root, st, globals) cfg, ok := root.Lookup(curdir) if ok { for _, assertCfg := range cfg.Node.Asserts { - assert, err := config.EvalAssert(evalctx.Context, assertCfg) + assert, err := config.EvalAssert(evalctx, assertCfg) if err != nil { errs.Append(err) } else { @@ -1183,24 +1199,24 @@ func loadAsserts(root *config.Root, st *config.Stack, globals *eval.Object) ([]c func loadStackCodeCfgs( root *config.Root, + evalctx *eval.Context, st *config.Stack, - globals *eval.Object, vendorDir project.Path, vendorRequests chan<- event.VendorRequest, ) ([]GenFile, error) { - asserts, err := loadAsserts(root, st, globals) + asserts, err := loadAsserts(root, evalctx, st) if err != nil { return nil, err } var genfilesConfigs []GenFile - genfiles, err := genfile.Load(root, st, globals, vendorDir, vendorRequests) + genfiles, err := genfile.Load(root, evalctx, st, vendorDir, vendorRequests) if err != nil { return nil, err } - genhcls, err := genhcl.Load(root, st, globals, vendorDir, vendorRequests) + genhcls, err := genhcl.Load(root, evalctx, st, vendorDir, vendorRequests) if err != nil { return nil, err } diff --git a/generate/generate_bench_test.go b/generate/generate_bench_test.go index 590c223f3..ec1560754 100644 --- a/generate/generate_bench_test.go +++ b/generate/generate_bench_test.go @@ -11,6 +11,7 @@ import ( "github.com/madlambda/spells/assert" "github.com/terramate-io/terramate/config" "github.com/terramate-io/terramate/generate" + "github.com/terramate-io/terramate/globals" "github.com/terramate-io/terramate/project" "github.com/terramate-io/terramate/test/sandbox" ) @@ -59,9 +60,11 @@ func BenchmarkGenerate(b *testing.B) { root, err := config.LoadRoot(s.RootDir()) assert.NoError(b, err) + globals := globals.NewResolver(root) + b.StartTimer() for i := 0; i < b.N; i++ { - report := generate.Do(root, project.NewPath("/vendor"), nil) + report := generate.Do(root, globals, project.NewPath("/vendor"), nil) if report.HasFailures() { b.Fatal(report.Full()) } @@ -114,9 +117,10 @@ func BenchmarkGenerateRegex(b *testing.B) { root, err := config.LoadRoot(s.RootDir()) assert.NoError(b, err) + globals := globals.NewResolver(root) b.StartTimer() for i := 0; i < b.N; i++ { - report := generate.Do(root, project.NewPath("/vendor"), nil) + report := generate.Do(root, globals, project.NewPath("/vendor"), nil) if report.HasFailures() { b.Fatal(report.Full()) } diff --git a/generate/generate_hcl_test.go b/generate/generate_hcl_test.go index 30382c38d..50b2e3136 100644 --- a/generate/generate_hcl_test.go +++ b/generate/generate_hcl_test.go @@ -1330,7 +1330,7 @@ func TestWontOverwriteManuallyDefinedTerraform(t *testing.T) { fmt.Sprintf("f:stack/%s:%s", genFilename, manualTfCode), }) - report := generate.Do(s.Config(), project.NewPath("/modules"), nil) + report := generate.Do(s.Config(), s.Globals(), project.NewPath("/modules"), nil) assert.EqualInts(t, 0, len(report.Successes), "want no success") assert.EqualInts(t, 1, len(report.Failures), "want single failure") assertReportHasError(t, report, errors.E(generate.ErrManualCodeExists)) diff --git a/generate/generate_test.go b/generate/generate_test.go index 4718e7aa6..736acce48 100644 --- a/generate/generate_test.go +++ b/generate/generate_test.go @@ -719,6 +719,7 @@ func testCodeGeneration(t *testing.T, tcases []testcase) { tcase := tc t.Run(tcase.name, func(t *testing.T) { + t.Helper() t.Parallel() if tcase.skipOn == runtime.GOOS { t.Skipf("skipping on GOOS %q", tcase.skipOn) @@ -760,7 +761,7 @@ func testCodeGeneration(t *testing.T, tcases []testcase) { if tcase.vendorDir != "" { vendorDir = project.NewPath(tcase.vendorDir) } - report := generate.Do(s.Config(), vendorDir, nil) + report := generate.Do(s.Config(), s.Globals(), vendorDir, nil) assertEqualReports(t, report, tcase.wantReport) assertGeneratedFiles(t) @@ -768,7 +769,7 @@ func testCodeGeneration(t *testing.T, tcases []testcase) { // piggyback on the tests to validate that regeneration doesn't // delete files or fail and has identical results. t.Run("regenerate", func(t *testing.T) { - report := generate.Do(s.Config(), vendorDir, nil) + report := generate.Do(s.Config(), s.Globals(), vendorDir, nil) // since we just generated everything, report should only contain // the same failures as previous code generation. assertEqualReports(t, report, generate.Report{ diff --git a/generate/genfile/genfile.go b/generate/genfile/genfile.go index 1bfeda01d..c151a4dfc 100644 --- a/generate/genfile/genfile.go +++ b/generate/genfile/genfile.go @@ -19,7 +19,6 @@ import ( "github.com/terramate-io/terramate/lets" "github.com/terramate-io/terramate/project" - "github.com/terramate-io/terramate/stack" "github.com/zclconf/go-cty/cty" ) @@ -119,8 +118,8 @@ func (f File) String() string { // The rootdir MUST be an absolute path. func Load( root *config.Root, + evalctx *eval.Context, st *config.Stack, - globals *eval.Object, vendorDir project.Path, vendorRequests chan<- event.VendorRequest, ) ([]File, error) { @@ -130,21 +129,19 @@ func Load( } var files []File - for _, genFileBlock := range genFileBlocks { if genFileBlock.Context != StackContext { continue } name := genFileBlock.Label - evalctx := stack.NewEvalCtx(root, st, globals) vendorTargetDir := project.NewPath(path.Join( st.Dir.String(), path.Dir(name))) evalctx.SetFunction(stdlib.Name("vendor"), stdlib.VendorFunc(vendorTargetDir, vendorDir, vendorRequests)) - file, err := Eval(genFileBlock, evalctx.Context) + file, err := Eval(genFileBlock, evalctx, st.Dir) if err != nil { return nil, err } @@ -159,12 +156,13 @@ func Load( } // Eval the generate_file block. -func Eval(block hcl.GenFileBlock, evalctx *eval.Context) (File, error) { +func Eval(block hcl.GenFileBlock, evalctx *eval.Context, _ project.Path) (File, error) { name := block.Label - err := lets.Load(block.Lets, evalctx) - if err != nil { - return File{}, err - } + letsResolver := lets.NewResolver(block.Lets) + evalctx.SetResolver(letsResolver) + defer func() { + evalctx.DeleteResolver("let") + }() condition := true if block.Condition != nil { diff --git a/generate/genfile/genfile_lets_map_test.go b/generate/genfile/genfile_lets_map_test.go index 59c26be6e..9ff433fe9 100644 --- a/generate/genfile/genfile_lets_map_test.go +++ b/generate/genfile/genfile_lets_map_test.go @@ -8,6 +8,7 @@ import ( "github.com/terramate-io/terramate/errors" "github.com/terramate-io/terramate/hcl" + "github.com/terramate-io/terramate/hcl/eval" "github.com/terramate-io/terramate/lets" maptest "github.com/terramate-io/terramate/mapexpr/test" . "github.com/terramate-io/terramate/test/hclwrite/hclutils" @@ -145,7 +146,7 @@ func TestGenFileLetsMap(t *testing.T) { ), }, }, - wantErr: errors.E(lets.ErrEval), + wantErr: errors.E(eval.ErrEval), }, { name: "lets with map block with incorrect value", @@ -167,7 +168,7 @@ func TestGenFileLetsMap(t *testing.T) { ), }, }, - wantErr: errors.E(lets.ErrEval), + wantErr: errors.E(eval.ErrEval), }, { name: "lets with simple map without using element", diff --git a/generate/genfile/genfile_test.go b/generate/genfile/genfile_test.go index 0e6e2bcd1..db3ef0aa4 100644 --- a/generate/genfile/genfile_test.go +++ b/generate/genfile/genfile_test.go @@ -12,9 +12,13 @@ import ( "github.com/terramate-io/terramate/config" "github.com/terramate-io/terramate/errors" "github.com/terramate-io/terramate/generate/genfile" + "github.com/terramate-io/terramate/globals" "github.com/terramate-io/terramate/hcl" + "github.com/terramate-io/terramate/hcl/eval" "github.com/terramate-io/terramate/hcl/info" "github.com/terramate-io/terramate/project" + "github.com/terramate-io/terramate/runtime" + "github.com/terramate-io/terramate/stdlib" "github.com/terramate-io/terramate/test" errtest "github.com/terramate-io/terramate/test/errors" infotest "github.com/terramate-io/terramate/test/hclutils/info" @@ -105,7 +109,8 @@ stack_path_basename=${terramate.stack.path.basename} stack_id=${tm_try(terramate.stack.id, "no-id")} stack_name=${terramate.stack.name} stack_description=${terramate.stack.description} -EOT`, +EOT +`, )), }, }, @@ -139,7 +144,8 @@ stack_description= Expr("content", `< = +``` + +A `globals` block is interpreted as 0 or more statements. +Example: + +```hcl +globals "a" "b" { + c = { + d = 1 + } + z = 2 +} +``` +is interpreted as the list of statements below: + +``` +global.a.b.c.d = 1 +global.z = 2 +``` + +- The _origin reference_ + +The _origin_ of a reference is the `globals` attribute that originated the +statement. Very often it's the same _ref_ as the statement lhs _ref_ itself but +that's not always the case: + +Example: + +``` +globals "a" { + b = 1 +} +``` + +In this case, it generates the statement below: +``` +global.a.b = 1 +``` +and the _origin reference_ (the attribute that creates it) is `global.a.b`. +But have a look at the example below: + +``` +globals a { + b = { + c = { + d = 1 + } + } +} +``` +In this case it generates the statements below: + +``` +global.a.b.c.d = 1 +``` +but the _origin reference_ is `global.a.b` because that's the attribute that +originated the internal refs. + +## Properties + +- The `globals` are blocks that define the single runtime `global` object. +- There could be multiple syntactic `globals` blocks defined in the same directory +as long as their LHS _refs_ are unique. + +The configuration below: + +```hcl +globals { + a = 1 +} + +globals { + b = 2 + c = 3 +} + +globals { + z = 4 +} +``` + +is interpreted as: + +``` +# ref = value +global.a = 1 +global.b = 2 +global.c = 3 +global.z = 4 +``` + +The literal objects are also constructed as ref based assignments: + + +```hcl +# /globals.tm +globals "a" "b" { + c = { + d = 1 + } +} + +globals { + a = { + b = { + c = { + e = 1 + z = 1 + } + } + } +} +``` + +interpreted as: + +``` +global.a.b.c.e = 1 +global.a.b.c.z = 1 +global.a.b.c.d = 1 +``` + +- The labels in the `globals` block is a syntax sugar for building a nested _LHS_ +`ref`, automatically building the intermediate object keys. + +```hcl +globals a b c { + val = 1 +} +``` + +is interpreted as: + +```js +if (!global.a) { + global.a = {} +} +if (!global.a.b) { + global.a.b = {} +} +if (!global.a.b.c) { + global.a.b.c = {} +} + +global.a.b.c.val = 1 +``` + +By the same definition above, a labeled globals block without any attributes +just build the intermediate refs: + +```hcl +globals "a" "b" {} +``` + +is interpreted as: + +```js +if (!global.a) { + global.a = {} +} +if (!global.a.b) { + global.a.b = {} +} +``` + +- scopes are defined by the directory hierarchy + +Each directory defines a new global scope which inherits parent globals. + +```hcl +# /root.tm +globals { + a = 1 +} +``` +and +```hcl +# /child/globals.tm +globals { + b = global.a +} +``` +and +```hcl +# /child/grand-child/globals.tm +globals { + c = global.b +} +``` + +The code above defines the scope tree below: + +```hcl +scope = { + global = { + a = 1 + } + scopes = { + child = { + global = { + b = global.a + }, + scopes = { + "grand-child": { + global = { + c = global.b + } + } + } + } + } +} +``` + +The `grand-child` scope inherits the `child` and `root` scopes. +The `child` scope inherits the `root` scope. + +- implicit order of evaluation + +The order of evaluation of global values are implicitly defined by their dependencies and _origin ref_ size (ie `global.a` evaluates before `global.a.b`). + +Case 1: + +```hcl +# /globals.tm +globals { + b = global.a + a = 1 +} +``` + +As `global.b` depends on `global.a`, then the order of evaluation is: + +``` +global.a = 1 +global.b = global.a +``` + +When multiple global lhs _references_ target the same object tree, then +statements with smaller _origin reference_ evaluates first: + +Example: + +```hcl +# /globals.tm +globals "a" "b" { + c = { + d = 1 + } +} + +globals { + a = { + b = { + c = { + e = 1 + z = 1 + } + } + } +} +``` + +interpreted as: + +``` +global.a.b.c.e = 1 # origin reference is global.a +global.a.b.c.z = 1 # origin reference is global.a +global.a.b.c.d = 1 # origin reference is global.a.b.c +``` + + # Evaluation + + ## Single evaluation + +When evaluating a single expression, just the _referenced_ globals are +evaluated. +If a target _ref_ is not found in the current scope, then it's looked +up in the parent scope until root is reached or a _origin reference_ +is found which is a subpath of the target _ref_. + +Case 1: _ref_ is in the same scope (no dependency). + +target ref: `global.a.b` in `/child` scope + +```hcl +# /root.tm +globals { + a = { + b = 1 + } + b = 1 + c = { + b = 1 + } +} +``` + +```hcl +# /child/globals.tm +globals a { + b = 2 +} +``` + +evaluates to `2` and only the statement `global.a.b = 2` from the `/child` scope is evaluated. + +Case 2: _ref_ is in the same scope (with dependencies). + +target ref: `global.a.b` in `/child` scope + +```hcl +# /root.tm +globals "a" { + b = 1 +} + +globals { + c = 2 +} +``` + +```hcl +# /child/globals.tm +globals "a" { + b = global.c +} +``` +evaluates to `2` because _origin ref_ `global.a.b` is found in the same scope, +then `global.c` is lookup in parent scope. + +Case 3: _ref_ is in the parent scope (no deps) + +target ref: `global.a.b` in `/child` scope + +```hcl +# /root.tm +globals "a" { + b = 1 +} +``` + +```hcl +# /child/globals.tm +``` + +evaluates to `1` and only the `global.a.b = 1` from `/` is evaluated. + +Case 4: lazy evaluation + +target ref: `global.a.b` from `/child`. + +```hcl +# /root.tm +globals "a" { + b = global.c +} + +globals { + c = 1 +} +``` + +```hcl +# /child/globals.tm +globals { + c = 2 +} +``` + +It evaluates to `2` and only `global.a.b = global.c` from `/` and `global.c` from `/child` are evaluated. + +## Evaluating all globals for a stack + +The evaluation of all globals consists of evaluating the `global` object. +The process is the same as evaluating a single exp \ No newline at end of file diff --git a/globals/eval_report.go b/globals/eval_report.go deleted file mode 100644 index 2a5b47973..000000000 --- a/globals/eval_report.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2023 Terramate GmbH -// SPDX-License-Identifier: MPL-2.0 - -package globals - -import ( - "github.com/terramate-io/terramate/errors" - "github.com/terramate-io/terramate/project" - - "github.com/terramate-io/terramate/hcl/eval" -) - -type ( - // EvalReport is the report for the evaluation globals. - EvalReport struct { - // Globals are the evaluated globals. - Globals *eval.Object - - // BootstrapErr is for the case of errors happening before the evaluation. - BootstrapErr error - - // Errors is a map of errors for each global. - Errors map[GlobalPathKey]EvalError // map of GlobalPath to its EvalError. - } - - // EvalError carries the error and the expression which resulted in it. - EvalError struct { - Expr Expr - Err error - } -) - -// NewEvalReport creates a new globals evaluation report. -func NewEvalReport() EvalReport { - return EvalReport{ - Globals: eval.NewObject(eval.Info{ - DefinedAt: project.NewPath("/"), - }), - Errors: make(map[GlobalPathKey]EvalError), - } -} - -// AsError returns an error != nil if there's any error in the report. -func (r *EvalReport) AsError() error { - if len(r.Errors) == 0 && r.BootstrapErr == nil { - return nil - } - - errs := errors.L(r.BootstrapErr) - for _, e := range r.Errors { - errs.AppendWrap(ErrEval, e.Err) - } - return errs.AsError() -} diff --git a/globals/globals.go b/globals/globals.go deleted file mode 100644 index c628399b1..000000000 --- a/globals/globals.go +++ /dev/null @@ -1,581 +0,0 @@ -// Copyright 2023 Terramate GmbH -// SPDX-License-Identifier: MPL-2.0 - -package globals - -import ( - "sort" - "strings" - - hhcl "github.com/hashicorp/hcl/v2" - "github.com/hashicorp/hcl/v2/hclsyntax" - "github.com/rs/zerolog/log" - "github.com/terramate-io/terramate/config" - "github.com/terramate-io/terramate/errors" - "github.com/terramate-io/terramate/hcl" - "github.com/terramate-io/terramate/mapexpr" - - "github.com/terramate-io/terramate/hcl/eval" - "github.com/terramate-io/terramate/hcl/info" - "github.com/terramate-io/terramate/project" - "github.com/zclconf/go-cty/cty" -) - -// Errors returned when parsing and evaluating globals. -const ( - ErrEval errors.Kind = "global eval" - ErrRedefined errors.Kind = "global redefined" -) - -type ( - // Expr is an unevaluated global expression. - Expr struct { - // Origin is the filename where this expression can be found. - Origin info.Range - - // ConfigDir is the directory which loaded this expression. - ConfigDir project.Path - - // LabelPath denotes the target accessor path which the expression must - // be assigned into. - LabelPath eval.ObjectPath - - hhcl.Expression - } - - // GlobalPathKey represents a global object accessor to be used as map key. - // The reason is that slices cannot be used as map key because the equality - // operator is not defined, then this type implements a fixed size struct. - GlobalPathKey struct { - path [project.MaxGlobalLabels]string - isattr bool - numPaths int - } -) - -// Path returns the global accessor path (labels + attribute name). -func (a GlobalPathKey) Path() []string { return a.path[:a.numPaths] } - -func (a GlobalPathKey) rootname() string { - if a.numPaths == 0 { - return "" - } - return a.path[0] -} - -func (a GlobalPathKey) name() string { - return strings.Join(a.path[:a.numPaths], ".") -} - -// ForDir loads all the globals from the cfgdir. -// It will navigate the configuration tree up from the dir until it reaches root, -// loading globals and merging them appropriately. -// -// More specific globals (closer or at the current dir) have precedence over -// less specific globals (closer or at the root dir). -func ForDir(root *config.Root, cfgdir project.Path, ctx *eval.Context) EvalReport { - tree, ok := root.Lookup(cfgdir) - if !ok { - return NewEvalReport() - } - - exprs, err := LoadExprs(tree) - if err != nil { - report := NewEvalReport() - report.BootstrapErr = err - return report - } - - return exprs.Eval(ctx) -} - -// ExprSet represents a set of globals loaded from a dir. -// The origin is the path of the dir from where all expressions were loaded. -type ExprSet struct { - origin project.Path - expressions map[GlobalPathKey]Expr -} - -// HierarchicalExprs contains all loaded global expressions from multiple -// configuration directories (the key). Each configuration dir path is mapped to -// its global expressions. -type HierarchicalExprs map[project.Path]*ExprSet - -func newExprSet(origin project.Path) *ExprSet { - return &ExprSet{ - origin: origin, - expressions: map[GlobalPathKey]Expr{}, - } -} - -// LoadExprs loads from the file system all globals expressions defined for -// the given directory. It will navigate the file system from dir until it -// reaches rootdir, loading globals expressions and merging them appropriately. -// More specific globals (closer or at the dir) have precedence over less -// specific globals (closer or at the root dir). -func LoadExprs(tree *config.Tree) (HierarchicalExprs, error) { - exprs := newExprSet(tree.Dir()) - - globalsBlocks := tree.Node.Globals.AsList() - for _, block := range globalsBlocks { - if len(block.Labels) > 0 && !hclsyntax.ValidIdentifier(block.Labels[0]) { - return nil, errors.E( - hcl.ErrTerramateSchema, - "first global label must be a valid identifier but got %s", - block.Labels[0], - ) - } - - attrs := block.Attributes.SortedList() - if len(block.Labels) > 0 && len(attrs) == 0 { - expr := &hclsyntax.ObjectConsExpr{ - SrcRange: block.RawOrigins[0].Range.ToHCLRange(), - } - key := NewGlobalExtendPath(block.Labels) - exprs.expressions[key] = Expr{ - Origin: block.RawOrigins[0].Range, - ConfigDir: tree.Dir(), - LabelPath: key.Path(), - Expression: expr, - } - } - - for _, varsBlock := range block.Blocks { - varName := varsBlock.Labels[0] - if _, ok := block.Attributes[varName]; ok { - return HierarchicalExprs{}, errors.E( - ErrRedefined, - "map label %s conflicts with global.%s attribute", varName, varName) - } - - key := NewGlobalAttrPath(block.Labels, varName) - expr, err := mapexpr.NewMapExpr(varsBlock) - if err != nil { - return HierarchicalExprs{}, errors.E(err, "failed to interpret map block") - } - exprs.expressions[key] = Expr{ - Origin: varsBlock.RawOrigins[0].Range, - LabelPath: key.Path(), - Expression: expr, - } - } - - for _, attr := range attrs { - - key := NewGlobalAttrPath(block.Labels, attr.Name) - exprs.expressions[key] = Expr{ - Origin: attr.Range, - ConfigDir: tree.Dir(), - LabelPath: key.Path(), - Expression: attr.Expr, - } - } - } - - globals := HierarchicalExprs{ - tree.Dir(): exprs, - } - - parent := tree.NonEmptyGlobalsParent() - if parent == nil { - return globals, nil - } - - parentGlobals, err := LoadExprs(parent) - if err != nil { - return nil, err - } - - globals.merge(parentGlobals) - return globals, nil -} - -// SetOverride sets a custom global at the specified directory, using the given -// global path and expr. The origin is only used for debugging purposes. -func (dirExprs HierarchicalExprs) SetOverride( - dir project.Path, - path GlobalPathKey, - expr hhcl.Expression, - origin info.Range, -) { - exprSet, ok := dirExprs[dir] - if !ok { - exprSet = newExprSet(origin.Path()) - dirExprs[dir] = exprSet - } - exprSet.expressions[path] = Expr{ - Origin: origin, - ConfigDir: dir, - LabelPath: path.Path(), - Expression: expr, - } -} - -// Returns a sorted loaded exprs, sorting it by config dir path. -// The loaded expressions are sorted by the config dir path -// from smaller (root) to more specific (stack). Eg: -// - / -// - /dir -// - /dir/stack -func (dirExprs HierarchicalExprs) sort() []*ExprSet { - cfgdirs := []project.Path{} - for cfgdir := range dirExprs { - cfgdirs = append(cfgdirs, cfgdir) - } - - sort.SliceStable(cfgdirs, func(i, j int) bool { - return len(cfgdirs[i].String()) < len(cfgdirs[j].String()) - }) - - res := []*ExprSet{} - for _, cfgdir := range cfgdirs { - res = append(res, dirExprs[cfgdir]) - } - return res -} - -// Returns the expressions access path sorted from the smallest to -// the biggest path. -// - global.a -// - global.a.b -// - global.a.b.c -func (dirExprs ExprSet) sort() []GlobalPathKey { - res := []GlobalPathKey{} - for globalPath := range dirExprs.expressions { - res = append(res, globalPath) - } - - sort.SliceStable(res, func(i, j int) bool { - return len(res[i].Path()) < len(res[j].Path()) - }) - - return res -} - -// Eval evaluates all global expressions and returns an EvalReport. -func (dirExprs HierarchicalExprs) Eval(ctx *eval.Context) EvalReport { - logger := log.With(). - Str("action", "HierarchicalExprs.Eval()"). - Logger() - - report := NewEvalReport() - globals := report.Globals - pendingExprsErrs := map[GlobalPathKey]*errors.List{} - - sortedLoadedExprs := dirExprs.sort() - pendingExprs := map[GlobalPathKey]Expr{} - - // Here we will override values, but since - // we ordered by config dir the more specific global expressions - // will override the parent ones. - for _, xp := range sortedLoadedExprs { - for k, v := range xp.expressions { - pendingExprs[k] = v - } - } - - // Here we will sort each set of globals from each dir independently - // So the final iteration order is parent first then child, and - // for each given config dir it is ordered by the length of the global path. - // So we guarantee that independent of the expression accessors length we always - // process parent expressions first, then the child ones, until reaching the stack. - type globalAccessors struct { - origin project.Path - accessors []GlobalPathKey - } - sortedGlobalAccessors := []globalAccessors{} - for _, exprset := range sortedLoadedExprs { - // for now we are allowing repeated access paths for different - // directories, should not affect results since pendingExprs already - // has the correct expression anyway. - sortedGlobalAccessors = append(sortedGlobalAccessors, globalAccessors{ - origin: exprset.origin, - accessors: exprset.sort(), - }) - } - - if !ctx.HasNamespace("global") { - ctx.SetNamespace("global", map[string]cty.Value{}) - } - - for len(pendingExprs) > 0 { - amountEvaluated := 0 - - for _, sortedGlobals := range sortedGlobalAccessors { - - pendingExpression: - for _, accessor := range sortedGlobals.accessors { - expr, ok := pendingExprs[accessor] - if !ok { - // Ignoring already evaluated expression - continue - } - - logger := logger.With(). - Stringer("origin", sortedGlobals.origin). - Strs("global", accessor.Path()). - Logger() - - traversal, diags := hhcl.AbsTraversalForExpr(expr.Expression) - if !diags.HasErrors() && len(traversal) == 1 && traversal.RootName() == "unset" { - if _, ok := globals.GetKeyPath(accessor.Path()); ok { - err := globals.DeleteAt(accessor.Path()) - if err != nil { - panic(errors.E(errors.ErrInternal, err)) - } - } - - amountEvaluated++ - delete(pendingExprs, accessor) - delete(pendingExprsErrs, accessor) - } - - pendingExprsErrs[accessor] = errors.L() - for _, namespace := range expr.Variables() { - if !ctx.HasNamespace(namespace.RootName()) { - pendingExprsErrs[accessor].Append(errors.E( - ErrEval, - namespace.SourceRange(), - "unknown variable namespace: %s", namespace.RootName(), - )) - - continue - } - - if namespace.RootName() != "global" || len(namespace) == 1 { - continue - } - - var varPaths []string - - for _, ns := range namespace[1:] { - switch attr := ns.(type) { - case hhcl.TraverseAttr: - varPaths = append(varPaths, attr.Name) - case hhcl.TraverseSplat: - // ignore - case hhcl.TraverseIndex: - if !attr.Key.Type().Equals(cty.String) { - break - } - - varPaths = append(varPaths, attr.Key.AsString()) - default: - panic(errors.E( - errors.ErrInternal, - "unexpected type of traversal - this is a BUG: %T", - attr, - )) - } - } - - min := func(a, b int) int { - if a < b { - return a - } - return b - } - - for accessPath := range pendingExprs { - found := true - accessPathPaths := accessPath.Path() - for i := min(len(accessPathPaths), len(varPaths)) - 1; i >= 0; i-- { - if accessPathPaths[i] != varPaths[i] { - found = false - break - } - } - if found { - continue pendingExpression - } - } - } - - // also checks if any part of the accessor is pending. - // Example: - // globals a { - // val = tm_try(global.pending, 1) - // } - // - // globals a b { - // c = 1 - // } - // - // The first global block would evaluate before but as it has - // pending variables, then we need to postpone the second block - // as well. - if len(accessor.Path()) > 1 { - for size := accessor.numPaths; size >= 1; size-- { - base := accessor.path[0 : size-1] - attr := accessor.path[size-1] - v, isPending := pendingExprs[newGlobalPath(base, attr)] - - if isPending && - // is not this global path - !isSameObjectPath(v.LabelPath, accessor.Path()) && - // dependent comes from same or higher level - strings.HasPrefix(sortedGlobals.origin.String(), v.ConfigDir.String()) { - continue pendingExpression - } - - } - } - - if err := pendingExprsErrs[accessor].AsError(); err != nil { - continue - } - - // This catches a schema error that cannot be detected at the parser. - // When a nested object is defined either by literal or funcalls, - // it can't be detected at the parser. - oldValue, hasOldValue := globals.GetKeyPath(accessor.Path()) - if hasOldValue && - accessor.isattr && - oldValue.Info().DefinedAt.Dir().String() == expr.Origin.Path().Dir().String() { - pendingExprsErrs[accessor].Append( - errors.E(hcl.ErrTerramateSchema, expr.Range(), - "global.%s attribute redefined: previously defined at %s", - accessor.name(), oldValue.Info().DefinedAt.String())) - - continue - } - - // This is to avoid setting a label defined extension on the child - // and later overwriting that with an object definition on the parent - - val, err := ctx.Eval(expr) - if err != nil { - pendingExprsErrs[accessor].Append(errors.E( - ErrEval, err, "global.%s (%t)", accessor.rootname(), accessor.isattr)) - continue - } - if hasOldValue && oldValue.IsObject() && !accessor.isattr { - // all the `attr = expr` inside global blocks become an entry - // in the globalExprs map but we have the special case that - // an empty globals block with labels must implicitly create - // the label defined object... - // then as it does not define any expression, an implicit - // expression for an empty object block is added to the map. - // This special entry sets the key accessor.isattr = false - // which means this expression doesn't come from an attribute. - - // this `if` happens for the general case, which we must not - // set the fake expression when extending an existing object. - logger.Trace().Msg("ignoring implicitly created empty global") - - } else { - - err := setGlobal(globals, accessor, eval.NewValue(val, - eval.Info{ - DefinedAt: expr.Origin.Path(), - Dir: sortedGlobals.origin, - }, - )) - - if err != nil { - pendingExprsErrs[accessor].Append(errors.E(err, "setting global")) - continue - } - } - - amountEvaluated++ - - delete(pendingExprs, accessor) - delete(pendingExprsErrs, accessor) - - ctx.SetNamespace("global", globals.AsValueMap()) - } - } - - if amountEvaluated == 0 { - break - } - } - - for accessor, expr := range pendingExprs { - err := pendingExprsErrs[accessor].AsError() - if err == nil { - err = errors.E(expr.Range(), "undefined global.%s", accessor.rootname()) - } - report.Errors[accessor] = EvalError{ - Expr: expr, - Err: errors.E(ErrEval, err), - } - } - - return report -} - -func (dirExprs HierarchicalExprs) merge(other HierarchicalExprs) { - for k, v := range other { - if _, ok := dirExprs[k]; !ok { - dirExprs[k] = v - } else { - panic(errors.E( - errors.ErrInternal, - "cant merge duplicated configuration %q", - k)) - } - } -} - -func isSameObjectPath(a, b eval.ObjectPath) bool { - if len(a) != len(b) { - return false - } - for i, v := range a { - if b[i] != v { - return false - } - } - return true -} - -// setGlobal sets the global accordingly to the hierarchical rules. -func setGlobal(globals *eval.Object, accessor GlobalPathKey, newVal eval.Value) error { - oldVal, hasOldVal := globals.GetKeyPath(accessor.Path()) - if !hasOldVal { - return globals.SetAt(accessor.Path(), newVal) - } - - newConfigDir := newVal.Info().Dir - oldConfigDir := oldVal.Info().Dir - - newDefinedDir := newVal.Info().DefinedAt.Dir() - oldDefinedDir := oldVal.Info().DefinedAt.Dir() - - if !newConfigDir.HasPrefix(oldConfigDir.String()) { - panic(errors.E(errors.ErrInternal, - "unexpected globals behavior: new value from dir %s and defined at %s: "+ - "old value from dir %s and defined at %s", - newConfigDir, newDefinedDir, - oldConfigDir, oldDefinedDir)) - } - - // newval comes from lower layer. - - return globals.SetAt(accessor.Path(), newVal) - -} - -func newGlobalPath(basepath []string, name string) GlobalPathKey { - accessor := GlobalPathKey{} - accessor.numPaths = len(basepath) - copy(accessor.path[:], basepath) - if name != "" { - accessor.path[len(basepath)] = name - accessor.numPaths++ - accessor.isattr = true - } - return accessor -} - -// NewGlobalAttrPath creates a new global path key for an attribute. -func NewGlobalAttrPath(basepath []string, name string) GlobalPathKey { - return newGlobalPath(basepath, name) -} - -// NewGlobalExtendPath creates a new global path key for extension purposes only. -func NewGlobalExtendPath(path []string) GlobalPathKey { - return newGlobalPath(path, "") -} diff --git a/globals/globals_map_test.go b/globals/globals_map_test.go index b7706523b..8f46af79c 100644 --- a/globals/globals_map_test.go +++ b/globals/globals_map_test.go @@ -9,6 +9,7 @@ import ( "github.com/terramate-io/terramate/errors" "github.com/terramate-io/terramate/globals" "github.com/terramate-io/terramate/hcl" + "github.com/terramate-io/terramate/hcl/eval" maptest "github.com/terramate-io/terramate/mapexpr/test" "github.com/terramate-io/terramate/test/hclwrite" . "github.com/terramate-io/terramate/test/hclwrite/hclutils" @@ -76,7 +77,7 @@ func TestGlobalsMap(t *testing.T) { ), }, }, - wantErr: errors.E(globals.ErrEval), + wantErr: errors.E(eval.ErrEval), }, { name: "invalid globals.map value", @@ -94,7 +95,7 @@ func TestGlobalsMap(t *testing.T) { ), }, }, - wantErr: errors.E(globals.ErrEval), + wantErr: errors.E(eval.ErrEval), }, { name: "simple globals.map without using element", diff --git a/globals/globals_resolver.go b/globals/globals_resolver.go new file mode 100644 index 000000000..d77845c3e --- /dev/null +++ b/globals/globals_resolver.go @@ -0,0 +1,195 @@ +// Copyright 2023 Terramate GmbH +// SPDX-License-Identifier: MPL-2.0 + +package globals + +import ( + "sort" + + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/terramate-io/terramate/config" + "github.com/terramate-io/terramate/errors" + "github.com/terramate-io/terramate/hcl" + "github.com/terramate-io/terramate/hcl/eval" + "github.com/terramate-io/terramate/mapexpr" + "github.com/terramate-io/terramate/project" + "github.com/zclconf/go-cty/cty" +) + +// ErrRedefined indicates the global is redefined. +const ErrRedefined errors.Kind = "global redefined" + +const nsName = "global" + +// Resolver is the globals resolver. +type Resolver struct { + root *config.Root + override map[eval.RefStr]eval.Stmt + // Scopes is a cache of scoped statements. + Scopes map[project.Path]cacheData +} + +type cacheData struct { + tree *config.Tree + stmts eval.Stmts +} + +// NewResolver creates a new globals resolver. +func NewResolver(root *config.Root, overrides ...eval.Stmt) *Resolver { + r := &Resolver{ + root: root, + Scopes: make(map[project.Path]cacheData), + override: make(map[eval.RefStr]eval.Stmt), + } + + for _, override := range overrides { + r.override[override.LHS.AsKey()] = override + } + + return r +} + +// Name of the variable. +func (*Resolver) Name() string { return nsName } + +// Prevalue is the predeclared globals. +func (r *Resolver) Prevalue() cty.Value { + return cty.EmptyObjectVal +} + +// LookupRef lookups global references. +func (r *Resolver) LookupRef(scope project.Path, ref eval.Ref) ([]eval.Stmts, error) { + return r.lookupStmtsAt(ref, scope, map[eval.RefStr]eval.Ref{}) +} + +func (r *Resolver) loadStmtsAt(scope project.Path) (eval.Stmts, *config.Tree, error) { + cache, ok := r.Scopes[scope] + if ok { + return cache.stmts, cache.tree, nil + } + + tree, ok := r.root.Lookup(scope) + if !ok { + panic(scope) + } + + overrideMap := map[eval.RefStr]struct{}{} + + var stmts eval.Stmts + + for _, override := range r.override { + stmts = append(stmts, override) + overrideMap[override.Origin.AsKey()] = struct{}{} + } + + for _, block := range tree.Node.Globals.AsList() { + if len(block.Labels) > 0 && !hclsyntax.ValidIdentifier(block.Labels[0]) { + return nil, nil, errors.E( + hcl.ErrTerramateSchema, + "first global label must be a valid identifier but got %s", + block.Labels[0], + ) + } + + attrs := block.Attributes.SortedList() + if len(block.Labels) > 0 { + scope := tree.Dir() + stmts = append(stmts, eval.NewExtendStmt( + eval.NewRef(nsName, block.Labels...), + eval.NewInfo(scope, block.RawOrigins[0].Range), + )) + } + + for _, varsBlock := range block.Blocks { + varName := varsBlock.Labels[0] + if _, ok := block.Attributes[varName]; ok { + return nil, nil, errors.E( + ErrRedefined, + "map label %s conflicts with global.%s attribute", varName, varName) + } + + origin := eval.NewRef(nsName, block.Labels...) + origin.Path = append(origin.Path, varName) + + if _, ok := overrideMap[origin.AsKey()]; ok { + continue + } + + expr, err := mapexpr.NewMapExpr(varsBlock) + if err != nil { + return nil, nil, errors.E(err, "failed to interpret map block") + } + + info := eval.NewInfo(tree.Dir(), varsBlock.RawOrigins[0].Range) + blockStmts, err := eval.StmtsOfExpr(info, origin, origin.Path, expr) + if err != nil { + return nil, nil, err + } + stmts = append(stmts, blockStmts...) + } + + for _, attr := range attrs { + origin := eval.NewRef(nsName, block.Labels...) + origin.Path = append(origin.Path, attr.Name) + + if _, ok := overrideMap[origin.AsKey()]; ok { + continue + } + + info := eval.NewInfo(tree.Dir(), attr.Range) + blockStmts, err := eval.StmtsOfExpr(info, origin, origin.Path, attr.Expr) + if err != nil { + return nil, nil, err + } + stmts = append(stmts, blockStmts...) + } + } + + // bigger refs -> smaller refs + sort.Slice(stmts, func(i, j int) bool { + if len(stmts[i].Origin.Path) != len(stmts[j].Origin.Path) { + return len(stmts[i].Origin.Path) > len(stmts[j].Origin.Path) + } + return len(stmts[i].LHS.Path) > len(stmts[j].LHS.Path) + }) + + r.Scopes[tree.Dir()] = cacheData{ + tree: tree, + stmts: stmts, + } + + return stmts, tree, nil +} + +func (r *Resolver) lookupStmtsAt(ref eval.Ref, scope project.Path, origins map[eval.RefStr]eval.Ref) ([]eval.Stmts, error) { + ret := make([]eval.Stmts, 0, 10) + stmts, tree, err := r.loadStmtsAt(scope) + if err != nil { + return nil, err + } + + filtered, found := stmts.SelectBy(ref, origins) + for _, s := range filtered { + if !s.Special { + origins[s.Origin.AsKey()] = s.Origin + } + } + + ret = append(ret, filtered) + + if found || tree.Parent == nil { + return ret, nil + } + + parent := tree.NonEmptyGlobalsParent() + if parent == nil { + return ret, nil + } + + parentStmts, err := r.lookupStmtsAt(ref, parent.Dir(), origins) + if err != nil { + return nil, err + } + ret = append(ret, parentStmts...) + return ret, nil +} diff --git a/globals/globals_test.go b/globals/globals_test.go index 0d9f136b5..f5afa698c 100644 --- a/globals/globals_test.go +++ b/globals/globals_test.go @@ -7,14 +7,19 @@ import ( "path/filepath" "testing" + "github.com/google/go-cmp/cmp" + hhclwrite "github.com/hashicorp/hcl/v2/hclwrite" "github.com/madlambda/spells/assert" "github.com/rs/zerolog" "github.com/terramate-io/terramate/config" "github.com/terramate-io/terramate/errors" "github.com/terramate-io/terramate/globals" "github.com/terramate-io/terramate/hcl" + "github.com/terramate-io/terramate/hcl/ast" "github.com/terramate-io/terramate/hcl/eval" + "github.com/terramate-io/terramate/runtime" "github.com/terramate-io/terramate/stack" + "github.com/terramate-io/terramate/stdlib" "github.com/terramate-io/terramate/test" errtest "github.com/terramate-io/terramate/test/errors" @@ -22,7 +27,6 @@ import ( "github.com/terramate-io/terramate/test/hclwrite" . "github.com/terramate-io/terramate/test/hclwrite/hclutils" "github.com/terramate-io/terramate/test/sandbox" - "github.com/zclconf/go-cty-debug/ctydebug" ) type ( @@ -305,7 +309,7 @@ func TestLoadGlobals(t *testing.T) { }, want: map[string]*hclwrite.Block{ "/stacks/stack-1": Globals( - EvalExpr(t, "stacks_list", `tolist(["/stacks/stack-1", "/stacks/stack-2"])`), + EvalExpr(t, "stacks_list", `["/stacks/stack-1", "/stacks/stack-2"]`), Str("stack_path_abs", "/stacks/stack-1"), Str("stack_path_rel", "stacks/stack-1"), Str("stack_path_to_root", "../.."), @@ -313,10 +317,10 @@ func TestLoadGlobals(t *testing.T) { Str("stack_id", "no-id"), Str("stack_name", "stack-1"), Str("stack_description", ""), - EvalExpr(t, "stack_tags", "tolist([])"), + EvalExpr(t, "stack_tags", "[]"), ), "/stacks/stack-2": Globals( - EvalExpr(t, "stacks_list", `tolist(["/stacks/stack-1", "/stacks/stack-2"])`), + EvalExpr(t, "stacks_list", `["/stacks/stack-1", "/stacks/stack-2"]`), Str("stack_path_abs", "/stacks/stack-2"), Str("stack_path_rel", "stacks/stack-2"), Str("stack_path_to_root", "../.."), @@ -324,7 +328,7 @@ func TestLoadGlobals(t *testing.T) { Str("stack_id", "stack-2-id"), Str("stack_name", "stack-2"), Str("stack_description", "someDescriptionStack2"), - EvalExpr(t, "stack_tags", `tolist(["tag1", "tag2", "tag3"])`), + EvalExpr(t, "stack_tags", `["tag1", "tag2", "tag3"]`), ), }, }, @@ -1647,10 +1651,10 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(eval.ErrCannotExtendObject), + wantErr: errors.E(eval.ErrEval), }, { - name: "extending parent list fails", + name: "extending parent list works -- override", layout: []string{ "s:stack", }, @@ -1669,7 +1673,13 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(eval.ErrCannotExtendObject), + want: map[string]*hclwrite.Block{ + "/stack": Globals( + Expr("lst", `{ + other = [] + }`), + ), + }, }, { name: "extending list from object fails", @@ -1691,7 +1701,7 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(eval.ErrCannotExtendObject), + wantErr: errors.E(eval.ErrEval), }, { name: "extending non-objects fails", @@ -1712,7 +1722,7 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(eval.ErrCannotExtendObject), + wantErr: errors.E(eval.ErrEval), }, { name: "extending nested literal object", @@ -1770,7 +1780,7 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(hcl.ErrTerramateSchema), + wantErr: errors.E(eval.ErrRedefined), }, { name: "funcall object with a conflict", @@ -1795,7 +1805,7 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(hcl.ErrTerramateSchema), + wantErr: errors.E(eval.ErrRedefined), }, { name: "globals hierarchically defined with different filenames", @@ -2070,7 +2080,7 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(globals.ErrEval), + wantErr: errors.E(eval.ErrEval), }, { name: "tm_vendor is not available on globals", @@ -2083,7 +2093,7 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(globals.ErrEval), + wantErr: errors.E(eval.ErrEval), }, { name: "global interpolating multiple lists fails", @@ -2097,7 +2107,7 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(globals.ErrEval), + wantErr: errors.E(eval.ErrEval), }, { name: "global interpolating list with space fails", @@ -2111,7 +2121,7 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(globals.ErrEval), + wantErr: errors.E(eval.ErrEval), }, { // This tests double check that interpolation on a single object/map @@ -2147,7 +2157,7 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(globals.ErrEval), + wantErr: errors.E(eval.ErrEval), }, { name: "global interpolating object with space fails", @@ -2161,7 +2171,7 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(globals.ErrEval), + wantErr: errors.E(eval.ErrEval), }, { name: "global interpolating undefined reference fails", @@ -2174,7 +2184,7 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(globals.ErrEval), + wantErr: errors.E(eval.ErrEval), }, { // This tests double check that interpolation on a single number @@ -2264,7 +2274,7 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - { + /*{ name: "global reference with try on root config and value defined on stack", layout: []string{"s:stack"}, configs: []hclconfig{ @@ -2289,7 +2299,7 @@ func TestLoadGlobals(t *testing.T) { EvalExpr(t, "team_def_try", `{ name = "awesome" }`), ), }, - }, + },*/ { name: "globals cant have blocks inside", layout: []string{"s:stack"}, @@ -2317,7 +2327,7 @@ func TestLoadGlobals(t *testing.T) { add: Globals(Str("stack", "whatever")), }, }, - wantErr: errors.E(globals.ErrEval), + wantErr: errors.E(eval.ErrEval), }, { name: "global undefined reference on stack", @@ -2328,7 +2338,7 @@ func TestLoadGlobals(t *testing.T) { add: Globals(Expr("field", "global.unknown")), }, }, - wantErr: errors.E(globals.ErrEval), + wantErr: errors.E(eval.ErrEval), }, { name: "global undefined references mixed on stack", @@ -2344,7 +2354,7 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(globals.ErrEval), + wantErr: errors.E(eval.ErrEval), }, { name: "global cyclic reference on stack", @@ -2359,7 +2369,7 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(globals.ErrEval), + wantErr: errors.E(eval.ErrCycle), }, { name: "global cyclic references across hierarchy", @@ -2378,7 +2388,7 @@ func TestLoadGlobals(t *testing.T) { add: Globals(Expr("c", "global.a")), }, }, - wantErr: errors.E(globals.ErrEval), + wantErr: errors.E(eval.ErrCycle), }, { name: "global redefined on different file on stack", @@ -2721,7 +2731,7 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(globals.ErrEval), + wantErr: errors.E(eval.ErrEval), }, { name: "operating unset and other type fails", @@ -2734,10 +2744,10 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(globals.ErrEval), + wantErr: errors.E(eval.ErrEval), }, { - name: "interpolating unset fails", + name: "interpolating unset", layout: []string{"s:stack"}, configs: []hclconfig{ { @@ -2747,7 +2757,9 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(globals.ErrEval), + want: map[string]*hclwrite.Block{ + "/stack": Globals(), + }, }, { name: "unset on list fails", @@ -2760,7 +2772,11 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(globals.ErrEval), + want: map[string]*hclwrite.Block{ + "/stack": Globals( + EvalExpr(t, "a", `[]`), + ), + }, }, { name: "unset on obj fails", @@ -2773,7 +2789,11 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(globals.ErrEval), + want: map[string]*hclwrite.Block{ + "/stack": Globals( + //EvalExpr(t, "a", `{}`), + ), + }, }, { name: "unset on extended global", @@ -2835,7 +2855,7 @@ func TestLoadGlobals(t *testing.T) { want: map[string]*hclwrite.Block{ "/stack": Globals( Bool("a", true), - EvalExpr(t, "val", `{}`), + Expr("val", `{}`), ), }, }, @@ -2851,7 +2871,12 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(globals.ErrEval), + want: map[string]*hclwrite.Block{ + "/stack": Globals( + Bool("a", true), + Expr("val", `[local.a]`), + ), + }, }, { name: "tm_try with only root traversal", @@ -2864,11 +2889,7 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - want: map[string]*hclwrite.Block{ - "/stack": Globals( - EvalExpr(t, "val", `{}`), - ), - }, + wantErr: errors.E(eval.ErrCycle), }, { name: "globals.map label conflicts with global name", @@ -2905,7 +2926,7 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(globals.ErrEval), + wantErr: errors.E(eval.ErrEval), }, { name: "invalid globals.map value", @@ -2923,7 +2944,7 @@ func TestLoadGlobals(t *testing.T) { ), }, }, - wantErr: errors.E(globals.ErrEval), + wantErr: errors.E(eval.ErrEval), }, { name: "simple globals.map without using element", @@ -4066,7 +4087,7 @@ func TestLoadGlobalsErrors(t *testing.T) { test.AppendFile(t, path, config.DefaultFilename, c.body) } - cfg, err := config.LoadTree(s.RootDir(), s.RootDir()) + cfg, err := config.LoadRoot(s.RootDir()) // TODO(i4k): this better not be tested here. if errors.IsKind(tcase.want, hcl.ErrHCLSyntax) { errtest.Assert(t, err, tcase.want) @@ -4076,11 +4097,17 @@ func TestLoadGlobalsErrors(t *testing.T) { return } - stacks, err := config.LoadAllStacks(cfg) + stacks, err := config.LoadAllStacks(cfg.Tree()) assert.NoError(t, err) for _, elem := range stacks { - report := globals.ForStack(s.Config(), elem.Stack) - errtest.Assert(t, report.AsError(), tcase.want) + tree, _ := cfg.Lookup(elem.Dir()) + evalctx := eval.New(elem.Dir(), globals.NewResolver(cfg)) + evalctx.SetFunctions(stdlib.Functions(evalctx, tree.HostDir())) + expr, err := ast.ParseExpression(`global`, `test.hcl`) + assert.NoError(t, err) + + _, err = evalctx.Eval(expr) + errtest.Assert(t, err, tcase.want) } }) } @@ -4102,13 +4129,13 @@ func testGlobals(t *testing.T, tcase testcase) { wantGlobals := tcase.want - cfg, err := config.LoadTree(s.RootDir(), s.RootDir()) + root, err := config.LoadRoot(s.RootDir()) if err != nil { errtest.Assert(t, err, tcase.wantErr) return } - stackEntries, err := stack.List(cfg) + stackEntries, err := stack.List(root.Tree()) assert.NoError(t, err) var stacks config.List[*config.SortableStack] @@ -4118,8 +4145,19 @@ func testGlobals(t *testing.T, tcase testcase) { t.Logf("loading globals for stack: %s", st.Dir) - gotReport := globals.ForStack(s.Config(), st) - errtest.Assert(t, gotReport.AsError(), tcase.wantErr) + tree, _ := root.Lookup(st.Dir) + + // TODO(i4k): optimize by moving resolver out of the loop + evalctx := eval.New(st.Dir, + runtime.NewResolver(root, st), + globals.NewResolver(root), + ) + + evalctx.SetFunctions(stdlib.Functions(evalctx, tree.HostDir())) + allExpr, err := ast.ParseExpression(`global`, "test.hcl") + assert.NoError(t, err) + got, err := evalctx.Eval(allExpr) + errtest.Assert(t, err, tcase.wantErr) if tcase.wantErr != nil { continue } @@ -4133,12 +4171,11 @@ func testGlobals(t *testing.T, tcase testcase) { // Could have one type for globals configs and another type // for wanted evaluated globals, but that would make // globals building more annoying (two sets of functions). - if want.HasExpressions() { - t.Errorf("wanted globals definition contains expressions, they should be defined only by evaluated values") - t.Fatalf("wanted globals definition:\n%s\n", want) - } + //if want.HasExpressions() { + // t.Fatal("wanted globals definition contains expressions, they should be defined only by evaluated values") + // t.Errorf("wanted globals definition:\n%s\n", want) + //} - got := gotReport.Globals gotAttrs := got.AsValueMap() wantAttrs := want.AttributesValues() @@ -4152,10 +4189,12 @@ func testGlobals(t *testing.T, tcase testcase) { t.Errorf("wanted global.%s is missing", name) continue } - if diff := ctydebug.DiffValues(wantVal, gotVal); diff != "" { + wantStr := string(hhclwrite.Format(ast.TokensForValue(wantVal).Bytes())) + gotStr := string(hhclwrite.Format(ast.TokensForValue(gotVal).Bytes())) + if diff := cmp.Diff(wantStr, gotStr); diff != "" { t.Errorf("global.%s doesn't match expectation", name) - t.Errorf("want: %s", ctydebug.ValueString(wantVal)) - t.Errorf("got: %s", ctydebug.ValueString(gotVal)) + t.Errorf("want: %s", wantStr) + t.Errorf("got: %s", gotStr) t.Errorf("diff:\n%s", diff) } } diff --git a/globals/globals_v2_test.go b/globals/globals_v2_test.go new file mode 100644 index 000000000..1e4e5ae52 --- /dev/null +++ b/globals/globals_v2_test.go @@ -0,0 +1,496 @@ +// Copyright 2023 Terramate GmbH +// SPDX-License-Identifier: MPL-2.0 + +package globals_test + +import ( + "path/filepath" + "testing" + + "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + hhclwrite "github.com/hashicorp/hcl/v2/hclwrite" + "github.com/madlambda/spells/assert" + "github.com/terramate-io/terramate/config" + "github.com/terramate-io/terramate/errors" + + "github.com/rs/zerolog" + "github.com/terramate-io/terramate/globals" + "github.com/terramate-io/terramate/hcl/ast" + "github.com/terramate-io/terramate/hcl/eval" + "github.com/terramate-io/terramate/project" + "github.com/terramate-io/terramate/stdlib" + "github.com/terramate-io/terramate/test" + errtest "github.com/terramate-io/terramate/test/errors" + "github.com/terramate-io/terramate/test/hclwrite" + . "github.com/terramate-io/terramate/test/hclwrite/hclutils" + "github.com/terramate-io/terramate/test/sandbox" +) + +func TestGlobals2(t *testing.T) { + type ( + hclconfig struct { + path string + filename string + add *hclwrite.Block + } + testcase struct { + name string + layout []string + configs []hclconfig + expr string + evalDir string + want string + wantErr error + } + ) + + for _, tc := range []testcase{ + { + name: "no globals", + expr: "1", + evalDir: "/", + want: "1", + }, + { + name: "no globals but with funcalls", + expr: `tm_upper("terramate is fun")`, + evalDir: "/", + want: `"TERRAMATE IS FUN"`, + }, + { + name: "empty labeled globals creates objects", + configs: []hclconfig{ + { + path: "/", + add: Globals( + Labels("obj"), + ), + }, + }, + expr: `global.obj`, + evalDir: "/", + want: `{}`, + }, + { + name: "empty labeled globals creates objects - multiple labels", + configs: []hclconfig{ + { + path: "/", + add: Globals( + Labels("obj", "a", "b", "c"), + ), + }, + }, + expr: `global.obj`, + evalDir: "/", + want: `{ + a = { + b = { + c = {} + } + } + }`, + }, + { + name: "single stack with a single global", + layout: []string{"s:stack"}, + configs: []hclconfig{ + { + path: "/stack", + add: Globals( + Str("a", "string"), + ), + }, + }, + evalDir: "/stack", + expr: `global.a`, + want: `"string"`, + }, + { + name: "extending global in the same scope", + layout: []string{"s:stack"}, + configs: []hclconfig{ + { + path: "/stack", + add: Doc( + Globals( + Labels("obj"), + Str("a", "string"), + ), + Globals( + Labels("obj"), + Str("b", "string"), + ), + ), + }, + }, + evalDir: "/stack", + expr: `global.obj`, + want: `{ + a = "string" + b = "string" + }`, + }, + { + name: "extended globals outside the target ref range are ignored", + layout: []string{"s:stack"}, + configs: []hclconfig{ + { + path: "/stack", + add: Doc( + Globals( + Labels("obj"), + Str("a", "string"), + ), + Globals( + Labels("obj"), + Expr("fail", "crash()"), + ), + ), + }, + }, + evalDir: "/stack", + expr: `global.obj.a`, + want: `"string"`, + }, + { + name: "not referenced globals are not evaluated", + layout: []string{"s:stack"}, + configs: []hclconfig{ + { + path: "/stack", + add: Globals( + Str("a", "value"), + Expr("fail_if_evaluated", `crash()`), + ), + }, + }, + evalDir: "/stack", + expr: `global.a`, + want: `"value"`, + }, + { + name: "single stack with target global depending on same scoped global", + layout: []string{"s:stack"}, + configs: []hclconfig{ + { + path: "/stack", + add: Globals( + Expr("a", `global.b`), + Expr("b", `tm_upper("terramate is fun")`), + ), + }, + }, + evalDir: "/stack", + expr: `global.a`, + want: `"TERRAMATE IS FUN"`, + }, + { + name: "extending parent globals", + layout: []string{"s:stack"}, + configs: []hclconfig{ + { + path: "/", + add: Globals( + Labels("obj"), + Expr("a", `"test"`), + Expr("b", `tm_upper("test")`), + ), + }, + { + path: "/stack", + add: Globals( + Labels("obj", "c"), + Expr("a", `"c.a"`), + ), + }, + }, + evalDir: "/stack", + expr: `global.obj`, + want: `{ + a = "test" + b = "TEST" + c = { + a = "c.a" + } + }`, + }, + { + name: "extending same key from parent globals", + layout: []string{"s:stack"}, + configs: []hclconfig{ + { + path: "/", + add: Globals( + Labels("obj"), + Expr("a", `"test"`), + Expr("b", `tm_upper("test")`), + ), + }, + { + path: "/stack", + add: Globals( + Labels("obj"), + Expr("a", `"stackval"`), + ), + }, + }, + evalDir: "/stack", + expr: `global.obj.a`, + want: `"stackval"`, + }, + { + name: "extending same key from parent globals but targeting root object", + layout: []string{"s:stack"}, + configs: []hclconfig{ + { + path: "/", + add: Globals( + Labels("obj"), + Expr("a", `"test"`), + Expr("b", `tm_upper("test")`), + ), + }, + { + path: "/stack", + add: Globals( + Labels("obj"), + Expr("a", `"stackval"`), + ), + }, + }, + evalDir: "/stack", + expr: `global.obj`, + want: `{ + a = "stackval" + b = "TEST" + }`, + }, + { + name: "extending parent globals but referencing child defined part -- should not descend into parent", + layout: []string{"s:stack"}, + configs: []hclconfig{ + { + path: "/", + add: Globals( + Labels("obj"), + Expr("a", `crash()`), + ), + }, + { + path: "/stack", + add: Globals( + Labels("obj", "c"), + Expr("a", `"c.a"`), + ), + }, + }, + evalDir: "/stack", + expr: `global.obj.c`, + want: `{ + a = "c.a" + }`, + }, + { + name: "single stack with target global depending on multiple same scoped globals", + layout: []string{"s:stack"}, + configs: []hclconfig{ + { + path: "/stack", + add: Globals( + Expr("cfg", `{ + name = global.name + domain = global.domain + }`), + Expr("name", `tm_upper("terramate")`), + Str("domain", `terramate.io`), + ), + }, + }, + evalDir: "/stack", + expr: `global.cfg`, + want: `{ + domain = "terramate.io" + name = "TERRAMATE" + }`, + }, + { + name: "globals with 2 dependency hops", + layout: []string{"s:stack"}, + configs: []hclconfig{ + { + path: "/stack", + add: Globals( + Expr("cfg", `{ + name = global.indirect + }`), + Expr("indirect", `tm_upper(global.name)`), + Str("name", `terramate`), + ), + }, + }, + evalDir: "/stack", + expr: `global.cfg`, + want: `{ + name = "TERRAMATE" + }`, + }, + { + name: "globals with 5 dependency hops", + layout: []string{"s:stack"}, + configs: []hclconfig{ + { + path: "/stack", + add: Globals( + Expr("obj", `{ + val = global.a1 + }`), + Expr("a1", `tm_upper(global.a2)`), + Expr("a2", `tm_lower(global.a3)`), + Expr("a3", `tm_upper(global.a4)`), + Expr("a4", `"a4"`), + ), + }, + }, + evalDir: "/stack", + expr: `global.obj`, + want: `{ + val = "A4" + }`, + }, + { + name: "single stack with global dependency from parent", + layout: []string{"s:stack"}, + configs: []hclconfig{ + { + path: "/", + add: Globals( + Str("name", "terramate"), + ), + }, + { + path: "/stack", + add: Globals( + Expr("a", `global.name`), + ), + }, + }, + evalDir: "/stack", + expr: `global.a`, + want: `"terramate"`, + }, + { + name: "global dependency from parent with multiple hops", + layout: []string{"s:stack"}, + configs: []hclconfig{ + { + path: "/", + add: Globals( + Expr("a1", `global.a2`), + Expr("a2", `"a2"`), + ), + }, + { + path: "/stack", + add: Globals( + Expr("a", `global.a1`), + ), + }, + }, + evalDir: "/stack", + expr: `global.a`, + want: `"a2"`, + }, + { + name: "global dependency from parent with lazy evaluation", + layout: []string{"s:stack"}, + configs: []hclconfig{ + { + path: "/", + add: Globals( + Expr("a1", `global.a2`), + Expr("a2", `global.stackval`), + ), + }, + { + path: "/stack", + add: Globals( + Expr("a", `global.a1`), + Str("stackval", "value from stack"), + ), + }, + }, + evalDir: "/stack", + expr: `global.a`, + want: `"value from stack"`, + }, + { + name: "globals with cycles in the same scope", + layout: []string{"s:stack"}, + configs: []hclconfig{ + { + path: "/stack", + add: Globals( + Expr("obj", `{ + val = global.a1 + }`), + Expr("a1", `tm_upper(global.a2)`), + Expr("a2", `tm_lower(global.a3)`), + Expr("a3", `tm_upper(global.a4)`), + Expr("a4", `global.a1`), + ), + }, + }, + evalDir: "/stack", + expr: `global.obj`, + wantErr: errors.E(eval.ErrCycle), + }, + } { + tc := tc + t.Run(tc.name, func(t *testing.T) { + s := sandbox.New(t) + s.BuildTree(tc.layout) + for _, globalBlock := range tc.configs { + path := filepath.Join(s.RootDir(), globalBlock.path) + filename := config.DefaultFilename + if globalBlock.filename != "" { + filename = globalBlock.filename + } + test.AppendFile(t, path, filename, globalBlock.add.String()) + } + + cfg, err := config.LoadRoot(s.RootDir()) + if err != nil { + errtest.Assert(t, err, tc.wantErr) + return + } + + tree, ok := cfg.Lookup(project.NewPath(tc.evalDir)) + if !ok { + t.Fatalf("evalDir %s not found", tc.evalDir) + } + + expr, diags := hclsyntax.ParseExpression([]byte(tc.expr), "test.hcl", hcl.InitialPos) + if diags.HasErrors() { + t.Fatal(diags.Error()) + } + + ctx := eval.New(tree.Dir(), globals.NewResolver(cfg)) + ctx.SetFunctions(stdlib.Functions(ctx, tree.HostDir())) + val, err := ctx.Eval(expr) + errtest.Assert(t, err, tc.wantErr) + + if tc.wantErr != nil { + return + } + + assert.EqualStrings(t, string(hhclwrite.Format([]byte(tc.want))), + string(hhclwrite.Format(ast.TokensForValue(val).Bytes()))) + }) + } +} + +func init() { + zerolog.SetGlobalLevel(zerolog.Disabled) +} diff --git a/globals/stack.go b/globals/stack.go deleted file mode 100644 index 04f200fcf..000000000 --- a/globals/stack.go +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2023 Terramate GmbH -// SPDX-License-Identifier: MPL-2.0 - -package globals - -import ( - "github.com/terramate-io/terramate/config" - "github.com/terramate-io/terramate/hcl/eval" - "github.com/terramate-io/terramate/stdlib" -) - -// ForStack loads from the config tree all globals defined for a given stack. -func ForStack(root *config.Root, stack *config.Stack) EvalReport { - ctx := eval.NewContext( - stdlib.Functions(stack.HostDir(root)), - ) - runtime := root.Runtime() - runtime.Merge(stack.RuntimeValues(root)) - ctx.SetNamespace("terramate", runtime) - return ForDir(root, stack.Dir, ctx) -} diff --git a/go.mod b/go.mod index d1b477c33..9864b8197 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/posener/complete v1.2.3 github.com/terramate-io/go-checkpoint v1.0.0 github.com/willabides/kongplete v0.2.0 + github.com/wk8/go-ordered-map/v2 v2.1.7 github.com/zclconf/go-cty v1.13.2 github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b go.lsp.dev/jsonrpc2 v0.10.0 @@ -35,11 +36,14 @@ require ( require ( github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect github.com/acomagu/bufpipe v1.0.4 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/cloudflare/circl v1.3.3 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect diff --git a/go.sum b/go.sum index 30b1c57c3..85b853201 100644 --- a/go.sum +++ b/go.sum @@ -108,6 +108,8 @@ github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgI github.com/aws/aws-sdk-go v1.15.78/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= github.com/aws/aws-sdk-go v1.31.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -117,6 +119,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bmatcuk/doublestar v1.1.5 h1:2bNwBOmhyFEFcoB3tGvTD5xanq+4kyOZlB8wFYbMjkk= github.com/bmatcuk/doublestar v1.1.5/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= @@ -339,6 +343,7 @@ github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeY github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/joyent/triton-go v0.0.0-20180313100802-d8f9c0314926/go.mod h1:U+RSyWxWd04xTqnuOQxnai7XGS2PrPY2cfGoDKtMHjA= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -375,6 +380,8 @@ github.com/lusis/go-artifactory v0.0.0-20160115162124-7e4ce345df82/go.mod h1:y54 github.com/madlambda/spells v0.4.2 h1:SZnqMRSgp65SoIiW1ky73wPL2ImkSg+sYZlWA+uyrx4= github.com/madlambda/spells v0.4.2/go.mod h1:Z8EUYIlBI+GfxQQGHHPkzt9YGu9olhVTbhcnxMGpgYo= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/masterzen/simplexml v0.0.0-20160608183007-4572e39b1ab9/go.mod h1:kCEbxUJlNDEBNbdQMkPSp6yaKcRXVI6f4ddk8Riv4bc= github.com/masterzen/simplexml v0.0.0-20190410153822-31eea3082786/go.mod h1:kCEbxUJlNDEBNbdQMkPSp6yaKcRXVI6f4ddk8Riv4bc= github.com/masterzen/winrm v0.0.0-20200615185753-c42b5136ff88/go.mod h1:a2HXwefeat3evJHxFXSayvRHpYEPJYtErl4uIzfaUqY= @@ -495,8 +502,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/tencentcloud/tencentcloud-sdk-go v3.0.82+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20190808065407-f07404cefc8c/go.mod h1:wk2XFUg6egk4tSDNZtXeKfe2G6690UVyt163PuUxBZk= github.com/terramate-io/go-checkpoint v1.0.0 h1:E+pBXByyRsLWQYyIeOgt25yWB4e7PN0zf8ePVaFO5vw= @@ -510,6 +517,8 @@ github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+ github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/willabides/kongplete v0.2.0 h1:C6wYVn+IPyA8rAGRGLLkuxhhSQTEECX4t8u3gi+fuD0= github.com/willabides/kongplete v0.2.0/go.mod h1:kFVw+PkQsqkV7O4tfIBo6iJ9qY94PJC8sPfMgFG5AdM= +github.com/wk8/go-ordered-map/v2 v2.1.7 h1:aUZ1xBMdbvY8wnNt77qqo4nyT3y0pX4Usat48Vm+hik= +github.com/wk8/go-ordered-map/v2 v2.1.7/go.mod h1:9Xvgm2mV2kSq2SAm0Y608tBmu8akTzI7c2bz7/G7ZN4= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= diff --git a/hcl/ast/expr.go b/hcl/ast/expr.go index 2a8782a3a..3db085fcb 100644 --- a/hcl/ast/expr.go +++ b/hcl/ast/expr.go @@ -15,6 +15,11 @@ import ( "github.com/zclconf/go-cty/cty" ) +// Tokenizer is an interface for expressions that knows how to tokenize themselves. +type Tokenizer interface { + Tokens() hclwrite.Tokens +} + // ParseExpression parses the expression str. func ParseExpression(str string, filename string) (hcl.Expression, error) { expr, diags := hclsyntax.ParseExpression([]byte(str), filename, hcl.InitialPos) @@ -32,6 +37,14 @@ func TokensForExpression(expr hcl.Expression) hclwrite.Tokens { return tokens } +// StringForExpression returns a string representation of the expression. +func StringForExpression(expr hcl.Expression) string { + if stringer, ok := expr.(fmt.Stringer); ok { + return stringer.String() + } + return string(TokensForExpression(expr).Bytes()) +} + // TokensForValue returns the tokens for the provided value. func TokensForValue(value cty.Value) hclwrite.Tokens { if value.Type() == customdecode.ExpressionClosureType { @@ -43,6 +56,11 @@ func TokensForValue(value cty.Value) hclwrite.Tokens { return hclwrite.TokensForValue(value) } +// StringForValue returns a string representation of the value. +func StringForValue(value cty.Value) string { + return string(TokensForValue(value).Bytes()) +} + func tokensForExpression(expr hcl.Expression) hclwrite.Tokens { builder := tokenBuilder{} builder.build(expr) @@ -58,6 +76,13 @@ func (builder *tokenBuilder) add(tokens ...*hclwrite.Token) { } func (builder *tokenBuilder) build(expr hcl.Expression) { + if tokenizer, ok := expr.(interface { + Tokens() hclwrite.Tokens + }); ok { + builder.add(tokenizer.Tokens()...) + return + } + switch e := expr.(type) { case *hclsyntax.LiteralValueExpr: builder.literalTokens(e) diff --git a/hcl/eval/eval.go b/hcl/eval/eval.go index 96cc6eeb6..89359973e 100644 --- a/hcl/eval/eval.go +++ b/hcl/eval/eval.go @@ -4,63 +4,430 @@ package eval import ( - "strings" + "reflect" "github.com/terramate-io/terramate/errors" + "github.com/terramate-io/terramate/hcl/ast" + "github.com/terramate-io/terramate/project" + orderedmap "github.com/wk8/go-ordered-map/v2" "github.com/zclconf/go-cty/cty" "github.com/zclconf/go-cty/cty/function" hhcl "github.com/hashicorp/hcl/v2" ) -// ErrEval indicates a failure during the evaluation process -const ErrEval errors.Kind = "eval expression" +// Errors returned when parsing and evaluating globals. +const ( + // ErrEval indicates a failure during the evaluation process + ErrEval errors.Kind = "eval expression" -// Context is used to evaluate HCL code. -type Context struct { - hclctx *hhcl.EvalContext + // ErrCycle indicates there's a cycle in the variable declarations. + ErrCycle errors.Kind = "cycle detected" + + // ErrRedefined indicates the variable was already defined in this scope. + ErrRedefined errors.Kind = "variable redefined" +) + +type ( + // Context is the variables evaluator. + Context struct { + scope project.Path + hclctx *hhcl.EvalContext + ns namespaces + + evaluators map[string]Resolver + } + + // Resolver resolves unknown variable references. + Resolver interface { + Name() string + Prevalue() cty.Value + LookupRef(scope project.Path, ref Ref) ([]Stmts, error) + } + + namespaces map[string]namespace + + value struct { + stmt Stmt + value cty.Value + info Info + } + + namespace struct { + byref map[RefStr]value + bykey *orderedmap.OrderedMap[string, any] + + persist bool // whether persistence into internal context is needed. + } +) + +var unset cty.Type + +func init() { + unset = cty.Capsule("unset", reflect.TypeOf(struct{}{})) } -// NewContext creates a new HCL evaluation context. -// The basedir is the base directory used by any interpolation functions that -// accept filesystem paths as arguments. -// The basedir must be an absolute path to a directory. -func NewContext(funcs map[string]function.Function) *Context { +// New evaluator. +// TODO(i4k): better document. +func New(scope project.Path, evaluators ...Resolver) *Context { hclctx := &hhcl.EvalContext{ - Functions: funcs, + Functions: map[string]function.Function{}, Variables: map[string]cty.Value{}, } - return &Context{ - hclctx: hclctx, + evalctx := &Context{ + scope: scope, + hclctx: hclctx, + evaluators: map[string]Resolver{}, + ns: namespaces{}, + } + + for _, ev := range evaluators { + evalctx.SetResolver(ev) + } + + unsetVal := cty.CapsuleVal(unset, &struct{}{}) + evalctx.hclctx.Variables["unset"] = unsetVal + + return evalctx +} + +// SetResolver sets the resolver ev into the context. +func (c *Context) SetResolver(ev Resolver) { + c.evaluators[ev.Name()] = ev + ns := newNamespace() + c.ns[ev.Name()] = ns + + prevalue := ev.Prevalue() + if prevalue.Type().IsObjectType() { + values := prevalue.AsValueMap() + for key, val := range values { + origin := NewRef(ev.Name(), []string{key}...) + err := c.set(NewValStmt(origin, val, newBuiltinInfo(c.scope)), val) + if err != nil { + panic(errors.E(errors.ErrInternal, "failed to initialize context")) + } + } + } else { + c.hclctx.Variables[ev.Name()] = prevalue } } +// DeleteResolver removes the resolver. +func (c *Context) DeleteResolver(name string) { + delete(c.evaluators, name) + delete(c.hclctx.Variables, name) +} + +// Eval the given expr and all of its dependency references (if needed) +// using a bottom-up algorithm (starts in the target g.tree directory +// and lookup references in parent directories when needed). +// The algorithm is reliable but it does the minimum required work to +// get the expr evaluated, and then it does not validate all globals +// scopes but only the ones it traversed into. +func (c *Context) Eval(expr hhcl.Expression) (cty.Value, error) { + return c.eval(expr, map[RefStr]hhcl.Expression{}) +} + +func (c *Context) eval(expr hhcl.Expression, visited map[RefStr]hhcl.Expression) (cty.Value, error) { + refs, refsMap := refsOf(expr) + unsetRefs := map[RefStr]bool{} + + for _, dep := range refs { + if dep.String() == "unset" { + continue + } + + if _, ok := c.ns.Get(dep); ok { + // dep already evaluated. + continue + } + + if previousExpr, ok := visited[dep.AsKey()]; ok { + return cty.NilVal, errors.E( + ErrCycle, + expr.Range(), + "variable have circular dependencies: "+ + "reference %s already evaluated in the expression %s", + dep, + ast.TokensForExpression(previousExpr).Bytes(), + ) + } + + visited[dep.AsKey()] = expr + + stmtResolver, ok := c.evaluators[dep.Object] + if !ok { + // ignore unknowns in partial expressions + continue + } + + scopeStmts, err := stmtResolver.LookupRef(c.scope, dep) + if err != nil { + return cty.NilVal, err + } + + for _, stmts := range scopeStmts { + for _, stmt := range stmts { + val, hasVal, err := c.evalStmt(stmt, visited) + if err != nil { + return cty.NilVal, err + } + + if !hasVal { + continue + } + + if val.Type().Equals(unset) { + unsetRefs[stmt.LHS.AsKey()] = true + continue + } + + if unsetRefs[stmt.LHS.AsKey()] { + continue + } + + err = c.set(stmt, val) + if err != nil { + return cty.NilVal, errors.E(ErrEval, err) + } + } + } + + if _, ok := c.ns.Get(dep); !ok { + delete(visited, dep.AsKey()) + } + } + + for nsname, ns := range c.ns { + if ns.persist { + if _, ok := refsMap[nsname]; ok { + c.SetNamespace(nsname, tocty(ns.bykey).AsValueMap()) + ns.persist = false + } + } + } + + val, diags := expr.Value(c.hclctx) + if diags.HasErrors() { + return cty.NilVal, errors.E(ErrEval, diags) + } + + return val, nil +} + +func (c *Context) evalStmt(stmt Stmt, visited map[RefStr]hhcl.Expression) (cty.Value, bool, error) { + if v, ok := c.ns.Get(stmt.LHS); ok { + if isRedefined(stmt, v) { + return cty.NilVal, false, errors.E(ErrRedefined, stmt.Info.DefinedAt, + "variable %s already set in the scope %s at %s", + stmt, stmt.Info.Scope, v.info.DefinedAt.String()) + } + if !v.value.Type().IsObjectType() || !v.stmt.Special { + + // stmt already evaluated + // This can happen when the current scope is overriding the parent + // object but still the target expr is looking for the entire object + // so we still have to ascent into parent scope and then the "already + // overridden" refs show up here. + return cty.NilVal, false, nil + } + } + + if stmt.Special { + err := c.setExtend(stmt) + if err != nil { + return cty.NilVal, false, errors.E(ErrEval, err) + } + return cty.NilVal, false, nil + } + + var val cty.Value + var err error + if stmt.RHS.IsEvaluated { + val = stmt.RHS.Value + } else { + val, err = c.eval(stmt.RHS.Expression, visited) + if err != nil { + return cty.NilVal, false, errors.E(err, "evaluating %s from %s scope", stmt.LHS, stmt.Info.Scope) + } + } + return val, true, nil +} + +func (c *Context) setExtend(stmt Stmt) error { + ref := stmt.LHS + ns, ok := c.ns[ref.Object] + if !ok { + panic(errors.E(errors.ErrInternal, "there's no evaluator for namespace %q", ref.Object)) + } + + obj, err := traverseObject(&ns, ref, stmt.Info) + if err != nil { + return err + } + + _, found := obj.Get(ref.LastAccessor()) + if found { + return nil + } + tempMap := orderedmap.New[string, any]() + obj.Set(ref.LastAccessor(), tempMap) + return nil +} + +func (c *Context) set(stmt Stmt, val cty.Value) error { + ref := stmt.LHS + + if val.Type().IsObjectType() { + origin := ref + stmts := StmtsOfValue(stmt.Info, origin, origin.Path, val) + for _, s := range stmts { + val, hasVal, err := c.evalStmt(s, map[RefStr]hhcl.Expression{}) + if err != nil { + return err + } + if !hasVal { + continue + } + + err = c.set(s, val) + if err != nil { + return err + } + } + if len(stmts) > 0 { + return nil + } + } + + ns, ok := c.ns[ref.Object] + if !ok { + panic(errors.E(errors.ErrInternal, "there's no evaluator for namespace %q", ref.Object)) + } + + oldval, hasold := ns.byref[ref.AsKey()] + + if hasold && len(oldval.info.Scope.String()) > len(stmt.Info.Scope.String()) { + return nil + } + + ns.byref[ref.AsKey()] = value{ + stmt: stmt, + value: val, + info: stmt.Info, + } + + ns.persist = true + + obj, err := traverseObject(&ns, ref, stmt.Info) + if err != nil { + return err + } + + if hasold && oldval.stmt.Special && oldval.info.Scope == stmt.Info.Scope { + return errors.E( + ErrEval, + "variable %s being extended but was previously evaluated as %s in the same scope", + stmt.LHS, ast.TokensForValue(oldval.value).Bytes(), + ) + } + ns.persist = true + obj.Set(ref.LastAccessor(), val) + return nil +} + +func traverseObject(ns *namespace, ref Ref, info Info) (*orderedmap.OrderedMap[string, any], error) { + obj := ns.bykey + + // len(path) >= 1 + + lastIndex := len(ref.Path) - 1 + for i, path := range ref.Path[:lastIndex] { + v, ok := obj.Get(path) + if ok { + switch vv := v.(type) { + case *orderedmap.OrderedMap[string, any]: + obj = vv + case cty.Value: + return nil, errors.E("%s points to a %s type but expects an object", ref, vv.Type().FriendlyName()) + default: + panic(vv) + } + } else { + r := ref + r.Path = r.Path[:i+1] + ns.byref[r.AsKey()] = value{ + stmt: NewExtendStmt(r, info), + value: cty.EmptyObjectVal, + info: info, + } + + tempMap := orderedmap.New[string, any]() + obj.Set(path, tempMap) + obj = tempMap + } + } + return obj, nil +} + +func tocty(globals *orderedmap.OrderedMap[string, any]) cty.Value { + ret := map[string]cty.Value{} + for pair := globals.Oldest(); pair != nil; pair = pair.Next() { + switch vv := pair.Value.(type) { + case *orderedmap.OrderedMap[string, any]: + ret[pair.Key] = tocty(vv) + case cty.Value: + if vv.Type().IsTupleType() { + var items []cty.Value + it := vv.ElementIterator() + for it.Next() { + _, elem := it.Element() + if !elem.Type().Equals(unset) { + items = append(items, elem) + } + } + if len(items) == 0 { + vv = cty.EmptyTupleVal + } else { + vv = cty.TupleVal(items) + } + } + ret[pair.Key] = vv + default: + panic(errors.E(errors.ErrInternal, "unexpected type %T", vv)) + } + } + return cty.ObjectVal(ret) +} + +func (ns namespaces) Get(ref Ref) (value, bool) { + if v, ok := ns[ref.Object]; ok { + if vv, ok := v.byref[ref.AsKey()]; ok { + return vv, true + } + } + return value{}, false +} + // SetNamespace will set the given values inside the given namespace on the // evaluation context. func (c *Context) SetNamespace(name string, vals map[string]cty.Value) { c.hclctx.Variables[name] = cty.ObjectVal(vals) } -// GetNamespace will retrieve the value for the given namespace. -func (c *Context) GetNamespace(name string) (cty.Value, bool) { - val, ok := c.hclctx.Variables[name] - return val, ok -} - // SetFunction sets the function in the context. func (c *Context) SetFunction(name string, fn function.Function) { c.hclctx.Functions[name] = fn } -// SetEnv sets the given environment on the env namespace of the evaluation context. -// environ must be on the same format as os.Environ(). -func (c *Context) SetEnv(environ []string) { - env := map[string]cty.Value{} - for _, v := range environ { - parsed := strings.Split(v, "=") - env[parsed[0]] = cty.StringVal(parsed[1]) - } - c.SetNamespace("env", env) +// DeleteFunction deletes the given function from the context. +func (c *Context) DeleteFunction(name string) { + delete(c.hclctx.Functions, name) +} + +// SetFunctions sets the functions of the context. +func (c *Context) SetFunctions(funcs map[string]function.Function) { + c.hclctx.Functions = funcs } // DeleteNamespace deletes the namespace name from the context. @@ -75,15 +442,6 @@ func (c *Context) HasNamespace(name string) bool { return has } -// Eval will evaluate an expression given its context. -func (c *Context) Eval(expr hhcl.Expression) (cty.Value, error) { - val, diag := expr.Value(c.hclctx) - if diag.HasErrors() { - return cty.NilVal, errors.E(ErrEval, diag) - } - return val, nil -} - // PartialEval evaluates only the terramate variable expressions from the list // of tokens, leaving all the rest as-is. It returns a modified list of tokens // with no reference to terramate namespaced variables (globals and terramate) @@ -119,3 +477,18 @@ func NewContextFrom(ctx *hhcl.EvalContext) *Context { hclctx: ctx, } } + +func newNamespace() namespace { + return namespace{ + persist: true, + byref: make(map[RefStr]value), + bykey: orderedmap.New[string, any](), + } +} + +func isRedefined(new Stmt, v value) bool { + return !new.Special && !v.stmt.Special && + v.info.Scope == new.Info.Scope && + v.info.DefinedAt.Path().Dir() == new.Info.DefinedAt.Path().Dir() && + v.info.DefinedAt != new.Info.DefinedAt +} diff --git a/hcl/eval/eval_test.go b/hcl/eval/eval_test.go index 49cbecd2e..e835c34f2 100644 --- a/hcl/eval/eval_test.go +++ b/hcl/eval/eval_test.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/terramate-io/terramate/errors" "github.com/terramate-io/terramate/hcl/eval" + "github.com/terramate-io/terramate/project" "github.com/terramate-io/terramate/stdlib" "github.com/terramate-io/terramate/test" errtest "github.com/terramate-io/terramate/test/errors" @@ -22,6 +23,7 @@ type want struct { err error value cty.Value } + type testcase struct { name string basedir string @@ -48,7 +50,7 @@ func TestEvalTmFuncall(t *testing.T) { }, { name: "tm_ternary - cond is false, with partial not evaluated", - expr: `tm_ternary(false, local.var, "world")`, + expr: `tm_ternary(false, unset, "world")`, want: want{ value: cty.StringVal("world"), }, @@ -79,7 +81,8 @@ func TestEvalTmFuncall(t *testing.T) { if basedir == "" { basedir = root(t) } - ctx := eval.NewContext(stdlib.Functions(basedir)) + evalctx := eval.New(project.RootPath) + evalctx.SetFunctions(stdlib.Functions(evalctx, basedir)) const attrname = "value" @@ -94,7 +97,7 @@ func TestEvalTmFuncall(t *testing.T) { body := file.Body.(*hclsyntax.Body) attr := body.Attributes[attrname] - got, err := ctx.Eval(attr.Expr) + got, err := evalctx.Eval(attr.Expr) errtest.Assert(t, err, tc.want.err) if tc.want.err == nil { diff --git a/hcl/eval/object.go b/hcl/eval/object.go deleted file mode 100644 index 71f3d83bb..000000000 --- a/hcl/eval/object.go +++ /dev/null @@ -1,342 +0,0 @@ -// Copyright 2023 Terramate GmbH -// SPDX-License-Identifier: MPL-2.0 - -package eval - -import ( - "github.com/terramate-io/terramate/errors" - "github.com/terramate-io/terramate/hcl/fmt" - "github.com/terramate-io/terramate/project" - "github.com/zclconf/go-cty/cty" -) - -// ErrCannotExtendObject is the error when an object cannot be extended. -const ErrCannotExtendObject errors.Kind = "cannot extend object" - -type ( - // Object is an object container for cty.Value values supporting set at - // arbitrary accessor paths. - // - // Eg.: - // obj := eval.NewObject(origin) - // obj.Set("val", eval.NewObject()) - // - // The snippet above creates the object below: - // { - // val = {} - // } - // - // Then values can be set inside obj.val by doing: - // - // obj.SetAt(ObjectPath{"val", "test"}, eval.NewValue(cty.StringVal("test"), origin)) - // - // Of which creates the object below: - // - // { - // val = { - // test = "test" - // } - // } - Object struct { - origin Info - // Keys is a map of key names to values. - Keys map[string]Value - } - - // Value is an evaluated value. - Value interface { - // Info provides extra information for the value. - Info() Info - - // IsObject tells if the value is an object. - IsObject() bool - } - - // CtyValue is a wrapper for a raw cty value. - CtyValue struct { - origin Info - cty.Value - } - - // Info provides extra information for the configuration value. - Info struct { - // Dir defines the directory from there the value is being instantiated, - // which means it's the scope directory (not the file where it's defined). - // For values that comes from imports, the Dir will be the directory - // which imports the value. - Dir project.Path - - // DefinedAt provides the source file where the value is defined. - DefinedAt project.Path - } - - // ObjectPath represents a path inside the object. - ObjectPath []string -) - -// NewObject creates a new object for configdir directory. -func NewObject(origin Info) *Object { - return &Object{ - origin: origin, - Keys: make(map[string]Value), - } -} - -// Set a key value into object. -func (obj *Object) Set(key string, value Value) { - obj.Keys[key] = value -} - -// GetKeyPath retrieves the value at path. -func (obj *Object) GetKeyPath(path ObjectPath) (Value, bool) { - key := path[0] - next := path[1:] - - v, ok := obj.Keys[key] - if !ok { - return nil, false - } - if len(next) == 0 { - return v, true - } - if !v.IsObject() { - return nil, false - } - - return v.(*Object).GetKeyPath(next) -} - -// Info provides extra information for the object value. -func (obj *Object) Info() Info { return obj.origin } - -// IsObject returns true for [Object] values. -func (obj *Object) IsObject() bool { return true } - -// SetFrom sets the object keys and values from the map. -func (obj *Object) SetFrom(values map[string]Value) *Object { - for k, v := range values { - obj.Set(k, v) - } - return obj -} - -// SetFromCtyValues sets the object from the values map. -func (obj *Object) SetFromCtyValues(values map[string]cty.Value, origin Info) *Object { - for k, v := range values { - if v.Type().IsObjectType() { - subtree := NewObject(origin) - subtree.SetFromCtyValues(v.AsValueMap(), origin) - obj.Set(k, subtree) - } else { - obj.Set(k, NewCtyValue(v, origin)) - } - } - return obj -} - -// SetAt sets a value at the specified path key. -func (obj *Object) SetAt(path ObjectPath, value Value) error { - target, key, err := computeTargetFrom(obj, path, value.Info()) - if err != nil { - return err - } - - target.Set(key, value) - return nil -} - -// MergeFailsIfKeyExists merge the value into obj but fails if any key in value exists in -// obj. -func (obj *Object) MergeFailsIfKeyExists(path ObjectPath, value Value) error { - target, key, err := computeTargetFrom(obj, path, value.Info()) - if err != nil { - return err - } - - old, ok := target.GetKeyPath([]string{key}) - if !ok { - target.Set(key, value) - return nil - } - - if old.IsObject() != value.IsObject() { - return errors.E("failed to merge object and value for key path %v", key) - } - if !value.IsObject() { - return errors.E("cannot overwrite key path %v", key) - } - valObj := value.(*Object) - oldObj := old.(*Object) - for k, v := range valObj.Keys { - _, ok := oldObj.GetKeyPath([]string{k}) - if ok { - return errors.E("cannot overwrite") - } - err := oldObj.SetAt([]string{k}, v) - if err != nil { - return err - } - } - return nil -} - -// MergeOverwrite merges value into obj by overwriting each key. -func (obj *Object) MergeOverwrite(path ObjectPath, value Value) error { - target, key, err := computeTargetFrom(obj, path, value.Info()) - if err != nil { - return err - } - - old, ok := target.GetKeyPath([]string{key}) - if !ok { - target.Set(key, value) - return nil - } - - if old.IsObject() != value.IsObject() { - target.Set(key, value) - return nil - } - if !old.IsObject() { - target.Set(key, value) - return nil - } - valObj := value.(*Object) - oldObj := old.(*Object) - for k, v := range valObj.Keys { - err := oldObj.SetAt([]string{k}, v) - if err != nil { - return err - } - } - return nil -} - -// MergeNewKeys merge the keys from value that doesn't exist in obj. -func (obj *Object) MergeNewKeys(path ObjectPath, value Value) error { - target, key, err := computeTargetFrom(obj, path, value.Info()) - if err != nil { - return err - } - - old, ok := target.GetKeyPath([]string{key}) - if !ok { - target.Set(key, value) - return nil - } - - if old.IsObject() != value.IsObject() { - return errors.E("failed to merge object and value") - } - if !value.IsObject() { - return errors.E("cannot overwrite") - } - valObj := value.(*Object) - oldObj := old.(*Object) - for k, v := range valObj.Keys { - _, ok := oldObj.GetKeyPath([]string{k}) - if !ok { - err := oldObj.SetAt([]string{k}, v) - if err != nil { - return err - } - } - } - return nil -} - -func computeTargetFrom(obj *Object, path ObjectPath, info Info) (*Object, string, error) { - for len(path) > 1 { - key := path[0] - subobj, ok := obj.Keys[key] - if !ok { - subobj = NewObject(info) - obj.Set(key, subobj) - } - if !subobj.IsObject() { - return nil, "", errors.E(ErrCannotExtendObject, - "path part %s (from %s) contains non-object parts in the path (%v is %T)", - key, path, key, subobj) - } - obj = subobj.(*Object) - path = path[1:] - } - return obj, path[0], nil -} - -// DeleteAt deletes the value at the specified path. -func (obj *Object) DeleteAt(path ObjectPath) error { - for len(path) > 1 { - key := path[0] - subobj, ok := obj.Keys[key] - if !ok { - return nil - } - if !subobj.IsObject() { - return errors.E(ErrCannotExtendObject, - "path part %s (from %v) contains non-object parts in the path (%s is %T)", - key, path, key, subobj) - } - obj = subobj.(*Object) - path = path[1:] - } - - delete(obj.Keys, path[0]) - return nil -} - -// AsValueMap returns a map of string to Hashicorp cty.Value. -func (obj *Object) AsValueMap() map[string]cty.Value { - vmap := map[string]cty.Value{} - for k, v := range obj.Keys { - switch vv := v.(type) { - case *Object: - subvmap := vv.AsValueMap() - vmap[k] = cty.ObjectVal(subvmap) - case CtyValue: - vmap[k] = vv.Raw() - default: - panic("unreachable") - } - } - return vmap -} - -// String representation of the object. -func (obj *Object) String() string { - return fmt.FormatAttributes(obj.AsValueMap()) -} - -// NewCtyValue creates a new cty.Value wrapper. -// Note: The cty.Value val is marked with the origin path and must be unmarked -// before use with any hashicorp API otherwise it panics. -func NewCtyValue(val cty.Value, origin Info) CtyValue { - val = val.Mark(origin) - return CtyValue{ - origin: origin, - Value: val, - } -} - -// NewValue returns a new object Value from a cty.Value. -// Note: this is not a wrapper as it returns an [Object] if val is a cty.Object. -func NewValue(val cty.Value, origin Info) Value { - if val.Type().IsObjectType() { - obj := NewObject(origin) - obj.SetFromCtyValues(val.AsValueMap(), origin) - return obj - } - return NewCtyValue(val, origin) -} - -// Info provides extra information for the value. -func (v CtyValue) Info() Info { return v.origin } - -// IsObject returns false for CtyValue values. -func (v CtyValue) IsObject() bool { return false } - -// Raw returns the original cty.Value value (unmarked). -func (v CtyValue) Raw() cty.Value { - val, _ := v.Value.Unmark() - return val -} diff --git a/hcl/eval/object_test.go b/hcl/eval/object_test.go deleted file mode 100644 index 3c4f751c7..000000000 --- a/hcl/eval/object_test.go +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright 2023 Terramate GmbH -// SPDX-License-Identifier: MPL-2.0 - -package eval_test - -import ( - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/terramate-io/terramate/errors" - "github.com/terramate-io/terramate/hcl/eval" - "github.com/terramate-io/terramate/project" - - errtest "github.com/terramate-io/terramate/test/errors" -) - -type strValue string - -func (s strValue) IsObject() bool { return false } -func (s strValue) Info() eval.Info { - return eval.Info{ - Dir: project.NewPath("/"), - } -} - -func TestCtyObjectSetAt(t *testing.T) { - t.Parallel() - type testcase struct { - name string - obj *eval.Object - val eval.Value - path eval.ObjectPath - want *eval.Object - wantErr error - } - - defaultOrigin := eval.Info{ - DefinedAt: project.NewPath("/file.tm"), - Dir: project.NewPath("/"), - } - - newobj := func(sets ...map[string]eval.Value) *eval.Object { - obj := eval.NewObject(defaultOrigin) - for _, set := range sets { - obj.SetFrom(set) - } - return obj - } - - for _, tc := range []testcase{ - { - name: "set at root, empty object", - obj: newobj(), - path: eval.ObjectPath{"key"}, - val: strValue("value"), - want: newobj(map[string]eval.Value{ - "key": strValue("value"), - }), - }, - { - name: "set at root, override value", - obj: newobj(map[string]eval.Value{ - "key": strValue("old value"), - }), - path: eval.ObjectPath{"key"}, - val: strValue("new value"), - want: newobj().SetFrom( - map[string]eval.Value{ - "key": strValue("new value"), - }, - ), - }, - { - name: "set at root, new value", - obj: newobj(map[string]eval.Value{ - "key": strValue("value"), - }, - ), - path: eval.ObjectPath{"other-key"}, - val: strValue("other value"), - want: newobj(map[string]eval.Value{ - "key": strValue("value"), - "other-key": strValue("other value"), - }), - }, - { - name: "set at an existing child object", - obj: newobj(map[string]eval.Value{ - "key": newobj(), - }), - path: eval.ObjectPath{"key", "test"}, - val: strValue("child value"), - want: newobj(map[string]eval.Value{ - "key": newobj(map[string]eval.Value{ - "test": strValue("child value"), - }), - }), - }, - { - name: "set at an existing child object", - obj: newobj(map[string]eval.Value{ - "key": newobj(), - }), - path: eval.ObjectPath{"key", "test"}, - val: strValue("child value"), - want: newobj(map[string]eval.Value{ - "key": newobj(map[string]eval.Value{ - "test": strValue("child value"), - }), - }), - }, - { - name: "set at a non-existent child object", - obj: newobj(), - path: eval.ObjectPath{"key", "test"}, - val: strValue("child value"), - want: newobj(map[string]eval.Value{ - "key": newobj(map[string]eval.Value{ - "test": strValue("child value"), - }), - }), - }, - { - name: "set at a non-existent deep child object", - obj: newobj(), - path: eval.ObjectPath{"a", "b", "c", "d", "test"}, - val: strValue("value"), - want: newobj(map[string]eval.Value{ - "a": newobj(map[string]eval.Value{ - "b": newobj(map[string]eval.Value{ - "c": newobj(map[string]eval.Value{ - "d": newobj(map[string]eval.Value{ - "test": strValue("value"), - }), - }), - }), - }), - }), - }, - { - name: "set at a non-object child - fails", - obj: newobj(map[string]eval.Value{ - "key": strValue("1"), - }), - path: eval.ObjectPath{"key", "test"}, - val: strValue("child value"), - wantErr: errors.E(eval.ErrCannotExtendObject), - }, - } { - tc := tc - t.Run(tc.name, func(t *testing.T) { - err := tc.obj.SetAt(tc.path, tc.val) - errtest.Assert(t, err, tc.wantErr) - if err == nil { - if diff := cmp.Diff(tc.obj, tc.want, cmpopts.IgnoreUnexported(eval.Object{})); diff != "" { - t.Fatalf("-(got) +(want):\n%s", diff) - } - } - }) - } -} diff --git a/hcl/eval/partial_eval.go b/hcl/eval/partial_eval.go index 664845470..bd8e592d2 100644 --- a/hcl/eval/partial_eval.go +++ b/hcl/eval/partial_eval.go @@ -200,7 +200,7 @@ func (c *Context) hasUnknownVars(expr hclsyntax.Expression) bool { func (c *Context) hasTerramateVars(expr hclsyntax.Expression) bool { for _, namespace := range expr.Variables() { - if c.HasNamespace(namespace.RootName()) { + if _, ok := c.evaluators[namespace.RootName()]; ok { return true } } @@ -306,9 +306,12 @@ func (c *Context) partialEvalScopeTrav(scope *hclsyntax.ScopeTraversalExpr, opts if !ok { return scope, nil } - if !c.HasNamespace(ns.Name) { + + // check if there's a resolver + if _, ok := c.evaluators[ns.Name]; !ok { return scope, nil } + forbidRootEval := false if len(opts) == 1 { forbidRootEval = opts[0].forbidRootEval diff --git a/hcl/eval/partial_eval_bench_test.go b/hcl/eval/partial_eval_bench_test.go index 2b7873e5c..a9aff1b56 100644 --- a/hcl/eval/partial_eval_bench_test.go +++ b/hcl/eval/partial_eval_bench_test.go @@ -8,42 +8,57 @@ import ( "strings" "testing" - "github.com/hashicorp/hcl/v2" + hhcl "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/terramate-io/terramate/globals" "github.com/terramate-io/terramate/hcl/eval" + "github.com/terramate-io/terramate/hcl/info" + "github.com/terramate-io/terramate/project" "github.com/terramate-io/terramate/stdlib" + "github.com/terramate-io/terramate/test/sandbox" "github.com/zclconf/go-cty/cty" ) -func setupContext() *eval.Context { - ctx := eval.NewContext(stdlib.Functions(os.TempDir())) - ctx.SetNamespace("global", map[string]cty.Value{ - "true": cty.True, - "false": cty.False, - "number": cty.NumberFloatVal(3.141516), - "string": cty.StringVal("terramate"), - "list": cty.ListVal([]cty.Value{ - cty.NumberIntVal(0), - cty.NumberIntVal(1), - cty.NumberIntVal(2), - cty.NumberIntVal(3), +func setupContext(b *testing.B) *eval.Context { + s := sandbox.New(b) + builtinInfo := eval.Info{ + Scope: project.NewPath("/"), + DefinedAt: info.NewRange(s.RootDir(), hhcl.Range{ + Start: hhcl.InitialPos, + End: hhcl.InitialPos, }), - "strings": cty.ListVal([]cty.Value{ - cty.StringVal("terramate"), - cty.StringVal("is"), - cty.StringVal("fun"), - }), - "obj": cty.ObjectVal(map[string]cty.Value{ - "a": cty.NumberIntVal(0), - "b": cty.ListVal([]cty.Value{cty.StringVal("terramate")}), - }), - }) + } + ctx := eval.New( + s.Config().Tree().Dir(), + globals.NewResolver( + s.Config(), + eval.NewValStmt(eval.NewRef("global", "true"), cty.True, builtinInfo), + eval.NewValStmt(eval.NewRef("global", "false"), cty.False, builtinInfo), + eval.NewValStmt(eval.NewRef("global", "number"), cty.NumberFloatVal(3.141516), builtinInfo), + eval.NewValStmt(eval.NewRef("global", "string"), cty.StringVal("terramate"), builtinInfo), + eval.NewValStmt(eval.NewRef("global", "list"), cty.ListVal([]cty.Value{ + cty.NumberIntVal(0), + cty.NumberIntVal(1), + cty.NumberIntVal(2), + cty.NumberIntVal(3), + }), builtinInfo), + eval.NewValStmt(eval.NewRef("global", "strings"), cty.ListVal([]cty.Value{ + cty.StringVal("terramate"), + cty.StringVal("is"), + cty.StringVal("fun"), + }), builtinInfo), + eval.NewValStmt(eval.NewRef("global", "obj"), cty.ObjectVal(map[string]cty.Value{ + "a": cty.NumberIntVal(0), + "b": cty.ListVal([]cty.Value{cty.StringVal("terramate")}), + }), builtinInfo), + )) + ctx.SetFunctions(stdlib.Functions(ctx, os.TempDir())) return ctx } func BenchmarkPartialEvalComplex(b *testing.B) { b.StopTimer() - ctx := setupContext() + ctx := setupContext(b) exprBytes := []byte(`[ { @@ -95,7 +110,7 @@ func BenchmarkPartialEvalComplex(b *testing.B) { b.StartTimer() for n := 0; n < b.N; n++ { - expr, diags := hclsyntax.ParseExpression(exprBytes, "", hcl.InitialPos) + expr, diags := hclsyntax.ParseExpression(exprBytes, "", hhcl.InitialPos) if diags.HasErrors() { b.Fatalf(diags.Error()) } @@ -108,13 +123,13 @@ func BenchmarkPartialEvalComplex(b *testing.B) { func BenchmarkPartialEvalSmallString(b *testing.B) { b.StopTimer() - ctx := setupContext() + ctx := setupContext(b) exprBytes := []byte(`"terramate is fun"`) b.StartTimer() for n := 0; n < b.N; n++ { - expr, diags := hclsyntax.ParseExpression(exprBytes, "", hcl.InitialPos) + expr, diags := hclsyntax.ParseExpression(exprBytes, "", hhcl.InitialPos) if diags.HasErrors() { b.Fatalf(diags.Error()) } @@ -127,13 +142,13 @@ func BenchmarkPartialEvalSmallString(b *testing.B) { func BenchmarkPartialEvalHugeString(b *testing.B) { b.StopTimer() - ctx := setupContext() + ctx := setupContext(b) exprBytes := []byte(`"` + strings.Repeat(`terramate is fun\n`, 1000) + `"`) b.StartTimer() for n := 0; n < b.N; n++ { - expr, diags := hclsyntax.ParseExpression(exprBytes, "", hcl.InitialPos) + expr, diags := hclsyntax.ParseExpression(exprBytes, "", hhcl.InitialPos) if diags.HasErrors() { b.Fatalf(diags.Error()) } @@ -146,13 +161,13 @@ func BenchmarkPartialEvalHugeString(b *testing.B) { func BenchmarkPartialEvalHugeInterpolatedString(b *testing.B) { b.StopTimer() - ctx := setupContext() + ctx := setupContext(b) exprBytes := []byte(`"` + strings.Repeat(`${global.string} is fun\n`, 1000) + `"`) b.StartTimer() for n := 0; n < b.N; n++ { - expr, diags := hclsyntax.ParseExpression(exprBytes, "", hcl.InitialPos) + expr, diags := hclsyntax.ParseExpression(exprBytes, "", hhcl.InitialPos) if diags.HasErrors() { b.Fatalf(diags.Error()) } @@ -165,7 +180,7 @@ func BenchmarkPartialEvalHugeInterpolatedString(b *testing.B) { func BenchmarkPartialEvalObject(b *testing.B) { b.StopTimer() - ctx := setupContext() + ctx := setupContext(b) exprBytes := []byte(`{ a = 1 @@ -176,7 +191,7 @@ func BenchmarkPartialEvalObject(b *testing.B) { b.StartTimer() for n := 0; n < b.N; n++ { - expr, diags := hclsyntax.ParseExpression(exprBytes, "", hcl.InitialPos) + expr, diags := hclsyntax.ParseExpression(exprBytes, "", hhcl.InitialPos) if diags.HasErrors() { b.Fatalf(diags.Error()) } diff --git a/hcl/eval/partial_eval_test.go b/hcl/eval/partial_eval_test.go index e3707c69c..abb3235d2 100644 --- a/hcl/eval/partial_eval_test.go +++ b/hcl/eval/partial_eval_test.go @@ -4,19 +4,21 @@ package eval_test import ( - "os" "testing" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/hashicorp/hcl/v2/hclwrite" "github.com/madlambda/spells/assert" + "github.com/terramate-io/terramate/config" "github.com/terramate-io/terramate/errors" + "github.com/terramate-io/terramate/globals" "github.com/terramate-io/terramate/hcl/ast" "github.com/terramate-io/terramate/hcl/eval" "github.com/terramate-io/terramate/stdlib" + "github.com/terramate-io/terramate/test" errtest "github.com/terramate-io/terramate/test/errors" - "github.com/zclconf/go-cty/cty" + "github.com/terramate-io/terramate/test/sandbox" ) func TestPartialEval(t *testing.T) { @@ -331,28 +333,23 @@ EOT }, } { tc := tc - t.Run(tc.expr, func(t *testing.T) { + t.Run(firstN(tc.expr, 10), func(t *testing.T) { t.Parallel() - ctx := eval.NewContext(stdlib.Functions(os.TempDir())) - ctx.SetNamespace("global", map[string]cty.Value{ - "number": cty.NumberIntVal(10), - "string": cty.StringVal("terramate"), - "list": cty.ListVal([]cty.Value{ - cty.NumberIntVal(0), - cty.NumberIntVal(1), - cty.NumberIntVal(2), - cty.NumberIntVal(3), - }), - "strings": cty.ListVal([]cty.Value{ - cty.StringVal("terramate"), - cty.StringVal("is"), - cty.StringVal("fun"), - }), - "obj": cty.ObjectVal(map[string]cty.Value{ - "a": cty.NumberIntVal(0), - "b": cty.ListVal([]cty.Value{cty.StringVal("terramate")}), - }), - }) + s := sandbox.New(t) + root, err := config.LoadRoot(s.RootDir()) + assert.NoError(t, err) + predefined := eval.Stmts{} + predefined = append(predefined, test.NewStmtFrom(t, "global.string", `"terramate"`)...) + predefined = append(predefined, test.NewStmtFrom(t, "global.number", `10`)...) + predefined = append(predefined, test.NewStmtFrom(t, "global.list", `[0, 1, 2, 3]`)...) + predefined = append(predefined, test.NewStmtFrom(t, "global.strings", `["terramate", "is", "fun"]`)...) + predefined = append(predefined, test.NewStmtFrom(t, "global.obj", `{ + a = 0 + b = ["terramate"] + }`)...) + ctx := eval.New(root.Tree().Dir(), globals.NewResolver(root, predefined...)) + ctx.SetFunctions(stdlib.Functions(ctx, s.RootDir())) + expr, diags := hclsyntax.ParseExpression([]byte(tc.expr), "test.hcl", hcl.InitialPos) if diags.HasErrors() { t.Fatalf(diags.Error()) @@ -371,3 +368,14 @@ EOT }) } } + +func firstN(s string, n int) string { + i := 0 + for j := range s { + if i == n { + return s[:j] + } + i++ + } + return s +} diff --git a/hcl/eval/partial_fuzz_test.go b/hcl/eval/partial_fuzz_test.go index 89f3cc9ef..80ece6673 100644 --- a/hcl/eval/partial_fuzz_test.go +++ b/hcl/eval/partial_fuzz_test.go @@ -1,12 +1,11 @@ // Copyright 2023 Terramate GmbH // SPDX-License-Identifier: MPL-2.0 -//go:build go1.18 && !windows +//go:build go1.18 -package eval +package eval_test import ( - "math/big" "regexp" "strings" "testing" @@ -14,9 +13,12 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/rs/zerolog" + "github.com/terramate-io/terramate/globals" "github.com/terramate-io/terramate/hcl/ast" - "github.com/zclconf/go-cty/cty" - "github.com/zclconf/go-cty/cty/function" + "github.com/terramate-io/terramate/hcl/eval" + "github.com/terramate-io/terramate/runtime" + "github.com/terramate-io/terramate/test" + "github.com/terramate-io/terramate/test/sandbox" ) func FuzzPartialEval(f *testing.F) { @@ -51,25 +53,8 @@ EOT`, f.Add(seed) } - globals := map[string]cty.Value{ - "str": cty.StringVal("mineiros.io"), - "bool": cty.BoolVal(true), - "list": cty.ListVal([]cty.Value{ - cty.NumberVal(big.NewFloat(1)), - cty.NumberVal(big.NewFloat(2)), - cty.NumberVal(big.NewFloat(3)), - }), - "obj": cty.ObjectVal(map[string]cty.Value{ - "a": cty.StringVal("b"), - "b": cty.StringVal("c"), - "c": cty.StringVal("d"), - }), - } - - terramate := map[string]cty.Value{ - "path": cty.StringVal("/my/project"), - "name": cty.StringVal("happy stack"), - } + s := sandbox.New(f) + root := s.Config() f.Fuzz(func(t *testing.T, str string) { // WHY? because HCL uses the big.Float library for numbers and then @@ -91,9 +76,29 @@ EOT`, if diags.HasErrors() { return } - ctx := NewContext(map[string]function.Function{}) - ctx.SetNamespace("global", globals) - ctx.SetNamespace("terramate", terramate) + + //resolver := globals.NewResolver(root.Tree()) + + globalsStmts := eval.Stmts{ + test.NewStmt(t, `global.str`, `"mineiros.io"`), + test.NewStmt(t, `global.bool`, `true`), + test.NewStmt(t, `global.list`, `[1, 2, 3]`), + } + + globalsStmts = append(globalsStmts, test.NewStmtFrom(t, `global.obj`, `{ + a = "b" + b = "c" + c = "d" + }`)...) + + ctx := eval.New( + root.Tree().Dir(), + runtime.NewResolver(root, nil), + globals.NewResolver( + root, + globalsStmts..., + ), + ) gotExpr, err := ctx.PartialEval(parsedExpr) if err != nil { diff --git a/hcl/eval/ref.go b/hcl/eval/ref.go new file mode 100644 index 000000000..67d415d64 --- /dev/null +++ b/hcl/eval/ref.go @@ -0,0 +1,187 @@ +// Copyright 2023 Mineiros GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eval + +import ( + "bytes" + "strconv" + + hhcl "github.com/hashicorp/hcl/v2" + "github.com/terramate-io/terramate/errors" + "github.com/zclconf/go-cty/cty" +) + +type ( + // Ref is a Terramate variable reference. + // It implements the `dot operator` or `member access` syntaxex like: + // global.a.b + // global[a][b] + // In the examples above, the `global` is the Object and Path is `["a", "b"]`. + Ref struct { + Object string + Path []string + + Range hhcl.Range + } + + // RefStr is a string representation of the ref used as map keys. + RefStr string + + // Refs is a list of references. + Refs []Ref +) + +// NewRef creates a new reference. +// The provided accessor is copied, and then safe to be modified. +func NewRef(varname string, accessor ...string) Ref { + r := Ref{ + Object: varname, + Path: accessor, + } + r.Path = make([]string, len(accessor)) + copy(r.Path, accessor) + return r +} + +// AsKey returns a ref suitable to be used as a map key. +func (ref Ref) AsKey() RefStr { return RefStr(ref.String()) } + +// Comb returns all sub references of this one. +func (ref Ref) Comb() Refs { + refs := Refs{} + for i := len(ref.Path) - 1; i >= 0; i-- { + newRef := NewRef(ref.Object, ref.Path[:i+1]...) + refs = append(refs, newRef) + } + return refs +} + +// LastAccessor returns the last part of the accessor. +// Eg.: for `global.a.b.c` it returns "c". +// Eg.: for `global` it returns "global". +func (ref Ref) LastAccessor() string { + if len(ref.Path) == 0 { + return ref.Object + } + return ref.Path[len(ref.Path)-1] +} + +// String returns a string representation of the Ref. +// Note that it does not represent the syntactic ref found in the source file. +// This is an internal representation that better fits the implementation design. +func (ref Ref) String() string { + var out bytes.Buffer + out.Grow( + len(ref.Object) + + len(ref.Path)*10 + /* average key size */ + +len(ref.Path)*2, /* brackets */ + ) + + // NOTE: the buffer methods never return errors and always write the full content. + // it panics if more memory cannot be allocated. + out.WriteString(ref.Object) + for _, p := range ref.Path { + out.WriteRune('[') + out.WriteString(strconv.Quote(p)) + out.WriteRune(']') + } + return out.String() +} + +// Has returns true if ref contains the other ref and false otherwise. +func (ref Ref) Has(other Ref) bool { + if ref.Object != other.Object { + return false + } + if len(ref.Path) < len(other.Path) { + return false + } + var max int + if len(ref.Path) == len(other.Path) { + max = len(ref.Path) + } else { + max = len(other.Path) + } + + for i := 0; i < max; i++ { + if ref.Path[i] != other.Path[i] { + return false + } + } + return true +} + +// equal tells if two refs are the same. +func (ref Ref) equal(other Ref) bool { + if ref.Object != other.Object || len(ref.Path) != len(other.Path) { + return false + } + for i, p := range ref.Path { + if other.Path[i] != p { + return false + } + } + return true +} + +func (refs Refs) equal(other Refs) bool { + if len(refs) != len(other) { + return false + } + for i, ref := range refs { + if !ref.equal(other[i]) { + return false + } + } + return true +} + +// refsOf returns a distinct set of Refs contained in the expression. +func refsOf(expr hhcl.Expression) (Refs, map[string]Ref) { + ret := Refs{} + uniqueRefs := map[string]Ref{} + refsObjects := map[string]Ref{} + for _, trav := range expr.Variables() { + // they are all root traversals + ref := Ref{ + Object: trav[0].(hhcl.TraverseRoot).Name, + Range: trav.SourceRange(), + } + + inner: + for _, tt := range trav[1:] { + switch t := tt.(type) { + case hhcl.TraverseAttr: + ref.Path = append(ref.Path, t.Name) + case hhcl.TraverseSplat: + break inner + case hhcl.TraverseIndex: + if !t.Key.Type().Equals(cty.String) { + break inner + } + ref.Path = append(ref.Path, t.Key.AsString()) + default: + panic(errors.E(errors.ErrInternal, "unexpected traversal: %v", t)) + } + } + + if _, ok := uniqueRefs[ref.String()]; !ok { + uniqueRefs[ref.String()] = ref + refsObjects[ref.Object] = ref + ret = append(ret, ref) + } + } + return ret, refsObjects +} diff --git a/hcl/eval/ref_test.go b/hcl/eval/ref_test.go new file mode 100644 index 000000000..cc6dd8a53 --- /dev/null +++ b/hcl/eval/ref_test.go @@ -0,0 +1,189 @@ +// Copyright 2023 Mineiros GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eval + +import ( + "fmt" + "testing" + + hhcl "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/madlambda/spells/assert" +) + +func TestRefsOf(t *testing.T) { + t.Parallel() + + type testcase struct { + expr string + want []Ref + } + + for _, tc := range []testcase{ + { + expr: `global.a`, + want: []Ref{ + {Object: "global", Path: []string{"a"}}, + }, + }, + { + expr: `global.a.b.c + global.x.y.z`, + want: []Ref{ + {Object: "global", Path: []string{"a", "b", "c"}}, + {Object: "global", Path: []string{"x", "y", "z"}}, + }, + }, + { + expr: `global.a + global.a * global.a`, + want: []Ref{ + // unique + {Object: "global", Path: []string{"a"}}, + }, + }, + { + expr: `global["a"]`, + want: []Ref{ + {Object: "global", Path: []string{"a"}}, + }, + }, + { + expr: `global["a"]["b"]`, + want: []Ref{ + {Object: "global", Path: []string{"a", "b"}}, + }, + }, + { + expr: `global["a"][global.b]`, + want: []Ref{ + {Object: "global", Path: []string{"a"}}, + {Object: "global", Path: []string{"b"}}, + }, + }, + { + expr: `global[global]`, + want: []Ref{ + {Object: "global"}, + }, + }, + { + expr: `{ + a = global.a + b = { + c = { + d = { + e = { + f = global.b + } + } + } + } + }`, + want: []Ref{ + {Object: "global", Path: []string{"a"}}, + {Object: "global", Path: []string{"b"}}, + }, + }, + { + expr: `tm_call(global.a)+tm_call(tm_other(tm_bleh(tm_a(hidden.thing))))`, + want: []Ref{ + {Object: "global", Path: []string{"a"}}, + {Object: "hidden", Path: []string{"thing"}}, + }, + }, + } { + tc := tc + t.Run(fmt.Sprintf("refsOf(%s)", tc.expr), func(t *testing.T) { + expr, diags := hclsyntax.ParseExpression([]byte(tc.expr), "test.hcl", hhcl.InitialPos) + if diags.HasErrors() { + t.Fatal(diags.Error()) + } + refs, _ := refsOf(expr) + if !refs.equal(tc.want) { + t.Fatalf(fmt.Sprintf("(%v != %v)", refs, tc.want)) + } + }) + } +} + +func TestRefEquals(t *testing.T) { + t.Parallel() + type testcase struct { + a, b Ref + want bool + } + + for _, tc := range []testcase{ + { + a: Ref{Object: "global"}, + b: Ref{Object: "terramate"}, + want: false, + }, + { + a: Ref{Object: "global"}, + b: Ref{Object: "global"}, + want: true, + }, + { + a: Ref{Object: "global", Path: nil}, + b: Ref{Object: "global", Path: []string{}}, + want: true, + }, + { + a: Ref{Object: "global", Path: []string{"a"}}, + b: Ref{Object: "global", Path: []string{}}, + want: false, + }, + { + a: Ref{Object: "global", Path: []string{"a", "b"}}, + b: Ref{Object: "global", Path: []string{"a", "b"}}, + want: true, + }, + } { + tc := tc + t.Run(fmt.Sprintf("%s == %s", tc.a, tc.b), func(t *testing.T) { + if tc.a.equal(tc.b) != tc.want { + t.Fatalf(fmt.Sprintf("(%s == %s) != %t", tc.a, tc.b, tc.want)) + } + }) + } +} + +func TestRefString(t *testing.T) { + t.Parallel() + type testcase struct { + in Ref + want string + } + + for _, tc := range []testcase{ + { + in: Ref{Object: "global"}, + want: `global`, + }, + { + in: Ref{Object: "global", Path: []string{"a", "b"}}, + want: `global["a"]["b"]`, + }, + { + in: Ref{Object: "global", Path: []string{"spaces and\nnew lines"}}, + want: "global[\"spaces and\\nnew lines\"]", + }, + } { + tc := tc + t.Run(fmt.Sprintf("object:%s, path:%v", tc.in.Object, tc.in.Path), func(t *testing.T) { + assert.EqualStrings(t, tc.want, tc.in.String()) + }) + } +} diff --git a/hcl/eval/stmt.go b/hcl/eval/stmt.go new file mode 100644 index 000000000..4cbbd9e86 --- /dev/null +++ b/hcl/eval/stmt.go @@ -0,0 +1,271 @@ +// Copyright 2023 Terramate GmbH +// SPDX-License-Identifier: MPL-2.0 + +package eval + +import ( + "fmt" + "strconv" + + hhcl "github.com/hashicorp/hcl/v2" + "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/hashicorp/hcl/v2/hclwrite" + "github.com/terramate-io/terramate/errors" + "github.com/terramate-io/terramate/hcl/ast" + "github.com/terramate-io/terramate/hcl/info" + "github.com/terramate-io/terramate/project" + "github.com/zclconf/go-cty/cty" +) + +type ( + // Stmt represents a `var-decl` stmt. + Stmt struct { + LHS Ref + RHS *RHS + + Special bool + + // Origin is the *origin ref*. + Origin Ref + + Info Info + } + + // RHS is the right-hand side of an statement. + // It could be an evaluated value or an expression. + RHS struct { + IsEvaluated bool + Expression hhcl.Expression + Value cty.Value + } + + // Stmts is a list of statements. + Stmts []Stmt + + // Info contains origin information for the statement. + Info struct { + Scope project.Path + DefinedAt info.Range + } +) + +// ensures RHS is a tokenizer +var _ ast.Tokenizer = NewExprRHS(nil) + +// NewExprStmt creates a new stmt. +func NewExprStmt(origin Ref, lhs Ref, rhs hhcl.Expression, info Info) Stmt { + return Stmt{ + Origin: origin, + LHS: lhs, + RHS: NewExprRHS(rhs), + Info: info, + } +} + +// NewValStmt creates a new stmt. +func NewValStmt(origin Ref, rhs cty.Value, info Info) Stmt { + return NewInnerValStmt(origin, origin, rhs, info) +} + +// NewInnerValStmt creates a new stmt. +func NewInnerValStmt(origin Ref, lhs Ref, rhs cty.Value, info Info) Stmt { + return Stmt{ + Origin: origin, + LHS: lhs, + RHS: NewValRHS(rhs), + Info: info, + } +} + +// NewValRHS creates a new RHS for an evaluated value. +func NewValRHS(val cty.Value) *RHS { + return &RHS{ + Value: val, + IsEvaluated: true, + } +} + +// NewExprRHS creates a new RHS for an unevaluated expression. +func NewExprRHS(expr hhcl.Expression) *RHS { + return &RHS{ + Expression: expr, + } +} + +// String returns the string representation of the RHS. +func (rhs *RHS) String() string { + if rhs.IsEvaluated { + return ast.StringForValue(rhs.Value) + } + return string(ast.TokensForExpression(rhs.Expression).Bytes()) +} + +// Tokens tokenizes the RHS. +func (rhs *RHS) Tokens() hclwrite.Tokens { + if rhs.IsEvaluated { + return ast.TokensForValue(rhs.Value) + } + return ast.TokensForExpression(rhs.Expression) +} + +// NewExtendStmt returns a statement that extend existent object accessors. +func NewExtendStmt(origin Ref, info Info) Stmt { + return Stmt{ + Origin: origin, + LHS: origin, + Special: true, + Info: info, + } +} + +// String representation of the statement. +// This function is only meant to be used in tests. +func (stmt Stmt) String() string { + var rhs string + if stmt.Special { + rhs = "{} (Extend)" + } else if stmt.RHS.IsEvaluated { + rhs = string(ast.TokensForValue(stmt.RHS.Value).Bytes()) + } else { + rhs = string(ast.TokensForExpression(stmt.RHS.Expression).Bytes()) + } + return fmt.Sprintf("%s = %s (scope=%s, definedAt=%s)", + stmt.LHS, + rhs, + stmt.Info.Scope, + stmt.Info.DefinedAt, + ) +} + +// StmtsOfValue returns all inners statements of the provided value. +func StmtsOfValue(info Info, origin Ref, base []string, val cty.Value) Stmts { + stmts := Stmts{} + if !val.Type().IsObjectType() { + stmts = append(stmts, NewInnerValStmt( + origin, + NewRef(origin.Object, base...), + val, + info, + )) + return stmts + } + + newbase := make([]string, len(base)+1) + copy(newbase, base) + last := len(newbase) - 1 + objMap := val.AsValueMap() + for key, value := range objMap { + newbase[last] = key + stmts = append(stmts, StmtsOfValue(info, origin, newbase, value)...) + } + return stmts +} + +// StmtsOfExpr returns all statements of the expr. +func StmtsOfExpr(info Info, origin Ref, base []string, expr hhcl.Expression) (Stmts, error) { + stmts := Stmts{} + newbase := make([]string, len(base)+1) + copy(newbase, base) + last := len(newbase) - 1 + switch e := expr.(type) { + case *hclsyntax.ObjectConsExpr: + for _, item := range e.Items { + var key string + switch v := item.KeyExpr.(type) { + case *hclsyntax.LiteralValueExpr: + if !v.Val.Type().Equals(cty.String) { + // TODO(i4k): test this. + panic(errors.E("unexpected key type %s", v.Val.Type().FriendlyName())) + } + + key = v.Val.AsString() + case *hclsyntax.ObjectConsKeyExpr: + if v.ForceNonLiteral { + panic("TODO") + } + + key = string(ast.TokensForExpression(v).Bytes()) + if key[0] == '"' { + // TODO(i4k): test this + key, _ = strconv.Unquote(key) + } + default: + // TODO(i4k): test this. + panic(errors.E("unexpected key type %T", v)) + } + + newbase[last] = key + newStmts, err := StmtsOfExpr(info, origin, newbase, item.ValueExpr) + if err != nil { + return nil, err + } + stmts = append(stmts, newStmts...) + } + default: + stmts = append(stmts, NewExprStmt( + origin, + NewRef(origin.Object, newbase[0:last]...), + expr, + info, + )) + } + + return stmts, nil +} + +// SelectBy selects the statements related to ref. +func (stmts Stmts) SelectBy(ref Ref, atChild map[RefStr]Ref) (Stmts, bool) { + found := false + contains := Stmts{} + isContainedBy := Stmts{} +outer: + for _, stmt := range stmts { + if stmt.LHS.Object != ref.Object { + // unrelated + continue + } + + if !stmt.Special { + for _, gotRef := range atChild { + if stmt.LHS.Has(gotRef) { + continue outer + } + } + } + + if stmt.LHS.Has(ref) { + contains = append(contains, stmt) + if stmt.Origin.equal(ref) || stmt.LHS.equal(ref) { + found = true + } + } else { + if found { + return contains, true + } + if ref.Has(stmt.LHS) { + isContainedBy = append(isContainedBy, stmt) + } + } + } + + if len(contains) == 0 { + return isContainedBy, false + } + + contains = append(contains, isContainedBy...) + return contains, false +} + +// NewInfo returns a new Info. +func NewInfo(scope project.Path, rng info.Range) Info { + return Info{ + Scope: scope, + DefinedAt: rng, + } +} + +func newBuiltinInfo(scope project.Path) Info { + return Info{ + Scope: scope, + } +} diff --git a/hcl/eval/stmt_filter_test.go b/hcl/eval/stmt_filter_test.go new file mode 100644 index 000000000..8fd8b82a3 --- /dev/null +++ b/hcl/eval/stmt_filter_test.go @@ -0,0 +1,206 @@ +// Copyright 2023 Mineiros GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package eval_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + hhcl "github.com/hashicorp/hcl/v2" + "github.com/zclconf/go-cty/cty" + + "github.com/terramate-io/terramate/hcl/eval" + "github.com/terramate-io/terramate/hcl/info" + "github.com/terramate-io/terramate/project" +) + +func TestStmtSelectBy(t *testing.T) { + t.Parallel() + + type testcase struct { + name string + ref eval.Ref + stmts eval.Stmts + want eval.Stmts + found bool + } + + for _, tc := range []testcase{ + { + name: "exact match with origin", + ref: eval.Ref{Object: "global", Path: []string{"a", "b"}}, + stmts: eval.Stmts{ + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"a", "b"}}, + Origin: eval.Ref{Object: "global", Path: []string{"a", "b"}}, + }, + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"c"}}, + Origin: eval.Ref{Object: "global", Path: []string{"c"}}, + }, + }, + want: eval.Stmts{ + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"a", "b"}}, + Origin: eval.Ref{Object: "global", Path: []string{"a", "b"}}, + }, + }, + found: true, + }, + { + name: "exact match with lhs", + ref: eval.Ref{Object: "global", Path: []string{"a", "b"}}, + stmts: eval.Stmts{ + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"a", "b"}}, + Origin: eval.Ref{Object: "global", Path: []string{"a"}}, + }, + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"c"}}, + Origin: eval.Ref{Object: "global", Path: []string{"c"}}, + }, + }, + want: eval.Stmts{ + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"a", "b"}}, + Origin: eval.Ref{Object: "global", Path: []string{"a"}}, + }, + }, + found: true, + }, + { + name: "partial match", + ref: eval.Ref{Object: "global", Path: []string{"a"}}, + stmts: eval.Stmts{ + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"a", "b"}}, + Origin: eval.Ref{Object: "global", Path: []string{"a", "b"}}, + }, + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"a", "c"}}, + Origin: eval.Ref{Object: "global", Path: []string{"a", "c"}}, + }, + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"c"}}, + Origin: eval.Ref{Object: "global", Path: []string{"c"}}, + }, + }, + want: eval.Stmts{ + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"a", "b"}}, + Origin: eval.Ref{Object: "global", Path: []string{"a", "b"}}, + }, + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"a", "c"}}, + Origin: eval.Ref{Object: "global", Path: []string{"a", "c"}}, + }, + }, + found: false, + }, + { + name: "no match -- in same branch", + ref: eval.Ref{Object: "global", Path: []string{"a", "b", "c"}}, + stmts: eval.Stmts{ + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"a", "b"}}, + Origin: eval.Ref{Object: "global", Path: []string{"a", "b"}}, + }, + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"a", "c"}}, + Origin: eval.Ref{Object: "global", Path: []string{"a", "c"}}, + }, + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"c"}}, + Origin: eval.Ref{Object: "global", Path: []string{"c"}}, + }, + }, + want: eval.Stmts{ + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"a", "b"}}, + Origin: eval.Ref{Object: "global", Path: []string{"a", "b"}}, + }, + }, + found: false, + }, + { + name: "no match -- in different branch", + ref: eval.Ref{Object: "global", Path: []string{"unknown"}}, + stmts: eval.Stmts{ + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"a", "b"}}, + Origin: eval.Ref{Object: "global", Path: []string{"a", "b"}}, + }, + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"a", "c"}}, + Origin: eval.Ref{Object: "global", Path: []string{"a", "c"}}, + }, + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"c"}}, + Origin: eval.Ref{Object: "global", Path: []string{"c"}}, + }, + }, + want: eval.Stmts{}, + found: false, + }, + { + name: "root match -- should always return all globals", + ref: eval.Ref{Object: "global"}, + stmts: eval.Stmts{ + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"a", "b"}}, + }, + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"a"}}, + }, + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"b"}}, + }, + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"c"}}, + }, + }, + want: eval.Stmts{ + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"a", "b"}}, + }, + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"a"}}, + }, + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"b"}}, + }, + eval.Stmt{ + LHS: eval.Ref{Object: "global", Path: []string{"c"}}, + }, + }, + }, + } { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + got, found := tc.stmts.SelectBy(tc.ref, map[eval.RefStr]eval.Ref{}) + if found != tc.found { + t.Fatalf("expected found=%t but got %t", found, tc.found) + } + if diff := cmp.Diff(got, tc.want, + cmp.AllowUnexported(eval.Stmt{}, project.Path{}, info.Range{}, info.Pos{}, hhcl.Range{}), + cmpopts.IgnoreTypes(cty.Value{})); diff != "" { + t.Fatal(diff) + } + }) + } +} diff --git a/hcl/hcl.go b/hcl/hcl.go index f614da407..8c224f2bf 100644 --- a/hcl/hcl.go +++ b/hcl/hcl.go @@ -20,6 +20,7 @@ import ( "github.com/terramate-io/terramate/hcl/ast" "github.com/terramate-io/terramate/hcl/eval" "github.com/terramate-io/terramate/hcl/info" + "github.com/terramate-io/terramate/project" "github.com/terramate-io/terramate/stdlib" "github.com/zclconf/go-cty/cty" "golang.org/x/exp/slices" @@ -222,24 +223,6 @@ type GenFileBlock struct { Asserts []AssertConfig } -// Evaluator represents a Terramate evaluator -type Evaluator interface { - // Eval evaluates the given expression returning a value. - Eval(hcl.Expression) (cty.Value, error) - - // PartialEval partially evaluates the given expression returning the - // tokens that form the result of the partial evaluation. Any unknown - // namespace access are ignored and left as is, while known namespaces - // are substituted by its value. - PartialEval(hcl.Expression) (hcl.Expression, error) - - // SetNamespace adds a new namespace, replacing any with the same name. - SetNamespace(name string, values map[string]cty.Value) - - // DeleteNamespace deletes a namespace. - DeleteNamespace(name string) -} - // TerramateParser is an HCL parser tailored for Terramate configuration schema. // As the Terramate configuration can span multiple files in the same directory, // this API allows you to define the exact set of files (and contents) that are @@ -306,6 +289,9 @@ func NewTerramateParser(rootdir string, dir string, experiments ...string) (*Ter return nil, errors.E("%s is not a directory", dir) } + evalctx := eval.New(project.RootPath) + evalctx.SetFunctions(stdlib.Functions(evalctx, dir)) + return &TerramateParser{ Config: NewTopLevelRawConfig(), Imported: NewTopLevelRawConfig(), @@ -315,7 +301,7 @@ func NewTerramateParser(rootdir string, dir string, experiments ...string) (*Ter files: map[string][]byte{}, hclparser: hclparse.NewParser(), parsedFiles: make(map[string]parsedFile), - evalctx: eval.NewContext(stdlib.Functions(dir)), + evalctx: evalctx, }, nil } diff --git a/lets/doc.go b/lets/doc.go new file mode 100644 index 000000000..8134c0793 --- /dev/null +++ b/lets/doc.go @@ -0,0 +1,5 @@ +// Copyright 2023 Terramate GmbH +// SPDX-License-Identifier: MPL-2.0 + +// Package lets provides functions for evaluating `lets` blocks. +package lets diff --git a/lets/lets.go b/lets/lets.go index 402ea1e21..c0d67cb3c 100644 --- a/lets/lets.go +++ b/lets/lets.go @@ -1,208 +1,114 @@ // Copyright 2023 Terramate GmbH // SPDX-License-Identifier: MPL-2.0 -// Package lets provides parsing and evaluation of lets blocks. package lets import ( - hhcl "github.com/hashicorp/hcl/v2" + "sort" + "github.com/terramate-io/terramate/errors" "github.com/terramate-io/terramate/hcl/ast" "github.com/terramate-io/terramate/hcl/eval" - "github.com/terramate-io/terramate/hcl/fmt" - "github.com/terramate-io/terramate/hcl/info" "github.com/terramate-io/terramate/mapexpr" + "github.com/terramate-io/terramate/project" "github.com/zclconf/go-cty/cty" ) -// Errors returned when parsing and evaluating lets. const ( - ErrEval errors.Kind = "lets eval" + // ErrRedefined indicates the lets variable is being redefined in the same + // scope. ErrRedefined errors.Kind = "lets redefined" -) - -type ( - // Expr is an unevaluated let expression. - Expr struct { - // Origin contains the information where the expr is defined. - Origin info.Range - - hhcl.Expression - } - - // Exprs is the map of unevaluated let expressions visible in a - // directory. - Exprs map[string]Expr - // Value is an evaluated let. - Value struct { - Origin info.Range - - cty.Value - } - - // Map is an evaluated lets map. - Map map[string]Value + nsName = "let" ) -// Load loads all the lets from the hcl blocks. -func Load(letblock *ast.MergedBlock, ctx *eval.Context) error { - exprs, err := loadExprs(letblock) - if err != nil { - return err - } +// Resolver is the lets resolver. +type Resolver struct { + scope project.Path + block *ast.MergedBlock - return exprs.Eval(ctx) + // cache of statements. + cached eval.Stmts } -// Eval evaluates all lets expressions and returns an EvalReport.. -func (letExprs Exprs) Eval(ctx *eval.Context) error { - lets := Map{} - pendingExprsErrs := map[string]*errors.List{} - pendingExprs := make(Exprs) - - copyexprs(pendingExprs, letExprs) - removeUnset(pendingExprs) - - if !ctx.HasNamespace("let") { - ctx.SetNamespace("let", map[string]cty.Value{}) +// NewResolver is a resolver for let.* references. +func NewResolver(block *ast.MergedBlock) *Resolver { + r := &Resolver{ + block: block, } - - for len(pendingExprs) > 0 { - amountEvaluated := 0 - - pendingExpression: - for name, expr := range pendingExprs { - vars := expr.Variables() - pendingExprsErrs[name] = errors.L() - - for _, namespace := range vars { - if !ctx.HasNamespace(namespace.RootName()) { - pendingExprsErrs[name].Append(errors.E( - ErrEval, - namespace.SourceRange(), - "unknown variable namespace: %s", namespace.RootName(), - )) - - continue - } - - if namespace.RootName() != "let" { - continue - } - - switch attr := namespace[1].(type) { - case hhcl.TraverseAttr: - if _, isPending := pendingExprs[attr.Name]; isPending { - continue pendingExpression - } - default: - panic("unexpected type of traversal - this is a BUG") - } - } - - if err := pendingExprsErrs[name].AsError(); err != nil { - continue - } - - val, err := ctx.Eval(expr) - if err != nil { - pendingExprsErrs[name].Append(errors.E(ErrEval, err, "let.%s", name)) - continue - } - - lets[name] = Value{ - Origin: expr.Origin, - Value: val, - } - - amountEvaluated++ - - delete(pendingExprs, name) - delete(pendingExprsErrs, name) - - ctx.SetNamespace("let", lets.Attributes()) - } - - if amountEvaluated == 0 { - break - } - } - - errs := errors.L() - for name, expr := range pendingExprs { - err := pendingExprsErrs[name].AsError() - if err == nil { - err = errors.E(expr.Range(), "undefined let %s", name) - } - errs.AppendWrap(ErrEval, err) - } - - return errs.AsError() + return r } -// String provides a string representation of the evaluated lets. -func (lets Map) String() string { - return fmt.FormatAttributes(lets.Attributes()) -} +// Name of the resolver. +func (*Resolver) Name() string { return nsName } -// Attributes returns all the lets attributes, the key in the map -// is the attribute name with its corresponding value mapped -func (lets Map) Attributes() map[string]cty.Value { - attrcopy := map[string]cty.Value{} - for k, v := range lets { - attrcopy[k] = v.Value - } - return attrcopy +// Prevalue returns predeclared lets variables. +func (r *Resolver) Prevalue() cty.Value { + return cty.EmptyObjectVal } -func removeUnset(exprs Exprs) { - for name, expr := range exprs { - traversal, diags := hhcl.AbsTraversalForExpr(expr.Expression) - if diags.HasErrors() { - continue - } - if len(traversal) != 1 { - continue - } - if traversal.RootName() == "unset" { - delete(exprs, name) - } +// LookupRef lookup the lets references. +func (r *Resolver) LookupRef(_ project.Path, ref eval.Ref) ([]eval.Stmts, error) { + stmts, err := r.loadStmts() + if err != nil { + return nil, err } -} - -func copyexprs(dst, src Exprs) { - for k, v := range src { - dst[k] = v + var filtered eval.Stmts + if len(ref.Path) == 0 { + filtered = stmts + } else { + filtered, _ = stmts.SelectBy(ref, map[eval.RefStr]eval.Ref{}) } -} -func loadExprs(letblock *ast.MergedBlock) (Exprs, error) { - letExprs := Exprs{} + return []eval.Stmts{filtered}, nil +} - for _, attr := range letblock.Attributes.SortedList() { - letExprs[attr.Name] = Expr{ - Origin: attr.Range, - Expression: attr.Expr, - } +func (r *Resolver) loadStmts() (eval.Stmts, error) { + stmts := r.cached + if stmts != nil { + return stmts, nil } - for _, mapBlock := range letblock.Blocks { - varName := mapBlock.Labels[0] - if _, ok := letblock.Attributes[varName]; ok { + for _, varsBlock := range r.block.Blocks { + varName := varsBlock.Labels[0] + if _, ok := r.block.Attributes[varName]; ok { return nil, errors.E( ErrRedefined, "map label %s conflicts with let.%s attribute", varName, varName) } - mapExpr, err := mapexpr.NewMapExpr(mapBlock) + + origin := eval.NewRef(nsName, varName) + + expr, err := mapexpr.NewMapExpr(varsBlock) + if err != nil { + return nil, errors.E(err, "failed to interpret map block") + } + + blockStmts, err := eval.StmtsOfExpr(eval.NewInfo(r.scope, varsBlock.RawOrigins[0].Range), origin, origin.Path, expr) if err != nil { - return nil, errors.E(ErrEval, err) + return nil, err } - letExprs[mapBlock.Labels[0]] = Expr{ - Origin: mapBlock.RawOrigins[0].Range, - Expression: mapExpr, + stmts = append(stmts, blockStmts...) + } + + attrs := r.block.Attributes.SortedList() + for _, attr := range attrs { + origin := eval.NewRef(nsName, attr.Name) + blockStmts, err := eval.StmtsOfExpr(eval.NewInfo(r.scope, attr.Range), origin, origin.Path, attr.Expr) + if err != nil { + return nil, err } + stmts = append(stmts, blockStmts...) } - return letExprs, nil + // bigger refs -> smaller refs + sort.Slice(stmts, func(i, j int) bool { + if len(stmts[i].Origin.Path) != len(stmts[j].Origin.Path) { + return len(stmts[i].Origin.Path) > len(stmts[j].Origin.Path) + } + return len(stmts[i].LHS.Path) > len(stmts[j].LHS.Path) + }) + + r.cached = stmts + return stmts, nil } diff --git a/project/project.go b/project/project.go index 797d27505..eb1b8f490 100644 --- a/project/project.go +++ b/project/project.go @@ -36,6 +36,9 @@ type Runtime map[string]cty.Value // TODO(i4k): get rid of this limit. const MaxGlobalLabels = 256 +// RootPath is the root ("/") path. +var RootPath = NewPath("/") + // NewPath creates a new project path. // It panics if a relative path is provided. func NewPath(p string) Path { diff --git a/run/env.go b/run/env.go index 5484e7874..504eae87b 100644 --- a/run/env.go +++ b/run/env.go @@ -7,10 +7,15 @@ import ( "os" "strings" + hhcl "github.com/hashicorp/hcl/v2" + "github.com/terramate-io/terramate/runtime" + "github.com/terramate-io/terramate/config" "github.com/terramate-io/terramate/errors" "github.com/terramate-io/terramate/globals" "github.com/terramate-io/terramate/hcl/eval" + "github.com/terramate-io/terramate/hcl/info" + "github.com/terramate-io/terramate/project" "github.com/terramate-io/terramate/stdlib" "github.com/rs/zerolog/log" @@ -18,9 +23,6 @@ import ( ) const ( - // ErrLoadingGlobals indicates that an error happened while loading globals. - ErrLoadingGlobals errors.Kind = "loading globals to evaluate terramate.config.run.env configuration" - // ErrEval indicates that an error happened while evaluating one of the // terramate.config.run.env attributes. ErrEval errors.Kind = "evaluating terramate.config.run.env attribute" @@ -49,17 +51,18 @@ func LoadEnv(root *config.Root, st *config.Stack) (EnvVars, error) { return nil, nil } - globalsReport := globals.ForStack(root, st) - if err := globalsReport.AsError(); err != nil { - return nil, errors.E(ErrLoadingGlobals, err) - } + tree, _ := root.Lookup(st.Dir) - evalctx := eval.NewContext(stdlib.Functions(st.HostDir(root))) - runtime := root.Runtime() - runtime.Merge(st.RuntimeValues(root)) - evalctx.SetNamespace("terramate", runtime) - evalctx.SetNamespace("global", globalsReport.Globals.AsValueMap()) - evalctx.SetEnv(os.Environ()) + evalctx := eval.New( + tree.Dir(), + globals.NewResolver(root), + runtime.NewResolver(root, st), + &resolver{ + rootdir: root.HostDir(), + env: os.Environ(), + }, + ) + evalctx.SetFunctions(stdlib.Functions(evalctx, st.HostDir(root))) envVars := EnvVars{} @@ -90,6 +93,49 @@ func LoadEnv(root *config.Root, st *config.Stack) (EnvVars, error) { return envVars, nil } +type resolver struct { + rootdir string + env []string +} + +func (r *resolver) Name() string { return "env" } + +func (r *resolver) Prevalue() cty.Value { return cty.EmptyObjectVal } + +func (r *resolver) loadStmts() (eval.Stmts, error) { + stmts := make(eval.Stmts, len(r.env)) + for _, env := range r.env { + nameval := strings.Split(env, "=") + + ref := eval.Ref{ + Object: r.Name(), + Path: []string{nameval[0]}, + } + + val := cty.StringVal(strings.Join(nameval[1:], "=")) + stmts = append(stmts, eval.Stmt{ + Origin: ref, + LHS: ref, + RHS: eval.NewValRHS(val), + Info: eval.NewInfo(project.NewPath("/"), info.NewRange(r.rootdir, hhcl.Range{ + Start: hhcl.InitialPos, + End: hhcl.InitialPos, + Filename: ``, + })), // env is root-scoped + }) + } + return stmts, nil +} + +func (r *resolver) LookupRef(_ project.Path, ref eval.Ref) ([]eval.Stmts, error) { + stmts, err := r.loadStmts() + if err != nil { + return nil, err + } + filtered, _ := stmts.SelectBy(ref, map[eval.RefStr]eval.Ref{}) + return []eval.Stmts{filtered}, nil +} + func getEnv(key string, environ []string) (string, bool) { for i := len(environ) - 1; i >= 0; i-- { env := environ[i] diff --git a/run/env_test.go b/run/env_test.go index d7a815ad8..1ff88ac6d 100644 --- a/run/env_test.go +++ b/run/env_test.go @@ -14,6 +14,7 @@ import ( "github.com/terramate-io/terramate/config" "github.com/terramate-io/terramate/errors" "github.com/terramate-io/terramate/hcl" + "github.com/terramate-io/terramate/hcl/eval" "github.com/terramate-io/terramate/project" "github.com/terramate-io/terramate/run" "github.com/terramate-io/terramate/test" @@ -172,7 +173,7 @@ func TestLoadRunEnv(t *testing.T) { }, want: map[string]result{ "stack": { - enverr: errors.E(run.ErrLoadingGlobals), + enverr: errors.E(eval.ErrEval), }, }, }, diff --git a/runtime/doc.go b/runtime/doc.go new file mode 100644 index 000000000..de78e421a --- /dev/null +++ b/runtime/doc.go @@ -0,0 +1,17 @@ +// Copyright 2023 Mineiros GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package runtime provides functions for the "terramate" variable namespace, +// the so-called "runtime" variable. +package runtime diff --git a/runtime/runtime_resolver.go b/runtime/runtime_resolver.go new file mode 100644 index 000000000..85491070c --- /dev/null +++ b/runtime/runtime_resolver.go @@ -0,0 +1,45 @@ +// Copyright 2023 Terramate GmbH +// SPDX-License-Identifier: MPL-2.0 + +package runtime + +import ( + "github.com/terramate-io/terramate/config" + "github.com/terramate-io/terramate/hcl/eval" + "github.com/terramate-io/terramate/project" + "github.com/zclconf/go-cty/cty" +) + +// Resolver is the runtime resolver. +// It resolves references to variables of form `terramate.` +type Resolver struct { + terramate cty.Value + scope project.Path +} + +// NewResolver returns a new resolver for terramate runtime variables. +func NewResolver(root *config.Root, stack *config.Stack) *Resolver { + runtime := root.Runtime() + var scope project.Path + if stack != nil { + runtime.Merge(stack.RuntimeValues(root)) + scope = stack.Dir + } else { + scope = project.NewPath("/") + } + return &Resolver{ + terramate: cty.ObjectVal(runtime), + scope: scope, + } +} + +// Name returns the variable name. +func (r *Resolver) Name() string { return "terramate" } + +// Prevalue returns a predeclared value. +func (r *Resolver) Prevalue() cty.Value { return r.terramate } + +// LookupRef lookup pending runtime variables. Not implemeneted at the moment. +func (r *Resolver) LookupRef(_ project.Path, _ eval.Ref) ([]eval.Stmts, error) { + return []eval.Stmts{}, nil +} diff --git a/scripts/cpu_profile.sh b/scripts/cpu_profile.sh new file mode 100644 index 000000000..678b55c65 --- /dev/null +++ b/scripts/cpu_profile.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Copyright 2023 Terramate GmbH +# SPDX-License-Identifier: MPL-2.0 + + +WORKSPACE_ROOT="$HOME/src" +TM="$WORKSPACE_ROOT/terramate" +IAC="$WORKSPACE_ROOT/iac-gcloud" + +delete_terraform_files() { + pushd "$IAC" + fd '_terramate.*tf' --exec rm + printf "Deleted generated terraform files...\n" + popd +} + +build_terramate() { + pushd "${WORKSPACE_ROOT}/terramate" + printf "Building Terramate from source...\n" + make test/build + printf "Built Terramate\n" + popd +} + +main() { + build_terramate + mv "${TM}/bin/test-terramate" "$IAC/terramate" + delete_terraform_files + + pushd "$IAC" + printf "Starting profiling run...\n\n" + ./terramate --cpu-profiling generate + printf "Profiling run done, moving to '$WORKSPACE_ROOT/terramate.prof'\n" + mv "./terramate.prof" "$WORKSPACE_ROOT" + popd +} + +main diff --git a/stack/eval.go b/stack/eval.go deleted file mode 100644 index d948bbcfc..000000000 --- a/stack/eval.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2023 Terramate GmbH -// SPDX-License-Identifier: MPL-2.0 - -package stack - -import ( - "github.com/terramate-io/terramate/config" - "github.com/terramate-io/terramate/hcl/eval" - "github.com/terramate-io/terramate/stdlib" -) - -// EvalCtx represents the evaluation context of a stack. -type EvalCtx struct { - *eval.Context - - root *config.Root -} - -// NewEvalCtx creates a new stack evaluation context. -func NewEvalCtx(root *config.Root, stack *config.Stack, globals *eval.Object) *EvalCtx { - evalctx := eval.NewContext(stdlib.Functions(stack.HostDir(root))) - evalwrapper := &EvalCtx{ - Context: evalctx, - root: root, - } - evalwrapper.SetMetadata(stack) - evalwrapper.SetGlobals(globals) - return evalwrapper -} - -// SetGlobals sets the given globals in the stack evaluation context. -func (e *EvalCtx) SetGlobals(g *eval.Object) { - e.SetNamespace("global", g.AsValueMap()) -} - -// SetMetadata sets the given metadata in the stack evaluation context. -func (e *EvalCtx) SetMetadata(st *config.Stack) { - runtime := e.root.Runtime() - runtime.Merge(st.RuntimeValues(e.root)) - e.SetNamespace("terramate", runtime) -} diff --git a/stack/manager.go b/stack/manager.go index d02402bfa..abd3dbe57 100644 --- a/stack/manager.go +++ b/stack/manager.go @@ -176,7 +176,7 @@ func (m *Manager) ListChanged() (*Report, error) { continue } - s, err := config.NewStackFromHCL(m.root.HostDir(), cfg.Node) + s, err := config.NewStackFromHCL(m.root.HostDir(), cfg) if err != nil { return nil, errors.E(errListChanged, err) } @@ -210,7 +210,7 @@ func (m *Manager) ListChanged() (*Report, error) { } } - s, err := config.NewStackFromHCL(m.root.HostDir(), stackTree.Node) + s, err := config.NewStackFromHCL(m.root.HostDir(), stackTree) if err != nil { return nil, errors.E(errListChanged, err) } diff --git a/stdlib/funcs.go b/stdlib/funcs.go index d1d65e265..975bdeac8 100644 --- a/stdlib/funcs.go +++ b/stdlib/funcs.go @@ -17,6 +17,7 @@ import ( "github.com/terramate-io/terramate/errors" "github.com/terramate-io/terramate/event" "github.com/terramate-io/terramate/hcl/ast" + "github.com/terramate-io/terramate/hcl/eval" "github.com/terramate-io/terramate/modvendor" "github.com/terramate-io/terramate/project" "github.com/terramate-io/terramate/tf" @@ -25,15 +26,20 @@ import ( "github.com/zclconf/go-cty/cty/function" ) -var regexCache map[string]*regexp.Regexp +type regexDataCache struct { + pattern *regexp.Regexp + retType cty.Type +} + +var regexCache map[string]regexDataCache func init() { - regexCache = map[string]*regexp.Regexp{} + regexCache = map[string]regexDataCache{} } // Functions returns all the Terramate default functions. // The `basedir` must be an absolute path for an existent directory or it panics. -func Functions(basedir string) map[string]function.Function { +func Functions(evalctx *eval.Context, basedir string) map[string]function.Function { if !filepath.IsAbs(basedir) { panic(errors.E(errors.ErrInternal, "context created with relative path: %q", basedir)) } @@ -65,7 +71,7 @@ func Functions(basedir string) map[string]function.Function { tmfuncs["tm_abspath"] = AbspathFunc(basedir) // sane ternary - tmfuncs["tm_ternary"] = TernaryFunc() + tmfuncs["tm_ternary"] = TernaryFunc(evalctx) tmfuncs["tm_version_match"] = VersionMatch() return tmfuncs @@ -73,8 +79,8 @@ func Functions(basedir string) map[string]function.Function { // NoFS returns all Terramate functions but excluding fs-related // functions. -func NoFS(basedir string) map[string]function.Function { - funcs := Functions(basedir) +func NoFS(evalctx *eval.Context, basedir string) map[string]function.Function { + funcs := Functions(evalctx, basedir) fsFuncNames := []string{ "tm_abspath", "tm_file", @@ -115,6 +121,12 @@ func Regex() function.Function { return cty.DynamicPseudoType, nil } + pattern := args[0].AsString() + cache, ok := regexCache[pattern] + if ok { + return cache.retType, nil + } + retTy, err := regexPatternResultType(args[0].AsString()) if err != nil { err = function.NewArgError(0, err) @@ -126,19 +138,19 @@ func Regex() function.Function { return cty.DynamicVal, nil } - re, ok := regexCache[args[0].AsString()] + pattern := args[0].AsString() + cache, ok := regexCache[pattern] if !ok { - panic("should be in the cache") + panic(errors.E(errors.ErrInternal, "pattern %s should be in the cache", pattern)) } str := args[1].AsString() - - captureIdxs := re.FindStringSubmatchIndex(str) + captureIdxs := cache.pattern.FindStringSubmatchIndex(str) if captureIdxs == nil { return cty.NilVal, fmt.Errorf("pattern did not match any part of the given string") } - return regexPatternResult(re, str, captureIdxs, retType), nil + return regexPatternResult(cache.pattern, str, captureIdxs, retType), nil }, }) } @@ -149,24 +161,32 @@ func Regex() function.Function { // // Returns an error if parsing fails or if the pattern uses a mixture of // named and unnamed capture groups, which is not permitted. -func regexPatternResultType(pattern string) (cty.Type, error) { - re, ok := regexCache[pattern] - if !ok { - var rawErr error - re, rawErr = regexp.Compile(pattern) - switch err := rawErr.(type) { - case *resyntax.Error: - return cty.NilType, fmt.Errorf("invalid regexp pattern: %s in %s", err.Code, err.Expr) - case error: - // Should never happen, since all regexp compile errors should - // be resyntax.Error, but just in case... - return cty.NilType, fmt.Errorf("error parsing pattern: %s", err) - } +func regexPatternResultType(pattern string) (retType cty.Type, err error) { + cache, ok := regexCache[pattern] + if ok { + panic(errors.E(errors.ErrInternal, "regex should not be cached at this point")) + } + var rawErr error + re, rawErr := regexp.Compile(pattern) + switch err := rawErr.(type) { + case *resyntax.Error: + return cty.NilType, fmt.Errorf("invalid regexp pattern: %s in %s", err.Code, err.Expr) + case error: + // Should never happen, since all regexp compile errors should + // be resyntax.Error, but just in case... + return cty.NilType, fmt.Errorf("error parsing pattern: %s", err) + } - regexCache[pattern] = re + cache = regexDataCache{ + pattern: re, } - allNames := re.SubexpNames()[1:] + defer func() { + cache.retType = retType + regexCache[pattern] = cache + }() + + allNames := cache.pattern.SubexpNames()[1:] var names []string unnamed := 0 for _, name := range allNames { diff --git a/stdlib/funcs_test.go b/stdlib/funcs_test.go index b9222021f..bdd12dc25 100644 --- a/stdlib/funcs_test.go +++ b/stdlib/funcs_test.go @@ -129,9 +129,10 @@ func TestTmVendor(t *testing.T) { vendordir := project.NewPath(tcase.vendorDir) targetdir := project.NewPath(tcase.targetDir) - funcs := stdlib.Functions(rootdir) + ctx := eval.New(project.NewPath("/")) + funcs := stdlib.Functions(ctx, rootdir) funcs[stdlib.Name("vendor")] = stdlib.VendorFunc(targetdir, vendordir, events) - ctx := eval.NewContext(funcs) + ctx.SetFunctions(funcs) gotEvents := []event.VendorRequest{} done := make(chan struct{}) @@ -162,9 +163,10 @@ func TestTmVendor(t *testing.T) { // piggyback on the current tests to validate that // it also works with a nil channel (no interest on events). t.Run("works with nil events channel", func(t *testing.T) { - funcs := stdlib.Functions(rootdir) + ctx := eval.New(project.NewPath("/")) + funcs := stdlib.Functions(ctx, rootdir) funcs["tm_vendor"] = stdlib.VendorFunc(targetdir, vendordir, nil) - ctx := eval.NewContext(funcs) + ctx.SetFunctions(funcs) val, err := ctx.Eval(test.NewExpr(t, tcase.expr)) assert.NoError(t, err) @@ -285,7 +287,9 @@ func TestStdlibTmVersionMatch(t *testing.T) { tc := tc t.Run(tc.expr, func(t *testing.T) { rootdir := test.TempDir(t) - ctx := eval.NewContext(stdlib.Functions(rootdir)) + ctx := eval.New(project.NewPath("/")) + funcs := stdlib.Functions(ctx, rootdir) + ctx.SetFunctions(funcs) val, err := ctx.Eval(test.NewExpr(t, tc.expr)) errors.Assert(t, err, tc.wantErr) if err != nil { @@ -304,7 +308,7 @@ func TestStdlibNewFunctionsMustPanicIfRelativeBaseDir(t *testing.T) { t.Fatal("eval.NewContext() did not panic with relative basedir") } }() - _ = stdlib.Functions("relative") + _ = stdlib.Functions(eval.New(project.NewPath("/")), "relative") } func TestStdlibNewFunctionsMustPanicIfBasedirIsNonExistent(t *testing.T) { @@ -315,7 +319,7 @@ func TestStdlibNewFunctionsMustPanicIfBasedirIsNonExistent(t *testing.T) { } }() - stdlib.Functions(filepath.Join(test.TempDir(t), "non-existent")) + stdlib.Functions(eval.New(project.NewPath("/")), filepath.Join(test.TempDir(t), "non-existent")) } func TestStdlibNewFunctionsFailIfBasedirIsNotADirectory(t *testing.T) { @@ -325,9 +329,8 @@ func TestStdlibNewFunctionsFailIfBasedirIsNotADirectory(t *testing.T) { t.Fatal("eval.NewContext() did not panic if basedir is not a dir") } }() - path := test.WriteFile(t, test.TempDir(t), "somefile.txt", ``) - _ = stdlib.Functions(path) + _ = stdlib.Functions(eval.New(project.NewPath("/")), path) } func init() { diff --git a/stdlib/ternary.go b/stdlib/ternary.go index 38ee09f28..ad8e5490f 100644 --- a/stdlib/ternary.go +++ b/stdlib/ternary.go @@ -17,7 +17,7 @@ import ( // TernaryFunc is the `tm_ternary` function implementation. // The `tm_ternary(cond, expr1, expr2)` will return expr1 if `cond` evaluates // to `true` and `expr2` otherwise. -func TernaryFunc() function.Function { +func TernaryFunc(evalctx *eval.Context) function.Function { return function.New(&function.Spec{ Params: []function.Parameter{ { @@ -34,30 +34,28 @@ func TernaryFunc() function.Function { }, }, Type: func(args []cty.Value) (cty.Type, error) { - v, err := ternary(args[0], args[1], args[2]) + v, err := ternary(evalctx, args[0], args[1], args[2]) if err != nil { return cty.NilType, err } return v.Type(), nil }, Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) { - return ternary(args[0], args[1], args[2]) + return ternary(evalctx, args[0], args[1], args[2]) }, }) } -func ternary(cond cty.Value, val1, val2 cty.Value) (cty.Value, error) { +func ternary(evalctx *eval.Context, cond cty.Value, val1, val2 cty.Value) (cty.Value, error) { if cond.True() { - return evalTernaryBranch(val1) + return evalTernaryBranch(evalctx, val1) } - return evalTernaryBranch(val2) + return evalTernaryBranch(evalctx, val2) } -func evalTernaryBranch(arg cty.Value) (cty.Value, error) { +func evalTernaryBranch(evalctx *eval.Context, arg cty.Value) (cty.Value, error) { closure := customdecode.ExpressionClosureFromVal(arg) - - ctx := eval.NewContextFrom(closure.EvalContext) - newexpr, err := ctx.PartialEval(&ast.CloneExpression{ + newexpr, err := evalctx.PartialEval(&ast.CloneExpression{ Expression: closure.Expression.(hclsyntax.Expression), }) if err != nil { diff --git a/test/eval.go b/test/eval.go new file mode 100644 index 000000000..98ed49c4d --- /dev/null +++ b/test/eval.go @@ -0,0 +1,59 @@ +// Copyright 2023 Terramate GmbH +// SPDX-License-Identifier: MPL-2.0 + +package test + +import ( + "strings" + "testing" + + "github.com/madlambda/spells/assert" + "github.com/terramate-io/terramate/hcl/ast" + "github.com/terramate-io/terramate/hcl/eval" + "github.com/terramate-io/terramate/project" +) + +// NewRef returns a new variable reference for testing purposes. +// It only allows basic dotted accessors. +func NewRef(t testing.TB, varname string) eval.Ref { + t.Helper() + paths := strings.Split(varname, ".") + if strings.Contains(varname, "[") { + t.Fatalf("invalid testing reference: %s", varname) + } + return eval.Ref{ + Object: paths[0], + Path: paths[1:], + } +} + +// NewStmtFrom is a testing purspose method that initializes a Stmt. +func NewStmtFrom(t testing.TB, lhs string, rhs string) eval.Stmts { + lhsRef := NewRef(t, lhs) + tmpExpr, err := ast.ParseExpression(rhs, ``) + assert.NoError(t, err) + + stmts, err := eval.StmtsOfExpr(newSameInfo("/"), lhsRef, lhsRef.Path, tmpExpr) + assert.NoError(t, err) + + return stmts +} + +// NewStmt is a testing purspose method that initializes a Stmt. +func NewStmt(t testing.TB, lhs string, rhs string) eval.Stmt { + lhsRef := NewRef(t, lhs) + rhsExpr, err := ast.ParseExpression(rhs, ``) + assert.NoError(t, err) + return eval.Stmt{ + Origin: lhsRef, + LHS: lhsRef, + RHS: eval.NewExprRHS(rhsExpr), + Info: newSameInfo("/"), + } +} + +func newSameInfo(path string) eval.Info { + return eval.Info{ + Scope: project.NewPath(path), + } +} diff --git a/test/hcl.go b/test/hcl.go index fd7d93b11..7877dfdeb 100644 --- a/test/hcl.go +++ b/test/hcl.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" hhcl "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclwrite" "github.com/madlambda/spells/assert" @@ -78,7 +79,10 @@ func AssertTerramateConfig(t *testing.T, got, want hcl.Config) { func AssertDiff(t *testing.T, got, want interface{}, msg ...interface{}) { t.Helper() - if diff := cmp.Diff(got, want, cmp.AllowUnexported(project.Path{})); diff != "" { + if diff := cmp.Diff(got, want, + cmp.AllowUnexported(project.Path{}), + cmpopts.IgnoreUnexported(config.Stack{}), + ); diff != "" { errmsg := fmt.Sprintf("-(got) +(want):\n%s", diff) if len(msg) > 0 { errmsg = fmt.Sprintf(msg[0].(string), msg[1:]...) + ": " + errmsg diff --git a/test/hclwrite/hclwrite.go b/test/hclwrite/hclwrite.go index db0cf3141..c0ea3b1e7 100644 --- a/test/hclwrite/hclwrite.go +++ b/test/hclwrite/hclwrite.go @@ -22,8 +22,10 @@ import ( "strings" "testing" + "github.com/hashicorp/hcl/v2/ext/customdecode" "github.com/hashicorp/hcl/v2/hclparse" "github.com/hashicorp/hcl/v2/hclsyntax" + "github.com/terramate-io/terramate/hcl/ast" "github.com/terramate-io/terramate/hcl/fmt" "github.com/zclconf/go-cty/cty" @@ -33,9 +35,8 @@ import ( // Block represents an HCL block. type Block struct { - name string - labels []string - hasexpr bool + name string + labels []string // Not cool to keep 2 copies of values but casting around // cty values is quite annoying, so this is a lazy solution. ctyvalues map[string]cty.Value @@ -49,9 +50,13 @@ func (b *Block) AddLabel(name string) { // AddExpr adds an expression to the block. The expressions are kept as is on the // final document. -func (b *Block) AddExpr(name string, expr string) { - b.hasexpr = true - b.addAttr(name, expr) +func (b *Block) AddExpr(name string, exprStr string) { + expr, err := ast.ParseExpression(exprStr, ``) + if err != nil { + panic(err) + } + b.ctyvalues[name] = customdecode.ExpressionVal(expr) + b.addAttr(name, exprStr) } // AddNumberInt adds a number to the block. @@ -83,12 +88,6 @@ func (b *Block) AttributesValues() map[string]cty.Value { return b.ctyvalues } -// HasExpressions returns true if block has any non-evaluated -// expressions. -func (b *Block) HasExpressions() bool { - return b.hasexpr -} - // Build builds the given parent block by adding itself on it. func (b *Block) Build(parent *Block) { parent.AddBlock(b) diff --git a/test/sandbox/sandbox.go b/test/sandbox/sandbox.go index 00dcf4db1..189eb9d08 100644 --- a/test/sandbox/sandbox.go +++ b/test/sandbox/sandbox.go @@ -31,7 +31,6 @@ import ( "github.com/terramate-io/terramate/generate" "github.com/terramate-io/terramate/globals" "github.com/terramate-io/terramate/hcl" - "github.com/terramate-io/terramate/hcl/eval" "github.com/terramate-io/terramate/project" "github.com/terramate-io/terramate/run" "github.com/terramate-io/terramate/stack" @@ -45,6 +44,7 @@ type S struct { git *Git rootdir string cfg *config.Root + globals *globals.Resolver Env []string } @@ -257,7 +257,7 @@ func (s S) GenerateWith(root *config.Root, vendorDir project.Path) generate.Repo t := s.t t.Helper() - report := generate.Do(root, vendorDir, nil) + report := generate.Do(root, globals.NewResolver(root), vendorDir, nil) for _, failure := range report.Failures { t.Errorf("Generate unexpected failure: %v", failure) } @@ -289,19 +289,6 @@ func (s S) LoadStacks() config.List[*config.SortableStack] { return stacks } -// LoadStackGlobals loads globals for specific stack on the sandbox. -// Fails the caller test if an error is found. -func (s S) LoadStackGlobals( - root *config.Root, - st *config.Stack, -) *eval.Object { - s.t.Helper() - - report := globals.ForStack(root, st) - assert.NoError(s.t, report.AsError()) - return report.Globals -} - // RootDir returns the root directory of the test env. All dirs/files created // through the test env will be included inside this dir. // @@ -325,10 +312,22 @@ func (s *S) Config() *config.Root { return cfg } +// Globals returns the globals resolver for the sandbox config. +func (s *S) Globals() *globals.Resolver { + s.t.Helper() + if s.globals != nil { + return s.globals + } + g := globals.NewResolver(s.Config()) + s.globals = g + return g +} + // ReloadConfig reloads the sandbox configuration. func (s *S) ReloadConfig() *config.Root { s.t.Helper() s.cfg = nil + s.globals = nil return s.Config() }