test: fix windows CI tests.

Signed-off-by: i4k <[email protected]>

.github/workflows/ci-experimental.yml → .github/workflows/ci-windows.yml

@@ -1,15 +1,9 @@
 # Copyright 2023 Terramate GmbH
 # SPDX-License-Identifier: MPL-2.0
 
-name: ci-experimental
+name: ci-windows
 on:
-  push:
-    paths-ignore:
-      - 'docs/**'
-      - 'makefiles/**'
-      - '.github/**'
-      - '**/*.tm.hcl'
-      - '.tool-versions'
+  pull_request:
 
 jobs:
   build_test:

cmd/terramate/cli/run.go

@@ -665,7 +665,7 @@ func (c *cli) runAll(
 
 				var err error
 				if !task.isSuccessExit(result.cmd.ProcessState.ExitCode()) {
-					err = errors.E(result.err, ErrRunFailed, "running %s (in %s)", result.cmd, run.Stack.Dir)
+					err = errors.E(result.err, ErrRunFailed, "running %s (in %s)", cmdStr, run.Stack.Dir)
 					errs.Append(err)
 				}
 

e2etests/cloud/run_script_cloud_preview_test.go

@@ -5,7 +5,6 @@ package cloud_test
 
 import (
 	"encoding/json"
-	"fmt"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -72,22 +71,20 @@ func TestScriptRunWithCloudSyncPreview(t *testing.T) {
 					}
 				  }
 				`,
-				fmt.Sprintf(`f:stack/preview.tm:
+				`f:stack/preview.tm:
 
 				  script "preview" {
 					description = "sync a preview"
 					job {
 					  commands = [
-						["%s", "plan", "-out=out.tfplan", "-no-color", "-detailed-exitcode", {
+						["terraform", "plan", "-out=out.tfplan", "-no-color", "-detailed-exitcode", {
 						  sync_preview             = true,
 						  terraform_plan_file = "out.tfplan",
 						}],
 					  ]
 					}
 				  }
 				  `,
-					TerraformTestPath,
-				),
 				`f:stack/main.tf:
 				  resource "local_file" "foo" {
 					content  = "test content"
@@ -129,22 +126,20 @@ func TestScriptRunWithCloudSyncPreview(t *testing.T) {
 					}
 				  }
 				`,
-				fmt.Sprintf(`f:stack/preview.tm:
+				`f:stack/preview.tm:
 
 				  script "preview" {
 					description = "sync a preview"
 					job {
 					  commands = [
-						["%s", "plan", "-out=out.tfplan", "-no-color", "-detailed-exitcode", {
+						["terraform", "plan", "-out=out.tfplan", "-no-color", "-detailed-exitcode", {
 						  sync_preview             = true,
 						  terraform_plan_file = "out.tfplan",
 						}],
 					  ]
 					}
 				  }
 				  `,
-					TerraformTestPath,
-				),
 				`f:stack/main.tf:
 				  resource "local_file" "foo" {
 					content  = "test content"
@@ -177,7 +172,7 @@ func TestScriptRunWithCloudSyncPreview(t *testing.T) {
 						{
 							ID:     "1",
 							Status: "changed",
-							Cmd:    []string{TerraformTestPath, "plan", "-out=out.tfplan", "-no-color", "-detailed-exitcode"},
+							Cmd:    []string{"terraform", "plan", "-out=out.tfplan", "-no-color", "-detailed-exitcode"},
 						},
 					},
 					ReviewRequest: &cloud.ReviewRequest{
@@ -239,23 +234,21 @@ func TestScriptRunWithCloudSyncPreview(t *testing.T) {
 					}
 				  }
 				`,
-				fmt.Sprintf(`f:stack/preview.tm:
+				`f:stack/preview.tm:
 
 				  script "preview" {
 					description = "sync a preview"
 					job {
 					  commands = [
 					    ["do-not-exist-command"],
-						["%s", "plan", "-out=out.tfplan", "-no-color", "-detailed-exitcode", {
+						["terraform", "plan", "-out=out.tfplan", "-no-color", "-detailed-exitcode", {
 						  sync_preview             = true,
 						  terraform_plan_file = "out.tfplan",
 						}],
 					  ]
 					}
 				  }
 				  `,
-					TerraformTestPath,
-				),
 				`f:stack/main.tf:
 				  resource "local_file" "foo" {
 					content  = "test content"
@@ -285,7 +278,7 @@ func TestScriptRunWithCloudSyncPreview(t *testing.T) {
 						{
 							ID:     "1",
 							Status: "failed",
-							Cmd:    []string{TerraformTestPath, "plan", "-out=out.tfplan", "-no-color", "-detailed-exitcode"},
+							Cmd:    []string{"terraform", "plan", "-out=out.tfplan", "-no-color", "-detailed-exitcode"},
 						},
 					},
 					ReviewRequest: &cloud.ReviewRequest{

e2etests/core/run_sharing_test.go

@@ -274,7 +274,7 @@ func TestRunSharing(t *testing.T) {
 					Labels("name"),
 					Expr("type", "terraform"),
 					Str("filename", "sharing.tf"),
-					Command(HelperPath, "exit", "1"),
+					Command("helper", "exit", "1"),
 				).String(),
 				"s:s1:id=s1",
 				"f:s1/main.tf:" + Doc(
@@ -335,7 +335,7 @@ func TestRunSharing(t *testing.T) {
 					Labels("name"),
 					Expr("type", "terraform"),
 					Str("filename", "sharing.tf"),
-					Command(HelperPath, "echo", "{}"),
+					Command("helper", "echo", "{}"),
 				).String(),
 				"s:s1:id=s1",
 				"f:s1/main.tf:" + Doc(
@@ -402,7 +402,7 @@ func TestRunSharing(t *testing.T) {
 					Labels("name"),
 					Expr("type", "terraform"),
 					Str("filename", "sharing.tf"),
-					Command(HelperPath, "echo", "$error"),
+					Command("helper", "echo", "$error"),
 				).String(),
 				"s:s1:id=s1",
 				"f:s1/main.tf:" + Doc(
@@ -503,7 +503,8 @@ func TestRunSharing(t *testing.T) {
 				s.BuildTree(layout)
 				tmcli := NewCLI(t, s.RootDir())
 				tmcli.PrependToPath(filepath.Dir(TerraformTestPath))
-				res := tmcli.Run("run", HelperPath, "echo", "hello")
+				tmcli.PrependToPath(filepath.Dir(HelperPath))
+				res := tmcli.Run("run", "helper", "echo", "hello")
 				if res.Status == 0 {
 					// generate safeguard must trigger
 					t.Fatal("run must fail if sharing is not generated")

e2etests/core/run_test.go

@@ -918,10 +918,9 @@ func TestCLIRunOrder(t *testing.T) {
 					t.Parallel()
 					copiedLayout := make([]string, len(tc.layout))
 					copy(copiedLayout, tc.layout)
-					if runtime.GOOS != "windows" {
-						copiedLayout = append(copiedLayout,
 
-							fmt.Sprintf(`file:script.tm:
+					copiedLayout = append(copiedLayout,
+						fmt.Sprintf(`file:script.tm:
 terramate {
 	config {
 		experiments = ["scripts"]
@@ -930,10 +929,14 @@ terramate {
 script "cmd" {
 	description = "test"
 	job {
-		command = ["%s", "stack-abs-path", "%s"]
+		command = ["helper", "stack-abs-path", "${tm_chomp(<<-EOF
+                    %s
+					EOF
+				)}"]
 	}
-}`, HelperPath, s.RootDir()))
-					}
+}`, s.RootDir()))
+					t.Logf("config: %v", copiedLayout)
+
 					s.BuildTree(copiedLayout)
 
 					wd := s.RootDir()
@@ -950,6 +953,7 @@ script "cmd" {
 					}
 
 					cli := NewCLI(t, wd)
+					cli.PrependToPath(filepath.Dir(HelperPath))
 					AssertRunResult(t, cli.StacksRunOrder(filterArgs...), tc.want)
 					runArgs := []string{
 						"--quiet", "run", "-X", // disable all safeguards
@@ -958,21 +962,19 @@ script "cmd" {
 					runArgs = append(runArgs, "--", HelperPath, "stack-abs-path", s.RootDir())
 					AssertRunResult(t, cli.Run(runArgs...), tc.want)
 
-					if runtime.GOOS != "windows" {
-						runScriptArgs := []string{
-							"--quiet",
-						}
-						runScriptArgs = append(runScriptArgs, filterArgs...)
-						runScriptArgs = append(runScriptArgs, "script", "run", "-X", "cmd") // disable all safeguards)
-						AssertRunResult(t, cli.Run(runScriptArgs...), RunExpected{
-							Status:        tc.want.Status,
-							IgnoreStderr:  true,
-							Stdout:        tc.want.Stdout,
-							IgnoreStdout:  tc.want.IgnoreStdout,
-							StdoutRegex:   tc.want.StdoutRegex,
-							StdoutRegexes: tc.want.StdoutRegexes,
-						})
+					runScriptArgs := []string{
+						"--quiet",
 					}
+					runScriptArgs = append(runScriptArgs, filterArgs...)
+					runScriptArgs = append(runScriptArgs, "script", "run", "-X", "cmd") // disable all safeguards)
+					AssertRunResult(t, cli.Run(runScriptArgs...), RunExpected{
+						Status:        tc.want.Status,
+						IgnoreStderr:  true,
+						Stdout:        tc.want.Stdout,
+						IgnoreStdout:  tc.want.IgnoreStdout,
+						StdoutRegex:   tc.want.StdoutRegex,
+						StdoutRegexes: tc.want.StdoutRegexes,
+					})
 				})
 			}
 		})
@@ -2611,25 +2613,25 @@ func TestRunOutput(t *testing.T) {
 	for _, tc := range []testcase{
 		{
 			name:    "run without eval",
-			runArgs: []string{HelperPath, "echo", "hello"},
+			runArgs: []string{"helper", "echo", "hello"},
 			want: RunExpected{
 				Stderr: "terramate: Entering stack in /stack" + "\n" +
-					fmt.Sprintf(`terramate: Executing command "%s echo hello"`, HelperPath) + "\n",
+					`terramate: Executing command "helper echo hello"` + "\n",
 				Stdout: "hello\n",
 			},
 		},
 		{
 			name:    "run with eval",
-			runArgs: []string{"--eval", HelperPath, "echo", "${terramate.stack.name}"},
+			runArgs: []string{"--eval", "helper", "echo", "${terramate.stack.name}"},
 			want: RunExpected{
 				Stderr: "terramate: Entering stack in /stack" + "\n" +
-					fmt.Sprintf(`terramate: Executing command "%s echo stack"`, HelperPath) + "\n",
+					`terramate: Executing command "helper echo stack"` + "\n",
 				Stdout: "stack\n",
 			},
 		},
 		{
 			name:    "run with eval with error",
-			runArgs: []string{"--eval", HelperPath, "echo", "${terramate.stack.abcabc}"},
+			runArgs: []string{"--eval", "helper", "echo", "${terramate.stack.abcabc}"},
 			want: RunExpected{
 				Stderr: "Error: unable to evaluate command" + "\n" +
 					`> <cmd arg>:1,19-26: eval expression: eval "${terramate.stack.abcabc}": This object does not have an attribute named "abcabc"` + ".\n",
@@ -2644,6 +2646,7 @@ func TestRunOutput(t *testing.T) {
 			git := s.Git()
 			git.CommitAll("first commit")
 			cli := NewCLI(t, s.RootDir())
+			cli.PrependToPath(filepath.Dir(HelperPath))
 			AssertRunResult(t,
 				cli.Run(append([]string{"run"}, tc.runArgs...)...),
 				tc.want,
@@ -2665,19 +2668,19 @@ func TestRunDryRun(t *testing.T) {
 	for _, tc := range []testcase{
 		{
 			name:    "dryrun without eval",
-			runArgs: []string{"--dry-run", HelperPath, "echo", "hello"},
+			runArgs: []string{"--dry-run", "helper", "echo", "hello"},
 			want: RunExpected{
 				Stderr: "terramate: (dry-run) Entering stack in /stack" + "\n" +
-					fmt.Sprintf(`terramate: (dry-run) Executing command "%s echo hello"`, HelperPath) + "\n",
+					`terramate: (dry-run) Executing command "helper echo hello"` + "\n",
 				Stdout: "",
 			},
 		},
 		{
 			name:    "dryrun with eval",
-			runArgs: []string{"--dry-run", "--eval", HelperPath, "echo", "${terramate.stack.name}"},
+			runArgs: []string{"--dry-run", "--eval", "helper", "echo", "${terramate.stack.name}"},
 			want: RunExpected{
 				Stderr: "terramate: (dry-run) Entering stack in /stack" + "\n" +
-					fmt.Sprintf(`terramate: (dry-run) Executing command "%s echo stack"`, HelperPath) + "\n",
+					`terramate: (dry-run) Executing command "helper echo stack"` + "\n",
 				Stdout: "",
 			},
 		},
@@ -2688,6 +2691,7 @@ func TestRunDryRun(t *testing.T) {
 			git := s.Git()
 			git.CommitAll("first commit")
 			cli := NewCLI(t, s.RootDir())
+			cli.PrependToPath(filepath.Dir(HelperPath))
 			AssertRunResult(t,
 				cli.Run(append([]string{"run"}, tc.runArgs...)...),
 				tc.want,

e2etests/core/script_run_test.go

@@ -6,6 +6,7 @@ package core_test
 import (
 	"os"
 	"path/filepath"
+	"regexp"
 	"testing"
 
 	. "github.com/terramate-io/terramate/e2etests/internal/runner"
@@ -204,14 +205,18 @@ func TestScriptRun(t *testing.T) {
 			args:      []string{"--continue-on-error"},
 			want: RunExpected{
 				Status: 1,
-				Stderr: `Script 0 at /stack-a/script.tm:2,5-13,6 having 3 job(s)` + "\n" +
-					"/stack-a (script:0 job:0.0)> echo hello1" + "\n" +
-					"/stack-a (script:0 job:1.0)> someunknowncommand" + "\n" +
-					"/stack-a/stack-b (script:0 job:0.0)> echo hello1" + "\n" +
-					"/stack-a/stack-b (script:0 job:1.0)> someunknowncommand" + "\n" +
-					"Error: one or more commands failed" + "\n" +
-					"> executable file not found in $PATH: running " + "`someunknowncommand`" + " in stack /stack-a: someunknowncommand" + "\n" +
-					"> executable file not found in $PATH: running " + "`someunknowncommand`" + " in stack /stack-a/stack-b: someunknowncommand" + "\n",
+				StderrRegexes: []string{
+					regexp.QuoteMeta(`Script 0 at /stack-a/script.tm:2,5-13,6 having 3 job(s)` + "\n" +
+						"/stack-a (script:0 job:0.0)> echo hello1" + "\n" +
+						"/stack-a (script:0 job:1.0)> someunknowncommand" + "\n" +
+						"/stack-a/stack-b (script:0 job:0.0)> echo hello1" + "\n" +
+						"/stack-a/stack-b (script:0 job:1.0)> someunknowncommand" + "\n" +
+						"Error: one or more commands failed" + "\n" +
+						"> executable file not found in "),
+					regexp.QuoteMeta(" running " + "`someunknowncommand`" + " in stack /stack-a: someunknowncommand" + "\n" +
+						"> executable file not found in "),
+					regexp.QuoteMeta("running " + "`someunknowncommand`" + " in stack /stack-a/stack-b: someunknowncommand" + "\n"),
+				},
 				Stdout: "hello1" + "\n" +
 					"hello1" + "\n",
 			},
@@ -228,7 +233,7 @@ func TestScriptRun(t *testing.T) {
 					command = ["echo", "hello1"]
 				  }
 				  job {
-					command = ["` + HelperPath + `", "false"]
+					command = ["helper", "false"]
 				  }
 				  job {
 					command = ["echo", "hello2"]
@@ -242,12 +247,12 @@ func TestScriptRun(t *testing.T) {
 				Status: 1,
 				Stderr: "Script 0 at /stack-a/script.tm:2,5-13,6 having 3 job(s)\n" +
 					"/stack-a (script:0 job:0.0)> echo hello1\n" +
-					"/stack-a (script:0 job:1.0)> " + HelperPath + " false\n" +
+					"/stack-a (script:0 job:1.0)> helper false\n" +
 					"/stack-a/stack-b (script:0 job:0.0)> echo hello1\n" +
-					"/stack-a/stack-b (script:0 job:1.0)> " + HelperPath + " false\n" +
+					"/stack-a/stack-b (script:0 job:1.0)> helper false\n" +
 					"Error: one or more commands failed\n" +
-					"> execution failed: running " + HelperPath + " false (in /stack-a): exit status 1\n" +
-					"> execution failed: running " + HelperPath + " false (in /stack-a/stack-b): exit status 1\n",
+					"> execution failed: running helper false (in /stack-a): exit status 1\n" +
+					"> execution failed: running helper false (in /stack-a/stack-b): exit status 1\n",
 				Stdout: "hello1" + "\n" +
 					"hello1" + "\n",
 			},
@@ -489,6 +494,7 @@ func TestScriptRun(t *testing.T) {
 					env = append(env, tc.env...)
 
 					cli := NewCLI(t, wd, env...)
+					cli.PrependToPath(filepath.Dir(HelperPath))
 					args := tc.args
 					args = append(args, tc.runScript...)
 					AssertRunResult(t, cli.RunScript(args...), tc.want)

e2etests/internal/runner/runner.go

@@ -430,8 +430,8 @@ func AssertRunResult(t *testing.T, got RunResult, want RunExpected) {
 				}
 			}
 		} else {
-			if want.Stderr != got.Stderr {
-				t.Errorf("stderr mismatch: got %q != want %q", got.Stderr, want.Stderr)
+			if diff := cmp.Diff(want.Stderr, got.Stderr); diff != "" {
+				t.Errorf("stderr mismatch (-want +got): %s", diff)
 			}
 		}
 	}