Feature/output secrets plugin (#35)

* godotenv update

* [feat]: [CI-12564]: env file support

* added tests

* feature flag utility

* deleting output files

* default outputs

* review comments

* formatting

* added fallback

api/api.go

@@ -127,9 +127,9 @@ type (
 	}
 
 	OutputV2 struct {
-		Key   string `json:"key,omitempty"`
-		Value string `json:"value,omitempty"`
-		Type  string `json:"type,omitempty"`
+		Key   string     `json:"key,omitempty"`
+		Value string     `json:"value,omitempty"`
+		Type  OutputType `json:"type,omitempty"`
 	}
 
 	DelegateMetaInfo struct {
@@ -190,3 +190,10 @@ type (
 		Paths []string `json:"paths,omitempty"`
 	}
 )
+
+type OutputType string
+
+const (
+	OutputTypeString OutputType = "STRING"
+	OutputTypeSecret OutputType = "SECRET"
+)

cli/certs/generate_certs.go

@@ -9,8 +9,8 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/harness/godotenv/v3"
 	"github.com/harness/harness-docker-runner/config"
-	"github.com/joho/godotenv"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"gopkg.in/alecthomas/kingpin.v2"

cli/client/client.go

@@ -16,7 +16,7 @@ import (
 	"github.com/harness/harness-docker-runner/engine/spec"
 	"github.com/harness/harness-docker-runner/logger"
 
-	"github.com/joho/godotenv"
+	"github.com/harness/godotenv/v3"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"gopkg.in/alecthomas/kingpin.v2"

cli/server/server.go

@@ -22,7 +22,7 @@ import (
 	"github.com/harness/harness-docker-runner/server"
 	"github.com/harness/harness-docker-runner/setup"
 
-	"github.com/joho/godotenv"
+	"github.com/harness/godotenv/v3"
 	"github.com/sirupsen/logrus"
 	"gopkg.in/alecthomas/kingpin.v2"
 )

engine/engine.go

@@ -25,6 +25,7 @@ const (
 	DockerSockVolName  = "_docker"
 	DockerSockUnixPath = "/var/run/docker.sock"
 	DockerSockWinPath  = `\\.\pipe\docker_engine`
+	trueValue          = "true"
 )
 
 type Engine struct {
@@ -199,3 +200,11 @@ func matchDockerSockPath(s string) bool {
 	}
 	return false
 }
+
+func (e *Engine) IsFeatureFlagEnabled(featureFlagName string) bool {
+	if e.pipelineConfig == nil || e.pipelineConfig.Envs == nil {
+		return false
+	}
+	val, ok := e.pipelineConfig.Envs[featureFlagName]
+	return ok && val == trueValue
+}

go.mod

@@ -15,14 +15,14 @@ require (
 	github.com/go-chi/chi v1.5.4
 	github.com/gofrs/uuid v4.3.1+incompatible
 	github.com/golang/mock v1.6.0
+	github.com/harness/godotenv/v3 v3.0.0
 	github.com/hashicorp/go-multierror v1.1.1
-	github.com/joho/godotenv v1.4.0
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/linkedin/goavro/v2 v2.12.0
 	github.com/mholt/archiver/v3 v3.5.1
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.9.0
-	github.com/stretchr/testify v1.8.1
+	github.com/stretchr/testify v1.9.0
 	golang.org/x/sync v0.1.0
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/yaml.v2 v2.4.0
@@ -68,7 +68,6 @@ require (
 )
 
 require (
-	github.com/kardianos/service v1.2.2 // indirect
 	github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Microsoft/go-winio v0.6.0 // indirect
@@ -84,6 +83,7 @@ require (
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/kardianos/service v1.2.2 // indirect
 	github.com/mattn/go-zglob v0.0.4
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/natessilva/dag v0.0.0-20180124060714-7194b8dcc5c4 // indirect

go.sum

@@ -553,6 +553,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/harness/godotenv/v3 v3.0.0 h1:1YJU9nUk4tQygHOYkm+NZ5jL9IZQ3UqyBspCqL3EMQQ=
+github.com/harness/godotenv/v3 v3.0.0/go.mod h1:UIXXJtTM7NkSYMYknHYOO2d8BfDlAWMYZRuRsXcDDR0=
 github.com/harness/lite-engine v0.4.9-0.20230131094531-9c0bf7135df4 h1:q7kJr+zF/Lg0zGvUSe9P7AqCL/4Av0+VbkPD2kMfwc8=
 github.com/harness/lite-engine v0.4.9-0.20230131094531-9c0bf7135df4/go.mod h1:RP+Bz5B/Cj/a1HjzU13GutbS6uWotX6OqZSe5Pp4XCw=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
@@ -596,8 +598,6 @@ github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbB
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8=
-github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
-github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
@@ -908,6 +908,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
@@ -1225,6 +1227,7 @@ golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

main.go

@@ -7,8 +7,8 @@ package main
 import (
 	"runtime"
 
+	_ "github.com/harness/godotenv/v3/autoload"
 	"github.com/harness/harness-docker-runner/cli"
-	_ "github.com/joho/godotenv/autoload"
 	"github.com/kardianos/service"
 )
 

pipeline/runtime/common.go

@@ -12,11 +12,21 @@ import (
 	"os"
 	"strings"
 
+	"github.com/harness/godotenv/v3"
 	"github.com/harness/harness-docker-runner/api"
+	"github.com/harness/harness-docker-runner/engine"
+	"github.com/harness/harness-docker-runner/engine/spec"
 	"github.com/harness/harness-docker-runner/logstream"
 	"github.com/sirupsen/logrus"
 )
 
+const (
+	ciEnablePluginOutputSecrets = "CI_ENABLE_PLUGIN_OUTPUT_SECRETS"
+	trueValue                   = "true"
+	outputDelimiterSpace        = " "
+	outputDelimiterEquals       = "="
+)
+
 func getNudges() []logstream.Nudge {
 	// <search-term> <resolution> <error-msg>
 	return []logstream.Nudge{
@@ -29,46 +39,54 @@ func getNudges() []logstream.Nudge {
 	}
 }
 
-func getOutputVarCmd(entrypoint, outputVars []string, outputFile string) string {
+func getOutputVarCmd(entrypoint, outputVars []string, outputFile string, shouldEnableDotEnvSupport bool) string {
 	isPsh := isPowershell(entrypoint)
 	isPython := isPython(entrypoint)
 
 	cmd := ""
+	delimiter := outputDelimiterSpace
+	if shouldEnableDotEnvSupport {
+		delimiter = outputDelimiterEquals
+	}
 	if isPsh {
 		cmd += fmt.Sprintf("\nNew-Item %s", outputFile)
 	} else if isPython {
 		cmd += "\nimport os\n"
 	}
 	for _, o := range outputVars {
 		if isPsh {
-			cmd += fmt.Sprintf("\n$val = \"%s $Env:%s\" \nAdd-Content -Path %s -Value $val", o, o, outputFile)
+			cmd += fmt.Sprintf("\n$val = \"%s%s$Env:%s\" \nAdd-Content -Path %s -Value $val", o, delimiter, o, outputFile)
 		} else if isPython {
-			cmd += fmt.Sprintf("with open('%s', 'a') as out_file:\n\tout_file.write('%s ' + os.getenv('%s') + '\\n')\n", outputFile, o, o)
+			cmd += fmt.Sprintf("with open('%s', 'a') as out_file:\n\tout_file.write('%s%s' + os.getenv('%s') + '\\n')\n", outputFile, o, delimiter, o)
 		} else {
-			cmd += fmt.Sprintf("\necho \"%s $%s\" >> %s", o, o, outputFile)
+			cmd += fmt.Sprintf("\necho \"%s%s$%s\" >> %s", o, delimiter, o, outputFile)
 		}
 	}
 
 	return cmd
 }
 
-func getOutputsCmd(entrypoint []string, outputVars []*api.OutputV2, outputFile string) string {
+func getOutputsCmd(entrypoint []string, outputVars []*api.OutputV2, outputFile string, shouldEnableDotEnvSupport bool) string {
 	isPsh := isPowershell(entrypoint)
 	isPython := isPython(entrypoint)
 
 	cmd := ""
+	delimiter := outputDelimiterSpace
+	if shouldEnableDotEnvSupport {
+		delimiter = outputDelimiterEquals
+	}
 	if isPsh {
 		cmd += fmt.Sprintf("\nNew-Item %s", outputFile)
 	} else if isPython {
 		cmd += "\nimport os\n"
 	}
 	for _, o := range outputVars {
 		if isPsh {
-			cmd += fmt.Sprintf("\n$val = \"%s $Env:%s\" \nAdd-Content -Path %s -Value $val", o.Key, o.Value, outputFile)
+			cmd += fmt.Sprintf("\n$val = \"%s%s$Env:%s\" \nAdd-Content -Path %s -Value $val", o.Key, delimiter, o.Value, outputFile)
 		} else if isPython {
-			cmd += fmt.Sprintf("with open('%s', 'a') as out_file:\n\tout_file.write('%s ' + os.getenv('%s') + '\\n')\n", outputFile, o.Key, o.Value)
+			cmd += fmt.Sprintf("with open('%s', 'a') as out_file:\n\tout_file.write('%s%s' + os.getenv('%s') + '\\n')\n", outputFile, o.Key, delimiter, o.Value)
 		} else {
-			cmd += fmt.Sprintf("\necho \"%s $%s\" >> %s", o.Key, o.Value, outputFile)
+			cmd += fmt.Sprintf("\necho \"%s%s$%s\" >> %s", o.Key, delimiter, o.Value, outputFile)
 		}
 	}
 
@@ -107,11 +125,15 @@ func isPython(entrypoint []string) bool {
 
 // Fetches map of env variable and value from OutputFile.
 // OutputFile stores all env variable and value
-func fetchOutputVariables(outputFile string, out io.Writer) (map[string]string, error) {
+func fetchOutputVariables(outputFile string, out io.Writer, isDotEnvFile bool) (map[string]string, error) {
 	log := logrus.New()
 	log.Out = out
 
 	outputs := make(map[string]string)
+	delimiter := outputDelimiterSpace
+	if isDotEnvFile {
+		delimiter = outputDelimiterEquals
+	}
 
 	// The output file maybe not exist - we don't consider that an error
 	if _, err := os.Stat(outputFile); os.IsNotExist(err) {
@@ -128,7 +150,7 @@ func fetchOutputVariables(outputFile string, out io.Writer) (map[string]string,
 	s := bufio.NewScanner(f)
 	for s.Scan() {
 		line := s.Text()
-		sa := strings.Split(line, " ")
+		sa := strings.Split(line, delimiter)
 		if len(sa) < 2 { // nolint:gomnd
 			log.WithField("variable", sa[0]).Warnln("output variable does not exist")
 		} else {
@@ -141,3 +163,49 @@ func fetchOutputVariables(outputFile string, out io.Writer) (map[string]string,
 	}
 	return outputs, nil
 }
+
+func fetchExportedVarsFromEnvFile(envFile string, out io.Writer) (map[string]string, error) {
+	log := logrus.New()
+	log.Out = out
+
+	defaultOutputs := make(map[string]string)
+	if _, err := os.Stat(envFile); errors.Is(err, os.ErrNotExist) {
+		return defaultOutputs, nil
+	}
+
+	var (
+		env map[string]string
+		err error
+	)
+	env, err = godotenv.Read(envFile)
+
+	if err != nil {
+		//fallback incase any parsing issue from godotenv package
+		fallbackEnv, fallbackErr := fetchOutputVariables(envFile, out, true)
+		if fallbackErr != nil {
+			content, ferr := os.ReadFile(envFile)
+			if ferr != nil {
+				log.WithError(ferr).WithField("envFile", envFile).Warnln("Unable to read exported env file")
+			}
+			log.WithError(err).WithField("envFile", envFile).WithField("content", string(content)).Warnln("failed to read exported env file")
+			if errors.Is(err, bufio.ErrTooLong) {
+				err = fmt.Errorf("output variable length is more than %d bytes", bufio.MaxScanTokenSize)
+			}
+			return nil, err
+		}
+		return fallbackEnv, nil
+
+	}
+	return env, nil
+}
+
+func IsFeatureFlagEnabled(featureFlagName string, engine *engine.Engine, step *spec.Step) bool {
+	if engine != nil && engine.IsFeatureFlagEnabled(featureFlagName) {
+		return true
+	}
+	if step == nil {
+		return false
+	}
+	val, ok := step.Envs[featureFlagName]
+	return ok && val == trueValue
+}