expose network and environment variable (#323)

* expose network and environment variable

builder/builder.go

@@ -39,6 +39,10 @@ func NewBuilder(pm *procmanager.ProcManager, debug bool, workspaceDir string) *B
 // RunTask executes a Task.
 func (b *Builder) RunTask(ctx context.Context, task *graph.Task) error {
 	for _, network := range task.Networks {
+		if network.SkipCreation {
+			log.Printf("Skip creating network: %s\n", network.Name)
+			continue
+		}
 		log.Printf("Creating Docker network: %s, driver: '%s'\n", network.Name, network.Driver)
 		if msg, err := network.Create(ctx, b.procManager); err != nil {
 			return fmt.Errorf("Failed to create network: %s, err: %v, msg: %s", network.Name, err, msg)
@@ -132,6 +136,10 @@ func (b *Builder) CleanTask(ctx context.Context, task *graph.Task) {
 	}
 
 	for _, network := range task.Networks {
+		if network.SkipCreation {
+			log.Printf("Skip deleting network: %s\n", network.Name)
+			continue
+		}
 		if msg, err := network.Delete(ctx, b.procManager); err != nil {
 			log.Printf("Failed to delete network: %s, err: %v, msg: %s\n", network.Name, err, msg)
 		}

cmd/exec.go

@@ -38,6 +38,8 @@ type execCmd struct {
 	registryUser   string
 	registryPw     string
 	defaultWorkDir string
+	network        string
+	envs           []string
 
 	opts *templating.BaseRenderOptions
 }
@@ -62,6 +64,9 @@ func newExecCmd(out io.Writer) *cobra.Command {
 	f.BoolVar(&e.dryRun, "dry-run", false, "evaluates the task but doesn't execute it")
 	f.StringVar(&e.defaultWorkDir, "working-directory", "", "the default working directory to use if the underlying Task doesn't have one specified")
 
+	f.StringVar(&e.network, "network", "", "the default network to use")
+	f.StringArrayVar(&e.envs, "env", []string{}, "the default environment variables which are applied to each step (use `--env` multiple times or use commas: env1=val1,env2=val2)")
+
 	AddBaseRenderingOptions(f, e.opts, cmd, true)
 	return cmd
 }
@@ -109,7 +114,8 @@ func (e *execCmd) run(cmd *cobra.Command, args []string) error {
 		fmt.Println(rendered)
 	}
 
-	task, err := graph.UnmarshalTaskFromString(rendered, e.opts.Registry, e.registryUser, e.registryPw, e.defaultWorkDir)
+	task, err := graph.UnmarshalTaskFromString(rendered, e.opts.Registry, e.registryUser, e.registryPw, e.defaultWorkDir, e.network, e.envs)
+
 	if err != nil {
 		return err
 	}

graph/dag_test.go

@@ -46,23 +46,25 @@ func TestDagCreation_ValidFile(t *testing.T) {
 		Keep:       true,
 		Isolation:  "default",
 		Network:    DefaultNetworkName,
+		Envs:       []string{"foo=taskEnv"},
 	}
 
 	bStep := &Step{
-		ID:           "B",
-		When:         []string{"C"},
-		Cmd:          "azure/images/git clone https://github.com/ehotinger/clone",
-		StepStatus:   Skipped,
-		Timeout:      defaultStepTimeoutInSeconds,
-		IgnoreErrors: true,
-		Network:      DefaultNetworkName,
+		ID:                              "B",
+		When:                            []string{"C"},
+		Cmd:                             "azure/images/git clone https://github.com/ehotinger/clone",
+		StepStatus:                      Skipped,
+		Timeout:                         defaultStepTimeoutInSeconds,
+		IgnoreErrors:                    true,
+		Network:                         DefaultNetworkName,
 		DisableWorkingDirectoryOverride: true,
+		Envs:                            []string{"foo=taskEnv"},
 	}
 
 	fooStep := &Step{
 		ID:         "build-foo",
 		Cmd:        "azure/images/acr-builder build -f Dockerfile https://github.com/ehotinger/foo --cache-from=ubuntu",
-		Envs:       []string{"eric=foo"},
+		Envs:       []string{"eric=foo", "foo=taskEnv"},
 		When:       []string{"build-qux"},
 		SecretEnvs: []string{"someAkvSecretEnv"},
 		StepStatus: Skipped,
@@ -77,6 +79,7 @@ func TestDagCreation_ValidFile(t *testing.T) {
 		StepStatus: Skipped,
 		Timeout:    defaultStepTimeoutInSeconds,
 		Network:    DefaultNetworkName,
+		Envs:       []string{"foo=taskEnv"},
 	}
 
 	quxStep := &Step{
@@ -88,6 +91,7 @@ func TestDagCreation_ValidFile(t *testing.T) {
 		Detach:     true,
 		StartDelay: 50,
 		Network:    DefaultNetworkName,
+		Envs:       []string{"foo=taskEnv"},
 	}
 
 	qazStep := &Step{
@@ -98,6 +102,7 @@ func TestDagCreation_ValidFile(t *testing.T) {
 		Privileged: true,
 		User:       "root",
 		Network:    "host",
+		Envs:       []string{"foo=taskEnv"},
 	}
 
 	dict := make(map[string]*Step)

graph/network.go

@@ -17,17 +17,21 @@ const (
 
 // Network defines a Docker network.
 type Network struct {
-	Name   string `yaml:"name"`
-	Ipv6   bool   `yaml:"ipv6"`
-	Driver string `yaml:"driver"`
+	Name         string `yaml:"name"`
+	Ipv6         bool   `yaml:"ipv6"`
+	Driver       string `yaml:"driver"`
+	SkipCreation bool   `yaml:"skipCreation"`
+	IsDefault    bool   `yaml:"isDefault"`
 }
 
 // NewNetwork creates a new network.
-func NewNetwork(name string, ipv6 bool, driver string) *Network {
+func NewNetwork(name string, ipv6 bool, driver string, skipCreation bool, isDefault bool) *Network {
 	return &Network{
-		Name:   name,
-		Ipv6:   ipv6,
-		Driver: driver,
+		Name:         name,
+		Ipv6:         ipv6,
+		Driver:       driver,
+		SkipCreation: skipCreation,
+		IsDefault:    isDefault,
 	}
 }
 

graph/network_test.go

@@ -16,28 +16,43 @@ func TestNetwork(t *testing.T) {
 		name               string
 		ipv6               bool
 		driver             string
+		skipCreation       bool
+		isDefault          bool
 		expectedCreateArgs []string
 		expectedDeleteArgs []string
 	}{
 		{
 			"foo",
 			true,
 			"",
+			false,
+			false,
 			[]string{"docker", "network", "create", "foo", "--ipv6"},
 			[]string{"docker", "network", "rm", "foo"},
 		},
 		{
 			"bar",
 			false,
 			"nat",
+			false,
+			false,
 			[]string{"docker", "network", "create", "bar", "--driver", "nat"},
 			[]string{"docker", "network", "rm", "bar"},
 		},
+		{
+			"foo",
+			false,
+			"",
+			true,
+			true,
+			[]string{"docker", "network", "create", "foo"},
+			[]string{"docker", "network", "rm", "foo"},
+		},
 	}
 	procManager := procmanager.NewProcManager(true)
 
 	for _, test := range tests {
-		network := NewNetwork(test.name, test.ipv6, test.driver)
+		network := NewNetwork(test.name, test.ipv6, test.driver, test.skipCreation, test.isDefault)
 		if network.Name != test.name {
 			t.Fatalf("Expected network name: %s but got %s", test.name, network.Name)
 		}

graph/task.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"runtime"
+	"strings"
 
 	"github.com/Azure/acr-builder/scan"
 	"github.com/Azure/acr-builder/util"
@@ -51,11 +52,12 @@ type Task struct {
 	RegistryUsername string
 	RegistryPassword string
 	Dag              *Dag
-	IsBuildTask      bool // Used to skip the default network creation for build.
+	IsBuildTask      bool     // Used to skip the default network creation for build.
+	Envs             []string `yaml:"envs,omitempty"`
 }
 
 // UnmarshalTaskFromString unmarshals a Task from a raw string.
-func UnmarshalTaskFromString(data, registry, user, pw, defaultWorkDir string) (*Task, error) {
+func UnmarshalTaskFromString(data, registry, user, pw, defaultWorkDir, network string, envs []string) (*Task, error) {
 	t := &Task{}
 	if err := yaml.Unmarshal([]byte(data), t); err != nil {
 		return t, errors.Wrap(err, "failed to deserialize task")
@@ -64,6 +66,16 @@ func UnmarshalTaskFromString(data, registry, user, pw, defaultWorkDir string) (*
 		t.WorkingDirectory = defaultWorkDir
 	}
 	t.setRegistryInfo(registry, user, pw)
+
+	t.Envs = envs
+
+	//External network parsed in from CLI will be set as default network, it will be used for any step if no network provide for them
+	//The external network is append at the end of the list of networks, later we will do reverse iteration to get this network
+	if network != "" {
+		externalNetwork := NewNetwork(network, false, "external", true, true)
+		t.Networks = append(t.Networks, externalNetwork)
+	}
+
 	err := t.initialize()
 	return t, err
 }
@@ -108,11 +120,23 @@ func NewTask(
 
 // initialize normalizes a Task's values.
 func (t *Task) initialize() error {
+	newDefaultNetworkName := DefaultNetworkName
+	addDefaultNetworkToSteps := false
+
+	// Reverse iterate the list to get the default network
+	for i := len(t.Networks) - 1; i >= 0; i-- {
+		network := t.Networks[i]
+		if network.IsDefault {
+			newDefaultNetworkName = network.Name
+			addDefaultNetworkToSteps = true
+			break
+		}
+	}
+
 	// Add the default network if none are specified.
 	// Only add the default network if we're using tasks.
-	addDefaultNetworkToSteps := false
 	if !t.IsBuildTask && len(t.Networks) <= 0 {
-		defaultNetwork := NewNetwork(DefaultNetworkName, false, "bridge")
+		defaultNetwork := NewNetwork(newDefaultNetworkName, false, "bridge", false, true)
 		if runtime.GOOS == "windows" {
 			defaultNetwork.Driver = "nat"
 		}
@@ -156,7 +180,13 @@ func (t *Task) initialize() error {
 		}
 
 		if addDefaultNetworkToSteps && s.Network == "" {
-			s.Network = DefaultNetworkName
+			s.Network = newDefaultNetworkName
+		}
+
+		if newEnvs, err := mergeEnvs(s.Envs, t.Envs); err != nil {
+			return fmt.Errorf("Bad format of environment variables, err: %v", err)
+		} else {
+			s.Envs = newEnvs
 		}
 
 		if s.ID == "" {
@@ -225,3 +255,46 @@ func getNormalizedDockerImageNames(dockerImages []string, registry string) []str
 
 	return normalizedDockerImages
 }
+
+// the step's environment variables should override the task's default ones if provided
+func mergeEnvs(stepEnvs []string, taskEnvs []string) ([]string, error) {
+	if len(taskEnvs) < 1 {
+		return stepEnvs, nil
+	}
+
+	//preprocess the comma case
+	var newTaskEnvs []string
+	for _, env := range taskEnvs {
+		newEnv := strings.Split(env, ",")
+		newTaskEnvs = append(newTaskEnvs, newEnv...)
+	}
+
+	var stepmap = make(map[string]string)
+	//parse stepEnvs into a map
+	for _, env := range stepEnvs {
+		pair := strings.SplitN(env, "=", 2)
+		if len(pair) != 2 {
+			err := fmt.Errorf("Can not parse step environment variable %s correctly", env)
+			return stepEnvs, err
+		}
+		stepmap[pair[0]] = pair[1]
+	}
+
+	//merge the unique taskEnvs into stepEnvs
+	for _, env := range newTaskEnvs {
+		pair := strings.SplitN(env, "=", 2)
+
+		if len(pair) != 2 {
+			err := fmt.Errorf("Can not parse task environment variable %s correctly", env)
+			return stepEnvs, err
+		}
+
+		//if the env has not been provided, add to step env
+		if _, ok := stepmap[pair[0]]; !ok {
+			stepEnvs = append(stepEnvs, pair[0]+"="+pair[1])
+		}
+
+	}
+
+	return stepEnvs, nil
+}

graph/task_test.go

@@ -1,6 +1,8 @@
 package graph
 
-import "testing"
+import (
+	"testing"
+)
 
 func TestUsingRegistryCreds(t *testing.T) {
 	tests := []struct {
@@ -30,3 +32,41 @@ func TestUsingRegistryCreds(t *testing.T) {
 		}
 	}
 }
+
+func TestMergingEnvs(t *testing.T) {
+	stepEnvsTests := [][]string{
+		{},
+		{"key1=newVal1", "key2=newVal2"},
+		{"key1=newVal1", "key2=newVal2", "key3=newVal3="},
+		{},
+		{"key1=newVal1", "key2=newVal2"},
+		{"key1=newVal1", "key2=newVal2", "key3=newVal3="},
+	}
+	taskEnvsTests := [][]string{
+		{"key1=val1", "key2=val2", "key3=val3"},
+		{"key1=val1", "key2=val2", "key3=val3"},
+		{"key1=val1,key2=val2,key3=val3"},
+		{"key1=val1,key2=val2,key3=val3"},
+		{"key1=val1,key2=val2", "key3=val3,key4=val4"},
+		{"key1=val1,key2=val2", "key3=val3,key4=val4"},
+	}
+
+	//Expect: stepEnvs should overwrite envs that exist in taskEnvs
+	expects := [][]string{
+		{"key1=val1", "key2=val2", "key3=val3"},
+		{"key1=newVal1", "key2=newVal2", "key3=val3"},
+		{"key1=newVal1", "key2=newVal2", "key3=newVal3="},
+		{"key1=val1", "key2=val2", "key3=val3"},
+		{"key1=newVal1", "key2=newVal2", "key3=val3", "key4=val4"},
+		{"key1=newVal1", "key2=newVal2", "key3=newVal3=", "key4=val4"},
+	}
+
+	for i := range taskEnvsTests {
+		mergeEnvs, _ := mergeEnvs(stepEnvsTests[i], taskEnvsTests[i])
+		for j := range mergeEnvs {
+			if expects[i][j] != mergeEnvs[j] {
+				t.Errorf("running test %v, expected merge of step and task envs to be %v but got %v", i, expects[i], mergeEnvs)
+			}
+		}
+	}
+}

graph/testdata/acb.yaml

@@ -60,4 +60,7 @@ steps:
     build: "-f Dockerfile https://github.com/ehotinger/qaz --cache-from=ubuntu"
     privileged: true
     user: root
-    network: "host"
+    network: "host"
+
+envs:
+  - "foo=taskEnv"