builder: initiate image transfers in parallel and make timeout configurable.

.web-docs/components/builder/digitalocean/README.md

@@ -76,20 +76,23 @@ each category, the available configuration keys are alphabetized.
   appear in your account. Defaults to `packer-{{timestamp}}` (see
   configuration templates for more info).
 
-- `snapshot_regions` ([]string) - The regions of the resulting
-  snapshot that will appear in your account.
+- `snapshot_regions` ([]string) - Additional regions that resulting snapshot should be distributed to.
+
+- `wait_snapshot_transfer` (\*bool) - When true, Packer will block until all snapshot transfers have been completed
+  and report errors. When false, Packer will initiate the snapshot transfers
+  and exit successfully without waiting for completion. Defaults to true.
+
+- `transfer_timeout` (duration string | ex: "1h5m2s") - How long to wait for a snapshot to be transferred to an additional region
+  before timing out. The default transfer timeout is "30m" (valid time units
+  include `s` for seconds, `m` for minutes, and `h` for hours).
 
 - `state_timeout` (duration string | ex: "1h5m2s") - The time to wait, as a duration string, for a
   droplet to enter a desired state (such as "active") before timing out. The
   default state timeout is "6m".
 
-- `snapshot_timeout` (duration string | ex: "1h5m2s") - How long to wait for an image to be published to the shared image
-  gallery before timing out. If your Packer build is failing on the
-  Publishing to Shared Image Gallery step with the error `Original Error:
-  context deadline exceeded`, but the image is present when you check your
-  Azure dashboard, then you probably need to increase this timeout from
-  its default of "60m" (valid time units include `s` for seconds, `m` for
-  minutes, and `h` for hours.)
+- `snapshot_timeout` (duration string | ex: "1h5m2s") - How long to wait for the Droplet snapshot to complete before timing out.
+  The default snapshot timeout is "60m" (valid time units include `s` for
+  seconds, `m` for minutes, and `h` for hours).
 
 - `droplet_name` (string) - The name assigned to the droplet. DigitalOcean
   sets the hostname of the machine to this value.

builder/digitalocean/builder.go

@@ -136,7 +136,9 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook)
 		new(stepShutdown),
 		new(stepPowerOff),
 		&stepSnapshot{
-			snapshotTimeout: b.config.SnapshotTimeout,
+			snapshotTimeout:         b.config.SnapshotTimeout,
+			transferTimeout:         b.config.TransferTimeout,
+			waitForSnapshotTransfer: *b.config.WaitSnapshotTransfer,
 		},
 	}
 

builder/digitalocean/builder_acc_test.go

@@ -3,8 +3,10 @@ package digitalocean
 import (
 	"context"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
+	"regexp"
 	"testing"
 
 	"github.com/digitalocean/godo"
@@ -68,6 +70,43 @@ func TestBuilderAcc_multiRegion(t *testing.T) {
 	})
 }
 
+func TestBuilderAcc_multiRegionNoWait(t *testing.T) {
+	if skip := testAccPreCheck(t); skip == true {
+		return
+	}
+	acctest.TestPlugin(t, &acctest.PluginTestCase{
+		Name:     "test-digitalocean-builder-multi-region",
+		Template: fmt.Sprintf(testBuilderAccMultiRegionNoWait, "ubuntu-20-04-x64"),
+		Check: func(buildCommand *exec.Cmd, logfile string) error {
+			if buildCommand.ProcessState != nil {
+				if buildCommand.ProcessState.ExitCode() != 0 {
+					return fmt.Errorf("Bad exit code. Logfile: %s", logfile)
+				}
+			}
+
+			logs, err := os.Open(logfile)
+			if err != nil {
+				return fmt.Errorf("Unable find %s", logfile)
+			}
+			defer logs.Close()
+
+			logsBytes, err := io.ReadAll(logs)
+			if err != nil {
+				return fmt.Errorf("Unable to read %s", logfile)
+			}
+			logsString := string(logsBytes)
+
+			notExpected := regexp.MustCompile(`Transfer to .* is complete.`)
+			matches := notExpected.FindStringSubmatch(logsString)
+			if len(matches) > 0 {
+				return fmt.Errorf("logs contains unexpected value: %v", matches)
+			}
+
+			return nil
+		},
+	})
+}
+
 func testAccPreCheck(t *testing.T) bool {
 	if os.Getenv(acctest.TestEnvVar) == "" {
 		t.Skipf("Acceptance tests skipped unless env '%s' set", acctest.TestEnvVar)
@@ -127,9 +166,7 @@ const (
 		"region": "nyc2",
 		"size": "s-1vcpu-1gb",
 		"image": "%v",
-		"ssh_username": "root",
-		"user_data": "",
-		"user_data_file": ""
+		"ssh_username": "root"
 	}]
 }
 `
@@ -142,9 +179,21 @@ const (
 		"size": "s-1vcpu-1gb",
 		"image": "%v",
 		"ssh_username": "root",
-		"user_data": "",
-		"user_data_file": "",
-		"snapshot_regions": ["nyc2", "nyc3"]
+		"snapshot_regions": ["nyc1", "nyc2", "nyc3"]
+	}]
+}
+`
+
+	testBuilderAccMultiRegionNoWait = `
+{
+	"builders": [{
+		"type": "digitalocean",
+		"region": "nyc2",
+		"size": "s-1vcpu-1gb",
+		"image": "%v",
+		"ssh_username": "root",
+		"snapshot_regions": ["nyc2", "nyc3"],
+		"wait_snapshot_transfer": false
 	}]
 }
 `

builder/digitalocean/config.go

@@ -73,20 +73,23 @@ type Config struct {
 	// appear in your account. Defaults to `packer-{{timestamp}}` (see
 	// configuration templates for more info).
 	SnapshotName string `mapstructure:"snapshot_name" required:"false"`
-	// The regions of the resulting
-	// snapshot that will appear in your account.
+	// Additional regions that resulting snapshot should be distributed to.
 	SnapshotRegions []string `mapstructure:"snapshot_regions" required:"false"`
+	// When true, Packer will block until all snapshot transfers have been completed
+	// and report errors. When false, Packer will initiate the snapshot transfers
+	// and exit successfully without waiting for completion. Defaults to true.
+	WaitSnapshotTransfer *bool `mapstructure:"wait_snapshot_transfer" required:"false"`
+	// How long to wait for a snapshot to be transferred to an additional region
+	// before timing out. The default transfer timeout is "30m" (valid time units
+	// include `s` for seconds, `m` for minutes, and `h` for hours).
+	TransferTimeout time.Duration `mapstructure:"transfer_timeout" required:"false"`
 	// The time to wait, as a duration string, for a
 	// droplet to enter a desired state (such as "active") before timing out. The
 	// default state timeout is "6m".
 	StateTimeout time.Duration `mapstructure:"state_timeout" required:"false"`
-	// How long to wait for an image to be published to the shared image
-	// gallery before timing out. If your Packer build is failing on the
-	// Publishing to Shared Image Gallery step with the error `Original Error:
-	// context deadline exceeded`, but the image is present when you check your
-	// Azure dashboard, then you probably need to increase this timeout from
-	// its default of "60m" (valid time units include `s` for seconds, `m` for
-	// minutes, and `h` for hours.)
+	// How long to wait for the Droplet snapshot to complete before timing out.
+	// The default snapshot timeout is "60m" (valid time units include `s` for
+	// seconds, `m` for minutes, and `h` for hours).
 	SnapshotTimeout time.Duration `mapstructure:"snapshot_timeout" required:"false"`
 	// The name assigned to the droplet. DigitalOcean
 	// sets the hostname of the machine to this value.
@@ -212,6 +215,14 @@ func (c *Config) Prepare(raws ...interface{}) ([]string, error) {
 		c.SnapshotTimeout = 60 * time.Minute
 	}
 
+	if c.TransferTimeout == 0 {
+		c.TransferTimeout = 30 * time.Minute
+	}
+
+	if c.WaitSnapshotTransfer == nil {
+		c.WaitSnapshotTransfer = godo.PtrTo(true)
+	}
+
 	if es := c.Comm.Prepare(&c.ctx); len(es) > 0 {
 		errs = packersdk.MultiErrorAppend(errs, es...)
 	}

builder/digitalocean/step_snapshot.go

@@ -10,10 +10,13 @@ import (
 	"github.com/digitalocean/godo"
 	"github.com/hashicorp/packer-plugin-sdk/multistep"
 	packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
+	"golang.org/x/sync/errgroup"
 )
 
 type stepSnapshot struct {
-	snapshotTimeout time.Duration
+	snapshotTimeout         time.Duration
+	transferTimeout         time.Duration
+	waitForSnapshotTransfer bool
 }
 
 func (s *stepSnapshot) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction {
@@ -65,61 +68,72 @@ func (s *stepSnapshot) Run(ctx context.Context, state multistep.StateBag) multis
 		return multistep.ActionHalt
 	}
 
+	var imageId int
+	if len(images) == 1 {
+		imageId = images[0].ID
+		log.Printf("Snapshot image ID: %d", imageId)
+	} else {
+		err := errors.New("Couldn't find snapshot to get the image ID. Bug?")
+		state.Put("error", err)
+		ui.Error(err.Error())
+		return multistep.ActionHalt
+	}
+
 	if len(c.SnapshotRegions) > 0 {
-		regionSet := make(map[string]struct{})
+		regionSet := make(map[string]bool)
 		regions := make([]string, 0, len(c.SnapshotRegions))
-		regionSet[c.Region] = struct{}{}
+		regionSet[c.Region] = true
 		for _, region := range c.SnapshotRegions {
 			// If we already saw the region, then don't look again
-			if _, ok := regionSet[region]; ok {
+			if regionSet[region] {
 				continue
 			}
 
 			// Mark that we saw the region
-			regionSet[region] = struct{}{}
+			regionSet[region] = true
 
 			regions = append(regions, region)
 		}
-		snapshotRegions = regions
-
-		for transfer := range snapshotRegions {
-			transferRequest := &godo.ActionRequest{
-				"type":   "transfer",
-				"region": snapshotRegions[transfer],
-			}
 
-			ui.Say(fmt.Sprintf("Transferring snapshot (ID: %d) to %s", images[0].ID, snapshotRegions[transfer]))
-			imageTransfer, _, err := client.ImageActions.Transfer(context.TODO(), images[0].ID, transferRequest)
-			if err != nil {
-				err := fmt.Errorf("Error transferring snapshot: %s", err)
-				state.Put("error", err)
-				ui.Error(err.Error())
-				return multistep.ActionHalt
-			}
+		eg, gCtx := errgroup.WithContext(ctx)
+		for _, r := range regions {
+			region := r
+			eg.Go(func() error {
+				transferRequest := &godo.ActionRequest{
+					"type":   "transfer",
+					"region": region,
+				}
+
+				ui.Say(fmt.Sprintf("Transferring snapshot (ID: %d) to %s...", imageId, region))
+				imageTransfer, _, err := client.ImageActions.Transfer(gCtx, imageId, transferRequest)
+				if err != nil {
+					return fmt.Errorf("Error transferring snapshot: %s", err)
+				}
+
+				if s.waitForSnapshotTransfer {
+					if err := WaitForImageState(
+						godo.ActionCompleted,
+						imageId,
+						imageTransfer.ID,
+						client, s.transferTimeout); err != nil {
+						return fmt.Errorf("Error waiting for snapshot transfer: %s", err)
+					}
+					ui.Say(fmt.Sprintf("Transfer to %s is complete.", region))
+				}
+
+				return nil
+			})
+		}
 
-			if err := WaitForImageState(godo.ActionCompleted, images[0].ID, imageTransfer.ID,
-				client, 20*time.Minute); err != nil {
-				// If we get an error the first time, actually report it
-				err := fmt.Errorf("Error waiting for snapshot transfer: %s", err)
-				state.Put("error", err)
-				ui.Error(err.Error())
-				return multistep.ActionHalt
-			}
+		if err := eg.Wait(); err != nil {
+			state.Put("error", err)
+			ui.Error(err.Error())
+			return multistep.ActionHalt
 		}
 	}
 
-	var imageId int
-	if len(images) == 1 {
-		imageId = images[0].ID
-	} else {
-		err := errors.New("Couldn't find snapshot to get the image ID. Bug?")
-		state.Put("error", err)
-		ui.Error(err.Error())
-		return multistep.ActionHalt
-	}
 	snapshotRegions = append(snapshotRegions, c.Region)
 
-	log.Printf("Snapshot image ID: %d", imageId)
 	state.Put("snapshot_image_id", imageId)
 	state.Put("snapshot_name", c.SnapshotName)
 	state.Put("regions", snapshotRegions)

docs-partials/builder/digitalocean/Config-not-required.mdx

@@ -30,20 +30,23 @@
   appear in your account. Defaults to `packer-{{timestamp}}` (see
   configuration templates for more info).
 
-- `snapshot_regions` ([]string) - The regions of the resulting
-  snapshot that will appear in your account.
+- `snapshot_regions` ([]string) - Additional regions that resulting snapshot should be distributed to.
+
+- `wait_snapshot_transfer` (\*bool) - When true, Packer will block until all snapshot transfers have been completed
+  and report errors. When false, Packer will initiate the snapshot transfers
+  and exit successfully without waiting for completion. Defaults to true.
+
+- `transfer_timeout` (duration string | ex: "1h5m2s") - How long to wait for a snapshot to be transferred to an additional region
+  before timing out. The default transfer timeout is "30m" (valid time units
+  include `s` for seconds, `m` for minutes, and `h` for hours).
 
 - `state_timeout` (duration string | ex: "1h5m2s") - The time to wait, as a duration string, for a
   droplet to enter a desired state (such as "active") before timing out. The
   default state timeout is "6m".
 
-- `snapshot_timeout` (duration string | ex: "1h5m2s") - How long to wait for an image to be published to the shared image
-  gallery before timing out. If your Packer build is failing on the
-  Publishing to Shared Image Gallery step with the error `Original Error:
-  context deadline exceeded`, but the image is present when you check your
-  Azure dashboard, then you probably need to increase this timeout from
-  its default of "60m" (valid time units include `s` for seconds, `m` for
-  minutes, and `h` for hours.)
+- `snapshot_timeout` (duration string | ex: "1h5m2s") - How long to wait for the Droplet snapshot to complete before timing out.
+  The default snapshot timeout is "60m" (valid time units include `s` for
+  seconds, `m` for minutes, and `h` for hours).
 
 - `droplet_name` (string) - The name assigned to the droplet. DigitalOcean
   sets the hostname of the machine to this value.

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/zclconf/go-cty v1.13.3
 	golang.org/x/oauth2 v0.1.0
+	golang.org/x/sync v0.4.0
 )
 
 require (