diff --git a/.web-docs/components/builder/digitalocean/README.md b/.web-docs/components/builder/digitalocean/README.md index 35350cf..514c4ee 100644 --- a/.web-docs/components/builder/digitalocean/README.md +++ b/.web-docs/components/builder/digitalocean/README.md @@ -76,20 +76,23 @@ each category, the available configuration keys are alphabetized. appear in your account. Defaults to `packer-{{timestamp}}` (see configuration templates for more info). -- `snapshot_regions` ([]string) - The regions of the resulting - snapshot that will appear in your account. +- `snapshot_regions` ([]string) - Additional regions that resulting snapshot should be distributed to. + +- `wait_snapshot_transfer` (\*bool) - When true, Packer will block until all snapshot transfers have been completed + and report errors. When false, Packer will initiate the snapshot transfers + and exit successfully without waiting for completion. Defaults to true. + +- `transfer_timeout` (duration string | ex: "1h5m2s") - How long to wait for a snapshot to be transferred to an additional region + before timing out. The default transfer timeout is "30m" (valid time units + include `s` for seconds, `m` for minutes, and `h` for hours). - `state_timeout` (duration string | ex: "1h5m2s") - The time to wait, as a duration string, for a droplet to enter a desired state (such as "active") before timing out. The default state timeout is "6m". -- `snapshot_timeout` (duration string | ex: "1h5m2s") - How long to wait for an image to be published to the shared image - gallery before timing out. If your Packer build is failing on the - Publishing to Shared Image Gallery step with the error `Original Error: - context deadline exceeded`, but the image is present when you check your - Azure dashboard, then you probably need to increase this timeout from - its default of "60m" (valid time units include `s` for seconds, `m` for - minutes, and `h` for hours.) +- `snapshot_timeout` (duration string | ex: "1h5m2s") - How long to wait for the Droplet snapshot to complete before timing out. + The default snapshot timeout is "60m" (valid time units include `s` for + seconds, `m` for minutes, and `h` for hours). - `droplet_name` (string) - The name assigned to the droplet. DigitalOcean sets the hostname of the machine to this value. diff --git a/builder/digitalocean/builder.go b/builder/digitalocean/builder.go index 0532410..e2383ed 100644 --- a/builder/digitalocean/builder.go +++ b/builder/digitalocean/builder.go @@ -136,7 +136,9 @@ func (b *Builder) Run(ctx context.Context, ui packersdk.Ui, hook packersdk.Hook) new(stepShutdown), new(stepPowerOff), &stepSnapshot{ - snapshotTimeout: b.config.SnapshotTimeout, + snapshotTimeout: b.config.SnapshotTimeout, + transferTimeout: b.config.TransferTimeout, + waitForSnapshotTransfer: *b.config.WaitSnapshotTransfer, }, } diff --git a/builder/digitalocean/builder_acc_test.go b/builder/digitalocean/builder_acc_test.go index f88fb1d..b779932 100644 --- a/builder/digitalocean/builder_acc_test.go +++ b/builder/digitalocean/builder_acc_test.go @@ -3,8 +3,10 @@ package digitalocean import ( "context" "fmt" + "io" "os" "os/exec" + "regexp" "testing" "github.com/digitalocean/godo" @@ -68,6 +70,43 @@ func TestBuilderAcc_multiRegion(t *testing.T) { }) } +func TestBuilderAcc_multiRegionNoWait(t *testing.T) { + if skip := testAccPreCheck(t); skip == true { + return + } + acctest.TestPlugin(t, &acctest.PluginTestCase{ + Name: "test-digitalocean-builder-multi-region", + Template: fmt.Sprintf(testBuilderAccMultiRegionNoWait, "ubuntu-20-04-x64"), + Check: func(buildCommand *exec.Cmd, logfile string) error { + if buildCommand.ProcessState != nil { + if buildCommand.ProcessState.ExitCode() != 0 { + return fmt.Errorf("Bad exit code. Logfile: %s", logfile) + } + } + + logs, err := os.Open(logfile) + if err != nil { + return fmt.Errorf("Unable find %s", logfile) + } + defer logs.Close() + + logsBytes, err := io.ReadAll(logs) + if err != nil { + return fmt.Errorf("Unable to read %s", logfile) + } + logsString := string(logsBytes) + + notExpected := regexp.MustCompile(`Transfer to .* is complete.`) + matches := notExpected.FindStringSubmatch(logsString) + if len(matches) > 0 { + return fmt.Errorf("logs contains unexpected value: %v", matches) + } + + return nil + }, + }) +} + func testAccPreCheck(t *testing.T) bool { if os.Getenv(acctest.TestEnvVar) == "" { t.Skipf("Acceptance tests skipped unless env '%s' set", acctest.TestEnvVar) @@ -127,9 +166,7 @@ const ( "region": "nyc2", "size": "s-1vcpu-1gb", "image": "%v", - "ssh_username": "root", - "user_data": "", - "user_data_file": "" + "ssh_username": "root" }] } ` @@ -142,9 +179,21 @@ const ( "size": "s-1vcpu-1gb", "image": "%v", "ssh_username": "root", - "user_data": "", - "user_data_file": "", - "snapshot_regions": ["nyc2", "nyc3"] + "snapshot_regions": ["nyc1", "nyc2", "nyc3"] + }] +} +` + + testBuilderAccMultiRegionNoWait = ` +{ + "builders": [{ + "type": "digitalocean", + "region": "nyc2", + "size": "s-1vcpu-1gb", + "image": "%v", + "ssh_username": "root", + "snapshot_regions": ["nyc2", "nyc3"], + "wait_snapshot_transfer": false }] } ` diff --git a/builder/digitalocean/config.go b/builder/digitalocean/config.go index 1671d71..06813a1 100644 --- a/builder/digitalocean/config.go +++ b/builder/digitalocean/config.go @@ -73,20 +73,23 @@ type Config struct { // appear in your account. Defaults to `packer-{{timestamp}}` (see // configuration templates for more info). SnapshotName string `mapstructure:"snapshot_name" required:"false"` - // The regions of the resulting - // snapshot that will appear in your account. + // Additional regions that resulting snapshot should be distributed to. SnapshotRegions []string `mapstructure:"snapshot_regions" required:"false"` + // When true, Packer will block until all snapshot transfers have been completed + // and report errors. When false, Packer will initiate the snapshot transfers + // and exit successfully without waiting for completion. Defaults to true. + WaitSnapshotTransfer *bool `mapstructure:"wait_snapshot_transfer" required:"false"` + // How long to wait for a snapshot to be transferred to an additional region + // before timing out. The default transfer timeout is "30m" (valid time units + // include `s` for seconds, `m` for minutes, and `h` for hours). + TransferTimeout time.Duration `mapstructure:"transfer_timeout" required:"false"` // The time to wait, as a duration string, for a // droplet to enter a desired state (such as "active") before timing out. The // default state timeout is "6m". StateTimeout time.Duration `mapstructure:"state_timeout" required:"false"` - // How long to wait for an image to be published to the shared image - // gallery before timing out. If your Packer build is failing on the - // Publishing to Shared Image Gallery step with the error `Original Error: - // context deadline exceeded`, but the image is present when you check your - // Azure dashboard, then you probably need to increase this timeout from - // its default of "60m" (valid time units include `s` for seconds, `m` for - // minutes, and `h` for hours.) + // How long to wait for the Droplet snapshot to complete before timing out. + // The default snapshot timeout is "60m" (valid time units include `s` for + // seconds, `m` for minutes, and `h` for hours). SnapshotTimeout time.Duration `mapstructure:"snapshot_timeout" required:"false"` // The name assigned to the droplet. DigitalOcean // sets the hostname of the machine to this value. @@ -212,6 +215,14 @@ func (c *Config) Prepare(raws ...interface{}) ([]string, error) { c.SnapshotTimeout = 60 * time.Minute } + if c.TransferTimeout == 0 { + c.TransferTimeout = 30 * time.Minute + } + + if c.WaitSnapshotTransfer == nil { + c.WaitSnapshotTransfer = godo.PtrTo(true) + } + if es := c.Comm.Prepare(&c.ctx); len(es) > 0 { errs = packersdk.MultiErrorAppend(errs, es...) } diff --git a/builder/digitalocean/config.hcl2spec.go b/builder/digitalocean/config.hcl2spec.go index 28ebd98..abb1871 100644 --- a/builder/digitalocean/config.hcl2spec.go +++ b/builder/digitalocean/config.hcl2spec.go @@ -81,6 +81,8 @@ type FlatConfig struct { IPv6 *bool `mapstructure:"ipv6" required:"false" cty:"ipv6" hcl:"ipv6"` SnapshotName *string `mapstructure:"snapshot_name" required:"false" cty:"snapshot_name" hcl:"snapshot_name"` SnapshotRegions []string `mapstructure:"snapshot_regions" required:"false" cty:"snapshot_regions" hcl:"snapshot_regions"` + WaitSnapshotTransfer *bool `mapstructure:"wait_snapshot_transfer" required:"false" cty:"wait_snapshot_transfer" hcl:"wait_snapshot_transfer"` + TransferTimeout *string `mapstructure:"transfer_timeout" required:"false" cty:"transfer_timeout" hcl:"transfer_timeout"` StateTimeout *string `mapstructure:"state_timeout" required:"false" cty:"state_timeout" hcl:"state_timeout"` SnapshotTimeout *string `mapstructure:"snapshot_timeout" required:"false" cty:"snapshot_timeout" hcl:"snapshot_timeout"` DropletName *string `mapstructure:"droplet_name" required:"false" cty:"droplet_name" hcl:"droplet_name"` @@ -175,6 +177,8 @@ func (*FlatConfig) HCL2Spec() map[string]hcldec.Spec { "ipv6": &hcldec.AttrSpec{Name: "ipv6", Type: cty.Bool, Required: false}, "snapshot_name": &hcldec.AttrSpec{Name: "snapshot_name", Type: cty.String, Required: false}, "snapshot_regions": &hcldec.AttrSpec{Name: "snapshot_regions", Type: cty.List(cty.String), Required: false}, + "wait_snapshot_transfer": &hcldec.AttrSpec{Name: "wait_snapshot_transfer", Type: cty.Bool, Required: false}, + "transfer_timeout": &hcldec.AttrSpec{Name: "transfer_timeout", Type: cty.String, Required: false}, "state_timeout": &hcldec.AttrSpec{Name: "state_timeout", Type: cty.String, Required: false}, "snapshot_timeout": &hcldec.AttrSpec{Name: "snapshot_timeout", Type: cty.String, Required: false}, "droplet_name": &hcldec.AttrSpec{Name: "droplet_name", Type: cty.String, Required: false}, diff --git a/builder/digitalocean/step_snapshot.go b/builder/digitalocean/step_snapshot.go index 372b2b4..ee6259e 100644 --- a/builder/digitalocean/step_snapshot.go +++ b/builder/digitalocean/step_snapshot.go @@ -10,10 +10,13 @@ import ( "github.com/digitalocean/godo" "github.com/hashicorp/packer-plugin-sdk/multistep" packersdk "github.com/hashicorp/packer-plugin-sdk/packer" + "golang.org/x/sync/errgroup" ) type stepSnapshot struct { - snapshotTimeout time.Duration + snapshotTimeout time.Duration + transferTimeout time.Duration + waitForSnapshotTransfer bool } func (s *stepSnapshot) Run(ctx context.Context, state multistep.StateBag) multistep.StepAction { @@ -65,61 +68,72 @@ func (s *stepSnapshot) Run(ctx context.Context, state multistep.StateBag) multis return multistep.ActionHalt } + var imageId int + if len(images) == 1 { + imageId = images[0].ID + log.Printf("Snapshot image ID: %d", imageId) + } else { + err := errors.New("Couldn't find snapshot to get the image ID. Bug?") + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt + } + if len(c.SnapshotRegions) > 0 { - regionSet := make(map[string]struct{}) + regionSet := make(map[string]bool) regions := make([]string, 0, len(c.SnapshotRegions)) - regionSet[c.Region] = struct{}{} + regionSet[c.Region] = true for _, region := range c.SnapshotRegions { // If we already saw the region, then don't look again - if _, ok := regionSet[region]; ok { + if regionSet[region] { continue } // Mark that we saw the region - regionSet[region] = struct{}{} + regionSet[region] = true regions = append(regions, region) } - snapshotRegions = regions - - for transfer := range snapshotRegions { - transferRequest := &godo.ActionRequest{ - "type": "transfer", - "region": snapshotRegions[transfer], - } - ui.Say(fmt.Sprintf("Transferring snapshot (ID: %d) to %s", images[0].ID, snapshotRegions[transfer])) - imageTransfer, _, err := client.ImageActions.Transfer(context.TODO(), images[0].ID, transferRequest) - if err != nil { - err := fmt.Errorf("Error transferring snapshot: %s", err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } + eg, gCtx := errgroup.WithContext(ctx) + for _, r := range regions { + region := r + eg.Go(func() error { + transferRequest := &godo.ActionRequest{ + "type": "transfer", + "region": region, + } + + ui.Say(fmt.Sprintf("Transferring snapshot (ID: %d) to %s...", imageId, region)) + imageTransfer, _, err := client.ImageActions.Transfer(gCtx, imageId, transferRequest) + if err != nil { + return fmt.Errorf("Error transferring snapshot: %s", err) + } + + if s.waitForSnapshotTransfer { + if err := WaitForImageState( + godo.ActionCompleted, + imageId, + imageTransfer.ID, + client, s.transferTimeout); err != nil { + return fmt.Errorf("Error waiting for snapshot transfer: %s", err) + } + ui.Say(fmt.Sprintf("Transfer to %s is complete.", region)) + } + + return nil + }) + } - if err := WaitForImageState(godo.ActionCompleted, images[0].ID, imageTransfer.ID, - client, 20*time.Minute); err != nil { - // If we get an error the first time, actually report it - err := fmt.Errorf("Error waiting for snapshot transfer: %s", err) - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } + if err := eg.Wait(); err != nil { + state.Put("error", err) + ui.Error(err.Error()) + return multistep.ActionHalt } } - var imageId int - if len(images) == 1 { - imageId = images[0].ID - } else { - err := errors.New("Couldn't find snapshot to get the image ID. Bug?") - state.Put("error", err) - ui.Error(err.Error()) - return multistep.ActionHalt - } snapshotRegions = append(snapshotRegions, c.Region) - log.Printf("Snapshot image ID: %d", imageId) state.Put("snapshot_image_id", imageId) state.Put("snapshot_name", c.SnapshotName) state.Put("regions", snapshotRegions) diff --git a/docs-partials/builder/digitalocean/Config-not-required.mdx b/docs-partials/builder/digitalocean/Config-not-required.mdx index 7494b5d..db1195b 100644 --- a/docs-partials/builder/digitalocean/Config-not-required.mdx +++ b/docs-partials/builder/digitalocean/Config-not-required.mdx @@ -30,20 +30,23 @@ appear in your account. Defaults to `packer-{{timestamp}}` (see configuration templates for more info). -- `snapshot_regions` ([]string) - The regions of the resulting - snapshot that will appear in your account. +- `snapshot_regions` ([]string) - Additional regions that resulting snapshot should be distributed to. + +- `wait_snapshot_transfer` (\*bool) - When true, Packer will block until all snapshot transfers have been completed + and report errors. When false, Packer will initiate the snapshot transfers + and exit successfully without waiting for completion. Defaults to true. + +- `transfer_timeout` (duration string | ex: "1h5m2s") - How long to wait for a snapshot to be transferred to an additional region + before timing out. The default transfer timeout is "30m" (valid time units + include `s` for seconds, `m` for minutes, and `h` for hours). - `state_timeout` (duration string | ex: "1h5m2s") - The time to wait, as a duration string, for a droplet to enter a desired state (such as "active") before timing out. The default state timeout is "6m". -- `snapshot_timeout` (duration string | ex: "1h5m2s") - How long to wait for an image to be published to the shared image - gallery before timing out. If your Packer build is failing on the - Publishing to Shared Image Gallery step with the error `Original Error: - context deadline exceeded`, but the image is present when you check your - Azure dashboard, then you probably need to increase this timeout from - its default of "60m" (valid time units include `s` for seconds, `m` for - minutes, and `h` for hours.) +- `snapshot_timeout` (duration string | ex: "1h5m2s") - How long to wait for the Droplet snapshot to complete before timing out. + The default snapshot timeout is "60m" (valid time units include `s` for + seconds, `m` for minutes, and `h` for hours). - `droplet_name` (string) - The name assigned to the droplet. DigitalOcean sets the hostname of the machine to this value. diff --git a/go.mod b/go.mod index 5b76e4a..8b2ce7a 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/zclconf/go-cty v1.13.3 golang.org/x/oauth2 v0.1.0 + golang.org/x/sync v0.4.0 ) require (