diff --git a/aws/provider.go b/aws/provider.go index 4feb7cd4181..bdeac35a97a 100644 --- a/aws/provider.go +++ b/aws/provider.go @@ -465,6 +465,7 @@ func Provider() terraform.ResourceProvider { "aws_elb_attachment": resourceAwsElbAttachment(), "aws_emr_cluster": resourceAwsEMRCluster(), "aws_emr_instance_group": resourceAwsEMRInstanceGroup(), + "aws_emr_instance_fleet": resourceAwsEMRInstanceFleet(), "aws_emr_security_configuration": resourceAwsEMRSecurityConfiguration(), "aws_flow_log": resourceAwsFlowLog(), "aws_gamelift_alias": resourceAwsGameliftAlias(), diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index 555d0b0e9e2..7f62fbc0dcf 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -89,6 +89,11 @@ func resourceAwsEMRCluster() *schema.Resource { ForceNew: true, Required: true, }, + "hadoop_version": { + Type: schema.TypeString, + ForceNew: true, + Optional: true, + }, "release_label": { Type: schema.TypeString, ForceNew: true, @@ -134,7 +139,7 @@ func resourceAwsEMRCluster() *schema.Resource { DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { // EMR uses a proprietary filesystem called EMRFS // and both s3n & s3 protocols are mapped to that FS - // so they're equvivalent in this context (confirmed by AWS support) + // so they're equivalent in this context (confirmed by AWS support) old = strings.Replace(old, "s3n://", "s3://", -1) return old == new }, @@ -176,7 +181,12 @@ func resourceAwsEMRCluster() *schema.Resource { "subnet_id": { Type: schema.TypeString, Optional: true, - ForceNew: true, + Computed: true, + }, + "subnet_ids": { + Type: schema.TypeString, + Optional: true, + Computed: true, }, "additional_master_security_groups": { Type: schema.TypeString, @@ -208,6 +218,16 @@ func resourceAwsEMRCluster() *schema.Resource { Optional: true, ForceNew: true, }, + "availability_zone": { + Type: schema.TypeString, + Optional: true, + Computed: true, + }, + "availability_zones": { + Type: schema.TypeString, + Optional: true, + Computed: true, + }, }, }, }, @@ -260,33 +280,16 @@ func resourceAwsEMRCluster() *schema.Resource { Type: schema.TypeString, Optional: true, }, + "ebs_optimized": { + Type: schema.TypeBool, + Optional: true, + ForceNew: true, + }, "ebs_config": { Type: schema.TypeSet, Optional: true, - Computed: true, - Elem: &schema.Resource{ - Schema: map[string]*schema.Schema{ - "iops": { - Type: schema.TypeInt, - Optional: true, - }, - "size": { - Type: schema.TypeInt, - Required: true, - }, - "type": { - Type: schema.TypeString, - Required: true, - ValidateFunc: validateAwsEmrEbsVolumeType(), - }, - "volumes_per_instance": { - Type: schema.TypeInt, - Optional: true, - Default: 1, - }, - }, - }, - Set: resourceAwsEMRClusterEBSConfigHash, + ForceNew: true, + Elem: ebsConfigurationSchema(), }, "instance_count": { Type: schema.TypeInt, @@ -328,6 +331,50 @@ func resourceAwsEMRCluster() *schema.Resource { }, Set: resourceAwsEMRClusterInstanceGroupHash, }, + "instance_fleet": { + Type: schema.TypeSet, + Optional: true, + ForceNew: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "instance_fleet_type": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + ValidateFunc: validateAwsEmrInstanceFleetType, + }, + "instance_type_configs": { + Type: schema.TypeSet, + Optional: true, + ForceNew: true, + Elem: instanceTypeConfigSchema(), + }, + "launch_specifications": { + Type: schema.TypeSet, + Optional: true, + ForceNew: true, + MinItems: 1, + MaxItems: 1, + Elem: launchSpecificationsSchema(), + }, + "name": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + }, + "target_on_demand_capacity": { + Type: schema.TypeInt, + Optional: true, + Default: 0, + }, + "target_spot_capacity": { + Type: schema.TypeInt, + Optional: true, + Default: 0, + }, + }, + }, + }, "bootstrap_action": { Type: schema.TypeSet, Optional: true, @@ -472,6 +519,62 @@ func resourceAwsEMRCluster() *schema.Resource { } } +func ebsConfigurationSchema() *schema.Resource { + return &schema.Resource{ + Schema: map[string]*schema.Schema{ + "iops": { + Type: schema.TypeInt, + Optional: true, + }, + "size": { + Type: schema.TypeInt, + Required: true, + }, + "type": { + Type: schema.TypeString, + Required: true, + ValidateFunc: validateAwsEmrEbsVolumeType(), + }, + "volumes_per_instance": { + Type: schema.TypeInt, + Optional: true, + }, + }, + } +} + +func expandEbsConfiguration(ebsConfigurations []interface{}) *emr.EbsConfiguration { + ebsConfig := &emr.EbsConfiguration{} + + ebsConfigs := make([]*emr.EbsBlockDeviceConfig, 0) + for _, ebsConfiguration := range ebsConfigurations { + cfg := ebsConfiguration.(map[string]interface{}) + + ebs := &emr.EbsBlockDeviceConfig{} + + volumeSpec := &emr.VolumeSpecification{ + SizeInGB: aws.Int64(int64(cfg["size"].(int))), + VolumeType: aws.String(cfg["type"].(string)), + } + + if v, ok := cfg["iops"].(int); ok && v != 0 { + volumeSpec.Iops = aws.Int64(int64(v)) + } + + if v, ok := cfg["volumes_per_instance"].(int); ok && v != 0 { + ebs.VolumesPerInstance = aws.Int64(int64(v)) + } + + ebs.VolumeSpecification = volumeSpec + + ebsConfigs = append(ebsConfigs, ebs) + } + + ebsConfig.EbsBlockDeviceConfigs = ebsConfigs + + return ebsConfig +} + func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error { conn := meta.(*AWSClient).emrconn @@ -520,6 +623,10 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error instanceConfig.InstanceGroups = append(instanceConfig.InstanceGroups, coreInstanceGroupConfig) } + if v, ok := d.GetOk("hadoop_version"); ok { + instanceConfig.HadoopVersion = aws.String(v.(string)) + } + var instanceProfile string if a, ok := d.GetOk("ec2_attributes"); ok { ec2Attributes := a.([]interface{}) @@ -528,10 +635,22 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error if v, ok := attributes["key_name"]; ok { instanceConfig.Ec2KeyName = aws.String(v.(string)) } + if v, ok := attributes["subnet_id"]; ok { instanceConfig.Ec2SubnetId = aws.String(v.(string)) } + if v, ok := attributes["subnet_ids"]; ok { + if len(v.(string)) > 0 { + strSlice := strings.Split(v.(string), ",") + for i, s := range strSlice { + strSlice[i] = strings.TrimSpace(s) + } + + instanceConfig.Ec2SubnetIds = aws.StringSlice(strSlice) + } + } + if v, ok := attributes["additional_master_security_groups"]; ok { strSlice := strings.Split(v.(string), ",") for i, s := range strSlice { @@ -551,6 +670,7 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error if v, ok := attributes["emr_managed_master_security_group"]; ok { instanceConfig.EmrManagedMasterSecurityGroup = aws.String(v.(string)) } + if v, ok := attributes["emr_managed_slave_security_group"]; ok { instanceConfig.EmrManagedSlaveSecurityGroup = aws.String(v.(string)) } @@ -562,6 +682,26 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error if v, ok := attributes["service_access_security_group"]; ok { instanceConfig.ServiceAccessSecurityGroup = aws.String(v.(string)) } + + if v, ok := attributes["availability_zone"]; ok { + instanceConfig.Placement = &emr.PlacementType{ + AvailabilityZone: aws.String(v.(string)), + } + } + + if v, ok := attributes["availability_zones"]; ok { + if len(v.(string)) > 0 { + strSlice := strings.Split(v.(string), ",") + for i, s := range strSlice { + strSlice[i] = strings.TrimSpace(s) + } + + instanceConfig.Placement = &emr.PlacementType{ + AvailabilityZones: aws.StringSlice(strSlice), + } + } + } + } if v, ok := d.GetOk("instance_group"); ok { instanceGroupConfigs := v.(*schema.Set).List() @@ -574,6 +714,11 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error instanceConfig.InstanceGroups = instanceGroups } + if v, ok := d.GetOk("instance_fleet"); ok { + instanceFleetConfigs := v.(*schema.Set).List() + instanceConfig.InstanceFleets = expandInstanceFleetConfigs(instanceFleetConfigs) + } + emrApps := expandApplications(applications) params := &emr.RunJobFlowInput{ @@ -700,7 +845,7 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error _, err = stateConf.WaitForState() if err != nil { - return fmt.Errorf("Error waiting for EMR Cluster state to be \"WAITING\" or \"RUNNING\": %s", err) + return fmt.Errorf("[WARN] Error waiting for EMR Cluster state to be %q or %q: %s", emr.ClusterStateWaiting, emr.ClusterStateRunning, err) } return resourceAwsEMRClusterRead(d, meta) @@ -759,6 +904,17 @@ func resourceAwsEMRClusterRead(d *schema.ResourceData, meta interface{}) error { } } + instanceFleets, err := fetchAllEMRInstanceFleets(emrconn, d.Id()) + if err == nil { + coreFleet := findCoreFleet(instanceFleets) + if coreFleet != nil { + d.Set("core_instance_type", coreFleet.InstanceFleetType) + } + if err := d.Set("instance_fleet", flattenInstanceFleets(instanceFleets)); err != nil { + log.Printf("[ERR] Error setting EMR instance fleets: %s", err) + } + } + d.Set("name", cluster.Name) d.Set("service_role", cluster.ServiceRole) @@ -792,7 +948,7 @@ func resourceAwsEMRClusterRead(d *schema.ResourceData, meta interface{}) error { } if err := d.Set("ec2_attributes", flattenEc2Attributes(cluster.Ec2InstanceAttributes)); err != nil { - return fmt.Errorf("error setting EMR Ec2 Attributes: %s", err) + log.Printf("[ERR] Error setting EMR EC2 Attributes: %s", err) } if err := d.Set("kerberos_attributes", flattenEmrKerberosAttributes(d, cluster.KerberosAttributes)); err != nil { @@ -1058,12 +1214,19 @@ func flattenEc2Attributes(ia *emr.Ec2InstanceAttributes) []map[string]interface{ if ia.Ec2KeyName != nil { attrs["key_name"] = *ia.Ec2KeyName } + if ia.Ec2SubnetId != nil { attrs["subnet_id"] = *ia.Ec2SubnetId } + if len(ia.RequestedEc2SubnetIds) > 0 { + strs := aws.StringValueSlice(ia.RequestedEc2SubnetIds) + attrs["subnet_ids"] = strings.Join(strs, ",") + } + if ia.IamInstanceProfile != nil { attrs["instance_profile"] = *ia.IamInstanceProfile } + if ia.EmrManagedMasterSecurityGroup != nil { attrs["emr_managed_master_security_group"] = *ia.EmrManagedMasterSecurityGroup } @@ -1084,6 +1247,14 @@ func flattenEc2Attributes(ia *emr.Ec2InstanceAttributes) []map[string]interface{ attrs["service_access_security_group"] = *ia.ServiceAccessSecurityGroup } + if ia.Ec2AvailabilityZone != nil { + attrs["availability_zone"] = *ia.Ec2AvailabilityZone + } + if len(ia.RequestedEc2AvailabilityZones) > 0 { + strs := aws.StringValueSlice(ia.RequestedEc2AvailabilityZones) + attrs["availability_zones"] = strings.Join(strs, ",") + } + result = append(result, attrs) return result @@ -1277,6 +1448,123 @@ func flattenEBSConfig(ebsBlockDevices []*emr.EbsBlockDevice) *schema.Set { return schema.NewSet(resourceAwsEMRClusterEBSConfigHash, ebsConfig) } +func flattenInstanceFleets(ifleets []*emr.InstanceFleet) []map[string]interface{} { + result := make([]map[string]interface{}, 0) + + for _, fleet := range ifleets { + attrs := make(map[string]interface{}) + + attrs["instance_fleet_type"] = *fleet.InstanceFleetType + + instanceTypeConfigs := make([]map[string]interface{}, 0) + for _, its := range fleet.InstanceTypeSpecifications { + itsAttrs := make(map[string]interface{}) + + if its.BidPrice != nil { + itsAttrs["bid_price"] = *its.BidPrice + } else { + itsAttrs["bid_price"] = "" + } + + if its.BidPriceAsPercentageOfOnDemandPrice != nil { + itsAttrs["bid_price_as_percentage_of_on_demand_price"] = *its.BidPriceAsPercentageOfOnDemandPrice + } + + configurations := make([]map[string]interface{}, 0) + if its.Configurations != nil { + for _, cfg := range its.Configurations { + configurationsAttrs := make(map[string]interface{}) + + configurationsAttrs["classification"] = *cfg.Classification + + additionalConfigurations := make([]map[string]interface{}, 0) + if cfg.Configurations != nil { + for _, additionalCfg := range cfg.Configurations { + additionalConfigurationsAttrs := make(map[string]interface{}) + + additionalConfigurationsAttrs["classification"] = *additionalCfg.Classification + + additionalProperties := make(map[string]string) + for k, v := range additionalCfg.Properties { + additionalProperties[k] = *v + } + additionalConfigurationsAttrs["properties"] = additionalProperties + + additionalConfigurations = append(additionalConfigurations, additionalConfigurationsAttrs) + } + } + configurationsAttrs["configurations"] = additionalConfigurations + + properties := make(map[string]string) + for k, v := range cfg.Properties { + properties[k] = *v + } + configurationsAttrs["properties"] = properties + + configurations = append(configurations, configurationsAttrs) + } + } + itsAttrs["configurations"] = configurations + + ebsConfiguration := make([]map[string]interface{}, 0) + if its.EbsBlockDevices != nil { + for _, ebsbd := range its.EbsBlockDevices { + ebscAttrs := make(map[string]interface{}) + + if ebsbd.VolumeSpecification != nil { + if ebsbd.VolumeSpecification.Iops != nil { + ebscAttrs["iops"] = *ebsbd.VolumeSpecification.Iops + } else { + ebscAttrs["iops"] = "" + } + ebscAttrs["size"] = *ebsbd.VolumeSpecification.SizeInGB + ebscAttrs["type"] = *ebsbd.VolumeSpecification.VolumeType + ebscAttrs["volumes_per_instance"] = 1 + } + + ebsConfiguration = append(ebsConfiguration, ebscAttrs) + } + } + if len(ebsConfiguration) > 0 { + itsAttrs["ebs_config"] = ebsConfiguration + } + + itsAttrs["instance_type"] = *its.InstanceType + itsAttrs["weighted_capacity"] = *its.WeightedCapacity + + instanceTypeConfigs = append(instanceTypeConfigs, itsAttrs) + } + if len(instanceTypeConfigs) > 0 { + attrs["instance_type_configs"] = instanceTypeConfigs + } + + launchSpecifications := make(map[string]interface{}) + if fleet.LaunchSpecifications != nil { + spotSpecification := make(map[string]interface{}) + + if fleet.LaunchSpecifications.SpotSpecification.BlockDurationMinutes != nil { + spotSpecification["block_duration_minutes"] = *fleet.LaunchSpecifications.SpotSpecification.BlockDurationMinutes + } else { + spotSpecification["block_duration_minutes"] = "" + } + + spotSpecification["timeout_action"] = *fleet.LaunchSpecifications.SpotSpecification.TimeoutAction + spotSpecification["timeout_duration_minutes"] = *fleet.LaunchSpecifications.SpotSpecification.TimeoutDurationMinutes + + launchSpecifications["spot_specification"] = spotSpecification + } + attrs["launch_specifications"] = launchSpecifications + + attrs["name"] = *fleet.Name + attrs["target_on_demand_capacity"] = *fleet.TargetOnDemandCapacity + attrs["target_spot_capacity"] = *fleet.TargetSpotCapacity + + result = append(result, attrs) + } + + return result +} + func flattenBootstrapArguments(actions []*emr.Command) []map[string]interface{} { result := make([]map[string]interface{}, 0) @@ -1300,6 +1588,17 @@ func emrCoreInstanceGroup(grps []*emr.InstanceGroup) *emr.InstanceGroup { return nil } +func findCoreFleet(flts []*emr.InstanceFleet) *emr.InstanceFleet { + for _, flt := range flts { + if flt.InstanceFleetType != nil { + if *flt.InstanceFleetType == emr.InstanceFleetTypeCore { + return flt + } + } + } + return nil +} + func expandTags(m map[string]interface{}) []*emr.Tag { var result []*emr.Tag for k, v := range m { @@ -1508,8 +1807,14 @@ func expandInstanceGroupConfigs(instanceGroupConfigs []interface{}) ([]*emr.Inst } instanceGroupConfig = append(instanceGroupConfig, config) - } + if v, ok := configAttributes["ebs_config"].(*schema.Set); ok && v.Len() == 1 { + config.EbsConfiguration = expandEbsConfiguration(v.List()) + if v, ok := configAttributes["ebs_optimized"].(bool); ok { + config.EbsConfiguration.EbsOptimized = aws.Bool(v) + } + } + } return instanceGroupConfig, nil } diff --git a/aws/resource_aws_emr_instance_fleet.go b/aws/resource_aws_emr_instance_fleet.go new file mode 100644 index 00000000000..f102634b0ee --- /dev/null +++ b/aws/resource_aws_emr_instance_fleet.go @@ -0,0 +1,541 @@ +package aws + +import ( + "errors" + "log" + "time" + + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/emr" + "github.com/hashicorp/terraform/helper/resource" + "github.com/hashicorp/terraform/helper/schema" +) + +var emrInstanceFleetNotFound = errors.New("no matching EMR Instance Fleet") + +func resourceAwsEMRInstanceFleet() *schema.Resource { + return &schema.Resource{ + Create: resourceAwsEMRInstanceFleetCreate, + Read: resourceAwsEMRInstanceFleetRead, + Update: resourceAwsEMRInstanceFleetUpdate, + Delete: resourceAwsEMRInstanceFleetDelete, + Schema: map[string]*schema.Schema{ + "cluster_id": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + }, + "instance_fleet_type": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + ValidateFunc: validateAwsEmrInstanceFleetType, + }, + "instance_type_configs": { + Type: schema.TypeSet, + Optional: true, + ForceNew: true, + Elem: instanceTypeConfigSchema(), + }, + "launch_specifications": { + Type: schema.TypeSet, + Optional: true, + ForceNew: true, + MinItems: 1, + MaxItems: 1, + Elem: launchSpecificationsSchema(), + }, + "name": { + Type: schema.TypeString, + Optional: true, + Computed: true, + }, + "target_on_demand_capacity": { + Type: schema.TypeInt, + Optional: true, + Default: 1, + }, + "target_spot_capacity": { + Type: schema.TypeInt, + Optional: true, + Default: 0, + }, + "provisioned_on_demand_capacity": { + Type: schema.TypeInt, + Computed: true, + }, + "provisioned_spot_capacity": { + Type: schema.TypeInt, + Computed: true, + }, + "status": { + Type: schema.TypeString, + Computed: true, + }, + }, + } +} + +func instanceTypeConfigSchema() *schema.Resource { + return &schema.Resource{ + Schema: map[string]*schema.Schema{ + "bid_price": { + Type: schema.TypeString, + Optional: true, + Required: false, + ForceNew: true, + }, + "bid_price_as_percentage_of_on_demand_price": { + Type: schema.TypeFloat, + Optional: true, + ForceNew: true, + }, + "configurations": { + Type: schema.TypeSet, + Optional: true, + ForceNew: true, + Elem: configurationSchema(), + }, + "ebs_optimized": { + Type: schema.TypeBool, + Optional: true, + ForceNew: true, + }, + "ebs_config": { + Type: schema.TypeSet, + Optional: true, + ForceNew: true, + Elem: ebsConfigurationSchema(), + }, + "instance_type": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + }, + "weighted_capacity": { + Type: schema.TypeInt, + Optional: true, + ForceNew: true, + Default: 1, + }, + }, + } +} + +func launchSpecificationsSchema() *schema.Resource { + return &schema.Resource{ + Schema: map[string]*schema.Schema{ + "spot_specification": { + Type: schema.TypeSet, + Required: true, + ForceNew: true, + MaxItems: 1, + MinItems: 1, + Elem: spotSpecificationSchema(), + }, + }, + } +} + +func spotSpecificationSchema() *schema.Resource { + return &schema.Resource{ + Schema: map[string]*schema.Schema{ + "block_duration_minutes": { + Type: schema.TypeInt, + Optional: true, + ForceNew: true, + Default: 0, + }, + "timeout_action": { + Type: schema.TypeString, + Required: true, + ValidateFunc: validateAwsEmrSpotProvisioningTimeOutAction, + }, + "timeout_duration_minutes": { + Type: schema.TypeInt, + Required: true, + }, + }, + } +} + +func configurationSchema() *schema.Resource { + return &schema.Resource{ + Schema: map[string]*schema.Schema{ + "classification": { + Type: schema.TypeString, + Optional: true, + }, + "configurations": { + Type: schema.TypeSet, + Optional: true, + Elem: additionalConfigurationSchema(), + }, + "properties": { + Type: schema.TypeMap, + Optional: true, + Elem: schema.TypeString, + }, + }, + } +} + +func additionalConfigurationSchema() *schema.Resource { + return &schema.Resource{ + Schema: map[string]*schema.Schema{ + "classification": { + Type: schema.TypeString, + Optional: true, + }, + "properties": { + Type: schema.TypeMap, + Optional: true, + Elem: schema.TypeString, + }, + }, + } +} + +func expandInstanceFleetConfig(data *schema.ResourceData) *emr.InstanceFleetConfig { + configInstanceFleetType := data.Get("instance_fleet_type").(string) + configName := data.Get("name").(string) + configTargetOnDemandCapacity := data.Get("target_on_demand_capacity").(int) + configTargetSpotCapacity := data.Get("target_spot_capacity").(int) + + config := &emr.InstanceFleetConfig{ + InstanceFleetType: aws.String(configInstanceFleetType), + Name: aws.String(configName), + TargetOnDemandCapacity: aws.Int64(int64(configTargetOnDemandCapacity)), + TargetSpotCapacity: aws.Int64(int64(configTargetSpotCapacity)), + } + + if v, ok := data.Get("instance_type_configs").(*schema.Set); ok && v.Len() > 0 { + config.InstanceTypeConfigs = expandInstanceTypeConfigs(v.List()) + } + + if v, ok := data.Get("launch_specifications").(*schema.Set); ok && v.Len() == 1 { + config.LaunchSpecifications = expandLaunchSpecification(v.List()[0]) + } + + return config +} + +func expandInstanceFleetConfigs(instanceFleetConfigs []interface{}) []*emr.InstanceFleetConfig { + configsOut := []*emr.InstanceFleetConfig{} + + for _, raw := range instanceFleetConfigs { + configAttributes := raw.(map[string]interface{}) + + configInstanceFleetType := configAttributes["instance_fleet_type"].(string) + configName := configAttributes["name"].(string) + configTargetOnDemandCapacity := configAttributes["target_on_demand_capacity"].(int) + configTargetSpotCapacity := configAttributes["target_spot_capacity"].(int) + + config := &emr.InstanceFleetConfig{ + InstanceFleetType: aws.String(configInstanceFleetType), + Name: aws.String(configName), + TargetOnDemandCapacity: aws.Int64(int64(configTargetOnDemandCapacity)), + TargetSpotCapacity: aws.Int64(int64(configTargetSpotCapacity)), + } + + if v, ok := configAttributes["instance_type_configs"].(*schema.Set); ok && v.Len() > 0 { + config.InstanceTypeConfigs = expandInstanceTypeConfigs(v.List()) + } + + if v, ok := configAttributes["launch_specifications"].(*schema.Set); ok && v.Len() == 1 { + config.LaunchSpecifications = expandLaunchSpecification(v.List()[0]) + } + + configsOut = append(configsOut, config) + } + + return configsOut +} + +func expandInstanceTypeConfigs(instanceTypeConfigs []interface{}) []*emr.InstanceTypeConfig { + configsOut := []*emr.InstanceTypeConfig{} + + for _, raw := range instanceTypeConfigs { + configAttributes := raw.(map[string]interface{}) + + configInstanceType := configAttributes["instance_type"].(string) + + config := &emr.InstanceTypeConfig{ + InstanceType: aws.String(configInstanceType), + } + + if bidPrice, ok := configAttributes["bid_price"]; ok { + if bidPrice != "" { + config.BidPrice = aws.String(bidPrice.(string)) + } + } + + if v, ok := configAttributes["bid_price_as_percentage_of_on_demand_price"].(float64); ok && v != 0 { + config.BidPriceAsPercentageOfOnDemandPrice = aws.Float64(v) + } + + if v, ok := configAttributes["weighted_capacity"].(int); ok { + config.WeightedCapacity = aws.Int64(int64(v)) + } + + if v, ok := configAttributes["configurations"].(*schema.Set); ok && v.Len() > 0 { + config.Configurations = expandConfigurations(v.List()) + } + + if v, ok := configAttributes["ebs_config"].(*schema.Set); ok && v.Len() == 1 { + config.EbsConfiguration = expandEbsConfiguration(v.List()) + + if v, ok := configAttributes["ebs_optimized"].(bool); ok { + config.EbsConfiguration.EbsOptimized = aws.Bool(v) + } + } + + configsOut = append(configsOut, config) + } + + return configsOut +} + +func expandConfigurations(configurations []interface{}) []*emr.Configuration { + configsOut := []*emr.Configuration{} + + for _, raw := range configurations { + configAttributes := raw.(map[string]interface{}) + + config := &emr.Configuration{} + + if v, ok := configAttributes["classification"].(string); ok { + config.Classification = aws.String(v) + } + + if rawConfig, ok := configAttributes["configurations"]; ok { + config.Configurations = expandConfigurations(rawConfig.([]interface{})) + } + + if v, ok := configAttributes["properties"]; ok { + properties := make(map[string]string) + for k, v := range v.(map[string]interface{}) { + properties[k] = v.(string) + } + config.Properties = aws.StringMap(properties) + } + + configsOut = append(configsOut, config) + } + + return configsOut +} + +func expandLaunchSpecification(launchSpecification interface{}) *emr.InstanceFleetProvisioningSpecifications { + configAttributes := launchSpecification.(map[string]interface{}) + + return &emr.InstanceFleetProvisioningSpecifications{ + SpotSpecification: expandSpotSpecification(configAttributes["spot_specification"].(*schema.Set).List()[0]), + } +} + +func expandSpotSpecification(spotSpecification interface{}) *emr.SpotProvisioningSpecification { + configAttributes := spotSpecification.(map[string]interface{}) + + spotProvisioning := &emr.SpotProvisioningSpecification{ + TimeoutAction: aws.String(configAttributes["timeout_action"].(string)), + TimeoutDurationMinutes: aws.Int64(int64(configAttributes["timeout_duration_minutes"].(int))), + } + + if v, ok := configAttributes["block_duration_minutes"]; ok && v != 0 { + spotProvisioning.BlockDurationMinutes = aws.Int64(int64(v.(int))) + } + + return spotProvisioning +} + +func resourceAwsEMRInstanceFleetCreate(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).emrconn + + clusterId := d.Get("cluster_id").(string) + instanceFleetConfig := expandInstanceFleetConfig(d) + + addInstanceFleetInput := &emr.AddInstanceFleetInput{ + ClusterId: aws.String(clusterId), + InstanceFleet: instanceFleetConfig, + } + + log.Printf("[DEBUG] Creating EMR instance fleet params: %s", addInstanceFleetInput) + resp, err := conn.AddInstanceFleet(addInstanceFleetInput) + if err != nil { + return err + } + + log.Printf("[DEBUG] Created EMR instance fleet finished: %#v", resp) + if resp == nil { + return fmt.Errorf("error creating instance fleet: no instance fleet returned") + } + d.SetId(*resp.InstanceFleetId) + + return nil +} + +func resourceAwsEMRInstanceFleetRead(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).emrconn + fleet, err := fetchEMRInstanceFleet(conn, d.Get("cluster_id").(string), d.Id()) + if err != nil { + switch err { + case emrInstanceFleetNotFound: + log.Printf("[DEBUG] EMR Instance Fleet (%s) not found, removing", d.Id()) + d.SetId("") + return nil + default: + return err + } + } + + // Guard against the chance of fetchEMRInstanceFleet returning nil fleet but + // not a emrInstanceFleetNotFound error + if fleet == nil { + log.Printf("[DEBUG] EMR Instance Fleet (%s) not found, removing", d.Id()) + d.SetId("") + return nil + } + + d.Set("name", fleet.Name) + d.Set("provisioned_on_demand_capacity", fleet.ProvisionedOnDemandCapacity) + d.Set("provisioned_spot_capacity", fleet.ProvisionedSpotCapacity) + if fleet.Status != nil && fleet.Status.State != nil { + d.Set("status", fleet.Status.State) + } + + return nil +} + +func fetchAllEMRInstanceFleets(conn *emr.EMR, clusterId string) ([]*emr.InstanceFleet, error) { + listInstanceFleetsInput := &emr.ListInstanceFleetsInput{ + ClusterId: aws.String(clusterId), + } + + var fleets []*emr.InstanceFleet + marker := aws.String("initial") + for marker != nil { + log.Printf("[DEBUG] EMR Cluster Instance Marker: %s", *marker) + respFleets, errFleets := conn.ListInstanceFleets(listInstanceFleetsInput) + if errFleets != nil { + return nil, fmt.Errorf("[ERR] Error reading EMR cluster (%s): %s", clusterId, errFleets) + } + if respFleets == nil { + return nil, fmt.Errorf("[ERR] Error reading EMR Instance Fleet for cluster (%s)", clusterId) + } + + if respFleets.InstanceFleets != nil { + fleets = append(fleets, respFleets.InstanceFleets...) + } else { + log.Printf("[DEBUG] EMR Instance Fleet list was empty") + } + marker = respFleets.Marker + } + + if len(fleets) == 0 { + return nil, fmt.Errorf("[WARN] No instance fleets found for EMR Cluster (%s)", clusterId) + } + + return fleets, nil +} + +func fetchEMRInstanceFleet(conn *emr.EMR, clusterId, fleetId string) (*emr.InstanceFleet, error) { + fleets, err := fetchAllEMRInstanceFleets(conn, clusterId) + if err != nil { + return nil, err + } + + var instanceFleet *emr.InstanceFleet + for _, fleet := range fleets { + if fleetId == *fleet.Id { + instanceFleet = fleet + break + } + } + + if instanceFleet != nil { + return instanceFleet, nil + } + + return nil, emrInstanceFleetNotFound +} + +func resourceAwsEMRInstanceFleetUpdate(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).emrconn + + log.Printf("[DEBUG] Modify EMR task fleet") + clusterId := d.Get("cluster_id").(string) + targetOnDemandCapacity := d.Get("target_on_demand_capacity").(int) + targetSpotCapacity := d.Get("target_spot_capacity").(int) + + modifyInstanceFleetInput := &emr.ModifyInstanceFleetInput{ + ClusterId: aws.String(clusterId), + InstanceFleet: &emr.InstanceFleetModifyConfig{ + InstanceFleetId: aws.String(d.Id()), + TargetOnDemandCapacity: aws.Int64(int64(targetOnDemandCapacity)), + TargetSpotCapacity: aws.Int64(int64(targetSpotCapacity)), + }, + } + + _, err := conn.ModifyInstanceFleet(modifyInstanceFleetInput) + if err != nil { + return err + } + + stateConf := &resource.StateChangeConf{ + Pending: []string{emr.InstanceFleetStateProvisioning, emr.InstanceFleetStateBootstrapping, emr.InstanceFleetStateResizing}, + Target: []string{emr.InstanceFleetStateRunning}, + Refresh: instanceFleetStateRefresh(conn, d.Get("cluster_id").(string), d.Id()), + Timeout: 10 * time.Minute, + Delay: 10 * time.Second, + MinTimeout: 3 * time.Second, + } + + _, err = stateConf.WaitForState() + if err != nil { + return fmt.Errorf("error waiting for instance (%s) to terminate: %s", d.Id(), err) + } + + return resourceAwsEMRInstanceFleetRead(d, meta) +} + +func instanceFleetStateRefresh(conn *emr.EMR, clusterID, ifID string) resource.StateRefreshFunc { + return func() (interface{}, string, error) { + fleet, err := fetchEMRInstanceFleet(conn, clusterID, ifID) + if err != nil { + return nil, "Not Found", err + } + + if fleet.Status == nil || fleet.Status.State == nil { + log.Printf("[WARN] ERM Instance Fleet found, but without state") + return nil, "Undefined", fmt.Errorf("undefined EMR Cluster Instance Fleet state") + } + + return fleet, *fleet.Status.State, nil + } +} + +func resourceAwsEMRInstanceFleetDelete(d *schema.ResourceData, meta interface{}) error { + log.Printf("[WARN] AWS EMR Instance Fleet does not support DELETE; resizing cluster to zero before removing from state") + conn := meta.(*AWSClient).emrconn + + clusterId := d.Get("cluster_id").(string) + + modifyInstanceFleetInput := &emr.ModifyInstanceFleetInput{ + ClusterId: aws.String(clusterId), + InstanceFleet: &emr.InstanceFleetModifyConfig{ + InstanceFleetId: aws.String(d.Id()), + TargetOnDemandCapacity: aws.Int64(0), + TargetSpotCapacity: aws.Int64(0), + }, + } + + _, err := conn.ModifyInstanceFleet(modifyInstanceFleetInput) + if err != nil { + return err + } + + return nil +} diff --git a/aws/resource_aws_emr_instance_fleet_test.go b/aws/resource_aws_emr_instance_fleet_test.go new file mode 100644 index 00000000000..f447a077d9a --- /dev/null +++ b/aws/resource_aws_emr_instance_fleet_test.go @@ -0,0 +1,588 @@ +package aws + +import ( + "fmt" + "log" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/emr" + "github.com/hashicorp/terraform/helper/acctest" + "github.com/hashicorp/terraform/helper/resource" + "github.com/hashicorp/terraform/terraform" +) + +func TestAccAWSEMRInstanceFleet_basic(t *testing.T) { + var fleet emr.InstanceFleet + rInt := acctest.RandInt() + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrInstanceFleetDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrInstanceFleetConfig(rInt), + Check: resource.ComposeTestCheckFunc(testAccCheckAWSEmrInstanceFleetExists("aws_emr_instance_fleet.task", &fleet), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "instance_fleet_type", "TASK"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "instance_type_configs.#", "1"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "target_on_demand_capacity", "0"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "target_spot_capacity", "1"), + ), + }, + }, + }) +} + +func TestAccAWSEMRInstanceFleet_zero_count(t *testing.T) { + var fleet emr.InstanceFleet + rInt := acctest.RandInt() + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrInstanceFleetDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrInstanceFleetConfig(rInt), + Check: resource.ComposeTestCheckFunc(testAccCheckAWSEmrInstanceFleetExists("aws_emr_instance_fleet.task", &fleet), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "instance_fleet_type", "TASK"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "instance_type_configs.#", "1"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "target_on_demand_capacity", "0"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "target_spot_capacity", "1"), + ), + }, + { + Config: testAccAWSEmrInstanceFleetConfigZeroCount(rInt), + Check: resource.ComposeTestCheckFunc(testAccCheckAWSEmrInstanceFleetExists("aws_emr_instance_fleet.task", &fleet), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "instance_fleet_type", "TASK"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "instance_type_configs.#", "1"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "target_on_demand_capacity", "0"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "target_spot_capacity", "0"), + ), + }, + }, + }) +} + +func TestAccAWSEMRInstanceFleet_ebsBasic(t *testing.T) { + var fleet emr.InstanceFleet + rInt := acctest.RandInt() + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrInstanceFleetDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrInstanceFleetConfigEbsBasic(rInt), + Check: resource.ComposeTestCheckFunc(testAccCheckAWSEmrInstanceFleetExists("aws_emr_instance_fleet.task", &fleet), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "instance_fleet_type", "TASK"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "instance_type_configs.#", "1"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "target_on_demand_capacity", "0"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "target_spot_capacity", "1"), + ), + }, + }, + }) +} + +func TestAccAWSEMRInstanceFleet_full(t *testing.T) { + var fleet emr.InstanceFleet + rInt := acctest.RandInt() + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrInstanceFleetDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrInstanceFleetConfigFull(rInt), + Check: resource.ComposeTestCheckFunc(testAccCheckAWSEmrInstanceFleetExists("aws_emr_instance_fleet.task", &fleet), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "instance_fleet_type", "TASK"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "instance_type_configs.#", "1"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "target_on_demand_capacity", "1"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "target_spot_capacity", "1"), + ), + }, + }, + }) +} + +func testAccCheckAWSEmrInstanceFleetDestroy(s *terraform.State) error { + conn := testAccProvider.Meta().(*AWSClient).emrconn + + for _, rs := range s.RootModule().Resources { + if rs.Type != "aws_emr_cluster" { + continue + } + + params := &emr.DescribeClusterInput{ + ClusterId: aws.String(rs.Primary.ID), + } + + describe, err := conn.DescribeCluster(params) + + if err == nil { + if describe.Cluster != nil && + *describe.Cluster.Status.State == "WAITING" { + return fmt.Errorf("EMR Cluster still exists") + } + } + + providerErr, ok := err.(awserr.Error) + if !ok { + return err + } + + log.Printf("[ERROR] %v", providerErr) + } + + return nil +} + +func testAccCheckAWSEmrInstanceFleetExists(n string, v *emr.InstanceFleet) resource.TestCheckFunc { + return func(s *terraform.State) error { + rs, ok := s.RootModule().Resources[n] + if !ok { + return fmt.Errorf("Not found: %s", n) + } + if rs.Primary.ID == "" { + return fmt.Errorf("No task fleet id set") + } + meta := testAccProvider.Meta() + conn := meta.(*AWSClient).emrconn + f, err := fetchEMRInstanceFleet(conn, rs.Primary.Attributes["cluster_id"], rs.Primary.ID) + if err != nil { + return fmt.Errorf("EMR error: %v", err) + } + + if f == nil { + return fmt.Errorf("No match found for (%s)", n) + } + + v = f + return nil + } +} + +const testAccAWSEmrInstanceFleetBase = ` +provider "aws" { + region = "us-west-2" +} + +resource "aws_emr_cluster" "tf-test-cluster" { + name = "emr-test-%d" + release_label = "emr-5.10.0" + applications = ["Spark"] + + ec2_attributes { + subnet_id = "${aws_subnet.main.id}" + emr_managed_master_security_group = "${aws_security_group.allow_all.id}" + emr_managed_slave_security_group = "${aws_security_group.allow_all.id}" + instance_profile = "${aws_iam_instance_profile.emr_profile.arn}" + } + + instance_fleet = [ + { + instance_fleet_type = "MASTER" + instance_type_configs = [ + { + instance_type = "m3.xlarge" + } + ] + target_on_demand_capacity = 1 + }, + { + instance_fleet_type = "CORE" + instance_type_configs = [ + { + bid_price_as_percentage_of_on_demand_price = 80 + ebs_optimized = true + ebs_config = [ + { + size = 100 + type = "gp2" + volumes_per_instance = 1 + } + ] + instance_type = "m3.xlarge" + weighted_capacity = 1 + } + ] + launch_specifications { + spot_specification { + block_duration_minutes = 60 + timeout_action = "SWITCH_TO_ON_DEMAND" + timeout_duration_minutes = 10 + } + } + name = "instance-fleet-test" + target_on_demand_capacity = 0 + target_spot_capacity = 1 + } + ] + + tags { + role = "rolename" + dns_zone = "env_zone" + env = "env" + name = "name-env" + } + + bootstrap_action { + path = "s3://elasticmapreduce/bootstrap-actions/run-if" + name = "runif" + args = ["instance.isMaster=true", "echo running on master node"] + } + + configurations = "test-fixtures/emr_configurations.json" + service_role = "${aws_iam_role.iam_emr_default_role.arn}" + + depends_on = ["aws_internet_gateway.gw"] +} + +resource "aws_security_group" "allow_all" { + name = "allow_all" + description = "Allow all inbound traffic" + vpc_id = "${aws_vpc.main.id}" + + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + depends_on = ["aws_subnet.main"] + + lifecycle { + ignore_changes = ["ingress", "egress"] + } +} + +resource "aws_vpc" "main" { + cidr_block = "168.31.0.0/16" + enable_dns_hostnames = true + + tags { + Name = "tf_acc_emr_tests" + } +} + +resource "aws_subnet" "main" { + vpc_id = "${aws_vpc.main.id}" + cidr_block = "168.31.0.0/20" + + # map_public_ip_on_launch = true +} + +resource "aws_internet_gateway" "gw" { + vpc_id = "${aws_vpc.main.id}" +} + +resource "aws_route_table" "r" { + vpc_id = "${aws_vpc.main.id}" + + route { + cidr_block = "0.0.0.0/0" + gateway_id = "${aws_internet_gateway.gw.id}" + } +} + +resource "aws_main_route_table_association" "a" { + vpc_id = "${aws_vpc.main.id}" + route_table_id = "${aws_route_table.r.id}" +} + +### + +# IAM role for EMR Service +resource "aws_iam_role" "iam_emr_default_role" { + name = "iam_emr_default_role_%d" + + assume_role_policy = < 256 { diff --git a/website/aws.erb b/website/aws.erb index f532d44d6c8..2ebaa2c9756 100644 --- a/website/aws.erb +++ b/website/aws.erb @@ -1351,6 +1351,10 @@ aws_emr_instance_group + > + aws_emr_instance_fleet + + > aws_emr_security_configuration diff --git a/website/docs/r/emr_cluster.html.markdown b/website/docs/r/emr_cluster.html.markdown index b5138ef3b46..1437cf2c3a8 100644 --- a/website/docs/r/emr_cluster.html.markdown +++ b/website/docs/r/emr_cluster.html.markdown @@ -172,6 +172,7 @@ resource "aws_emr_cluster" "example" { The following arguments are supported: * `name` - (Required) The name of the job flow +* `hadoop_version` - (Optional) The Hadoop version for the cluster. * `release_label` - (Required) The release label for the Amazon EMR release * `master_instance_type` - (Optional) The EC2 instance type of the master node. Exactly one of `master_instance_type` and `instance_group` must be specified. * `scale_down_behavior` - (Optional) The way that individual Amazon EC2 instances terminate when an automatic scale-in activity occurs or an `instance group` is resized. @@ -232,6 +233,8 @@ Attributes for the Amazon EC2 instances running the job flow * `emr_managed_slave_security_group` - (Optional) Identifier of the Amazon EC2 EMR-Managed security group for the slave nodes * `service_access_security_group` - (Optional) Identifier of the Amazon EC2 service-access security group - required when the cluster runs on a private subnet * `instance_profile` - (Required) Instance Profile for EC2 instances of the cluster assume this role +* `availability_zone` - (Optional) The Amazon EC2 Availability Zone for the cluster. `availability_zone` is used for uniform instance groups, while `availability_zones` (plural) is used for instance fleets. +* `availability_zones` - (Optional) When multiple Availability Zones are specified, Amazon EMR evaluates them and launches instances in the optimal Availability Zone. `availability_zones` is used for instance fleets, while `availability_zone` (singular) is used for uniform instance groups. ~> **NOTE on EMR-Managed security groups:** These security groups will have any missing inbound or outbound access rules added and maintained by AWS, to ensure @@ -267,9 +270,22 @@ Attributes for each task instance group in the cluster * `instance_count` - (Optional) Target number of instances for the instance group * `name` - (Optional) Friendly name given to the instance group * `bid_price` - (Optional) If set, the bid price for each EC2 instance in the instance group, expressed in USD. By setting this attribute, the instance group is being declared as a Spot Instance, and will implicitly create a Spot request. Leave this blank to use On-Demand Instances. `bid_price` can not be set for the `MASTER` instance group, since that group must always be On-Demand +* `ebs_optimized` (Optional) Indicates whether an Amazon EBS volume is EBS-optimized. Changing this forces a new resource to be created. * `ebs_config` - (Optional) A list of attributes for the EBS volumes attached to each instance in the instance group. Each `ebs_config` defined will result in additional EBS volumes being attached to _each_ instance in the instance group. Defined below * `autoscaling_policy` - (Optional) The autoscaling policy document. This is a JSON formatted string. See [EMR Auto Scaling](https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-automatic-scaling.html) + +## instance_fleet + +Attributes for each instance fleet in the cluster (See aws_emr_instance_fleet for more details) + +* `instance_fleet_type` - (Required) The node type that the instance fleet hosts. Valid values are `MASTER`, `CORE`, and `TASK`. Changing this forces a new resource to be created. +* `instance_type_configs` - (Optional) The instance type configurations that define the EC2 instances in the instance fleet. Array of `instance_type_config` blocks. +* `launch_specifications` - (Optional) The launch specification for the instance fleet. +* `name` - (Optional) The friendly name of the instance fleet. +* `target_on_demand_capacity` - (Optional) The target capacity of On-Demand units for the instance fleet, which determines how many On-Demand instances to provision. +* `target_spot_capacity` - (Optional) The target capacity of Spot units for the instance fleet, which determines how many Spot instances to provision. + ## ebs_config Attributes for the EBS volumes attached to each EC2 instance in the `instance_group` diff --git a/website/docs/r/emr_instance_fleet.html.md b/website/docs/r/emr_instance_fleet.html.md new file mode 100644 index 00000000000..b6dbbfde2ee --- /dev/null +++ b/website/docs/r/emr_instance_fleet.html.md @@ -0,0 +1,157 @@ +--- +layout: "aws" +page_title: "AWS: aws_emr_instance_fleet" +sidebar_current: "docs-aws-resource-emr-instance-fleet" +description: |- + Provides an Elastic MapReduce Cluster Instance Fleet +--- + +# aws_emr_instance_fleet + +Provides an Elastic MapReduce Cluster Instance Fleet configuration. +See [Amazon Elastic MapReduce Documentation](https://aws.amazon.com/documentation/emr/) for more information. + +~> **NOTE:** At this time, Instance Fleets cannot be destroyed through the API nor +web interface. Instance Fleets are destroyed when the EMR Cluster is destroyed. +Terraform will resize any Instance Fleet to zero when destroying the resource. + +## Example Usage + +```hcl +resource "aws_emr_instance_fleet" "task" { + cluster_id = "${aws_emr_cluster.tf-test-cluster.id}" + instance_fleet_type = "TASK" + instance_type_configs [ + { + bid_price_as_percentage_of_on_demand_price = 100 + configurations = [] + ebs_optimized = true + ebs_config = [ + { + iops = 300 + size = 10 + type = "gp2" + volumes_per_instance = 1 + } + ] + + instance_type = "m3.xlarge" + "weighted_capacity" = 8 + } + ], + launch_specifications { + spot_specification { + block_duration_minutes = 60 + timeout_action = "TERMINATE_CLUSTER" + timeout_duration_minutes = 10 + } + }, + name = "my little instance fleet" + target_on_demand_capacity = 1 + target_spot_capacity = 1 +} +``` + +## Argument Reference + +The following arguments are supported: + +* `cluster_id` - (Required) ID of the EMR Cluster to attach to. Changing this forces a new resource to be created. + +* `instance_fleet_type` - (Required) The node type that the instance fleet hosts. Valid values are `MASTER`, `CORE`, and `TASK`. Changing this forces a new resource to be created. + +* `instance_type_configs` - (Optional) The instance type configurations that define the EC2 instances in the instance fleet. List of `instance_type_config` blocks. + +* `launch_specifications` - (Optional) The launch specification for the instance fleet. + + * `spot_specification` - (Required) The launch specification for Spot instances in the fleet, which determines the + defined duration and provisioning timeout behavior. + +* `name` - (Optional) The friendly name of the instance fleet. + +* `target_on_demand_capacity` - (Optional) The target capacity of On-Demand units for the instance fleet, which determines how many On-Demand instances to provision. + +* `target_spot_capacity` - (Optional) The target capacity of Spot units for the instance fleet, which determines how many Spot instances to provision. + + + +`instance_type_config` supports the following: + +* `bid_price` - (Optional) The bid price for each EC2 Spot instance type as defined by `instance_type`. +Expressed in USD. If neither `bid_price` nor `bid_price_as_percentage_of_on_demand_price` is provided, +`bid_price_as_percentage_of_on_demand_price` defaults to 100%. + +* `bid_price_as_percentage_of_on_demand_price` - (Optional) The bid price, as a percentage of On-Demand price, +for each EC2 Spot instance as defined by `instance_type`. Expressed as a number (for example, 20 specifies 20%). +If neither `bid_price` nor `bid_price_as_percentage_of_on_demand_price` is provided, +`bid_price_as_percentage_of_on_demand_price` defaults to 100%. + +* `configurations` - (Optional) A configuration classification that applies when provisioning cluster instances, +which can include configurations for applications and software that run on the cluster. List of `configuration` blocks. + +* `ebs_optimized` - (Optional) Indicates whether an Amazon EBS volume is EBS-optimized. + +* `ebs_config` - (Optional) The configuration of Amazon Elastic Block Storage (EBS) attached to each instance as +defined by `instance_type`. + +* `instance_type` - (Required) An EC2 instance type, such as m3.xlarge. + +* `weighted_capacity` - (Optional) The number of units that a provisioned instance of this type provides toward +fulfilling the target capacities defined in `aws_emr_instance_fleet`. This value is 1 for a master instance fleet, +and must be 1 or greater for core and task instance fleets. Defaults to 1 if not specified. + + + +`spot_specification` supports the following: + +* `block_duration_minutes` - (Optional) The defined duration for Spot instances (also known as Spot blocks) in minutes. +When specified, the Spot instance does not terminate before the defined duration expires, and defined duration pricing +for Spot instances applies. Valid values are 60, 120, 180, 240, 300, or 360. The duration period starts as soon as a +Spot instance receives its instance ID. At the end of the duration, Amazon EC2 marks the Spot instance for termination +and provides a Spot instance termination notice, which gives the instance a two-minute warning before it terminates. + +* `timeout_action` - (Required) The action to take when `target_spot_capacity` has not been fulfilled when the +`timeout_duration_minutes` has expired. Spot instances are not uprovisioned within the Spot provisioining timeout. +Valid values are `TERMINATE_CLUSTER` and `SWITCH_TO_ON_DEMAND`. `SWITCH_TO_ON_DEMAND` specifies that if no Spot +instances are available, On-Demand Instances should be provisioned to fulfill any remaining Spot capacity. + +* `timeout_duration_minutes` - (Required) The spot provisioning timeout period in minutes. If Spot instances are not +provisioned within this time period, the `timeout_action` is taken. Minimum value is 5 and maximum value is 1440. +The timeout applies only during initial provisioning, when the cluster is first created. + + + +`configuration` supports the following: + +* `classification` - (Optional) The classification within a configuration. + +* `configurations` - (Optional) A list of additional configurations to apply within a configuration object. + +* `properties` - (Optional) A set of properties specified within a configuration classification. + + + +`ebs_config` EBS volume specifications such as volume type, IOPS, and size (GiB) that will be requested for the EBS volume attached to an EC2 instance in the cluster. + +* `iops` - (Optional) The number of I/O operations per second (IOPS) that the volume supports. + +* `size_in_gb` - (Required) The volume size, in gibibytes (GiB). This can be a number from 1 - 1024. If the volume type is EBS-optimized, the minimum value is 10. + +* `volume_type` - (Required) The volume type. Volume types supported are `gp2`, `io1`, `standard`. + +* `volumes_per_instance` - (Optional) Number of EBS volumes with a specific volume configuration that will be associated with every instance in the instance group + + +## Attributes Reference + +The following attributes are exported: + +* `id` - The unique identifier of the instance fleet. + +* `provisioned_on_demand_capacity` The number of On-Demand units that have been provisioned for the instance +fleet to fulfill TargetOnDemandCapacity. This provisioned capacity might be less than or greater than TargetOnDemandCapacity. + +* `provisioned_spot_capacity` The number of Spot units that have been provisioned for this instance fleet +to fulfill TargetSpotCapacity. This provisioned capacity might be less than or greater than TargetSpotCapacity. + +* `status` The current status of the instance fleet.