Merge remote-tracking branch 'oss/master' into lazy-load-expiration

* oss/master:
  Add 'discard' target to file audit backend (#3262)
  changelog++
  auth/aws: Allow wildcard in bound_iam_principal_id (#3213)
  changelog++
  Add option to set cluster TLS cipher suites. (#3228)

CHANGELOG.md

@@ -13,12 +13,16 @@ FEATURES:
   verification through the SSH CA backend, if enabled.
 
 IMPROVEMENTS:
+
  * audit/file: Allow specifying `stdout` as the `file_path` to log to standard
    output [GH-3235]
+ * auth/aws: Allow wildcards in `bound_iam_principal_id` [GH-3213]
  * auth/okta: Compare groups case-insensitively since Okta is only
    case-preserving [GH-3240]
  * cli: Add subcommand autocompletion that can be enabled with 
    `vault -autocomplete-install` [GH-3223]
+ * core: TLS cipher suites used for cluster behavior can now be set via
+   `cluster_cipher_suites` in configuration [GH-3228]
  * storage/gcp: Use application default credentials if they exist [GH-3248]
 
 BUG FIXES:

builtin/audit/file/backend.go

@@ -2,6 +2,7 @@ package file
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -33,6 +34,9 @@ func Factory(conf *audit.BackendConfig) (audit.Backend, error) {
 	if strings.ToLower(path) == "stdout" {
 		path = "stdout"
 	}
+	if strings.ToLower(path) == "discard" {
+		path = "discard"
+	}
 
 	format, ok := conf.Config["format"]
 	if !ok {
@@ -99,8 +103,8 @@ func Factory(conf *audit.BackendConfig) (audit.Backend, error) {
 	}
 
 	switch path {
-	case "stdout":
-		// no need to test opening file if outputting to stdout
+	case "stdout", "discard":
+		// no need to test opening file if outputting to stdout or discarding
 	default:
 		// Ensure that the file can be successfully opened for writing;
 		// otherwise it will be too late to catch later without problems
@@ -166,8 +170,11 @@ func (b *Backend) LogRequest(auth *logical.Auth, req *logical.Request, outerErr
 	b.fileLock.Lock()
 	defer b.fileLock.Unlock()
 
-	if b.path == "stdout" {
+	switch b.path {
+	case "stdout":
 		return b.formatter.FormatRequest(os.Stdout, b.formatConfig, auth, req, outerErr)
+	case "discard":
+		return b.formatter.FormatRequest(ioutil.Discard, b.formatConfig, auth, req, outerErr)
 	}
 
 	if err := b.open(); err != nil {
@@ -198,8 +205,11 @@ func (b *Backend) LogResponse(
 	b.fileLock.Lock()
 	defer b.fileLock.Unlock()
 
-	if b.path == "stdout" {
+	switch b.path {
+	case "stdout":
 		return b.formatter.FormatResponse(os.Stdout, b.formatConfig, auth, req, resp, err)
+	case "discard":
+		return b.formatter.FormatResponse(ioutil.Discard, b.formatConfig, auth, req, resp, err)
 	}
 
 	if err := b.open(); err != nil {
@@ -251,7 +261,8 @@ func (b *Backend) open() error {
 }
 
 func (b *Backend) Reload() error {
-	if b.path == "stdout" {
+	switch b.path {
+	case "stdout", "discard":
 		return nil
 	}
 

builtin/credential/aws/backend.go

@@ -10,6 +10,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/iam"
 	"github.com/hashicorp/vault/logical"
 	"github.com/hashicorp/vault/logical/framework"
+	"github.com/patrickmn/go-cache"
 )
 
 func Factory(conf *logical.BackendConfig) (logical.Backend, error) {
@@ -60,6 +61,11 @@ type backend struct {
 	// will be flushed. The empty STS role signifies the master account
 	IAMClientsMap map[string]map[string]*iam.IAM
 
+	// Map of AWS unique IDs to the full ARN corresponding to that unique ID
+	// This avoids the overhead of an AWS API hit for every login request
+	// using the IAM auth method when bound_iam_principal_arn contains a wildcard
+	iamUserIdToArnCache *cache.Cache
+
 	// AWS Account ID of the "default" AWS credentials
 	// This cache avoids the need to call GetCallerIdentity repeatedly to learn it
 	// We can't store this because, in certain pathological cases, it could change
@@ -74,9 +80,10 @@ func Backend(conf *logical.BackendConfig) (*backend, error) {
 	b := &backend{
 		// Setting the periodic func to be run once in an hour.
 		// If there is a real need, this can be made configurable.
-		tidyCooldownPeriod: time.Hour,
-		EC2ClientsMap:      make(map[string]map[string]*ec2.EC2),
-		IAMClientsMap:      make(map[string]map[string]*iam.IAM),
+		tidyCooldownPeriod:  time.Hour,
+		EC2ClientsMap:       make(map[string]map[string]*ec2.EC2),
+		IAMClientsMap:       make(map[string]map[string]*iam.IAM),
+		iamUserIdToArnCache: cache.New(7*24*time.Hour, 24*time.Hour),
 	}
 
 	b.resolveArnToUniqueIDFunc = b.resolveArnToRealUniqueId

builtin/credential/aws/backend_test.go

@@ -1522,22 +1522,14 @@ func TestBackendAcc_LoginWithCallerIdentity(t *testing.T) {
 		t.Fatalf("bad: expected valid login: resp:%#v", resp)
 	}
 
-	renewReq := &logical.Request{
-		Storage: storage,
-		Auth:    &logical.Auth{},
-	}
+	renewReq := generateRenewRequest(storage, resp.Auth)
+	// dump a fake ARN into the metadata to ensure that we ONLY look
+	// at the unique ID that has been generated
+	renewReq.Auth.Metadata["canonical_arn"] = "fake_arn"
 	empty_login_fd := &framework.FieldData{
 		Raw:    map[string]interface{}{},
 		Schema: pathLogin(b).Fields,
 	}
-	renewReq.Auth.InternalData = resp.Auth.InternalData
-	renewReq.Auth.Metadata = resp.Auth.Metadata
-	renewReq.Auth.LeaseOptions = resp.Auth.LeaseOptions
-	renewReq.Auth.Policies = resp.Auth.Policies
-	renewReq.Auth.IssueTime = time.Now()
-	// dump a fake ARN into the metadata to ensure that we ONLY look
-	// at the unique ID that has been generated
-	renewReq.Auth.Metadata["canonical_arn"] = "fake_arn"
 	// ensure we can renew
 	resp, err = b.pathLoginRenew(renewReq, empty_login_fd)
 	if err != nil {
@@ -1571,5 +1563,57 @@ func TestBackendAcc_LoginWithCallerIdentity(t *testing.T) {
 	if err == nil || (resp != nil && !resp.IsError()) {
 		t.Errorf("bad: expected failed renew due to changed AWS role ID: resp: %#v", resp, err)
 	}
+	// Undo the fake resolver...
+	b.resolveArnToUniqueIDFunc = b.resolveArnToRealUniqueId
+
+	// Now test that wildcard matching works
+	wildcardRoleName := "valid_wildcard"
+	wildcardEntity := *entity
+	wildcardEntity.FriendlyName = "*"
+	roleData["bound_iam_principal_arn"] = wildcardEntity.canonicalArn()
+	roleRequest.Path = "role/" + wildcardRoleName
+	resp, err = b.HandleRequest(roleRequest)
+	if err != nil || (resp != nil && resp.IsError()) {
+		t.Fatalf("bad: failed to create wildcard role: resp:%#v\nerr:%v", resp, err)
+	}
+
+	loginData["role"] = wildcardRoleName
+	resp, err = b.HandleRequest(loginRequest)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp == nil || resp.Auth == nil || resp.IsError() {
+		t.Fatalf("bad: expected valid login: resp:%#v", resp)
+	}
+	// and ensure we can renew
+	renewReq = generateRenewRequest(storage, resp.Auth)
+	resp, err = b.pathLoginRenew(renewReq, empty_login_fd)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp == nil {
+		t.Fatal("got nil response from renew")
+	}
+	if resp.IsError() {
+		t.Fatalf("got error when renewing: %#v", *resp)
+	}
+	// ensure the cache is populated
+	cachedArn := b.getCachedUserId(resp.Auth.Metadata["client_user_id"])
+	if cachedArn == "" {
+		t.Errorf("got empty ARN back from user ID cache; expected full arn")
+	}
+}
+
+func generateRenewRequest(s logical.Storage, auth *logical.Auth) *logical.Request {
+	renewReq := &logical.Request{
+		Storage: s,
+		Auth:    &logical.Auth{},
+	}
+	renewReq.Auth.InternalData = auth.InternalData
+	renewReq.Auth.Metadata = auth.Metadata
+	renewReq.Auth.LeaseOptions = auth.LeaseOptions
+	renewReq.Auth.Policies = auth.Policies
+	renewReq.Auth.IssueTime = time.Now()
 
+	return renewReq
 }

builtin/credential/aws/client.go

@@ -141,6 +141,25 @@ func (b *backend) flushCachedIAMClients() {
 	}
 }
 
+// Gets an entry out of the user ID cache
+func (b *backend) getCachedUserId(userId string) string {
+	if userId == "" {
+		return ""
+	}
+	if entry, ok := b.iamUserIdToArnCache.Get(userId); ok {
+		b.iamUserIdToArnCache.SetDefault(userId, entry)
+		return entry.(string)
+	}
+	return ""
+}
+
+// Sets an entry in the user ID cache
+func (b *backend) setCachedUserId(userId, arn string) {
+	if userId != "" {
+		b.iamUserIdToArnCache.SetDefault(userId, arn)
+	}
+}
+
 func (b *backend) stsRoleForAccount(s logical.Storage, accountID string) (string, error) {
 	// Check if an STS configuration exists for the AWS account
 	sts, err := b.lockedAwsStsEntry(s, accountID)

builtin/credential/aws/path_login.go

@@ -928,6 +928,12 @@ func (b *backend) pathLoginRenewIam(
 		}
 	}
 
+	// Note that the error messages below can leak a little bit of information about the role information
+	// For example, if on renew, the client gets the "error parsing ARN..." error message, the client
+	// will know that it's a wildcard bind (but not the actual bind), even if the client can't actually
+	// read the role directly to know what the bind is. It's a relatively small amount of leakage, in
+	// some fairly corner cases, and in the most likely error case (role has been changed to a new ARN),
+	// the error message is identical.
 	if roleEntry.BoundIamPrincipalARN != "" {
 		// We might not get here if all bindings were on the inferred entity, which we've already validated
 		// above
@@ -936,10 +942,31 @@ func (b *backend) pathLoginRenewIam(
 			// Resolving unique IDs is enabled and the auth metadata contains the unique ID, so checking the
 			// unique ID is authoritative at this stage
 			if roleEntry.BoundIamPrincipalID != clientUserId {
-				return nil, fmt.Errorf("role no longer bound to ID %q", clientUserId)
+				return nil, fmt.Errorf("role no longer bound to ARN %q", canonicalArn)
+			}
+		} else if strings.HasSuffix(roleEntry.BoundIamPrincipalARN, "*") {
+			fullArn := b.getCachedUserId(clientUserId)
+			if fullArn == "" {
+				entity, err := parseIamArn(canonicalArn)
+				if err != nil {
+					return nil, fmt.Errorf("error parsing ARN %q: %v", canonicalArn, err)
+				}
+				fullArn, err = b.fullArn(entity, req.Storage)
+				if err != nil {
+					return nil, fmt.Errorf("error looking up full ARN of entity %v: %v", entity, err)
+				}
+				if fullArn == "" {
+					return nil, fmt.Errorf("got empty string back when looking up full ARN of entity %v", entity)
+				}
+				if clientUserId != "" {
+					b.setCachedUserId(clientUserId, fullArn)
+				}
+			}
+			if !strutil.GlobbedStringsMatch(roleEntry.BoundIamPrincipalARN, fullArn) {
+				return nil, fmt.Errorf("role no longer bound to ARN %q", canonicalArn)
 			}
 		} else if roleEntry.BoundIamPrincipalARN != canonicalArn {
-			return nil, fmt.Errorf("role no longer bound to arn %q", canonicalArn)
+			return nil, fmt.Errorf("role no longer bound to ARN %q", canonicalArn)
 		}
 	}
 
@@ -1129,7 +1156,7 @@ func (b *backend) pathLoginUpdateIam(
 	callerUniqueId := strings.Split(callerID.UserId, ":")[0]
 	entity, err := parseIamArn(callerID.Arn)
 	if err != nil {
-		return logical.ErrorResponse(fmt.Sprintf("Error parsing arn: %v", err)), nil
+		return logical.ErrorResponse(fmt.Sprintf("error parsing arn %q: %v", callerID.Arn, err)), nil
 	}
 
 	roleName := data.Get("role").(string)
@@ -1158,8 +1185,27 @@ func (b *backend) pathLoginUpdateIam(
 		if callerUniqueId != roleEntry.BoundIamPrincipalID {
 			return logical.ErrorResponse(fmt.Sprintf("expected IAM %s %s to resolve to unique AWS ID %q but got %q instead", entity.Type, entity.FriendlyName, roleEntry.BoundIamPrincipalID, callerUniqueId)), nil
 		}
-	} else if roleEntry.BoundIamPrincipalARN != "" && roleEntry.BoundIamPrincipalARN != entity.canonicalArn() {
-		return logical.ErrorResponse(fmt.Sprintf("IAM Principal %q does not belong to the role %q", callerID.Arn, roleName)), nil
+	} else if roleEntry.BoundIamPrincipalARN != "" {
+		if strings.HasSuffix(roleEntry.BoundIamPrincipalARN, "*") {
+			fullArn := b.getCachedUserId(callerUniqueId)
+			if fullArn == "" {
+				fullArn, err = b.fullArn(entity, req.Storage)
+				if err != nil {
+					return logical.ErrorResponse(fmt.Sprintf("error looking up full ARN of entity %v: %v", entity, err)), nil
+				}
+				if fullArn == "" {
+					return logical.ErrorResponse(fmt.Sprintf("got empty string back when looking up full ARN of entity %v", entity)), nil
+				}
+				b.setCachedUserId(callerUniqueId, fullArn)
+			}
+			if !strutil.GlobbedStringsMatch(roleEntry.BoundIamPrincipalARN, fullArn) {
+				// Note: Intentionally giving the exact same error message as a few lines below. Otherwise, we might leak information
+				// about whether the bound IAM principal ARN is a wildcard or not, and what that wildcard is.
+				return logical.ErrorResponse(fmt.Sprintf("IAM Principal %q does not belong to the role %q", callerID.Arn, roleName)), nil
+			}
+		} else if roleEntry.BoundIamPrincipalARN != entity.canonicalArn() {
+			return logical.ErrorResponse(fmt.Sprintf("IAM Principal %q does not belong to the role %q", callerID.Arn, roleName)), nil
+		}
 	}
 
 	policies := roleEntry.Policies
@@ -1508,6 +1554,7 @@ type iamEntity struct {
 	SessionInfo   string
 }
 
+// Returns a Vault-internal canonical ARN for referring to an IAM entity
 func (e *iamEntity) canonicalArn() string {
 	entityType := e.Type
 	// canonicalize "assumed-role" into "role"
@@ -1522,6 +1569,46 @@ func (e *iamEntity) canonicalArn() string {
 	return fmt.Sprintf("arn:%s:iam::%s:%s/%s", e.Partition, e.AccountNumber, entityType, e.FriendlyName)
 }
 
+// This returns the "full" ARN of an iamEntity, how it would be referred to in AWS proper
+func (b *backend) fullArn(e *iamEntity, s logical.Storage) (string, error) {
+	// Not assuming path is reliable for any entity types
+	client, err := b.clientIAM(s, getAnyRegionForAwsPartition(e.Partition).ID(), e.AccountNumber)
+	if err != nil {
+		return "", fmt.Errorf("error creating IAM client: %v", err)
+	}
+
+	switch e.Type {
+	case "user":
+		input := iam.GetUserInput{
+			UserName: aws.String(e.FriendlyName),
+		}
+		resp, err := client.GetUser(&input)
+		if err != nil {
+			return "", fmt.Errorf("error fetching user %q: %v", e.FriendlyName, err)
+		}
+		if resp == nil {
+			return "", fmt.Errorf("nil response from GetUser")
+		}
+		return *(resp.User.Arn), nil
+	case "assumed-role":
+		fallthrough
+	case "role":
+		input := iam.GetRoleInput{
+			RoleName: aws.String(e.FriendlyName),
+		}
+		resp, err := client.GetRole(&input)
+		if err != nil {
+			return "", fmt.Errorf("error fetching role %q: %v", e.FriendlyName, err)
+		}
+		if resp == nil {
+			return "", fmt.Errorf("nil response form GetRole")
+		}
+		return *(resp.Role.Arn), nil
+	default:
+		return "", fmt.Errorf("unrecognized entity type: %s", e.Type)
+	}
+}
+
 const iamServerIdHeader = "X-Vault-AWS-IAM-Server-ID"
 
 const pathLoginSyn = `

builtin/credential/aws/path_role.go

@@ -318,7 +318,8 @@ func (b *backend) upgradeRoleEntry(s logical.Storage, roleEntry *awsRoleEntry) (
 	if roleEntry.AuthType == iamAuthType &&
 		roleEntry.ResolveAWSUniqueIDs &&
 		roleEntry.BoundIamPrincipalARN != "" &&
-		roleEntry.BoundIamPrincipalID == "" {
+		roleEntry.BoundIamPrincipalID == "" &&
+		!strings.HasSuffix(roleEntry.BoundIamPrincipalARN, "*") {
 		principalId, err := b.resolveArnToUniqueIDFunc(s, roleEntry.BoundIamPrincipalARN)
 		if err != nil {
 			return false, err
@@ -493,14 +494,17 @@ func (b *backend) pathRoleCreateUpdate(
 		// This allows the user to sumbit an update with the same ARN to force Vault
 		// to re-resolve the ARN to the unique ID, in case an entity was deleted and
 		// recreated
-		if roleEntry.ResolveAWSUniqueIDs {
+		if roleEntry.ResolveAWSUniqueIDs && !strings.HasSuffix(roleEntry.BoundIamPrincipalARN, "*") {
 			principalID, err := b.resolveArnToUniqueIDFunc(req.Storage, principalARN)
 			if err != nil {
 				return logical.ErrorResponse(fmt.Sprintf("failed updating the unique ID of ARN %#v: %#v", principalARN, err)), nil
 			}
 			roleEntry.BoundIamPrincipalID = principalID
+		} else {
+			// Need to handle the case where we're switching from a non-wildcard principal to a wildcard principal
+			roleEntry.BoundIamPrincipalID = ""
 		}
-	} else if roleEntry.ResolveAWSUniqueIDs && roleEntry.BoundIamPrincipalARN != "" {
+	} else if roleEntry.ResolveAWSUniqueIDs && roleEntry.BoundIamPrincipalARN != "" && !strings.HasSuffix(roleEntry.BoundIamPrincipalARN, "*") {
 		// we're turning on resolution on this role, so ensure we update it
 		principalID, err := b.resolveArnToUniqueIDFunc(req.Storage, roleEntry.BoundIamPrincipalARN)
 		if err != nil {