testing: provide test coverage for backend selection

Add a scenario where we mimic a full setup and where we verify which backend is reached.

.github/workflows/go.yml

@@ -56,6 +56,9 @@ jobs:
     - name: Build
       run: go build -v ./...
 
+    - name: Setup test dependencies
+      run: make setup-test-dependencies && make start-test-s3-servers
+
     - name: Test
       # As we use config files from time to time we always want to run without cache
       run: go clean -testcache && go test -v ./...

.gitignore

@@ -1,4 +1,8 @@
 **__debug_bin*
 **/.idea
 
-**/*.private
+**/*.private
+
+testing/venv/**
+testing/**.err
+testing/**.log

Makefile

@@ -6,3 +6,16 @@ run-container-s3:
 
 run-container-sts:
 	podman run --rm -v ./etc.private:/etc/fakes3pp:Z -p 8444:8444 --env HOME=${HOME} -it localhost/fakes3pp:latest proxysts --dot-env /etc/fakes3pp/.env.docker
+
+setup-test-dependencies:
+	[ ! -f testing/venv/moto ] && python3 -m venv testing/venv/moto
+	./testing/venv/moto/bin/pip3 install -r testing/requirements.txt
+
+start-test-s3-servers:
+	./testing/venv/moto/bin/moto_server -p 5000 >testing/server_5000.log 2>testing/server_5000.err &
+	./testing/venv/moto/bin/moto_server -p 5001 >testing/server_5001.log 2>testing/server_5001.err &
+	./testing/venv/moto/bin/python3 testing/bootstrap_backend.py testing/backends/tst-1 http://localhost:5000
+	./testing/venv/moto/bin/python3 testing/bootstrap_backend.py testing/backends/eu-test-2 http://localhost:5001
+
+stop-test-s3-servers:
+	for pid in `ps -ef | grep testing/venv/moto/bin/python3 | grep -v grep | awk '{print $$2}'`; do kill "$${pid}"; done

cmd/almost-e2e_test.go

@@ -0,0 +1,146 @@
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+)
+
+
+const testRegion1 = "tst-1"
+const testRegion2 = "eu-test-2"
+var backendTestRegions = []string{testRegion1, testRegion2}
+
+var testingBackendsConfig = []byte(fmt.Sprintf(`
+# This is a test file check backend-config.yaml if you want to create a configuration
+s3backends:
+  - region: %s
+    credentials:
+      file: ../etc/creds/cfc_creds.yaml
+    endpoint: http://localhost:5000
+  - region: %s
+    credentials:
+      file: ../etc/creds/otc_creds.yaml
+    endpoint: http://localhost:5001
+default:  %s
+`, testRegion1, testRegion2, testRegion2))
+
+
+//Set the configurations as expected for the testingbackends
+//See testing/README.md for details on testing setup
+func setTestingBackendsConfig(t *testing.T) {
+	cfg, err :=  getBackendsConfigFromBytes(testingBackendsConfig)
+  if err != nil {
+    t.Error(err)
+    t.FailNow()
+  }
+  globalBackendsConfig = cfg
+}
+
+//This is the testing fixture. It starts an sts and s3 proxy which
+//are configured with the S3 backends detailed in testing/README.md.
+func testingFixture(t *testing.T) (tearDown func ()(), getToken func(subject string, d time.Duration, tags AWSSessionTags) string){
+  //Configure backends to be the testing S3 backends
+  setTestingBackendsConfig(t)
+	//Given valid server config
+  teardownSuiteSTS := setupSuiteProxySTS(t)
+  teardownSuiteS3 := setupSuiteProxyS3(t, justProxied)
+
+  //function to stop the setup of the fixture
+  tearDownProxies := func () {
+    teardownSuiteSTS(t)
+    teardownSuiteS3(t)
+  }
+
+  _, err := loadOidcConfig([]byte(testConfigFakeTesting))
+	if err != nil {
+		t.Error(err)
+	}
+
+	signingKey, err := getTestSigningKey()
+	if err != nil {
+		t.Error("Could not get test signing key")
+		t.FailNow()
+	}
+
+  //function to get a valid token that can be exchanged for credentials
+  getSignedToken := func(subject string, d time.Duration, tags AWSSessionTags) string {
+    token, err := CreateSignedToken(createRS256PolicyTokenWithSessionTags(testFakeIssuer, subject, d, tags), signingKey)
+    if err != nil {
+      t.Errorf("Could create signed token with subject %s and tags %v: %s", subject, tags, err)
+      t.FailNow()
+    }
+    return token
+  }
+
+
+  return tearDownProxies, getSignedToken
+}
+
+func getCredentialsFromTestStsProxy(t *testing.T, token, sessionName, roleArn string) aws.Credentials {
+	result, err := assumeRoleWithWebIdentityAgainstTestStsProxy(t, token, sessionName, roleArn)
+	if err != nil {
+		t.Errorf("encountered error when assuming role: %s", err)
+	}
+  creds := result.Credentials
+  awsCreds := aws.Credentials{
+    AccessKeyID: *creds.AccessKeyId,
+    SecretAccessKey: *creds.SecretAccessKey,
+    SessionToken: *creds.SessionToken,
+    Expires: *creds.Expiration,
+    CanExpire: true,
+  }
+  return awsCreds
+}
+
+//region object is setup in the backends and matches the region name of the backend
+func getRegionObjectContent(t *testing.T, region string, creds aws.Credentials) string{
+  client := getS3ClientAgainstS3Proxy(t, region, creds)
+
+	max1Sec, cancel := context.WithTimeout(context.Background(), 1000 * time.Second)
+  var bucketName = "backenddetails"
+  var objectKey = "region.txt"
+	input := s3.GetObjectInput{
+		Bucket: &bucketName,
+    Key: &objectKey,
+	}
+	defer cancel()
+	s3ObjectOutput, err := client.GetObject(max1Sec, &input)
+	if err != nil {
+		t.Errorf("encountered error getting region file for %s: %s", region, err)
+	}
+  bytes, err := io.ReadAll(s3ObjectOutput.Body)
+  if err != nil {
+		t.Errorf("encountered error reading region file content for %s: %s", region, err)
+	}
+  return string(bytes)
+}
+
+
+//Backend selection is done by chosing a region. The enpdoint we use is fixed
+//to our testing S3Proxy and therefore the hostname is the same. In each backend
+//we have a bucket with the same name and region.txt which holds the actual region
+//name which we can use to validate that our request went to the right backend.
+func TestMakeSureCorrectBackendIsSelected(t *testing.T) {
+  tearDown, getSignedToken := testingFixture(t)
+  defer tearDown()
+  token := getSignedToken("mySubject", time.Minute * 20, AWSSessionTags{PrincipalTags: map[string][]string{"org": {"a"}}})
+  print(token)
+  //Given the policy Manager that has roleArn for the testARN
+	pm = *NewTestPolicyManagerAllowAll()
+  //Given credentials for that role
+  creds := getCredentialsFromTestStsProxy(t, token, "my-session", testPolicyAllowAllARN)
+
+
+  for _, backendRegion := range backendTestRegions {
+    regionContent := getRegionObjectContent(t, backendRegion, creds)
+    if regionContent != backendRegion {
+      t.Errorf("when retrieving region file for %s we got %s", backendRegion, regionContent)
+    }
+  }
+}

cmd/proxys3_test.go

@@ -173,6 +173,35 @@ func getS3ProxyUrl() string{
 	return fmt.Sprintf("%s://%s:%d/", getProxyProtocol(), viper.GetString(s3ProxyFQDN), viper.GetInt(s3ProxyPort))
 }
 
+
+func adapterCredentialsToCredentialsProvider(creds aws.Credentials) aws.CredentialsProviderFunc {
+	return func(ctx context.Context) (aws.Credentials, error) {
+		return creds, nil
+	}
+}
+
+func adapterAwsCredentialsToCredentials(creds AWSCredentials) aws.Credentials {
+	return aws.Credentials{
+		AccessKeyID: creds.AccessKey,
+		SecretAccessKey: creds.SecretKey,
+		SessionToken: creds.SessionToken,
+	}
+}
+
+
+func getS3ClientAgainstS3Proxy(t *testing.T, region string, creds aws.Credentials) (*s3.Client) {
+	cfg := getTestAwsConfig(t)
+
+	client := s3.NewFromConfig(cfg, func (o *s3.Options) {
+		o.BaseEndpoint = aws.String(getS3ProxyUrl())
+		o.Credentials = adapterCredentialsToCredentialsProvider(creds)
+		o.Region = region
+		o.UsePathStyle = true
+	})
+
+	return client
+}
+
 func TestWithValidCredsButNoAccess(t *testing.T) {
 	teardownSuite := setupSuiteProxyS3(t, testStubJustProxy)
 	defer teardownSuite(t)
@@ -183,14 +212,8 @@ func TestWithValidCredsButNoAccess(t *testing.T) {
 		t.Error(err)
 	}
 
-	cfg := getTestAwsConfig(t)
-
-	client := s3.NewFromConfig(cfg, func (o *s3.Options) {
-		o.BaseEndpoint = aws.String(getS3ProxyUrl())
-		o.Credentials = cred
-		o.Region = "eu-west-1"
-		o.UsePathStyle = true
-	})
+	client := getS3ClientAgainstS3Proxy(t, "eu-west-1", adapterAwsCredentialsToCredentials(*cred))
+
 	max1Sec, cancel := context.WithTimeout(context.Background(), 1000 * time.Second)
 	testPrefix := testAllowedPrefix
 	input := s3.ListObjectsV2Input{

cmd/proxysts_test.go

@@ -111,13 +111,14 @@ func TestProxyStsAssumeRoleWithWebIdentityBasicToken(t *testing.T) {
 	}
 }
 
-func createRS256PolicyTokenWithSessionTags(issuer, subject string, expiry time.Duration) (*jwt.Token) {
-	tags := AWSSessionTags{
-		PrincipalTags: map[string][]string{
-			"custom_id": {"idA"},
-		},
-		TransitiveTagKeys: []string{"custom_id"},
-	}
+var testSessionTagsCustomIdA = AWSSessionTags{
+	PrincipalTags: map[string][]string{
+		"custom_id": {"idA"},
+	},
+	TransitiveTagKeys: []string{"custom_id"},
+}
+
+func createRS256PolicyTokenWithSessionTags(issuer, subject string, expiry time.Duration, tags AWSSessionTags) (*jwt.Token) { 
 	claims := newIDPClaims(issuer, subject, expiry, tags)
 
 	token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims)
@@ -138,7 +139,7 @@ func TestProxyStsAssumeRoleWithWebIdentitySessionTagsToken(t *testing.T) {
 		t.Error("Could not get test signing key")
 		t.FailNow()
 	}
-	token, err := CreateSignedToken(createRS256PolicyTokenWithSessionTags(testFakeIssuer, testSubject, 20 * time.Minute), signingKey)
+	token, err := CreateSignedToken(createRS256PolicyTokenWithSessionTags(testFakeIssuer, testSubject, 20 * time.Minute, testSessionTagsCustomIdA), signingKey)
 	if err != nil {
 		t.Error("Could create signed token")
 		t.FailNow()
@@ -204,10 +205,7 @@ func getTestAwsConfig(t *testing.T) (aws.Config) {
 	return cfg
 }
 
-func TestProxyStsViaSTSClient(t *testing.T) {
-	teardownSuite := setupSuiteProxySTS(t)
-	defer teardownSuite(t)
-
+func assumeRoleWithWebIdentityAgainstTestStsProxy(t *testing.T, token, roleSessionName, roleArn string) (*sts.AssumeRoleWithWebIdentityOutput, error) {
 	cfg := getTestAwsConfig(t)
 	secure := viper.GetBool(secure)
 	var protocol string
@@ -222,24 +220,30 @@ func TestProxyStsViaSTSClient(t *testing.T) {
 			fmt.Sprintf("%s://%s:%d/", protocol, viper.GetString(stsProxyFQDN), viper.GetInt(stsProxyPort)),
 		)
 	})
-
-	token := getTestingToken(t)
-	//Given the policy Manager that has roleArn for the testARN
-	initializePolicyManager()
-	roleSessionName := "my-session"
-	var arnToAssume string = testARN
-
 	input := &sts.AssumeRoleWithWebIdentityInput{
 		RoleSessionName: &roleSessionName,
 		WebIdentityToken: &token,
-		RoleArn: &arnToAssume,
+		RoleArn: &roleArn,
 	}
 
 	max1Sec, cancel := context.WithTimeout(context.Background(), 1000 * time.Second)
 	defer cancel()
-	_, err := client.AssumeRoleWithWebIdentity(
+	result, err := client.AssumeRoleWithWebIdentity(
 		max1Sec, input,
 	)
+
+	return result, err
+}
+
+func TestProxyStsViaSTSClient(t *testing.T) {
+	teardownSuite := setupSuiteProxySTS(t)
+	defer teardownSuite(t)
+
+	token := getTestingToken(t)
+	//Given the policy Manager that has roleArn for the testARN
+	initializePolicyManager()
+
+	_, err := assumeRoleWithWebIdentityAgainstTestStsProxy(t, token, "my-session", testARN)
 	if err != nil {
 		t.Errorf("encountered error when assuming role: %s", err)
 	}

testing/README.md

@@ -0,0 +1,31 @@
+# testing setup
+
+In order to allow more extensive testing we run basic implementations of S3 servers as part of testing and 
+allow using them as part of the tests. For simplicity there are not separate hostnames so they do run on
+localhost but they get distinguished by port number.
+
+The downside of this is that we make assumptions on the development environment. This directory should help
+developers setup their local environment to have the S3 servers running.
+
+## Overview
+
+From code it might be harder to understand what we are trying to simulate. But we are just trying to simulate
+S3 servers which corresponds to different regions and thus 2 separate S3 stacks. These regions do not share any state.
+
+In every region we create a bucket "backenddetails" that contains a file region.txt with the region name.
+
+Currently we bootstrap the following regions:
+ - tst-1 : available on port 5000
+ - eu-test-2 : available on port 5001`
+
+
+## Dependencies
+
+
+### Dependencies bootstrap
+Assumed dependencies are to have a modern Python3 runtime which supports virtual environments and pip.
+By executing `make setup-test-dependencies` the required packages get downloaded and installed in the virtual
+environment.
+
+### Dependencies runtimes
+In order to run the S3 servers and have them populated with the test files run `make start-test-s3-servers`

testing/backends/eu-test-2/backenddetails/region.txt

@@ -0,0 +1 @@
+eu-test-2

testing/backends/tst-1/backenddetails/region.txt

@@ -0,0 +1 @@
+tst-1