From 6c42e427e9360b5213494b169ac26a81975d0092 Mon Sep 17 00:00:00 2001 From: Peter Van Bouwel Date: Wed, 20 Nov 2024 08:12:07 +0100 Subject: [PATCH] testing: provide test coverage for backend selection Add a scenario where we mimic a full setup and where we verify which backend is reached. --- .github/workflows/go.yml | 3 + .gitignore | 6 +- Makefile | 13 ++ cmd/almost-e2e_test.go | 146 ++++++++++++++++++ cmd/proxys3_test.go | 39 ++++- cmd/proxysts_test.go | 46 +++--- testing/README.md | 31 ++++ .../eu-test-2/backenddetails/region.txt | 1 + .../backends/tst-1/backenddetails/region.txt | 1 + testing/bootstrap_backend.py | 69 +++++++++ testing/requirements.txt | 60 +++++++ 11 files changed, 385 insertions(+), 30 deletions(-) create mode 100644 cmd/almost-e2e_test.go create mode 100644 testing/README.md create mode 100644 testing/backends/eu-test-2/backenddetails/region.txt create mode 100644 testing/backends/tst-1/backenddetails/region.txt create mode 100644 testing/bootstrap_backend.py create mode 100644 testing/requirements.txt diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 93e224f..2525e4b 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -56,6 +56,9 @@ jobs: - name: Build run: go build -v ./... + - name: Setup test dependencies + run: make setup-test-dependencies && make start-test-s3-servers + - name: Test # As we use config files from time to time we always want to run without cache run: go clean -testcache && go test -v ./... diff --git a/.gitignore b/.gitignore index 83fab6b..4d2ae71 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,8 @@ **__debug_bin* **/.idea -**/*.private \ No newline at end of file +**/*.private + +testing/venv/** +testing/**.err +testing/**.log \ No newline at end of file diff --git a/Makefile b/Makefile index a2a7a49..a336549 100644 --- a/Makefile +++ b/Makefile @@ -6,3 +6,16 @@ run-container-s3: run-container-sts: podman run --rm -v ./etc.private:/etc/fakes3pp:Z -p 8444:8444 --env HOME=${HOME} -it localhost/fakes3pp:latest proxysts --dot-env /etc/fakes3pp/.env.docker + +setup-test-dependencies: + [ ! -f testing/venv/moto ] && python3 -m venv testing/venv/moto + ./testing/venv/moto/bin/pip3 install -r testing/requirements.txt + +start-test-s3-servers: + ./testing/venv/moto/bin/moto_server -p 5000 >testing/server_5000.log 2>testing/server_5000.err & + ./testing/venv/moto/bin/moto_server -p 5001 >testing/server_5001.log 2>testing/server_5001.err & + ./testing/venv/moto/bin/python3 testing/bootstrap_backend.py testing/backends/tst-1 http://localhost:5000 + ./testing/venv/moto/bin/python3 testing/bootstrap_backend.py testing/backends/eu-test-2 http://localhost:5001 + +stop-test-s3-servers: + for pid in `ps -ef | grep testing/venv/moto/bin/python3 | grep -v grep | awk '{print $$2}'`; do kill "$${pid}"; done diff --git a/cmd/almost-e2e_test.go b/cmd/almost-e2e_test.go new file mode 100644 index 0000000..fa78a14 --- /dev/null +++ b/cmd/almost-e2e_test.go @@ -0,0 +1,146 @@ +package cmd + +import ( + "context" + "fmt" + "io" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" +) + + +const testRegion1 = "tst-1" +const testRegion2 = "eu-test-2" +var backendTestRegions = []string{testRegion1, testRegion2} + +var testingBackendsConfig = []byte(fmt.Sprintf(` +# This is a test file check backend-config.yaml if you want to create a configuration +s3backends: + - region: %s + credentials: + file: ../etc/creds/cfc_creds.yaml + endpoint: http://localhost:5000 + - region: %s + credentials: + file: ../etc/creds/otc_creds.yaml + endpoint: http://localhost:5001 +default: %s +`, testRegion1, testRegion2, testRegion2)) + + +//Set the configurations as expected for the testingbackends +//See testing/README.md for details on testing setup +func setTestingBackendsConfig(t *testing.T) { + cfg, err := getBackendsConfigFromBytes(testingBackendsConfig) + if err != nil { + t.Error(err) + t.FailNow() + } + globalBackendsConfig = cfg +} + +//This is the testing fixture. It starts an sts and s3 proxy which +//are configured with the S3 backends detailed in testing/README.md. +func testingFixture(t *testing.T) (tearDown func ()(), getToken func(subject string, d time.Duration, tags AWSSessionTags) string){ + //Configure backends to be the testing S3 backends + setTestingBackendsConfig(t) + //Given valid server config + teardownSuiteSTS := setupSuiteProxySTS(t) + teardownSuiteS3 := setupSuiteProxyS3(t, justProxied) + + //function to stop the setup of the fixture + tearDownProxies := func () { + teardownSuiteSTS(t) + teardownSuiteS3(t) + } + + _, err := loadOidcConfig([]byte(testConfigFakeTesting)) + if err != nil { + t.Error(err) + } + + signingKey, err := getTestSigningKey() + if err != nil { + t.Error("Could not get test signing key") + t.FailNow() + } + + //function to get a valid token that can be exchanged for credentials + getSignedToken := func(subject string, d time.Duration, tags AWSSessionTags) string { + token, err := CreateSignedToken(createRS256PolicyTokenWithSessionTags(testFakeIssuer, subject, d, tags), signingKey) + if err != nil { + t.Errorf("Could create signed token with subject %s and tags %v: %s", subject, tags, err) + t.FailNow() + } + return token + } + + + return tearDownProxies, getSignedToken +} + +func getCredentialsFromTestStsProxy(t *testing.T, token, sessionName, roleArn string) aws.Credentials { + result, err := assumeRoleWithWebIdentityAgainstTestStsProxy(t, token, sessionName, roleArn) + if err != nil { + t.Errorf("encountered error when assuming role: %s", err) + } + creds := result.Credentials + awsCreds := aws.Credentials{ + AccessKeyID: *creds.AccessKeyId, + SecretAccessKey: *creds.SecretAccessKey, + SessionToken: *creds.SessionToken, + Expires: *creds.Expiration, + CanExpire: true, + } + return awsCreds +} + +//region object is setup in the backends and matches the region name of the backend +func getRegionObjectContent(t *testing.T, region string, creds aws.Credentials) string{ + client := getS3ClientAgainstS3Proxy(t, region, creds) + + max1Sec, cancel := context.WithTimeout(context.Background(), 1000 * time.Second) + var bucketName = "backenddetails" + var objectKey = "region.txt" + input := s3.GetObjectInput{ + Bucket: &bucketName, + Key: &objectKey, + } + defer cancel() + s3ObjectOutput, err := client.GetObject(max1Sec, &input) + if err != nil { + t.Errorf("encountered error getting region file for %s: %s", region, err) + } + bytes, err := io.ReadAll(s3ObjectOutput.Body) + if err != nil { + t.Errorf("encountered error reading region file content for %s: %s", region, err) + } + return string(bytes) +} + + +//Backend selection is done by chosing a region. The enpdoint we use is fixed +//to our testing S3Proxy and therefore the hostname is the same. In each backend +//we have a bucket with the same name and region.txt which holds the actual region +//name which we can use to validate that our request went to the right backend. +func TestMakeSureCorrectBackendIsSelected(t *testing.T) { + tearDown, getSignedToken := testingFixture(t) + defer tearDown() + token := getSignedToken("mySubject", time.Minute * 20, AWSSessionTags{PrincipalTags: map[string][]string{"org": {"a"}}}) + print(token) + //Given the policy Manager that has roleArn for the testARN + pm = *NewTestPolicyManagerAllowAll() + //Given credentials for that role + creds := getCredentialsFromTestStsProxy(t, token, "my-session", testPolicyAllowAllARN) + + + for _, backendRegion := range backendTestRegions { + regionContent := getRegionObjectContent(t, backendRegion, creds) + if regionContent != backendRegion { + t.Errorf("when retrieving region file for %s we got %s", backendRegion, regionContent) + } + } +} \ No newline at end of file diff --git a/cmd/proxys3_test.go b/cmd/proxys3_test.go index ffda73b..afa3a71 100644 --- a/cmd/proxys3_test.go +++ b/cmd/proxys3_test.go @@ -173,6 +173,35 @@ func getS3ProxyUrl() string{ return fmt.Sprintf("%s://%s:%d/", getProxyProtocol(), viper.GetString(s3ProxyFQDN), viper.GetInt(s3ProxyPort)) } + +func adapterCredentialsToCredentialsProvider(creds aws.Credentials) aws.CredentialsProviderFunc { + return func(ctx context.Context) (aws.Credentials, error) { + return creds, nil + } +} + +func adapterAwsCredentialsToCredentials(creds AWSCredentials) aws.Credentials { + return aws.Credentials{ + AccessKeyID: creds.AccessKey, + SecretAccessKey: creds.SecretKey, + SessionToken: creds.SessionToken, + } +} + + +func getS3ClientAgainstS3Proxy(t *testing.T, region string, creds aws.Credentials) (*s3.Client) { + cfg := getTestAwsConfig(t) + + client := s3.NewFromConfig(cfg, func (o *s3.Options) { + o.BaseEndpoint = aws.String(getS3ProxyUrl()) + o.Credentials = adapterCredentialsToCredentialsProvider(creds) + o.Region = region + o.UsePathStyle = true + }) + + return client +} + func TestWithValidCredsButNoAccess(t *testing.T) { teardownSuite := setupSuiteProxyS3(t, testStubJustProxy) defer teardownSuite(t) @@ -183,14 +212,8 @@ func TestWithValidCredsButNoAccess(t *testing.T) { t.Error(err) } - cfg := getTestAwsConfig(t) - - client := s3.NewFromConfig(cfg, func (o *s3.Options) { - o.BaseEndpoint = aws.String(getS3ProxyUrl()) - o.Credentials = cred - o.Region = "eu-west-1" - o.UsePathStyle = true - }) + client := getS3ClientAgainstS3Proxy(t, "eu-west-1", adapterAwsCredentialsToCredentials(*cred)) + max1Sec, cancel := context.WithTimeout(context.Background(), 1000 * time.Second) testPrefix := testAllowedPrefix input := s3.ListObjectsV2Input{ diff --git a/cmd/proxysts_test.go b/cmd/proxysts_test.go index e162e6b..e9baf5d 100644 --- a/cmd/proxysts_test.go +++ b/cmd/proxysts_test.go @@ -111,13 +111,14 @@ func TestProxyStsAssumeRoleWithWebIdentityBasicToken(t *testing.T) { } } -func createRS256PolicyTokenWithSessionTags(issuer, subject string, expiry time.Duration) (*jwt.Token) { - tags := AWSSessionTags{ - PrincipalTags: map[string][]string{ - "custom_id": {"idA"}, - }, - TransitiveTagKeys: []string{"custom_id"}, - } +var testSessionTagsCustomIdA = AWSSessionTags{ + PrincipalTags: map[string][]string{ + "custom_id": {"idA"}, + }, + TransitiveTagKeys: []string{"custom_id"}, +} + +func createRS256PolicyTokenWithSessionTags(issuer, subject string, expiry time.Duration, tags AWSSessionTags) (*jwt.Token) { claims := newIDPClaims(issuer, subject, expiry, tags) token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims) @@ -138,7 +139,7 @@ func TestProxyStsAssumeRoleWithWebIdentitySessionTagsToken(t *testing.T) { t.Error("Could not get test signing key") t.FailNow() } - token, err := CreateSignedToken(createRS256PolicyTokenWithSessionTags(testFakeIssuer, testSubject, 20 * time.Minute), signingKey) + token, err := CreateSignedToken(createRS256PolicyTokenWithSessionTags(testFakeIssuer, testSubject, 20 * time.Minute, testSessionTagsCustomIdA), signingKey) if err != nil { t.Error("Could create signed token") t.FailNow() @@ -204,10 +205,7 @@ func getTestAwsConfig(t *testing.T) (aws.Config) { return cfg } -func TestProxyStsViaSTSClient(t *testing.T) { - teardownSuite := setupSuiteProxySTS(t) - defer teardownSuite(t) - +func assumeRoleWithWebIdentityAgainstTestStsProxy(t *testing.T, token, roleSessionName, roleArn string) (*sts.AssumeRoleWithWebIdentityOutput, error) { cfg := getTestAwsConfig(t) secure := viper.GetBool(secure) var protocol string @@ -222,24 +220,30 @@ func TestProxyStsViaSTSClient(t *testing.T) { fmt.Sprintf("%s://%s:%d/", protocol, viper.GetString(stsProxyFQDN), viper.GetInt(stsProxyPort)), ) }) - - token := getTestingToken(t) - //Given the policy Manager that has roleArn for the testARN - initializePolicyManager() - roleSessionName := "my-session" - var arnToAssume string = testARN - input := &sts.AssumeRoleWithWebIdentityInput{ RoleSessionName: &roleSessionName, WebIdentityToken: &token, - RoleArn: &arnToAssume, + RoleArn: &roleArn, } max1Sec, cancel := context.WithTimeout(context.Background(), 1000 * time.Second) defer cancel() - _, err := client.AssumeRoleWithWebIdentity( + result, err := client.AssumeRoleWithWebIdentity( max1Sec, input, ) + + return result, err +} + +func TestProxyStsViaSTSClient(t *testing.T) { + teardownSuite := setupSuiteProxySTS(t) + defer teardownSuite(t) + + token := getTestingToken(t) + //Given the policy Manager that has roleArn for the testARN + initializePolicyManager() + + _, err := assumeRoleWithWebIdentityAgainstTestStsProxy(t, token, "my-session", testARN) if err != nil { t.Errorf("encountered error when assuming role: %s", err) } diff --git a/testing/README.md b/testing/README.md new file mode 100644 index 0000000..e099cfd --- /dev/null +++ b/testing/README.md @@ -0,0 +1,31 @@ +# testing setup + +In order to allow more extensive testing we run basic implementations of S3 servers as part of testing and +allow using them as part of the tests. For simplicity there are not separate hostnames so they do run on +localhost but they get distinguished by port number. + +The downside of this is that we make assumptions on the development environment. This directory should help +developers setup their local environment to have the S3 servers running. + +## Overview + +From code it might be harder to understand what we are trying to simulate. But we are just trying to simulate +S3 servers which corresponds to different regions and thus 2 separate S3 stacks. These regions do not share any state. + +In every region we create a bucket "backenddetails" that contains a file region.txt with the region name. + +Currently we bootstrap the following regions: + - tst-1 : available on port 5000 + - eu-test-2 : available on port 5001` + + +## Dependencies + + +### Dependencies bootstrap +Assumed dependencies are to have a modern Python3 runtime which supports virtual environments and pip. +By executing `make setup-test-dependencies` the required packages get downloaded and installed in the virtual +environment. + +### Dependencies runtimes +In order to run the S3 servers and have them populated with the test files run `make start-test-s3-servers` \ No newline at end of file diff --git a/testing/backends/eu-test-2/backenddetails/region.txt b/testing/backends/eu-test-2/backenddetails/region.txt new file mode 100644 index 0000000..b8a4a09 --- /dev/null +++ b/testing/backends/eu-test-2/backenddetails/region.txt @@ -0,0 +1 @@ +eu-test-2 \ No newline at end of file diff --git a/testing/backends/tst-1/backenddetails/region.txt b/testing/backends/tst-1/backenddetails/region.txt new file mode 100644 index 0000000..5434859 --- /dev/null +++ b/testing/backends/tst-1/backenddetails/region.txt @@ -0,0 +1 @@ +tst-1 \ No newline at end of file diff --git a/testing/bootstrap_backend.py b/testing/bootstrap_backend.py new file mode 100644 index 0000000..d5add8c --- /dev/null +++ b/testing/bootstrap_backend.py @@ -0,0 +1,69 @@ +import boto3 +import sys +from glob import glob +from pathlib import Path + + +if len(sys.argv) != 3: + raise RuntimeError("Expect invocation bootstrap_backend.py ") +else: + endpoint_url = sys.argv[2] + local_backend_path = Path(sys.argv[1]) + print(f"Bootstrapping {endpoint_url} using {local_backend_path}") + if not local_backend_path.is_dir(): + raise RuntimeError(f"Expect to be a directory got {local_backend_path}") + region_name = local_backend_path.stem + + +def get_s3client(): + session = boto3.session.Session() + + return session.client( + service_name='s3', + endpoint_url=endpoint_url, + region_name=region_name, + ) + + +def create_bucket(s3_client, bucket_name: str): + try: + return s3_client.create_bucket( + Bucket=bucket_name, + CreateBucketConfiguration={ + 'LocationConstraint': region_name, + }, + ) + except s3_client.exceptions.BucketAlreadyOwnedByYou: + # For idempotency we do not fail if a bucket already exists + return None + + +def copy_file_to_bucket(s3_client, source_file: Path, bucket_name:str, object_key: str): + print(f"Moving {source_file} to s3://{bucket_name}/{object_key}") + s3_client.upload_file(source_file, bucket_name, object_key) + + +def process_bucket(s3_client, bucket_dir: str): + """ + Bootstrap a bucket based on a directory that looks like the bucket contents + """ + bucket_path = Path(bucket_dir) + bucket_name = bucket_path.stem + print(f"Processing bucket {bucket_name} in {region_name}") + create_bucket(s3_client, bucket_name) + for file in glob(str(bucket_path.joinpath("*")), recursive=True): + file_path = Path(file) + if file_path.is_dir(): + continue # We don´t mimic the directories in our object store + if not file_path.is_file(): + raise RuntimeError(f"Unsupported filesystem object {file_path}") + print(f"Processing file {file}") + object_key = str(file).replace(f"{bucket_dir}/", "") + copy_file_to_bucket(s3_client, file_path, bucket_name, object_key) + + +if __name__ == '__main__': + s3_client = get_s3client() + for bucket_dir in glob(str(local_backend_path.joinpath("*"))): + process_bucket(s3_client, bucket_dir) + diff --git a/testing/requirements.txt b/testing/requirements.txt new file mode 100644 index 0000000..905bc53 --- /dev/null +++ b/testing/requirements.txt @@ -0,0 +1,60 @@ +annotated-types==0.7.0 +antlr4-python3-runtime==4.13.2 +attrs==24.2.0 +aws-sam-translator==1.92.0 +aws-xray-sdk==2.14.0 +blinker==1.9.0 +boto3==1.35.64 +botocore==1.35.64 +certifi==2024.8.30 +cffi==1.17.1 +cfn-lint==1.20.0 +charset-normalizer==3.4.0 +click==8.1.7 +cryptography==43.0.3 +docker==7.1.0 +Flask==3.1.0 +Flask-Cors==5.0.0 +graphql-core==3.2.5 +idna==3.10 +itsdangerous==2.2.0 +Jinja2==3.1.4 +jmespath==1.0.1 +joserfc==1.0.0 +jsondiff==2.2.1 +jsonpatch==1.33 +jsonpath-ng==1.7.0 +jsonpointer==3.0.0 +jsonschema==4.23.0 +jsonschema-path==0.3.3 +jsonschema-specifications==2023.12.1 +lazy-object-proxy==1.10.0 +MarkupSafe==3.0.2 +moto==5.0.21 +mpmath==1.3.0 +networkx==3.4.2 +openapi-schema-validator==0.6.2 +openapi-spec-validator==0.7.1 +pathable==0.4.3 +ply==3.11 +py-partiql-parser==0.5.6 +pycparser==2.22 +pydantic==2.9.2 +pydantic_core==2.23.4 +pyparsing==3.2.0 +python-dateutil==2.9.0.post0 +PyYAML==6.0.2 +referencing==0.35.1 +regex==2024.11.6 +requests==2.32.3 +responses==0.25.3 +rfc3339-validator==0.1.4 +rpds-py==0.21.0 +s3transfer==0.10.3 +six==1.16.0 +sympy==1.13.3 +typing_extensions==4.12.2 +urllib3==2.2.3 +Werkzeug==3.1.3 +wrapt==1.16.0 +xmltodict==0.14.2