From fe354001effd09c791737d50811a100269ead153 Mon Sep 17 00:00:00 2001 From: Peter Van Bouwel Date: Sun, 17 Nov 2024 15:18:13 +0100 Subject: [PATCH] !feature: support multiple S3 backends This implements support for multiple S3 backends. Based on the region that is provided in the request the proxy will select the backend to send the request to. BREAKING CHANGE: remove AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and FAKES3PP_S3_PROXY_TARGET environment variables in favor of backend-config.yaml which allows specifying similar config per proxy backend. Same functionality can be achieved but the configuration needs to be specified in backend-config.yaml which is pointed to by environment variable `FAKES3PP_S3_BACKEND_CONFIG`. --- README.md | 7 +- cmd/backendconfig.go | 195 +++++++++++++++++++++++++++ cmd/backendconfig_test.go | 56 ++++++++ cmd/config.go | 36 ++--- cmd/handler-builder.go | 53 ++++---- cmd/presign.go | 49 ++++--- cmd/proxys3.go | 14 +- cmd/proxys3_test.go | 33 ++++- etc/.env | 3 +- etc/.env.docker | 4 +- etc/backend-config.yaml | 22 +++ etc/creds/cfc_creds.yaml | 2 + etc/creds/otc_creds.yaml | 3 + presign/s3v4.go | 17 +-- presign/s3v4query.go | 3 +- presign/s3v4query_test.go | 2 +- requestutils/amz-credential-value.go | 10 ++ 17 files changed, 401 insertions(+), 108 deletions(-) create mode 100644 cmd/backendconfig.go create mode 100644 cmd/backendconfig_test.go create mode 100644 etc/backend-config.yaml create mode 100644 etc/creds/cfc_creds.yaml create mode 100644 etc/creds/otc_creds.yaml diff --git a/README.md b/README.md index 5276713..4e17a23 100644 --- a/README.md +++ b/README.md @@ -54,10 +54,9 @@ cp -R etc etc.private ``` Next adapt the config under etc.private. To get a minimal working local proxy you need to change at least: - - etc.private/.env.docker - - AWS_ACCESS_KEY_ID: The access key id for the object store that you are proxying - - AWS_SECRET_ACCESS_KEY: The secret access key for the object store that you are proxying - - FAKES3PP_S3_PROXY_TARGET: The hostname of the object store you are proxying (e.g. `s3.waw3-1.cloudferro.com`) + - etc.private/backend-config.yaml + - The example shows config for 2 S3 backends. Remove and add config as per your use case. + - The additional files you can create under etc.private as they get mounted under /etc/fakes3pp ### Production configuration diff --git a/cmd/backendconfig.go b/cmd/backendconfig.go new file mode 100644 index 0000000..a514afb --- /dev/null +++ b/cmd/backendconfig.go @@ -0,0 +1,195 @@ +package cmd + +import ( + "errors" + "fmt" + "os" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/spf13/viper" + "sigs.k8s.io/yaml" +) + +type backendConfigFileEntry struct { + RegionName string `yaml:"region" json:"region"` + Credentials map[string]string `yaml:"credentials" json:"credentials"` + Endpoint string `yaml:"endpoint" json:"endpoint"` +} + + +type awsBackendCredentialFile struct { + AccessKey string `yaml:"aws_access_key_id" json:"aws_access_key_id"` + SecretKey string `yaml:"aws_secret_access_key" json:"aws_secret_access_key"` + SessionToken string `yaml:"aws_session_token,omitempty" json:"aws_session_token,omitempty"` +} + +//The config file could host different types of credentials. Check cases 1 by one +//and fail if there was no valid type of credentials found +func (entry backendConfigFileEntry) getCredentials() (creds aws.Credentials, err error) { + filePath, ok := entry.Credentials["file"] + if ok { + // We are indeed a file + buf, err := os.ReadFile(filePath) + if err != nil { + return creds, fmt.Errorf("could not read credentials file %s; %s", filePath, err) + } + + c := &awsBackendCredentialFile{} + err = yaml.Unmarshal(buf, c) + if err != nil { + return creds, fmt.Errorf("error unmarshalling file %s; %s", filePath, err) + } + if c.AccessKey == "" { + return creds, errors.New("invalid credentials file, missing access key") + } + creds.AccessKeyID = c.AccessKey + if c.SecretKey == "" { + return creds, errors.New("invalid credentials file, missing secret key") + } + creds.SecretAccessKey = c.SecretKey + if c.SessionToken != "" { + creds.SessionToken = c.SessionToken + creds.CanExpire = true + } + return creds, nil + } + return creds, errors.New("unable to find a valid type of credentials") +} + +type backendsConfigFile struct { + Backends []backendConfigFileEntry `yaml:"s3backends" json:"s3backends"` + Default string `yaml:"default" json:"default"` +} + + +func getBackendsConfig() (*backendsConfig, error) { + buf, err := os.ReadFile(viper.GetString(s3BackendConfigFile)) + if err != nil { + return nil, err + } + return getBackendsConfigFromBytes(buf) +} + +func getBackendsConfigFromBytes(inputBytes []byte) (*backendsConfig, error) { + c := &backendsConfigFile{} + err := yaml.Unmarshal(inputBytes, c) + if err != nil { + return nil, err + } + + result := backendsConfig{ + backends: map[string]backendConfigEntry{}, + } + + for _, backendRawCfg := range c.Backends { + backendCfg := backendConfigEntry{} + err = backendCfg.fromBackendConfigFileEntry(backendRawCfg) + if err != nil { + return nil, fmt.Errorf("invalid config %v resulted in %s", backendRawCfg, err) + } + result.backends[backendRawCfg.RegionName] = backendCfg + } + + defaultBackend := c.Default + _, defaultExists := result.backends[defaultBackend] + if !defaultExists { + return nil, fmt.Errorf("default backend %s does not exist", defaultBackend) + } + result.defaultBackend = defaultBackend + + return &result, err +} + +type backendConfigEntry struct { + credentials aws.Credentials + endpoint endpoint +} + +//A dedicated type for endpoint allows to have the semantics of endpoints of the config. +//When creating these endpoints we do certain checks so by typing them we can assume these +//checks had passed whenever we encounter an endpoint later on +type endpoint string + +func buildEndpoint(uri string) (endpoint, error) { + if !strings.HasPrefix(uri, "https://") && !strings.HasPrefix(uri, "http://") { + return "", errors.New("endpoint URIs must start with https:// or http://") + } + return endpoint(uri), nil +} + +//This method is to get rid of the protocol from the endpoint specification +func (e endpoint) getHost() string { + uriString := string(e) + return strings.Split(uriString, "://")[1] +} + +//The endpoint base URI is of form protocol://hostname and can be used to identify the backend +//service +func (e endpoint) getBaseURI() string { + return string(e) +} + +func (bce *backendConfigEntry) fromBackendConfigFileEntry(input backendConfigFileEntry) error { + endpoint, err := buildEndpoint(input.Endpoint) + bce.endpoint = endpoint + + awsCredentials, err := input.getCredentials() + bce.credentials = awsCredentials + return err +} + +type backendsConfig struct { + backends map[string]backendConfigEntry + defaultBackend string +} + +func (cfg* backendsConfig) getBackendConfig(backendId string) (cfgEntry backendConfigEntry, err error) { + if cfg == nil { + return cfgEntry, errors.New("backendsConfig not initialised") + } + if backendId == "" { + backendId = cfg.defaultBackend + } + backendCfg, ok := cfg.backends[backendId] + if ok { + return backendCfg, nil + } else { + return cfgEntry, fmt.Errorf("no such backend: %s", backendId) + } +} + +//Get credentials for a backendId. +func (cfg *backendsConfig) getBackendCredentials(backendId string) (creds aws.Credentials, err error) { + backendCfg, err := cfg.getBackendConfig(backendId) + if err != nil { + return creds, err + } + creds = backendCfg.credentials + return +} + +var globalBackendsConfig *backendsConfig + +//Get the server credentials for a specific backend identified by its identifier +//At this time we use the region name and we do support the empty string in case the region +//cannot be determined and the default backend should be used. +func getBackendCredentials(backendId string) (creds aws.Credentials, err error) { + return globalBackendsConfig.getBackendCredentials(backendId) +} + +//Get endpoint for a backend. The endpoint contains the protocol and the hostname +//to arrive at the backend. +func (cfg *backendsConfig) getBackendEndpoint(backendId string) (endpoint, error) { + backendCfg, err := cfg.getBackendConfig(backendId) + if err != nil { + return "", err + } + return backendCfg.endpoint, nil +} + +//Get endpoint for a backend. The endpoint contains the protocol and the hostname +//to arrive at the backend. +func getBackendEndpoint(backendId string) (endpoint, error) { + return globalBackendsConfig.getBackendEndpoint(backendId) +} \ No newline at end of file diff --git a/cmd/backendconfig_test.go b/cmd/backendconfig_test.go new file mode 100644 index 0000000..6405788 --- /dev/null +++ b/cmd/backendconfig_test.go @@ -0,0 +1,56 @@ +package cmd + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" +) + + +var testDefaultBackendRegion = "waw3-1" +var testSecondBackendRegion = "eu-nl" +var testDefaultBackendCredentials = aws.Credentials{ + AccessKeyID: "fake_key_id", + SecretAccessKey: "fake_secret", +} +var testSecondaryBackendCredentials = aws.Credentials{ + AccessKeyID: "fake_key_id_otc", + SecretAccessKey: "fake_secret_otc", + SessionToken: "fakeSessionTokOtc1", + CanExpire: true, +} + +func TestLoadingOfExampleConfig(t *testing.T) { + BindEnvVariables(proxys3) + cfg , err := getBackendsConfig() + if err != nil { + t.Error("Could not load S3 backend config") + t.Fail() + } + if cfg.defaultBackend != testDefaultBackendRegion { + t.Errorf("Incorrect default backend. Got %s, Expected %s", cfg.defaultBackend, testDefaultBackendRegion) + } + _, err = cfg.getBackendConfig(testDefaultBackendRegion) + if err != nil { + t.Error("Default backend config is not available") + } + creds1, err := cfg.getBackendCredentials(testDefaultBackendRegion) + if err != nil { + t.Error("Default backend credentials are not available") + } + if creds1 != testDefaultBackendCredentials { + t.Error("Default backend credentials are not correctly loaded") + } + + _, err = cfg.getBackendConfig(testSecondBackendRegion) + if err != nil { + t.Error("Secondary backend config is not available") + } + creds2, err := cfg.getBackendCredentials(testSecondBackendRegion) + if err != nil { + t.Error("Secondary backend credentials are not available") + } + if creds2 != testSecondaryBackendCredentials { + t.Error("Secondary backend credentials are not correctly loaded") + } +} \ No newline at end of file diff --git a/cmd/config.go b/cmd/config.go index d9ae0a2..b66d002 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -32,15 +32,12 @@ func (e envVarDef) shouldBeSetFor(cmd string) (bool) { } const( - awsAccessKeyId = "awsAccessKeyId" - awsSecretAccessKey = "awsSecretAccessKey" s3ProxyFQDN = "s3ProxyFQDN" s3ProxyPort = "s3ProxyPort" s3ProxyCertFile = "s3ProxyCertFile" s3ProxyKeyFile = "s3ProxyKeyFile" s3ProxyJwtPublicRSAKey = "s3ProxyJwtPublicRSAKey" s3ProxyJwtPrivateRSAKey = "s3ProxyJwtPrivateRSAKey" - s3ProxyTarget = "s3ProxyTarget" stsProxyFQDN = "stsProxyFQDN" stsProxyPort = "stsProxyPort" stsProxyCertFile = "stsProxyCertFile" @@ -48,46 +45,31 @@ const( rolePolicyPath = "rolePolicyPath" secure = "secure" stsOIDCConfigFile = "stsOIDCConfigFile" + s3BackendConfigFile = "s3BackendConfigFile" stsMaxDurationSeconds = "stsMaxDurationSeconds" signedUrlGraceTimeSeconds = "signedUrlGraceTimeSeconds" //Environment variables are upper cased //Unless they are wellknown environment variables they should be prefixed - AWS_ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID" - AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY" FAKES3PP_S3_PROXY_FQDN = "FAKES3PP_S3_PROXY_FQDN" FAKES3PP_S3_PROXY_PORT = "FAKES3PP_S3_PROXY_PORT" FAKES3PP_S3_PROXY_CERT_FILE = "FAKES3PP_S3_PROXY_CERT_FILE" FAKES3PP_S3_PROXY_KEY_FILE = "FAKES3PP_S3_PROXY_KEY_FILE" FAKES3PP_S3_PROXY_JWT_PUBLIC_RSA_KEY = "FAKES3PP_S3_PROXY_JWT_PUBLIC_RSA_KEY" FAKES3PP_S3_PROXY_JWT_PRIVATE_RSA_KEY = "FAKES3PP_S3_PROXY_JWT_PRIVATE_RSA_KEY" - FAKES3PP_S3_PROXY_TARGET = "FAKES3PP_S3_PROXY_TARGET" FAKES3PP_STS_PROXY_FQDN = "FAKES3PP_STS_PROXY_FQDN" FAKES3PP_STS_PROXY_PORT = "FAKES3PP_STS_PROXY_PORT" FAKES3PP_STS_PROXY_CERT_FILE = "FAKES3PP_STS_PROXY_CERT_FILE" FAKES3PP_STS_PROXY_KEY_FILE = "FAKES3PP_STS_PROXY_KEY_FILE" FAKES3PP_SECURE = "FAKES3PP_SECURE" FAKES3PP_STS_OIDC_CONFIG = "FAKES3PP_STS_OIDC_CONFIG" + FAKES3PP_S3_BACKEND_CONFIG = "FAKES3PP_S3_BACKEND_CONFIG" FAKES3PP_ROLE_POLICY_PATH = "FAKES3PP_ROLE_POLICY_PATH" FAKES3PP_STS_MAX_DURATION_SECONDS = "FAKES3PP_STS_MAX_DURATION_SECONDS" FAKES3PP_SIGNEDURL_GRACE_TIME_SECONDS = "FAKES3PP_SIGNEDURL_GRACE_TIME_SECONDS" ) var envVarDefs = []envVarDef{ - { - awsAccessKeyId, - AWS_ACCESS_KEY_ID, - true, - "The AWS_ACCESS_KEY_ID that will be used to go to the upstream S3 API.", - []string{proxys3}, - }, - { - awsSecretAccessKey, - AWS_SECRET_ACCESS_KEY, - true, - "The AWS_SECRET_ACCESS_KEY that will be used to go to the upstream S3 API.", - []string{proxys3}, - }, { s3ProxyFQDN, FAKES3PP_S3_PROXY_FQDN, @@ -130,13 +112,6 @@ var envVarDefs = []envVarDef{ "The key file used for signing JWT tokens", []string{proxys3, proxysts}, }, - { - s3ProxyTarget, - FAKES3PP_S3_PROXY_TARGET, - true, - "The hostname to be used to connect to the backend.", - []string{proxys3}, - }, { stsProxyFQDN, FAKES3PP_STS_PROXY_FQDN, @@ -179,6 +154,13 @@ var envVarDefs = []envVarDef{ "The configuration of which issuers are trusted for OIDC tokens", []string{proxysts}, }, + { + s3BackendConfigFile, + FAKES3PP_S3_BACKEND_CONFIG, + true, + "The configuration of the backends that are proxied. See the sample start config for details how to configure these backends", + []string{proxys3}, + }, { rolePolicyPath, FAKES3PP_ROLE_POLICY_PATH, diff --git a/cmd/handler-builder.go b/cmd/handler-builder.go index aa47785..ba83c5e 100644 --- a/cmd/handler-builder.go +++ b/cmd/handler-builder.go @@ -8,7 +8,6 @@ import ( "log/slog" "net/http" "net/url" - "os" "strings" "time" @@ -255,7 +254,7 @@ func (hb handlerBuilder) Build(action S3ApiAction, presigned bool) (http.Handler SecretAccessKey: secretAccessKey, SessionToken: sessionToken, } - err = presign.SignWithCreds(ctx, clonedReq, creds) + err = presign.SignWithCreds(ctx, clonedReq, creds, "ThisShouldNotBeUsedForSigv4Requests258") if err != nil { slog.Error("Could not sign request", "error", err, xRequestIDStr, getRequestID(ctx)) writeS3ErrorResponse(ctx, w, ErrS3InternalError, nil) @@ -331,10 +330,22 @@ func logRequest(ctx context.Context, apiAction string, r *http.Request) { } func justProxy(ctx context.Context, w http.ResponseWriter, r *http.Request) { - reTargetRequest(r) - err := signRequest(ctx, r) + targetRegion := requestutils.GetRegionFromRequest(r, globalBackendsConfig.defaultBackend) + err := reTargetRequest(r, targetRegion) if err != nil { - slog.Error("Could not sign request with permanent credentials", "error", err, xRequestIDStr, getRequestID(ctx)) + slog.Error("Could not re-target request with permanent credentials", "error", err, xRequestIDStr, getRequestID(ctx), "backendId", targetRegion) + writeS3ErrorResponse(ctx, w, ErrS3InternalError, nil) + return + } + creds, err := getBackendCredentials(targetRegion) + if err != nil { + slog.Error("Could not get credentials for request", "error", err, xRequestIDStr, getRequestID(ctx), "backendId", targetRegion) + writeS3ErrorResponse(ctx, w, ErrS3InternalError, nil) + return + } + err = presign.SignWithCreds(ctx, r, creds, targetRegion) + if err != nil { + slog.Error("Could not sign request with permanent credentials", "error", err, xRequestIDStr, getRequestID(ctx), "backendId", targetRegion) writeS3ErrorResponse(ctx, w, ErrS3InternalError, nil) return } @@ -368,38 +379,30 @@ func justProxy(ctx context.Context, w http.ResponseWriter, r *http.Request) { // Adapt Host to the new target // We also have to clear RequestURI and set URL appropriately as explained in // https://stackoverflow.com/questions/19595860/http-request-requesturi-field-when-making-request-in-go -func reTargetRequest(r *http.Request) { +func reTargetRequest(r *http.Request, backendId string) (error) { // Old signature r.Header.Del("Authorization") // Old session token r.Header.Del(constants.AmzSecurityTokenKey) r.Header.Del("Host") - r.Header.Add("Host", s3TargetHost) - r.Host = s3TargetHost + endpoint, err := getBackendEndpoint(backendId) + if err != nil { + return err + } + r.Header.Add("Host", endpoint.getHost()) + r.Host = endpoint.getHost() origRawQuery := r.URL.RawQuery slog.Info("Stored orig RawQuery", "raw_query", origRawQuery) - u, err := url.Parse(fmt.Sprintf("https://%s%s", s3TargetHost, r.RequestURI)) + u, err := url.Parse(fmt.Sprintf("%s%s", endpoint.getBaseURI(), r.RequestURI)) if err != nil { - panic(err) + return err } - r.RequestURI = "" + r.RequestURI = "" r.RemoteAddr = "" r.URL = u r.URL.RawQuery = origRawQuery slog.Info("RawQuery that is put in place", "raw_query", r.URL.RawQuery) -} - -func signRequest(ctx context.Context, req *http.Request) error{ - accessKey := os.Getenv("AWS_ACCESS_KEY_ID") - secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY") - - creds := aws.Credentials{ - AccessKeyID: accessKey, - SecretAccessKey: secretKey, - } - - return presign.SignWithCreds(ctx, req, creds) -} - + return nil +} \ No newline at end of file diff --git a/cmd/presign.go b/cmd/presign.go index 072dcf6..1454666 100644 --- a/cmd/presign.go +++ b/cmd/presign.go @@ -12,7 +12,7 @@ import ( "time" "github.com/VITObelgium/fakes3pp/presign" - "github.com/aws/aws-sdk-go-v2/aws" + "github.com/VITObelgium/fakes3pp/requestutils" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -28,9 +28,9 @@ Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, Run: func(cmd *cobra.Command, args []string) { - signedURI, err := PreSignRequestForGet(bucket, key, time.Now(), expiry) + signedURI, err := PreSignRequestForGet(cliBucket, cliKey, cliRegion, time.Now(), cliExpiry) if err != nil { - fmt.Printf("Encountered erorr %s when trying to creatue url for s3://%s/%s an expiry of %d\n", err, bucket, key, expiry) + fmt.Printf("Encountered erorr %s when trying to creatue url for s3://%s/%s an expiry of %d\n", err, cliBucket, cliKey, cliExpiry) os.Exit(1) } fmt.Println(signedURI) @@ -38,61 +38,60 @@ to quickly create a Cobra application.`, } func checkPresignRequiredFlags() { - err := cobra.MarkFlagRequired(presignCmd.Flags(), bucket) + err := cobra.MarkFlagRequired(presignCmd.Flags(), cliBucket) if err != nil { slog.Debug("Missing required flag", "error", err) } - err = cobra.MarkFlagRequired(presignCmd.Flags(), key) + err = cobra.MarkFlagRequired(presignCmd.Flags(), cliKey) if err != nil { slog.Debug("Missing required flag", "error", err) } } -var bucket string -var key string -var expiry int +var cliBucket string +var cliKey string +var cliExpiry int +var cliRegion string func init() { rootCmd.AddCommand(presignCmd) - presignCmd.Flags().StringVar(&bucket, "bucket", "", "The bucket for which to create a pre-signed URL.") - presignCmd.Flags().StringVar(&key, "key", "", "The key for the object for which to create a pre-signed URL.") - presignCmd.Flags().IntVar(&expiry, "expiry", 600, "The amount of seconds before the URL will expire") + presignCmd.Flags().StringVar(&cliBucket, "bucket", "", "The bucket for which to create a pre-signed URL.") + presignCmd.Flags().StringVar(&cliKey, "key", "", "The key for the object for which to create a pre-signed URL.") + presignCmd.Flags().IntVar(&cliExpiry, "expiry", 600, "The amount of seconds before the URL will expire.") + presignCmd.Flags().StringVar(&cliRegion, "region", "waw3-1", "The default region to be used.") checkPresignRequiredFlags() } -func getServerCreds() aws.Credentials { - accessKey := viper.GetString(awsAccessKeyId) - secretKey := viper.GetString(awsSecretAccessKey) - - return aws.Credentials{ - AccessKeyID: accessKey, - SecretAccessKey: secretKey, - } -} - //Pre-sign the requests with the credentials that are used by the proxy itself -func PreSignRequestWithServerCreds(req *http.Request, exiryInSeconds int, signingTime time.Time) (signedURI string, signedHeaders http.Header, err error){ +func PreSignRequestWithServerCreds(req *http.Request, exiryInSeconds int, signingTime time.Time, defaultRegion string) (signedURI string, signedHeaders http.Header, err error){ ctx := context.Background() + region := requestutils.GetRegionFromRequest(req, defaultRegion) + creds, err := getBackendCredentials(region) + if err != nil { + return + } + return presign.PreSignRequestWithCreds( ctx, req, exiryInSeconds, signingTime, - getServerCreds(), + creds, + region, ) } -func PreSignRequestForGet(bucket, key string, signingTime time.Time, expirySeconds int) (string, error) { +func PreSignRequestForGet(bucket, key, region string, signingTime time.Time, expirySeconds int) (string, error) { url := fmt.Sprintf("https://%s:%d/%s/%s", viper.Get(s3ProxyFQDN), viper.GetInt(s3ProxyPort), bucket, key) req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return "", fmt.Errorf("error when creating a request context for url: %s", err) } - signedURI, _ , err := PreSignRequestWithServerCreds(req, expirySeconds, signingTime) + signedURI, _ , err := PreSignRequestWithServerCreds(req, expirySeconds, signingTime, region) return signedURI, err } \ No newline at end of file diff --git a/cmd/proxys3.go b/cmd/proxys3.go index 53bc349..5b63918 100644 --- a/cmd/proxys3.go +++ b/cmd/proxys3.go @@ -12,7 +12,6 @@ import ( ) const proxys3 = "proxys3" -var s3TargetHost string // proxys3Cmd represents the proxyS3 command var proxys3Cmd = &cobra.Command{ @@ -27,15 +26,20 @@ var proxys3Cmd = &cobra.Command{ if err != nil { panic(err) //Fail hard } - s3TargetHost = viper.GetString(s3ProxyTarget) - if s3TargetHost == "" { - slog.Error("S3 target host not defined") - panic(err) //Fail hard + + if err := initializeGlobalBackendsConfig(); err != nil { + panic(err) //Fail hard as no valid backends are configured } s3Proxy() }, } +func initializeGlobalBackendsConfig() error { + cfg, err := getBackendsConfig() + globalBackendsConfig = cfg + return err +} + func createAndStartS3Proxy(proxyHB handlerBuilderI) (*sync.WaitGroup, *http.Server, error) { s3ProxyDone := &sync.WaitGroup{} s3ProxyDone.Add(1) diff --git a/cmd/proxys3_test.go b/cmd/proxys3_test.go index 26e36f5..ffda73b 100644 --- a/cmd/proxys3_test.go +++ b/cmd/proxys3_test.go @@ -24,17 +24,31 @@ func TestMain(m *testing.M) { m.Run() } +func getTestServerCreds(t *testing.T) aws.Credentials{ + creds, err := getBackendCredentials("waw3-1") + if err != nil { + t.Errorf("Could not get test credentials") + t.FailNow() + } + return creds +} + func TestValidPreSignWithServerCreds(t *testing.T) { //Given valid server config BindEnvVariables("proxys3") + //Pre-sign with server creds so must initialize backend config for testing + if err := initializeGlobalBackendsConfig(); err != nil { + t.Error(err) //Fail hard as no valid backends are configured + t.FailNow() + } //Given we have a valid signed URI valid for 1 second - signedURI, err := PreSignRequestForGet("pvb-test", "onnx_dependencies_1.16.3.zip", time.Now(), 60) + signedURI, err := PreSignRequestForGet("pvb-test", "onnx_dependencies_1.16.3.zip", testDefaultBackendRegion, time.Now(), 60) if err != nil { t.Errorf("could not presign request: %s\n", err) } //When we check the signature within 1 second - isValid, err := presign.IsPresignedUrlWithValidSignature(context.Background(), signedURI, getServerCreds()) + isValid, err := presign.IsPresignedUrlWithValidSignature(context.Background(), signedURI, getTestServerCreds(t)) //Then it is a valid signature if err != nil { t.Errorf("Url should have been valid but %s", err) @@ -68,7 +82,7 @@ func TestValidPreSignWithTempCreds(t *testing.T) { t.Errorf("error when creating a request context for url: %s", err) } - uri, _, err := presign.PreSignRequestWithCreds(context.Background(), req, 100, time.Now(), creds) + uri, _, err := presign.PreSignRequestWithCreds(context.Background(), req, 100, time.Now(), creds, testDefaultBackendRegion) if err != nil { t.Errorf("error when signing request with creds: %s", err) } @@ -88,14 +102,19 @@ func TestValidPreSignWithTempCreds(t *testing.T) { func TestExpiredPreSign(t *testing.T) { //Given valid server config BindEnvVariables("proxys3") + //Pre-sign with server creds so must initialize backend config for testing + if err := initializeGlobalBackendsConfig(); err != nil { + t.Error(err) //Fail hard as no valid backends are configured + t.FailNow() + } //Given we have a valid signed URI valid for 1 second - signedURI, err := PreSignRequestForGet("pvb-test", "onnx_dependencies_1.16.3.zip", time.Now(), 1) + signedURI, err := PreSignRequestForGet("pvb-test", "onnx_dependencies_1.16.3.zip", testDefaultBackendRegion, time.Now(), 1) if err != nil { t.Errorf("could not presign request: %s\n", err) } //When we would check the url after 1 second time.Sleep(1 * time.Second) - isValid, err := presign.IsPresignedUrlWithValidSignature(context.Background(), signedURI, getServerCreds()) + isValid, err := presign.IsPresignedUrlWithValidSignature(context.Background(), signedURI, getTestServerCreds(t)) //Then it is no longer a valid signature TODO check if err != nil { t.Errorf("Url should have been valid but %s", err) @@ -371,7 +390,7 @@ func TestWithValidCredsButProxyHeaders(t *testing.T) { req.Header.Add("User-Agent", "aws-cli/2.15.40 Python/3.11.8 Linux/6.8.0-40-generic exe/x86_64.ubuntu.12 prompt/off command/s3.ls") req.Header.Add("X-Amz-Content-SHA256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") ctx = buildContextWithRequestID(req) - err = presign.SignWithCreds(ctx, req, awsCred) + err = presign.SignWithCreds(ctx, req, awsCred, testDefaultBackendRegion) if err != nil { t.Error(err) t.FailNow() @@ -426,7 +445,7 @@ func TestWithValidCredsButUntrustedHeaders(t *testing.T) { req.Header.Add("User-Agent", "aws-cli/2.15.40 Python/3.11.8 Linux/6.8.0-40-generic exe/x86_64.ubuntu.12 prompt/off command/s3.ls") req.Header.Add("X-Amz-Content-SHA256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") ctx = buildContextWithRequestID(req) - err = presign.SignWithCreds(ctx, req, awsCred) + err = presign.SignWithCreds(ctx, req, awsCred, testDefaultBackendRegion) if err != nil { t.Error(err) t.FailNow() diff --git a/etc/.env b/etc/.env index 26f8514..3504d2f 100644 --- a/etc/.env +++ b/etc/.env @@ -1,5 +1,3 @@ -AWS_ACCESS_KEY_ID=fake_key_id -AWS_SECRET_ACCESS_KEY=fake_secret FAKES3PP_S3_PROXY_FQDN=localhost FAKES3PP_S3_PROXY_PORT=8443 FAKES3PP_S3_PROXY_KEY_FILE=../etc/key.pem @@ -13,4 +11,5 @@ FAKES3PP_STS_PROXY_KEY_FILE=../etc/key.pem FAKES3PP_STS_PROXY_CERT_FILE=../etc/cert.pem FAKES3PP_SECURE="true" FAKES3PP_STS_OIDC_CONFIG="../etc/oidc-config.yaml" +FAKES3PP_S3_BACKEND_CONFIG="../etc/backend-config.yaml" FAKES3PP_ROLE_POLICY_PATH="../etc/policies" \ No newline at end of file diff --git a/etc/.env.docker b/etc/.env.docker index d241af8..cfa36c4 100644 --- a/etc/.env.docker +++ b/etc/.env.docker @@ -1,16 +1,14 @@ -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= FAKES3PP_S3_PROXY_FQDN=localhost FAKES3PP_S3_PROXY_PORT=8443 FAKES3PP_S3_PROXY_KEY_FILE=/etc/fakes3pp/key.pem FAKES3PP_S3_PROXY_CERT_FILE=/etc/fakes3pp/cert.pem FAKES3PP_S3_PROXY_JWT_PRIVATE_RSA_KEY=/etc/fakes3pp/jwt_testing_rsa FAKES3PP_S3_PROXY_JWT_PUBLIC_RSA_KEY=/etc/fakes3pp/jwt_testing_rsa.pub -FAKES3PP_S3_PROXY_TARGET="s3.example.com" FAKES3PP_STS_PROXY_FQDN=localhost FAKES3PP_STS_PROXY_PORT=8444 FAKES3PP_STS_PROXY_KEY_FILE=/etc/fakes3pp/key.pem FAKES3PP_STS_PROXY_CERT_FILE=/etc/fakes3pp/cert.pem FAKES3PP_SECURE="true" FAKES3PP_STS_OIDC_CONFIG="/etc/fakes3pp/oidc-config.yaml" +FAKES3PP_S3_BACKEND_CONFIG="/etc/fakes3pp/backend-config.yaml" FAKES3PP_ROLE_POLICY_PATH="/etc/fakes3pp/policies" \ No newline at end of file diff --git a/etc/backend-config.yaml b/etc/backend-config.yaml new file mode 100644 index 0000000..d2664cf --- /dev/null +++ b/etc/backend-config.yaml @@ -0,0 +1,22 @@ +# This file contains the configurations of all the proxied backends. +# In order to allow proxying multiple backends the region name of the backend +# is used to distinguish between configuration (see https://github.com/VITObelgium/fakes3pp/issues/3) +s3backends: + # A mapping of the region name to the details required to use the backend: + # * Credentials could over time be provided in different ways. Available are: + # * A yaml file that contains a map with aws_access_key_id, aws_secret_access_key and optionally aws_session_token + # given that the contents of this file is sensitive it should be exclusive to the user running the s3proxy + # * endpoint should be protocol and hostname of how to reach the backend S3 API + - region: waw3-1 + credentials: + file: ../etc/creds/cfc_creds.yaml + endpoint: https://s3.waw3-1.cloudferro.com + - region: eu-nl + credentials: + file: ../etc/creds/otc_creds.yaml + endpoint: https://obs.eu-nl.otc.t-systems.com +# The default backend is the backend that will be used if the system cannot determine which backend is intended. +# When using multiple backends sigv4 presigned URLs should be preferred as they do contain the region in the +# X-Amz-Credential query parameter. Normal requests are also signed with sigv4 and have the region. +# The only known case where we need this are presigned hmacv1 query URLs since those do not specify the region. +default: waw3-1 \ No newline at end of file diff --git a/etc/creds/cfc_creds.yaml b/etc/creds/cfc_creds.yaml new file mode 100644 index 0000000..323f97d --- /dev/null +++ b/etc/creds/cfc_creds.yaml @@ -0,0 +1,2 @@ +aws_access_key_id: fake_key_id +aws_secret_access_key: fake_secret \ No newline at end of file diff --git a/etc/creds/otc_creds.yaml b/etc/creds/otc_creds.yaml new file mode 100644 index 0000000..73e01af --- /dev/null +++ b/etc/creds/otc_creds.yaml @@ -0,0 +1,3 @@ +aws_access_key_id: fake_key_id_otc +aws_secret_access_key: fake_secret_otc +aws_session_token: fakeSessionTokOtc1 \ No newline at end of file diff --git a/presign/s3v4.go b/presign/s3v4.go index 6b5dcc5..92146a6 100644 --- a/presign/s3v4.go +++ b/presign/s3v4.go @@ -16,20 +16,20 @@ import ( //This file just contains helpers to presign for S3 with sigv4 -func PreSignRequestWithCreds(ctx context.Context, req *http.Request, expiryInSeconds int, signingTime time.Time, creds aws.Credentials) (signedURI string, signedHeaders http.Header, err error){ +func PreSignRequestWithCreds(ctx context.Context, req *http.Request, expiryInSeconds int, signingTime time.Time, creds aws.Credentials, defaultRegion string) (signedURI string, signedHeaders http.Header, err error){ if expiryInSeconds <= 0 { return "", nil, errors.New("expiryInSeconds must be bigger than 0 for presigned requests") } signer := v4.NewSigner() - ctx, creds, req, payloadHash, service, region, signingTime := GetS3SignRequestParams(ctx, req, expiryInSeconds, signingTime, creds) + ctx, creds, req, payloadHash, service, region, signingTime := GetS3SignRequestParams(ctx, req, expiryInSeconds, signingTime, creds, defaultRegion) return signer.PresignHTTP(ctx, creds, req, payloadHash, service, region, signingTime) } -func SignRequestWithCreds(ctx context.Context, req *http.Request, expiryInSeconds int, signingTime time.Time, creds aws.Credentials) (err error){ +func SignRequestWithCreds(ctx context.Context, req *http.Request, expiryInSeconds int, signingTime time.Time, creds aws.Credentials, defaultRegion string) (err error){ signer := v4.NewSigner() - ctx, creds, req, payloadHash, service, region, signingTime := GetS3SignRequestParams(ctx, req, expiryInSeconds, signingTime, creds) + ctx, creds, req, payloadHash, service, region, signingTime := GetS3SignRequestParams(ctx, req, expiryInSeconds, signingTime, creds, defaultRegion) return signer.SignHTTP(ctx, creds, req, payloadHash, service, region, signingTime) } @@ -44,8 +44,9 @@ var signatureQueryParamNames []string = []string{ //Sign an HTTP request with a sigv4 signature. If expiry in seconds is bigger than zero then the signature has an explicit limited lifetime //use a negative value to not set an explicit expiry time -func GetS3SignRequestParams(ctx context.Context, req *http.Request, expiryInSeconds int, signingTime time.Time, creds aws.Credentials) (context.Context, aws.Credentials, *http.Request, string, string, string, time.Time){ - region := "eu-west-1" +//The requests gets checked to determine the region but if the request does not specify it the defaultRegion aruement will be used as fallback +func GetS3SignRequestParams(ctx context.Context, req *http.Request, expiryInSeconds int, signingTime time.Time, creds aws.Credentials, defaultRegion string) (context.Context, aws.Credentials, *http.Request, string, string, string, time.Time){ + region := defaultRegion regionName, err := requestutils.GetSignatureCredentialPartFromRequest(req, requestutils.CredentialPartRegionName) if err == nil { region = regionName @@ -73,7 +74,7 @@ func GetS3SignRequestParams(ctx context.Context, req *http.Request, expiryInSeco } -func SignWithCreds(ctx context.Context, req *http.Request, creds aws.Credentials) error{ +func SignWithCreds(ctx context.Context, req *http.Request, creds aws.Credentials, defaultRegion string) error{ var signingTime time.Time amzDate := req.Header.Get(constants.AmzDateKey) if amzDate == "" { @@ -87,5 +88,5 @@ func SignWithCreds(ctx context.Context, req *http.Request, creds aws.Credentials } } - return SignRequestWithCreds(ctx, req, -1, signingTime, creds) + return SignRequestWithCreds(ctx, req, -1, signingTime, creds, defaultRegion) } diff --git a/presign/s3v4query.go b/presign/s3v4query.go index 03f1bbd..eacd5dc 100644 --- a/presign/s3v4query.go +++ b/presign/s3v4query.go @@ -90,7 +90,8 @@ func (u presignedUrlS3V4Query) GetPresignedUrlDetails(ctx context.Context, deriv c.Header.Add("Host", c.Host) } CleanHeadersTo(ctx, c, u.getSignedHeaders()) - signedUri, _, err := PreSignRequestWithCreds(ctx, c, expirySeconds, signDate, creds) + defaultRegion := "" // A Sigv4 always has a region specified as part of the X-amz-credentials parameter so no fallback needed. + signedUri, _, err := PreSignRequestWithCreds(ctx, c, expirySeconds, signDate, creds, defaultRegion) if err != nil { err = fmt.Errorf("InvalidSignature: encountered error trying to sign a similar req: %s", err) return diff --git a/presign/s3v4query_test.go b/presign/s3v4query_test.go index 2eb211b..a9e27d4 100644 --- a/presign/s3v4query_test.go +++ b/presign/s3v4query_test.go @@ -68,7 +68,7 @@ func TestAwsCliGeneratedURLMustWork(t *testing.T) { t.FailNow() } - signedUri, _, err := PreSignRequestWithCreds(ctx, req, testExpirySeconds, signDate, creds) + signedUri, _, err := PreSignRequestWithCreds(ctx, req, testExpirySeconds, signDate, creds, "eu-west-1") if err != nil { t.Errorf("Did not expect error. Got %s", err) } diff --git a/requestutils/amz-credential-value.go b/requestutils/amz-credential-value.go index 68012eb..f9eb7a6 100644 --- a/requestutils/amz-credential-value.go +++ b/requestutils/amz-credential-value.go @@ -59,6 +59,16 @@ func GetSignatureCredentialPartFromRequest(r *http.Request, credentialPart Crede return GetCredentialPart(credentialString, credentialPart) } +//Get region name from a request and return fallback if the information is not in the request +func GetRegionFromRequest(req *http.Request, fallback string) (region string) { + region = fallback + regionName, err := GetSignatureCredentialPartFromRequest(req, CredentialPartRegionName) + if err == nil { + region = regionName + } + return +} + // Gets a part of the Credential value that is passed via the authorization header func getSignatureCredentialStringFromRequestAuthHeader(authorizationHeader string) (string, error) { if authorizationHeader == "" {