diff --git a/backend.go b/backend.go index 582997e..b72f604 100644 --- a/backend.go +++ b/backend.go @@ -2,7 +2,6 @@ package gofakes3 import ( "context" - "encoding/hex" "io" "net/http" "time" @@ -122,13 +121,6 @@ func (p ListBucketPage) IsEmpty() bool { return p == ListBucketPage{} } -type PutObjectResult struct { - // If versioning is enabled on the bucket, this should be set to the - // created version ID. If versioning is not enabled, this should be - // empty. - VersionID VersionID -} - // Backend provides a set of operations to be implemented in order to support // gofakes3. // @@ -323,13 +315,13 @@ type VersionedBackend interface { // gets finalised and pushed to the backend. type MultipartBackend interface { CreateMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string) (UploadID, error) - UploadPart(ctx context.Context, bucket, object string, id UploadID, partNumber int, contentLength int64, input io.Reader) (etag string, err error) + UploadPart(ctx context.Context, bucket, object string, id UploadID, partNumber int, contentLength int64, input io.Reader) (*UploadPartResult, error) ListMultipartUploads(ctx context.Context, bucket string, marker *UploadListMarker, prefix Prefix, limit int64) (*ListMultipartUploadsResult, error) ListParts(ctx context.Context, bucket, object string, uploadID UploadID, marker int, limit int64) (*ListMultipartUploadPartsResult, error) AbortMultipartUpload(ctx context.Context, bucket, object string, id UploadID) error - CompleteMultipartUpload(ctx context.Context, bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (versionID VersionID, etag string, err error) + CompleteMultipartUpload(ctx context.Context, bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (*CompleteMultipartUploadResult, error) } type AuthenticatedBackend interface { @@ -346,13 +338,13 @@ func CopyObject(ctx context.Context, db Backend, srcBucket, srcKey, dstBucket, d } defer c.Contents.Close() - _, err = db.PutObject(ctx, dstBucket, dstKey, meta, c.Contents, c.Size) + res, err := db.PutObject(ctx, dstBucket, dstKey, meta, c.Contents, c.Size) if err != nil { return } return CopyObjectResult{ - ETag: `"` + hex.EncodeToString(c.Hash) + `"`, + ETag: res.ETag, LastModified: NewContentTime(time.Now()), }, nil } diff --git a/backend/s3mem/backend.go b/backend/s3mem/backend.go index d1565b8..27f557f 100644 --- a/backend/s3mem/backend.go +++ b/backend/s3mem/backend.go @@ -258,6 +258,7 @@ func (db *Backend) PutObject(ctx context.Context, bucketName, objectName string, result.VersionID = item.versionID } + result.ETag = item.etag return result, nil } diff --git a/gofakes3.go b/gofakes3.go index 573e68c..2bbea67 100644 --- a/gofakes3.go +++ b/gofakes3.go @@ -650,7 +650,11 @@ func (g *GoFakeS3) createObjectBrowserUpload(bucket string, w http.ResponseWrite w.Header().Set("x-amz-version-id", string(result.VersionID)) } - w.Header().Set("ETag", `"`+hex.EncodeToString(rdr.Sum(nil))+`"`) + etag := result.ETag + if etag == "" { + etag = formatETag(hex.EncodeToString(rdr.Sum(nil))) + } + w.Header().Set("ETag", etag) return nil } @@ -712,7 +716,12 @@ func (g *GoFakeS3) createObject(bucket, object string, w http.ResponseWriter, r g.log.Print(LogInfo, "CREATED VERSION:", bucket, object, result.VersionID) w.Header().Set("x-amz-version-id", string(result.VersionID)) } - w.Header().Set("ETag", `"`+hex.EncodeToString(rdr.Sum(nil))+`"`) + + etag := result.ETag + if etag == "" { + etag = formatETag(hex.EncodeToString(rdr.Sum(nil))) + } + w.Header().Set("ETag", etag) return nil } @@ -767,6 +776,12 @@ func (g *GoFakeS3) copyObject(bucket, object string, meta map[string]string, w h w.Header().Set("x-amz-version-id", string(srcObj.VersionID)) } + etag := result.ETag + if etag == "" { + etag = formatETag(hex.EncodeToString(srcObj.Hash)) + } + w.Header().Set("ETag", etag) + return g.xmlEncoder(w).Encode(result) } @@ -927,12 +942,12 @@ func (g *GoFakeS3) putMultipartUploadPart(bucket, object string, uploadID Upload } } - etag, err := g.uploader.UploadPart(r.Context(), bucket, object, uploadID, int(partNumber), r.ContentLength, rdr) + res, err := g.uploader.UploadPart(r.Context(), bucket, object, uploadID, int(partNumber), r.ContentLength, rdr) if err != nil { return err } - w.Header().Add("ETag", etag) + w.Header().Add("ETag", res.ETag) return nil } @@ -953,17 +968,17 @@ func (g *GoFakeS3) completeMultipartUpload(bucket, object string, uploadID Uploa return err } - versionID, etag, err := g.uploader.CompleteMultipartUpload(r.Context(), bucket, object, uploadID, &in) + res, err := g.uploader.CompleteMultipartUpload(r.Context(), bucket, object, uploadID, &in) if err != nil { return err } - if versionID != "" { - w.Header().Set("x-amz-version-id", string(versionID)) + if res.VersionID != "" { + w.Header().Set("x-amz-version-id", string(res.VersionID)) } - return g.xmlEncoder(w).Encode(&CompleteMultipartUploadResult{ - ETag: etag, + return g.xmlEncoder(w).Encode(&CompleteMultipartUploadResponse{ + ETag: res.ETag, Bucket: bucket, Key: object, }) @@ -1189,3 +1204,7 @@ func listBucketVersionsPageFromQuery(query url.Values) (page ListBucketVersionsP return page, nil } + +func formatETag(etag string) string { + return fmt.Sprintf("\"%s\"", etag) +} diff --git a/messages.go b/messages.go index 17138fe..1075421 100644 --- a/messages.go +++ b/messages.go @@ -69,7 +69,7 @@ func (c CompleteMultipartUploadRequest) partIDs() []int { return inParts } -type CompleteMultipartUploadResult struct { +type CompleteMultipartUploadResponse struct { Location string `xml:"Location"` Bucket string `xml:"Bucket"` Key string `xml:"Key"` @@ -365,6 +365,20 @@ func (b *ListBucketVersionsResult) AddPrefix(prefix string) { b.CommonPrefixes = append(b.CommonPrefixes, CommonPrefix{Prefix: prefix}) } +type UploadPartResult struct { + ETag string `xml:"ETag,omitempty"` +} + +type CompleteMultipartUploadResult struct { + // If versioning is enabled on the bucket, this should be set to the + // created version ID. If versioning is not enabled, this should be + // empty. + VersionID VersionID `xml:"VersionId,omitempty"` + + // ETag is the value of the ETag header returned by the backend. + ETag string `xml:"ETag,omitempty"` +} + type ListMultipartUploadsResult struct { Bucket string `xml:"Bucket"` @@ -430,6 +444,17 @@ type ListMultipartUploadPartItem struct { Size int64 `xml:"Size"` } +// PutObjectResult contains the response from a PutObject operation. +type PutObjectResult struct { + // If versioning is enabled on the bucket, this should be set to the + // created version ID. If versioning is not enabled, this should be + // empty. + VersionID VersionID `xml:"VersionId,omitempty"` + + // ETag is the value of the ETag header returned by the backend. + ETag string `xml:"ETag,omitempty"` +} + // CopyObjectResult contains the response from a CopyObject operation. type CopyObjectResult struct { XMLName xml.Name `xml:"CopyObjectResult"` diff --git a/uploader.go b/uploader.go index 63af0a1..c4f4718 100644 --- a/uploader.go +++ b/uploader.go @@ -368,17 +368,17 @@ func (u *uploader) AbortMultipartUpload(_ context.Context, bucket, object string return nil } -func (u *uploader) UploadPart(_ context.Context, bucket, object string, id UploadID, partNumber int, contentLength int64, input io.Reader) (etag string, err error) { +func (u *uploader) UploadPart(_ context.Context, bucket, object string, id UploadID, partNumber int, contentLength int64, input io.Reader) (*UploadPartResult, error) { body, err := io.ReadAll(input) if err != nil { - return "", err + return nil, err } if len(body) != int(contentLength) { - return "", ErrIncompleteBody + return nil, ErrIncompleteBody } mpu, err := u.getUnlocked(bucket, object, id) if err != nil { - return "", err + return nil, err } mpu.mu.Lock() @@ -388,7 +388,7 @@ func (u *uploader) UploadPart(_ context.Context, bucket, object string, id Uploa // from guaranteed unique input: hash := md5.New() hash.Write([]byte(body)) - etag = fmt.Sprintf(`"%s"`, hex.EncodeToString(hash.Sum(nil))) + etag := formatETag(hex.EncodeToString(hash.Sum(nil))) part := multipartUploadPart{ PartNumber: partNumber, @@ -400,13 +400,14 @@ func (u *uploader) UploadPart(_ context.Context, bucket, object string, id Uploa mpu.parts = append(mpu.parts, make([]*multipartUploadPart, partNumber-len(mpu.parts)+1)...) } mpu.parts[partNumber] = &part - return etag, nil + + return &UploadPartResult{ETag: etag}, nil } -func (u *uploader) CompleteMultipartUpload(ctx context.Context, bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (version VersionID, etag string, err error) { +func (u *uploader) CompleteMultipartUpload(ctx context.Context, bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (*CompleteMultipartUploadResult, error) { mpu, err := u.getUnlocked(bucket, object, id) if err != nil { - return "", "", err + return nil, err } mpu.mu.Lock() @@ -418,23 +419,23 @@ func (u *uploader) CompleteMultipartUpload(ctx context.Context, bucket, object s // end up uploading more parts than you need to assemble, so it should // probably just ignore that? if len(input.Parts) > mpuPartsLen { - return "", "", ErrInvalidPart + return nil, ErrInvalidPart } if !input.partsAreSorted() { - return "", "", ErrInvalidPartOrder + return nil, ErrInvalidPartOrder } var size int64 for _, inPart := range input.Parts { if inPart.PartNumber >= mpuPartsLen || mpu.parts[inPart.PartNumber] == nil { - return "", "", ErrorMessagef(ErrInvalidPart, "unexpected part number %d in complete request", inPart.PartNumber) + return nil, ErrorMessagef(ErrInvalidPart, "unexpected part number %d in complete request", inPart.PartNumber) } upPart := mpu.parts[inPart.PartNumber] if strings.Trim(inPart.ETag, "\"") != strings.Trim(upPart.ETag, "\"") { - return "", "", ErrorMessagef(ErrInvalidPart, "unexpected part etag for number %d in complete request", inPart.PartNumber) + return nil, ErrorMessagef(ErrInvalidPart, "unexpected part etag for number %d in complete request", inPart.PartNumber) } size += int64(len(upPart.Body)) @@ -445,16 +446,22 @@ func (u *uploader) CompleteMultipartUpload(ctx context.Context, bucket, object s body = append(body, mpu.parts[part.PartNumber].Body...) } - hash := fmt.Sprintf("%x", md5.Sum(body)) - result, err := u.storage.PutObject(ctx, bucket, object, mpu.Meta, bytes.NewReader(body), int64(len(body))) if err != nil { - return "", "", err + return nil, err + } + + etag := result.ETag + if etag == "" { + etag = formatETag(fmt.Sprintf("%x", md5.Sum(body))) } // if getUnlocked succeeded, so will this: u.buckets[bucket].remove(id) - return result.VersionID, hash, nil + return &CompleteMultipartUploadResult{ + VersionID: result.VersionID, + ETag: etag, + }, nil } func (u *uploader) getUnlocked(bucket, object string, id UploadID) (mu *multipartUpload, err error) {