diff --git a/backend.go b/backend.go index bb5cc5e..f5334b1 100644 --- a/backend.go +++ b/backend.go @@ -1,7 +1,6 @@ package gofakes3 import ( - "encoding/hex" "io" "time" @@ -120,17 +119,6 @@ func (p ListBucketPage) IsEmpty() bool { return p == ListBucketPage{} } -type PutObjectResult struct { - // If versioning is enabled on the bucket, this should be set to the - // created version ID. If versioning is not enabled, this should be - // empty. - VersionID VersionID - - // ETag is the value of the ETag header returned by the backend, stripped of - // its quotation marks. - ETag string -} - // Backend provides a set of operations to be implemented in order to support // gofakes3. // @@ -324,13 +312,13 @@ type VersionedBackend interface { // gets finalised and pushed to the backend. type MultipartBackend interface { CreateMultipartUpload(bucket, object string, meta map[string]string) (UploadID, error) - UploadPart(bucket, object string, id UploadID, partNumber int, contentLength int64, input io.Reader) (etag string, err error) + UploadPart(bucket, object string, id UploadID, partNumber int, contentLength int64, input io.Reader) (*UploadPartResult, error) ListMultipartUploads(bucket string, marker *UploadListMarker, prefix Prefix, limit int64) (*ListMultipartUploadsResult, error) ListParts(bucket, object string, uploadID UploadID, marker int, limit int64) (*ListMultipartUploadPartsResult, error) AbortMultipartUpload(bucket, object string, id UploadID) error - CompleteMultipartUpload(bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (versionID VersionID, etag string, err error) + CompleteMultipartUpload(bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (*CompleteMultipartResult, error) } // CopyObject is a helper function useful for quickly implementing CopyObject on @@ -343,13 +331,13 @@ func CopyObject(db Backend, srcBucket, srcKey, dstBucket, dstKey string, meta ma } defer c.Contents.Close() - _, err = db.PutObject(dstBucket, dstKey, meta, c.Contents, c.Size) + res, err := db.PutObject(dstBucket, dstKey, meta, c.Contents, c.Size) if err != nil { return } return CopyObjectResult{ - ETag: `"` + hex.EncodeToString(c.Hash) + `"`, + ETag: res.ETag, LastModified: NewContentTime(time.Now()), }, nil } diff --git a/gofakes3.go b/gofakes3.go index b4d7468..8fe0456 100644 --- a/gofakes3.go +++ b/gofakes3.go @@ -663,7 +663,11 @@ func (g *GoFakeS3) createObjectBrowserUpload(bucket string, w http.ResponseWrite w.Header().Set("x-amz-version-id", string(result.VersionID)) } - w.Header().Set("ETag", `"`+hex.EncodeToString(rdr.Sum(nil))+`"`) + etag := result.ETag + if etag == "" { + etag = formatETag(hex.EncodeToString(rdr.Sum(nil))) + } + w.Header().Set("ETag", etag) return nil } @@ -728,9 +732,9 @@ func (g *GoFakeS3) createObject(bucket, object string, w http.ResponseWriter, r etag := result.ETag if etag == "" { - etag = hex.EncodeToString(rdr.Sum(nil)) + etag = formatETag(hex.EncodeToString(rdr.Sum(nil))) } - w.Header().Set("ETag", fmt.Sprintf("\"%s\"", etag)) + w.Header().Set("ETag", etag) return nil } @@ -953,7 +957,7 @@ func (g *GoFakeS3) putMultipartUploadPart(bucket, object string, uploadID Upload return err } - w.Header().Add("ETag", etag) + w.Header().Add("ETag", formatETag(etag)) return nil } @@ -974,17 +978,17 @@ func (g *GoFakeS3) completeMultipartUpload(bucket, object string, uploadID Uploa return err } - versionID, etag, err := g.uploader.CompleteMultipartUpload(bucket, object, uploadID, &in) + res, err := g.uploader.CompleteMultipartUpload(bucket, object, uploadID, &in) if err != nil { return err } - if versionID != "" { - w.Header().Set("x-amz-version-id", string(versionID)) + if res.VersionID != "" { + w.Header().Set("x-amz-version-id", string(res.VersionID)) } return g.xmlEncoder(w).Encode(&CompleteMultipartUploadResult{ - ETag: etag, + ETag: res.ETag, Bucket: bucket, Key: object, }) @@ -1210,3 +1214,7 @@ func listBucketVersionsPageFromQuery(query url.Values) (page ListBucketVersionsP return page, nil } + +func formatETag(etag string) string { + return fmt.Sprintf("\"%s\"", etag) +} diff --git a/messages.go b/messages.go index 17138fe..6d51a2f 100644 --- a/messages.go +++ b/messages.go @@ -365,6 +365,20 @@ func (b *ListBucketVersionsResult) AddPrefix(prefix string) { b.CommonPrefixes = append(b.CommonPrefixes, CommonPrefix{Prefix: prefix}) } +type UploadPartResult struct { + ETag string `xml:"ETag,omitempty"` +} + +type CompleteMultipartResult struct { + // If versioning is enabled on the bucket, this should be set to the + // created version ID. If versioning is not enabled, this should be + // empty. + VersionID VersionID `xml:"VersionId,omitempty"` + + // ETag is the value of the ETag header returned by the backend. + ETag string `xml:"ETag,omitempty"` +} + type ListMultipartUploadsResult struct { Bucket string `xml:"Bucket"` @@ -430,6 +444,17 @@ type ListMultipartUploadPartItem struct { Size int64 `xml:"Size"` } +// PutObjectResult contains the response from a PutObject operation. +type PutObjectResult struct { + // If versioning is enabled on the bucket, this should be set to the + // created version ID. If versioning is not enabled, this should be + // empty. + VersionID VersionID `xml:"VersionId,omitempty"` + + // ETag is the value of the ETag header returned by the backend. + ETag string `xml:"ETag,omitempty"` +} + // CopyObjectResult contains the response from a CopyObject operation. type CopyObjectResult struct { XMLName xml.Name `xml:"CopyObjectResult"` diff --git a/uploader.go b/uploader.go index 48b3af2..c77cb85 100644 --- a/uploader.go +++ b/uploader.go @@ -367,17 +367,17 @@ func (u *uploader) AbortMultipartUpload(bucket, object string, id UploadID) erro return nil } -func (u *uploader) UploadPart(bucket, object string, id UploadID, partNumber int, contentLength int64, input io.Reader) (etag string, err error) { +func (u *uploader) UploadPart(bucket, object string, id UploadID, partNumber int, contentLength int64, input io.Reader) (*UploadPartResult, error) { body, err := io.ReadAll(input) if err != nil { - return "", err + return nil, err } if len(body) != int(contentLength) { - return "", ErrIncompleteBody + return nil, ErrIncompleteBody } mpu, err := u.getUnlocked(bucket, object, id) if err != nil { - return "", err + return nil, err } mpu.mu.Lock() @@ -387,7 +387,7 @@ func (u *uploader) UploadPart(bucket, object string, id UploadID, partNumber int // from guaranteed unique input: hash := md5.New() hash.Write([]byte(body)) - etag = fmt.Sprintf(`"%s"`, hex.EncodeToString(hash.Sum(nil))) + etag := fmt.Sprintf(`"%s"`, hex.EncodeToString(hash.Sum(nil))) part := multipartUploadPart{ PartNumber: partNumber, @@ -399,13 +399,14 @@ func (u *uploader) UploadPart(bucket, object string, id UploadID, partNumber int mpu.parts = append(mpu.parts, make([]*multipartUploadPart, partNumber-len(mpu.parts)+1)...) } mpu.parts[partNumber] = &part - return etag, nil + + return &UploadPartResult{ETag: formatETag(etag)}, nil } -func (u *uploader) CompleteMultipartUpload(bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (version VersionID, etag string, err error) { +func (u *uploader) CompleteMultipartUpload(bucket, object string, id UploadID, input *CompleteMultipartUploadRequest) (*CompleteMultipartResult, error) { mpu, err := u.getUnlocked(bucket, object, id) if err != nil { - return "", "", err + return nil, err } mpu.mu.Lock() @@ -417,23 +418,23 @@ func (u *uploader) CompleteMultipartUpload(bucket, object string, id UploadID, i // end up uploading more parts than you need to assemble, so it should // probably just ignore that? if len(input.Parts) > mpuPartsLen { - return "", "", ErrInvalidPart + return nil, ErrInvalidPart } if !input.partsAreSorted() { - return "", "", ErrInvalidPartOrder + return nil, ErrInvalidPartOrder } var size int64 for _, inPart := range input.Parts { if inPart.PartNumber >= mpuPartsLen || mpu.parts[inPart.PartNumber] == nil { - return "", "", ErrorMessagef(ErrInvalidPart, "unexpected part number %d in complete request", inPart.PartNumber) + return nil, ErrorMessagef(ErrInvalidPart, "unexpected part number %d in complete request", inPart.PartNumber) } upPart := mpu.parts[inPart.PartNumber] if strings.Trim(inPart.ETag, "\"") != strings.Trim(upPart.ETag, "\"") { - return "", "", ErrorMessagef(ErrInvalidPart, "unexpected part etag for number %d in complete request", inPart.PartNumber) + return nil, ErrorMessagef(ErrInvalidPart, "unexpected part etag for number %d in complete request", inPart.PartNumber) } size += int64(len(upPart.Body)) @@ -444,16 +445,22 @@ func (u *uploader) CompleteMultipartUpload(bucket, object string, id UploadID, i body = append(body, mpu.parts[part.PartNumber].Body...) } - hash := fmt.Sprintf("%x", md5.Sum(body)) - result, err := u.storage.PutObject(bucket, object, mpu.Meta, bytes.NewReader(body), int64(len(body))) if err != nil { - return "", "", err + return nil, err + } + + etag := result.ETag + if etag == "" { + etag = formatETag(fmt.Sprintf("%x", md5.Sum(body))) } // if getUnlocked succeeded, so will this: u.buckets[bucket].remove(id) - return result.VersionID, hash, nil + return &CompleteMultipartResult{ + VersionID: result.VersionID, + ETag: etag, + }, nil } func (u *uploader) getUnlocked(bucket, object string, id UploadID) (mu *multipartUpload, err error) {