From c1c47b5037fbfa1c10ea07c3918f90325ef34960 Mon Sep 17 00:00:00 2001 From: Pablo Chacin Date: Mon, 17 Jun 2024 11:20:29 +0200 Subject: [PATCH] retrive object if already exists in cache Signed-off-by: Pablo Chacin --- cache.go | 75 ++++++++++++++++++++++++++++++++++++++------------- cache_test.go | 67 +++++++++++++++++++++++++++++++++++++++++++-- go.mod | 2 +- go.sum | 4 +-- service.go | 18 ++++++++++++- 5 files changed, 142 insertions(+), 24 deletions(-) diff --git a/cache.go b/cache.go index adc1ad8..6e07d66 100644 --- a/cache.go +++ b/cache.go @@ -4,12 +4,20 @@ import ( "bytes" "context" "crypto/sha256" + "errors" "fmt" "io" "os" "path/filepath" ) +var ( + ErrObjectNotFound = errors.New("object not found") + ErrAccessingObject = errors.New("accessing object") + ErrCreatingObject = errors.New("creating object") + ErrInitializingCache = errors.New("initializing cache") +) + // Object represents an object stored in the Cache // TODO: add metadata (e.g creation data, size) type Object struct { @@ -21,6 +29,8 @@ type Object struct { // Cache defines an interface for storing blobs type Cache interface { + // Get retrieves an objects if exists in the cache or an error otherwise + Get(ctx context.Context, id string) (Object, error) // Store stores the object and returns the metadata Store(ctx context.Context, id string, content io.Reader) (Object, error) } @@ -31,23 +41,14 @@ type fileObjectStore struct { } func NewTempFileCache() (Cache, error) { - cacheDir, err := os.MkdirTemp(os.TempDir(), "buildcache*") - if err != nil { - return nil, fmt.Errorf("creating cache directory %w", err) - } - - return NewFileCache(cacheDir) + return NewFileCache(filepath.Join(os.TempDir(), "buildcache")) } // NewFileCache creates an cached backed by a directory func NewFileCache(path string) (Cache, error) { - fileInfo, err := os.Stat(path) + err := os.MkdirAll(path, 0o777) if err != nil { - return nil, fmt.Errorf("invalid file path: %w", err) - } - - if !fileInfo.IsDir() { - return nil, fmt.Errorf("must be a directory: %s", path) + return nil, fmt.Errorf("%w: %w", ErrInitializingCache, err) } return &fileObjectStore{ @@ -56,9 +57,16 @@ func NewFileCache(path string) (Cache, error) { } func (f *fileObjectStore) Store(ctx context.Context, id string, content io.Reader) (Object, error) { - objectFile, err := os.Create(filepath.Join(f.path, id)) + objectDir := filepath.Join(f.path, id) + // TODO: check permissions + err := os.MkdirAll(objectDir, 0o777) if err != nil { - return Object{}, fmt.Errorf("creating object %w", err) + return Object{}, fmt.Errorf("%w: %w", ErrCreatingObject, err) + } + + objectFile, err := os.Create(filepath.Join(objectDir, "data")) + if err != nil { + return Object{}, fmt.Errorf("%w: %w", ErrCreatingObject, err) } // write content to object file and copy to buffer to calculate checksum @@ -66,16 +74,47 @@ func (f *fileObjectStore) Store(ctx context.Context, id string, content io.Reade buff := bytes.Buffer{} _, err = io.Copy(objectFile, io.TeeReader(content, &buff)) if err != nil { - return Object{}, fmt.Errorf("creating object %w", err) + return Object{}, fmt.Errorf("%w: %w", ErrCreatingObject, err) } // calculate checksum - checksum := sha256.New() - checksum.Sum(buff.Bytes()) + checksumHash := sha256.New() + checksumHash.Sum(buff.Bytes()) + checksum := fmt.Sprintf("%x", checksumHash.Sum(nil)) + + // write metadata + err = os.WriteFile(filepath.Join(objectDir, "checksum"), []byte(checksum), 0o644) + if err != nil { + return Object{}, fmt.Errorf("%w: %w", ErrCreatingObject, err) + } return Object{ ID: id, - Checksum: string(fmt.Sprintf("%x", checksum.Sum(nil))), + Checksum: checksum, URL: fmt.Sprintf("file://%s", objectFile.Name()), }, nil } + +func (f *fileObjectStore) Get(ctx context.Context, id string) (Object, error) { + objectDir := filepath.Join(f.path, id) + _, err := os.Stat(objectDir) + + if errors.Is(err, os.ErrNotExist) { + return Object{}, fmt.Errorf("%w: %s", ErrObjectNotFound, id) + } + + if err != nil { + return Object{}, fmt.Errorf("%w: %w", ErrAccessingObject, err) + } + + checksum, err := os.ReadFile(filepath.Join(objectDir, "checksum")) + if err != nil { + return Object{}, fmt.Errorf("%w: %w", ErrAccessingObject, err) + } + + return Object{ + ID: id, + Checksum: string(checksum), + URL: fmt.Sprintf("file://%s", filepath.Join(objectDir, "data")), + }, nil +} diff --git a/cache_test.go b/cache_test.go index be11bbf..0462555 100644 --- a/cache_test.go +++ b/cache_test.go @@ -9,12 +9,13 @@ import ( "testing" ) -func TestFileCache(t *testing.T) { +func TestCreateObject(t *testing.T) { t.Parallel() testCases := []struct { title string content []byte + id string expectErr error }{ { @@ -38,7 +39,7 @@ func TestFileCache(t *testing.T) { obj, err := cache.Store(context.TODO(), "object", bytes.NewBuffer(tc.content)) if !errors.Is(err, tc.expectErr) { - t.Fatalf("expected %v got %v", tc, err) + t.Fatalf("expected %v got %v", tc.expectErr, err) } fileUrl, err := url.Parse(obj.URL) @@ -57,3 +58,65 @@ func TestFileCache(t *testing.T) { }) } } + +func TestGetObjectCache(t *testing.T) { + t.Parallel() + + testCases := []struct { + title string + id string + expectErr error + }{ + { + title: "retrieve existing", + id: "object", + expectErr: nil, + }, + { + title: "retrieve non existing object", + id: "object2", + expectErr: ErrObjectNotFound, + }, + } + + cache, err := NewFileCache(t.TempDir()) + if err != nil { + t.Fatalf("test setup %v", err) + } + + content := []byte("content") + _, err = cache.Store(context.TODO(), "object", bytes.NewBuffer(content)) + if err != nil { + t.Fatalf("test setup %v", err) + } + + for _, tc := range testCases { + t.Run(tc.title, func(t *testing.T) { + t.Parallel() + + obj, err := cache.Get(context.TODO(), tc.id) + if !errors.Is(err, tc.expectErr) { + t.Fatalf("expected %v got %v", tc.expectErr, err) + } + + // if expected error, don't check returned object + if tc.expectErr != nil { + return + } + + fileUrl, err := url.Parse(obj.URL) + if err != nil { + t.Fatalf("invalid url %v", err) + } + + data, err := os.ReadFile(fileUrl.Path) + if err != nil { + t.Fatalf("reading object url %v", err) + } + + if !bytes.Equal(data, content) { + t.Fatalf("expected %v got %v", data, content) + } + }) + } +} diff --git a/go.mod b/go.mod index 58f4bcd..fb61777 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/grafana/k6build go 1.22.2 require ( - github.com/grafana/k6catalog v0.0.0-20240614130953-c7b8fc289822 + github.com/grafana/k6catalog v0.1.0 github.com/grafana/k6foundry v0.1.0 github.com/spf13/cobra v1.8.0 golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 diff --git a/go.sum b/go.sum index 143ddec..ddc799b 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/grafana/k6catalog v0.0.0-20240614130953-c7b8fc289822 h1:O2+6uF5Z4V/UsIR8KI03gYGJHWqLHPu+yIFXtbhBm64= -github.com/grafana/k6catalog v0.0.0-20240614130953-c7b8fc289822/go.mod h1:8R9eXAh2nb69+drkj0rZ4aemso0jcwCbPP6Q3E5LqCw= +github.com/grafana/k6catalog v0.1.0 h1:jLmbmB3EUJ+zyQG3hWy6dWbtMjvTkvJNx1d4LX8it6I= +github.com/grafana/k6catalog v0.1.0/go.mod h1:8R9eXAh2nb69+drkj0rZ4aemso0jcwCbPP6Q3E5LqCw= github.com/grafana/k6foundry v0.1.0 h1:hbpFFqZMjIOqsQbdn0aoCF+LdofBv5UfTOyXdZJSpE8= github.com/grafana/k6foundry v0.1.0/go.mod h1:b6n4InFgXl+3yPobmlyJfcJmLozU9CI9IIUuq8YqEiM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/service.go b/service.go index 10eb6bf..09982e2 100644 --- a/service.go +++ b/service.go @@ -5,6 +5,7 @@ import ( "bytes" "context" "crypto/sha1" + "errors" "fmt" "sort" @@ -129,13 +130,28 @@ func (b *buildsrv) Build(ctx context.Context, platform string, k6Constrains stri } id := fmt.Sprintf("%x", hash.Sum(nil)) + artifactObject, err := b.cache.Get(ctx, id) + if err == nil { + return Artifact{ + ID: id, + Checksum: artifactObject.Checksum, + URL: artifactObject.URL, + Dependencies: resolved, + Platform: platform, + }, nil + } + + if !errors.Is(err, ErrObjectNotFound) { + return Artifact{}, fmt.Errorf("accessing artifact %w", err) + } + artifactBuffer := &bytes.Buffer{} err = b.builder.Build(ctx, buildPlatform, k6Mod.Version, mods, []string{}, artifactBuffer) if err != nil { return Artifact{}, fmt.Errorf("building artifact %w", err) } - artifactObject, err := b.cache.Store(ctx, id, artifactBuffer) + artifactObject, err = b.cache.Store(ctx, id, artifactBuffer) if err != nil { return Artifact{}, fmt.Errorf("creating object %w", err) }