retrive object if already exists in cache

Signed-off-by: Pablo Chacin <[email protected]>

cache.go

@@ -4,12 +4,20 @@ import (
 	"bytes"
 	"context"
 	"crypto/sha256"
+	"errors"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 )
 
+var (
+	ErrObjectNotFound    = errors.New("object not found")
+	ErrAccessingObject   = errors.New("accessing object")
+	ErrCreatingObject    = errors.New("creating object")
+	ErrInitializingCache = errors.New("initializing cache")
+)
+
 // Object represents an object stored in the Cache
 // TODO: add metadata (e.g creation data, size)
 type Object struct {
@@ -21,6 +29,8 @@ type Object struct {
 
 // Cache defines an interface for storing blobs
 type Cache interface {
+	// Get retrieves an objects if exists in the cache or an error otherwise
+	Get(ctx context.Context, id string) (Object, error)
 	// Store stores the object and returns the metadata
 	Store(ctx context.Context, id string, content io.Reader) (Object, error)
 }
@@ -31,23 +41,14 @@ type fileObjectStore struct {
 }
 
 func NewTempFileCache() (Cache, error) {
-	cacheDir, err := os.MkdirTemp(os.TempDir(), "buildcache*")
-	if err != nil {
-		return nil, fmt.Errorf("creating cache directory %w", err)
-	}
-
-	return NewFileCache(cacheDir)
+	return NewFileCache(filepath.Join(os.TempDir(), "buildcache"))
 }
 
 // NewFileCache creates an cached backed by a directory
 func NewFileCache(path string) (Cache, error) {
-	fileInfo, err := os.Stat(path)
+	err := os.MkdirAll(path, 0o777)
 	if err != nil {
-		return nil, fmt.Errorf("invalid file path: %w", err)
-	}
-
-	if !fileInfo.IsDir() {
-		return nil, fmt.Errorf("must be a directory: %s", path)
+		return nil, fmt.Errorf("%w: %w", ErrInitializingCache, err)
 	}
 
 	return &fileObjectStore{
@@ -56,26 +57,64 @@ func NewFileCache(path string) (Cache, error) {
 }
 
 func (f *fileObjectStore) Store(ctx context.Context, id string, content io.Reader) (Object, error) {
-	objectFile, err := os.Create(filepath.Join(f.path, id))
+	objectDir := filepath.Join(f.path, id)
+	// TODO: check permissions
+	err := os.MkdirAll(objectDir, 0o777)
 	if err != nil {
-		return Object{}, fmt.Errorf("creating object %w", err)
+		return Object{}, fmt.Errorf("%w: %w", ErrCreatingObject, err)
+	}
+
+	objectFile, err := os.Create(filepath.Join(objectDir, "data"))
+	if err != nil {
+		return Object{}, fmt.Errorf("%w: %w", ErrCreatingObject, err)
 	}
 
 	// write content to object file and copy to buffer to calculate checksum
 	// TODO: optimize memory by copying content in blocks
 	buff := bytes.Buffer{}
 	_, err = io.Copy(objectFile, io.TeeReader(content, &buff))
 	if err != nil {
-		return Object{}, fmt.Errorf("creating object %w", err)
+		return Object{}, fmt.Errorf("%w: %w", ErrCreatingObject, err)
 	}
 
 	// calculate checksum
-	checksum := sha256.New()
-	checksum.Sum(buff.Bytes())
+	checksumHash := sha256.New()
+	checksumHash.Sum(buff.Bytes())
+	checksum := fmt.Sprintf("%x", checksumHash.Sum(nil))
+
+	// write metadata
+	err = os.WriteFile(filepath.Join(objectDir, "checksum"), []byte(checksum), 0o644)
+	if err != nil {
+		return Object{}, fmt.Errorf("%w: %w", ErrCreatingObject, err)
+	}
 
 	return Object{
 		ID:       id,
-		Checksum: string(fmt.Sprintf("%x", checksum.Sum(nil))),
+		Checksum: checksum,
 		URL:      fmt.Sprintf("file://%s", objectFile.Name()),
 	}, nil
 }
+
+func (f *fileObjectStore) Get(ctx context.Context, id string) (Object, error) {
+	objectDir := filepath.Join(f.path, id)
+	_, err := os.Stat(objectDir)
+
+	if errors.Is(err, os.ErrNotExist) {
+		return Object{}, fmt.Errorf("%w: %s", ErrObjectNotFound, id)
+	}
+
+	if err != nil {
+		return Object{}, fmt.Errorf("%w: %w", ErrAccessingObject, err)
+	}
+
+	checksum, err := os.ReadFile(filepath.Join(objectDir, "checksum"))
+	if err != nil {
+		return Object{}, fmt.Errorf("%w: %w", ErrAccessingObject, err)
+	}
+
+	return Object{
+		ID:       id,
+		Checksum: string(checksum),
+		URL:      fmt.Sprintf("file://%s", filepath.Join(objectDir, "data")),
+	}, nil
+}

cache_test.go

@@ -9,12 +9,13 @@ import (
 	"testing"
 )
 
-func TestFileCache(t *testing.T) {
+func TestCreateObject(t *testing.T) {
 	t.Parallel()
 
 	testCases := []struct {
 		title     string
 		content   []byte
+		id        string
 		expectErr error
 	}{
 		{
@@ -38,7 +39,7 @@ func TestFileCache(t *testing.T) {
 
 			obj, err := cache.Store(context.TODO(), "object", bytes.NewBuffer(tc.content))
 			if !errors.Is(err, tc.expectErr) {
-				t.Fatalf("expected %v got %v", tc, err)
+				t.Fatalf("expected %v got %v", tc.expectErr, err)
 			}
 
 			fileUrl, err := url.Parse(obj.URL)
@@ -57,3 +58,65 @@ func TestFileCache(t *testing.T) {
 		})
 	}
 }
+
+func TestGetObjectCache(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		title     string
+		id        string
+		expectErr error
+	}{
+		{
+			title:     "retrieve existing",
+			id:        "object",
+			expectErr: nil,
+		},
+		{
+			title:     "retrieve non existing object",
+			id:        "object2",
+			expectErr: ErrObjectNotFound,
+		},
+	}
+
+	cache, err := NewFileCache(t.TempDir())
+	if err != nil {
+		t.Fatalf("test setup %v", err)
+	}
+
+	content := []byte("content")
+	_, err = cache.Store(context.TODO(), "object", bytes.NewBuffer(content))
+	if err != nil {
+		t.Fatalf("test setup %v", err)
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.title, func(t *testing.T) {
+			t.Parallel()
+
+			obj, err := cache.Get(context.TODO(), tc.id)
+			if !errors.Is(err, tc.expectErr) {
+				t.Fatalf("expected %v got %v", tc.expectErr, err)
+			}
+
+			// if expected error, don't check returned object
+			if tc.expectErr != nil {
+				return
+			}
+
+			fileUrl, err := url.Parse(obj.URL)
+			if err != nil {
+				t.Fatalf("invalid url %v", err)
+			}
+
+			data, err := os.ReadFile(fileUrl.Path)
+			if err != nil {
+				t.Fatalf("reading object url %v", err)
+			}
+
+			if !bytes.Equal(data, content) {
+				t.Fatalf("expected %v got %v", data, content)
+			}
+		})
+	}
+}

go.mod

@@ -3,7 +3,7 @@ module github.com/grafana/k6build
 go 1.22.2
 
 require (
-	github.com/grafana/k6catalog v0.0.0-20240614130953-c7b8fc289822
+	github.com/grafana/k6catalog v0.1.0
 	github.com/grafana/k6foundry v0.1.0
 	github.com/spf13/cobra v1.8.0
 	golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8

go.sum

@@ -6,8 +6,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/grafana/k6catalog v0.0.0-20240614130953-c7b8fc289822 h1:O2+6uF5Z4V/UsIR8KI03gYGJHWqLHPu+yIFXtbhBm64=
-github.com/grafana/k6catalog v0.0.0-20240614130953-c7b8fc289822/go.mod h1:8R9eXAh2nb69+drkj0rZ4aemso0jcwCbPP6Q3E5LqCw=
+github.com/grafana/k6catalog v0.1.0 h1:jLmbmB3EUJ+zyQG3hWy6dWbtMjvTkvJNx1d4LX8it6I=
+github.com/grafana/k6catalog v0.1.0/go.mod h1:8R9eXAh2nb69+drkj0rZ4aemso0jcwCbPP6Q3E5LqCw=
 github.com/grafana/k6foundry v0.1.0 h1:hbpFFqZMjIOqsQbdn0aoCF+LdofBv5UfTOyXdZJSpE8=
 github.com/grafana/k6foundry v0.1.0/go.mod h1:b6n4InFgXl+3yPobmlyJfcJmLozU9CI9IIUuq8YqEiM=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=

service.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/sha1"
+	"errors"
 	"fmt"
 	"sort"
 
@@ -129,13 +130,28 @@ func (b *buildsrv) Build(ctx context.Context, platform string, k6Constrains stri
 	}
 	id := fmt.Sprintf("%x", hash.Sum(nil))
 
+	artifactObject, err := b.cache.Get(ctx, id)
+	if err == nil {
+		return Artifact{
+			ID:           id,
+			Checksum:     artifactObject.Checksum,
+			URL:          artifactObject.URL,
+			Dependencies: resolved,
+			Platform:     platform,
+		}, nil
+	}
+
+	if !errors.Is(err, ErrObjectNotFound) {
+		return Artifact{}, fmt.Errorf("accessing artifact %w", err)
+	}
+
 	artifactBuffer := &bytes.Buffer{}
 	err = b.builder.Build(ctx, buildPlatform, k6Mod.Version, mods, []string{}, artifactBuffer)
 	if err != nil {
 		return Artifact{}, fmt.Errorf("building artifact  %w", err)
 	}
 
-	artifactObject, err := b.cache.Store(ctx, id, artifactBuffer)
+	artifactObject, err = b.cache.Store(ctx, id, artifactBuffer)
 	if err != nil {
 		return Artifact{}, fmt.Errorf("creating object  %w", err)
 	}