Merge pull request #193 from lets-cli/add-remote-mixins

add remote mixins (experimental)

.github/workflows/test.yaml

@@ -25,10 +25,11 @@ jobs:
           go-version: 1.18.x
       - name: Checkout code
         uses: actions/checkout@v2
+      - run: go install gotest.tools/gotestsum@latest
       - name: Test unit
         env:
           LETS_CONFIG_DIR: ..
-        run: go test ./... -v
+        run: gotestsum --format testname -- ./... -coverprofile=coverage.out
 
   test-bats:
     runs-on: ubuntu-latest

docker/Dockerfile → Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.18.3-bullseye
+FROM golang:1.18.3-bullseye as builder
 
 ENV GOPROXY https://proxy.golang.org
 WORKDIR /app
@@ -21,3 +21,7 @@ COPY go.mod .
 COPY go.sum .
 
 RUN go mod download
+
+FROM golangci/golangci-lint:v1.45-alpine as linter
+
+RUN mkdir -p /.cache && chmod -R 777 /.cache

checksum/checksum.go

@@ -15,7 +15,6 @@ import (
 const (
 	DefaultChecksumKey      = "__default_checksum__"
 	DefaultChecksumFileName = "lets_default_checksum"
-	checksumsDir            = "checksums"
 )
 
 var checksumCache = make(map[string][]byte)
@@ -144,8 +143,8 @@ func CalculateChecksumFromSources(workDir string, checksumSources map[string][]s
 	return checksumMap, nil
 }
 
-func ReadChecksumFromDisk(dotLetsDir, cmdName, checksumName string) (string, error) {
-	_, checksumFilePath := getChecksumPath(dotLetsDir, cmdName, checksumName)
+func ReadChecksumFromDisk(checksumsDir, cmdName, checksumName string) (string, error) {
+	_, checksumFilePath := getChecksumPath(checksumsDir, cmdName, checksumName)
 
 	fileData, err := os.ReadFile(checksumFilePath)
 	if err != nil {
@@ -155,32 +154,27 @@ func ReadChecksumFromDisk(dotLetsDir, cmdName, checksumName string) (string, err
 	return string(fileData), nil
 }
 
-func getCmdChecksumPath(dotLetsDir string, cmdName string) string {
-	return filepath.Join(dotLetsDir, checksumsDir, cmdName)
+func getCmdChecksumPath(checksumsDir string, cmdName string) string {
+	return filepath.Join(checksumsDir, cmdName)
 }
 
 // returns dir path and full file path to checksum
 // (.lets/checksums/[command_name]/, .lets/checksums/[command_name]/[checksum_name]).
-func getChecksumPath(dotLetsDir string, cmdName string, checksumName string) (string, string) {
-	dirPath := getCmdChecksumPath(dotLetsDir, cmdName)
+func getChecksumPath(checksumsDir string, cmdName string, checksumName string) (string, string) {
+	dirPath := getCmdChecksumPath(checksumsDir, cmdName)
 
 	return dirPath, filepath.Join(dirPath, checksumName)
 }
 
 // TODO maybe checksumMap has to be separate struct ?
-func PersistCommandsChecksumToDisk(dotLetsDir string, checksumMap map[string]string, cmdName string) error {
-	checksumPath := filepath.Join(dotLetsDir, checksumsDir)
-	if err := util.SafeCreateDir(checksumPath); err != nil {
-		return fmt.Errorf("can not create %s: %w", checksumPath, err)
-	}
-
+func PersistCommandsChecksumToDisk(checksumsDir string, checksumMap map[string]string, cmdName string) error {
 	// TODO if at least one write failed do we have to revert all writes ???
 	for checksumName, checksum := range checksumMap {
 		filename := checksumName
 		if checksumName == DefaultChecksumKey {
 			filename = DefaultChecksumFileName
 		}
-		err := persistOneChecksum(dotLetsDir, cmdName, filename, checksum)
+		err := persistOneChecksum(checksumsDir, cmdName, filename, checksum)
 		if err != nil {
 			return err
 		}
@@ -189,8 +183,8 @@ func PersistCommandsChecksumToDisk(dotLetsDir string, checksumMap map[string]str
 	return nil
 }
 
-func persistOneChecksum(dotLetsDir string, cmdName string, checksumName string, checksum string) error {
-	checksumDirPath, checksumFilePath := getChecksumPath(dotLetsDir, cmdName, checksumName)
+func persistOneChecksum(checksumsDir string, cmdName string, checksumName string, checksum string) error {
+	checksumDirPath, checksumFilePath := getChecksumPath(checksumsDir, cmdName, checksumName)
 	if err := util.SafeCreateDir(checksumDirPath); err != nil {
 		return fmt.Errorf("can not create checksum dir at %s: %w", checksumDirPath, err)
 	}
@@ -209,9 +203,9 @@ func persistOneChecksum(dotLetsDir string, cmdName string, checksumName string,
 }
 
 // IsChecksumForCmdPersisted checks if checksums for cmd exists and persisted.
-func IsChecksumForCmdPersisted(dotLetsDir string, cmdName string) bool {
+func IsChecksumForCmdPersisted(checksumsDir string, cmdName string) bool {
 	// check if checksums for cmd exists
-	if _, err := os.Stat(getCmdChecksumPath(dotLetsDir, cmdName)); err != nil {
+	if _, err := os.Stat(getCmdChecksumPath(checksumsDir, cmdName)); err != nil {
 		return !os.IsNotExist(err)
 	}
 

config/config/command.go

@@ -145,15 +145,15 @@ func (cmd *Command) GetPersistedChecksums() map[string]string {
 }
 
 // ReadChecksumsFromDisk reads all checksums for cmd into map.
-func (cmd *Command) ReadChecksumsFromDisk(dotLetsDir string, cmdName string, checksumMap map[string]string) error {
+func (cmd *Command) ReadChecksumsFromDisk(checksumsDir string, cmdName string, checksumMap map[string]string) error {
 	checksums := make(map[string]string, len(checksumMap)+1)
 
 	for checksumName := range checksumMap {
 		filename := checksumName
 		if checksumName == checksum.DefaultChecksumKey {
 			filename = checksum.DefaultChecksumFileName
 		}
-		checksumResult, err := checksum.ReadChecksumFromDisk(dotLetsDir, cmdName, filename)
+		checksumResult, err := checksum.ReadChecksumFromDisk(checksumsDir, cmdName, filename)
 		if err != nil {
 			return err
 		}

config/config/config.go

@@ -1,6 +1,12 @@
 package config
 
-import "github.com/lets-cli/lets/set"
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/lets-cli/lets/set"
+	"github.com/lets-cli/lets/util"
+)
 
 var (
 	// COMMANDS is a top-level directive. Includes all commands to run.
@@ -36,21 +42,43 @@ type Config struct {
 	isMixin bool // if true, we consider config as mixin and apply different parsing and validation
 	// absolute path to .lets
 	DotLetsDir string
+	// absolute path to .lets/checksums
+	ChecksumsDir string
+	// absolute path to .lets/mixins
+	MixinsDir string
 }
 
 func NewConfig(workDir string, configAbsPath string, dotLetsDir string) *Config {
 	return &Config{
-		Commands:   make(map[string]Command),
-		Env:        make(map[string]string),
-		WorkDir:    workDir,
-		FilePath:   configAbsPath,
-		DotLetsDir: dotLetsDir,
+		Commands:     make(map[string]Command),
+		Env:          make(map[string]string),
+		WorkDir:      workDir,
+		FilePath:     configAbsPath,
+		DotLetsDir:   dotLetsDir,
+		ChecksumsDir: filepath.Join(dotLetsDir, "checksums"),
+		MixinsDir:    filepath.Join(dotLetsDir, "mixins"),
+	}
+}
+
+func NewMixinConfig(cfg *Config, configAbsPath string) *Config {
+	mixin := NewConfig(cfg.WorkDir, configAbsPath, cfg.DotLetsDir)
+	mixin.isMixin = true
+
+	return mixin
+}
+
+func (c *Config) CreateChecksumsDir() error {
+	if err := util.SafeCreateDir(c.ChecksumsDir); err != nil {
+		return fmt.Errorf("can not create %s: %w", c.ChecksumsDir, err)
 	}
+
+	return nil
 }
 
-func NewMixinConfig(workDir string, configAbsPath string, dotLetsDir string) *Config {
-	cfg := NewConfig(workDir, configAbsPath, dotLetsDir)
-	cfg.isMixin = true
+func (c *Config) CreateMixinsDir() error {
+	if err := util.SafeCreateDir(c.MixinsDir); err != nil {
+		return fmt.Errorf("can not create %s: %w", c.MixinsDir, err)
+	}
 
-	return cfg
+	return nil
 }

config/find.go

@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 
 	"github.com/lets-cli/lets/config/path"
+	"github.com/lets-cli/lets/util"
 	"github.com/lets-cli/lets/workdir"
 	log "github.com/sirupsen/logrus"
 )
@@ -71,6 +72,10 @@ func FindConfig(configName string, configDir string) (PathInfo, error) {
 		return PathInfo{}, fmt.Errorf("can not get .lets absolute path: %w", err)
 	}
 
+	if err := util.SafeCreateDir(dotLetsDir); err != nil {
+		return PathInfo{}, fmt.Errorf("can not create .lets dir: %w", err)
+	}
+
 	pathInfo := PathInfo{
 		AbsPath:    configAbsPath,
 		WorkDir:    workDir,

config/parser/parser.go

@@ -2,10 +2,16 @@ package parser
 
 import (
 	"bytes"
+	"context"
+	"crypto/sha256"
 	"errors"
 	"fmt"
+	"io"
+	"net/http"
 	"os"
+	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/lets-cli/lets/config/config"
 	"github.com/lets-cli/lets/config/path"
@@ -193,15 +199,113 @@ func isIgnoredMixin(filename string) bool {
 	return strings.HasPrefix(filename, "-")
 }
 
+type RemoteMixin struct {
+	URL     string
+	Version string
+
+	mixinsDir string
+}
+
+// Filename is name of mixin file (hash from url).
+func (rm *RemoteMixin) Filename() string {
+	hasher := sha256.New()
+	hasher.Write([]byte(rm.URL))
+
+	if rm.Version != "" {
+		hasher.Write([]byte(rm.Version))
+	}
+
+	return fmt.Sprintf("%x", hasher.Sum(nil))
+}
+
+// Path is abs path to mixin file (.lets/mixins/<filename>).
+func (rm *RemoteMixin) Path() string {
+	return filepath.Join(rm.mixinsDir, rm.Filename())
+}
+
+func (rm *RemoteMixin) persist(data []byte) error {
+	f, err := os.OpenFile(rm.Path(), os.O_CREATE|os.O_WRONLY, 0o755)
+	if err != nil {
+		return fmt.Errorf("can not open file %s to persist mixin: %w", rm.Path(), err)
+	}
+
+	_, err = f.Write(data)
+	if err != nil {
+		return fmt.Errorf("can not write mixin to file %s: %w", rm.Path(), err)
+	}
+
+	return nil
+}
+
+func (rm *RemoteMixin) exists() bool {
+	return util.FileExists(rm.Path())
+}
+
+func (rm *RemoteMixin) tryRead() ([]byte, error) {
+	if !rm.exists() {
+		return nil, nil
+	}
+	data, err := os.ReadFile(rm.Path())
+	if err != nil {
+		return nil, fmt.Errorf("can not read mixin config file at %s: %w", rm.Path(), err)
+	}
+
+	return data, nil
+}
+
+func (rm *RemoteMixin) download() ([]byte, error) {
+	// TODO: maybe create a client for this?
+	ctx, cancel := context.WithTimeout(context.Background(), 60*5*time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(
+		ctx,
+		"GET",
+		rm.URL,
+		nil,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	client := &http.Client{
+		Timeout: 15 * 60 * time.Second, // TODO: move to client struct
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to make request: %w", err)
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusNotFound {
+		return nil, fmt.Errorf("no such file at: %s", rm.URL)
+	} else if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		return nil, fmt.Errorf("network error: %s", resp.Status)
+	}
+
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response: %w", err)
+	}
+
+	return data, nil
+}
+
 func readAndValidateMixins(mixins []interface{}, cfg *config.Config) error {
-	for _, filename := range mixins {
-		if filename, ok := filename.(string); ok { //nolint:nestif
+	if err := cfg.CreateMixinsDir(); err != nil {
+		return err
+	}
+
+	for _, mixin := range mixins {
+		if filename, ok := mixin.(string); ok { //nolint:nestif
 			configAbsPath, err := path.GetFullConfigPath(normalizeMixinFilename(filename), cfg.WorkDir)
 			if err != nil {
 				if isIgnoredMixin(filename) && errors.Is(err, path.ErrFileNotExists) {
 					continue
 				} else {
-					// complain non-existed mixin only if its filename does not starts with dash `-`
+					// complain non-existed mixin only if its filename does not start with dash `-`
 					return fmt.Errorf("failed to read mixin config: %w", err)
 				}
 			}
@@ -210,14 +314,52 @@ func readAndValidateMixins(mixins []interface{}, cfg *config.Config) error {
 				return fmt.Errorf("can not read mixin config file: %w", err)
 			}
 
-			mixinCfg := config.NewMixinConfig(cfg.WorkDir, filename, cfg.DotLetsDir)
+			mixinCfg := config.NewMixinConfig(cfg, filename)
 			if err := parseMixinConfig(fileData, mixinCfg); err != nil {
 				return fmt.Errorf("failed to load mixin config '%s': %w", filename, err)
 			}
 
 			if err := mergeConfigs(cfg, mixinCfg); err != nil {
 				return fmt.Errorf("failed to merge mixin config %s with main config: %w", filename, err)
 			}
+		} else if mixinMapping, ok := mixin.(map[string]interface{}); ok {
+			rm := &RemoteMixin{mixinsDir: cfg.MixinsDir}
+			if url, ok := mixinMapping["url"]; ok {
+				// TODO check if url is valid
+				rm.URL, _ = url.(string)
+			}
+
+			if version, ok := mixinMapping["version"]; ok {
+				rm.Version, _ = version.(string)
+			}
+
+			data, err := rm.tryRead()
+			if err != nil {
+				return err
+			}
+
+			if data == nil {
+				data, err = rm.download()
+				if err != nil {
+					return err
+				}
+			}
+
+			// TODO: what if multiple mixins have same commands
+			//  1 option - fail and suggest use to namespace all commands in remote mixin
+			//  2 option - namespace it (this may require specifying namespace in mixin config or in main config mixin section)
+			mixinCfg := config.NewMixinConfig(cfg, rm.Filename())
+			if err := parseMixinConfig(data, mixinCfg); err != nil {
+				return fmt.Errorf("failed to load remote mixin config '%s': %w", rm.URL, err)
+			}
+
+			if err := mergeConfigs(cfg, mixinCfg); err != nil {
+				return fmt.Errorf("failed to merge remote mixin config %s with main config: %w", rm.URL, err)
+			}
+
+			if err := rm.persist(data); err != nil {
+				return fmt.Errorf("failed to persist remote mixin config %s: %w", rm.URL, err)
+			}
 		} else {
 			return newConfigParseError(
 				"must be a string",