diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e83e9ff1..4a08d814 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -25,10 +25,11 @@ jobs: go-version: 1.18.x - name: Checkout code uses: actions/checkout@v2 + - run: go install gotest.tools/gotestsum@latest - name: Test unit env: LETS_CONFIG_DIR: .. - run: go test ./... -v + run: gotestsum --format testname -- ./... -coverprofile=coverage.out test-bats: runs-on: ubuntu-latest diff --git a/docker/Dockerfile b/Dockerfile similarity index 80% rename from docker/Dockerfile rename to Dockerfile index 945a3408..5f358b21 100644 --- a/docker/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.18.3-bullseye +FROM golang:1.18.3-bullseye as builder ENV GOPROXY https://proxy.golang.org WORKDIR /app @@ -21,3 +21,7 @@ COPY go.mod . COPY go.sum . RUN go mod download + +FROM golangci/golangci-lint:v1.45-alpine as linter + +RUN mkdir -p /.cache && chmod -R 777 /.cache diff --git a/checksum/checksum.go b/checksum/checksum.go index 409df7d7..7475397f 100644 --- a/checksum/checksum.go +++ b/checksum/checksum.go @@ -15,7 +15,6 @@ import ( const ( DefaultChecksumKey = "__default_checksum__" DefaultChecksumFileName = "lets_default_checksum" - checksumsDir = "checksums" ) var checksumCache = make(map[string][]byte) @@ -144,8 +143,8 @@ func CalculateChecksumFromSources(workDir string, checksumSources map[string][]s return checksumMap, nil } -func ReadChecksumFromDisk(dotLetsDir, cmdName, checksumName string) (string, error) { - _, checksumFilePath := getChecksumPath(dotLetsDir, cmdName, checksumName) +func ReadChecksumFromDisk(checksumsDir, cmdName, checksumName string) (string, error) { + _, checksumFilePath := getChecksumPath(checksumsDir, cmdName, checksumName) fileData, err := os.ReadFile(checksumFilePath) if err != nil { @@ -155,32 +154,27 @@ func ReadChecksumFromDisk(dotLetsDir, cmdName, checksumName string) (string, err return string(fileData), nil } -func getCmdChecksumPath(dotLetsDir string, cmdName string) string { - return filepath.Join(dotLetsDir, checksumsDir, cmdName) +func getCmdChecksumPath(checksumsDir string, cmdName string) string { + return filepath.Join(checksumsDir, cmdName) } // returns dir path and full file path to checksum // (.lets/checksums/[command_name]/, .lets/checksums/[command_name]/[checksum_name]). -func getChecksumPath(dotLetsDir string, cmdName string, checksumName string) (string, string) { - dirPath := getCmdChecksumPath(dotLetsDir, cmdName) +func getChecksumPath(checksumsDir string, cmdName string, checksumName string) (string, string) { + dirPath := getCmdChecksumPath(checksumsDir, cmdName) return dirPath, filepath.Join(dirPath, checksumName) } // TODO maybe checksumMap has to be separate struct ? -func PersistCommandsChecksumToDisk(dotLetsDir string, checksumMap map[string]string, cmdName string) error { - checksumPath := filepath.Join(dotLetsDir, checksumsDir) - if err := util.SafeCreateDir(checksumPath); err != nil { - return fmt.Errorf("can not create %s: %w", checksumPath, err) - } - +func PersistCommandsChecksumToDisk(checksumsDir string, checksumMap map[string]string, cmdName string) error { // TODO if at least one write failed do we have to revert all writes ??? for checksumName, checksum := range checksumMap { filename := checksumName if checksumName == DefaultChecksumKey { filename = DefaultChecksumFileName } - err := persistOneChecksum(dotLetsDir, cmdName, filename, checksum) + err := persistOneChecksum(checksumsDir, cmdName, filename, checksum) if err != nil { return err } @@ -189,8 +183,8 @@ func PersistCommandsChecksumToDisk(dotLetsDir string, checksumMap map[string]str return nil } -func persistOneChecksum(dotLetsDir string, cmdName string, checksumName string, checksum string) error { - checksumDirPath, checksumFilePath := getChecksumPath(dotLetsDir, cmdName, checksumName) +func persistOneChecksum(checksumsDir string, cmdName string, checksumName string, checksum string) error { + checksumDirPath, checksumFilePath := getChecksumPath(checksumsDir, cmdName, checksumName) if err := util.SafeCreateDir(checksumDirPath); err != nil { return fmt.Errorf("can not create checksum dir at %s: %w", checksumDirPath, err) } @@ -209,9 +203,9 @@ func persistOneChecksum(dotLetsDir string, cmdName string, checksumName string, } // IsChecksumForCmdPersisted checks if checksums for cmd exists and persisted. -func IsChecksumForCmdPersisted(dotLetsDir string, cmdName string) bool { +func IsChecksumForCmdPersisted(checksumsDir string, cmdName string) bool { // check if checksums for cmd exists - if _, err := os.Stat(getCmdChecksumPath(dotLetsDir, cmdName)); err != nil { + if _, err := os.Stat(getCmdChecksumPath(checksumsDir, cmdName)); err != nil { return !os.IsNotExist(err) } diff --git a/config/config/command.go b/config/config/command.go index 9edbce46..26666c8e 100644 --- a/config/config/command.go +++ b/config/config/command.go @@ -145,7 +145,7 @@ func (cmd *Command) GetPersistedChecksums() map[string]string { } // ReadChecksumsFromDisk reads all checksums for cmd into map. -func (cmd *Command) ReadChecksumsFromDisk(dotLetsDir string, cmdName string, checksumMap map[string]string) error { +func (cmd *Command) ReadChecksumsFromDisk(checksumsDir string, cmdName string, checksumMap map[string]string) error { checksums := make(map[string]string, len(checksumMap)+1) for checksumName := range checksumMap { @@ -153,7 +153,7 @@ func (cmd *Command) ReadChecksumsFromDisk(dotLetsDir string, cmdName string, che if checksumName == checksum.DefaultChecksumKey { filename = checksum.DefaultChecksumFileName } - checksumResult, err := checksum.ReadChecksumFromDisk(dotLetsDir, cmdName, filename) + checksumResult, err := checksum.ReadChecksumFromDisk(checksumsDir, cmdName, filename) if err != nil { return err } diff --git a/config/config/config.go b/config/config/config.go index 812f9f54..12decffc 100644 --- a/config/config/config.go +++ b/config/config/config.go @@ -1,6 +1,12 @@ package config -import "github.com/lets-cli/lets/set" +import ( + "fmt" + "path/filepath" + + "github.com/lets-cli/lets/set" + "github.com/lets-cli/lets/util" +) var ( // COMMANDS is a top-level directive. Includes all commands to run. @@ -36,21 +42,43 @@ type Config struct { isMixin bool // if true, we consider config as mixin and apply different parsing and validation // absolute path to .lets DotLetsDir string + // absolute path to .lets/checksums + ChecksumsDir string + // absolute path to .lets/mixins + MixinsDir string } func NewConfig(workDir string, configAbsPath string, dotLetsDir string) *Config { return &Config{ - Commands: make(map[string]Command), - Env: make(map[string]string), - WorkDir: workDir, - FilePath: configAbsPath, - DotLetsDir: dotLetsDir, + Commands: make(map[string]Command), + Env: make(map[string]string), + WorkDir: workDir, + FilePath: configAbsPath, + DotLetsDir: dotLetsDir, + ChecksumsDir: filepath.Join(dotLetsDir, "checksums"), + MixinsDir: filepath.Join(dotLetsDir, "mixins"), + } +} + +func NewMixinConfig(cfg *Config, configAbsPath string) *Config { + mixin := NewConfig(cfg.WorkDir, configAbsPath, cfg.DotLetsDir) + mixin.isMixin = true + + return mixin +} + +func (c *Config) CreateChecksumsDir() error { + if err := util.SafeCreateDir(c.ChecksumsDir); err != nil { + return fmt.Errorf("can not create %s: %w", c.ChecksumsDir, err) } + + return nil } -func NewMixinConfig(workDir string, configAbsPath string, dotLetsDir string) *Config { - cfg := NewConfig(workDir, configAbsPath, dotLetsDir) - cfg.isMixin = true +func (c *Config) CreateMixinsDir() error { + if err := util.SafeCreateDir(c.MixinsDir); err != nil { + return fmt.Errorf("can not create %s: %w", c.MixinsDir, err) + } - return cfg + return nil } diff --git a/config/find.go b/config/find.go index 39b12f35..0e1b0095 100644 --- a/config/find.go +++ b/config/find.go @@ -5,6 +5,7 @@ import ( "path/filepath" "github.com/lets-cli/lets/config/path" + "github.com/lets-cli/lets/util" "github.com/lets-cli/lets/workdir" log "github.com/sirupsen/logrus" ) @@ -71,6 +72,10 @@ func FindConfig(configName string, configDir string) (PathInfo, error) { return PathInfo{}, fmt.Errorf("can not get .lets absolute path: %w", err) } + if err := util.SafeCreateDir(dotLetsDir); err != nil { + return PathInfo{}, fmt.Errorf("can not create .lets dir: %w", err) + } + pathInfo := PathInfo{ AbsPath: configAbsPath, WorkDir: workDir, diff --git a/config/parser/parser.go b/config/parser/parser.go index b54ef243..20dfa28b 100644 --- a/config/parser/parser.go +++ b/config/parser/parser.go @@ -2,10 +2,16 @@ package parser import ( "bytes" + "context" + "crypto/sha256" "errors" "fmt" + "io" + "net/http" "os" + "path/filepath" "strings" + "time" "github.com/lets-cli/lets/config/config" "github.com/lets-cli/lets/config/path" @@ -193,15 +199,113 @@ func isIgnoredMixin(filename string) bool { return strings.HasPrefix(filename, "-") } +type RemoteMixin struct { + URL string + Version string + + mixinsDir string +} + +// Filename is name of mixin file (hash from url). +func (rm *RemoteMixin) Filename() string { + hasher := sha256.New() + hasher.Write([]byte(rm.URL)) + + if rm.Version != "" { + hasher.Write([]byte(rm.Version)) + } + + return fmt.Sprintf("%x", hasher.Sum(nil)) +} + +// Path is abs path to mixin file (.lets/mixins/). +func (rm *RemoteMixin) Path() string { + return filepath.Join(rm.mixinsDir, rm.Filename()) +} + +func (rm *RemoteMixin) persist(data []byte) error { + f, err := os.OpenFile(rm.Path(), os.O_CREATE|os.O_WRONLY, 0o755) + if err != nil { + return fmt.Errorf("can not open file %s to persist mixin: %w", rm.Path(), err) + } + + _, err = f.Write(data) + if err != nil { + return fmt.Errorf("can not write mixin to file %s: %w", rm.Path(), err) + } + + return nil +} + +func (rm *RemoteMixin) exists() bool { + return util.FileExists(rm.Path()) +} + +func (rm *RemoteMixin) tryRead() ([]byte, error) { + if !rm.exists() { + return nil, nil + } + data, err := os.ReadFile(rm.Path()) + if err != nil { + return nil, fmt.Errorf("can not read mixin config file at %s: %w", rm.Path(), err) + } + + return data, nil +} + +func (rm *RemoteMixin) download() ([]byte, error) { + // TODO: maybe create a client for this? + ctx, cancel := context.WithTimeout(context.Background(), 60*5*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext( + ctx, + "GET", + rm.URL, + nil, + ) + if err != nil { + return nil, err + } + + client := &http.Client{ + Timeout: 15 * 60 * time.Second, // TODO: move to client struct + } + + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to make request: %w", err) + } + + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + return nil, fmt.Errorf("no such file at: %s", rm.URL) + } else if resp.StatusCode < 200 || resp.StatusCode > 299 { + return nil, fmt.Errorf("network error: %s", resp.Status) + } + + data, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response: %w", err) + } + + return data, nil +} + func readAndValidateMixins(mixins []interface{}, cfg *config.Config) error { - for _, filename := range mixins { - if filename, ok := filename.(string); ok { //nolint:nestif + if err := cfg.CreateMixinsDir(); err != nil { + return err + } + + for _, mixin := range mixins { + if filename, ok := mixin.(string); ok { //nolint:nestif configAbsPath, err := path.GetFullConfigPath(normalizeMixinFilename(filename), cfg.WorkDir) if err != nil { if isIgnoredMixin(filename) && errors.Is(err, path.ErrFileNotExists) { continue } else { - // complain non-existed mixin only if its filename does not starts with dash `-` + // complain non-existed mixin only if its filename does not start with dash `-` return fmt.Errorf("failed to read mixin config: %w", err) } } @@ -210,7 +314,7 @@ func readAndValidateMixins(mixins []interface{}, cfg *config.Config) error { return fmt.Errorf("can not read mixin config file: %w", err) } - mixinCfg := config.NewMixinConfig(cfg.WorkDir, filename, cfg.DotLetsDir) + mixinCfg := config.NewMixinConfig(cfg, filename) if err := parseMixinConfig(fileData, mixinCfg); err != nil { return fmt.Errorf("failed to load mixin config '%s': %w", filename, err) } @@ -218,6 +322,44 @@ func readAndValidateMixins(mixins []interface{}, cfg *config.Config) error { if err := mergeConfigs(cfg, mixinCfg); err != nil { return fmt.Errorf("failed to merge mixin config %s with main config: %w", filename, err) } + } else if mixinMapping, ok := mixin.(map[string]interface{}); ok { + rm := &RemoteMixin{mixinsDir: cfg.MixinsDir} + if url, ok := mixinMapping["url"]; ok { + // TODO check if url is valid + rm.URL, _ = url.(string) + } + + if version, ok := mixinMapping["version"]; ok { + rm.Version, _ = version.(string) + } + + data, err := rm.tryRead() + if err != nil { + return err + } + + if data == nil { + data, err = rm.download() + if err != nil { + return err + } + } + + // TODO: what if multiple mixins have same commands + // 1 option - fail and suggest use to namespace all commands in remote mixin + // 2 option - namespace it (this may require specifying namespace in mixin config or in main config mixin section) + mixinCfg := config.NewMixinConfig(cfg, rm.Filename()) + if err := parseMixinConfig(data, mixinCfg); err != nil { + return fmt.Errorf("failed to load remote mixin config '%s': %w", rm.URL, err) + } + + if err := mergeConfigs(cfg, mixinCfg); err != nil { + return fmt.Errorf("failed to merge remote mixin config %s with main config: %w", rm.URL, err) + } + + if err := rm.persist(data); err != nil { + return fmt.Errorf("failed to persist remote mixin config %s: %w", rm.URL, err) + } } else { return newConfigParseError( "must be a string", diff --git a/docker-compose.yml b/docker-compose.yml index 2e00e3b7..fcaaa282 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: image: lets build: context: . - dockerfile: docker/Dockerfile + dockerfile: Dockerfile working_dir: /app volumes: - ./:/app diff --git a/docker/lint.Dockerfile b/docker/lint.Dockerfile deleted file mode 100644 index 814cabbb..00000000 --- a/docker/lint.Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM golangci/golangci-lint:v1.45-alpine - -RUN mkdir -p /.cache && chmod -R 777 /.cache diff --git a/docs/docs/best_practices.md b/docs/docs/best_practices.md index 9fa23f53..5b176745 100644 --- a/docs/docs/best_practices.md +++ b/docs/docs/best_practices.md @@ -3,6 +3,38 @@ id: best_practices title: Best practices --- +### Naming conventions + +Prefer single word over plural. + +It is better to leverage semantics of `lets` as an intention to do something. For example it is natural saying `lets test` or `lets build` something. + +`bad` + +``` +lets runs +``` + +`good` + +``` +lets run +``` + +--- + +`bad` + +``` +lets tests +``` + +`good` + +``` +lets test +``` + ### Default commands If you have many projects (lets say - microservices) - it would be great to have one way to run and operate them when developing diff --git a/docs/docs/config.md b/docs/docs/config.md index 8292507c..31378754 100644 --- a/docs/docs/config.md +++ b/docs/docs/config.md @@ -133,13 +133,25 @@ commands: `key: mixins` -`type: list of string` +`type:` +- `list of strings` +- `list of map` + + +`Example` + +``` +mixins: + - lets.build.yaml + - url: https://raw.githubusercontent.com/lets-cli/lets/master/lets.build.yaml + version: 1 +``` Allows to split `lets.yaml` into mixins (mixin config files). To make `lets.yaml` small and readable it is convenient to split main config into many smaller ones and include them -Example: +`Full example` ```yaml # in lets.yaml @@ -159,7 +171,6 @@ commands: cmd: echo Testing... ``` -And `lets test` works fine. ### Ignored mixins @@ -178,6 +189,25 @@ mixins: Now if `my.yaml` exists - it will be loaded as a mixin. If it is not exist - `lets` will skip it. +### Remote mixins `(experimental)` + +It is possible to specify mixin as url. Lets will download it and load it as a mixin. +File will be stored in `.lets/mixins` directory. + +By default mixin filename will be sha256 hash of url. + +You can specify `version` key. If url is not versioned, lets will use `version` for filename hash as well (`hash(url) + hash(version)`). + +For example: + +```yaml +shell: bash +mixins: + - url: https://raw.githubusercontent.com/lets-cli/lets/master/lets.build.yaml + version: 1 +``` + + ### Commands `key: commands` diff --git a/lets.build.yaml b/lets.build.yaml index 561b3084..52d24caa 100644 --- a/lets.build.yaml +++ b/lets.build.yaml @@ -4,8 +4,8 @@ env: commands: build-lets-image: description: Build lets docker image - cmd: docker build -t lets -f docker/Dockerfile . + cmd: docker build -t lets -f Dockerfile --target builder . build-lint-image: description: Build lets lint docker image - cmd: docker build -t lets-lint -f docker/lint.Dockerfile . + cmd: docker build -t lets-lint -f Dockerfile --target linter . diff --git a/lets.yaml b/lets.yaml index 9aefba1e..a99379a8 100644 --- a/lets.yaml +++ b/lets.yaml @@ -65,12 +65,6 @@ commands: go tool cover -func=coverage.out fi - staticcheck: - description: Run staticcheck for lets. Staticcheck is go vet on steroids - depends: [build-lets-image] - cmd: - docker-compose run --rm staticheck - lint: description: Run golint-ci depends: [build-lint-image] diff --git a/main.go b/main.go index eb5cb69f..82824d9c 100644 --- a/main.go +++ b/main.go @@ -11,7 +11,6 @@ import ( "github.com/lets-cli/lets/env" "github.com/lets-cli/lets/logging" "github.com/lets-cli/lets/runner" - "github.com/lets-cli/lets/workdir" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) @@ -31,11 +30,6 @@ func main() { var rootCmd *cobra.Command if cfg != nil { rootCmd = cmd.CreateRootCommandWithConfig(os.Stdout, cfg, version) - - if err := workdir.CreateDotLetsDir(cfg.WorkDir); err != nil { - log.Error(err) - os.Exit(1) - } } else { rootCmd = cmd.CreateRootCommand(version) } diff --git a/runner/run.go b/runner/run.go index a179c2ec..eb591cda 100644 --- a/runner/run.go +++ b/runner/run.go @@ -217,8 +217,8 @@ func (r *Runner) initCmd() error { // if command declared as persist_checksum we must read current persisted checksums into memory if r.cmd.PersistChecksum { - if checksum.IsChecksumForCmdPersisted(r.cfg.DotLetsDir, r.cmd.Name) { - err := r.cmd.ReadChecksumsFromDisk(r.cfg.DotLetsDir, r.cmd.Name, r.cmd.ChecksumMap) + if checksum.IsChecksumForCmdPersisted(r.cfg.ChecksumsDir, r.cmd.Name) { + err := r.cmd.ReadChecksumsFromDisk(r.cfg.ChecksumsDir, r.cmd.Name, r.cmd.ChecksumMap) if err != nil { return fmt.Errorf("failed to read persisted checksum for command '%s': %w", r.cmd.Name, err) } @@ -359,8 +359,12 @@ func (r *Runner) persistChecksum() error { if r.cmd.PersistChecksum { debugf("persisting checksum for command '%s'", r.cmd.Name) + if err := r.cfg.CreateChecksumsDir(); err != nil { + return err + } + err := checksum.PersistCommandsChecksumToDisk( - r.cfg.DotLetsDir, + r.cfg.ChecksumsDir, r.cmd.ChecksumMap, r.cmd.Name, ) diff --git a/workdir/workdir.go b/workdir/workdir.go index 33288ff3..11475a70 100644 --- a/workdir/workdir.go +++ b/workdir/workdir.go @@ -5,13 +5,12 @@ import ( "os" "path/filepath" - "github.com/lets-cli/lets/util" log "github.com/sirupsen/logrus" ) const dotLetsDir = ".lets" -const defaultLetsYaml = `version: %s +const defaultLetsYaml = `version: "%s" shell: bash commands: @@ -25,17 +24,6 @@ commands: cmd: echo Hello, "${LETSOPT_NAME:-world}"! ` -// CreateDotLetsDir creates .lets dir where lets.yaml located. -// If directory already exists - skip creation. -func CreateDotLetsDir(workDir string) error { - fullPath, err := GetDotLetsDir(workDir) - if err != nil { - return err - } - - return util.SafeCreateDir(fullPath) -} - func GetDotLetsDir(workDir string) (string, error) { return filepath.Abs(filepath.Join(workDir, dotLetsDir)) }