diff --git a/README.md b/README.md index 5d43f178..a0af25f5 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,8 @@ lets run --debug --level=info Config schema * [shell](#shell) +* [mixins](#mixins) +* [env](#global-env) * [commands](#commands) * [description](#description) * [cmd](#cmd) @@ -64,6 +66,8 @@ Config schema ### Top-level directives: #### `shell` +`key: shell` + `type: string` Specify shell to use when running commands @@ -74,7 +78,55 @@ Example: shell: bash ``` +#### `global env` +`key: env` + +`type: string` + +Specify global env for all commands. + +Example: + +```sh +shell: bash +env: + MY_GLOBAL_ENV: "123" +``` + +#### `mixins` +`key: mixins` + +`type: list of string` + +Allows to split `lets.yaml` into mixins (mixin config files). + +To make `lets.yaml` small and readable its convenient to split main config into many smaller ones and include them + +Example: + +```sh +# in lets.yaml +... +shell: bash +mixins: + - test.yaml + +commands: + echo: + cmd: echo Hi + +# in test.yaml +... +commands: + test: + cmd: echo Testing... +``` + +And `lets test` works fine. + #### `commands` +`key: commands` + `type: mapping` Mapping of all available commands @@ -90,6 +142,8 @@ commands: ### Command directives: ##### `description` +`key: description` + `type: string` Short description of command - shown in help message @@ -103,6 +157,8 @@ commands: ``` ##### `cmd` +`key: cmd` + `type: string or array of strings` Actual command to run in shell. @@ -156,6 +212,8 @@ lets test -v the `-v` will be appended, so the resulting command to run will be `go test ./... -v` ##### `depends` +`key: depends` + `type: array of string` Specify what commands to run before the actual command. May be useful, when have one shared command. @@ -184,6 +242,8 @@ commands: ##### `options` +`key: options` + `type: string (multiline string)` One of the most cool things about `lets` than it has built in docopt parsing. @@ -247,6 +307,8 @@ echo LETSCLI_DEBUG=${LETSCLI_DEBUG} # LETSCLI_DEBUG=--debug ##### `env` +`key: env` + `type: mapping string => string` Env is as simple as it sounds. Define additional env for a commmand: @@ -265,6 +327,8 @@ commands: ##### `eval_env` +`key: eval_env` + `type: mapping string => string` Same as env but allows you to dynamically compute env: @@ -285,6 +349,8 @@ Value will be executed in shell and result will be saved in env. ##### `checksum` +`key: checksum` + `type: array of string` Checksum used for computing file hashed. It is useful when you depend on some files content changes. @@ -360,19 +426,20 @@ Yet there is no binaries - [ ] global checksums (check if some commands use checksum so we can skip its calculation) - [ ] multiple checksums in one command (kv) - [x] depends on other commands -- [ ] inherit configs +- [x] inherit configs - [x] LETS_DEBUG env for debugging logs - [ ] command to only calculate checksum - [x] capture env from shell - [ ] env as a list of strings `- key=val` - [ ] env computing - - [ ] global env + - [ ] global eval_env + - [x] global env - [x] command env - [ ] dogfood on ci - [x] add version flag to lets - [ ] add verbose flag to lets - [x] add LETSCLI_OPTION - options as is -- [ ] add all env vars event if no options were passed +- [x] add all env vars event if no options were passed - [ ] BUG - when run git commit, lets complains that no config is found for git - [x] Print usage if wrong opt passed for options - [ ] Bash/zsh completion diff --git a/build.yaml b/build.yaml new file mode 100644 index 00000000..c5618ac2 --- /dev/null +++ b/build.yaml @@ -0,0 +1,6 @@ +env: + DOCKER_BUILDKIT: "1" + +commands: + build-docker: + cmd: docker build -t lets -f docker/Dockerfile . diff --git a/commands/command/cmd.go b/commands/command/cmd.go index 949bd5d0..4ebb91cf 100644 --- a/commands/command/cmd.go +++ b/commands/command/cmd.go @@ -44,7 +44,12 @@ func parseAndValidateCmd(cmd interface{}, newCmd *Command) error { cmdList = append(cmdList, fmt.Sprintf("%s", v)) } // cut binary path and command name - cmdList = append(cmdList, os.Args[2:]...) + if len(os.Args) > 1 { + cmdList = append(cmdList, os.Args[2:]...) + } else if len(os.Args) == 1 { + cmdList = append(cmdList, os.Args[1:]...) + } + var escapedCmdList []string for _, val := range cmdList { escapedCmdList = append(escapedCmdList, escapeFlagValue(val)) diff --git a/commands/command/cmd_test.go b/commands/command/cmd_test.go index ec5b0687..12cb89cb 100644 --- a/commands/command/cmd_test.go +++ b/commands/command/cmd_test.go @@ -7,6 +7,22 @@ import ( ) func TestCommandFieldCmd(t *testing.T) { + t.Run("so subcommand in os.Args", func(t *testing.T) { + testCmd := NewCommand("test-cmd") + cmdArgs := "echo Hello" + // mock args + os.Args = []string{"bin_to_run"} + err := parseAndValidateCmd(cmdArgs, &testCmd) + + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + if testCmd.Cmd != cmdArgs { + t.Errorf("wrong output. \nexpect %s \ngot: %s", cmdArgs, testCmd.Cmd) + } + }) + t.Run("as string", func(t *testing.T) { testCmd := NewCommand("test-cmd") cmdArgs := "echo Hello" diff --git a/commands/run.go b/commands/run.go index 3af249fb..82c7c1f2 100644 --- a/commands/run.go +++ b/commands/run.go @@ -25,22 +25,14 @@ func RunCommand(cmdToRun command.Command, cfg *config.Config, out io.Writer) err return runCmd(cmdToRun, cfg, out, "") } -func convertEnvForCmd(envMap map[string]string) []string { - envList := make([]string, len(envMap)) +func convertEnvMapToList(envMap map[string]string) []string { + var envList []string for name, value := range envMap { envList = append(envList, fmt.Sprintf("%s=%s", name, value)) } return envList } -func convertOptsToEnvForCmd(opts map[string]string) []string { - envList := make([]string, len(opts)) - for name, value := range opts { - envList = append(envList, fmt.Sprintf("%s=%s", name, value)) - } - return envList -} - func convertChecksumToEnvForCmd(checksum string) []string { return []string{fmt.Sprintf("LETS_CHECKSUM=%s", checksum)} } @@ -79,11 +71,14 @@ func runCmd(cmdToRun command.Command, cfg *config.Config, out io.Writer, parentN cmdToRun.CliOptions = command.OptsToLetsCli(opts) // setup env for command - env := convertEnvForCmd(cmdToRun.Env) - optsEnv := convertOptsToEnvForCmd(cmdToRun.Options) - cliOptsEnv := convertOptsToEnvForCmd(cmdToRun.CliOptions) - checksumEnv := convertChecksumToEnvForCmd(cmdToRun.Checksum) - cmd.Env = composeEnvs(os.Environ(), env, optsEnv, cliOptsEnv, checksumEnv) + cmd.Env = composeEnvs( + os.Environ(), + convertEnvMapToList(cfg.Env), + convertEnvMapToList(cmdToRun.Env), + convertEnvMapToList(cmdToRun.Options), + convertEnvMapToList(cmdToRun.CliOptions), + convertChecksumToEnvForCmd(cmdToRun.Checksum), + ) if parentName == "" { logging.Log.Debugf( "Executing command\nname: %s\ncmd: %s\nenv:\n%s", diff --git a/commands/run_test.go b/commands/run_test.go new file mode 100644 index 00000000..0c85dc7d --- /dev/null +++ b/commands/run_test.go @@ -0,0 +1,19 @@ +package commands + +import ( + "testing" +) + + +func TestConvertEnvMapToList(t *testing.T) { + t.Run("should convert map to list of key=val", func(t *testing.T) { + env := make(map[string]string) + env["ONE"] = "1" + envList := convertEnvMapToList(env) + + exp := "ONE=1" + if envList[0] != exp { + t.Errorf("failed to convert env map to list. \nexp: %s\ngot: %s", exp, envList[0]) + } + }) +} \ No newline at end of file diff --git a/config/config.go b/config/config.go index 0b912455..caa8601b 100644 --- a/config/config.go +++ b/config/config.go @@ -14,9 +14,12 @@ var ( // COMMANDS is a top-level directive. Includes all commands to run COMMANDS = "commands" SHELL = "shell" + ENV = "env" + MIXINS = "mixins" ) -var validFields = strings.Join([]string{COMMANDS, SHELL}, " ") +var validConfigFields = strings.Join([]string{COMMANDS, SHELL, ENV, MIXINS}, " ") +var validMixinConfigFields = strings.Join([]string{COMMANDS, ENV}, " ") // Config is a struct for loaded config file type Config struct { @@ -24,6 +27,48 @@ type Config struct { FilePath string Commands map[string]command.Command Shell string + Env map[string]string + isMixin bool // if true, we consider config as mixin and apply different parsing and validation +} + +type ParseError struct { + Path struct { + Full string + Field string + } + Err error +} + +func (e *ParseError) Error() string { + return fmt.Sprintf("failed to parse config: %s", e.Err) +} + +func newConfigParseError(msg string, name string, field string) error { + fields := []string{name, field} + fullPath := strings.Join(fields, ".") + return &ParseError{ + Path: struct { + Full string + Field string + }{ + Full: fullPath, + Field: field, + }, + Err: fmt.Errorf("field %s: %s", fullPath, msg), + } +} + +func newConfig() *Config { + return &Config{ + Commands: make(map[string]command.Command), + Env: make(map[string]string), + } +} + +func newMixinConfig() *Config { + cfg := newConfig() + cfg.isMixin = true + return cfg } // Load a config from file @@ -72,10 +117,18 @@ func loadConfig(filename string) (*Config, error) { return config, nil } -func newConfig() *Config { - return &Config{ - Commands: make(map[string]command.Command), +func loadMixinConfig(filename string) (*Config, error) { + fileData, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err } + + config := newMixinConfig() + err = yaml.Unmarshal(fileData, config) + if err != nil { + return nil, err + } + return config, nil } // UnmarshalYAML unmarshals a config @@ -85,26 +138,123 @@ func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { if err := unmarshal(&rawKeyValue); err != nil { return err } - - if err := validateTopLevelFields(rawKeyValue, validFields); err != nil { - return err + if c.isMixin { + return unmarshalMixinConfig(rawKeyValue, c) } + return unmarshalConfig(rawKeyValue, c) +} + +func unmarshalConfigGeneral(rawKeyValue map[string]interface{}, cfg *Config) error { if cmds, ok := rawKeyValue[COMMANDS]; ok { - if err := c.loadCommands(cmds.(map[interface{}]interface{})); err != nil { + if err := cfg.loadCommands(cmds.(map[interface{}]interface{})); err != nil { + return err + } + } + if env, ok := rawKeyValue[ENV]; ok { + env, ok := env.(map[interface{}]interface{}) + if !ok { + return fmt.Errorf("env must be a mapping") + } + err := parseAndValidateEnv(env, cfg) + if err != nil { return err } } + return nil +} + +func unmarshalConfig(rawKeyValue map[string]interface{}, cfg *Config) error { + if err := validateTopLevelFields(rawKeyValue, validConfigFields); err != nil { + return err + } + + if err := unmarshalConfigGeneral(rawKeyValue, cfg); err != nil { + return err + } if shell, ok := rawKeyValue[SHELL]; ok { shell, ok := shell.(string) if !ok { return fmt.Errorf("shell must be a string") } - c.Shell = shell + cfg.Shell = shell } else { - return fmt.Errorf("shell must be specified in config") + return fmt.Errorf("'shell' field is required") } + if mixins, ok := rawKeyValue[MIXINS]; ok { + mixins, ok := mixins.([]interface{}) + if !ok { + return fmt.Errorf("mixins must be a list of string") + } + err := readAndValidateMixins(mixins, cfg) + if err != nil { + return err + } + } + + return nil +} + +func unmarshalMixinConfig(rawKeyValue map[string]interface{}, cfg *Config) error { + if err := validateTopLevelFields(rawKeyValue, validMixinConfigFields); err != nil { + return err + } + return unmarshalConfigGeneral(rawKeyValue, cfg) +} + +func readAndValidateMixins(mixins []interface{}, cfg *Config) error { + for _, filename := range mixins { + if filename, ok := filename.(string); ok { + mixinCfg, err := loadMixinConfig(filename) + if err != nil { + return fmt.Errorf("failed to load mixin config: %s", err) + } + if err := mergeConfigs(cfg, mixinCfg); err != nil { + return fmt.Errorf("failed to merge mixin config %s with main config: %s", filename, err) + } + } else { + return newConfigParseError( + "must be a string", + MIXINS, + "list item", + ) + } + } + return nil +} + +// Merge main and mixin configs. If there is a conflict - return error as we do not override values +// TODO add test +func mergeConfigs(mainCfg *Config, mixinCfg *Config) error { + for _, mixinCmd := range mixinCfg.Commands { + if _, conflict := mainCfg.Commands[mixinCmd.Name]; conflict { + return fmt.Errorf("command %s from mixin is already declared in main config's commands", mixinCmd.Name) + } + mainCfg.Commands[mixinCmd.Name] = mixinCmd + } + for mixinEnvKey, mixinEnvVal := range mixinCfg.Env { + if _, conflict := mainCfg.Env[mixinEnvKey]; conflict { + return fmt.Errorf("env %s from mixin is already declared in main config's env", mixinEnvKey) + } + mainCfg.Env[mixinEnvKey] = mixinEnvVal + } + return nil +} + +func parseAndValidateEnv(env map[interface{}]interface{}, cfg *Config) error { + for name, value := range env { + nameKey := name.(string) + if value, ok := value.(string); ok { + cfg.Env[nameKey] = value + } else { + return newConfigParseError( + "must be a string", + ENV, + nameKey, + ) + } + } return nil } diff --git a/config/validate.go b/config/validate.go index c05e0fdb..fa674da0 100644 --- a/config/validate.go +++ b/config/validate.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "github.com/kindritskyiMax/lets/commands/command" "strings" ) @@ -14,13 +15,15 @@ func Validate(config *Config) error { return validateCircularDepends(config) } +// if any two commands have each other command in deps, raise error +// TODO not optimal function but works for now func validateCircularDepends(cfg *Config) error { for _, cmdA := range cfg.Commands { for _, cmdB := range cfg.Commands { - depsA := strings.Join(cmdA.Depends, " ") - depsB := strings.Join(cmdB.Depends, " ") - if strings.Contains(depsB, cmdA.Name) && - strings.Contains(depsA, cmdB.Name) { + if cmdA.Name == cmdB.Name { + continue + } + if yes := depsIntersect(cmdA, cmdB); yes { return fmt.Errorf( "command '%s' have circular depends on command '%s'", fmt.Sprintf(NoticeColor, cmdA.Name), @@ -32,6 +35,18 @@ func validateCircularDepends(cfg *Config) error { return nil } +func depsIntersect(cmdA command.Command, cmdB command.Command) bool { + isNameInDeps := func(name string, deps []string) bool { + for _, dep := range deps { + if dep == name { + return true + } + } + return false + } + return isNameInDeps(cmdA.Name, cmdB.Depends) && isNameInDeps(cmdB.Name, cmdA.Depends) +} + func validateTopLevelFields(rawKeyValue map[string]interface{}, validFields string) error { for k := range rawKeyValue { if !strings.Contains(validFields, k) { diff --git a/config/validate_test.go b/config/validate_test.go new file mode 100644 index 00000000..d2c16fdd --- /dev/null +++ b/config/validate_test.go @@ -0,0 +1,76 @@ +package config + +import ( + "github.com/kindritskyiMax/lets/commands/command" + "testing" +) + +func TestValidateCircularDeps(t *testing.T) { + t.Run("command skip itself", func(t *testing.T) { + + testCfg := &Config{ + Commands: make(map[string]command.Command), + } + testCfg.Commands["a-cmd"] = command.Command{ + Name: "a-cmd", + Depends: []string{"noop"}, + } + testCfg.Commands["b-cmd"] = command.Command{ + Name: "b-cmd", + Depends: []string{"noop"}, + } + err := validateCircularDepends(testCfg) + + if err != nil { + t.Errorf("checked itself when validation circular depends. got: %s", err) + } + }) + + t.Run("command with similar name should not fail validation", func(t *testing.T) { + + testCfg := &Config{ + Commands: make(map[string]command.Command), + } + testCfg.Commands["a-cmd"] = command.Command{ + Name: "a-cmd", + Depends: []string{"b1-cmd"}, + } + testCfg.Commands["b"] = command.Command{ + Name: "b", + Depends: []string{"a-cmd"}, + } + testCfg.Commands["b1-cmd"] = command.Command{ + Name: "b1-cmd", + Depends: []string{"noop"}, + } + err := validateCircularDepends(testCfg) + + if err != nil { + t.Errorf("checked itself when validation circular depends. got: %s", err) + } + }) + + t.Run("validation should fail", func(t *testing.T) { + + testCfg := &Config{ + Commands: make(map[string]command.Command), + } + testCfg.Commands["a-cmd"] = command.Command{ + Name: "a-cmd", + Depends: []string{"b1-cmd"}, + } + testCfg.Commands["b"] = command.Command{ + Name: "b", + Depends: []string{"a-cmd"}, + } + testCfg.Commands["b1-cmd"] = command.Command{ + Name: "b1-cmd", + Depends: []string{"a-cmd"}, + } + err := validateCircularDepends(testCfg) + + if err == nil { + t.Errorf("validation should fail. got: %s", err) + } + }) +} diff --git a/docker/Dockerfile b/docker/Dockerfile index a1f7a7fd..cbb7aba6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,6 @@ FROM golang:1.13.7 +ENV GOPROXY https://proxy.golang.org WORKDIR /app RUN apt-get update && apt-get install git gcc diff --git a/lets-test.yaml b/lets-test.yaml new file mode 100644 index 00000000..dfbf6087 --- /dev/null +++ b/lets-test.yaml @@ -0,0 +1,57 @@ +shell: bash + +commands: + test-options: + description: Test options + options: | + Usage: + lets test-options [--kv-opt=] [--bool-opt] [--attr=...] [...] + + Options: + ... Positional args in the end + --bool-opt, -b Boolean opt + --kv-opt=, -K Key value opt + --attr=... Repeated kv args + cmd: | + echo "Flags command" + echo LETSOPT_KV_OPT=${LETSOPT_KV_OPT} + echo LETSOPT_BOOL_OPT=${LETSOPT_BOOL_OPT} + echo LETSOPT_ARGS=${LETSOPT_ARGS} + echo LETSOPT_ATTR=${LETSOPT_ATTR} + echo LETSCLI_KV_OPT=${LETSCLI_KV_OPT} + echo LETSCLI_BOOL_OPT=${LETSCLI_BOOL_OPT} + echo LETSCLI_ARGS=${LETSCLI_ARGS} + echo LETSCLI_ATTR=${LETSCLI_ATTR} + + test-env: + description: Test env + env: + MY_ENV: "my env val" + cmd: echo ${LETSOPT_MY_ENV} + + test-eval_env: + description: Test eval_env + eval_env: + CHECKSUM: echo "evaled_checksum" + + test-depends: + description: Test depends + depends: + - test-env + - test-eval_env + + test-cmd-as-list: + description: Test cmd as list + cmd: + - echo + - test + - cmd + + test-checksum: + description: test checksum + checksum: + - README.md + - LICENSE + cmd: + - echo + - "${LETS_CHECKSUM}" diff --git a/lets.yaml b/lets.yaml index fa96a866..ae6c6857 100644 --- a/lets.yaml +++ b/lets.yaml @@ -1,61 +1,9 @@ shell: bash -commands: - test-options: - description: Test options - options: | - Usage: - lets test-options [--kv-opt=] [--bool-opt] [--attr=...] [...] - - Options: - ... Positional args in the end - --bool-opt, -b Boolean opt - --kv-opt=, -K Key value opt - --attr=... Repeated kv args - cmd: | - echo "Flags command" - echo LETSOPT_KV_OPT=${LETSOPT_KV_OPT} - echo LETSOPT_BOOL_OPT=${LETSOPT_BOOL_OPT} - echo LETSOPT_ARGS=${LETSOPT_ARGS} - echo LETSOPT_ATTR=${LETSOPT_ATTR} - echo LETSCLI_KV_OPT=${LETSCLI_KV_OPT} - echo LETSCLI_BOOL_OPT=${LETSCLI_BOOL_OPT} - echo LETSCLI_ARGS=${LETSCLI_ARGS} - echo LETSCLI_ATTR=${LETSCLI_ATTR} - - test-env: - description: Test env - env: - MY_ENV: "my env val" - cmd: echo ${LETSOPT_MY_ENV} - - test-eval_env: - description: Test eval_env - eval_env: - CHECKSUM: echo "evaled_checksum" - - test-depends: - description: Test depends - depends: - - test-env - - test-eval_env - - test-cmd-as-list: - description: Test cmd as list - cmd: - - echo - - test - - cmd - - test-checksum: - description: test checksum - checksum: - - README.md - - LICENSE - cmd: - - echo - - "${LETS_CHECKSUM}" +mixins: + - build.yaml +commands: version: description: Update version.go file with latest tag options: | @@ -67,11 +15,13 @@ commands: test: description: Run tests for lets + depends: [build-docker] cmd: docker-compose run --rm test staticcheck: description: Run staticcheck for lets + depends: [build-docker] cmd: docker-compose run --rm staticheck diff --git a/test/config.go b/test/config.go index a89ec8ca..4eeb5f9d 100644 --- a/test/config.go +++ b/test/config.go @@ -7,7 +7,7 @@ import ( ) func GetTestConfig() *config.Config { - conf, err := config.Load("lets.yaml", "..") + conf, err := config.Load("lets-test.yaml", "..") if err != nil { fmt.Printf("can not read test config: %s", err) os.Exit(1)