From 064dd7442a87408b5daf346e5030e3050719db98 Mon Sep 17 00:00:00 2001 From: Gergo Torcsvari Date: Sun, 23 Jan 2022 20:07:44 +0100 Subject: [PATCH] list rules (groups) added --- README.md | 38 ++++++++++++++++++++----- go.mod | 8 +++++- go.sum | 6 ---- pkg/cli-utils/rule-command.go | 17 ++++++------ pkg/cli-utils/run-command.go | 2 ++ pkg/cli-utils/tm-command.go | 2 ++ pkg/parser/parser.go | 52 +++++++++++++++++++++-------------- pkg/parser/rule-list.go | 41 +++++++++++++++++++++++++++ pkg/parser/rule.go | 25 +++++++++++++---- rules/osx.yaml | 31 +++++++++++++++++++++ test-rules/node-multi.yaml | 25 +++++++++++++++++ test-scripts/list-tester.sh | 31 +++++++++++++++++++++ 12 files changed, 230 insertions(+), 48 deletions(-) create mode 100644 pkg/parser/rule-list.go create mode 100644 rules/osx.yaml create mode 100644 test-rules/node-multi.yaml create mode 100644 test-scripts/list-tester.sh diff --git a/README.md b/README.md index a11e4a1..95f1f92 100644 --- a/README.md +++ b/README.md @@ -68,11 +68,12 @@ heptapod prune -a ``` ### Dictionary - - search path - a path that we want to process - - ignore path - path that we don't process (further) - - exclude path - path that we don't want in our TM saves - - include path - path that we want in our TM saves - - ignore rule - a rule that parse git/docker ignore file format + - **path** - a file or a folder + - **search path** - a path that we want to process + - **ignore path** - path that we don't process (further) + - **exclude path** - path that we don't want in our TM saves + - **include path** - path that we want in our TM saves + - **ignore rule** - a rule that parse git/docker ignore file format ### Notes from TM migrating to a new machine When you try to migrate your TM state to a new machine @@ -89,7 +90,7 @@ There are two ways to exclude a dir from backups; - keeps the given flag when moved - can not be reliably list them (`mdfind com_apple_backup_excludeItem = 'com.apple.backupd'` is a close call, but some folders are excluded by mdfind too) -This tool excludes by flag! You can check any folder manually with `tmutil isexcluded`. If you delete a folder, it will be deleted with its flag. You don't need to clean up ever. +This tool excludes by flag! You can check any folder manually with `tmutil isexcluded`. **If you delete a folder, it will be deleted with its flag. You don't need to clean up ever.** Also, you can only exclude nonexcluded files with tmutil, so we only add them if they are exists and if they are not already added. ### Rules @@ -103,6 +104,8 @@ Every rule has a searchPaths, ignorePaths. - `file-trigger` - `regexp` - `ignore-file` + - `global` + - `list` - `settings` other type setting see below #### Ignore file (not yet implemented) @@ -123,6 +126,24 @@ Ignores files/dirs based on other files existence, made for easy language dep ig - `fileTrigger` like `package.json` or `.git` - `excludePaths` like `node-modules` or `.` +#### Global +Globally ignores or excludes path (overrides other rules walk ignores). + - `path` the global path we want to evaluate (like `~/.npm`) + - `handleWith` a string enum + - `ignore` - ignores globally + - `exclude` - ignores globally and excludes from backups + - any other value - do nothing + +#### List +Can group multiple rules to a single file. + - `subRules` the array of subRules (can be recursive with another `list`) + +Notes; + - while `rules ls` works, `enable`, `disable` and the path commands not working for the subrules, you need to edit the file manually + - if the list is disabled, all of it subs are handled as disables + - the group `searchPaths` and `ignorePaths` are currently do nothing + - probably this will change for easier customisations + ### Credits - [asimov](https://github.com/stevegrunwell/asimov) - [tmignore](https://github.com/samuelmeuli/tmignore) @@ -145,9 +166,12 @@ done: - brew package - ghactions - rule manage commands (list/enable/disable/ignoreAdd/ignoreRemove) +- global rule type +- list rule type (for better rule grouping) todos: -- handle global deps (m2, ivy, nvm, npm) +- reorganise rules +- fix the rule manage commands for lists - support tmignore functionality - support tmignore like funcionality with dockerignore - regexp pattern diff --git a/go.mod b/go.mod index 386c17f..3333cca 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,13 @@ go 1.17 require ( github.com/olekukonko/tablewriter v0.0.5 - github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 github.com/urfave/cli/v2 v2.3.0 gopkg.in/yaml.v2 v2.4.0 ) + +require ( + github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect + github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/russross/blackfriday/v2 v2.0.1 // indirect + github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect +) diff --git a/go.sum b/go.sum index 8a2b16f..e11a244 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= @@ -10,12 +9,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI= -github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= @@ -23,4 +18,3 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/cli-utils/rule-command.go b/pkg/cli-utils/rule-command.go index 0c216a5..d407d59 100644 --- a/pkg/cli-utils/rule-command.go +++ b/pkg/cli-utils/rule-command.go @@ -13,7 +13,7 @@ import ( ) var RuleCommands = &cli.Command{ - Name: "rules", + Name: "rules", Aliases: []string{}, Usage: "rule related functions", Subcommands: []*cli.Command{ @@ -111,7 +111,6 @@ func ruleEnable(pathIn string, enables []string) error { return nil } - func ruleDisable(pathIn string, enables []string) error { path, err := utils.FixupPathsToHandleHome(pathIn) if err != nil { @@ -192,7 +191,7 @@ func ruleIgnoreRemoveAll(pathIn string, excludePaths []string) error { continue } for _, n := range excludePaths { - rule.IgnorePaths = utils.Filter(rule.IgnorePaths, func(s string) bool {return s == n}) + rule.IgnorePaths = utils.Filter(rule.IgnorePaths, func(s string) bool { return s == n }) } err2 := parser.RuleWrite(*rule, fp) if err2 != nil { @@ -239,26 +238,26 @@ func writeErrorRules(paths []string) { table.Render() } -func writeTypeErrorRules(tes map[string]parser.Rule) { +func writeTypeErrorRules(tes []parser.Rule) { table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"path", "name", "type"}) table.SetBorder(false) - for k, v := range tes { - table.Append([]string{k, v.Name, v.RuleType}) + for _, v := range tes { + table.Append([]string{v.FileName, v.Name, v.RuleType}) } table.SetAutoMergeCells(false) table.Render() } -func writeRules(tes map[string]parser.Rule) { +func writeRules(tes []parser.Rule) { table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"path", "name", "type", "search", "ignore"}) table.SetBorder(false) - for k, v := range tes { - table.Append([]string{k, v.Name, v.RuleType, strings.Join(v.SearchPaths, ", "), strings.Join(v.IgnorePaths, ", ")}) + for _, v := range tes { + table.Append([]string{v.FileName, v.Name, v.RuleType, strings.Join(v.SearchPaths, ", "), strings.Join(v.IgnorePaths, ", ")}) } table.SetAutoMergeCells(false) diff --git a/pkg/cli-utils/run-command.go b/pkg/cli-utils/run-command.go index 36277c5..37e350f 100644 --- a/pkg/cli-utils/run-command.go +++ b/pkg/cli-utils/run-command.go @@ -6,6 +6,7 @@ import ( "github.com/tg44/heptapod/pkg/tmutil" "github.com/urfave/cli/v2" "log" + "runtime" "strings" ) @@ -25,6 +26,7 @@ var RunCommand = &cli.Command{ }, }, Action: func(c *cli.Context) error { + runtime.GOMAXPROCS(par) if dry { res := pkg.GetExcludedPaths(rulePath, par, buffer, verbose) fmt.Println("-----") diff --git a/pkg/cli-utils/tm-command.go b/pkg/cli-utils/tm-command.go index 1b4a3e0..3d5ca73 100644 --- a/pkg/cli-utils/tm-command.go +++ b/pkg/cli-utils/tm-command.go @@ -6,6 +6,7 @@ import ( "github.com/tg44/heptapod/pkg/tmutil" "github.com/urfave/cli/v2" "log" + "runtime" ) var file string @@ -75,6 +76,7 @@ var TmPrune = &cli.Command{ }, }, Action: func(c *cli.Context) error { + runtime.GOMAXPROCS(par) if current { res := pkg.GetExcludedPaths(rulePath, par, buffer, verbose) tmutil.RemovePathsFromTM(res, buffer, verbose) diff --git a/pkg/parser/parser.go b/pkg/parser/parser.go index 6e77d6b..85c5e3a 100644 --- a/pkg/parser/parser.go +++ b/pkg/parser/parser.go @@ -23,9 +23,9 @@ func ParseFromDir(path string) ([]walker.WalkJob, error) { type RuleGroups struct { FileErrors []string - TypeErrors map[string]Rule - Enabled map[string]Rule - Disabled map[string]Rule + TypeErrors []Rule + Enabled []Rule + Disabled []Rule } func ParseRulesFromDir(pathIn string) (*RuleGroups, error) { @@ -38,9 +38,9 @@ func ParseRulesFromDir(pathIn string) (*RuleGroups, error) { return nil, err } errors := []string{} - typeErrors := map[string]Rule{} - enabled := map[string]Rule{} - disabled := map[string]Rule{} + typeErrors := []Rule{} + enabled := []Rule{} + disabled := []Rule{} for _, file := range files { if strings.HasSuffix(file.Name(), ".yml") || strings.HasSuffix(file.Name(), ".yaml") { fp := filepath.Join(path, file.Name()) @@ -49,13 +49,17 @@ func ParseRulesFromDir(pathIn string) (*RuleGroups, error) { errors = append(errors, file.Name()) continue } - rt, _ := parseRuleTypes(*rule) + rt, _, _ := parseRuleTypes(rule) if len(rt) == 0 { - typeErrors[file.Name()] = *rule - } else if !rule.Enabled { - disabled[file.Name()] = *rule + typeErrors = append(typeErrors, *rule) } else { - enabled[file.Name()] = *rule + for _, r := range rule.flattened() { + if !r.Enabled { + disabled = append(disabled, r) + } else { + enabled = append(enabled, r) + } + } } } } @@ -89,34 +93,42 @@ func parse(ruleFile string) ([]walker.WalkJob, []string) { return []walker.WalkJob{}, []string{} } - return parseRuleTypes(*rule) + w, i, _ := parseRuleTypes(rule) + return w, i } -func parseRuleTypes(rule Rule) ([]walker.WalkJob, []string) { +func parseRuleTypes(rule *Rule) ([]walker.WalkJob, []string, []Rule) { if rule.RuleType == "file-trigger" { settings, err2 := fileTriggerSettingsParse(rule.RuleSettings) if err2 != nil { log.Println(err2) - return []walker.WalkJob{}, []string{} + return []walker.WalkJob{}, []string{}, []Rule{} } tasks := []walker.WalkJob{} - walkerFun := fileTriggerWalker(rule, *settings) + walkerFun := fileTriggerWalker(*rule, *settings) for _, p := range rule.SearchPaths { tasks = append(tasks, walker.WalkJob{p, []walker.Walker{walkerFun}, []string{}}) } - return tasks, []string{} + return tasks, []string{}, []Rule{} } else if rule.RuleType == "global" { settings, err2 := globalSettingsParse(rule.RuleSettings) if err2 != nil { log.Println(err2) - return []walker.WalkJob{}, []string{} + return []walker.WalkJob{}, []string{}, []Rule{} } tasks := []walker.WalkJob{} - walkerFun := globalWalker(rule, *settings) + walkerFun := globalWalker(*rule, *settings) tasks = append(tasks, walker.WalkJob{"/", []walker.Walker{walkerFun}, []string{}}) - return tasks, getGlobalIgnore(*settings) + return tasks, getGlobalIgnore(*settings), []Rule{} + } else if rule.RuleType == "list" { + settings, err2 := listSettingsParse(rule.RuleSettings, parseRuleTypes) + if err2 != nil { + log.Println(err2) + } + rule.SubRules = settings.SubRules + return settings.walkers, settings.globalIgnores, settings.SubRules } - return []walker.WalkJob{}, []string{} + return []walker.WalkJob{}, []string{}, []Rule{} } func mergeJobs(works []walker.WalkJob, globalIgnores []string) []walker.WalkJob { diff --git a/pkg/parser/rule-list.go b/pkg/parser/rule-list.go new file mode 100644 index 0000000..c4389a4 --- /dev/null +++ b/pkg/parser/rule-list.go @@ -0,0 +1,41 @@ +package parser + +import ( + "errors" + "github.com/tg44/heptapod/pkg/walker" + "gopkg.in/yaml.v2" + "log" +) + +type ListSettings struct { + SubRules []Rule + walkers []walker.WalkJob + globalIgnores []string +} + +func listSettingsParse(i map[string]interface{}, parser func(*Rule) ([]walker.WalkJob, []string, []Rule)) (*ListSettings, error) { + subsI, found1 := i["subRules"].([]interface{}) + walkers := []walker.WalkJob{} + ignores := []string{} + rules := []Rule{} + for _, v := range subsI { + var rule Rule + data, err := yaml.Marshal(&v) + if err != nil { + log.Println("subrule cannot be marshalled back") + continue + } + err2 := yaml.Unmarshal(data, &rule) + if err2 != nil { + log.Println("subrule cannot be parsed back as a rule") + } + w, i, _ := parser(&rule) + walkers = append(walkers, w...) + ignores = append(ignores, i...) + rules = append(rules, rule) + } + if found1 { + return &ListSettings{rules, walkers, ignores}, nil + } + return nil, errors.New("The given input can't be parsed to ListSettings!") +} diff --git a/pkg/parser/rule.go b/pkg/parser/rule.go index 9d6fbb1..fbebd1b 100644 --- a/pkg/parser/rule.go +++ b/pkg/parser/rule.go @@ -7,12 +7,14 @@ import ( ) type Rule struct { - Name string `yaml:"name"` - Enabled bool `yaml:"enabled"` - SearchPaths []string `yaml:"searchPaths"` - IgnorePaths []string `yaml:"ignorePaths"` - RuleType string `yaml:"ruleType"` + Name string `yaml:"name"` + Enabled bool `yaml:"enabled"` + SearchPaths []string `yaml:"searchPaths"` + IgnorePaths []string `yaml:"ignorePaths"` + RuleType string `yaml:"ruleType"` RuleSettings map[string]interface{} `yaml:"ruleSettings"` + SubRules []Rule + FileName string } func RuleParse(fileName string) (*Rule, error) { @@ -30,6 +32,7 @@ func RuleParse(fileName string) (*Rule, error) { return nil, err } + rule.FileName = fileName return &rule, nil } @@ -47,3 +50,15 @@ func RuleWrite(rule Rule, file string) error { } return nil } + +func (r Rule) flattened() []Rule { + ret := []Rule{r} + for _, sr := range r.SubRules { + srm := sr + srm.Name = r.Name + "/" + sr.Name + srm.FileName = r.FileName + srm.Enabled = sr.Enabled && r.Enabled + ret = append(ret, srm.flattened()...) + } + return ret +} diff --git a/rules/osx.yaml b/rules/osx.yaml new file mode 100644 index 0000000..45fe144 --- /dev/null +++ b/rules/osx.yaml @@ -0,0 +1,31 @@ +name: "node-multi" +enabled: true +searchPaths: [] +ignorePaths: [] +ruleType: "list" +ruleSettings: + subRules: + - name: "library" + enabled: true + searchPaths: [] + ignorePaths: [] + ruleType: "global" + ruleSettings: + path: "~/Library" + handleWith: "ignore" + - name: "downloads" + enabled: false + searchPaths: [ ] + ignorePaths: [ ] + ruleType: "global" + ruleSettings: + path: "~/Downloads" + handleWith: "exclude" + - name: "trash" + enabled: false + searchPaths: [ ] + ignorePaths: [ ] + ruleType: "global" + ruleSettings: + path: "~/.Trash" + handleWith: "exclude" diff --git a/test-rules/node-multi.yaml b/test-rules/node-multi.yaml new file mode 100644 index 0000000..f1c2e72 --- /dev/null +++ b/test-rules/node-multi.yaml @@ -0,0 +1,25 @@ +name: "node-multi" +enabled: true +searchPaths: [] +ignorePaths: [] +ruleType: "list" +ruleSettings: + subRules: + - name: "node" + enabled: true + searchPaths: [ "~/dev" ] + ignorePaths: [ "~/dev/alpakka", "~/dev/akka" ] + ruleType: "file-trigger" + ruleSettings: + fileTrigger: "package.json" + excludePaths: + - "node_modules" + - name: "node-npm-global" + enabled: true + searchPaths: [ ] + ignorePaths: [ ] + ruleType: "global" + ruleSettings: + path: "~/.npm" + handleWith: "ignore" + diff --git a/test-scripts/list-tester.sh b/test-scripts/list-tester.sh new file mode 100644 index 0000000..9c39731 --- /dev/null +++ b/test-scripts/list-tester.sh @@ -0,0 +1,31 @@ +mkdir -p ~/test_hepta/rules +cat << EOF > ~/test_hepta/rules/test-rule.yaml +name: "node-multi" +enabled: true +searchPaths: [] +ignorePaths: [] +ruleType: "list" +ruleSettings: + subRules: + - name: "test-node" + enabled: true + searchPaths: [] + ignorePaths: [] + ruleType: "global" + ruleSettings: + path: "~/.npm" + handleWith: "exclude" + - name: "test-node" + enabled: true + searchPaths: ["~"] + ignorePaths: ["~/dev", "~/go", "~/tmp", "~/temp", "~/sdk", "~/Library"] + ruleType: "file-trigger" + ruleSettings: + fileTrigger: "test.file" + excludePaths: + - "node_modules" + - ".venv" + +EOF +./heptapod -r ~/test_hepta/rules -v 4 run -d +rm -rf ~/test_hepta