list rules (groups) added

README.md

@@ -68,11 +68,12 @@ heptapod prune -a
 ```
 
 ### Dictionary
- - search path - a path that we want to process
- - ignore path - path that we don't process (further)
- - exclude path - path that we don't want in our TM saves
- - include path - path that we want in our TM saves
- - ignore rule - a rule that parse git/docker ignore file format
+ - **path** - a file or a folder
+ - **search path** - a path that we want to process
+ - **ignore path** - path that we don't process (further)
+ - **exclude path** - path that we don't want in our TM saves
+ - **include path** - path that we want in our TM saves
+ - **ignore rule** - a rule that parse git/docker ignore file format
 
 ### Notes from TM migrating to a new machine
 When you try to migrate your TM state to a new machine
@@ -89,7 +90,7 @@ There are two ways to exclude a dir from backups;
   - keeps the given flag when moved
   - can not be reliably list them (`mdfind com_apple_backup_excludeItem = 'com.apple.backupd'` is a close call, but some folders are excluded by mdfind too)
 
-This tool excludes by flag! You can check any folder manually with `tmutil isexcluded`. If you delete a folder, it will be deleted with its flag. You don't need to clean up ever.
+This tool excludes by flag! You can check any folder manually with `tmutil isexcluded`. **If you delete a folder, it will be deleted with its flag. You don't need to clean up ever.**
 Also, you can only exclude nonexcluded files with tmutil, so we only add them if they are exists and if they are not already added.
 
 ### Rules
@@ -103,6 +104,8 @@ Every rule has a searchPaths, ignorePaths.
    - `file-trigger`
    - `regexp`
    - `ignore-file`
+   - `global`
+   - `list`
  - `settings` other type setting see below
 
 #### Ignore file (not yet implemented)
@@ -123,6 +126,24 @@ Ignores files/dirs based on other files existence, made for easy language dep ig
  - `fileTrigger` like `package.json` or `.git`
  - `excludePaths` like `node-modules` or `.`
 
+#### Global
+Globally ignores or excludes path (overrides other rules walk ignores).
+ - `path` the global path we want to evaluate (like `~/.npm`)
+ - `handleWith` a string enum
+    - `ignore` - ignores globally
+    - `exclude` - ignores globally and excludes from backups
+    - any other value - do nothing
+
+#### List
+Can group multiple rules to a single file.
+ - `subRules` the array of subRules (can be recursive with another `list`)
+
+Notes;
+ - while `rules ls` works, `enable`, `disable` and the path commands not working for the subrules, you need to edit the file manually
+ - if the list is disabled, all of it subs are handled as disables
+ - the group `searchPaths` and `ignorePaths` are currently do nothing
+   - probably this will change for easier customisations 
+
 ### Credits
  - [asimov](https://github.com/stevegrunwell/asimov)
  - [tmignore](https://github.com/samuelmeuli/tmignore)
@@ -145,9 +166,12 @@ done:
 - brew package
 - ghactions
 - rule manage commands (list/enable/disable/ignoreAdd/ignoreRemove)
+- global rule type
+- list rule type (for better rule grouping)
 
 todos:
-- handle global deps (m2, ivy, nvm, npm)
+- reorganise rules
+- fix the rule manage commands for lists
 - support tmignore functionality
 - support tmignore like funcionality with dockerignore
 - regexp pattern

go.mod

@@ -4,7 +4,13 @@ go 1.17
 
 require (
 	github.com/olekukonko/tablewriter v0.0.5
-	github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06
 	github.com/urfave/cli/v2 v2.3.0
 	gopkg.in/yaml.v2 v2.4.0
 )
+
+require (
+	github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/russross/blackfriday/v2 v2.0.1 // indirect
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+)

go.sum

@@ -1,7 +1,6 @@
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
@@ -10,17 +9,12 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 h1:OkMGxebDjyw0ULyrTYWeN0UNCCkmCWfjPnIA2W6oviI=
-github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06/go.mod h1:+ePHsJ1keEjQtpvf9HHw0f4ZeJ0TLRsxhunSI2hYJSs=
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
 github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/cli-utils/rule-command.go

@@ -13,7 +13,7 @@ import (
 )
 
 var RuleCommands = &cli.Command{
-	Name: "rules",
+	Name:    "rules",
 	Aliases: []string{},
 	Usage:   "rule related functions",
 	Subcommands: []*cli.Command{
@@ -111,7 +111,6 @@ func ruleEnable(pathIn string, enables []string) error {
 	return nil
 }
 
-
 func ruleDisable(pathIn string, enables []string) error {
 	path, err := utils.FixupPathsToHandleHome(pathIn)
 	if err != nil {
@@ -192,7 +191,7 @@ func ruleIgnoreRemoveAll(pathIn string, excludePaths []string) error {
 				continue
 			}
 			for _, n := range excludePaths {
-				rule.IgnorePaths = utils.Filter(rule.IgnorePaths, func(s string) bool {return s == n})
+				rule.IgnorePaths = utils.Filter(rule.IgnorePaths, func(s string) bool { return s == n })
 			}
 			err2 := parser.RuleWrite(*rule, fp)
 			if err2 != nil {
@@ -239,26 +238,26 @@ func writeErrorRules(paths []string) {
 	table.Render()
 }
 
-func writeTypeErrorRules(tes map[string]parser.Rule) {
+func writeTypeErrorRules(tes []parser.Rule) {
 	table := tablewriter.NewWriter(os.Stdout)
 	table.SetHeader([]string{"path", "name", "type"})
 	table.SetBorder(false)
 
-	for k, v := range tes {
-		table.Append([]string{k, v.Name, v.RuleType})
+	for _, v := range tes {
+		table.Append([]string{v.FileName, v.Name, v.RuleType})
 	}
 
 	table.SetAutoMergeCells(false)
 	table.Render()
 }
 
-func writeRules(tes map[string]parser.Rule) {
+func writeRules(tes []parser.Rule) {
 	table := tablewriter.NewWriter(os.Stdout)
 	table.SetHeader([]string{"path", "name", "type", "search", "ignore"})
 	table.SetBorder(false)
 
-	for k, v := range tes {
-		table.Append([]string{k, v.Name, v.RuleType, strings.Join(v.SearchPaths, ", "), strings.Join(v.IgnorePaths, ", ")})
+	for _, v := range tes {
+		table.Append([]string{v.FileName, v.Name, v.RuleType, strings.Join(v.SearchPaths, ", "), strings.Join(v.IgnorePaths, ", ")})
 	}
 
 	table.SetAutoMergeCells(false)

pkg/cli-utils/run-command.go

@@ -6,6 +6,7 @@ import (
 	"github.com/tg44/heptapod/pkg/tmutil"
 	"github.com/urfave/cli/v2"
 	"log"
+	"runtime"
 	"strings"
 )
 
@@ -25,6 +26,7 @@ var RunCommand = &cli.Command{
 		},
 	},
 	Action: func(c *cli.Context) error {
+		runtime.GOMAXPROCS(par)
 		if dry {
 			res := pkg.GetExcludedPaths(rulePath, par, buffer, verbose)
 			fmt.Println("-----")

pkg/cli-utils/tm-command.go

@@ -6,6 +6,7 @@ import (
 	"github.com/tg44/heptapod/pkg/tmutil"
 	"github.com/urfave/cli/v2"
 	"log"
+	"runtime"
 )
 
 var file string
@@ -75,6 +76,7 @@ var TmPrune = &cli.Command{
 		},
 	},
 	Action: func(c *cli.Context) error {
+		runtime.GOMAXPROCS(par)
 		if current {
 			res := pkg.GetExcludedPaths(rulePath, par, buffer, verbose)
 			tmutil.RemovePathsFromTM(res, buffer, verbose)

pkg/parser/parser.go

@@ -23,9 +23,9 @@ func ParseFromDir(path string) ([]walker.WalkJob, error) {
 
 type RuleGroups struct {
 	FileErrors []string
-	TypeErrors map[string]Rule
-	Enabled    map[string]Rule
-	Disabled   map[string]Rule
+	TypeErrors []Rule
+	Enabled    []Rule
+	Disabled   []Rule
 }
 
 func ParseRulesFromDir(pathIn string) (*RuleGroups, error) {
@@ -38,9 +38,9 @@ func ParseRulesFromDir(pathIn string) (*RuleGroups, error) {
 		return nil, err
 	}
 	errors := []string{}
-	typeErrors := map[string]Rule{}
-	enabled := map[string]Rule{}
-	disabled := map[string]Rule{}
+	typeErrors := []Rule{}
+	enabled := []Rule{}
+	disabled := []Rule{}
 	for _, file := range files {
 		if strings.HasSuffix(file.Name(), ".yml") || strings.HasSuffix(file.Name(), ".yaml") {
 			fp := filepath.Join(path, file.Name())
@@ -49,13 +49,17 @@ func ParseRulesFromDir(pathIn string) (*RuleGroups, error) {
 				errors = append(errors, file.Name())
 				continue
 			}
-			rt, _ := parseRuleTypes(*rule)
+			rt, _, _ := parseRuleTypes(rule)
 			if len(rt) == 0 {
-				typeErrors[file.Name()] = *rule
-			} else if !rule.Enabled {
-				disabled[file.Name()] = *rule
+				typeErrors = append(typeErrors, *rule)
 			} else {
-				enabled[file.Name()] = *rule
+				for _, r := range rule.flattened() {
+					if !r.Enabled {
+						disabled = append(disabled, r)
+					} else {
+						enabled = append(enabled, r)
+					}
+				}
 			}
 		}
 	}
@@ -89,34 +93,42 @@ func parse(ruleFile string) ([]walker.WalkJob, []string) {
 		return []walker.WalkJob{}, []string{}
 	}
 
-	return parseRuleTypes(*rule)
+	w, i, _ := parseRuleTypes(rule)
+	return w, i
 }
 
-func parseRuleTypes(rule Rule) ([]walker.WalkJob, []string) {
+func parseRuleTypes(rule *Rule) ([]walker.WalkJob, []string, []Rule) {
 	if rule.RuleType == "file-trigger" {
 		settings, err2 := fileTriggerSettingsParse(rule.RuleSettings)
 		if err2 != nil {
 			log.Println(err2)
-			return []walker.WalkJob{}, []string{}
+			return []walker.WalkJob{}, []string{}, []Rule{}
 		}
 		tasks := []walker.WalkJob{}
-		walkerFun := fileTriggerWalker(rule, *settings)
+		walkerFun := fileTriggerWalker(*rule, *settings)
 		for _, p := range rule.SearchPaths {
 			tasks = append(tasks, walker.WalkJob{p, []walker.Walker{walkerFun}, []string{}})
 		}
-		return tasks, []string{}
+		return tasks, []string{}, []Rule{}
 	} else if rule.RuleType == "global" {
 		settings, err2 := globalSettingsParse(rule.RuleSettings)
 		if err2 != nil {
 			log.Println(err2)
-			return []walker.WalkJob{}, []string{}
+			return []walker.WalkJob{}, []string{}, []Rule{}
 		}
 		tasks := []walker.WalkJob{}
-		walkerFun := globalWalker(rule, *settings)
+		walkerFun := globalWalker(*rule, *settings)
 		tasks = append(tasks, walker.WalkJob{"/", []walker.Walker{walkerFun}, []string{}})
-		return tasks, getGlobalIgnore(*settings)
+		return tasks, getGlobalIgnore(*settings), []Rule{}
+	} else if rule.RuleType == "list" {
+		settings, err2 := listSettingsParse(rule.RuleSettings, parseRuleTypes)
+		if err2 != nil {
+			log.Println(err2)
+		}
+		rule.SubRules = settings.SubRules
+		return settings.walkers, settings.globalIgnores, settings.SubRules
 	}
-	return []walker.WalkJob{}, []string{}
+	return []walker.WalkJob{}, []string{}, []Rule{}
 }
 
 func mergeJobs(works []walker.WalkJob, globalIgnores []string) []walker.WalkJob {

pkg/parser/rule-list.go

@@ -0,0 +1,41 @@
+package parser
+
+import (
+	"errors"
+	"github.com/tg44/heptapod/pkg/walker"
+	"gopkg.in/yaml.v2"
+	"log"
+)
+
+type ListSettings struct {
+	SubRules      []Rule
+	walkers       []walker.WalkJob
+	globalIgnores []string
+}
+
+func listSettingsParse(i map[string]interface{}, parser func(*Rule) ([]walker.WalkJob, []string, []Rule)) (*ListSettings, error) {
+	subsI, found1 := i["subRules"].([]interface{})
+	walkers := []walker.WalkJob{}
+	ignores := []string{}
+	rules := []Rule{}
+	for _, v := range subsI {
+		var rule Rule
+		data, err := yaml.Marshal(&v)
+		if err != nil {
+			log.Println("subrule cannot be marshalled back")
+			continue
+		}
+		err2 := yaml.Unmarshal(data, &rule)
+		if err2 != nil {
+			log.Println("subrule cannot be parsed back as a rule")
+		}
+		w, i, _ := parser(&rule)
+		walkers = append(walkers, w...)
+		ignores = append(ignores, i...)
+		rules = append(rules, rule)
+	}
+	if found1 {
+		return &ListSettings{rules, walkers, ignores}, nil
+	}
+	return nil, errors.New("The given input can't be parsed to ListSettings!")
+}

pkg/parser/rule.go

@@ -7,12 +7,14 @@ import (
 )
 
 type Rule struct {
-	Name       string `yaml:"name"`
-	Enabled bool `yaml:"enabled"`
-	SearchPaths []string `yaml:"searchPaths"`
-	IgnorePaths []string `yaml:"ignorePaths"`
-	RuleType string `yaml:"ruleType"`
+	Name         string                 `yaml:"name"`
+	Enabled      bool                   `yaml:"enabled"`
+	SearchPaths  []string               `yaml:"searchPaths"`
+	IgnorePaths  []string               `yaml:"ignorePaths"`
+	RuleType     string                 `yaml:"ruleType"`
 	RuleSettings map[string]interface{} `yaml:"ruleSettings"`
+	SubRules     []Rule
+	FileName     string
 }
 
 func RuleParse(fileName string) (*Rule, error) {
@@ -30,6 +32,7 @@ func RuleParse(fileName string) (*Rule, error) {
 		return nil, err
 	}
 
+	rule.FileName = fileName
 	return &rule, nil
 }
 
@@ -47,3 +50,15 @@ func RuleWrite(rule Rule, file string) error {
 	}
 	return nil
 }
+
+func (r Rule) flattened() []Rule {
+	ret := []Rule{r}
+	for _, sr := range r.SubRules {
+		srm := sr
+		srm.Name = r.Name + "/" + sr.Name
+		srm.FileName = r.FileName
+		srm.Enabled = sr.Enabled && r.Enabled
+		ret = append(ret, srm.flattened()...)
+	}
+	return ret
+}