Merge pull request #169 from lets-cli/fix-duplicate-files-for-checksum

fix duplicated files for checksum

checksum/checksum.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"sort"
 
+	"github.com/lets-cli/lets/set"
 	"github.com/lets-cli/lets/util"
 )
 
@@ -26,7 +27,7 @@ var checksumCache = make(map[string][]byte)
 //
 // return sorted list of files read by glob patterns.
 func readFilesFromPatterns(workDir string, patterns []string) ([]string, error) {
-	var files []string
+	filesSet := set.NewStringSet()
 
 	for _, pattern := range patterns {
 		absPatternPath := pattern
@@ -39,9 +40,10 @@ func readFilesFromPatterns(workDir string, patterns []string) ([]string, error)
 			return []string{}, fmt.Errorf("can not read file to calculate checksum: %w", err)
 		}
 
-		files = append(files, matches...)
+		filesSet.AddMany(matches)
 	}
 	// sort files list
+	files := filesSet.ToList()
 	sort.Strings(files)
 
 	return files, nil

config/config/config.go

@@ -1,5 +1,7 @@
 package config
 
+import "github.com/lets-cli/lets/set"
+
 var (
 	// COMMANDS is a top-level directive. Includes all commands to run.
 	COMMANDS = "commands"
@@ -12,8 +14,12 @@ var (
 )
 
 var (
-	ValidConfigFields      = []string{COMMANDS, SHELL, ENV, EvalEnv, MIXINS, VERSION, BEFORE}
-	ValidMixinConfigFields = []string{COMMANDS, ENV, EvalEnv, BEFORE}
+	ValidConfigDirectives = set.NewStringSetWithValues(
+		[]string{COMMANDS, SHELL, ENV, EvalEnv, MIXINS, VERSION, BEFORE},
+	)
+	ValidMixinConfigDirectives = set.NewStringSetWithValues(
+		[]string{COMMANDS, ENV, EvalEnv, BEFORE},
+	)
 )
 
 // Config is a struct for loaded config file.

config/parser/command.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 
 	"github.com/lets-cli/lets/config/config"
-	"github.com/lets-cli/lets/util"
+	"github.com/lets-cli/lets/set"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -24,7 +24,7 @@ var (
 	ARGS            = "args"
 )
 
-var validFields = []string{
+var directives = set.NewStringSetWithValues([]string{
 	CMD,
 	DESCRIPTION,
 	WORKDIR,
@@ -38,11 +38,11 @@ var validFields = []string{
 	AFTER,
 	REF,
 	ARGS,
-}
+})
 
 // parseCommand parses and validates unmarshaled yaml.
 func parseCommand(newCmd *config.Command, rawCommand map[string]interface{}, cfg *config.Config) error {
-	if err := validateCommandFields(rawCommand, validFields); err != nil {
+	if err := validateCommendDirectives(rawCommand); err != nil {
 		return err
 	}
 
@@ -158,9 +158,9 @@ func parseCommand(newCmd *config.Command, rawCommand map[string]interface{}, cfg
 	return nil
 }
 
-func validateCommandFields(rawKeyValue map[string]interface{}, validFields []string) error {
+func validateCommendDirectives(rawKeyValue map[string]interface{}) error {
 	for key := range rawKeyValue {
-		if !util.IsStringInList(key, validFields) {
+		if !directives.Contains(key) {
 			return fmt.Errorf("unknown command field '%s'", key)
 		}
 	}

config/parser/parser.go

@@ -126,21 +126,13 @@ func parseConfigGeneral(rawKeyValue map[string]interface{}, cfg *config.Config)
 	return nil
 }
 
-func validateTopLevelFields(rawKeyValue map[string]interface{}, validFields []string) error {
+func parseConfig(rawKeyValue map[string]interface{}, cfg *config.Config) error {
 	for key := range rawKeyValue {
-		if !util.IsStringInList(key, validFields) {
+		if !config.ValidConfigDirectives.Contains(key) {
 			return fmt.Errorf("unknown top-level field '%s'", key)
 		}
 	}
 
-	return nil
-}
-
-func parseConfig(rawKeyValue map[string]interface{}, cfg *config.Config) error {
-	if err := validateTopLevelFields(rawKeyValue, config.ValidConfigFields); err != nil {
-		return err
-	}
-
 	if err := parseConfigGeneral(rawKeyValue, cfg); err != nil {
 		return err
 	}
@@ -243,8 +235,10 @@ func parseMixinConfig(data []byte, mixinCfg *config.Config) error {
 		return fmt.Errorf("can not decode mixin config file: %w", err)
 	}
 
-	if err := validateTopLevelFields(rawKeyValue, config.ValidMixinConfigFields); err != nil {
-		return err
+	for key := range rawKeyValue {
+		if !config.ValidMixinConfigDirectives.Contains(key) {
+			return fmt.Errorf("unknown top-level field '%s'", key)
+		}
 	}
 
 	return parseConfigGeneral(rawKeyValue, mixinCfg)

docs/docs/changelog.md

@@ -5,17 +5,20 @@ title: Changelog
 
 ## [Unreleased]
 
+* `[Fixed]` **`Breaking change`** Fix duplicate files for checksum.
+  This will change checksum output if the same file has been read multiple times.
+
 ## [0.0.44](https://github.com/lets-cli/lets/releases/tag/v0.0.44)
 
-* ``[Fixed]`` Run ref declared in `depends` directive.
+* `[Fixed]` Run ref declared in `depends` directive.
 
 ## [0.0.43](https://github.com/lets-cli/lets/releases/tag/v0.0.43)
 
 * `[Noop]` Same as 0.0.42, deployed by accident.
 
 ## [0.0.42](https://github.com/lets-cli/lets/releases/tag/v0.0.42)
 
-* ``[Fixed]`` Fixed publish to `aur` repository.
+* `[Fixed]` Fixed publish to `aur` repository.
 
 ## [0.0.41](https://github.com/lets-cli/lets/releases/tag/v0.0.41)
 

set/set.go

@@ -0,0 +1,49 @@
+package set
+
+type StringSet struct {
+	entryMap map[string]struct{}
+}
+
+func (s *StringSet) Add(value string) {
+	s.entryMap[value] = struct{}{}
+}
+
+func (s *StringSet) AddMany(values []string) {
+	for _, value := range values {
+		s.entryMap[value] = struct{}{}
+	}
+}
+
+func (s *StringSet) ToList() []string {
+	values := make([]string, 0, len(s.entryMap))
+	for k := range s.entryMap {
+		values = append(values, k)
+	}
+
+	return values
+}
+
+func (s *StringSet) Remove(value string) {
+	delete(s.entryMap, value)
+}
+
+func (s *StringSet) Contains(value string) bool {
+	_, c := s.entryMap[value]
+
+	return c
+}
+
+func NewStringSet() *StringSet {
+	return &StringSet{
+		entryMap: make(map[string]struct{}),
+	}
+}
+
+func NewStringSetWithValues(values []string) *StringSet {
+	set := &StringSet{
+		entryMap: make(map[string]struct{}),
+	}
+	set.AddMany(values)
+
+	return set
+}

set/set_test.go

@@ -0,0 +1,63 @@
+package set
+
+import (
+	"reflect"
+	"sort"
+	"testing"
+)
+
+func TestStringSet(t *testing.T) {
+	t.Run("add string to set", func(t *testing.T) {
+		set := NewStringSet()
+
+		set.Add("a")
+		set.Add("b")
+		set.Add("a")
+		set.Add("c")
+
+		values := set.ToList()
+		sort.Strings(values)
+		if !reflect.DeepEqual(values, []string{"a", "b", "c"}) {
+			t.Errorf("set must contain only unique elements, got: %s", values)
+		}
+	})
+	t.Run("add many strings at once to set", func(t *testing.T) {
+		set := NewStringSet()
+
+		set.AddMany([]string{"a", "b", "c"})
+		set.Add("c")
+
+		values := set.ToList()
+		sort.Strings(values)
+		if !reflect.DeepEqual(values, []string{"a", "b", "c"}) {
+			t.Errorf("set must contain only unique elements, got: %s", values)
+		}
+	})
+
+	t.Run("remove string from set", func(t *testing.T) {
+		set := NewStringSet()
+
+		set.Add("a")
+		set.Add("b")
+		set.Add("c")
+		set.Remove("c")
+
+		values := set.ToList()
+		sort.Strings(values)
+		if !reflect.DeepEqual(values, []string{"a", "b"}) {
+			t.Errorf("set contains element which must be deleted, got: %s", values)
+		}
+	})
+
+	t.Run("remove string from set", func(t *testing.T) {
+		set := NewStringSet()
+
+		set.Add("a")
+		set.Add("b")
+		set.Add("c")
+
+		if !set.Contains("c") {
+			t.Errorf("set must contain element which was added, got: %s", set.ToList())
+		}
+	})
+}