Bug fixes and new features

.github/workflows/tests.yml

@@ -0,0 +1,53 @@
+name: k8s
+on:
+  push:
+    branches:
+      - "*"
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  tests:
+    name: e2e testing
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Install soft
+      run: |
+        sudo apt update
+        sudo apt install -y make jq curl git zip
+        curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+        sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+        wget https://releases.hashicorp.com/terraform/1.4.4/terraform_1.4.4_linux_amd64.zip
+        unzip terraform_1.4.4_linux_amd64.zip
+        sudo cp terraform /usr/local/bin/
+
+    - name: Code checkout
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 2
+
+    - uses: rinx/[email protected]
+      name: Deploy k3d cluster
+
+    - name: Build cdev
+      run: |
+        git config --global --add safe.directory /__w/cluster.dev/cluster.dev
+        git fetch --prune --unshallow --tags
+        cp Dockerfile-alpine Dockerfile
+        docker build -t tmp-cdev:latest .
+        id=$(docker create tmp-cdev:latest)
+        docker cp $id:/bin/cdev /usr/local/bin/
+        docker rm -v $id
+
+    - name: Run AWS deploy/destroy and 'cdev deep graph' tests
+      run: cd tests/test-project/ && cdev apply --force -l debug && cdev destroy --force -l debug
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+    - name: Run kubernetes/k8s-manifest/helm tests
+      run: |
+        export KUBE_CONTEXT=$(kubectl config current-context)
+        cd tests/k8s/ && cdev apply --force -l debug && cdev destroy --force -l debug

internal/cmd/cdev/apply.go

@@ -23,10 +23,10 @@ var applyCmd = &cobra.Command{
 			return NewCmdErr(project, "apply", err)
 		}
 		err = project.LockState()
-		defer project.UnLockState()
 		if err != nil {
 			return NewCmdErr(project, "apply", err)
 		}
+		defer project.UnLockState()
 		err = project.Apply()
 		if err != nil {
 			return NewCmdErr(project, "apply", err)

internal/cmd/cdev/destroy.go

@@ -18,10 +18,10 @@ var destroyCmd = &cobra.Command{
 			return NewCmdErr(project, "destroy", err)
 		}
 		err = project.LockState()
-		defer project.UnLockState()
 		if err != nil {
 			return NewCmdErr(project, "destroy", err)
 		}
+		defer project.UnLockState()
 		err = project.Destroy()
 		if err != nil {
 			return NewCmdErr(project, "destroy", err)

internal/project/commands.go

@@ -111,6 +111,7 @@ func (p *Project) StartSigTrap(stop chan struct{}) {
 				os.Exit(0)
 			case <-stop:
 				close(p.HupUnlockChan)
+				signal.Stop(p.HupUnlockChan)
 				return
 			}
 		}
@@ -200,10 +201,7 @@ func applyRoutine(graphUnit *UnitPlanningStatus, finFunc func(error), p *Project
 	p.ProcessedUnitsCount++
 	err = graphUnit.UnitPtr.Apply()
 	if err != nil {
-		state, _ := utils.JSONEncode(graphUnit.UnitPtr)
-		log.Warnf("applyRoutine: %v", string(state))
 		if graphUnit.UnitPtr.IsTainted() {
-			// log.Warnf("applyRoutine: tainted %v", graphUnit.UnitPtr.Key())
 			p.OwnState.UpdateUnit(graphUnit.UnitPtr)
 		}
 		finFunc(fmt.Errorf("apply unit: %v", err.Error()))

internal/project/helpers.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"path/filepath"
 	"reflect"
 	"regexp"
 	"strings"
@@ -234,25 +233,6 @@ func printVersion() string {
 	return config.Global.Version
 }
 
-func removeDirContent(dir string) error {
-	d, err := os.Open(dir)
-	if err != nil {
-		return err
-	}
-	defer d.Close()
-	names, err := d.Readdirnames(-1)
-	if err != nil {
-		return err
-	}
-	for _, name := range names {
-		err = os.RemoveAll(filepath.Join(dir, name))
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 // ScanMarkers use marker scanner function to replace templated markers.
 func ScanMarkers(data interface{}, procFunc MarkerScanner, unit Unit) error {
 	if data == nil {

internal/project/project.go

@@ -79,7 +79,6 @@ func NewEmptyProject() *Project {
 	log.Info("Checking for newer releases...")
 	err := DiscoverCdevLastRelease()
 	if err != nil {
-		log.Warnf("Version check: %v.", err)
 		project.NewVersionMessage = fmt.Sprintf("Version check: %v", err.Error())
 	}
 	return project
@@ -273,16 +272,16 @@ func (p *Project) ClearCacheDir() error {
 		return nil
 	}
 	if !config.Global.UseCache {
-		log.Debugf("Removes all old content: './%s'", relPath)
+		log.Debugf("Removes all old content: './%s'", p.CodeCacheDir)
 
-		err := removeDirContent(p.CodeCacheDir)
+		err := utils.RemoveDirContent(p.CodeCacheDir)
 		if err != nil {
 			return err
 		}
 		if _, err := os.Stat(config.Global.TemplatesCacheDir); os.IsNotExist(err) {
 			return nil
 		}
-		return removeDirContent(config.Global.TemplatesCacheDir)
+		return utils.RemoveDirContent(config.Global.TemplatesCacheDir)
 	}
 	return nil
 }

internal/project/project_config.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"strings"
 
 	"github.com/apex/log"
 	"github.com/shalb/cluster.dev/internal/config"
@@ -13,6 +14,7 @@ import (
 )
 
 const defaultProjectName = "default-project"
+const ignoreFileName = ".cdevignore"
 
 func (p *Project) parseProjectConfig() error {
 
@@ -60,9 +62,26 @@ func (p *Project) readManifests() (err error) {
 	files = append(files, filesYML...)
 	objFiles := make(map[string][]byte)
 
+	ignoreFileFullPath := filepath.Join(config.Global.WorkingDir, ignoreFileName)
+	ignoreData, _ := os.ReadFile(ignoreFileFullPath) // Ignore error, its ok
+	ignoreList := strings.Split(string(ignoreData), "\n")
+
+	ignoreFileCheck := func(filename string) bool {
+		for _, ignoreFile := range ignoreList {
+			if ignoreFile == filename {
+				return true
+			}
+		}
+		return false
+	}
+
 	for _, file := range files {
-		// log.Warnf("Read Files: %v", file)
+		// log.Warnf("Read Files: %v, list: %v", file, ignoreList)
 		fileName, _ := filepath.Rel(config.Global.WorkingDir, file)
+		if ignoreFileCheck(fileName) {
+			log.Debugf("File ignored: %v", fileName)
+			continue
+		}
 		isProjectConfig := regexp.MustCompile(ConfigFilePattern).MatchString(fileName)
 		if isProjectConfig {
 			p.configDataFile, err = os.ReadFile(file)

internal/project/stack.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/apex/log"
 	"github.com/shalb/cluster.dev/internal/config"
@@ -155,8 +156,26 @@ func (s *Stack) ReadTemplate(src string) (err error) {
 	if err != nil {
 		return err
 	}
+	ignoreFileFullPath := filepath.Join(s.TemplateDir, ignoreFileName)
+	ignoreData, _ := os.ReadFile(ignoreFileFullPath) // Ignore error, its ok
+	ignoreList := strings.Split(string(ignoreData), "\n")
+
+	ignoreFileCheck := func(filename string) bool {
+		for _, ignoreFile := range ignoreList {
+			if ignoreFile == filename {
+				return true
+			}
+		}
+		return false
+	}
 	s.Templates = []stackTemplate{}
 	for _, fn := range templatesFilesList {
+		// if ignoreFileCheck(fn)
+		checkFileName, _ := filepath.Rel(s.TemplateDir, fn)
+		if ignoreFileCheck(checkFileName) {
+			log.Debugf("Ignore stackTemplate file: %v", fn)
+			continue
+		}
 		tmplData, err := os.ReadFile(fn)
 		if err != nil {
 			return err

internal/project/stack_template.go

@@ -47,12 +47,12 @@ func NewStackTemplate(data []byte) (*stackTemplate, error) {
 		log.Debug("Checking client version...")
 		reqVerConstraints, err := semver.NewConstraint(iTmpl.ReqClientVersion)
 		if err != nil {
-			return nil, fmt.Errorf("parsing template: can't parse required client version: %v", iTmpl.ReqClientVersion)
+			return nil, fmt.Errorf("parsing template: can't parse required client version: %v, err: %v", iTmpl.ReqClientVersion, err)
 		}
 		ver, err := semver.NewVersion(config.Global.Version)
 		if err != nil {
 			// Invalid current cli version. Maybe test revision.
-			return nil, fmt.Errorf("parsing template: can't parse client version: %v", iTmpl.ReqClientVersion)
+			return nil, fmt.Errorf("parsing template: can't parse client version: %v", err)
 		}
 		ok, messages := reqVerConstraints.Validate(ver)
 		errReasons := ""

internal/project/state.go

@@ -57,7 +57,6 @@ func (p *Project) SaveState() error {
 	}
 	// log.Errorf("units links: %+v\n Project: %+v", st.UnitLinks, p.UnitLinks)
 	for key, unit := range p.Units {
-		log.Warnf("SaveState %v", key)
 		st.Units[key] = unit.GetState()
 	}
 	// Remove all unit links, that not have a target unit.
@@ -192,7 +191,7 @@ func (p *Project) LoadState() (*StateProject, error) {
 			return nil, fmt.Errorf("load state: create state cache dir: %w", err)
 		}
 	}
-	err := removeDirContent(config.Global.StateCacheDir)
+	err := utils.RemoveDirContent(config.Global.StateCacheDir)
 	if err != nil {
 		return nil, fmt.Errorf("load state: remove state cache dir: %w", err)
 	}

internal/project/ui/generator.go

@@ -408,7 +408,11 @@ func ClearScreen() {
 
 func GetTemplateGenerators(tmplSrc string) (tmplDir string, err error) {
 	if !config.Global.UseCache {
-		os.RemoveAll(config.Global.TemplatesCacheDir)
+		utils.RemoveDirContent(config.Global.TemplatesCacheDir)
+		err = os.Remove(config.Global.TemplatesCacheDir)
+		if err != nil {
+			return
+		}
 	}
 	err = os.MkdirAll(config.Global.TemplatesCacheDir, os.ModePerm)
 	if err != nil {

internal/project/ulinks.go

@@ -3,8 +3,6 @@ package project
 import (
 	"fmt"
 	"sync"
-
-	"github.com/apex/log"
 )
 
 // ULinkT describe unit link betwen one target unit and multiple cli units, which uses this unit (output or remote state, or custom unit dependency).
@@ -214,9 +212,7 @@ func (o *UnitLinksT) UniqUnits() map[string]Unit {
 		if unit != nil {
 			continue
 		}
-		if el.Unit == nil {
-			log.Warnf("Dev debug. Nil unit pointer %v. Pls check.", el.UnitKey())
-		}
+
 		res[el.UnitKey()] = el.Unit
 	}
 	return res

internal/secrets/aws_secretmanager/main.go

@@ -111,7 +111,7 @@ func (s *smDriver) Create(files map[string][]byte) error {
 		command := fmt.Sprintf("%s %s", editor, filename)
 		err = runner.RunWithTty(command)
 		if err != nil {
-			os.RemoveAll(filename)
+			os.Remove(filename)
 			return fmt.Errorf("secrets: create secret: %v", err.Error())
 		}
 	}

internal/secrets/sops/main.go

@@ -85,13 +85,13 @@ func (s *sopsDriver) Create(files map[string][]byte) error {
 		command := fmt.Sprintf("sops -e --encrypted-regex ^encrypted_data$ -i %s", filename)
 		err = runner.RunWithTty(command)
 		if err != nil {
-			os.RemoveAll(filename)
+			os.Remove(filename)
 			return fmt.Errorf("create sops secret: %v", err.Error())
 		}
 		command = fmt.Sprintf("sops %s", filename)
 		err = runner.RunWithTty(command)
 		if err != nil && err.Error() != "exit status 200" {
-			os.RemoveAll(filename)
+			os.Remove(filename)
 			log.Debugf("err %+v", err)
 			return fmt.Errorf("create sops secret: %v", err.Error())
 		}

internal/units/shell/common/unit.go

@@ -81,9 +81,8 @@ type Unit struct {
 func (u *Unit) SetTainted(newValue bool, err error) {
 	u.Tainted = newValue
 	u.ExecErr = err
-	log.Warnf("SetTainted %v", u.Key())
 	if u.SavedState != nil {
-		log.Warnf("SetTainted %v", u.SavedState.Key())
+		log.Debugf("SetTainted %v", u.SavedState.Key())
 		u.SavedState.SetTainted(newValue, err)
 	}
 }