From 84363be0e882bc8443629ac0536d656efc718bed Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Wed, 27 Nov 2024 09:27:28 +0100 Subject: [PATCH 1/7] Fix Readme after merging master into develop --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index c228107..ecb04bc 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ -
Katenary Logo -
[![Documentation Status](https://readthedocs.org/projects/katenary/badge/?version=latest)](https://katenary.readthedocs.io/en/latest/?badge=latest) From 628b35d4714993f22bfd6da24fa483107cb1770a Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Sun, 1 Dec 2024 08:48:33 +0100 Subject: [PATCH 2/7] doc(readme): fix the schema location --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ecb04bc..2a0d6b0 100644 --- a/README.md +++ b/README.md @@ -260,7 +260,7 @@ return { settings = { yaml = { schemas = { - ["https://raw.githubusercontent.com/metal3d/katenary/refs/heads/master/katenary.json"] = "katenary.yaml", + ["https://raw.githubusercontent.com/metal3d/katenary/master/katenary.json"] = "katenary.yaml", }, }, }, @@ -276,12 +276,12 @@ Use this address to validate the `katenary.yaml` file in VSCode: ```json { "yaml.schemas": { - "https://raw.githubusercontent.com/metal3d/katenary/refs/heads/master/katenary.json": "katenary.yaml" + "https://raw.githubusercontent.com/metal3d/katenary/master/katenary.json": "katenary.yaml" } } ``` -> You can, of course, replace the `refs/heads/master` with a specific tag or branch. +> You can, of course, replace the `master` with a specific tag or branch. ## What a name… From d458cdbd734a52a796772a44be9b3175a8fd1ab9 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Tue, 3 Dec 2024 13:50:58 +0100 Subject: [PATCH 3/7] chore(configmap): Manage binary data in configMap We should be now able to detect and manage binary files to be injected in configMaps --- generator/configMap.go | 35 ++++++++++++++++--- generator/volume_test.go | 72 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 5 deletions(-) diff --git a/generator/configMap.go b/generator/configMap.go index d65811d..61cfd23 100644 --- a/generator/configMap.go +++ b/generator/configMap.go @@ -9,6 +9,7 @@ import ( "path/filepath" "regexp" "strings" + "unicode/utf8" "github.com/compose-spec/compose-go/types" corev1 "k8s.io/api/core/v1" @@ -140,7 +141,7 @@ func NewConfigMapFromDirectory(service types.ServiceConfig, appName, path string // cumulate the path to the WorkingDir path = filepath.Join(service.WorkingDir, path) path = filepath.Clean(path) - cm.AppendDir(path) + cm.AppenddDir(path) return cm } @@ -149,9 +150,17 @@ func (c *ConfigMap) AddData(key, value string) { c.Data[key] = value } +// AddBinaryData adds binary data to the configmap. Append or overwrite the value if the key already exists. +func (c *ConfigMap) AddBinaryData(key string, value []byte) { + if c.BinaryData == nil { + c.BinaryData = make(map[string][]byte) + } + c.BinaryData[key] = value +} + // AddFile adds files from given path to the configmap. It is not recursive, to add all files in a directory, // you need to call this function for each subdirectory. -func (c *ConfigMap) AppendDir(path string) { +func (c *ConfigMap) AppenddDir(path string) { // read all files in the path and add them to the configmap stat, err := os.Stat(path) if err != nil { @@ -165,6 +174,7 @@ func (c *ConfigMap) AppendDir(path string) { } for _, file := range files { if file.IsDir() { + utils.Warn("Subdirectories are ignored for the moment, skipping", filepath.Join(path, file.Name())) continue } path := filepath.Join(path, file.Name()) @@ -174,7 +184,12 @@ func (c *ConfigMap) AppendDir(path string) { } // remove the path from the file filename := filepath.Base(path) - c.AddData(filename, string(content)) + if utf8.Valid(content) { + c.AddData(filename, string(content)) + } else { + c.AddBinaryData(filename, content) + } + } } else { // add the file to the configmap @@ -182,7 +197,12 @@ func (c *ConfigMap) AppendDir(path string) { if err != nil { log.Fatal(err) } - c.AddData(filepath.Base(path), string(content)) + filename := filepath.Base(path) + if utf8.Valid(content) { + c.AddData(filename, string(content)) + } else { + c.AddBinaryData(filename, content) + } } } @@ -199,7 +219,12 @@ func (c *ConfigMap) AppendFile(path string) { if err != nil { log.Fatal(err) } - c.AddData(filepath.Base(path), string(content)) + if utf8.Valid(content) { + c.AddData(filepath.Base(path), string(content)) + } else { + c.AddBinaryData(filepath.Base(path), content) + } + } } diff --git a/generator/volume_test.go b/generator/volume_test.go index ef365be..700fa6d 100644 --- a/generator/volume_test.go +++ b/generator/volume_test.go @@ -2,8 +2,13 @@ package generator import ( "fmt" + "image" + "image/color" + "image/png" "katenary/generator/labels" + "log" "os" + "path/filepath" "testing" v1 "k8s.io/api/apps/v1" @@ -149,6 +154,73 @@ services: } } +func TestBinaryMount(t *testing.T) { + composeFile := ` +services: + web: + image: nginx + volumes: + - ./images/foo.png:/var/www/foo + labels: + %[1]s/configmap-files: |- + - ./images/foo.png +` + composeFile = fmt.Sprintf(composeFile, labels.KatenaryLabelPrefix) + tmpDir := setup(composeFile) + log.Println(tmpDir) + defer teardown(tmpDir) + + os.Mkdir(filepath.Join(tmpDir, "images"), 0o755) + + // create a png image + pngFile := tmpDir + "/images/foo.png" + w, h := 100, 100 + img := image.NewRGBA(image.Rect(0, 0, w, h)) + red := color.RGBA{255, 0, 0, 255} + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + img.Set(x, y, red) + } + } + + blue := color.RGBA{0, 0, 255, 255} + for y := 30; y < 70; y++ { + for x := 30; x < 70; x++ { + img.Set(x, y, blue) + } + } + currentDir, _ := os.Getwd() + os.Chdir(tmpDir) + defer os.Chdir(currentDir) + + f, err := os.Create(pngFile) + if err != nil { + t.Fatal(err) + } + png.Encode(f, img) + f.Close() + output := internalCompileTest(t, "-s", "templates/web/deployment.yaml") + d := v1.Deployment{} + yaml.Unmarshal([]byte(output), &d) + volumes := d.Spec.Template.Spec.Volumes + if len(volumes) != 1 { + t.Errorf("Expected 1 volume, got %d", len(volumes)) + } + + cm := corev1.ConfigMap{} + cmContent, err := helmTemplate(ConvertOptions{ + OutputDir: "chart", + }, "-s", "templates/web/statics/images/configmap.yaml") + yaml.Unmarshal([]byte(cmContent), &cm) + if im, ok := cm.BinaryData["foo.png"]; !ok { + t.Errorf("Expected foo.png to be in the configmap") + } else { + if len(im) == 0 { + t.Errorf("Expected image to be non-empty") + } + } +} + func TestBindFrom(t *testing.T) { composeFile := ` services: From 6dc92df4b50832672a93fc6993a63497ce56738d Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Tue, 3 Dec 2024 13:51:39 +0100 Subject: [PATCH 4/7] chore(icons): Better icons for warning and info --- utils/icons.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/icons.go b/utils/icons.go index 999e082..5028c24 100644 --- a/utils/icons.go +++ b/utils/icons.go @@ -9,13 +9,13 @@ type Icon string const ( IconSuccess Icon = "✅" IconFailure Icon = "❌" - IconWarning Icon = "⚠️'" + IconWarning Icon = "❕" IconNote Icon = "📝" IconWorld Icon = "🌐" IconPlug Icon = "🔌" IconPackage Icon = "📦" IconCabinet Icon = "🗄️" - IconInfo Icon = "❕" + IconInfo Icon = "🔵" IconSecret Icon = "🔒" IconConfig Icon = "🔧" IconDependency Icon = "🔗" From 3833037862ea436bec0b6a1b6a88b0761a62996a Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Tue, 3 Dec 2024 13:52:22 +0100 Subject: [PATCH 5/7] doc(refresh): after changing names and adding functions --- doc/docs/packages/generator.md | 41 +++++++++++++++++++++------------- doc/docs/packages/utils.md | 4 ++-- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/doc/docs/packages/generator.md b/doc/docs/packages/generator.md index 4bf6f23..cd4f809 100644 --- a/doc/docs/packages/generator.md +++ b/doc/docs/packages/generator.md @@ -119,7 +119,7 @@ type ChartTemplate struct { ``` -## type [ConfigMap]() +## type [ConfigMap]() ConfigMap is a kubernetes ConfigMap. Implements the DataMap interface. @@ -131,7 +131,7 @@ type ConfigMap struct { ``` -### func [NewConfigMap]() +### func [NewConfigMap]() ```go func NewConfigMap(service types.ServiceConfig, appName string, forFile bool) *ConfigMap @@ -140,7 +140,7 @@ func NewConfigMap(service types.ServiceConfig, appName string, forFile bool) *Co NewConfigMap creates a new ConfigMap from a compose service. The appName is the name of the application taken from the project name. The ConfigMap is filled by environment variables and labels "map\-env". -### func [NewConfigMapFromDirectory]() +### func [NewConfigMapFromDirectory]() ```go func NewConfigMapFromDirectory(service types.ServiceConfig, appName, path string) *ConfigMap @@ -148,26 +148,26 @@ func NewConfigMapFromDirectory(service types.ServiceConfig, appName, path string NewConfigMapFromDirectory creates a new ConfigMap from a compose service. This path is the path to the file or directory. If the path is a directory, all files in the directory are added to the ConfigMap. Each subdirectory are ignored. Note that the Generate\(\) function will create the subdirectories ConfigMaps. - -### func \(\*ConfigMap\) [AddData]() + +### func \(\*ConfigMap\) [AddBinaryData]() ```go -func (c *ConfigMap) AddData(key, value string) +func (c *ConfigMap) AddBinaryData(key string, value []byte) ``` -AddData adds a key value pair to the configmap. Append or overwrite the value if the key already exists. +AddBinaryData adds binary data to the configmap. Append or overwrite the value if the key already exists. - -### func \(\*ConfigMap\) [AppendDir]() + +### func \(\*ConfigMap\) [AddData]() ```go -func (c *ConfigMap) AppendDir(path string) +func (c *ConfigMap) AddData(key, value string) ``` -AddFile adds files from given path to the configmap. It is not recursive, to add all files in a directory, you need to call this function for each subdirectory. +AddData adds a key value pair to the configmap. Append or overwrite the value if the key already exists. -### func \(\*ConfigMap\) [AppendFile]() +### func \(\*ConfigMap\) [AppendFile]() ```go func (c *ConfigMap) AppendFile(path string) @@ -175,8 +175,17 @@ func (c *ConfigMap) AppendFile(path string) + +### func \(\*ConfigMap\) [AppenddDir]() + +```go +func (c *ConfigMap) AppenddDir(path string) +``` + +AddFile adds files from given path to the configmap. It is not recursive, to add all files in a directory, you need to call this function for each subdirectory. + -### func \(\*ConfigMap\) [Filename]() +### func \(\*ConfigMap\) [Filename]() ```go func (c *ConfigMap) Filename() string @@ -185,7 +194,7 @@ func (c *ConfigMap) Filename() string Filename returns the filename of the configmap. If the configmap is used for files, the filename contains the path. -### func \(\*ConfigMap\) [SetData]() +### func \(\*ConfigMap\) [SetData]() ```go func (c *ConfigMap) SetData(data map[string]string) @@ -194,7 +203,7 @@ func (c *ConfigMap) SetData(data map[string]string) SetData sets the data of the configmap. It replaces the entire data. -### func \(\*ConfigMap\) [Yaml]() +### func \(\*ConfigMap\) [Yaml]() ```go func (c *ConfigMap) Yaml() ([]byte, error) @@ -421,7 +430,7 @@ func (d *Deployment) Yaml() ([]byte, error) Yaml returns the yaml representation of the deployment. -## type [FileMapUsage]() +## type [FileMapUsage]() FileMapUsage is the usage of the filemap. diff --git a/doc/docs/packages/utils.md b/doc/docs/packages/utils.md index 2f0ad01..35a62ba 100644 --- a/doc/docs/packages/utils.md +++ b/doc/docs/packages/utils.md @@ -196,13 +196,13 @@ type Icon string const ( IconSuccess Icon = "✅" IconFailure Icon = "❌" - IconWarning Icon = "⚠️'" + IconWarning Icon = "❕" IconNote Icon = "📝" IconWorld Icon = "🌐" IconPlug Icon = "🔌" IconPackage Icon = "📦" IconCabinet Icon = "🗄️" - IconInfo Icon = "❕" + IconInfo Icon = "🔵" IconSecret Icon = "🔒" IconConfig Icon = "🔧" IconDependency Icon = "🔗" From eb760d429983b0dcf517af7a101034b4c7b00296 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Tue, 3 Dec 2024 14:03:36 +0100 Subject: [PATCH 6/7] test(subdir): Add globally mount binary files --- generator/volume_test.go | 67 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/generator/volume_test.go b/generator/volume_test.go index 700fa6d..9c9a97a 100644 --- a/generator/volume_test.go +++ b/generator/volume_test.go @@ -221,6 +221,73 @@ services: } } +func TestGloballyBinaryMount(t *testing.T) { + composeFile := ` +services: + web: + image: nginx + volumes: + - ./images:/var/www/foo + labels: + %[1]s/configmap-files: |- + - ./images +` + composeFile = fmt.Sprintf(composeFile, labels.KatenaryLabelPrefix) + tmpDir := setup(composeFile) + log.Println(tmpDir) + defer teardown(tmpDir) + + os.Mkdir(filepath.Join(tmpDir, "images"), 0o755) + + // create a png image + pngFile := tmpDir + "/images/foo.png" + w, h := 100, 100 + img := image.NewRGBA(image.Rect(0, 0, w, h)) + red := color.RGBA{255, 0, 0, 255} + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + img.Set(x, y, red) + } + } + + blue := color.RGBA{0, 0, 255, 255} + for y := 30; y < 70; y++ { + for x := 30; x < 70; x++ { + img.Set(x, y, blue) + } + } + currentDir, _ := os.Getwd() + os.Chdir(tmpDir) + defer os.Chdir(currentDir) + + f, err := os.Create(pngFile) + if err != nil { + t.Fatal(err) + } + png.Encode(f, img) + f.Close() + output := internalCompileTest(t, "-s", "templates/web/deployment.yaml") + d := v1.Deployment{} + yaml.Unmarshal([]byte(output), &d) + volumes := d.Spec.Template.Spec.Volumes + if len(volumes) != 1 { + t.Errorf("Expected 1 volume, got %d", len(volumes)) + } + + cm := corev1.ConfigMap{} + cmContent, err := helmTemplate(ConvertOptions{ + OutputDir: "chart", + }, "-s", "templates/web/statics/images/configmap.yaml") + yaml.Unmarshal([]byte(cmContent), &cm) + if im, ok := cm.BinaryData["foo.png"]; !ok { + t.Errorf("Expected foo.png to be in the configmap") + } else { + if len(im) == 0 { + t.Errorf("Expected image to be non-empty") + } + } +} + func TestBindFrom(t *testing.T) { composeFile := ` services: From e574a2e2a869b7d3fa02f48da7941db8a8f8c975 Mon Sep 17 00:00:00 2001 From: Patrice Ferlet Date: Tue, 3 Dec 2024 14:37:13 +0100 Subject: [PATCH 7/7] chore(errors): Better error management We must remove all "Fatal" calls and use errors instead, to be returned and managed globally. This is the first step, but it is, at this time, a real problem. Tests are complicated without this. --- cmd/katenary/main.go | 4 ++-- generator/configMap.go | 21 ++++++++++++--------- generator/configMap_test.go | 17 +++++++++++++++++ generator/converter.go | 13 +++++++------ generator/tools_test.go | 4 +++- 5 files changed, 41 insertions(+), 18 deletions(-) diff --git a/cmd/katenary/main.go b/cmd/katenary/main.go index dc0c9ca..8a88969 100644 --- a/cmd/katenary/main.go +++ b/cmd/katenary/main.go @@ -141,11 +141,11 @@ func generateConvertCommand() *cobra.Command { convertCmd := &cobra.Command{ Use: "convert", Short: "Converts a docker-compose file to a Helm Chart", - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { if givenAppVersion != "" { appVersion = &givenAppVersion } - generator.Convert(generator.ConvertOptions{ + return generator.Convert(generator.ConvertOptions{ Force: force, OutputDir: outputDir, Profiles: profiles, diff --git a/generator/configMap.go b/generator/configMap.go index 61cfd23..5d79449 100644 --- a/generator/configMap.go +++ b/generator/configMap.go @@ -1,6 +1,7 @@ package generator import ( + "fmt" "katenary/generator/labels" "katenary/generator/labels/labelStructs" "katenary/utils" @@ -141,7 +142,7 @@ func NewConfigMapFromDirectory(service types.ServiceConfig, appName, path string // cumulate the path to the WorkingDir path = filepath.Join(service.WorkingDir, path) path = filepath.Clean(path) - cm.AppenddDir(path) + cm.AppendDir(path) return cm } @@ -160,17 +161,17 @@ func (c *ConfigMap) AddBinaryData(key string, value []byte) { // AddFile adds files from given path to the configmap. It is not recursive, to add all files in a directory, // you need to call this function for each subdirectory. -func (c *ConfigMap) AppenddDir(path string) { +func (c *ConfigMap) AppendDir(path string) error { // read all files in the path and add them to the configmap stat, err := os.Stat(path) if err != nil { - log.Fatalf("Path %s does not exist\n", path) + return fmt.Errorf("Path %s does not exist, %w\n", path, err) } // recursively read all files in the path and add them to the configmap if stat.IsDir() { files, err := os.ReadDir(path) if err != nil { - log.Fatal(err) + return err } for _, file := range files { if file.IsDir() { @@ -180,7 +181,7 @@ func (c *ConfigMap) AppenddDir(path string) { path := filepath.Join(path, file.Name()) content, err := os.ReadFile(path) if err != nil { - log.Fatal(err) + return err } // remove the path from the file filename := filepath.Base(path) @@ -195,7 +196,7 @@ func (c *ConfigMap) AppenddDir(path string) { // add the file to the configmap content, err := os.ReadFile(path) if err != nil { - log.Fatal(err) + return err } filename := filepath.Base(path) if utf8.Valid(content) { @@ -204,20 +205,21 @@ func (c *ConfigMap) AppenddDir(path string) { c.AddBinaryData(filename, content) } } + return nil } -func (c *ConfigMap) AppendFile(path string) { +func (c *ConfigMap) AppendFile(path string) error { // read all files in the path and add them to the configmap stat, err := os.Stat(path) if err != nil { - log.Fatalf("Path %s does not exist\n", path) + return fmt.Errorf("Path %s doesn not exists, %w", path, err) } // recursively read all files in the path and add them to the configmap if !stat.IsDir() { // add the file to the configmap content, err := os.ReadFile(path) if err != nil { - log.Fatal(err) + return err } if utf8.Valid(content) { c.AddData(filepath.Base(path), string(content)) @@ -226,6 +228,7 @@ func (c *ConfigMap) AppendFile(path string) { } } + return nil } // Filename returns the filename of the configmap. If the configmap is used for files, the filename contains the path. diff --git a/generator/configMap_test.go b/generator/configMap_test.go index a0b5f80..6e880e6 100644 --- a/generator/configMap_test.go +++ b/generator/configMap_test.go @@ -6,6 +6,7 @@ import ( "os" "testing" + "github.com/compose-spec/compose-go/types" v1 "k8s.io/api/core/v1" "sigs.k8s.io/yaml" ) @@ -73,3 +74,19 @@ services: t.Errorf("Expected FOO to be baz, got %s", v) } } + +func TestAppendBadFile(t *testing.T) { + cm := NewConfigMap(types.ServiceConfig{}, "app", true) + err := cm.AppendFile("foo") + if err == nil { + t.Errorf("Expected error, got nil") + } +} + +func TestAppendBadDir(t *testing.T) { + cm := NewConfigMap(types.ServiceConfig{}, "app", true) + err := cm.AppendDir("foo") + if err == nil { + t.Errorf("Expected error, got nil") + } +} diff --git a/generator/converter.go b/generator/converter.go index cd0d11d..1d55e46 100644 --- a/generator/converter.go +++ b/generator/converter.go @@ -90,7 +90,7 @@ var keyRegExp = regexp.MustCompile(`^\s*[^#]+:.*`) // Convert a compose (docker, podman...) project to a helm chart. // It calls Generate() to generate the chart and then write it to the disk. -func Convert(config ConvertOptions, dockerComposeFile ...string) { +func Convert(config ConvertOptions, dockerComposeFile ...string) error { var ( templateDir = filepath.Join(config.OutputDir, "templates") helpersPath = filepath.Join(config.OutputDir, "templates", "_helpers.tpl") @@ -105,7 +105,7 @@ func Convert(config ConvertOptions, dockerComposeFile ...string) { // go to the root of the project if err := os.Chdir(filepath.Dir(dockerComposeFile[0])); err != nil { fmt.Println(utils.IconFailure, err) - os.Exit(1) + return err } defer os.Chdir(currentDir) // after the generation, go back to the original directory @@ -118,13 +118,13 @@ func Convert(config ConvertOptions, dockerComposeFile ...string) { project, err := parser.Parse(config.Profiles, config.EnvFiles, dockerComposeFile...) if err != nil { fmt.Println(err) - os.Exit(1) + return err } // check older version of labels if err := checkOldLabels(project); err != nil { fmt.Println(utils.IconFailure, err) - os.Exit(1) + return err } // TODO: use katenary.yaml file here to set the labels @@ -140,7 +140,7 @@ func Convert(config ConvertOptions, dockerComposeFile ...string) { ) if !overwrite { fmt.Println("Aborting") - os.Exit(126) // 126 is the exit code for "Command invoked cannot execute" + return nil } } fmt.Println() // clean line @@ -150,7 +150,7 @@ func Convert(config ConvertOptions, dockerComposeFile ...string) { chart, err := Generate(project) if err != nil { fmt.Println(err) - os.Exit(1) + return err } // if the app version is set from the command line, use it @@ -194,6 +194,7 @@ func Convert(config ConvertOptions, dockerComposeFile ...string) { // call helm update if needed callHelmUpdate(config) + return nil } func addChartDoc(values []byte, project *types.Project) []byte { diff --git a/generator/tools_test.go b/generator/tools_test.go index 9d1591e..2395411 100644 --- a/generator/tools_test.go +++ b/generator/tools_test.go @@ -48,7 +48,9 @@ func internalCompileTest(t *testing.T, options ...string) string { AppVersion: appVersion, ChartVersion: chartVersion, } - Convert(convertOptions, "compose.yml") + if err := Convert(convertOptions, "compose.yml"); err != nil { + return err.Error() + } // launch helm lint to check the generated chart if helmLint(convertOptions) != nil {