Remove concept of glob key

Closes #2.

build/docs/render.adoc

@@ -1,13 +1,13 @@
 The purpose of this task is to render a asciidoc template located in the repository into a PDF. In addition to just transforming the asciidoc file to PDF, the task is also able to render information gathered from YAML/JSON files (such as ODS Pipeline artifacts) into the asciidoc file before transforming it to PDF.
 
-The task expects a glob pattern pointing to one or more Go template files (given by parameter `template`). It renders each found Go template with data gathered from files matching the `data-sources` parameter, which defaults to `artifacts:.ods/artifacts/\*/*.json;artifacts:.ods/artifacts/\*/*.yaml`. The asciidoc template can then access data parsed from these files under the key specified before the colon of each glob pattern. For example, if file `.ods/artifacts/org.foo/some.json` contains:
+The task expects a glob pattern pointing to one or more Go template files (given by parameter `template`). It renders each found Go template with data gathered from files matching the `data-sources` parameter, which defaults to `.ods/*;.ods/repos/*/.ods/*.ods/artifacts/*/*.json;.ods/artifacts/*/*.yaml`. The asciidoc template can then access data parsed from these files. For example, if file `.ods/artifacts/org.foo/some.json` contains:
 
 ```
 {"a":"b"}
 ```
 
-The asciidoc template can access the value of the field `a` by referencing `{{.artifacts.org_foo.some.a}}`. Note that any non-alphanumeric character in the file path is replaced with an underscore. In general, field access in templates is possible via `<key>.<path>.<file>.<field>`. In the special case where the file is located at the root of the directory, the `<path>` section is set to `root`. For example, the glob pattern `metadata:*.yaml` might match the file `foo.yaml` in the root of the repository - its data will be accessible as `metadata.root.foo.fieldname`.
+The asciidoc template can access the value of the field `a` by referencing `{{.ods.artifacts.org_foo.some.a}}`. Note that any non-alphanumeric character in the file path is replaced with an underscore, and leading or trailing underscores are trimmed.
 
-Note that only JSON and YAML formats are recognized. If a matching file does not end in either `.json` or `.y(a)ml`, its entire content is made available under the key `value`. For example, the glob pattern `log:*.log` might match the file `pipeline-run.log`, which would expose the content of the file as `log.root.pipeline_run.value` to the template.
+Note that only JSON and YAML formats are recognized as such. If a matching file does not end in either `.json` or `.y(a)ml`, its entire content is made available under the key `value`. For example, the glob pattern `*.log` might match the file `pipeline-run.log`, which would expose the content of the file as `pipeline_run.value` to the template.
 
 After the Go template has been rendered, link:https://github.com/asciidoctor/asciidoctor-pdf[asciidoctor-pdf] is used to turn each rendered asciidoc file into a PDF file. The resulting files are placed into the directory specified by `output-dir` (defaulting to `.ods/artifacts/org.opendevstack.pipeline.adoc.pdf` so that created PDFs are preserved as artifacts in Nexus). Theming is possible by specifying the `pdf-theme` parameter as explained in the link:https://docs.asciidoctor.org/pdf-converter/latest/theme/apply-theme/#theme-and-font-directories[Theme and font directories] documentation.

build/tasks/render.yaml

@@ -27,10 +27,8 @@ spec:
       description: >-
         Glob patterns from where to source data.
         Multiple glob patterns are separated by semicolons.
-        Each glob pattern must specify a key, followed by a colon, followed by the pattern.
-        The key is used to disambiguate the parsed data in the Go templates.
       type: string
-      default: "artifacts:.ods/artifacts/*/*.json;artifacts:.ods/artifacts/*/*.yaml"
+      default: ".ods/*;.ods/repos/*/.ods/*.ods/artifacts/*/*.json;.ods/artifacts/*/*.yaml"
     - name: pdf-theme
       description: >-
         The name or file path of the Asciidoctor PDF theme to load.

cmd/render/render.go

@@ -10,12 +10,6 @@ import (
 	"text/template"
 )
 
-// fileDataMap maps a "filename" to arbitary data.
-type fileDataMap map[string]map[string]interface{}
-
-// keyedFileDataMap maps a key to a fileDataMap
-type keyedFileDataMap map[string]fileDataMap
-
 var nonAlphanumericRegex = regexp.MustCompile(`[^a-zA-Z0-9]+`)
 
 func render(baseDir, templateGlob, outputDir string, dataSourceGlobs []string) error {
@@ -58,7 +52,7 @@ func render(baseDir, templateGlob, outputDir string, dataSourceGlobs []string) e
 	return nil
 }
 
-func renderTemplate(outputDir, templateFile string, tmpl *template.Template, data keyedFileDataMap) error {
+func renderTemplate(outputDir, templateFile string, tmpl *template.Template, data map[string]interface{}) error {
 	outFile := strings.TrimSuffix(templateFile, filepath.Ext(templateFile)) + ".out"
 	w, err := os.Create(filepath.Join(outputDir, outFile))
 	if err != nil {
@@ -68,72 +62,58 @@ func renderTemplate(outputDir, templateFile string, tmpl *template.Template, dat
 }
 
 func safeMapKey(str string) string {
-	return nonAlphanumericRegex.ReplaceAllString(str, "_")
+	return strings.Trim(nonAlphanumericRegex.ReplaceAllString(str, "_"), "_")
 }
 
-func extractDataFromSources(baseDir string, dataSourceGlobs []string) (keyedFileDataMap, error) {
-	data := make(keyedFileDataMap)
-	for _, keyedGlob := range dataSourceGlobs {
-		globKey, globPattern, found := strings.Cut(keyedGlob, ":")
-		if !found {
-			return nil, fmt.Errorf("no colon found in %q", keyedGlob)
-		}
-		log.Printf("Collecting data from files matching glob %q under key %q ...", globPattern, globKey)
-		if _, ok := data[globKey]; !ok {
-			data[globKey] = make(fileDataMap)
-		}
-		fdm, err := collectDataFromMatchingFiles(baseDir, globPattern)
+func extractDataFromSources(baseDir string, dataSourceGlobs []string) (map[string]interface{}, error) {
+	data := make(map[string]interface{})
+	for _, globPattern := range dataSourceGlobs {
+		log.Printf("Collecting data from files matching glob %q ...", globPattern)
+		err := collectDataFromMatchingFiles(baseDir, globPattern, data)
 		if err != nil {
 			return nil, err
 		}
-		for fileKey, fileData := range fdm {
-			if _, ok := data[globKey][fileKey]; !ok {
-				data[globKey][fileKey] = fileData
-			} else {
-				for k, v := range fileData {
-					data[globKey][fileKey][k] = v
-				}
-			}
-		}
 	}
 	return data, nil
 }
 
-func collectDataFromMatchingFiles(baseDir, glob string) (fileDataMap, error) {
-	result := make(fileDataMap)
+// buildMapPath builds map keys in m corresponding to p.
+// p is expected to be a filepath using slashes without an extension, e.g. "a/b/c/d".
+func buildMapPath(m map[string]interface{}, p string) map[string]interface{} {
+	elems := strings.SplitN(p, "/", 2)
+	dir := safeMapKey(elems[0])
+	if _, ok := m[dir]; !ok {
+		m[dir] = make(map[string]interface{})
+	}
+	leaf := m[dir].(map[string]interface{})
+	if len(elems) > 1 {
+		return buildMapPath(leaf, elems[1])
+	}
+	return leaf
+}
+
+func collectDataFromMatchingFiles(baseDir, glob string, data map[string]interface{}) error {
 	matches, err := filepath.Glob(filepath.Join(baseDir, glob))
 	if err != nil {
-		return nil, err
+		return err
 	}
 	for _, m := range matches {
-		ext := filepath.Ext(m)
-		decoderFunc := selectNewDecoderFunc(ext)
-		matchForKey := strings.TrimPrefix(m, baseDir)
-		fileName := filepath.Base(matchForKey)
-		kind := filepath.Base(strings.TrimSuffix(matchForKey, fileName))
-		fKey := strings.TrimSuffix(fileName, ext)
+		p := filepath.ToSlash(strings.TrimPrefix(m, baseDir))
+		p = strings.TrimSuffix(p, filepath.Ext(p))
+		fileData := buildMapPath(data, p)
+		decoderFunc := selectNewDecoderFunc(filepath.Ext(m))
 
 		f, err := os.Open(m)
 		if err != nil {
-			return nil, err
+			return err
 		}
 		defer f.Close()
 
 		dec := decoderFunc(f)
-		res := make(map[string]interface{})
-		err = dec.Decode(&res)
+		err = dec.Decode(&fileData)
 		if err != nil {
-			return nil, err
-		}
-
-		mk := safeMapKey(kind)
-		if mk == "" || mk == "_" {
-			mk = "root"
-		}
-		if _, ok := result[mk]; !ok {
-			result[mk] = make(map[string]interface{})
+			return err
 		}
-		result[mk][fKey] = res
 	}
-	return result, err
+	return err
 }

cmd/render/render_test.go

@@ -2,11 +2,43 @@ package main
 
 import (
 	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/opendevstack/ods-pipeline-adoc/internal/testhelper"
 )
 
+func TestBuildPath(t *testing.T) {
+	m := make(map[string]interface{})
+	p := "a/b/c/d.txt"
+	p = strings.TrimSuffix(p, filepath.Ext(p))
+	p = filepath.ToSlash(p)
+	buildMapPath(m, p)
+	if _, ok := m["a"]; !ok {
+		t.Fatal("expect key a")
+	}
+	a := m["a"].(map[string]interface{})
+	if _, ok := a["b"]; !ok {
+		t.Fatal("expect key b")
+	}
+	b := a["b"].(map[string]interface{})
+	if _, ok := b["c"]; !ok {
+		t.Fatal("expect key c")
+	}
+	c := b["c"].(map[string]interface{})
+	if _, ok := c["d"]; !ok {
+		t.Fatal("expect key d")
+	}
+}
+
+func TestSafeMapKey(t *testing.T) {
+	want := "ods_artifacts_org_some_example"
+	got := safeMapKey(".ods/artifacts/org.some-example")
+	if want != got {
+		t.Fatalf("want %s, got %s", want, got)
+	}
+}
+
 func TestRender(t *testing.T) {
 	tempDir := testhelper.MkdirTempDir(t)
 	defer testhelper.RmTempDir(tempDir)
@@ -16,10 +48,10 @@ func TestRender(t *testing.T) {
 		"sample.adoc.tmpl",
 		tempDir,
 		[]string{
-			"artifacts:sample-artifacts/*/*.json",
-			"artifacts:sample-artifacts/*/*.yaml",
-			"artifacts:sample-artifacts/*/*.txt",
-			"data:*.yaml",
+			".ods/artifacts/*/*.json",
+			".ods/artifacts/*/*.yaml",
+			".ods/artifacts/*/*.txt",
+			"*.yaml",
 		},
 	); err != nil {
 		t.Fatal(err)

docs/render.adoc

@@ -4,15 +4,15 @@
 
 The purpose of this task is to render a asciidoc template located in the repository into a PDF. In addition to just transforming the asciidoc file to PDF, the task is also able to render information gathered from YAML/JSON files (such as ODS Pipeline artifacts) into the asciidoc file before transforming it to PDF.
 
-The task expects a glob pattern pointing to one or more Go template files (given by parameter `template`). It renders each found Go template with data gathered from files matching the `data-sources` parameter, which defaults to `artifacts:.ods/artifacts/\*/*.json;artifacts:.ods/artifacts/\*/*.yaml`. The asciidoc template can then access data parsed from these files under the key specified before the colon of each glob pattern. For example, if file `.ods/artifacts/org.foo/some.json` contains:
+The task expects a glob pattern pointing to one or more Go template files (given by parameter `template`). It renders each found Go template with data gathered from files matching the `data-sources` parameter, which defaults to `.ods/*;.ods/repos/*/.ods/*.ods/artifacts/*/*.json;.ods/artifacts/*/*.yaml`. The asciidoc template can then access data parsed from these files. For example, if file `.ods/artifacts/org.foo/some.json` contains:
 
 ```
 {"a":"b"}
 ```
 
-The asciidoc template can access the value of the field `a` by referencing `{{.artifacts.org_foo.some.a}}`. Note that any non-alphanumeric character in the file path is replaced with an underscore. In general, field access in templates is possible via `<key>.<path>.<file>.<field>`. In the special case where the file is located at the root of the directory, the `<path>` section is set to `root`. For example, the glob pattern `metadata:*.yaml` might match the file `foo.yaml` in the root of the repository - its data will be accessible as `metadata.root.foo.fieldname`.
+The asciidoc template can access the value of the field `a` by referencing `{{.ods.artifacts.org_foo.some.a}}`. Note that any non-alphanumeric character in the file path is replaced with an underscore, and leading or trailing underscores are trimmed.
 
-Note that only JSON and YAML formats are recognized. If a matching file does not end in either `.json` or `.y(a)ml`, its entire content is made available under the key `value`. For example, the glob pattern `log:*.log` might match the file `pipeline-run.log`, which would expose the content of the file as `log.root.pipeline_run.value` to the template.
+Note that only JSON and YAML formats are recognized as such. If a matching file does not end in either `.json` or `.y(a)ml`, its entire content is made available under the key `value`. For example, the glob pattern `*.log` might match the file `pipeline-run.log`, which would expose the content of the file as `pipeline_run.value` to the template.
 
 After the Go template has been rendered, link:https://github.com/asciidoctor/asciidoctor-pdf[asciidoctor-pdf] is used to turn each rendered asciidoc file into a PDF file. The resulting files are placed into the directory specified by `output-dir` (defaulting to `.ods/artifacts/org.opendevstack.pipeline.adoc.pdf` so that created PDFs are preserved as artifacts in Nexus). Theming is possible by specifying the `pdf-theme` parameter as explained in the link:https://docs.asciidoctor.org/pdf-converter/latest/theme/apply-theme/#theme-and-font-directories[Theme and font directories] documentation.
 
@@ -41,8 +41,8 @@ without leading `./` and trailing `/`.
 
 
 | data-sources
-| artifacts:.ods/artifacts/*/*.json;artifacts:.ods/artifacts/*/*.yaml
-| Glob patterns from where to source data. Multiple glob patterns are separated by semicolons. Each glob pattern must specify a key, followed by a colon, followed by the pattern. The key is used to disambiguate the parsed data in the Go templates.
+| .ods/*;.ods/repos/*/.ods/*.ods/artifacts/*/*.json;.ods/artifacts/*/*.yaml
+| Glob patterns from where to source data. Multiple glob patterns are separated by semicolons.
 
 
 | pdf-theme

tasks/render.yaml

@@ -29,10 +29,8 @@ spec:
       description: >-
         Glob patterns from where to source data.
         Multiple glob patterns are separated by semicolons.
-        Each glob pattern must specify a key, followed by a colon, followed by the pattern.
-        The key is used to disambiguate the parsed data in the Go templates.
       type: string
-      default: "artifacts:.ods/artifacts/*/*.json;artifacts:.ods/artifacts/*/*.yaml"
+      default: ".ods/*;.ods/repos/*/.ods/*.ods/artifacts/*/*.json;.ods/artifacts/*/*.yaml"
     - name: pdf-theme
       description: >-
         The name or file path of the Asciidoctor PDF theme to load.

test/testdata/fixtures/sample.adoc.tmpl

@@ -4,17 +4,17 @@ This is a sample adoc file.
 
 Actually, it is a Go template, with some placeholders like:
 
-* {{.artifacts.com_github_opendevstack_ods_pipeline_go_foo.result.foo}}
-* {{.artifacts.com_github_opendevstack_ods_pipeline_go_foo.result.bar}}
-* {{.artifacts.com_github_opendevstack_ods_pipeline_go_bar.result.foo}}
+* {{.ods.artifacts.org_opendevstack_pipeline_go_foo.result.foo}}
+* {{.ods.artifacts.org_opendevstack_pipeline_go_foo.result.bar}}
+* {{.ods.artifacts.org_opendevstack_pipeline_go_bar.result.foo}}
 
 Also with longer text:
-{{.artifacts.com_github_opendevstack_ods_pipeline_go_bar.info.value}}
+{{.ods.artifacts.org_opendevstack_pipeline_go_bar.info.value}}
 
 Even nested things work:
 
-Hello {{.artifacts.com_github_opendevstack_ods_pipeline_go_foo.other.hello.msg}}!
+Hello {{.ods.artifacts.org_opendevstack_pipeline_go_foo.other.hello.msg}}!
 
 And other keys can be passed as well:
 
-{{.data.root.metadata.foo}}
+{{.metadata.foo}}

test/testdata/workspaces/sample-app/templates/one.adoc.tmpl

@@ -2,6 +2,6 @@
 
 This is a sample adoc file.
 
-Output: {{.artifacts.foo.out.result}}
+Output: {{.ods.artifacts.foo.out.result}}
 
-Result: {{.artifacts.bar.result.some.sample.value}}
+Result: {{.ods.artifacts.bar.result.some.sample.value}}

test/testdata/workspaces/sample-app/templates/two.adoc.tmpl

@@ -2,4 +2,4 @@
 
 This is a second adoc file.
 
-{{.artifacts.bar.result.other.value}}
+{{.ods.artifacts.bar.result.other.value}}