Remove data-source parameter and introduce template helpers instead

build/docs/render.adoc

@@ -1,20 +1,14 @@
-The purpose of this task is to render a asciidoc template located in the repository into a PDF. In addition to just transforming the asciidoc file to PDF, the task is also able to render information gathered from YAML/JSON/XML files (such as ODS Pipeline artifacts) into the asciidoc file before transforming it to PDF.
-
-The task expects a glob pattern pointing to one or more Go template files (given by parameter `template`). It renders each found Go template with data gathered from files matching the `data-sources` parameter, which defaults to `.ods/\*,.ods/repos/*/.ods/\*,.ods/artifacts/*/\*.json,.ods/artifacts/*/*.yaml`. The asciidoc template can then access data parsed from these files. For example, if file `.ods/artifacts/org.foo/some.json` contains:
-
-```
-{"a":"b"}
-```
-
-The asciidoc template can access the value of the field `a` by referencing `{{.ods.artifacts.org_foo.some.a}}`. Note that any non-alphanumeric character in the file path is replaced with an underscore, and leading or trailing underscores are trimmed.
-
-Note that only YAML, JSON, and XML formats are recognized as such. If a matching file does not end in `.y(a)ml`, `.json` or `.xml`, its entire content is made available under the key `value`. For example, the glob pattern `*.log` might match the file `pipeline-run.log`, which would expose the content of the file as `pipeline_run.value` to the template.
-
-The Go template can make use of template functions provided by link:http://masterminds.github.io/sprig/[sprig], as well as the following helper functions:
-
-* `fromMultiYAML`. Turns a string of multiple YAML documents (separated with `---`) into a slice of maps.
-* `toYAML`. Turns the given object into a YAML string.
-* `toSentence`. Turns a slice into a string enumerating its items. The words are connected with commas, except for the last two words, which are connected with "and".
-* `keys`. Returns a slice of all keys of given map.
+The purpose of this task is to render a asciidoc template located in the repository into a PDF. In addition to just transforming the asciidoc file to PDF, the task is also able to render information, for example gathered from YAML/JSON/XML files such as ODS Pipeline artifacts, into the asciidoc file before transforming it to PDF.
+
+The task expects a glob pattern pointing to one or more Go template files (given by parameter `template`). The templates can make use of template functions provided by link:http://masterminds.github.io/sprig/[sprig], as well as the following helper functions:
+
+* `data`. Parses a `.y(a)ml`, `.json` or `.xml` file into a map. Example: `{{$metadata := data "metadata.yaml"}}`
+* `content`. Reads the content of the given file. Example: `{{content ".ods/project"}}`
+* `contents`. Reads the content of all files matching the given glob pattern. Example: `{{$ods := contents ".ods/*"}}`
+* `directories`. Returns a slice of directories at given path. Example: `{{$repos := data ".ods/repos"}}`
+* `exists`. Checks whether given path exists. Example: `{{if exists ".ods/artifacts/foo/bar.json"}}artifact exists{{end}}`
+* `fromMultiYAML`. Turns a string of multiple YAML documents (separated with `---`) into a slice of maps. Example: `{{$manifest := fromMultiYAML $helm_status.manifest}}`
+* `toYAML`. Turns the given object into a YAML string. Example: `{{toYAML $helm_status.config}}`
+* `toSentence`. Turns a slice into a string enumerating its items. The words are connected with commas, except for the last two words, which are connected with "and". Example: `{{toSentence $repos}}`
 
 After the Go template has been rendered, link:https://github.com/asciidoctor/asciidoctor-pdf[asciidoctor-pdf] is used to turn each rendered asciidoc file into a PDF file. The resulting files are placed into the directory specified by `output-dir` (defaulting to `.ods/artifacts/org.opendevstack.pipeline.adoc.pdf` so that created PDFs are preserved as artifacts in Nexus). Theming is possible by specifying the `pdf-theme` parameter as explained in the link:https://docs.asciidoctor.org/pdf-converter/latest/theme/apply-theme/#theme-and-font-directories[Theme and font directories] documentation.

build/tasks/render.yaml

@@ -8,10 +8,9 @@ spec:
 
     See https://github.com/opendevstack/ods-pipeline-adoc/blob/v{{.Version}}/docs/tasks/render.adoc
   params:
-    - name: working-dir
+    - name: base-dir
       description: |
-        Working directory. The path must be relative to the root of the repository,
-        without leading `./` and trailing `/`.
+        Base directory. Paths given to template helpers are interprested relative to this directory.
       type: string
       default: "."
     - name: template
@@ -23,17 +22,10 @@ spec:
         Output directory where to place the rendered PDF files.
       type: string
       default: ".ods/artifacts/org.opendevstack.pipeline.adoc.pdf"
-    - name: data-sources
-      description: >-
-        Glob patterns from where to source data.
-        Multiple glob patterns are separated by commas.
-      type: string
-      default: ".ods/*,.ods/repos/*/.ods/*,.ods/artifacts/*/*.json,.ods/artifacts/*/*.yaml"
     - name: set
       description: >-
         Set template data values directly.
-        Multiple key=value pairs are separated by commas. Keys specified have
-        precedence over those discovered through `data-sources`.
+        Multiple key=value pairs are separated by commas.
       type: string
       default: ""
     - name: pdf-theme
@@ -62,8 +54,8 @@ spec:
       script: |
         render-template \
           --template='$(params.template)' \
+          --base-dir='$(params.base-dir)' \
           --output-dir=$(params.output-dir) \
-          --data-source='$(params.data-sources)' \
           --set='$(params.set)'
 
         asciidoctor_pdf_flags='--failure-level ERROR'

cmd/render/main.go

@@ -2,11 +2,7 @@
 //
 //		go run github.com/opendevstack/ods-pipeline-adoc/cmd/render \
 //			-template=sample.adoc.tmpl \
-//	        -output-dir=rendered \
-//			-data-source=records:sample-artifacts/*/*.json \
-//			-data-source=records:sample-artifacts/*/*.yaml
-//
-// Parsing of data is only supported for .json and .y(a)ml files.
+//	        -output-dir=rendered
 package main
 
 import (
@@ -17,17 +13,16 @@ import (
 
 func main() {
 	templateGlob := flag.String("template", "", "Glob pattern from where to source templates")
+	baseDir := flag.String("base-dir", ".", "Base directory from which to interpret filepaths passed to helper functions")
 	outputDir := flag.String("output-dir", "", "Output directory where to place the rendered files")
-	var dataSourceFlags multiFlag
-	flag.Var(&dataSourceFlags, "data-source", "Glob pattern from where to source data (may be specified multiple times)")
 	var setFlags multiFlag
 	flag.Var(&setFlags, "set", "Set values on the command line (can specify multiple or separate values with commas: key1=val1,key2=val2)")
 	flag.Parse()
 	wd, err := os.Getwd()
 	if err != nil {
 		log.Fatal(err)
 	}
-	if err := render(wd, *templateGlob, *outputDir, dataSourceFlags, setFlags); err != nil {
+	if err := render(wd, *baseDir, *templateGlob, *outputDir, setFlags); err != nil {
 		log.Fatal(err)
 	}
 }

cmd/render/render.go

@@ -14,23 +14,19 @@ import (
 
 var nonAlphanumericRegex = regexp.MustCompile(`[^a-zA-Z0-9]+`)
 
-func render(baseDir, templateGlob, outputDir string, dataSourceGlobs, setFlags []string) error {
-	if !strings.HasSuffix(baseDir, "/") {
-		baseDir = baseDir + "/"
+func render(workingDir, baseDir, templateGlob, outputDir string, setFlags []string) error {
+	if !strings.HasSuffix(workingDir, "/") {
+		workingDir = workingDir + "/"
 	}
 	if !filepath.IsAbs(outputDir) {
-		outputDir = filepath.Join(baseDir, outputDir)
+		outputDir = filepath.Join(workingDir, outputDir)
 	}
 	err := os.MkdirAll(outputDir, 0755)
 	if err != nil {
 		return err
 	}
 
-	data, err := extractDataFromSources(baseDir, dataSourceGlobs)
-	if err != nil {
-		return err
-	}
-	// Add key=value paris specified via --set.
+	data := map[string]any{}
 	for _, v := range setFlags {
 		key, value, found := strings.Cut(v, "=")
 		if !found {
@@ -39,32 +35,27 @@ func render(baseDir, templateGlob, outputDir string, dataSourceGlobs, setFlags [
 		data[key] = value
 	}
 
-	matches, err := filepath.Glob(filepath.Join(baseDir, templateGlob))
+	matches, err := filepath.Glob(filepath.Join(workingDir, templateGlob))
 	if err != nil {
 		return err
 	}
 	for _, templateFile := range matches {
 		log.Printf(
 			"Rendering template %q into %q ...",
-			strings.TrimPrefix(templateFile, baseDir),
-			strings.TrimPrefix(outputDir, baseDir),
+			strings.TrimPrefix(templateFile, workingDir),
+			strings.TrimPrefix(outputDir, workingDir),
 		)
 		templateBase := filepath.Base(templateFile)
 		tmpl, err := template.
 			New(templateBase).
-			Funcs(templateFuncs).
+			Funcs(templateFuncs(baseDir)).
 			Funcs(sprig.FuncMap()).
 			ParseFiles(templateFile)
 		if err != nil {
 			return fmt.Errorf("parse template %q: %s", templateFile, err)
 		}
 		err = renderTemplate(outputDir, templateBase, tmpl, data)
 		if err != nil {
-			if strings.Contains(err.Error(), "map has no entry for key") {
-				res := []string{}
-				walkMap(data, &res, []string{}, assembleRef)
-				return fmt.Errorf("render template %q: %s.\nValid references are:\n%s", templateBase, err, strings.Join(res, "\n"))
-			}
 			return fmt.Errorf("render template %q: %s", templateBase, err)
 		}
 	}
@@ -79,89 +70,3 @@ func renderTemplate(outputDir, templateFile string, tmpl *template.Template, dat
 	}
 	return tmpl.Option("missingkey=error").Execute(w, data)
 }
-
-func safeMapKey(str string) string {
-	return strings.Trim(nonAlphanumericRegex.ReplaceAllString(str, "_"), "_")
-}
-
-func extractDataFromSources(baseDir string, dataSourceGlobs []string) (map[string]interface{}, error) {
-	data := make(map[string]interface{})
-	for _, globPattern := range dataSourceGlobs {
-		log.Printf("Collecting data from files matching glob %q ...", globPattern)
-		err := collectDataFromMatchingFiles(baseDir, globPattern, data)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return data, nil
-}
-
-// buildMapPath builds map keys in m corresponding to p.
-// p is expected to be a filepath using slashes without an extension, e.g. "a/b/c/d".
-func buildMapPath(m map[string]interface{}, p string) map[string]interface{} {
-	elems := strings.SplitN(p, "/", 2)
-	dir := safeMapKey(elems[0])
-	if _, ok := m[dir]; !ok {
-		m[dir] = make(map[string]interface{})
-	}
-	leaf := m[dir].(map[string]interface{})
-	if len(elems) > 1 {
-		return buildMapPath(leaf, elems[1])
-	}
-	return leaf
-}
-
-func collectDataFromMatchingFiles(baseDir, glob string, data map[string]interface{}) error {
-	matches, err := filepath.Glob(filepath.Join(baseDir, glob))
-	if err != nil {
-		return err
-	}
-	for _, m := range matches {
-		p := filepath.ToSlash(strings.TrimPrefix(m, baseDir))
-		p = strings.TrimSuffix(p, filepath.Ext(p))
-		fileData := buildMapPath(data, p)
-		decoderFunc := selectNewDecoderFunc(filepath.Ext(m))
-
-		f, err := os.Open(m)
-		if err != nil {
-			return err
-		}
-		defer f.Close()
-
-		fInfo, err := f.Stat()
-		if err != nil {
-			return err
-		}
-		if fInfo.IsDir() {
-			continue
-		}
-
-		dec := decoderFunc(f)
-		err = dec.Decode(&fileData)
-		if err != nil {
-			return err
-		}
-	}
-	return err
-}
-
-type visitFunc func(path []string, key any, value any) string
-
-func assembleRef(path []string, key, value any) string {
-	if len(path) == 0 {
-		return fmt.Sprintf(".%s", key)
-	}
-	return fmt.Sprintf(".%s.%s", strings.Join(path, "."), key)
-}
-
-func walkMap(data map[string]any, paths *[]string, path []string, visit visitFunc) {
-	for key, value := range data {
-		if child, isMap := value.(map[string]any); isMap {
-			path = append(path, key)
-			walkMap(child, paths, path, visit)
-			path = path[:len(path)-1]
-		} else {
-			*paths = append(*paths, visit(path, key, child))
-		}
-	}
-}

cmd/render/render_test.go

@@ -2,72 +2,20 @@ package main
 
 import (
 	"path/filepath"
-	"strings"
 	"testing"
 
-	"github.com/google/go-cmp/cmp"
 	"github.com/opendevstack/ods-pipeline-adoc/internal/testhelper"
 )
 
-func TestBuildMapPath(t *testing.T) {
-	m := make(map[string]interface{})
-
-	// build map path for first file
-	p := "a/b/c/d.txt"
-	p = strings.TrimSuffix(p, filepath.Ext(p))
-	p = filepath.ToSlash(p)
-	buildMapPath(m, p)
-	if _, ok := m["a"]; !ok {
-		t.Fatal("expect key a")
-	}
-	a := m["a"].(map[string]interface{})
-	if _, ok := a["b"]; !ok {
-		t.Fatal("expect key b")
-	}
-	b := a["b"].(map[string]interface{})
-	if _, ok := b["c"]; !ok {
-		t.Fatal("expect key c")
-	}
-	c := b["c"].(map[string]interface{})
-	if _, ok := c["d"]; !ok {
-		t.Fatal("expect key d")
-	}
-
-	// build map path for second file which overlaps first path
-	p = "a/x.txt"
-	p = strings.TrimSuffix(p, filepath.Ext(p))
-	p = filepath.ToSlash(p)
-	buildMapPath(m, p)
-	if _, ok := a["x"]; !ok {
-		t.Fatal("expect key x")
-	}
-	if _, ok := a["b"]; !ok {
-		t.Fatal("expect key b")
-	}
-}
-
-func TestSafeMapKey(t *testing.T) {
-	want := "ods_artifacts_org_some_example"
-	got := safeMapKey(".ods/artifacts/org.some-example")
-	if want != got {
-		t.Fatalf("want %s, got %s", want, got)
-	}
-}
-
 func TestRender(t *testing.T) {
 	tempDir := testhelper.MkdirTempDir(t)
 	defer testhelper.RmTempDir(tempDir)
 
 	if err := render(
+		"../../test/testdata/fixtures",
 		"../../test/testdata/fixtures",
 		"sample.adoc.tmpl",
 		tempDir,
-		[]string{
-			".ods/artifacts/*/*.json",
-			".ods/artifacts/*/*.yaml",
-			".ods/artifacts/*/*.txt",
-			"*.yaml",
-		},
 		[]string{
 			"keyfoo=valbar",
 			"keybar=valbaz",
@@ -85,42 +33,13 @@ func TestRenderFailsOnMissingKeys(t *testing.T) {
 	tempDir := testhelper.MkdirTempDir(t)
 	defer testhelper.RmTempDir(tempDir)
 	err := render(
+		"../../test/testdata/fixtures",
 		"../../test/testdata/fixtures",
 		"error.adoc.tmpl",
 		tempDir,
-		[]string{
-			".ods/artifacts/*/*.json",
-			".ods/artifacts/*/*.yaml",
-			".ods/artifacts/*/*.txt",
-			"*.yaml",
-		},
 		[]string{},
 	)
 	if err == nil {
-		t.Error("Fixture template error.adoc.tmpl includes non-existent reference")
-	} else if !strings.Contains(err.Error(), ".ods.artifacts.org_opendevstack_pipeline_go_foo.result.foo") {
-		t.Errorf("Error must list valid references, got:\n%s", err)
-	}
-}
-
-func TestWalkMap(t *testing.T) {
-	got := []string{}
-	data := map[string]any{
-		"foo": "bar",
-		"baz": map[string]any{
-			"one": 1,
-			"two": map[string]any{
-				"x": "y",
-			},
-		},
-	}
-	walkMap(data, &got, []string{}, assembleRef)
-	want := []string{
-		".foo",
-		".baz.one",
-		".baz.two.x",
-	}
-	if diff := cmp.Diff(want, got); diff != "" {
-		t.Errorf("walkMap() mismatch (-want +got):\n%s", diff)
+		t.Error("Fixture template error.adoc.tmpl includes non-existent reference, rendering should fail")
 	}
 }