From 1c7192e13ac9089ae1b54330395467453eff7987 Mon Sep 17 00:00:00 2001 From: Michael Sauter Date: Wed, 25 Oct 2023 09:24:12 +0200 Subject: [PATCH] Remove concept of glob key Closes #2. --- build/docs/render.adoc | 6 +- build/tasks/render.yaml | 4 +- cmd/render/render.go | 84 +++++++------------ cmd/render/render_test.go | 40 ++++++++- docs/render.adoc | 10 +-- tasks/render.yaml | 4 +- .../info.txt | 0 .../result.json | 0 .../other.yaml | 0 .../result.json | 0 test/testdata/fixtures/sample.adoc.tmpl | 12 +-- .../sample-app/templates/one.adoc.tmpl | 4 +- .../sample-app/templates/two.adoc.tmpl | 2 +- 13 files changed, 87 insertions(+), 79 deletions(-) rename test/testdata/fixtures/{sample-artifacts/com.github.opendevstack.ods-pipeline-go.bar => .ods/artifacts/org.opendevstack.pipeline-go.bar}/info.txt (100%) rename test/testdata/fixtures/{sample-artifacts/com.github.opendevstack.ods-pipeline-go.bar => .ods/artifacts/org.opendevstack.pipeline-go.bar}/result.json (100%) rename test/testdata/fixtures/{sample-artifacts/com.github.opendevstack.ods-pipeline-go.foo => .ods/artifacts/org.opendevstack.pipeline-go.foo}/other.yaml (100%) rename test/testdata/fixtures/{sample-artifacts/com.github.opendevstack.ods-pipeline-go.foo => .ods/artifacts/org.opendevstack.pipeline-go.foo}/result.json (100%) diff --git a/build/docs/render.adoc b/build/docs/render.adoc index 06be1c5..8f310b0 100644 --- a/build/docs/render.adoc +++ b/build/docs/render.adoc @@ -1,13 +1,13 @@ The purpose of this task is to render a asciidoc template located in the repository into a PDF. In addition to just transforming the asciidoc file to PDF, the task is also able to render information gathered from YAML/JSON files (such as ODS Pipeline artifacts) into the asciidoc file before transforming it to PDF. -The task expects a glob pattern pointing to one or more Go template files (given by parameter `template`). It renders each found Go template with data gathered from files matching the `data-sources` parameter, which defaults to `artifacts:.ods/artifacts/\*/*.json;artifacts:.ods/artifacts/\*/*.yaml`. The asciidoc template can then access data parsed from these files under the key specified before the colon of each glob pattern. For example, if file `.ods/artifacts/org.foo/some.json` contains: +The task expects a glob pattern pointing to one or more Go template files (given by parameter `template`). It renders each found Go template with data gathered from files matching the `data-sources` parameter, which defaults to `.ods/*;.ods/repos/*/.ods/*.ods/artifacts/*/*.json;.ods/artifacts/*/*.yaml`. The asciidoc template can then access data parsed from these files. For example, if file `.ods/artifacts/org.foo/some.json` contains: ``` {"a":"b"} ``` -The asciidoc template can access the value of the field `a` by referencing `{{.artifacts.org_foo.some.a}}`. Note that any non-alphanumeric character in the file path is replaced with an underscore. In general, field access in templates is possible via `...`. In the special case where the file is located at the root of the directory, the `` section is set to `root`. For example, the glob pattern `metadata:*.yaml` might match the file `foo.yaml` in the root of the repository - its data will be accessible as `metadata.root.foo.fieldname`. +The asciidoc template can access the value of the field `a` by referencing `{{.ods.artifacts.org_foo.some.a}}`. Note that any non-alphanumeric character in the file path is replaced with an underscore, and leading or trailing underscores are trimmed. -Note that only JSON and YAML formats are recognized. If a matching file does not end in either `.json` or `.y(a)ml`, its entire content is made available under the key `value`. For example, the glob pattern `log:*.log` might match the file `pipeline-run.log`, which would expose the content of the file as `log.root.pipeline_run.value` to the template. +Note that only JSON and YAML formats are recognized as such. If a matching file does not end in either `.json` or `.y(a)ml`, its entire content is made available under the key `value`. For example, the glob pattern `*.log` might match the file `pipeline-run.log`, which would expose the content of the file as `pipeline_run.value` to the template. After the Go template has been rendered, link:https://github.com/asciidoctor/asciidoctor-pdf[asciidoctor-pdf] is used to turn each rendered asciidoc file into a PDF file. The resulting files are placed into the directory specified by `output-dir` (defaulting to `.ods/artifacts/org.opendevstack.pipeline.adoc.pdf` so that created PDFs are preserved as artifacts in Nexus). Theming is possible by specifying the `pdf-theme` parameter as explained in the link:https://docs.asciidoctor.org/pdf-converter/latest/theme/apply-theme/#theme-and-font-directories[Theme and font directories] documentation. diff --git a/build/tasks/render.yaml b/build/tasks/render.yaml index 0278b82..cc85916 100644 --- a/build/tasks/render.yaml +++ b/build/tasks/render.yaml @@ -27,10 +27,8 @@ spec: description: >- Glob patterns from where to source data. Multiple glob patterns are separated by semicolons. - Each glob pattern must specify a key, followed by a colon, followed by the pattern. - The key is used to disambiguate the parsed data in the Go templates. type: string - default: "artifacts:.ods/artifacts/*/*.json;artifacts:.ods/artifacts/*/*.yaml" + default: ".ods/*;.ods/repos/*/.ods/*.ods/artifacts/*/*.json;.ods/artifacts/*/*.yaml" - name: pdf-theme description: >- The name or file path of the Asciidoctor PDF theme to load. diff --git a/cmd/render/render.go b/cmd/render/render.go index e48eb33..671973a 100644 --- a/cmd/render/render.go +++ b/cmd/render/render.go @@ -10,12 +10,6 @@ import ( "text/template" ) -// fileDataMap maps a "filename" to arbitary data. -type fileDataMap map[string]map[string]interface{} - -// keyedFileDataMap maps a key to a fileDataMap -type keyedFileDataMap map[string]fileDataMap - var nonAlphanumericRegex = regexp.MustCompile(`[^a-zA-Z0-9]+`) func render(baseDir, templateGlob, outputDir string, dataSourceGlobs []string) error { @@ -58,7 +52,7 @@ func render(baseDir, templateGlob, outputDir string, dataSourceGlobs []string) e return nil } -func renderTemplate(outputDir, templateFile string, tmpl *template.Template, data keyedFileDataMap) error { +func renderTemplate(outputDir, templateFile string, tmpl *template.Template, data map[string]interface{}) error { outFile := strings.TrimSuffix(templateFile, filepath.Ext(templateFile)) + ".out" w, err := os.Create(filepath.Join(outputDir, outFile)) if err != nil { @@ -68,72 +62,58 @@ func renderTemplate(outputDir, templateFile string, tmpl *template.Template, dat } func safeMapKey(str string) string { - return nonAlphanumericRegex.ReplaceAllString(str, "_") + return strings.Trim(nonAlphanumericRegex.ReplaceAllString(str, "_"), "_") } -func extractDataFromSources(baseDir string, dataSourceGlobs []string) (keyedFileDataMap, error) { - data := make(keyedFileDataMap) - for _, keyedGlob := range dataSourceGlobs { - globKey, globPattern, found := strings.Cut(keyedGlob, ":") - if !found { - return nil, fmt.Errorf("no colon found in %q", keyedGlob) - } - log.Printf("Collecting data from files matching glob %q under key %q ...", globPattern, globKey) - if _, ok := data[globKey]; !ok { - data[globKey] = make(fileDataMap) - } - fdm, err := collectDataFromMatchingFiles(baseDir, globPattern) +func extractDataFromSources(baseDir string, dataSourceGlobs []string) (map[string]interface{}, error) { + data := make(map[string]interface{}) + for _, globPattern := range dataSourceGlobs { + log.Printf("Collecting data from files matching glob %q ...", globPattern) + err := collectDataFromMatchingFiles(baseDir, globPattern, data) if err != nil { return nil, err } - for fileKey, fileData := range fdm { - if _, ok := data[globKey][fileKey]; !ok { - data[globKey][fileKey] = fileData - } else { - for k, v := range fileData { - data[globKey][fileKey][k] = v - } - } - } } return data, nil } -func collectDataFromMatchingFiles(baseDir, glob string) (fileDataMap, error) { - result := make(fileDataMap) +// buildMapPath builds map keys in m corresponding to p. +// p is expected to be a filepath using slashes without an extension, e.g. "a/b/c/d". +func buildMapPath(m map[string]interface{}, p string) map[string]interface{} { + elems := strings.SplitN(p, "/", 2) + dir := safeMapKey(elems[0]) + if _, ok := m[dir]; !ok { + m[dir] = make(map[string]interface{}) + } + leaf := m[dir].(map[string]interface{}) + if len(elems) > 1 { + return buildMapPath(leaf, elems[1]) + } + return leaf +} + +func collectDataFromMatchingFiles(baseDir, glob string, data map[string]interface{}) error { matches, err := filepath.Glob(filepath.Join(baseDir, glob)) if err != nil { - return nil, err + return err } for _, m := range matches { - ext := filepath.Ext(m) - decoderFunc := selectNewDecoderFunc(ext) - matchForKey := strings.TrimPrefix(m, baseDir) - fileName := filepath.Base(matchForKey) - kind := filepath.Base(strings.TrimSuffix(matchForKey, fileName)) - fKey := strings.TrimSuffix(fileName, ext) + p := filepath.ToSlash(strings.TrimPrefix(m, baseDir)) + p = strings.TrimSuffix(p, filepath.Ext(p)) + fileData := buildMapPath(data, p) + decoderFunc := selectNewDecoderFunc(filepath.Ext(m)) f, err := os.Open(m) if err != nil { - return nil, err + return err } defer f.Close() dec := decoderFunc(f) - res := make(map[string]interface{}) - err = dec.Decode(&res) + err = dec.Decode(&fileData) if err != nil { - return nil, err - } - - mk := safeMapKey(kind) - if mk == "" || mk == "_" { - mk = "root" - } - if _, ok := result[mk]; !ok { - result[mk] = make(map[string]interface{}) + return err } - result[mk][fKey] = res } - return result, err + return err } diff --git a/cmd/render/render_test.go b/cmd/render/render_test.go index e9d91ff..e3fdf21 100644 --- a/cmd/render/render_test.go +++ b/cmd/render/render_test.go @@ -2,11 +2,43 @@ package main import ( "path/filepath" + "strings" "testing" "github.com/opendevstack/ods-pipeline-adoc/internal/testhelper" ) +func TestBuildPath(t *testing.T) { + m := make(map[string]interface{}) + p := "a/b/c/d.txt" + p = strings.TrimSuffix(p, filepath.Ext(p)) + p = filepath.ToSlash(p) + buildMapPath(m, p) + if _, ok := m["a"]; !ok { + t.Fatal("expect key a") + } + a := m["a"].(map[string]interface{}) + if _, ok := a["b"]; !ok { + t.Fatal("expect key b") + } + b := a["b"].(map[string]interface{}) + if _, ok := b["c"]; !ok { + t.Fatal("expect key c") + } + c := b["c"].(map[string]interface{}) + if _, ok := c["d"]; !ok { + t.Fatal("expect key d") + } +} + +func TestSafeMapKey(t *testing.T) { + want := "ods_artifacts_org_some_example" + got := safeMapKey(".ods/artifacts/org.some-example") + if want != got { + t.Fatalf("want %s, got %s", want, got) + } +} + func TestRender(t *testing.T) { tempDir := testhelper.MkdirTempDir(t) defer testhelper.RmTempDir(tempDir) @@ -16,10 +48,10 @@ func TestRender(t *testing.T) { "sample.adoc.tmpl", tempDir, []string{ - "artifacts:sample-artifacts/*/*.json", - "artifacts:sample-artifacts/*/*.yaml", - "artifacts:sample-artifacts/*/*.txt", - "data:*.yaml", + ".ods/artifacts/*/*.json", + ".ods/artifacts/*/*.yaml", + ".ods/artifacts/*/*.txt", + "*.yaml", }, ); err != nil { t.Fatal(err) diff --git a/docs/render.adoc b/docs/render.adoc index ddba8af..cf728d8 100644 --- a/docs/render.adoc +++ b/docs/render.adoc @@ -4,15 +4,15 @@ The purpose of this task is to render a asciidoc template located in the repository into a PDF. In addition to just transforming the asciidoc file to PDF, the task is also able to render information gathered from YAML/JSON files (such as ODS Pipeline artifacts) into the asciidoc file before transforming it to PDF. -The task expects a glob pattern pointing to one or more Go template files (given by parameter `template`). It renders each found Go template with data gathered from files matching the `data-sources` parameter, which defaults to `artifacts:.ods/artifacts/\*/*.json;artifacts:.ods/artifacts/\*/*.yaml`. The asciidoc template can then access data parsed from these files under the key specified before the colon of each glob pattern. For example, if file `.ods/artifacts/org.foo/some.json` contains: +The task expects a glob pattern pointing to one or more Go template files (given by parameter `template`). It renders each found Go template with data gathered from files matching the `data-sources` parameter, which defaults to `.ods/*;.ods/repos/*/.ods/*.ods/artifacts/*/*.json;.ods/artifacts/*/*.yaml`. The asciidoc template can then access data parsed from these files. For example, if file `.ods/artifacts/org.foo/some.json` contains: ``` {"a":"b"} ``` -The asciidoc template can access the value of the field `a` by referencing `{{.artifacts.org_foo.some.a}}`. Note that any non-alphanumeric character in the file path is replaced with an underscore. In general, field access in templates is possible via `...`. In the special case where the file is located at the root of the directory, the `` section is set to `root`. For example, the glob pattern `metadata:*.yaml` might match the file `foo.yaml` in the root of the repository - its data will be accessible as `metadata.root.foo.fieldname`. +The asciidoc template can access the value of the field `a` by referencing `{{.ods.artifacts.org_foo.some.a}}`. Note that any non-alphanumeric character in the file path is replaced with an underscore, and leading or trailing underscores are trimmed. -Note that only JSON and YAML formats are recognized. If a matching file does not end in either `.json` or `.y(a)ml`, its entire content is made available under the key `value`. For example, the glob pattern `log:*.log` might match the file `pipeline-run.log`, which would expose the content of the file as `log.root.pipeline_run.value` to the template. +Note that only JSON and YAML formats are recognized as such. If a matching file does not end in either `.json` or `.y(a)ml`, its entire content is made available under the key `value`. For example, the glob pattern `*.log` might match the file `pipeline-run.log`, which would expose the content of the file as `pipeline_run.value` to the template. After the Go template has been rendered, link:https://github.com/asciidoctor/asciidoctor-pdf[asciidoctor-pdf] is used to turn each rendered asciidoc file into a PDF file. The resulting files are placed into the directory specified by `output-dir` (defaulting to `.ods/artifacts/org.opendevstack.pipeline.adoc.pdf` so that created PDFs are preserved as artifacts in Nexus). Theming is possible by specifying the `pdf-theme` parameter as explained in the link:https://docs.asciidoctor.org/pdf-converter/latest/theme/apply-theme/#theme-and-font-directories[Theme and font directories] documentation. @@ -41,8 +41,8 @@ without leading `./` and trailing `/`. | data-sources -| artifacts:.ods/artifacts/*/*.json;artifacts:.ods/artifacts/*/*.yaml -| Glob patterns from where to source data. Multiple glob patterns are separated by semicolons. Each glob pattern must specify a key, followed by a colon, followed by the pattern. The key is used to disambiguate the parsed data in the Go templates. +| .ods/*;.ods/repos/*/.ods/*.ods/artifacts/*/*.json;.ods/artifacts/*/*.yaml +| Glob patterns from where to source data. Multiple glob patterns are separated by semicolons. | pdf-theme diff --git a/tasks/render.yaml b/tasks/render.yaml index 568250f..93621ab 100644 --- a/tasks/render.yaml +++ b/tasks/render.yaml @@ -29,10 +29,8 @@ spec: description: >- Glob patterns from where to source data. Multiple glob patterns are separated by semicolons. - Each glob pattern must specify a key, followed by a colon, followed by the pattern. - The key is used to disambiguate the parsed data in the Go templates. type: string - default: "artifacts:.ods/artifacts/*/*.json;artifacts:.ods/artifacts/*/*.yaml" + default: ".ods/*;.ods/repos/*/.ods/*.ods/artifacts/*/*.json;.ods/artifacts/*/*.yaml" - name: pdf-theme description: >- The name or file path of the Asciidoctor PDF theme to load. diff --git a/test/testdata/fixtures/sample-artifacts/com.github.opendevstack.ods-pipeline-go.bar/info.txt b/test/testdata/fixtures/.ods/artifacts/org.opendevstack.pipeline-go.bar/info.txt similarity index 100% rename from test/testdata/fixtures/sample-artifacts/com.github.opendevstack.ods-pipeline-go.bar/info.txt rename to test/testdata/fixtures/.ods/artifacts/org.opendevstack.pipeline-go.bar/info.txt diff --git a/test/testdata/fixtures/sample-artifacts/com.github.opendevstack.ods-pipeline-go.bar/result.json b/test/testdata/fixtures/.ods/artifacts/org.opendevstack.pipeline-go.bar/result.json similarity index 100% rename from test/testdata/fixtures/sample-artifacts/com.github.opendevstack.ods-pipeline-go.bar/result.json rename to test/testdata/fixtures/.ods/artifacts/org.opendevstack.pipeline-go.bar/result.json diff --git a/test/testdata/fixtures/sample-artifacts/com.github.opendevstack.ods-pipeline-go.foo/other.yaml b/test/testdata/fixtures/.ods/artifacts/org.opendevstack.pipeline-go.foo/other.yaml similarity index 100% rename from test/testdata/fixtures/sample-artifacts/com.github.opendevstack.ods-pipeline-go.foo/other.yaml rename to test/testdata/fixtures/.ods/artifacts/org.opendevstack.pipeline-go.foo/other.yaml diff --git a/test/testdata/fixtures/sample-artifacts/com.github.opendevstack.ods-pipeline-go.foo/result.json b/test/testdata/fixtures/.ods/artifacts/org.opendevstack.pipeline-go.foo/result.json similarity index 100% rename from test/testdata/fixtures/sample-artifacts/com.github.opendevstack.ods-pipeline-go.foo/result.json rename to test/testdata/fixtures/.ods/artifacts/org.opendevstack.pipeline-go.foo/result.json diff --git a/test/testdata/fixtures/sample.adoc.tmpl b/test/testdata/fixtures/sample.adoc.tmpl index e0efe19..8802d3d 100644 --- a/test/testdata/fixtures/sample.adoc.tmpl +++ b/test/testdata/fixtures/sample.adoc.tmpl @@ -4,17 +4,17 @@ This is a sample adoc file. Actually, it is a Go template, with some placeholders like: -* {{.artifacts.com_github_opendevstack_ods_pipeline_go_foo.result.foo}} -* {{.artifacts.com_github_opendevstack_ods_pipeline_go_foo.result.bar}} -* {{.artifacts.com_github_opendevstack_ods_pipeline_go_bar.result.foo}} +* {{.ods.artifacts.org_opendevstack_pipeline_go_foo.result.foo}} +* {{.ods.artifacts.org_opendevstack_pipeline_go_foo.result.bar}} +* {{.ods.artifacts.org_opendevstack_pipeline_go_bar.result.foo}} Also with longer text: -{{.artifacts.com_github_opendevstack_ods_pipeline_go_bar.info.value}} +{{.ods.artifacts.org_opendevstack_pipeline_go_bar.info.value}} Even nested things work: -Hello {{.artifacts.com_github_opendevstack_ods_pipeline_go_foo.other.hello.msg}}! +Hello {{.ods.artifacts.org_opendevstack_pipeline_go_foo.other.hello.msg}}! And other keys can be passed as well: -{{.data.root.metadata.foo}} +{{.metadata.foo}} diff --git a/test/testdata/workspaces/sample-app/templates/one.adoc.tmpl b/test/testdata/workspaces/sample-app/templates/one.adoc.tmpl index 43d50f8..66359e4 100644 --- a/test/testdata/workspaces/sample-app/templates/one.adoc.tmpl +++ b/test/testdata/workspaces/sample-app/templates/one.adoc.tmpl @@ -2,6 +2,6 @@ This is a sample adoc file. -Output: {{.artifacts.foo.out.result}} +Output: {{.ods.artifacts.foo.out.result}} -Result: {{.artifacts.bar.result.some.sample.value}} +Result: {{.ods.artifacts.bar.result.some.sample.value}} diff --git a/test/testdata/workspaces/sample-app/templates/two.adoc.tmpl b/test/testdata/workspaces/sample-app/templates/two.adoc.tmpl index 9ed07b6..76e930e 100644 --- a/test/testdata/workspaces/sample-app/templates/two.adoc.tmpl +++ b/test/testdata/workspaces/sample-app/templates/two.adoc.tmpl @@ -2,4 +2,4 @@ This is a second adoc file. -{{.artifacts.bar.result.other.value}} +{{.ods.artifacts.bar.result.other.value}}