[WIP] readmegen: A utility for generating readme from specifications

cmd/readmegen/main.go

@@ -0,0 +1,179 @@
+// Copyright © 2024 Meroxa, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"bytes"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"text/template"
+
+	"golang.org/x/mod/modfile"
+)
+
+var (
+	pkg               = flag.String("pkg", "", "The full package import path for the connector. By default, readmegen will try to detect the package import path based on the go.mod file it finds in the directory or any parent directory.")
+	debugIntermediary = flag.Bool("debug-intermediary", false, "Print the intermediary generated program to stdout instead of writing it to a file")
+	readmePath        = flag.String("f", "README.md", "The path to the readme file")
+	write             = flag.Bool("w", false, "Overwrite readme file instead of printing to stdout")
+)
+
+func main() {
+	flag.Parse()
+
+	if *pkg == "" {
+		var err error
+		*pkg, err = detectPackage()
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+	}
+
+	err := generate()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func detectPackage() (string, error) {
+	currentDir, err := filepath.Abs(".")
+	if err != nil {
+		return "", fmt.Errorf("could not detect absolute path to current directory: %w", err)
+	}
+	for {
+		dat, err := os.ReadFile(filepath.Join(currentDir, "go.mod"))
+		if os.IsNotExist(err) {
+			if currentDir == filepath.Dir(currentDir) {
+				// at the root
+				break
+			}
+			currentDir = filepath.Dir(currentDir)
+			continue
+		} else if err != nil {
+			return "", err
+		}
+		modulePath := modfile.ModulePath(dat)
+		if modulePath == "" {
+			break // no module path found
+		}
+		return modulePath, nil
+	}
+	return "", errors.New("could not detect package, make sure you are in the root of the connector directory or provide the -package flag manually")
+}
+
+func generate() (err error) {
+	if *debugIntermediary {
+		return executeTemplate(os.Stdout)
+	}
+
+	tmpDir, err := os.MkdirTemp(".", "readmegen_tmp")
+	if err != nil {
+		return fmt.Errorf("could not create temporary directory: %w", err)
+	}
+	defer func() {
+		removeErr := os.RemoveAll(tmpDir)
+		if removeErr != nil {
+			removeErr = fmt.Errorf("could not remove temporary directory: %w", removeErr)
+			err = errors.Join(err, removeErr)
+		}
+	}()
+
+	f, err := os.Create(filepath.Join(tmpDir, "main.go"))
+	if err != nil {
+		return fmt.Errorf("could not create temporary file: %w", err)
+	}
+	defer f.Close()
+
+	err = executeTemplate(f)
+	if err != nil {
+		return fmt.Errorf("could not execute template: %w", err)
+	}
+	f.Close()
+
+	cmd := exec.Command("go", "run", "main.go")
+	cmd.Dir = tmpDir
+	cmd.Stdout = os.Stdout // pipe directly to stdout
+	cmd.Stderr = os.Stderr // pipe directly to stderr
+
+	if *write {
+		buf := new(bytes.Buffer)
+		cmd.Stdout = buf
+		defer func() {
+			if err != nil {
+				os.Stdout.Write(buf.Bytes())
+				return
+			}
+			err = os.WriteFile(*readmePath, buf.Bytes(), 0644)
+		}()
+	}
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("intermediary program failed: %w", err)
+	}
+
+	return nil
+}
+
+func executeTemplate(out io.Writer) error {
+	t, err := template.New("").Parse(tmpl)
+	if err != nil {
+		return fmt.Errorf("could not parse template: %w", err)
+	}
+	p, err := filepath.Abs(*readmePath)
+	if err != nil {
+		return fmt.Errorf("could not get absolute path to readme: %w", err)
+	}
+	return t.Execute(out, data{
+		ImportPath: *pkg,
+		ReadmePath: p,
+	})
+}
+
+type data struct {
+	ImportPath string
+	ReadmePath string
+}
+
+const tmpl = `
+package main
+
+import (
+	"fmt"
+	"os"
+
+	conn "{{ .ImportPath }}"
+	"github.com/conduitio/conduit-connector-sdk/cmd/readmegen/util"
+)
+
+func main() {
+	err := util.Generate(
+		conn.Connector,
+		util.GenerateOptions{
+			ReadmePath: "{{ .ReadmePath }}",
+			Output:     os.Stdout,
+		},
+	)
+	if err != nil {
+		fmt.Fprint(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+`

cmd/readmegen/util/generate.go

@@ -0,0 +1,148 @@
+// Copyright © 2024 Meroxa, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+import (
+	"embed"
+	_ "embed"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+	"text/template"
+
+	"github.com/Masterminds/sprig/v3"
+	sdk "github.com/conduitio/conduit-connector-sdk"
+)
+
+var (
+	//go:embed templates/*
+	templates embed.FS
+)
+
+type GenerateOptions struct {
+	ReadmePath string
+	Output     io.Writer
+}
+
+func Generate(conn sdk.Connector, opts GenerateOptions) error {
+	readme, err := os.ReadFile(opts.ReadmePath)
+	if err != nil {
+		return fmt.Errorf("could not read readme file %v: %w", opts.ReadmePath, err)
+	}
+	readmeTmpl, err := Preprocess(string(readme))
+	if err != nil {
+		return fmt.Errorf("could not preprocess readme file %v: %w", opts.ReadmePath, err)
+	}
+
+	t := template.New("readme").Funcs(funcMap).Funcs(sprig.FuncMap())
+	t = template.Must(t.ParseFS(templates, "templates/*.tmpl"))
+	t = template.Must(t.Parse(readmeTmpl))
+
+	data := map[string]any{}
+	if conn.NewSpecification != nil {
+		data["specification"] = conn.NewSpecification()
+	}
+	if conn.NewSource != nil {
+		data["sourceParams"] = conn.NewSource().Parameters()
+	}
+	if conn.NewDestination != nil {
+		data["destinationParams"] = conn.NewDestination().Parameters()
+	}
+
+	return t.Execute(opts.Output, data)
+}
+
+var funcMap = template.FuncMap{
+	"formatCommentYAML": formatCommentYAML,
+	"args":              args,
+}
+
+func args(kvs ...any) (map[string]any, error) {
+	if len(kvs)%2 != 0 {
+		return nil, errors.New("args requires even number of arguments")
+	}
+	m := make(map[string]any)
+	for i := 0; i < len(kvs); i += 2 {
+		s, ok := kvs[i].(string)
+		if !ok {
+			return nil, errors.New("even args must be strings")
+		}
+		m[s] = kvs[i+1]
+	}
+	return m, nil
+}
+
+// formatCommentYAML takes a markdown text and formats it as a comment in a YAML
+// file. The comment is prefixed with the given indent level and "# ". The lines
+// are wrapped at 80 characters.
+func formatCommentYAML(text string, indent int) string {
+	const (
+		prefix     = "# "
+		lineLen    = 80
+		tmpNewLine = "〠"
+	)
+
+	// remove markdown new lines
+	text = strings.ReplaceAll(text, "\n\n", tmpNewLine)
+	text = strings.ReplaceAll(text, "\n", " ")
+	text = strings.ReplaceAll(text, tmpNewLine, "\n")
+
+	comment := formatMultiline(text, strings.Repeat(" ", indent)+prefix, lineLen)
+	// remove first indent and last new line
+	comment = comment[indent : len(comment)-1]
+	return comment
+}
+
+func formatMultiline(
+	input string,
+	prefix string,
+	maxLineLen int,
+) string {
+	textLen := maxLineLen - len(prefix)
+
+	// split the input into lines of length textLen
+	lines := strings.Split(input, "\n")
+	var formattedLines []string
+	for _, line := range lines {
+		if len(line) <= textLen {
+			formattedLines = append(formattedLines, line)
+			continue
+		}
+
+		// split the line into multiple lines, don't break words
+		words := strings.Fields(line)
+		var formattedLine string
+		for _, word := range words {
+			if len(formattedLine)+len(word) > textLen {
+				formattedLines = append(formattedLines, formattedLine[1:])
+				formattedLine = ""
+			}
+			formattedLine += " " + word
+		}
+		if formattedLine != "" {
+			formattedLines = append(formattedLines, formattedLine[1:])
+		}
+	}
+
+	// combine lines including indent and prefix
+	var formatted string
+	for _, line := range formattedLines {
+		formatted += prefix + line + "\n"
+	}
+
+	return formatted
+}

cmd/readmegen/util/generate_test.go

@@ -0,0 +1,21 @@
+// Copyright © 2024 Meroxa, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package util
+
+import "testing"
+
+func TestGenerate(t *testing.T) {
+
+}