From 0cd4814e9c95678ffcfe27604c9c50f24697b9bf Mon Sep 17 00:00:00 2001
From: Luc Talatinian <102624213+lucix-aws@users.noreply.github.com>
Date: Mon, 25 Mar 2024 08:43:12 -0400
Subject: [PATCH] move doc extractor to main (#2576)

---
 internal/docextractor/cmd/extract.go   | 302 +++++++++++++++++++++++++
 internal/docextractor/cmd/index.go     |  91 ++++++++
 internal/docextractor/cmd/jewelry.go   |  49 ++++
 internal/docextractor/cmd/main.go      |  69 ++++++
 internal/docextractor/cmd/serialize.go |  75 ++++++
 internal/docextractor/cmd/util.go      |  69 ++++++
 internal/docextractor/doc.go           |   2 +
 internal/docextractor/go.mod           |   3 +
 8 files changed, 660 insertions(+)
 create mode 100644 internal/docextractor/cmd/extract.go
 create mode 100644 internal/docextractor/cmd/index.go
 create mode 100644 internal/docextractor/cmd/jewelry.go
 create mode 100644 internal/docextractor/cmd/main.go
 create mode 100644 internal/docextractor/cmd/serialize.go
 create mode 100644 internal/docextractor/cmd/util.go
 create mode 100644 internal/docextractor/doc.go
 create mode 100644 internal/docextractor/go.mod

diff --git a/internal/docextractor/cmd/extract.go b/internal/docextractor/cmd/extract.go
new file mode 100644
index 00000000000..060054b5b94
--- /dev/null
+++ b/internal/docextractor/cmd/extract.go
@@ -0,0 +1,302 @@
+package main
+
+import (
+	"fmt"
+	"path/filepath"
+	"go/token"
+	"io/fs"
+	"go/ast"
+	"go/parser"
+	"strings"
+	"log"
+)
+
+
+// Extract will extract documentation from serviceDir and all sub-directories,
+// populate items with newly-created JewelryItem(s).
+// The overall strategy is to do a 
+func Extract(servicePath string, serviceDir fs.DirEntry, items map[string]JewelryItem) {
+
+	if serviceDir.Name() == "service" {
+		return
+	}
+
+	packageName := serviceDir.Name()
+
+
+	filepath.WalkDir(servicePath,
+		func(path string, d fs.DirEntry, e error) error {
+			if !d.IsDir() {
+				return nil
+			}
+
+			isInternal := strings.Count(path, "/internal") > 0
+			if isInternal {
+				return nil
+			}
+
+
+			fset := token.NewFileSet()
+			directory, err := parser.ParseDir(fset, path, nil, parser.ParseComments)
+			if err != nil {
+				panic(err)
+			}
+		
+			index := astIndex{
+				Types: map[string]*ast.TypeSpec{},
+				Functions: map[string]*ast.FuncDecl{},
+				Fields: map[string]*ast.Field{},
+				Other: []*ast.GenDecl{},
+			}
+
+		
+			for _, p := range directory {
+				removeTestFiles(p.Files)
+
+				packageItem, err := getPackageItem(packageName, p.Files)
+				if err == nil {
+					items["packageDocumentation"] = packageItem
+				}
+		
+				indexFromAst(p, &index)
+
+				err = extractTypes(packageName, index.Types, items)
+				if err != nil {
+					log.Fatal(err)
+				}
+
+				err = extractFunctions(packageName, index.Types, index.Functions, items)
+				if err != nil {
+					log.Fatal(err)
+				}
+			}
+
+
+
+			return nil
+		})
+
+	serialize(packageName, items)
+}
+
+// extractType iterates through 
+func extractTypes(packageName string, types map[string]*ast.TypeSpec, items map[string]JewelryItem) error {
+	for kt, vt := range types {
+		summary := ""
+		if vt.Doc != nil {
+			summary = vt.Doc.Text()
+		}
+		typeName := vt.Name.Name
+
+		item := JewelryItem{
+			Name:    typeName,
+			Summary: summary,
+			Members: []JewelryItem{},
+			Tags: []string{},
+			OtherBlocks: map[string]string{},
+			Params: []JewelryParam{},
+			BreadCrumbs: []BreadCrumb{
+				{
+					Name: packageName,
+					Kind: PACKAGE,
+				},
+			},
+		}
+		members := []JewelryItem{}
+
+		st, ok := vt.Type.(*ast.StructType)
+
+		if !ok {
+			item.Type = INTERFACE
+
+			bc := item.BreadCrumbs
+			bc = append(bc, BreadCrumb{
+				Name: typeName,
+				Kind: INTERFACE,
+			})
+			item.BreadCrumbs = bc
+			item.Signature = TypeSignature{
+				Signature: fmt.Sprintf("type %v interface", typeName),
+			}
+
+		} else {
+			item.Type = STRUCT
+			bc := item.BreadCrumbs
+			bc = append(bc, BreadCrumb{
+				Name: typeName,
+				Kind: STRUCT,
+			})
+			item.BreadCrumbs = bc
+			item.Signature = TypeSignature{
+				Signature: fmt.Sprintf("type %v struct", typeName),
+			}
+		}
+
+		if ok && st.Fields != nil && st.Fields.List != nil {
+			for _, vf := range st.Fields.List {
+				namesNum := len(vf.Names)
+				for i := 0; i < namesNum; i++ {
+					if !isExported(vf.Names[i].Name) {
+						break
+					}
+					fieldName := vf.Names[i].Name
+					var fieldItem JewelryItem
+					if vf.Doc == nil || vf.Doc.List == nil || vf.Doc.List[i] == nil {
+						fieldItem = JewelryItem{
+							Name:    fieldName,
+							Tags: []string{},
+							OtherBlocks: map[string]string{},
+							Params: []JewelryParam{},
+							Members: []JewelryItem{},
+							Summary: "",
+						}
+
+					} else {
+						fieldItem = JewelryItem{
+							Name:    fieldName,
+							Tags: []string{},
+							OtherBlocks: map[string]string{},
+							Params: []JewelryParam{},
+							Members: []JewelryItem{},
+							Summary: vf.Doc.List[i].Text,
+						}
+					}
+					fieldItem.Type = FIELD
+					fieldItem.BreadCrumbs = []BreadCrumb{
+						{
+							Name: packageName,
+							Kind: PACKAGE,
+						},
+						{
+							Name: typeName,
+							Kind: STRUCT,
+						},
+						{
+							Name: fieldName,
+							Kind: FIELD,
+						},
+					}
+					se, ok := vf.Type.(*ast.StarExpr)
+					if ok {
+						ident, ok := se.X.(*ast.Ident)
+						if ok {
+							fieldItem.Signature = TypeSignature{
+								Signature: ident.Name,
+							}
+						}
+					}
+					members = append(members, fieldItem)
+				}
+			}
+		}
+		item.Members = members
+		items[kt] = item
+	}
+	return nil
+}
+
+
+func extractFunctions(packageName string, types map[string]*ast.TypeSpec, functions map[string]*ast.FuncDecl, items map[string]JewelryItem) error {
+	for _, vf := range functions {
+		
+		// extract top-level functions
+		if vf.Recv == nil {
+			functionName := vf.Name.Name
+			items[functionName] = JewelryItem{
+				Type:    FUNCTION,
+				Name:    functionName,
+				Tags: []string{},
+				OtherBlocks: map[string]string{},
+				Params: []JewelryParam{},
+				Members: []JewelryItem{},
+				Summary: vf.Doc.Text(),
+				BreadCrumbs: []BreadCrumb{
+					{
+						Name: packageName,
+						Kind: PACKAGE,
+					},
+					{
+						Name: functionName,
+						Kind: FUNCTION,
+					},
+				},
+			}
+			continue
+		}
+		var receiverName string
+		switch r := vf.Recv.List[0].Type.(type) {
+		case *ast.StarExpr:
+			rName, _ := r.X.(*ast.Ident)
+			receiverName = rName.Name
+		case *ast.Ident:
+			receiverName = r.Name
+		}
+
+		// grab existing type
+		_, ok := types[receiverName]
+		if !ok {
+			// type doesnt exist
+			continue
+		}
+
+		methodName := vf.Name.Name
+
+		i := items[receiverName]
+
+		params := []JewelryParam{}
+		returns := ""
+
+		// extract operations
+		// assumes that all receiver methods on Client are
+		// service API operations except for the Options method.
+		if receiverName == "Client" && methodName != "Options" {
+			inputItem := items[fmt.Sprintf("%vInput", methodName)]
+			input := JewelryParam{
+				JewelryItem: JewelryItem{
+					Name: inputItem.Name,
+					Summary: inputItem.Summary,
+					Type: inputItem.Type,
+					Members: inputItem.Members,
+					BreadCrumbs: inputItem.BreadCrumbs,
+					Signature: inputItem.Signature,
+				},
+				IsOptional: false,
+				IsReadonly: false,
+			}
+			params = append(params, input)
+			returns = fmt.Sprintf("%vOutput", methodName)
+		}
+
+		members := i.Members
+		members = append(members,
+			JewelryItem{
+				Type:    METHOD,
+				Name:    methodName,
+				Members: []JewelryItem{},
+				Tags: []string{},
+				OtherBlocks: map[string]string{},
+				Params: params,
+				Returns: returns,
+				Summary: vf.Doc.Text(),
+				BreadCrumbs: []BreadCrumb{
+					{
+						Name: packageName,
+						Kind: PACKAGE,
+					},
+					{
+						Name: receiverName,
+						Kind: STRUCT,
+					},
+					{
+						Name: methodName,
+						Kind: METHOD,
+					},
+				},
+			},
+		)
+		i.Members = members
+		items[receiverName] = i
+	}
+	
+	return nil
+}
\ No newline at end of file
diff --git a/internal/docextractor/cmd/index.go b/internal/docextractor/cmd/index.go
new file mode 100644
index 00000000000..e578589a937
--- /dev/null
+++ b/internal/docextractor/cmd/index.go
@@ -0,0 +1,91 @@
+package main
+
+import (
+	"go/ast"
+	"go/token"
+)
+
+type astIndex struct {
+	Types map[string]*ast.TypeSpec
+	Functions map[string]*ast.FuncDecl
+	Fields map[string]*ast.Field
+	Other []*ast.GenDecl
+}
+
+
+func indexFromAst(p *ast.Package, index *astIndex) {
+	ast.Inspect(p, func(n ast.Node) bool {
+		switch x := n.(type) {
+		case *ast.FuncDecl:
+
+			// remove unexported items
+			if !isExported(x.Name.Name) {
+				break
+			}
+			name := x.Name.Name
+			index.Functions[name] = x
+
+		// use TypeSpec (over like StructType) because
+		// StructType doesnt have the name of the thing for some reason
+		// and TypeSpec contains the StructType obj as a field.
+		case *ast.TypeSpec:
+
+			if !isExported(x.Name.Name) {
+				break
+			}
+
+			// if a type exists AND it has a doc comment, then
+			// dont add anything -- were good.
+			// if not, then just add whatever.
+			name := x.Name.Name
+			if _, ok := index.Types[name]; ok && index.Types[name].Doc.Text() != "" {
+				break
+			}
+
+			index.Types[name] = x
+		case *ast.Field:
+			namesNum := len(x.Names)
+			for i := 0; i < namesNum; i++ {
+				if !isExported(x.Names[i].Name) {
+					break
+				}
+				name := x.Names[i].Name
+				index.Fields[name] = x
+			}
+		case *ast.GenDecl:
+
+			// for some reason, the same type will show up in the AST node list
+			// one with documentation and one without documentation
+			if x.Tok == token.TYPE {
+				xt, _ := x.Specs[0].(*ast.TypeSpec)
+
+				name := xt.Name.Name
+				if !isExported(name) {
+					break
+				}
+
+				// if a type exists AND it has a doc comment, then
+				// dont add anything -- were good.
+				// if not, then just add whatever.
+				if _, ok := index.Types[name]; ok && index.Types[name].Doc.Text() != "" {
+					break
+				}
+
+				// its a comment group, and each item in the list
+				// is a line
+				// summary := ""
+				if x.Doc != nil && x.Doc.List != nil {
+					xt.Doc = x.Doc
+					// for _, line := range x.Doc.List {
+					// 	summary += line.Text
+					// }
+				}
+
+				index.Types[name] = xt
+			} else {
+				index.Other = append(index.Other, x)
+			}
+		}
+		return true
+	})
+}
\ No newline at end of file
diff --git a/internal/docextractor/cmd/jewelry.go b/internal/docextractor/cmd/jewelry.go
new file mode 100644
index 00000000000..aee6c628b2f
--- /dev/null
+++ b/internal/docextractor/cmd/jewelry.go
@@ -0,0 +1,49 @@
+package main
+
+
+type JewelryItemKind string
+
+const (
+	PACKAGE   JewelryItemKind = "Package"
+	STRUCT    JewelryItemKind = "Struct"
+	INTERFACE JewelryItemKind = "Interface"
+	FUNCTION  JewelryItemKind = "Function"
+	METHOD    JewelryItemKind = "Method"
+	FIELD     JewelryItemKind = "Field"
+	OTHER     JewelryItemKind = "Other"
+)
+
+type BreadCrumb struct {
+	Name string          `json:"name"`
+	Kind JewelryItemKind `json:"kind"`
+}
+
+type TypeSignature struct {
+	Signature string `json:"signature"`
+	Location string `json:"location"`
+}
+
+type JewelryParam struct {
+	JewelryItem
+	IsOptional bool
+	IsReadonly bool
+	IsEventProperty bool
+}
+
+type JewelryItem struct {
+	Name        string          `json:"name"`
+	Summary     string          `json:"summary"`
+	Type        JewelryItemKind `json:"type"`
+	Members     []JewelryItem   `json:"members"`
+	BreadCrumbs []BreadCrumb    `json:"breadcrumbs"`
+	Signature TypeSignature `json:"typeSignature"`
+	Tags []string `json:"tags"`
+	Params []JewelryParam `json:"params"`
+	Returns string `json:"returns"`
+	// // optional (used only for JewelryOperations)
+	// // since no out-of-box thing in Go for union types
+	// Input string
+	// // optional. see above.
+	// Output string
+	OtherBlocks map[string]string `json:"otherBlocks"`
+}
\ No newline at end of file
diff --git a/internal/docextractor/cmd/main.go b/internal/docextractor/cmd/main.go
new file mode 100644
index 00000000000..68902e07821
--- /dev/null
+++ b/internal/docextractor/cmd/main.go
@@ -0,0 +1,69 @@
+package main
+
+import (
+	"fmt"
+	"flag"
+	"os"
+	"log"
+	"path/filepath"
+	"strings"
+	"io/fs"
+)
+
+var (
+	maxDepth int
+	servicePath string
+)
+
+// Visit function passed to filepath.WalkDir. 
+// Ensures that Exctract is only called on service client
+// directories and not internal directories or files.
+func extract(path string, d fs.DirEntry, err error) error {
+	if err != nil {
+		return err
+	}
+	if !d.IsDir() {
+		return nil
+	}
+
+	isInternal := strings.Count(path, "/internal") > 0
+	if isInternal {
+		fmt.Printf("Skipping %v\n", path)
+		return nil
+	}
+
+	currentDepth := strings.Count(path, string(os.PathSeparator))
+	serviceDepth := strings.Count(servicePath, string(os.PathSeparator))
+
+	if currentDepth > (serviceDepth + 1) || isInternal {
+		return fs.SkipDir
+	}
+
+	items := map[string]JewelryItem{}
+	Extract(path, d, items)
+	fmt.Printf("processed %v\n", path)
+	return nil
+}
+
+func init() {
+	flag.StringVar(&servicePath, "servicePath", "",
+		"Root directory that is direct parent of all service client directories")
+
+}
+
+func main() {
+	flag.Parse()
+	if servicePath == "" {
+		log.Fatalf("need service dir and max depth")
+	}
+
+	log.Println(
+		fmt.Sprintf("Processing service path %v", servicePath),
+	)
+
+
+	err := filepath.WalkDir(servicePath, extract)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
diff --git a/internal/docextractor/cmd/serialize.go b/internal/docextractor/cmd/serialize.go
new file mode 100644
index 00000000000..b316a444c13
--- /dev/null
+++ b/internal/docextractor/cmd/serialize.go
@@ -0,0 +1,75 @@
+package main
+
+import (
+	"fmt"
+	"encoding/json"
+	"os"
+)
+
+
+
+func serialize(packageName string, items map[string]JewelryItem) {
+	// there are single files getting through here
+	clientData := map[string]JewelryItem{}
+	for k, v := range items {
+		if k == "packageDocumentation" {
+			clientData[k] = v
+			continue
+		}
+		clientData[v.Name] = v
+		if v.Name == "Client" {
+			for _, m := range v.Members {
+				if m.Name == "Options" {
+					continue
+				}
+				// m.Input = fmt.Sprintf("%vInput", m.Name)
+				// m.Output = fmt.Sprintf("%vOutput", m.Name)
+				clientData[m.Name] = m
+			}
+		}
+	}
+	content, err := json.Marshal(clientData)
+	if err != nil {
+		fmt.Println(err)
+	}
+	err = os.WriteFile(fmt.Sprintf("../clients/%v.json", packageName), content, 0644)
+	if err != nil {
+		panic(err)
+	}
+
+	for _, item := range clientData {
+		if item.Name == "" || item.Name == "packageDocumentation" {
+			continue
+		}
+		content, err := json.Marshal(item)
+		if err != nil {
+			fmt.Println(err)
+		}
+		err = os.WriteFile(fmt.Sprintf("../public/members/-aws-sdk-client-%v.%v.%v.json", packageName, item.Name, string(item.Type)), content, 0644)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+
+	typeData := map[string][]string{}
+	for _, item := range clientData {
+		if item.Name == "" || item.Name == "packageDocumentation" {
+			continue
+		}
+		val, ok := typeData[string(item.Type)]
+		if !ok {
+			val = []string{}
+		}
+		val = append(val, item.Name)
+		typeData[string(item.Type)] = val
+	}
+	content, err = json.Marshal(typeData)
+	if err != nil {
+		fmt.Println(err)
+	}
+	err = os.WriteFile(fmt.Sprintf("../public/members/-aws-sdk-client-%v.json", packageName), content, 0644)
+	if err != nil {
+		panic(err)
+	}
+}
\ No newline at end of file
diff --git a/internal/docextractor/cmd/util.go b/internal/docextractor/cmd/util.go
new file mode 100644
index 00000000000..327592c6a1c
--- /dev/null
+++ b/internal/docextractor/cmd/util.go
@@ -0,0 +1,69 @@
+package main
+
+import (
+	"fmt"
+	"go/ast"
+	"regexp"
+)
+
+func getPackageItem(packageName string, files map[string]*ast.File) (JewelryItem, error) {
+	packageDoc := ""
+	docRe := regexp.MustCompile(`.*/doc.go`)
+
+	for k, f := range files {
+		matched := docRe.Match([]byte(k))
+		if !matched {
+			continue
+		}
+		if f.Doc != nil && f.Doc.List != nil {
+			for _, line := range f.Doc.List {
+				packageDoc += line.Text
+			}
+		}
+		return JewelryItem{
+			Tags: []string{},
+			OtherBlocks: map[string]string{},
+			Members: []JewelryItem{},
+			Params: []JewelryParam{},
+			BreadCrumbs: []BreadCrumb{},
+			Summary: packageDoc,
+		}, nil
+	}
+	return JewelryItem{}, fmt.Errorf("no doc.go")
+}
+
+func hasDocFile(p *ast.Package) bool {
+	docRe := regexp.MustCompile(`.*/doc.go`)
+
+	for k, _ := range p.Files {
+		matched := docRe.Match([]byte(k))
+		if !matched {
+			continue
+		}
+		return true
+	}
+	return false	
+}
+
+func isExported(name string) bool {
+	if name == "" {
+		return false
+	}
+	firstChar := name[0]
+	if firstChar >= 'a' && firstChar <= 'z' {
+		return false
+	}
+	return true
+}
+
+func removeTestFiles(files map[string]*ast.File) error {
+	testRe := regexp.MustCompile(`.*_test.go`)
+	for key := range files {
+		matched := testRe.Match([]byte(key))
+		if !matched {
+			continue
+		}
+		delete(files, key)
+	}
+	return nil
+}
\ No newline at end of file
diff --git a/internal/docextractor/doc.go b/internal/docextractor/doc.go
new file mode 100644
index 00000000000..4f942ba8fbc
--- /dev/null
+++ b/internal/docextractor/doc.go
@@ -0,0 +1,2 @@
+// Package docextractor is used by the Go SDK release system to generate API Reference documentation.
+package docextractor
diff --git a/internal/docextractor/go.mod b/internal/docextractor/go.mod
new file mode 100644
index 00000000000..d0e3a1fbfbb
--- /dev/null
+++ b/internal/docextractor/go.mod
@@ -0,0 +1,3 @@
+module github.com/aws/aws-sdk-go-v2/internal/docextractor
+
+go 1.21.6