Golang change to external go list (#9)

* golang: change from using package tools, to calling the external go list command

* mvn: suppress exit status as it always returns errors

* golang: return correct error when timeout

Signed-off-by: mcoops <[email protected]>

deplist.go

@@ -97,22 +97,13 @@ func GetDeps(fullPath string) ([]Dependency, Bitmask, error) {
 					foundTypes.DepFoundAddFlag(LangGolang)
 				}
 
-				for _, p := range pkgs {
+				for path, goPkg := range pkgs {
+
 					d := Dependency{
 						DepType: LangGolang,
-						Path:    p.PkgPath,
-						Files:   p.GoFiles,
-					}
-					if p.Module != nil {
-						d.Version = p.Module.Version
-						/* if replace is specified, then use that version
-						 * version only may exist here too
-						 * not seen when version and replace.version are differnt
-						 * but just in case
-						 */
-						if p.Module.Replace != nil {
-							d.Version = p.Module.Replace.Version
-						}
+						Path:    path,
+						Files:   goPkg.Gofiles,
+						Version: goPkg.Version,
 					}
 					deps = append(deps, d)
 				}

internal/scan/golang.go

@@ -1,62 +1,136 @@
 package scan
 
 import (
-	"os"
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"os/exec"
 	"path/filepath"
-	"sort"
+	"strings"
+	"time"
 
+	"golang.org/x/mod/semver"
 	"golang.org/x/tools/go/packages"
 )
 
+type GoListDeps struct {
+	ImportPath string `json:"ImportPath"`
+	Module     struct {
+		Version string `json:"Version"`
+		Replace struct {
+			Version string `json:"Version"`
+		} `json:"Replace"`
+	} `json:"Module"`
+	GoFiles []string `json:"GoFiles"`
+}
+
+type GoPkg struct {
+	Version string
+	Gofiles []string
+}
+
 const defaultOptions = packages.NeedDeps |
 	packages.NeedImports |
 	packages.NeedModule |
 	packages.NeedFiles |
 	packages.NeedName
 
-func GetGolangDeps(path string) ([]*packages.Package, error) {
-	dirPath := filepath.Dir(path) // force directory
+func getVersion(deps GoListDeps) string {
+	/* if replace is specified, then use that version
+	* not seen when version and replace.version are differnt
+	* but just in case
+	 */
+	if deps.Module.Replace.Version != "" {
+		// due to the way we're loading the json this time, this just works
+		return deps.Module.Replace.Version
+	}
+	return deps.Module.Version
+}
+
+func runCmd(path string, mod bool) ([]byte, error) {
+	// go list -f '{{if not .Standard}}{{.Module}}{{end}}' -json -deps ./...
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+
+	modVendor := ""
+	if mod {
+		modVendor = "-mod=vendor"
+	}
+	// go list -f '{{if not .Standard}}{{.Module}}{{end}}' -json -deps ./...
+	cmd := exec.CommandContext(ctx, "go", "list", modVendor, "-f", "'{{if not .Standard}}{{.Module}}{{end}}'", "-json", "-deps", "./...")
+	cmd.Dir = filepath.Dir(path) // // force directory
+	out, err := cmd.Output()
+
+	if ctx.Err() == context.DeadlineExceeded {
+		return nil, ctx.Err()
+	}
 
-	cfg := packages.Config{Mode: defaultOptions, Dir: dirPath}
+	if err != nil {
+		// assume some retrival error, we have to redo the cmd with mod=vendor
+		return nil, err
+	}
 
-	pkgs, err := packages.Load(&cfg, "./...")
+	return out, nil
+}
 
+/*
+* Need to support re-running the go list with and without -mod=vendor
+* First run defaults to without, if any kind of error we'll just retry the run
+ */
+func runGoList(path string) ([]byte, error) {
+	out, err := runCmd(path, false)
 	if err != nil {
-		cfg.Env = append(os.Environ(), "GOFLAGS=-mod=vendor")
-		pkgs, err = packages.Load(&cfg, "./...")
+		// rerun
+		out, err = runCmd(path, true)
 		if err != nil {
-			return nil, err
+			return nil, errors.New("error running go list")
 		}
 	}
 
-	// based off the original https://github.com/golang/tools/blob/e140590b16906206021525faa5a48c7314806569/go/packages/gopackages/main.go#L99
-	// todo: should just get this put into the go/packages repo instead
-	var all []*packages.Package // postorder
-	seen := make(map[*packages.Package]bool)
+	return out, nil
+}
 
-	var visit func(*packages.Package)
-	visit = func(lpkg *packages.Package) {
-		if !seen[lpkg] {
-			seen[lpkg] = true
+func GetGolangDeps(path string) (map[string]GoPkg, error) {
+	// need to use a map as we'll get lots of duplicate entries
+	gathered := make(map[string]GoPkg)
 
-			// visit imports
-			var importPaths []string
-			for path := range lpkg.Imports {
-				importPaths = append(importPaths, path)
-			}
-			sort.Strings(importPaths) // for determinism
-			for _, path := range importPaths {
-				visit(lpkg.Imports[path])
-			}
+	out, err := runGoList(path)
 
-			all = append(all, lpkg)
-		}
+	if err != nil {
+		return nil, err
 	}
 
-	for _, pkg := range pkgs {
-		visit(pkg)
-	}
-	pkgs = all
+	/* we cann't just marshall the json as go list returns multiple json
+	 * documents not an array of json - which is annoying
+	 */
+	decoder := json.NewDecoder(strings.NewReader(string(out)))
 
-	return pkgs, nil
+	for {
+		var goListDeps GoListDeps
+		err := decoder.Decode(&goListDeps)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		importPath := goListDeps.ImportPath
+
+		if _, ok := gathered[importPath]; ok {
+			if gathered[importPath].Version != semver.Max(gathered[importPath].Version, getVersion(goListDeps)) {
+				gathered[importPath] = GoPkg{
+					Version: getVersion(goListDeps),
+					Gofiles: goListDeps.GoFiles,
+				}
+			}
+		} else {
+			gathered[importPath] = GoPkg{
+				Version: getVersion(goListDeps),
+				Gofiles: goListDeps.GoFiles,
+			}
+		}
+	}
+	return gathered, nil
 }

internal/scan/maven.go

@@ -41,11 +41,8 @@ func GetMvnDeps(path string) (map[string]string, error) {
 	cmd := exec.Command("mvn", "--no-transfer-progress", "dependency:tree", "-DoutputType=dot")
 	cmd.Dir = dirPath
 
-	data, err := cmd.Output()
-
-	if err != nil {
-		fmt.Println(err)
-	}
+	// supress error, it always returns errors
+	data, _ := cmd.Output()
 
 	res := strings.Split(string(data), "\n")