From 5d57af5c235f98047a7265f6e0d28f6005a58ccd Mon Sep 17 00:00:00 2001 From: Riccardo Schirone Date: Wed, 25 May 2022 14:47:09 +0200 Subject: [PATCH 1/2] Add a IsBundled dependency property When possible differentiate between runtime dependencies and bundled code. By bundled code we can refer to copy-pasted code, vendored code (e.g. github, node_modules), copy-pasted code, webpacked, etc. --- cmd/deplist/deplist.go | 12 ++++++++-- dependencies.go | 9 +++---- deplist.go | 52 ++++++++++++++++++++++++++++------------- internal/scan/nodejs.go | 24 +++++++++++++++++++ internal/utils/utils.go | 2 -- 5 files changed, 75 insertions(+), 24 deletions(-) diff --git a/cmd/deplist/deplist.go b/cmd/deplist/deplist.go index bb79c79..664ea64 100644 --- a/cmd/deplist/deplist.go +++ b/cmd/deplist/deplist.go @@ -36,13 +36,21 @@ func main() { version := dep.Version inst, _ := purl.FromString(fmt.Sprintf("pkg:%s/%s@%s", deplist.GetLanguageStr(dep.DepType), dep.Path, version)) - fmt.Println(inst) + fmt.Print(inst) + if dep.IsBundled { + fmt.Print(" [bundled]") + } + fmt.Println() } } else { deptype := deplist.Bitmask(*deptypePtr) for _, dep := range deps { if (dep.DepType & deptype) == deptype { - fmt.Printf("%s@%s\n", dep.Path, dep.Version) + fmt.Printf("%s@%s", dep.Path, dep.Version) + if dep.IsBundled { + fmt.Print(" [bundled]") + } + fmt.Println() } } } diff --git a/dependencies.go b/dependencies.go index 3cf8c2e..fbdceac 100644 --- a/dependencies.go +++ b/dependencies.go @@ -5,10 +5,11 @@ type Bitmask uint32 // Dependency per dependency info type Dependency struct { - DepType Bitmask // golang, nodejs, python etc - Path string // the module path, github.com/teris-io/shortid - Version string // v0.0.0-20171029131806-771a37caa5cf - Files []string // if available, list of all files for a package + DepType Bitmask // golang, nodejs, python etc + Path string // the module path, github.com/teris-io/shortid + Version string // v0.0.0-20171029131806-771a37caa5cf + Files []string // if available, list of all files for a package + IsBundled bool // whether the dependency is bundled somehow in the dependency (e.g. vendored code, copy-pasted code, embedded code, etc.) // /usr/lib/go-1.13/src/regexp/syntax/compile.go // /usr/lib/go-1.13/src/regexp/syntax/doc.go } diff --git a/deplist.go b/deplist.go index 53597ac..406be56 100644 --- a/deplist.go +++ b/deplist.go @@ -80,16 +80,32 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) { } if info.IsDir() { - // prevent walking down the vendors, docs, etc + // prevent walking down the docs, .git, tests, etc. if utils.BelongsToIgnoreList(info.Name()) { return filepath.SkipDir } } else { // Two checks, one for filenames and the second switch for full // paths. Useful if we're looking for top of repo - switch filename := info.Name(); filename { // for now only go for yarn and npm + case "package.json": + pkg, err := scan.GetNodeJSPackage(path) + if err != nil { + log.Debugf("failed to scan for nodejs package: %s", path) + return nil + } + + foundTypes.DepFoundAddFlag(LangNodeJS) + + deps = append(deps, + Dependency{ + DepType: LangNodeJS, + Path: pkg.Name, + Version: pkg.Version, + Files: []string{}, + IsBundled: true, + }) case "package-lock.json": // if theres not a yarn.lock fall thru if _, err := os.Stat( @@ -160,10 +176,11 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) { if !strings.HasSuffix(version, "-javadoc") && !strings.HasSuffix(version, "-sources") { deps = append(deps, Dependency{ - DepType: LangJava, - Path: name, - Version: version, - Files: []string{}, + DepType: LangJava, + Path: name, + Version: version, + Files: []string{}, + IsBundled: true, }) } } @@ -184,10 +201,11 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) { for path, goPkg := range pkgs { d := Dependency{ - DepType: LangGolang, - Path: path, - Files: goPkg.Gofiles, - Version: goPkg.Version, + DepType: LangGolang, + Path: path, + Files: goPkg.Gofiles, + Version: goPkg.Version, + IsBundled: true, } deps = append(deps, d) } @@ -202,9 +220,10 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) { } for _, goPkg := range pkgs { d := Dependency{ - DepType: LangGolang, - Path: goPkg.Name, - Version: goPkg.Version, + DepType: LangGolang, + Path: goPkg.Name, + Version: goPkg.Version, + IsBundled: true, } deps = append(deps, d) } @@ -219,9 +238,10 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) { } for _, goPkg := range pkgs { d := Dependency{ - DepType: LangGolang, - Path: goPkg.Name, - Version: goPkg.Version, + DepType: LangGolang, + Path: goPkg.Name, + Version: goPkg.Version, + IsBundled: true, } deps = append(deps, d) } diff --git a/internal/scan/nodejs.go b/internal/scan/nodejs.go index 275cac1..d4e10c0 100644 --- a/internal/scan/nodejs.go +++ b/internal/scan/nodejs.go @@ -3,6 +3,7 @@ package scan import ( "encoding/json" "fmt" + "os" "os/exec" "path/filepath" "strings" @@ -25,6 +26,10 @@ type yarnOutput struct { } } +type packageJsonFormat struct { + Name string `json:"name"` + Version string `json:"version"` +} type npmDependency struct { Version string `json:"version"` Dependencies map[string]npmDependency `json:"dependencies"` @@ -121,6 +126,25 @@ func GetNodeJSDeps(path string) (map[string]NodeJSGather, error) { return nil, fmt.Errorf("unknown NodeJS dependency file %q", path) } +func GetNodeJSPackage(path string) (NodeJSGather, error) { + log.Debugf("GetNodeJSPackage %s", path) + + data, err := os.ReadFile(path) + if err != nil { + return NodeJSGather{}, err + } + + var packageJson packageJsonFormat + err = json.Unmarshal(data, &packageJson) + if err != nil { + return NodeJSGather{}, err + } + if packageJson.Name == "" { + return NodeJSGather{}, fmt.Errorf("Empty package") + } + return NodeJSGather{Name: packageJson.Name, Version: packageJson.Version}, nil +} + func getYarnDeps(path string) (map[string]NodeJSGather, error) { var yarnOutput yarnOutput gatheredNode = make(map[string]NodeJSGather) diff --git a/internal/utils/utils.go b/internal/utils/utils.go index ae8b1d7..7521c3c 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -9,8 +9,6 @@ import ( func BelongsToIgnoreList(needle string) bool { switch needle { case - "node_modules", - "vendor", "scripts", "docs", "test", From 0c9c28c418424c10cffc8a2d62eab220e290ef9a Mon Sep 17 00:00:00 2001 From: Riccardo Schirone Date: Wed, 13 Jul 2022 10:10:45 +0200 Subject: [PATCH 2/2] Add cmd flag to choose which type of dependencies to print --- cmd/deplist/deplist.go | 51 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/cmd/deplist/deplist.go b/cmd/deplist/deplist.go index 664ea64..85678c9 100644 --- a/cmd/deplist/deplist.go +++ b/cmd/deplist/deplist.go @@ -3,18 +3,56 @@ package main import ( "flag" "fmt" + "strings" "github.com/RedHatProductSecurity/deplist" purl "github.com/mcoops/packageurl-go" log "github.com/sirupsen/logrus" ) +type searchModeFlag []string + +func (i *searchModeFlag) String() string { + return strings.Join(*i, ", ") +} + +func (i *searchModeFlag) Set(value string) error { + values := strings.Split(value, ",") + for _, value := range values { + switch value { + case "deps": + *i = append(*i, "deps") + case "bundled": + *i = append(*i, "bundled") + } + } + return nil +} + +func (i *searchModeFlag) ShouldHandleDep(dep deplist.Dependency) bool { + res := false + for _, searchMode := range *i { + switch searchMode { + case "deps": + res = res || !dep.IsBundled + case "bundled": + res = res || dep.IsBundled + } + } + return res +} + func main() { deptypePtr := flag.Int("deptype", -1, "golang, nodejs, python etc") debugPtr := flag.Bool("debug", false, "debug logging (default false)") + var searchModes searchModeFlag + flag.Var(&searchModes, "modes", "search mode (bundled, deps)") flag.Parse() + if len(searchModes) == 0 { + searchModes = []string{"deps", "bundled"} + } if *debugPtr == true { log.SetLevel(log.DebugLevel) } @@ -33,23 +71,24 @@ func main() { if *deptypePtr == -1 { for _, dep := range deps { + if !searchModes.ShouldHandleDep(dep) { + continue + } version := dep.Version inst, _ := purl.FromString(fmt.Sprintf("pkg:%s/%s@%s", deplist.GetLanguageStr(dep.DepType), dep.Path, version)) fmt.Print(inst) - if dep.IsBundled { - fmt.Print(" [bundled]") - } fmt.Println() } } else { deptype := deplist.Bitmask(*deptypePtr) for _, dep := range deps { + if !searchModes.ShouldHandleDep(dep) { + continue + } + if (dep.DepType & deptype) == deptype { fmt.Printf("%s@%s", dep.Path, dep.Version) - if dep.IsBundled { - fmt.Print(" [bundled]") - } fmt.Println() } }