Skip to content

Commit

Permalink
Add an Embedded dependency property
Browse files Browse the repository at this point in the history 
When possible (right now only for NodeJS) differentiate between actual
dependencies and embedded code that is actually included in the source
files.
  • Loading branch information
Riccardo Schirone committed May 25, 2022
1 parent 5dd6e22 commit 589a994
Show file tree
Hide file tree
Showing 4 changed files with 129 additions and 73 deletions.
12 changes: 10 additions & 2 deletions cmd/deplist/deplist.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,21 @@ func main() {
version := dep.Version

inst, _ := purl.FromString(fmt.Sprintf("pkg:%s/%s@%s", deplist.GetLanguageStr(dep.DepType), dep.Path, version))
fmt.Println(inst)
fmt.Print(inst)
if dep.Embedded {
fmt.Print(" [embedded]")
}
fmt.Println()
}
} else {
deptype := deplist.Bitmask(*deptypePtr)
for _, dep := range deps {
if (dep.DepType & deptype) == deptype {
fmt.Printf("%s@%s\n", dep.Path, dep.Version)
fmt.Printf("%s@%s", dep.Path, dep.Version)
if dep.Embedded {
fmt.Print(" [embedded]")
}
fmt.Println()
}
}
}
Expand Down
9 changes: 5 additions & 4 deletions dependencies.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,11 @@ type Bitmask uint32

// Dependency per dependency info
type Dependency struct {
DepType Bitmask // golang, nodejs, python etc
Path string // the module path, github.com/teris-io/shortid
Version string // v0.0.0-20171029131806-771a37caa5cf
Files []string // if available, list of all files for a package
DepType Bitmask // golang, nodejs, python etc
Path string // the module path, github.com/teris-io/shortid
Version string // v0.0.0-20171029131806-771a37caa5cf
Files []string // if available, list of all files for a package
Embedded bool // whether the dependency is embedded in the source or not
// /usr/lib/go-1.13/src/regexp/syntax/compile.go
// /usr/lib/go-1.13/src/regexp/syntax/doc.go
}
Expand Down
157 changes: 90 additions & 67 deletions deplist.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,92 +79,115 @@ func getDeps(fullPath string) ([]Dependency, Bitmask, error) {
return err
}

if info.IsDir() {
// prevent walking down the vendors, docs, etc
if utils.BelongsToIgnoreList(info.Name()) {
return filepath.SkipDir
}
} else {
if !info.IsDir() {
// Two checks, one for filenames and the second switch for full
// paths. Useful if we're looking for top of repo

switch filename := info.Name(); filename {
// for now only go for yarn and npm
case "package-lock.json":
// if theres not a yarn.lock fall thru
if _, err := os.Stat(
filepath.Join(
filepath.Dir(path),
"yarn.lock")); err == nil {
return nil
relpath, _ := filepath.Rel(path, fullPath)
components := filepath.SplitList(relpath)
isInIgnoredList := false
for _, c := range components {
if utils.BelongsToIgnoreList(c) {
isInIgnoredList = true
}
fallthrough
}

case "yarn.lock":
pkgs, err := scan.GetNodeJSDeps(path)
if err != nil {
// ignore error
log.Debugf("failed to scan for nodejs: %s", path)
return nil
}
filename := info.Name()
if !isInIgnoredList || filename == "package.json" {
switch filename {
// for now only go for yarn and npm
case "package.json":
pkg, err := scan.GetNodeJSPackage(path)
if err != nil {
log.Debugf("failed to scan for nodejs package: %s", path)
return nil
}

if len(pkgs) > 0 {
foundTypes.DepFoundAddFlag(LangNodeJS)
}

for _, p := range pkgs {
deps = append(deps,
Dependency{
DepType: LangNodeJS,
Path: p.Name,
Version: p.Version,
Files: []string{},
DepType: LangNodeJS,
Path: pkg.Name,
Version: pkg.Version,
Files: []string{},
Embedded: true,
})
}
default:
ext := filepath.Ext(filename)
// java
switch ext {
case ".zip":
// be more aggressive with zip files, must contain something java ish
if ok, _ := utils.ZipContainsJava(path); !ok {
case "package-lock.json":
// if theres not a yarn.lock fall thru
if _, err := os.Stat(
filepath.Join(
filepath.Dir(path),
"yarn.lock")); err == nil {
return nil
}
fallthrough
case ".jar":
fallthrough
case ".war":
fallthrough
case ".ear":
fallthrough
case ".adm":
fallthrough
case ".hpi":
file := strings.Replace(filepath.Base(path), ext, "", 1) // get filename, check if we can ignore
if strings.HasSuffix(file, "-sources") || strings.HasSuffix(file, "-javadoc") {

case "yarn.lock":
pkgs, err := scan.GetNodeJSDeps(path)
if err != nil {
// ignore error
log.Debugf("failed to scan for nodejs: %s", path)
return nil
}

pkgs, err := scan.GetJarDeps(path)
if err == nil {
if len(pkgs) > 0 {
foundTypes.DepFoundAddFlag(LangNodeJS)
}

if len(pkgs) > 0 {
foundTypes.DepFoundAddFlag(LangJava)
for _, p := range pkgs {
deps = append(deps,
Dependency{
DepType: LangNodeJS,
Path: p.Name,
Version: p.Version,
Files: []string{},
})
}
default:
ext := filepath.Ext(filename)
// java
switch ext {
case ".zip":
// be more aggressive with zip files, must contain something java ish
if ok, _ := utils.ZipContainsJava(path); !ok {
return nil
}
fallthrough
case ".jar":
fallthrough
case ".war":
fallthrough
case ".ear":
fallthrough
case ".adm":
fallthrough
case ".hpi":
file := strings.Replace(filepath.Base(path), ext, "", 1) // get filename, check if we can ignore
if strings.HasSuffix(file, "-sources") || strings.HasSuffix(file, "-javadoc") {
return nil
}

for name, version := range pkgs {
// just in case we report the full path to the dep
name = strings.Replace(name, fullPath, "", 1)

// if the dep ends with -javadoc or -sources, not really interested
if !strings.HasSuffix(version, "-javadoc") && !strings.HasSuffix(version, "-sources") {
deps = append(deps,
Dependency{
DepType: LangJava,
Path: name,
Version: version,
Files: []string{},
})
pkgs, err := scan.GetJarDeps(path)
if err == nil {

if len(pkgs) > 0 {
foundTypes.DepFoundAddFlag(LangJava)
}

for name, version := range pkgs {
// just in case we report the full path to the dep
name = strings.Replace(name, fullPath, "", 1)

// if the dep ends with -javadoc or -sources, not really interested
if !strings.HasSuffix(version, "-javadoc") && !strings.HasSuffix(version, "-sources") {
deps = append(deps,
Dependency{
DepType: LangJava,
Path: name,
Version: version,
Files: []string{},
})
}
}
}
}
Expand Down
24 changes: 24 additions & 0 deletions internal/scan/nodejs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package scan
import (
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
Expand All @@ -25,6 +26,10 @@ type yarnOutput struct {
}
}

type packageJsonFormat struct {
Name string `json:"name"`
Version string `json:"version"`
}
type npmDependency struct {
Version string `json:"version"`
Dependencies map[string]npmDependency `json:"dependencies"`
Expand Down Expand Up @@ -121,6 +126,25 @@ func GetNodeJSDeps(path string) (map[string]NodeJSGather, error) {
return nil, fmt.Errorf("unknown NodeJS dependency file %q", path)
}

func GetNodeJSPackage(path string) (NodeJSGather, error) {
log.Debugf("GetNodeJSPackage %s", path)

data, err := os.ReadFile(path)
if err != nil {
return NodeJSGather{}, err
}

var packageJson packageJsonFormat
err = json.Unmarshal(data, &packageJson)
if err != nil {
return NodeJSGather{}, err
}
if packageJson.Name == "" {
return NodeJSGather{}, fmt.Errorf("Empty package")
}
return NodeJSGather{Name: packageJson.Name, Version: packageJson.Version}, nil
}

func getYarnDeps(path string) (map[string]NodeJSGather, error) {
var yarnOutput yarnOutput
gatheredNode = make(map[string]NodeJSGather)
Expand Down

0 comments on commit 589a994

Please sign in to comment.