diff --git a/cli/scancommands.go b/cli/scancommands.go index 5493556d..97bba6a9 100644 --- a/cli/scancommands.go +++ b/cli/scancommands.go @@ -418,6 +418,7 @@ func CurationCmd(c *components.Context) error { return err } curationAuditCommand.SetServerDetails(serverDetails). + SetIsCurationCmd(true). SetExcludeTestDependencies(c.GetBoolFlagValue(flags.ExcludeTestDeps)). SetOutputFormat(format). SetUseWrapper(c.GetBoolFlagValue(flags.UseWrapper)). diff --git a/commands/audit/sca/common_test.go b/commands/audit/sca/common_test.go index 1fb1c2d7..a99117f4 100644 --- a/commands/audit/sca/common_test.go +++ b/commands/audit/sca/common_test.go @@ -1,6 +1,7 @@ package sca import ( + "golang.org/x/exp/maps" "reflect" "testing" @@ -12,13 +13,13 @@ import ( ) func TestBuildXrayDependencyTree(t *testing.T) { - treeHelper := make(map[string][]string) - rootDep := []string{"topDep1", "topDep2", "topDep3"} - topDep1 := []string{"midDep1", "midDep2"} - topDep2 := []string{"midDep2", "midDep3"} - midDep1 := []string{"bottomDep1"} - midDep2 := []string{"bottomDep2", "bottomDep3"} - bottomDep3 := []string{"leafDep"} + treeHelper := make(map[string]coreXray.DepTreeNode) + rootDep := coreXray.DepTreeNode{Children: []string{"topDep1", "topDep2", "topDep3"}} + topDep1 := coreXray.DepTreeNode{Children: []string{"midDep1", "midDep2"}} + topDep2 := coreXray.DepTreeNode{Children: []string{"midDep2", "midDep3"}} + midDep1 := coreXray.DepTreeNode{Children: []string{"bottomDep1"}} + midDep2 := coreXray.DepTreeNode{Children: []string{"bottomDep2", "bottomDep3"}} + bottomDep3 := coreXray.DepTreeNode{Children: []string{"leafDep"}} treeHelper["rootDep"] = rootDep treeHelper["topDep1"] = topDep1 treeHelper["topDep2"] = topDep2 @@ -69,7 +70,7 @@ func TestBuildXrayDependencyTree(t *testing.T) { tree, uniqueDeps := coreXray.BuildXrayDependencyTree(treeHelper, "rootDep") - assert.ElementsMatch(t, expectedUniqueDeps, uniqueDeps) + assert.ElementsMatch(t, expectedUniqueDeps, maps.Keys(uniqueDeps)) assert.True(t, tests.CompareTree(tree, rootNode)) } diff --git a/commands/audit/sca/npm/npm.go b/commands/audit/sca/npm/npm.go index fcf3a088..ad9c2003 100644 --- a/commands/audit/sca/npm/npm.go +++ b/commands/audit/sca/npm/npm.go @@ -11,6 +11,7 @@ import ( "github.com/jfrog/jfrog-cli-security/utils" "github.com/jfrog/jfrog-client-go/utils/log" xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" + "golang.org/x/exp/maps" "golang.org/x/exp/slices" ) @@ -67,6 +68,7 @@ func configNpmResolutionServerIfNeeded(params utils.AuditParams) (clearResolutio if params.DepsRepo() == "" { return } + serverDetails, err := params.ServerDetails() if err != nil { return @@ -103,19 +105,22 @@ func addIgnoreScriptsFlag(npmArgs []string) []string { // Parse the dependencies into an Xray dependency tree format func parseNpmDependenciesList(dependencies []buildinfo.Dependency, packageInfo *biutils.PackageInfo) (*xrayUtils.GraphNode, []string) { - treeMap := make(map[string][]string) + treeMap := make(map[string]coreXray.DepTreeNode) for _, dependency := range dependencies { dependencyId := utils.NpmPackageTypeIdentifier + dependency.Id for _, requestedByNode := range dependency.RequestedBy { parent := utils.NpmPackageTypeIdentifier + requestedByNode[0] - if children, ok := treeMap[parent]; ok { - treeMap[parent] = appendUniqueChild(children, dependencyId) + depTreeNode, ok := treeMap[parent] + if ok { + depTreeNode.Children = appendUniqueChild(depTreeNode.Children, dependencyId) } else { - treeMap[parent] = []string{dependencyId} + depTreeNode.Children = []string{dependencyId} } + treeMap[parent] = depTreeNode } } - return coreXray.BuildXrayDependencyTree(treeMap, utils.NpmPackageTypeIdentifier+packageInfo.BuildInfoModuleId()) + graph, nodeMapTypes := coreXray.BuildXrayDependencyTree(treeMap, utils.NpmPackageTypeIdentifier+packageInfo.BuildInfoModuleId()) + return graph, maps.Keys(nodeMapTypes) } func appendUniqueChild(children []string, candidateDependency string) []string { diff --git a/commands/audit/sca/nuget/nuget.go b/commands/audit/sca/nuget/nuget.go index 8dd0d841..63cae11f 100644 --- a/commands/audit/sca/nuget/nuget.go +++ b/commands/audit/sca/nuget/nuget.go @@ -16,6 +16,7 @@ import ( "github.com/jfrog/jfrog-client-go/utils/io/fileutils" "github.com/jfrog/jfrog-client-go/utils/log" xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils" + "golang.org/x/exp/maps" "io/fs" "os" "os/exec" @@ -215,19 +216,21 @@ func runDotnetRestore(wd string, params utils.AuditParams, toolType bidotnet.Too func parseNugetDependencyTree(buildInfo *entities.BuildInfo) (nodes []*xrayUtils.GraphNode, allUniqueDeps []string) { uniqueDepsSet := datastructures.MakeSet[string]() for _, module := range buildInfo.Modules { - treeMap := make(map[string][]string) + treeMap := make(map[string]coreXray.DepTreeNode) for _, dependency := range module.Dependencies { dependencyId := nugetPackageTypeIdentifier + dependency.Id parent := nugetPackageTypeIdentifier + dependency.RequestedBy[0][0] - if children, ok := treeMap[parent]; ok { - treeMap[parent] = append(children, dependencyId) + depTreeNode, ok := treeMap[parent] + if ok { + depTreeNode.Children = append(depTreeNode.Children, dependencyId) } else { - treeMap[parent] = []string{dependencyId} + depTreeNode.Children = []string{dependencyId} } + treeMap[parent] = depTreeNode } dependencyTree, uniqueDeps := coreXray.BuildXrayDependencyTree(treeMap, nugetPackageTypeIdentifier+module.Id) nodes = append(nodes, dependencyTree) - for _, uniqueDep := range uniqueDeps { + for _, uniqueDep := range maps.Keys(uniqueDeps) { uniqueDepsSet.Add(uniqueDep) } } diff --git a/commands/audit/sca/yarn/yarn.go b/commands/audit/sca/yarn/yarn.go index 025da9d2..c70e623c 100644 --- a/commands/audit/sca/yarn/yarn.go +++ b/commands/audit/sca/yarn/yarn.go @@ -3,6 +3,7 @@ package yarn import ( "errors" "fmt" + "golang.org/x/exp/maps" "path/filepath" "github.com/jfrog/build-info-go/build" @@ -199,7 +200,7 @@ func runYarnInstallAccordingToVersion(curWd, yarnExecPath string, installCommand // Parse the dependencies into a Xray dependency tree format func parseYarnDependenciesMap(dependencies map[string]*biutils.YarnDependency, rootXrayId string) (*xrayUtils.GraphNode, []string) { - treeMap := make(map[string][]string) + treeMap := make(map[string]coreXray.DepTreeNode) for _, dependency := range dependencies { xrayDepId := getXrayDependencyId(dependency) var subDeps []string @@ -207,10 +208,11 @@ func parseYarnDependenciesMap(dependencies map[string]*biutils.YarnDependency, r subDeps = append(subDeps, getXrayDependencyId(dependencies[biutils.GetYarnDependencyKeyFromLocator(subDepPtr.Locator)])) } if len(subDeps) > 0 { - treeMap[xrayDepId] = subDeps + treeMap[xrayDepId] = coreXray.DepTreeNode{Children: subDeps} } } - return coreXray.BuildXrayDependencyTree(treeMap, rootXrayId) + graph, uniqDeps := coreXray.BuildXrayDependencyTree(treeMap, rootXrayId) + return graph, maps.Keys(uniqDeps) } func getXrayDependencyId(yarnDependency *biutils.YarnDependency) string { diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index eb8789ed..da1a961f 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -190,19 +190,33 @@ func GetTechDependencyTree(params xrayutils.AuditParams, tech coreutils.Technolo if params.Progress() != nil { params.Progress().SetHeadlineMsg(logMessage) } - serverDetails, err := params.ServerDetails() + err = SetResolutionRepoIfExists(params, tech) if err != nil { return } - err = SetResolutionRepoIfExists(params, tech) + serverDetails, err := params.ServerDetails() if err != nil { return } var uniqueDeps []string + var uniqDepsWithTypes map[string][]string startTime := time.Now() switch tech { case coreutils.Maven, coreutils.Gradle: - fullDependencyTrees, uniqueDeps, err = java.BuildDependencyTree(serverDetails, params.DepsRepo(), params.UseWrapper(), params.IsMavenDepTreeInstalled(), tech) + curationCacheFolder := "" + if params.IsCurationCmd() { + curationCacheFolder, err = xrayutils.GetCurationMavenCacheFolder() + if err != nil { + return + } + } + fullDependencyTrees, uniqDepsWithTypes, err = java.BuildDependencyTree(java.DepTreeParams{ + Server: serverDetails, + DepsRepo: params.DepsRepo(), + IsMavenDepTreeInstalled: params.IsMavenDepTreeInstalled(), + IsCurationCmd: params.IsCurationCmd(), + CurationCacheFolder: curationCacheFolder, + }, tech) case coreutils.Npm: fullDependencyTrees, uniqueDeps, err = npm.BuildDependencyTree(params) case coreutils.Yarn: @@ -220,17 +234,21 @@ func GetTechDependencyTree(params xrayutils.AuditParams, tech coreutils.Technolo default: err = errorutils.CheckErrorf("%s is currently not supported", string(tech)) } - if err != nil || len(uniqueDeps) == 0 { + if err != nil || (len(uniqueDeps) == 0 && len(uniqDepsWithTypes) == 0) { return } log.Debug(fmt.Sprintf("Created '%s' dependency tree with %d nodes. Elapsed time: %.1f seconds.", tech.ToFormal(), len(uniqueDeps), time.Since(startTime).Seconds())) + if len(uniqDepsWithTypes) > 0 { + flatTree, err = createFlatTreeWithTypes(uniqDepsWithTypes) + return + } flatTree, err = createFlatTree(uniqueDeps) return } // Associates a technology with another of a different type in the structure. // Docker is not present, as there is no docker-config command and, consequently, no docker.yaml file we need to operate on. -var techType = map[coreutils.Technology]project.ProjectType{ +var TechType = map[coreutils.Technology]project.ProjectType{ coreutils.Maven: project.Maven, coreutils.Gradle: project.Gradle, coreutils.Npm: project.Npm, coreutils.Yarn: project.Yarn, coreutils.Go: project.Go, coreutils.Pip: project.Pip, coreutils.Pipenv: project.Pipenv, coreutils.Poetry: project.Poetry, coreutils.Nuget: project.Nuget, coreutils.Dotnet: project.Dotnet, } @@ -241,7 +259,7 @@ func SetResolutionRepoIfExists(params xrayutils.AuditParams, tech coreutils.Tech return } - configFilePath, exists, err := project.GetProjectConfFilePath(techType[tech]) + configFilePath, exists, err := project.GetProjectConfFilePath(TechType[tech]) if err != nil { err = fmt.Errorf("failed while searching for %s.yaml config file: %s", tech.String(), err.Error()) return @@ -250,7 +268,7 @@ func SetResolutionRepoIfExists(params xrayutils.AuditParams, tech coreutils.Tech // Nuget and Dotnet are identified similarly in the detection process. To prevent redundancy, Dotnet is filtered out earlier in the process, focusing solely on detecting Nuget. // Consequently, it becomes necessary to verify the presence of dotnet.yaml when Nuget detection occurs. if tech == coreutils.Nuget { - configFilePath, exists, err = project.GetProjectConfFilePath(techType[coreutils.Dotnet]) + configFilePath, exists, err = project.GetProjectConfFilePath(TechType[coreutils.Dotnet]) if err != nil { err = fmt.Errorf("failed while searching for %s.yaml config file: %s", tech.String(), err.Error()) return @@ -292,14 +310,21 @@ func SetResolutionRepoIfExists(params xrayutils.AuditParams, tech coreutils.Tech return } +func createFlatTreeWithTypes(uniqueDeps map[string][]string) (*xrayCmdUtils.GraphNode, error) { + if err := logDeps(uniqueDeps); err != nil { + return nil, err + } + var uniqueNodes []*xrayCmdUtils.GraphNode + for uniqueDep, types := range uniqueDeps { + p := types + uniqueNodes = append(uniqueNodes, &xrayCmdUtils.GraphNode{Id: uniqueDep, Types: &p}) + } + return &xrayCmdUtils.GraphNode{Id: "root", Nodes: uniqueNodes}, nil +} + func createFlatTree(uniqueDeps []string) (*xrayCmdUtils.GraphNode, error) { - if log.GetLogger().GetLogLevel() == log.DEBUG { - // Avoid printing and marshaling if not on DEBUG mode. - jsonList, err := json.Marshal(uniqueDeps) - if errorutils.CheckError(err) != nil { - return nil, err - } - log.Debug("Unique dependencies list:\n" + clientutils.IndentJsonArray(jsonList)) + if err := logDeps(uniqueDeps); err != nil { + return nil, err } uniqueNodes := []*xrayCmdUtils.GraphNode{} for _, uniqueDep := range uniqueDeps { @@ -307,3 +332,17 @@ func createFlatTree(uniqueDeps []string) (*xrayCmdUtils.GraphNode, error) { } return &xrayCmdUtils.GraphNode{Id: "root", Nodes: uniqueNodes}, nil } + +func logDeps(uniqueDeps any) (err error) { + if log.GetLogger().GetLogLevel() != log.DEBUG { + // Avoid printing and marshaling if not on DEBUG mode. + return + } + jsonList, err := json.Marshal(uniqueDeps) + if errorutils.CheckError(err) != nil { + return err + } + log.Debug("Unique dependencies list:\n" + clientutils.IndentJsonArray(jsonList)) + + return +} diff --git a/commands/curation/curationaudit.go b/commands/curation/curationaudit.go index 91ba0cf5..91c65710 100644 --- a/commands/curation/curationaudit.go +++ b/commands/curation/curationaudit.go @@ -53,8 +53,11 @@ const ( var CurationOutputFormats = []string{string(outFormat.Table), string(outFormat.Json)} -var supportedTech = map[coreutils.Technology]struct{}{ - coreutils.Npm: {}, +var supportedTech = map[coreutils.Technology]func() (bool, error){ + coreutils.Npm: func() (bool, error) { return true, nil }, + coreutils.Maven: func() (bool, error) { + return clientutils.GetBoolEnvValue(utils.CurationMavenSupport, false) + }, } type ErrorsResp struct { @@ -186,10 +189,20 @@ func (ca *CurationAuditCommand) Run() (err error) { func (ca *CurationAuditCommand) doCurateAudit(results map[string][]*PackageStatus) error { techs := coreutils.DetectedTechnologiesList() for _, tech := range techs { - if _, ok := supportedTech[coreutils.Technology(tech)]; !ok { + supportedFunc, ok := supportedTech[coreutils.Technology(tech)] + if !ok { log.Info(fmt.Sprintf(errorTemplateUnsupportedTech, tech)) continue } + supported, err := supportedFunc() + if err != nil { + return err + } + if !supported { + log.Info(fmt.Sprintf(errorTemplateUnsupportedTech, tech)) + continue + } + if err := ca.auditTree(coreutils.Technology(tech), results); err != nil { return err } @@ -198,21 +211,24 @@ func (ca *CurationAuditCommand) doCurateAudit(results map[string][]*PackageStatu } func (ca *CurationAuditCommand) getAuditParamsByTech(tech coreutils.Technology) utils.AuditParams { - if tech == coreutils.Npm { + switch tech { + case coreutils.Npm: return utils.AuditNpmParams{AuditParams: ca.AuditParams}. SetNpmIgnoreNodeModules(true). SetNpmOverwritePackageLock(true) + case coreutils.Maven: + ca.AuditParams.SetIsMavenDepTreeInstalled(true) } return ca.AuditParams } func (ca *CurationAuditCommand) auditTree(tech coreutils.Technology, results map[string][]*PackageStatus) error { - flattenGraph, fullDependenciesTree, err := audit.GetTechDependencyTree(ca.getAuditParamsByTech(tech), tech) + flattenGraph, fullDependenciesTrees, err := audit.GetTechDependencyTree(ca.getAuditParamsByTech(tech), tech) if err != nil { return err } // Validate the graph isn't empty. - if len(fullDependenciesTree) == 0 { + if len(fullDependenciesTrees) == 0 { return errorutils.CheckErrorf("found no dependencies for the audited project using '%v' as the package manager", tech.String()) } if err = ca.SetRepo(tech); err != nil { @@ -231,8 +247,8 @@ func (ca *CurationAuditCommand) auditTree(tech coreutils.Technology, results map if err != nil { return err } - rootNode := fullDependenciesTree[0] - _, projectName, projectScope, projectVersion := getUrlNameAndVersionByTech(tech, rootNode.Id, "", "") + rootNode := fullDependenciesTrees[0] + _, projectName, projectScope, projectVersion := getUrlNameAndVersionByTech(tech, rootNode, "", "") if ca.Progress() != nil { ca.Progress().SetHeadlineMsg(fmt.Sprintf("Fetch curation status for %s graph with %v nodes project name: %s:%s", tech.ToFormal(), len(flattenGraph.Nodes)-1, projectName, projectVersion)) } @@ -253,11 +269,17 @@ func (ca *CurationAuditCommand) auditTree(tech coreutils.Technology, results map tech: tech, parallelRequests: ca.parallelRequests, } - packagesStatusMap := sync.Map{} + + rootNodes := map[string]struct{}{} + for _, tree := range fullDependenciesTrees { + rootNodes[tree.Id] = struct{}{} + } // Fetch status for each node from a flatten graph which, has no duplicate nodes. - err = analyzer.fetchNodesStatus(flattenGraph, &packagesStatusMap, rootNode.Id) - analyzer.fillGraphRelations(rootNode, &packagesStatusMap, - &packagesStatus, "", "", datastructures.MakeSet[string](), true) + packagesStatusMap := sync.Map{} + // if error returned we still want to produce a report, so we don't fail the next step + err = analyzer.fetchNodesStatus(flattenGraph, &packagesStatusMap, rootNodes) + analyzer.GraphsRelations(fullDependenciesTrees, &packagesStatusMap, + &packagesStatus) sort.Slice(packagesStatus, func(i, j int) bool { return packagesStatus[i].ParentName < packagesStatus[j].ParentName }) @@ -327,35 +349,42 @@ func (ca *CurationAuditCommand) CommandName() string { } func (ca *CurationAuditCommand) SetRepo(tech coreutils.Technology) error { - switch tech { - case coreutils.Npm: - configFilePath, exists, err := project.GetProjectConfFilePath(project.Npm) - if err != nil { - return err - } - if !exists { - return errorutils.CheckErrorf("no config file was found! Before running the npm command on a " + - "project for the first time, the project should be configured using the 'jf npmc' command") - } - vConfig, err := project.ReadConfigFile(configFilePath, project.YAML) - if err != nil { - return err - } - resolverParams, err := project.GetRepoConfigByPrefix(configFilePath, project.ProjectConfigResolverPrefix, vConfig) - if err != nil { - return err - } - ca.setPackageManagerConfig(resolverParams) - default: - return errorutils.CheckErrorf(errorTemplateUnsupportedTech, tech.String()) + resolverParams, err := ca.getRepoParams(audit.TechType[tech]) + if err != nil { + return err } + ca.setPackageManagerConfig(resolverParams) return nil } +func (ca *CurationAuditCommand) getRepoParams(projectType project.ProjectType) (*project.RepositoryConfig, error) { + configFilePath, exists, err := project.GetProjectConfFilePath(projectType) + if err != nil { + return nil, err + } + if !exists { + return nil, errorutils.CheckErrorf("no config file was found! Before running the " + projectType.String() + " command on a " + + "project for the first time, the project should be configured using the 'jf " + projectType.String() + "c' command") + } + vConfig, err := project.ReadConfigFile(configFilePath, project.YAML) + if err != nil { + return nil, err + } + return project.GetRepoConfigByPrefix(configFilePath, project.ProjectConfigResolverPrefix, vConfig) +} + +func (nc *treeAnalyzer) GraphsRelations(fullDependenciesTrees []*xrayUtils.GraphNode, preProcessMap *sync.Map, packagesStatus *[]*PackageStatus) { + visited := datastructures.MakeSet[string]() + for _, node := range fullDependenciesTrees { + nc.fillGraphRelations(node, preProcessMap, + packagesStatus, "", "", visited, true) + } +} + func (nc *treeAnalyzer) fillGraphRelations(node *xrayUtils.GraphNode, preProcessMap *sync.Map, packagesStatus *[]*PackageStatus, parent, parentVersion string, visited *datastructures.Set[string], isRoot bool) { for _, child := range node.Nodes { - packageUrl, name, scope, version := getUrlNameAndVersionByTech(nc.tech, child.Id, nc.url, nc.repo) + packageUrls, name, scope, version := getUrlNameAndVersionByTech(nc.tech, child, nc.url, nc.repo) if isRoot { parent = name parentVersion = version @@ -368,31 +397,34 @@ func (nc *treeAnalyzer) fillGraphRelations(node *xrayUtils.GraphNode, preProcess } visited.Add(scope + name + version + "-" + parent + parentVersion) - if pkgStatus, exist := preProcessMap.Load(packageUrl); exist { - relation := indirectRelation - if isRoot { - relation = directRelation - } - pkgStatusCast, isPkgStatus := pkgStatus.(*PackageStatus) - if isPkgStatus { - pkgStatusClone := *pkgStatusCast - pkgStatusClone.DepRelation = relation - pkgStatusClone.ParentName = parent - pkgStatusClone.ParentVersion = parentVersion - *packagesStatus = append(*packagesStatus, &pkgStatusClone) + for _, packageUrl := range packageUrls { + if pkgStatus, exist := preProcessMap.Load(packageUrl); exist { + relation := indirectRelation + if isRoot { + relation = directRelation + } + pkgStatusCast, isPkgStatus := pkgStatus.(*PackageStatus) + if isPkgStatus { + pkgStatusClone := *pkgStatusCast + pkgStatusClone.DepRelation = relation + pkgStatusClone.ParentName = parent + pkgStatusClone.ParentVersion = parentVersion + *packagesStatus = append(*packagesStatus, &pkgStatusClone) + } } } nc.fillGraphRelations(child, preProcessMap, packagesStatus, parent, parentVersion, visited, false) } } -func (nc *treeAnalyzer) fetchNodesStatus(graph *xrayUtils.GraphNode, p *sync.Map, rootNodeId string) error { + +func (nc *treeAnalyzer) fetchNodesStatus(graph *xrayUtils.GraphNode, p *sync.Map, rootNodeIds map[string]struct{}) error { var multiErrors error consumerProducer := parallel.NewBounedRunner(nc.parallelRequests, false) errorsQueue := clientutils.NewErrorsQueue(1) go func() { defer consumerProducer.Done() for _, node := range graph.Nodes { - if node.Id == rootNodeId { + if _, ok := rootNodeIds[node.Id]; ok { continue } getTask := func(node xrayUtils.GraphNode) func(threadId int) error { @@ -413,29 +445,34 @@ func (nc *treeAnalyzer) fetchNodesStatus(graph *xrayUtils.GraphNode, p *sync.Map } func (nc *treeAnalyzer) fetchNodeStatus(node xrayUtils.GraphNode, p *sync.Map) error { - packageUrl, name, scope, version := getUrlNameAndVersionByTech(nc.tech, node.Id, nc.url, nc.repo) + packageUrls, name, scope, version := getUrlNameAndVersionByTech(nc.tech, &node, nc.url, nc.repo) + if len(packageUrls) == 0 { + return nil + } if scope != "" { name = scope + "/" + name } - resp, _, err := nc.rtManager.Client().SendHead(packageUrl, &nc.httpClientDetails) - if err != nil { - if resp != nil && resp.StatusCode >= 400 { - return errorutils.CheckErrorf(errorTemplateHeadRequest, packageUrl, name, version, resp.StatusCode, err) - } - if resp == nil || resp.StatusCode != http.StatusForbidden { - return err - } - } - if resp != nil && resp.StatusCode >= 400 && resp.StatusCode != http.StatusForbidden { - return errorutils.CheckErrorf(errorTemplateHeadRequest, packageUrl, name, version, resp.StatusCode, err) - } - if resp.StatusCode == http.StatusForbidden { - pkStatus, err := nc.getBlockedPackageDetails(packageUrl, name, version) + for _, packageUrl := range packageUrls { + resp, _, err := nc.rtManager.Client().SendHead(packageUrl, &nc.httpClientDetails) if err != nil { - return err + if resp != nil && resp.StatusCode >= 400 { + return errorutils.CheckErrorf(errorTemplateHeadRequest, packageUrl, name, version, resp.StatusCode, err) + } + if resp == nil || resp.StatusCode != http.StatusForbidden { + return err + } + } + if resp != nil && resp.StatusCode >= 400 && resp.StatusCode != http.StatusForbidden { + return errorutils.CheckErrorf(errorTemplateHeadRequest, packageUrl, name, version, resp.StatusCode, err) } - if pkStatus != nil { - p.Store(pkStatus.BlockedPackageUrl, pkStatus) + if resp.StatusCode == http.StatusForbidden { + pkStatus, err := nc.getBlockedPackageDetails(packageUrl, name, version) + if err != nil { + return err + } + if pkStatus != nil { + p.Store(pkStatus.BlockedPackageUrl, pkStatus) + } } } return nil @@ -514,16 +551,43 @@ func makeLegiblePolicyDetails(explanation, recommendation string) (string, strin return explanation, recommendation } -func getUrlNameAndVersionByTech(tech coreutils.Technology, nodeId, artiUrl, repo string) (downloadUrl string, name string, scope string, version string) { - if tech == coreutils.Npm { - return getNpmNameScopeAndVersion(nodeId, artiUrl, repo, coreutils.Npm.String()) +func getUrlNameAndVersionByTech(tech coreutils.Technology, node *xrayUtils.GraphNode, artiUrl, repo string) (downloadUrls []string, name string, scope string, version string) { + switch tech { + case coreutils.Npm: + return getNpmNameScopeAndVersion(node.Id, artiUrl, repo, coreutils.Npm.String()) + case coreutils.Maven: + return getMavenNameScopeAndVersion(node.Id, artiUrl, repo, node.Types) } return } +// input- id: gav://org.apache.tomcat.embed:tomcat-embed-jasper:8.0.33 +// input - repo: libs-release +// output - downloadUrl: /libs-release/org/apache/tomcat/embed/tomcat-embed-jasper/8.0.33/tomcat-embed-jasper-8.0.33.jar +func getMavenNameScopeAndVersion(id, artiUrl, repo string, types *[]string) (downloadUrls []string, name, scope, version string) { + id = strings.TrimPrefix(id, "gav://") + allParts := strings.Split(id, ":") + if len(allParts) < 3 { + return + } + nameVersion := allParts[1] + "-" + allParts[2] + packagePath := strings.Join(strings.Split(allParts[0], "."), "/") + "/" + + allParts[1] + "/" + allParts[2] + "/" + nameVersion + if types != nil { + for _, fileType := range *types { + // curation service supports maven only for jar and war file types. + if fileType == "jar" || fileType == "war" { + downloadUrls = append(downloadUrls, strings.TrimSuffix(artiUrl, "/")+"/"+repo+"/"+packagePath+"."+fileType) + } + + } + } + return downloadUrls, strings.Join(allParts[:2], ":"), "", allParts[2] +} + // The graph holds, for each node, the component ID (xray representation) // from which we extract the package name, version, and construct the Artifactory download URL. -func getNpmNameScopeAndVersion(id, artiUrl, repo, tech string) (downloadUrl, name, scope, version string) { +func getNpmNameScopeAndVersion(id, artiUrl, repo, tech string) (downloadUrl []string, name, scope, version string) { id = strings.TrimPrefix(id, tech+"://") nameVersion := strings.Split(id, ":") @@ -539,14 +603,14 @@ func getNpmNameScopeAndVersion(id, artiUrl, repo, tech string) (downloadUrl, nam return buildNpmDownloadUrl(artiUrl, repo, name, scope, version), name, scope, version } -func buildNpmDownloadUrl(url, repo, name, scope, version string) string { +func buildNpmDownloadUrl(url, repo, name, scope, version string) []string { var packageUrl string if scope != "" { packageUrl = fmt.Sprintf("%s/api/npm/%s/%s/%s/-/%s-%s.tgz", strings.TrimSuffix(url, "/"), repo, scope, name, name, version) } else { packageUrl = fmt.Sprintf("%s/api/npm/%s/%s/-/%s-%s.tgz", strings.TrimSuffix(url, "/"), repo, name, name, version) } - return packageUrl + return []string{packageUrl} } func DetectNumOfThreads(threadsCount int) (int, error) { diff --git a/commands/curation/curationaudit_test.go b/commands/curation/curationaudit_test.go index 759b1bf6..fbe6b410 100644 --- a/commands/curation/curationaudit_test.go +++ b/commands/curation/curationaudit_test.go @@ -159,8 +159,8 @@ func TestGetNameScopeAndVersion(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotDownloadUrl, gotName, gotScope, gotVersion := getNpmNameScopeAndVersion(tt.componentId, tt.artiUrl, tt.repo, tt.repo) - assert.Equal(t, tt.wantDownloadUrl, gotDownloadUrl, "getNameScopeAndVersion() gotDownloadUrl = %v, want %v", gotDownloadUrl, tt.wantDownloadUrl) + gotDownloadUrls, gotName, gotScope, gotVersion := getNpmNameScopeAndVersion(tt.componentId, tt.artiUrl, tt.repo, tt.repo) + assert.Equal(t, tt.wantDownloadUrl, gotDownloadUrls[0], "getNameScopeAndVersion() gotDownloadUrl = %v, want %v", gotDownloadUrls[0], tt.wantDownloadUrl) assert.Equal(t, tt.wantName, gotName, "getNpmNameScopeAndVersion() gotName = %v, want %v", gotName, tt.wantName) assert.Equal(t, tt.wantScope, gotScope, "getNpmNameScopeAndVersion() gotScope = %v, want %v", gotScope, tt.wantScope) assert.Equal(t, tt.wantVersion, gotVersion, "getNpmNameScopeAndVersion() gotVersion = %v, want %v", gotVersion, tt.wantVersion) @@ -435,11 +435,12 @@ func TestDoCurationAudit(t *testing.T) { assert.EqualError(t, gotError, errMsgExpected) } // Add the mock server to the expected blocked message url - for index := range tt.expectedResp { - tt.expectedResp[index].BlockedPackageUrl = fmt.Sprintf("%s%s", strings.TrimSuffix(config.GetArtifactoryUrl(), "/"), tt.expectedResp[index].BlockedPackageUrl) + for key := range tt.expectedResp { + for index := range tt.expectedResp[key] { + tt.expectedResp[key][index].BlockedPackageUrl = fmt.Sprintf("%s%s", strings.TrimSuffix(config.GetArtifactoryUrl(), "/"), tt.expectedResp[key][index].BlockedPackageUrl) + } } - gotResults := results["npm_test:1.0.0"] - assert.Equal(t, tt.expectedResp, gotResults) + assert.Equal(t, tt.expectedResp, results) for _, requestDone := range tt.expectedRequest { assert.True(t, requestDone) } @@ -451,7 +452,7 @@ func getTestCasesForDoCurationAudit() []struct { name string expectedRequest map[string]bool requestToFail map[string]bool - expectedResp []*PackageStatus + expectedResp map[string][]*PackageStatus requestToError map[string]bool expectedError string } { @@ -459,7 +460,7 @@ func getTestCasesForDoCurationAudit() []struct { name string expectedRequest map[string]bool requestToFail map[string]bool - expectedResp []*PackageStatus + expectedResp map[string][]*PackageStatus requestToError map[string]bool expectedError string }{ @@ -472,21 +473,23 @@ func getTestCasesForDoCurationAudit() []struct { requestToFail: map[string]bool{ "/api/npm/npms/underscore/-/underscore-1.13.6.tgz": false, }, - expectedResp: []*PackageStatus{ - { - Action: "blocked", - ParentVersion: "1.13.6", - ParentName: "underscore", - BlockedPackageUrl: "/api/npm/npms/underscore/-/underscore-1.13.6.tgz", - PackageName: "underscore", - PackageVersion: "1.13.6", - BlockingReason: "Policy violations", - PkgType: "npm", - DepRelation: "direct", - Policy: []Policy{ - { - Policy: "pol1", - Condition: "cond1", + expectedResp: map[string][]*PackageStatus{ + "npm_test:1.0.0": { + { + Action: "blocked", + ParentVersion: "1.13.6", + ParentName: "underscore", + BlockedPackageUrl: "/api/npm/npms/underscore/-/underscore-1.13.6.tgz", + PackageName: "underscore", + PackageVersion: "1.13.6", + BlockingReason: "Policy violations", + PkgType: "npm", + DepRelation: "direct", + Policy: []Policy{ + { + Policy: "pol1", + Condition: "cond1", + }, }, }, }, @@ -504,21 +507,23 @@ func getTestCasesForDoCurationAudit() []struct { requestToError: map[string]bool{ "/api/npm/npms/lightweight/-/lightweight-0.1.0.tgz": false, }, - expectedResp: []*PackageStatus{ - { - Action: "blocked", - ParentVersion: "1.13.6", - ParentName: "underscore", - BlockedPackageUrl: "/api/npm/npms/underscore/-/underscore-1.13.6.tgz", - PackageName: "underscore", - PackageVersion: "1.13.6", - BlockingReason: "Policy violations", - PkgType: "npm", - DepRelation: "direct", - Policy: []Policy{ - { - Policy: "pol1", - Condition: "cond1", + expectedResp: map[string][]*PackageStatus{ + "npm_test:1.0.0": { + { + Action: "blocked", + ParentVersion: "1.13.6", + ParentName: "underscore", + BlockedPackageUrl: "/api/npm/npms/underscore/-/underscore-1.13.6.tgz", + PackageName: "underscore", + PackageVersion: "1.13.6", + BlockingReason: "Policy violations", + PkgType: "npm", + DepRelation: "direct", + Policy: []Policy{ + { + Policy: "pol1", + Condition: "cond1", + }, }, }, }, @@ -580,3 +585,75 @@ func WriteServerDetailsConfigFileBytes(t *testing.T, url string, configPath stri assert.NoError(t, os.WriteFile(confFilePath, detailsByte, 0644)) return confFilePath } + +func Test_getMavenNameScopeAndVersion(t *testing.T) { + type args struct { + id string + artiUrl string + repo string + types *[]string + } + tests := []struct { + name string + args args + wantDownloadUrls []string + wantName string + wantScope string + wantVersion string + }{ + { + name: "maven url jar", + args: args{ + id: "gav://org.apache.tomcat.embed:tomcat-embed-jasper:8.0.33", + artiUrl: "http://test:9000/artifactory", + repo: "maven-remote", + types: &[]string{"jar"}, + }, + wantDownloadUrls: []string{"http://test:9000/artifactory/maven-remote/org/apache/tomcat/embed/tomcat-embed-jasper/8.0.33/tomcat-embed-jasper-8.0.33.jar"}, + wantName: "org.apache.tomcat.embed:tomcat-embed-jasper", + wantVersion: "8.0.33", + }, + { + name: "maven url jar and war", + args: args{ + id: "gav://org.apache.tomcat.embed:tomcat-embed-jasper:8.0.33", + artiUrl: "http://test:9000/artifactory", + repo: "maven-remote", + types: &[]string{"jar", "war"}, + }, + wantDownloadUrls: []string{"http://test:9000/artifactory/maven-remote/org/apache/tomcat/embed/tomcat-embed-jasper/8.0.33/tomcat-embed-jasper-8.0.33.jar", + "http://test:9000/artifactory/maven-remote/org/apache/tomcat/embed/tomcat-embed-jasper/8.0.33/tomcat-embed-jasper-8.0.33.war"}, + wantName: "org.apache.tomcat.embed:tomcat-embed-jasper", + wantVersion: "8.0.33", + }, + { + name: "maven url pom - no expected url", + args: args{ + id: "gav://org.apache.tomcat.embed:tomcat-embed-jasper:8.0.33", + artiUrl: "http://test:9000/artifactory", + repo: "maven-remote", + types: &[]string{"pom"}, + }, + wantName: "org.apache.tomcat.embed:tomcat-embed-jasper", + wantVersion: "8.0.33", + }, + { + name: "bad id", + args: args{ + id: "gav://org.apache.tomcat.embed:8.0.33", + artiUrl: "http://test:9000/artifactory", + repo: "maven-remote", + types: &[]string{"jar"}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotDownloadUrls, gotName, gotScope, gotVersion := getMavenNameScopeAndVersion(tt.args.id, tt.args.artiUrl, tt.args.repo, tt.args.types) + assert.Equalf(t, tt.wantDownloadUrls, gotDownloadUrls, "getMavenNameScopeAndVersion(%v, %v, %v, %v)", tt.args.id, tt.args.artiUrl, tt.args.repo, tt.args.types) + assert.Equalf(t, tt.wantName, gotName, "getMavenNameScopeAndVersion(%v, %v, %v, %v)", tt.args.id, tt.args.artiUrl, tt.args.repo, tt.args.types) + assert.Equalf(t, tt.wantScope, gotScope, "getMavenNameScopeAndVersion(%v, %v, %v, %v)", tt.args.id, tt.args.artiUrl, tt.args.repo, tt.args.types) + assert.Equalf(t, tt.wantVersion, gotVersion, "getMavenNameScopeAndVersion(%v, %v, %v, %v)", tt.args.id, tt.args.artiUrl, tt.args.repo, tt.args.types) + }) + } +} diff --git a/go.mod b/go.mod index 0471ee5f..263f4da1 100644 --- a/go.mod +++ b/go.mod @@ -98,6 +98,6 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect ) -// replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 dev +replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240213075115-4bf1fe83505d replace github.com/jfrog/jfrog-client-go => github.com/orz25/jfrog-client-go v0.0.0-20240204100437-b823bf27a759 diff --git a/go.sum b/go.sum index bd80c417..ae6227e3 100644 --- a/go.sum +++ b/go.sum @@ -93,8 +93,8 @@ github.com/jfrog/gofrog v1.5.1 h1:2AXL8hHu1jJFMIoCqTp2OyRUfEqEp4nC7J8fwn6KtwE= github.com/jfrog/gofrog v1.5.1/go.mod h1:SZ1EPJUruxrVGndOzHd+LTiwWYKMlHqhKD+eu+v5Hqg= github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY= github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w= -github.com/jfrog/jfrog-cli-core/v2 v2.47.12 h1:xsEVdzbdhNGkI8Ey4Othx5+zpgCMnT99Uy71LOn+Q7k= -github.com/jfrog/jfrog-cli-core/v2 v2.47.12/go.mod h1:RVn4pIkR5fPUnr8gFXt61ou3pCNrrDdRQUpcolP4lhw= +github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240213075115-4bf1fe83505d h1:9efTE8NyZV6XtF9XoGq0g3XiEIYjCPdiHVEanxhhnlk= +github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240213075115-4bf1fe83505d/go.mod h1:+eraSKhahQf7tj09+g3rAA2Z+XPnZGfMc0y8uUDecZw= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= diff --git a/utils/auditbasicparams.go b/utils/auditbasicparams.go index 56418089..fee9e5c5 100644 --- a/utils/auditbasicparams.go +++ b/utils/auditbasicparams.go @@ -34,6 +34,8 @@ type AuditParams interface { SetIgnoreConfigFile(ignoreConfigFile bool) *AuditBasicParams IsMavenDepTreeInstalled() bool SetIsMavenDepTreeInstalled(isMavenDepTreeInstalled bool) *AuditBasicParams + IsCurationCmd() bool + SetIsCurationCmd(bool) *AuditBasicParams } type AuditBasicParams struct { @@ -45,6 +47,7 @@ type AuditBasicParams struct { insecureTls bool ignoreConfigFile bool isMavenDepTreeInstalled bool + isCurationCmd bool pipRequirementsFile string depsRepo string installCommandName string @@ -192,3 +195,11 @@ func (abp *AuditBasicParams) SetIsMavenDepTreeInstalled(isMavenDepTreeInstalled abp.isMavenDepTreeInstalled = isMavenDepTreeInstalled return abp } + +func (abp *AuditBasicParams) IsCurationCmd() bool { + return abp.isCurationCmd +} +func (abp *AuditBasicParams) SetIsCurationCmd(isCurationCmd bool) *AuditBasicParams { + abp.isCurationCmd = isCurationCmd + return abp +} diff --git a/utils/paths.go b/utils/paths.go new file mode 100644 index 00000000..05752d3d --- /dev/null +++ b/utils/paths.go @@ -0,0 +1,45 @@ +package utils + +import ( + "github.com/jfrog/jfrog-cli-core/v2/utils/coreutils" + "github.com/jfrog/jfrog-client-go/utils" + "os" + "path/filepath" +) + +const ( + JfrogCurationDirName = "curation" + + CurationsDir = "JFROG_CLI_CURATION_DIR" + + // #nosec G101 -- Not credentials. + CurationMavenSupport = "JFROG_CLI_CURATION_MAVEN" +) + +func getJfrogCurationFolder() (string, error) { + dependenciesDir := os.Getenv(CurationsDir) + if dependenciesDir != "" { + return utils.AddTrailingSlashIfNeeded(dependenciesDir), nil + } + jfrogHome, err := coreutils.GetJfrogHomeDir() + if err != nil { + return "", err + } + return filepath.Join(jfrogHome, JfrogCurationDirName), nil +} + +func getCurationCacheFolder() (string, error) { + curationFolder, err := getJfrogCurationFolder() + if err != nil { + return "", err + } + return filepath.Join(curationFolder, "cache"), nil +} + +func GetCurationMavenCacheFolder() (string, error) { + curationFolder, err := getCurationCacheFolder() + if err != nil { + return "", err + } + return filepath.Join(curationFolder, "maven"), nil +}