diff --git a/cli/docs/flags.go b/cli/docs/flags.go index 7132117d..cce3a3e2 100644 --- a/cli/docs/flags.go +++ b/cli/docs/flags.go @@ -2,7 +2,6 @@ package docs import ( "fmt" - "github.com/jfrog/jfrog-cli-security/commands" "strings" "github.com/jfrog/jfrog-cli-core/v2/common/cliutils" @@ -105,8 +104,7 @@ const ( WorkingDirs = "working-dirs" // Unique curation flags - CurationOutput = "curation-format" - CurationThreads = "curation-threads" + CurationOutput = "curation-format" ) // Mapping between security commands (key) and their flags (key). @@ -129,7 +127,7 @@ var commandFlags = map[string][]string{ Pnpm, Yarn, Go, Nuget, Pip, Pipenv, Poetry, MinSeverity, FixableOnly, ThirdPartyContextualAnalysis, Threads, }, CurationAudit: { - CurationOutput, WorkingDirs, CurationThreads, RequirementsFile, + CurationOutput, WorkingDirs, Threads, RequirementsFile, }, // TODO: Deprecated commands (remove at next CLI major version) AuditMvn: { @@ -220,7 +218,6 @@ var flagsMap = map[string]components.Flag{ components.SetHiddenBoolFlag(), ), RequirementsFile: components.NewStringFlag(RequirementsFile, "[Pip] Defines pip requirements file name. For example: 'requirements.txt'."), - CurationThreads: components.NewStringFlag(Threads, "Number of working threads.", components.WithIntDefaultValue(commands.TotalConcurrentRequests)), CurationOutput: components.NewStringFlag(OutputFormat, "Defines the output format of the command. Acceptable values are: table, json.", components.WithStrDefaultValue("table")), } diff --git a/cli/scancommands.go b/cli/scancommands.go index f6b9735b..3655827d 100644 --- a/cli/scancommands.go +++ b/cli/scancommands.go @@ -3,7 +3,6 @@ package cli import ( "fmt" "github.com/jfrog/jfrog-cli-core/v2/utils/usage" - "github.com/jfrog/jfrog-cli-security/commands" "os" "strings" @@ -331,15 +330,11 @@ func AuditCmd(c *components.Context) error { } } auditCmd.SetTechnologies(technologies) - threadsFlag, err := c.GetIntFlagValue(flags.Threads) - if err != nil { - return err - } - threads, err := commands.DetectNumOfThreads(threadsFlag) + threads, err := pluginsCommon.GetThreadsCount(c) if err != nil { return err } - auditCmd.SetParallelScans(threads) + auditCmd.SetThreads(threads) err = progressbar.ExecWithProgress(auditCmd) // Reporting error if Xsc service is enabled reportErrorIfExists(err, auditCmd) @@ -436,11 +431,7 @@ func AuditSpecificCmd(c *components.Context, technology coreutils.Technology) er } func CurationCmd(c *components.Context) error { - threadsFlag, err := c.GetIntFlagValue(flags.Threads) - if err != nil { - return err - } - threads, err := commands.DetectNumOfThreads(threadsFlag) + threads, err := pluginsCommon.GetThreadsCount(c) if err != nil { return err } diff --git a/commands/audit/audit.go b/commands/audit/audit.go index 3bbb3f6c..af205e7f 100644 --- a/commands/audit/audit.go +++ b/commands/audit/audit.go @@ -27,21 +27,10 @@ type AuditCommand struct { Fail bool PrintExtendedTable bool analyticsMetricsService *xrayutils.AnalyticsMetricsService - ParallelScans int + Threads int AuditParams } -type CommonGraphScanParams struct { - repoPath string - projectKey string - watches []string - scanType services.ScanType - includeVulnerabilities bool - includeLicenses bool - xscVersion string - multiScanId string -} - func NewGenericAuditCommand() *AuditCommand { return &AuditCommand{AuditParams: *NewAuditParams()} } @@ -86,33 +75,33 @@ func (auditCmd *AuditCommand) SetAnalyticsMetricsService(analyticsMetricsService return auditCmd } -func (auditCmd *AuditCommand) SetParallelScans(threads int) *AuditCommand { - auditCmd.ParallelScans = threads +func (auditCmd *AuditCommand) SetThreads(threads int) *AuditCommand { + auditCmd.Threads = threads return auditCmd } -func (auditCmd *AuditCommand) CreateCommonGraphScanParams() *CommonGraphScanParams { - commonParams := &CommonGraphScanParams{ - repoPath: auditCmd.targetRepoPath, - watches: auditCmd.watches, - scanType: services.Dependency, +func (auditCmd *AuditCommand) CreateCommonGraphScanParams() *scangraph.CommonGraphScanParams { + commonParams := &scangraph.CommonGraphScanParams{ + RepoPath: auditCmd.targetRepoPath, + Watches: auditCmd.watches, + ScanType: services.Dependency, } if auditCmd.projectKey == "" { - commonParams.projectKey = os.Getenv(coreutils.Project) + commonParams.ProjectKey = os.Getenv(coreutils.Project) } else { - commonParams.projectKey = auditCmd.projectKey + commonParams.ProjectKey = auditCmd.projectKey } - commonParams.includeVulnerabilities = auditCmd.IncludeVulnerabilities - commonParams.includeLicenses = auditCmd.IncludeLicenses - commonParams.multiScanId = auditCmd.analyticsMetricsService.GetMsi() - if commonParams.multiScanId != "" { + commonParams.IncludeVulnerabilities = auditCmd.IncludeVulnerabilities + commonParams.IncludeLicenses = auditCmd.IncludeLicenses + commonParams.MultiScanId = auditCmd.analyticsMetricsService.GetMsi() + if commonParams.MultiScanId != "" { xscManager := auditCmd.analyticsMetricsService.XscManager() if xscManager != nil { version, err := xscManager.GetVersion() if err != nil { log.Debug(fmt.Sprintf("Can't get XSC version for xray graph scan params. Cause: %s", err.Error())) } - commonParams.xscVersion = version + commonParams.XscVersion = version } } return commonParams @@ -135,7 +124,7 @@ func (auditCmd *AuditCommand) Run() (err error) { SetGraphBasicParams(auditCmd.AuditBasicParams). SetCommonGraphScanParams(auditCmd.CreateCommonGraphScanParams()). SetThirdPartyApplicabilityScan(auditCmd.thirdPartyApplicabilityScan). - SetParallelScans(auditCmd.ParallelScans) + SetThreads(auditCmd.Threads) auditParams.SetIsRecursiveScan(isRecursiveScan).SetExclusions(auditCmd.Exclusions()) auditResults, err := RunAudit(auditParams) @@ -202,9 +191,9 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) return } - results.MultiScanId = auditParams.commonGraphScanParams.multiScanId + results.MultiScanId = auditParams.commonGraphScanParams.MultiScanId - auditParallelRunner := utils.CreateAuditParallelRunner(auditParams.numOfParallelScans) + auditParallelRunner := utils.CreateAuditParallelRunner(auditParams.threads) JFrogAppsConfig, err := jas.CreateJFrogAppsConfig(auditParams.workingDirs) if err != nil { return results, fmt.Errorf("failed to create JFrogAppsConfig: %s", err.Error()) @@ -216,13 +205,12 @@ func RunAudit(auditParams *AuditParams) (results *xrayutils.Results, err error) return downloadAnalyzerManagerAndRunScanners(auditParallelRunner, results, serverDetails, auditParams, JFrogAppsConfig, threadId) }, auditParallelRunner.AddErrorToChan) if jasErr != nil { - auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to creat AM and jas scanners task: %s", jasErr.Error())) + auditParallelRunner.AddErrorToChan(fmt.Errorf("failed to create AM downloading task, skipping JAS scans...: %s", jasErr.Error())) } } // The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine. - scaScanErr := runScaScan(auditParallelRunner, auditParams, results) - if scaScanErr != nil { + if scaScanErr := runScaScan(auditParallelRunner, auditParams, results); scaScanErr != nil { auditParallelRunner.AddErrorToChan(scaScanErr) } go func() { @@ -256,10 +244,8 @@ func downloadAnalyzerManagerAndRunScanners(auditParallelRunner *utils.AuditParal defer func() { auditParallelRunner.JasWg.Done() }() - err = utils.DownloadAnalyzerManagerIfNeeded(threadId) - if err != nil { + if err = utils.DownloadAnalyzerManagerIfNeeded(threadId); err != nil { return } - err = RunJasScannersAndSetResults(auditParallelRunner, scanResults, serverDetails, auditParams, jfrogAppsConfig, scanResults.MultiScanId) - return + return RunJasScannersAndSetResults(auditParallelRunner, scanResults, serverDetails, auditParams, jfrogAppsConfig, scanResults.MultiScanId) } diff --git a/commands/audit/auditparams.go b/commands/audit/auditparams.go index 2b85474b..8d2be912 100644 --- a/commands/audit/auditparams.go +++ b/commands/audit/auditparams.go @@ -1,14 +1,14 @@ package audit import ( + "github.com/jfrog/jfrog-cli-security/scangraph" xrayutils "github.com/jfrog/jfrog-cli-security/utils" "github.com/jfrog/jfrog-client-go/xray/services" ) type AuditParams struct { - xrayGraphScanParams *services.XrayGraphScanParams - // common params to all scan routines - commonGraphScanParams *CommonGraphScanParams + // Common params to all scan routines + commonGraphScanParams *scangraph.CommonGraphScanParams workingDirs []string installFunc func(tech string) error fixableOnly bool @@ -17,13 +17,12 @@ type AuditParams struct { xrayVersion string // Include third party dependencies source code in the applicability scan. thirdPartyApplicabilityScan bool - numOfParallelScans int + threads int } func NewAuditParams() *AuditParams { return &AuditParams{ - xrayGraphScanParams: &services.XrayGraphScanParams{}, - AuditBasicParams: &xrayutils.AuditBasicParams{}, + AuditBasicParams: &xrayutils.AuditBasicParams{}, } } @@ -31,10 +30,6 @@ func (params *AuditParams) InstallFunc() func(tech string) error { return params.installFunc } -func (params *AuditParams) XrayGraphScanParams() *services.XrayGraphScanParams { - return params.xrayGraphScanParams -} - func (params *AuditParams) WorkingDirs() []string { return params.workingDirs } @@ -86,12 +81,25 @@ func (params *AuditParams) SetDepsRepo(depsRepo string) *AuditParams { return params } -func (params *AuditParams) SetParallelScans(numOfParallelScans int) *AuditParams { - params.numOfParallelScans = numOfParallelScans +func (params *AuditParams) SetThreads(threads int) *AuditParams { + params.threads = threads return params } -func (params *AuditParams) SetCommonGraphScanParams(commonParams *CommonGraphScanParams) *AuditParams { +func (params *AuditParams) SetCommonGraphScanParams(commonParams *scangraph.CommonGraphScanParams) *AuditParams { params.commonGraphScanParams = commonParams return params } + +func (params *AuditParams) createXrayGraphScanParams() *services.XrayGraphScanParams { + return &services.XrayGraphScanParams{ + RepoPath: params.commonGraphScanParams.RepoPath, + Watches: params.commonGraphScanParams.Watches, + ScanType: params.commonGraphScanParams.ScanType, + ProjectKey: params.commonGraphScanParams.ProjectKey, + IncludeVulnerabilities: params.commonGraphScanParams.IncludeVulnerabilities, + IncludeLicenses: params.commonGraphScanParams.IncludeLicenses, + XscVersion: params.commonGraphScanParams.XscVersion, + MultiScanId: params.commonGraphScanParams.MultiScanId, + } +} diff --git a/commands/audit/jas/applicability/applicabilitymanager.go b/commands/audit/jas/applicability/applicabilitymanager.go index 0dd1d67d..e4999a4a 100644 --- a/commands/audit/jas/applicability/applicabilitymanager.go +++ b/commands/audit/jas/applicability/applicabilitymanager.go @@ -1,18 +1,17 @@ package applicability import ( + "github.com/jfrog/gofrog/datastructures" jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-security/commands/audit/jas" - "path/filepath" - "strconv" - - "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/jfrog-cli-security/utils" + clientutils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/log" "github.com/jfrog/jfrog-client-go/xray/services" "github.com/owenrumney/go-sarif/v2/sarif" "golang.org/x/exp/maps" "golang.org/x/exp/slices" + "path/filepath" ) const ( @@ -49,17 +48,17 @@ func RunApplicabilityScan(auditParallelRunner *utils.AuditParallelRunner, xrayRe } applicabilityScanManager := newApplicabilityScanManager(xrayResults, directDependencies, scanner, thirdPartyContextualAnalysis, scannerTempDir) if !applicabilityScanManager.cvesExists() { - log.Debug("[thread_id: " + strconv.Itoa(threadId) + "] We couldn't find any vulnerable dependencies. Skipping....") + log.Debug(clientutils.GetLogMsgPrefix(threadId, false), "We couldn't find any vulnerable dependencies. Skipping....") return } - log.Info("[thread_id: " + strconv.Itoa(threadId) + "] Running applicability scanning...") + log.Info(clientutils.GetLogMsgPrefix(threadId, false), "Running applicability scanning...") if err = applicabilityScanManager.scanner.Run(applicabilityScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.Applicability, err) return } - auditParallelRunner.Mu.Lock() + auditParallelRunner.ResultsMu.Lock() extendedScanResults.ApplicabilityScanResults = applicabilityScanManager.applicabilityScanResults - auditParallelRunner.Mu.Unlock() + auditParallelRunner.ResultsMu.Unlock() return } diff --git a/commands/audit/jas/common.go b/commands/audit/jas/common.go index 5bca84fd..4a03e2a2 100644 --- a/commands/audit/jas/common.go +++ b/commands/audit/jas/common.go @@ -4,9 +4,9 @@ import ( "errors" "fmt" clientutils "github.com/jfrog/jfrog-client-go/utils" - "golang.org/x/exp/rand" "os" "path/filepath" + "strconv" "strings" "testing" "time" @@ -211,7 +211,8 @@ var FakeBasicXrayResults = []services.ScanResponse{ func InitJasTest(t *testing.T, workingDirs ...string) (*JasScanner, func()) { assert.NoError(t, utils.DownloadAnalyzerManagerIfNeeded(0)) - jfrogAppsConfigForTest, _ := CreateJFrogAppsConfig(workingDirs) + jfrogAppsConfigForTest, err := CreateJFrogAppsConfig(workingDirs) + assert.NoError(t, err) scanner, err := NewJasScanner(&FakeServerDetails, jfrogAppsConfigForTest) assert.NoError(t, err) return scanner, func() { @@ -302,13 +303,7 @@ func CreateScannerTempDirectory(scanner *JasScanner, scanType string) (string, e if scanner.TempDir == "" { return "", errors.New("scanner temp dir cannot be created in an empty base dir") } - rand.Seed(uint64(time.Now().UnixNano())) - randomString := "" - for i := 0; i < 4; i++ { - randomDigit := rand.Intn(10) - randomString += fmt.Sprintf("%d", randomDigit) - } - scannerTempDir := scanner.TempDir + "/" + scanType + "_" + randomString + scannerTempDir := scanner.TempDir + "/" + scanType + "_" + strconv.FormatInt(time.Now().Unix(), 10) err := os.MkdirAll(scannerTempDir, 0777) if err != nil { return "", err diff --git a/commands/audit/jas/common_test.go b/commands/audit/jas/common_test.go index 3128e2f1..0b67f03a 100644 --- a/commands/audit/jas/common_test.go +++ b/commands/audit/jas/common_test.go @@ -131,8 +131,7 @@ func TestSetAnalyticsMetricsDataForAnalyzerManager(t *testing.T) { } func TestCreateScannerTempDirectory(t *testing.T) { - scanner, cleanUp := InitJasTest(t) - defer cleanUp() + scanner := &JasScanner{TempDir: "path"} tempDir, err := CreateScannerTempDirectory(scanner, string(utils.Applicability)) assert.NoError(t, err) assert.NotEmpty(t, tempDir) @@ -143,10 +142,7 @@ func TestCreateScannerTempDirectory(t *testing.T) { } func TestCreateScannerTempDirectory_baseDirIsEmpty(t *testing.T) { - scanner, cleanUp := InitJasTest(t) - defer cleanUp() - - scanner.TempDir = "" + scanner := &JasScanner{TempDir: ""} _, err := CreateScannerTempDirectory(scanner, string(utils.Applicability)) assert.Error(t, err) } diff --git a/commands/audit/jas/iac/iacscanner.go b/commands/audit/jas/iac/iacscanner.go index 0f524b9d..8e4798e2 100644 --- a/commands/audit/jas/iac/iacscanner.go +++ b/commands/audit/jas/iac/iacscanner.go @@ -1,11 +1,10 @@ package iac import ( - "path/filepath" - "strconv" - jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-security/commands/audit/jas" + clientutils "github.com/jfrog/jfrog-client-go/utils" + "path/filepath" "github.com/jfrog/jfrog-cli-security/utils" "github.com/jfrog/jfrog-client-go/utils/log" @@ -39,18 +38,18 @@ func RunIacScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.Jas return } iacScanManager := newIacScanManager(scanner, scannerTempDir) - log.Info("[thread_id: " + strconv.Itoa(threadId) + "] Running IaC scanning...") + log.Info(clientutils.GetLogMsgPrefix(threadId, false) + "Running IaC scan...") if err = iacScanManager.scanner.Run(iacScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.IaC, err) return } if len(iacScanManager.iacScannerResults) > 0 { - log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Found", utils.GetResultsLocationCount(iacScanManager.iacScannerResults...), "IaC vulnerabilities") + log.Info(clientutils.GetLogMsgPrefix(threadId, false)+"Found", utils.GetResultsLocationCount(iacScanManager.iacScannerResults...), "IaC vulnerabilities") } results := iacScanManager.iacScannerResults - auditParallelRunner.Mu.Lock() + auditParallelRunner.ResultsMu.Lock() extendedScanResults.IacScanResults = append(extendedScanResults.IacScanResults, results...) - auditParallelRunner.Mu.Unlock() + auditParallelRunner.ResultsMu.Unlock() return } diff --git a/commands/audit/jas/sast/sastscanner.go b/commands/audit/jas/sast/sastscanner.go index 8d8f930d..82c78a8f 100644 --- a/commands/audit/jas/sast/sastscanner.go +++ b/commands/audit/jas/sast/sastscanner.go @@ -5,11 +5,11 @@ import ( jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" "github.com/jfrog/jfrog-cli-security/commands/audit/jas" "github.com/jfrog/jfrog-cli-security/utils" + clientutils "github.com/jfrog/jfrog-client-go/utils" "github.com/jfrog/jfrog-client-go/utils/log" "github.com/owenrumney/go-sarif/v2/sarif" "golang.org/x/exp/maps" "path/filepath" - "strconv" ) const ( @@ -31,18 +31,18 @@ func RunSastScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.Ja return } sastScanManager := newSastScanManager(scanner, scannerTempDir) - log.Info("[thread_id: " + strconv.Itoa(threadId) + "] Running SAST scanning...") + log.Info(clientutils.GetLogMsgPrefix(threadId, false) + "Running SAST scanning...") if err = sastScanManager.scanner.Run(sastScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.Sast, err) return } if len(sastScanManager.sastScannerResults) > 0 { - log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Found", utils.GetResultsLocationCount(sastScanManager.sastScannerResults...), "SAST vulnerabilities") + log.Info(clientutils.GetLogMsgPrefix(threadId, false)+"Found", utils.GetResultsLocationCount(sastScanManager.sastScannerResults...), "SAST vulnerabilities") } results := sastScanManager.sastScannerResults - auditParallelRunner.Mu.Lock() + auditParallelRunner.ResultsMu.Lock() extendedScanResults.SastScanResults = append(extendedScanResults.SastScanResults, results...) - auditParallelRunner.Mu.Unlock() + auditParallelRunner.ResultsMu.Unlock() return } diff --git a/commands/audit/jas/secrets/secretsscanner.go b/commands/audit/jas/secrets/secretsscanner.go index 1d2e71bd..b52c90e8 100644 --- a/commands/audit/jas/secrets/secretsscanner.go +++ b/commands/audit/jas/secrets/secretsscanner.go @@ -1,8 +1,8 @@ package secrets import ( + clientutils "github.com/jfrog/jfrog-client-go/utils" "path/filepath" - "strconv" "strings" jfrogappsconfig "github.com/jfrog/jfrog-apps-config/go" @@ -39,18 +39,18 @@ func RunSecretsScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas return } secretScanManager := newSecretsScanManager(scanner, scannerTempDir) - log.Info("[thread_id: " + strconv.Itoa(threadId) + "] Running secrets scanning...") + log.Info(clientutils.GetLogMsgPrefix(threadId, false) + "Running secrets scanning...") if err = secretScanManager.scanner.Run(secretScanManager, module); err != nil { err = utils.ParseAnalyzerManagerError(utils.Secrets, err) return } results := secretScanManager.secretsScannerResults if len(results) > 0 { - log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Found", utils.GetResultsLocationCount(results...), "secrets") + log.Info(clientutils.GetLogMsgPrefix(threadId, false)+"Found", utils.GetResultsLocationCount(results...), "secrets") } - auditParallelRunner.Mu.Lock() + auditParallelRunner.ResultsMu.Lock() extendedScanResults.SecretsScanResults = append(extendedScanResults.SecretsScanResults, results...) - auditParallelRunner.Mu.Unlock() + auditParallelRunner.ResultsMu.Unlock() return } diff --git a/commands/audit/jasrunner.go b/commands/audit/jasrunner.go index ad4a5f8a..1589644a 100644 --- a/commands/audit/jasrunner.go +++ b/commands/audit/jasrunner.go @@ -37,26 +37,14 @@ func RunJasScannersAndSetResults(auditParallelRunner *utils.AuditParallelRunner, // Don't execute other scanners when scanning third party dependencies. if !auditParams.thirdPartyApplicabilityScan { for _, module := range scanner.JFrogAppsConfig.Modules { - if !jas.ShouldSkipScanner(module, utils.Secrets) { - auditParallelRunner.ScannersWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(runSecretsScan(auditParallelRunner, scanner, scanResults, module), auditParallelRunner.AddErrorToChan) - if err != nil { - return fmt.Errorf("failed to create secrets scan task: %s", err.Error()) - } + if err = addModuleJasScanTask(module, utils.Secrets, auditParallelRunner, runSecretsScan(auditParallelRunner, scanner, scanResults, module)); err != nil { + return } - if !jas.ShouldSkipScanner(module, utils.IaC) { - auditParallelRunner.ScannersWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(runIacScan(auditParallelRunner, scanner, scanResults, module), auditParallelRunner.AddErrorToChan) - if err != nil { - return fmt.Errorf("failed to create iac scan task: %s", err.Error()) - } + if err = addModuleJasScanTask(module, utils.IaC, auditParallelRunner, runIacScan(auditParallelRunner, scanner, scanResults, module)); err != nil { + return } - if !jas.ShouldSkipScanner(module, utils.Sast) { - auditParallelRunner.ScannersWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(runSastScan(auditParallelRunner, scanner, scanResults, module), auditParallelRunner.AddErrorToChan) - if err != nil { - return fmt.Errorf("failed to create sast scan task: %s", err.Error()) - } + if err = addModuleJasScanTask(module, utils.Sast, auditParallelRunner, runSastScan(auditParallelRunner, scanner, scanResults, module)); err != nil { + return } } } @@ -64,17 +52,24 @@ func RunJasScannersAndSetResults(auditParallelRunner *utils.AuditParallelRunner, // Wait for sca scan to complete auditParallelRunner.ScaScansWg.Wait() for _, module := range scanner.JFrogAppsConfig.Modules { - if !jas.ShouldSkipScanner(module, utils.Applicability) { - auditParallelRunner.ScannersWg.Add(1) - _, err = auditParallelRunner.Runner.AddTaskWithError(runContextualScan(auditParallelRunner, scanner, scanResults, module, auditParams), auditParallelRunner.AddErrorToChan) - if err != nil { - return fmt.Errorf("failed to create contextual scan task: %s", err.Error()) - } + if err = addModuleJasScanTask(module, utils.Applicability, auditParallelRunner, runContextualScan(auditParallelRunner, scanner, scanResults, module, auditParams)); err != nil { + return } } return err } +func addModuleJasScanTask(module jfrogappsconfig.Module, scanType utils.JasScanType, auditParallelRunner *utils.AuditParallelRunner, task parallel.TaskFunc) (err error) { + if jas.ShouldSkipScanner(module, scanType) { + return + } + auditParallelRunner.ScannersWg.Add(1) + if _, err = auditParallelRunner.Runner.AddTaskWithError(task, auditParallelRunner.AddErrorToChan); err != nil { + err = fmt.Errorf("failed to create %s scan task: %s", scanType, err.Error()) + } + return +} + func runSecretsScan(auditParallelRunner *utils.AuditParallelRunner, scanner *jas.JasScanner, scanResults *utils.Results, module jfrogappsconfig.Module) parallel.TaskFunc { return func(threadId int) (err error) { diff --git a/commands/audit/scarunner.go b/commands/audit/scarunner.go index d185cb3f..86e7f7f6 100644 --- a/commands/audit/scarunner.go +++ b/commands/audit/scarunner.go @@ -6,7 +6,6 @@ import ( "fmt" "github.com/jfrog/build-info-go/utils/pythonutils" "github.com/jfrog/jfrog-client-go/utils/io/fileutils" - "strconv" "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/gofrog/parallel" @@ -68,11 +67,11 @@ func runScaScan(auditParallelRunner *utils.AuditParallelRunner, auditParams *Aud } // Create sca scan task auditParallelRunner.ScaScansWg.Add(1) - _, taskErr := auditParallelRunner.Runner.AddTaskWithError(executeScaScan(auditParallelRunner, serverDetails, auditParams, scan, *treeResult.FlatTree, treeResult.FullDepTrees), func(err error) { + _, taskErr := auditParallelRunner.Runner.AddTaskWithError(executeScaScan(auditParallelRunner, serverDetails, auditParams, scan, treeResult), func(err error) { auditParallelRunner.AddErrorToChan(fmt.Errorf("audit command in '%s' failed:\n%s", scan.WorkingDirectory, err.Error())) }) if taskErr != nil { - return fmt.Errorf("failed to creat sca scan task: %s", taskErr.Error()) + return fmt.Errorf("failed to create sca scan task for '%s': %s", scan.WorkingDirectory, taskErr.Error()) } // Add the scan to the results results.ScaResults = append(results.ScaResults, scan) @@ -119,32 +118,31 @@ func getRequestedDescriptors(params *AuditParams) map[coreutils.Technology][]str // Preform the SCA scan for the given scan information. func executeScaScan(auditParallelRunner *utils.AuditParallelRunner, serverDetails *config.ServerDetails, auditParams *AuditParams, - scan *xrayutils.ScaScanResult, flatTree xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode) parallel.TaskFunc { + scan *xrayutils.ScaScanResult, treeResult *DependencyTreeResult) parallel.TaskFunc { return func(threadId int) (err error) { - log.Info("[thread_id: "+strconv.Itoa(threadId)+"] Running SCA scan for", scan.Technology, "vulnerable dependencies in", scan.WorkingDirectory, "directory...") + log.Info(clientutils.GetLogMsgPrefix(threadId, false)+"Running SCA scan for", scan.Technology, "vulnerable dependencies in", scan.WorkingDirectory, "directory...") defer func() { auditParallelRunner.ScaScansWg.Done() }() // Scan the dependency tree. - xrayGraphScanParams := createXrayGraphScanParams(auditParams) - scanResults, xrayErr := runScaWithTech(scan.Technology, auditParams, xrayGraphScanParams, serverDetails, flatTree, fullDependencyTrees) + scanResults, xrayErr := runScaWithTech(scan.Technology, auditParams, serverDetails, *treeResult.FlatTree, treeResult.FullDepTrees) if xrayErr != nil { return fmt.Errorf("'%s' Xray dependency tree scan request failed:\n%s", scan.Technology, xrayErr.Error()) } - scan.IsMultipleRootProject = clientutils.Pointer(len(fullDependencyTrees) > 1) - addThirdPartyDependenciesToParams(auditParams, scan.Technology, &flatTree, fullDependencyTrees) - auditParallelRunner.Mu.Lock() + scan.IsMultipleRootProject = clientutils.Pointer(len(treeResult.FullDepTrees) > 1) + addThirdPartyDependenciesToParams(auditParams, scan.Technology, treeResult.FlatTree, treeResult.FullDepTrees) + auditParallelRunner.ResultsMu.Lock() scan.XrayResults = append(scan.XrayResults, scanResults...) - auditParallelRunner.Mu.Unlock() + auditParallelRunner.ResultsMu.Unlock() return } } -func runScaWithTech(tech coreutils.Technology, params *AuditParams, xrayGraphScanParams *services.XrayGraphScanParams, serverDetails *config.ServerDetails, +func runScaWithTech(tech coreutils.Technology, params *AuditParams, serverDetails *config.ServerDetails, flatTree xrayCmdUtils.GraphNode, fullDependencyTrees []*xrayCmdUtils.GraphNode) (techResults []services.ScanResponse, err error) { scanGraphParams := scangraph.NewScanGraphParams(). SetServerDetails(serverDetails). - SetXrayGraphScanParams(xrayGraphScanParams). + SetXrayGraphScanParams(params.createXrayGraphScanParams()). SetXrayVersion(params.xrayVersion). SetFixableOnly(params.fixableOnly). SetSeverityLevel(params.minSeverityFilter) @@ -406,16 +404,3 @@ func buildDependencyTree(scan *utils.ScaScanResult, params *AuditParams) (*Depen } return &treeResult, nil } - -func createXrayGraphScanParams(auditParams *AuditParams) *services.XrayGraphScanParams { - return &services.XrayGraphScanParams{ - RepoPath: auditParams.commonGraphScanParams.repoPath, - Watches: auditParams.commonGraphScanParams.watches, - ScanType: auditParams.commonGraphScanParams.scanType, - ProjectKey: auditParams.commonGraphScanParams.projectKey, - IncludeVulnerabilities: auditParams.commonGraphScanParams.includeVulnerabilities, - IncludeLicenses: auditParams.commonGraphScanParams.includeLicenses, - XscVersion: auditParams.commonGraphScanParams.xscVersion, - MultiScanId: auditParams.commonGraphScanParams.multiScanId, - } -} diff --git a/commands/curation/curationaudit.go b/commands/curation/curationaudit.go index 81ae6a34..044e9b1f 100644 --- a/commands/curation/curationaudit.go +++ b/commands/curation/curationaudit.go @@ -4,8 +4,8 @@ import ( "encoding/json" "errors" "fmt" + "github.com/jfrog/jfrog-cli-core/v2/common/cliutils" config "github.com/jfrog/jfrog-cli-core/v2/utils/config" - "github.com/jfrog/jfrog-cli-security/commands" "net/http" "os" "path/filepath" @@ -55,8 +55,6 @@ const ( MinArtiPassThroughSupport = "7.82.0" MinXrayPassTHroughSupport = "3.92.0" - MinArtiMavenSupport = "7.82.0" - MinArtiXraySupport = "3.92.0" ) var CurationOutputFormats = []string{string(outFormat.Table), string(outFormat.Json)} @@ -322,7 +320,7 @@ func (ca *CurationAuditCommand) auditTree(tech coreutils.Technology, results map projectName = projectScope + "/" + projectName } if ca.parallelRequests == 0 { - ca.parallelRequests = commands.TotalConcurrentRequests + ca.parallelRequests = cliutils.Threads } var packagesStatus []*PackageStatus analyzer := treeAnalyzer{ diff --git a/scangraph/params.go b/scangraph/params.go index 76e0106e..e6547cd9 100644 --- a/scangraph/params.go +++ b/scangraph/params.go @@ -13,6 +13,17 @@ type ScanGraphParams struct { severityLevel int } +type CommonGraphScanParams struct { + RepoPath string + ProjectKey string + Watches []string + ScanType services.ScanType + IncludeVulnerabilities bool + IncludeLicenses bool + XscVersion string + MultiScanId string +} + func NewScanGraphParams() *ScanGraphParams { return &ScanGraphParams{} } diff --git a/utils/analyzermanager.go b/utils/analyzermanager.go index d8f3a500..7ec32a41 100644 --- a/utils/analyzermanager.go +++ b/utils/analyzermanager.go @@ -3,11 +3,11 @@ package utils import ( "errors" "fmt" + clientutils "github.com/jfrog/jfrog-client-go/utils" "os" "os/exec" "path" "path/filepath" - "strconv" "strings" "github.com/jfrog/jfrog-cli-core/v2/utils/config" @@ -257,7 +257,7 @@ func DownloadAnalyzerManagerIfNeeded(threadId int) error { } } // Download & unzip the analyzer manager files - log.Debug("[thread_id: " + strconv.Itoa(threadId) + "] The 'Analyzer Manager' app is not cached locally. Downloading it now...") + log.Debug(clientutils.GetLogMsgPrefix(threadId, false) + "The 'Analyzer Manager' app is not cached locally. Downloading it now...") if err = dependencies.DownloadDependency(artDetails, remotePath, filepath.Join(analyzerManagerDir, AnalyzerManagerZipName), true); err != nil { return err } diff --git a/utils/parallel_runner.go b/utils/parallel_runner.go index 22151fe2..93aa0454 100644 --- a/utils/parallel_runner.go +++ b/utils/parallel_runner.go @@ -8,7 +8,7 @@ import ( type AuditParallelRunner struct { Runner parallel.Runner ErrorsQueue chan error - Mu sync.Mutex + ResultsMu sync.Mutex ScaScansWg sync.WaitGroup // verify that the sca scan routines are done before running contextual scan ScannersWg sync.WaitGroup // verify that all scanners routines are done before cleaning temp dir JasWg sync.WaitGroup // verify that downloading analyzer manager and running all scanners are done