Skip to content

Commit

Permalink
Add Violations support for JAS scanners (#241)
Browse files Browse the repository at this point in the history
  • Loading branch information
@eranturgeman
eranturgeman authored Dec 31, 2024
1 parent 218edb5 commit fd5e5f3
Show file tree
Hide file tree
Showing 76 changed files with 4,382 additions and 8,481 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ jobs:

# Test and generate code coverage
- name: Run tests
run: go test ${{ env.GO_COMMON_TEST_ARGS }} -cover -coverprofile=cover-unit-tests --test.unit
run: go test ${{ env.GO_COMMON_TEST_ARGS }} -covermode atomic -coverprofile=cover-unit-tests --test.unit

- name: Archive Code Coverage Results
uses: actions/upload-artifact@v4
Expand Down Expand Up @@ -246,7 +246,7 @@ jobs:
pull-requests: write # write permission needed to comment on PR
steps:
- name: Generate Unit Tests Code Coverage Report
uses: fgrosse/go-coverage-report@v1.2.0
uses: fgrosse/go-coverage-report@v1.1.0
with:
coverage-artifact-name: unit-tests-code-coverage
coverage-file-name: cover-unit-tests
13 changes: 7 additions & 6 deletions CONTRIBUTING.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,13 @@ go test -v github.com/jfrog/jfrog-cli-security [test-types] [flags]

### The available flags are:

| Flag | Equivalent Env vars | Description |
| ---------------------- | --------------------------------------------- | --------------------------------------------------------------------------------------------------------------- |
| `-jfrog.url` | `JFROG_SECURITY_CLI_TESTS_JFROG_URL` | [Default: http://localhost:8083] JFrog platform URL |
| `-jfrog.user` | `JFROG_SECURITY_CLI_TESTS_JFROG_USER` | [Default: admin] JFrog platform username |
| `-jfrog.password` | `JFROG_SECURITY_CLI_TESTS_JFROG_PASSWORD` | [Default: password] JFrog platform password |
| `-jfrog.adminToken` | `JFROG_SECURITY_CLI_TESTS_JFROG_ACCESS_TOKEN` | [Optional] JFrog platform admin token |
| Flag | Equivalent Env vars | Description |
| ---------------------- | ----------------------------------------------------- | --------------------------------------------------------------------------------------------------------------- |
| `-jfrog.url` | `JFROG_SECURITY_CLI_TESTS_JFROG_URL` | [Default: http://localhost:8083] JFrog platform URL |
| `-jfrog.user` | `JFROG_SECURITY_CLI_TESTS_JFROG_USER` | [Default: admin] JFrog platform username |
| `-jfrog.password` | `JFROG_SECURITY_CLI_TESTS_JFROG_PASSWORD` | [Default: password] JFrog platform password |
| `-jfrog.adminToken` | `JFROG_SECURITY_CLI_TESTS_JFROG_ACCESS_TOKEN` | [Optional] JFrog platform admin token |
| `-jfrog.projectKey` | `JFROG_SECURITY_CLI_TESTS_JFROG_PLATFORM_PROJECT_KEY` | [Optional] JFrog platform project key |
| `-ci.runId` | - | [Optional] A unique identifier used as a suffix to create repositories and builds in the tests. |
| `-jfrog.sshKeyPath` | - | [Optional] Path to the SSH key file. Use this flag only if the Artifactory URL format is `ssh://[domain]:port`. |
| `-jfrog.sshPassphrase` | - | [Optional] Passphrase for the SSH key. |
Expand Down
262 changes: 143 additions & 119 deletions audit_test.go
View file

Large diffs are not rendered by default.

9 changes: 6 additions & 3 deletions cli/scancommands.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,13 +211,15 @@ func ScanCmd(c *components.Context) error {
return err
}
var specFile *spec.SpecFiles
repoPath := ""
if c.IsFlagSet(flags.SpecFlag) && len(c.GetStringFlagValue(flags.SpecFlag)) > 0 {
specFile, err = pluginsCommon.GetFileSystemSpec(c)
if err != nil {
return err
}
} else {
specFile = createDefaultScanSpec(c, addTrailingSlashToRepoPathIfNeeded(c))
repoPath = addTrailingSlashToRepoPathIfNeeded(c)
specFile = createDefaultScanSpec(c, repoPath)
}
err = spec.ValidateSpec(specFile.Files, false, false)
if err != nil {
Expand All @@ -244,6 +246,7 @@ func ScanCmd(c *components.Context) error {
SetSpec(specFile).
SetOutputFormat(format).
SetProject(getProject(c)).
SetBaseRepoPath(repoPath).
SetIncludeVulnerabilities(c.GetBoolFlagValue(flags.Vuln) || shouldIncludeVulnerabilities(c)).
SetIncludeLicenses(c.GetBoolFlagValue(flags.Licenses)).
SetFail(c.GetBoolFlagValue(flags.Fail)).
Expand Down Expand Up @@ -472,7 +475,7 @@ func CreateAuditCmd(c *components.Context) (string, string, *coreConfig.ServerDe

auditCmd.SetTargetRepoPath(addTrailingSlashToRepoPathIfNeeded(c)).
SetProject(getProject(c)).
SetIncludeVulnerabilities(c.GetBoolFlagValue(flags.Vuln) || shouldIncludeVulnerabilities(c)).
SetIncludeVulnerabilities(c.GetBoolFlagValue(flags.Vuln)).
SetIncludeLicenses(c.GetBoolFlagValue(flags.Licenses)).
SetFail(c.GetBoolFlagValue(flags.Fail)).
SetPrintExtendedTable(c.GetBoolFlagValue(flags.ExtendedTable)).
Expand Down Expand Up @@ -728,12 +731,12 @@ func DockerScan(c *components.Context, image string) error {
return err
}
containerScanCommand.SetImageTag(image).
SetTargetRepoPath(addTrailingSlashToRepoPathIfNeeded(c)).
SetServerDetails(serverDetails).
SetXrayVersion(xrayVersion).
SetXscVersion(xscVersion).
SetOutputFormat(format).
SetProject(getProject(c)).
SetBaseRepoPath(addTrailingSlashToRepoPathIfNeeded(c)).
SetIncludeVulnerabilities(c.GetBoolFlagValue(flags.Vuln) || shouldIncludeVulnerabilities(c)).
SetIncludeLicenses(c.GetBoolFlagValue(flags.Licenses)).
SetFail(c.GetBoolFlagValue(flags.Fail)).
Expand Down
110 changes: 83 additions & 27 deletions commands/audit/audit.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,12 @@ import (
"github.com/jfrog/jfrog-client-go/xray"
"github.com/jfrog/jfrog-client-go/xray/services"
xscservices "github.com/jfrog/jfrog-client-go/xsc/services"
xscutils "github.com/jfrog/jfrog-client-go/xsc/services/utils"
)

type AuditCommand struct {
watches []string
gitRepoHttpsCloneUrl string
projectKey string
targetRepoPath string
IncludeVulnerabilities bool
Expand All @@ -53,6 +55,11 @@ func (auditCmd *AuditCommand) SetWatches(watches []string) *AuditCommand {
return auditCmd
}

func (auditCmd *AuditCommand) SetGitRepoHttpsCloneUrl(gitRepoHttpsCloneUrl string) *AuditCommand {
auditCmd.gitRepoHttpsCloneUrl = gitRepoHttpsCloneUrl
return auditCmd
}

func (auditCmd *AuditCommand) SetProject(project string) *AuditCommand {
auditCmd.projectKey = project
return auditCmd
Expand Down Expand Up @@ -88,16 +95,49 @@ func (auditCmd *AuditCommand) SetThreads(threads int) *AuditCommand {
return auditCmd
}

func (auditCmd *AuditCommand) CreateCommonGraphScanParams() *scangraph.CommonGraphScanParams {
commonParams := &scangraph.CommonGraphScanParams{
RepoPath: auditCmd.targetRepoPath,
Watches: auditCmd.watches,
ScanType: services.Dependency,
// Create a results context based on the provided parameters. resolves conflicts between the parameters based on the retrieved platform watches.
func CreateAuditResultsContext(serverDetails *config.ServerDetails, xrayVersion string, watches []string, artifactoryRepoPath, projectKey, gitRepoHttpsCloneUrl string, includeVulnerabilities, includeLicenses bool) (context results.ResultContext) {
context = results.ResultContext{
RepoPath: artifactoryRepoPath,
Watches: watches,
ProjectKey: projectKey,
IncludeVulnerabilities: shouldIncludeVulnerabilities(includeVulnerabilities, watches, artifactoryRepoPath, projectKey, ""),
IncludeLicenses: includeLicenses,
}
if err := clientutils.ValidateMinimumVersion(clientutils.Xray, xrayVersion, services.MinXrayVersionGitRepoKey); err != nil {
// Git repo key is not supported by the Xray version.
return
}
if gitRepoHttpsCloneUrl == "" {
// No git repo key was provided, no need to check anything else.
log.Debug("Git repo key was not provided, jas violations will not be checked for this resource.")
return
}
// Get the defined and active watches from the platform.
manager, err := xsc.CreateXscService(serverDetails)
if err != nil {
log.Warn(fmt.Sprintf("Failed to create Xray services manager: %s", err.Error()))
return
}
if context.PlatformWatches, err = manager.GetResourceWatches(xscutils.GetGitRepoUrlKey(gitRepoHttpsCloneUrl), projectKey); err != nil {
log.Warn(fmt.Sprintf("Failed to get active defined watches: %s", err.Error()))
return
}
commonParams.ProjectKey = auditCmd.projectKey
commonParams.IncludeVulnerabilities = auditCmd.IncludeVulnerabilities
commonParams.IncludeLicenses = auditCmd.IncludeLicenses
return commonParams
// Set git repo key and check if it has any watches defined in the platform.
context.GitRepoHttpsCloneUrl = gitRepoHttpsCloneUrl
if len(context.PlatformWatches.GitRepositoryWatches) == 0 && len(watches) == 0 && projectKey == "" {
log.Debug(fmt.Sprintf("No watches were found in the platform for the given git repo key (%s), and no watches were given by the user (using watches or project flags). Calculating vulnerabilities...", context.GitRepoHttpsCloneUrl))
context.GitRepoHttpsCloneUrl = ""
}
// We calculate again this time also taking into account the final git repo key value.
// (if there are no watches defined on the git repo and no other context was given, we should include vulnerabilities)
context.IncludeVulnerabilities = shouldIncludeVulnerabilities(includeVulnerabilities, watches, artifactoryRepoPath, projectKey, context.GitRepoHttpsCloneUrl)
return
}

// If the user requested to include vulnerabilities, or if the user didn't provide any watches, project key, artifactory repo path or git repo key, we should include vulnerabilities.
func shouldIncludeVulnerabilities(includeVulnerabilities bool, watches []string, artifactoryRepoPath, projectKey, gitRepoHttpsCloneUrl string) bool {
return includeVulnerabilities || !(len(watches) > 0 || projectKey != "" || artifactoryRepoPath != "" || gitRepoHttpsCloneUrl != "")
}

func (auditCmd *AuditCommand) Run() (err error) {
Expand All @@ -124,7 +164,16 @@ func (auditCmd *AuditCommand) Run() (err error) {
SetMinSeverityFilter(auditCmd.minSeverityFilter).
SetFixableOnly(auditCmd.fixableOnly).
SetGraphBasicParams(auditCmd.AuditBasicParams).
SetCommonGraphScanParams(auditCmd.CreateCommonGraphScanParams()).
SetResultsContext(CreateAuditResultsContext(
serverDetails,
auditCmd.GetXrayVersion(),
auditCmd.watches,
auditCmd.targetRepoPath,
auditCmd.projectKey,
auditCmd.gitRepoHttpsCloneUrl,
auditCmd.IncludeVulnerabilities,
auditCmd.IncludeLicenses,
)).
SetThirdPartyApplicabilityScan(auditCmd.thirdPartyApplicabilityScan).
SetThreads(auditCmd.Threads).
SetScansResultsOutputDir(auditCmd.scanResultsOutputDir).SetStartTime(startTime).SetMultiScanId(multiScanId)
Expand All @@ -144,13 +193,10 @@ func (auditCmd *AuditCommand) Run() (err error) {
messages = []string{coreutils.PrintTitle("The ‘jf audit’ command also supports JFrog Advanced Security features, such as 'Contextual Analysis', 'Secret Detection', 'IaC Scan' and ‘SAST’.\nThis feature isn't enabled on your system. Read more - ") + coreutils.PrintLink(utils.JasInfoURL)}
}
if err = output.NewResultsWriter(auditResults).
SetHasViolationContext(auditCmd.HasViolationContext()).
SetIncludeVulnerabilities(auditCmd.IncludeVulnerabilities).
SetIncludeLicenses(auditCmd.IncludeLicenses).
SetOutputFormat(auditCmd.OutputFormat()).
SetPrintExtendedTable(auditCmd.PrintExtendedTable).
SetExtraMessages(messages).
SetSubScansPreformed(auditCmd.ScansToPerform()).
SetSubScansPerformed(auditCmd.ScansToPerform()).
PrintScanResults(); err != nil {
return errors.Join(err, auditResults.GetErrors())
}
Expand All @@ -170,10 +216,6 @@ func (auditCmd *AuditCommand) CommandName() string {
return "generic_audit"
}

func (auditCmd *AuditCommand) HasViolationContext() bool {
return len(auditCmd.watches) > 0 || auditCmd.projectKey != "" || auditCmd.targetRepoPath != ""
}

// Runs an audit scan based on the provided auditParams.
// Returns an audit Results object containing all the scan results.
// If the current server is entitled for JAS, the advanced security results will be included in the scan results.
Expand All @@ -192,14 +234,14 @@ func RunAudit(auditParams *AuditParams) (cmdResults *results.SecurityCommandResu
var jasScanner *jas.JasScanner
var generalJasScanErr error
if jasScanner, generalJasScanErr = RunJasScans(auditParallelRunner, auditParams, cmdResults, jfrogAppsConfig); generalJasScanErr != nil {
cmdResults.AddGeneralError(fmt.Errorf("An error has occurred during JAS scan process. JAS scan is skipped for the following directories: %s\n%s", strings.Join(cmdResults.GetTargetsPaths(), ","), generalJasScanErr.Error()), auditParams.AllowPartialResults())
cmdResults.AddGeneralError(fmt.Errorf("error has occurred during JAS scan process. JAS scan is skipped for the following directories: %s\n%s", strings.Join(cmdResults.GetTargetsPaths(), ","), generalJasScanErr.Error()), auditParams.AllowPartialResults())
}
if auditParams.Progress() != nil {
auditParams.Progress().SetHeadlineMsg("Scanning for issues")
}
// The sca scan doesn't require the analyzer manager, so it can run separately from the analyzer manager download routine.
if generalScaScanError := buildDepTreeAndRunScaScan(auditParallelRunner, auditParams, cmdResults); generalScaScanError != nil {
cmdResults.AddGeneralError(fmt.Errorf("An error has occurred during SCA scan process. SCA scan is skipped for the following directories: %s\n%s", strings.Join(cmdResults.GetTargetsPaths(), ","), generalScaScanError.Error()), auditParams.AllowPartialResults())
cmdResults.AddGeneralError(fmt.Errorf("error has occurred during SCA scan process. SCA scan is skipped for the following directories: %s\n%s", strings.Join(cmdResults.GetTargetsPaths(), ","), generalScaScanError.Error()), auditParams.AllowPartialResults())
}
go func() {
auditParallelRunner.ScaScansWg.Wait()
Expand Down Expand Up @@ -234,7 +276,19 @@ func RunJasScans(auditParallelRunner *utils.SecurityParallelRunner, auditParams
return
}
auditParallelRunner.ResultsMu.Lock()
jasScanner, err = jas.CreateJasScanner(serverDetails, scanResults.SecretValidation, auditParams.minSeverityFilter, jas.GetAnalyzerManagerXscEnvVars(auditParams.GetMultiScanId(), scanResults.GetTechnologies()...), auditParams.Exclusions()...)
jasScanner, err = jas.CreateJasScanner(
serverDetails,
scanResults.SecretValidation,
auditParams.minSeverityFilter,
jas.GetAnalyzerManagerXscEnvVars(
auditParams.GetMultiScanId(),
jas.GetGitRepoUrlKey(auditParams.resultsContext.GitRepoHttpsCloneUrl),
auditParams.resultsContext.ProjectKey,
auditParams.resultsContext.Watches,
scanResults.GetTechnologies()...,
),
auditParams.Exclusions()...,
)
auditParallelRunner.ResultsMu.Unlock()
if err != nil {
generalError = fmt.Errorf("failed to create jas scanner: %s", err.Error())
Expand Down Expand Up @@ -276,7 +330,7 @@ func createJasScansTasks(auditParallelRunner *utils.SecurityParallelRunner, scan
Scanner: scanner,
Module: *module,
ConfigProfile: auditParams.configProfile,
ScansToPreform: auditParams.ScansToPerform(),
ScansToPerform: auditParams.ScansToPerform(),
SecretsScanType: secrets.SecretsScannerType,
DirectDependencies: auditParams.DirectDependencies(),
ThirdPartyApplicabilityScan: auditParams.thirdPartyApplicabilityScan,
Expand Down Expand Up @@ -310,11 +364,13 @@ func initAuditCmdResults(params *AuditParams) (cmdResults *results.SecurityComma
cmdResults.SetXscVersion(params.GetXscVersion())
cmdResults.SetMultiScanId(params.GetMultiScanId())
cmdResults.SetStartTime(params.StartTime())
// Send entitlement requests
cmdResults.SetResultsContext(params.resultsContext)

xrayManager, err := xrayutils.CreateXrayServiceManager(serverDetails)
if err != nil {
return cmdResults.AddGeneralError(err, false)
}
// Send entitlement requests
entitledForJas, err := isEntitledForJas(xrayManager, params)
if err != nil {
return cmdResults.AddGeneralError(err, false)
Expand All @@ -330,11 +386,11 @@ func initAuditCmdResults(params *AuditParams) (cmdResults *results.SecurityComma
// No SCA targets were detected, add the root directory as a target for JAS scans.
cmdResults.NewScanResults(results.ScanTarget{Target: params.workingDirs[0]})
}
scanInfo, err := coreutils.GetJsonIndent(cmdResults)
scanInfo, err := coreutils.GetJsonIndent(cmdResults.GetTargets())
if err != nil {
return
}
log.Info(fmt.Sprintf("Preforming scans on %d targets:\n%s", len(cmdResults.Targets), scanInfo))
log.Info(fmt.Sprintf("Performing scans on %d targets:\n%s", len(cmdResults.Targets), scanInfo))
return
}

Expand All @@ -350,14 +406,14 @@ func detectScanTargets(cmdResults *results.SecurityCommandResults, params *Audit
log.Warn("Couldn't detect technologies in", requestedDirectory, "directory.", err.Error())
continue
}
// Create scans to preform
// Create scans to perform
for tech, workingDirs := range techToWorkingDirs {
if tech == techutils.Dotnet {
// We detect Dotnet and Nuget the same way, if one detected so does the other.
// We don't need to scan for both and get duplicate results.
continue
}
// No technology was detected, add scan without descriptors. (so no sca scan will be preformed and set at target level)
// No technology was detected, add scan without descriptors. (so no sca scan will be performed and set at target level)
if len(workingDirs) == 0 {
// Requested technology (from params) descriptors/indicators were not found or recursive scan with NoTech value, add scan without descriptors.
cmdResults.NewScanResults(results.ScanTarget{Target: requestedDirectory, Technology: tech})
Expand Down
Loading

0 comments on commit fd5e5f3

Please sign in to comment.