improve test to check sub attribs in validations

commands/audit/audit_test.go

@@ -439,24 +439,21 @@ func TestAuditWithConfigProfile(t *testing.T) {
 			summary, err := conversion.NewCommandResultsConvertor(conversion.ResultConvertParams{IncludeVulnerabilities: true}).ConvertToSummary(auditResults)
 			assert.NoError(t, err)
 
-			var ScaResultsCount int
+			var scaResultsCount int
 			// When checking Applicability results with ExactResultsMatch = true, the sum of all statuses should equal total Sca results amount. Else, we check the provided Sca issues amount
 			if testcase.expectedCaApplicable > 0 || testcase.expectedCaNotApplicable > 0 || testcase.expectedCaNotCovered > 0 || testcase.expectedCaUndetermined > 0 {
-				ScaResultsCount = testcase.expectedCaApplicable + testcase.expectedCaNotApplicable + testcase.expectedCaNotCovered + testcase.expectedCaUndetermined
+				scaResultsCount = testcase.expectedCaApplicable + testcase.expectedCaNotApplicable + testcase.expectedCaNotCovered + testcase.expectedCaUndetermined
 			} else {
-				ScaResultsCount = testcase.expectedScaIssues
+				scaResultsCount = testcase.expectedScaIssues
 			}
 			validations.ValidateCommandSummaryOutput(t, validations.ValidationParams{
-				Actual:                       summary,
-				ExactResultsMatch:            true,
-				Vulnerabilities:              testcase.expectedSastIssues + testcase.expectedSecretsIssues + testcase.expectedIacIssues + ScaResultsCount,
-				SastVulnerabilities:          testcase.expectedSastIssues,
-				SecretsVulnerabilities:       testcase.expectedSecretsIssues,
-				IacVulnerabilities:           testcase.expectedIacIssues,
-				ApplicableVulnerabilities:    testcase.expectedCaApplicable,
-				NotApplicableVulnerabilities: testcase.expectedCaNotApplicable,
-				NotCoveredVulnerabilities:    testcase.expectedCaNotCovered,
-				UndeterminedVulnerabilities:  testcase.expectedCaUndetermined,
+				Actual:            summary,
+				ExactResultsMatch: true,
+				Total:             &validations.TotalCount{Vulnerabilities: testcase.expectedSastIssues + testcase.expectedSecretsIssues + testcase.expectedIacIssues + scaResultsCount},
+				Vulnerabilities: &validations.VulnerabilityCount{
+					ValidateScan:                &validations.ScanCount{Sca: scaResultsCount, Sast: testcase.expectedSastIssues, Secrets: testcase.expectedSecretsIssues, Iac: testcase.expectedIacIssues},
+					ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Applicable: testcase.expectedCaApplicable, NotApplicable: testcase.expectedCaNotApplicable, NotCovered: testcase.expectedCaNotCovered, Undetermined: testcase.expectedCaUndetermined},
+				},
 			})
 		})
 	}

scans_test.go

@@ -45,18 +45,15 @@ func TestXrayBinaryScanJson(t *testing.T) {
 	integration.InitScanTest(t, scangraph.GraphScanMinXrayVersion)
 	output := testXrayBinaryScan(t, string(format.Json), false)
 	validations.VerifyJsonResults(t, output, validations.ValidationParams{
-		Vulnerabilities: 1,
-		Licenses:        1,
+		Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1},
 	})
 }
 
 func TestXrayBinaryScanSimpleJson(t *testing.T) {
 	integration.InitScanTest(t, scangraph.GraphScanMinXrayVersion)
 	output := testXrayBinaryScan(t, string(format.SimpleJson), true)
 	validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
-		Vulnerabilities:       1,
-		ScaSecurityViolations: 1,
-		Licenses:              1,
+		Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1, Violations: 1},
 	})
 }
 
@@ -66,8 +63,7 @@ func TestXrayBinaryScanJsonWithProgress(t *testing.T) {
 	defer callback()
 	output := testXrayBinaryScan(t, string(format.Json), false)
 	validations.VerifyJsonResults(t, output, validations.ValidationParams{
-		Vulnerabilities: 1,
-		Licenses:        1,
+		Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1},
 	})
 }
 
@@ -77,9 +73,7 @@ func TestXrayBinaryScanSimpleJsonWithProgress(t *testing.T) {
 	defer callback()
 	output := testXrayBinaryScan(t, string(format.SimpleJson), true)
 	validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
-		Vulnerabilities:       1,
-		ScaSecurityViolations: 1,
-		Licenses:              1,
+		Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1, Violations: 1},
 	})
 }
 
@@ -110,8 +104,7 @@ func TestXrayBinaryScanWithBypassArchiveLimits(t *testing.T) {
 	scanArgs = append(scanArgs, "--bypass-archive-limits")
 	output := securityTests.PlatformCli.RunCliCmdWithOutput(t, scanArgs...)
 	validations.VerifyJsonResults(t, output, validations.ValidationParams{
-		Vulnerabilities: 1,
-		Licenses:        1,
+		Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1},
 	})
 }
 
@@ -172,9 +165,11 @@ func runDockerScan(t *testing.T, testCli *coreTests.JfrogCli, imageName, watchNa
 		output := testCli.WithoutCredentials().RunCliCmdWithOutput(t, cmdArgs...)
 		if assert.NotEmpty(t, output) {
 			if validateSecrets {
-				validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{InactiveVulnerabilities: minInactives})
+				validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
+					Vulnerabilities: &validations.VulnerabilityCount{ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Inactive: minInactives}},
+				})
 			} else {
-				validations.VerifyJsonResults(t, output, validations.ValidationParams{Vulnerabilities: minVulnerabilities, Licenses: minLicenses})
+				validations.VerifyJsonResults(t, output, validations.ValidationParams{Total: &validations.TotalCount{Vulnerabilities: minVulnerabilities, Licenses: minLicenses}})
 			}
 		}
 		// Run docker scan on image with watch
@@ -184,7 +179,7 @@ func runDockerScan(t *testing.T, testCli *coreTests.JfrogCli, imageName, watchNa
 		cmdArgs = append(cmdArgs, "--watches="+watchName)
 		output = testCli.WithoutCredentials().RunCliCmdWithOutput(t, cmdArgs...)
 		if assert.NotEmpty(t, output) {
-			validations.VerifyJsonResults(t, output, validations.ValidationParams{ScaSecurityViolations: minViolations})
+			validations.VerifyJsonResults(t, output, validations.ValidationParams{Total: &validations.TotalCount{Violations: minViolations}})
 		}
 	}
 }

utils/results/conversion/convertor_test.go

@@ -32,46 +32,43 @@ func getAuditValidationParams() validations.ValidationParams {
 	return validations.ValidationParams{
 		ExactResultsMatch: true,
 
-		Vulnerabilities:              19,
-		ApplicableVulnerabilities:    1,
-		NotApplicableVulnerabilities: 7,
-		NotCoveredVulnerabilities:    4,
-		SastVulnerabilities:          4,
-		SecretsVulnerabilities:       3,
-
-		Violations:              7,
-		ScaSecurityViolations:   5,
-		ApplicableViolations:    1,
-		NotApplicableViolations: 4,
-		SastViolations:          1,
-		SecretsViolations:       1,
+		Total: &validations.TotalCount{Vulnerabilities: 19, Violations: 7},
+
+		Vulnerabilities: &validations.VulnerabilityCount{
+			ValidateScan:                &validations.ScanCount{Sca: 12, Sast: 4, Secrets: 3},
+			ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Applicable: 1, NotApplicable: 7, NotCovered: 4},
+		},
+
+		Violations: &validations.ViolationCount{
+			ValidateScan:                &validations.ScanCount{Sca: 5, Sast: 1, Secrets: 1},
+			ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Applicable: 1, NotApplicable: 4},
+		},
 	}
 }
 
 // For Summary we count unique CVE finding (issueId), for SARIF and SimpleJson we count all findings (pair of issueId+impactedComponent)
 // We have in the result 2 CVE with 2 impacted components each
 func getDockerScanValidationParams(unique bool) validations.ValidationParams {
 	params := validations.ValidationParams{
-		ExactResultsMatch:      true,
-		SecretsVulnerabilities: 3,
-
-		Violations:             3,
-		ScaSecurityViolations:  1,
-		UndeterminedViolations: 1,
-		SecretsViolations:      2,
+		ExactResultsMatch: true,
+		Total:             &validations.TotalCount{Violations: 3},
+		Violations: &validations.ViolationCount{
+			ValidateScan:                &validations.ScanCount{Sca: 1, Secrets: 2},
+			ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Undetermined: 1},
+		},
 	}
 	if unique {
-		params.Vulnerabilities = 11
-		params.ApplicableVulnerabilities = 3
-		params.NotApplicableVulnerabilities = 3
-		params.NotCoveredVulnerabilities = 1
-		params.UndeterminedVulnerabilities = 1
+		params.Total.Vulnerabilities = 11
+		params.Vulnerabilities = &validations.VulnerabilityCount{
+			ValidateScan:                &validations.ScanCount{Sca: 8, Secrets: 3},
+			ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Applicable: 3, NotApplicable: 3, NotCovered: 1, Undetermined: 1},
+		}
 	} else {
-		params.Vulnerabilities = 14
-		params.ApplicableVulnerabilities = 5
-		params.NotApplicableVulnerabilities = 4
-		params.NotCoveredVulnerabilities = 1
-		params.UndeterminedVulnerabilities = 1
+		params.Total.Vulnerabilities = 14
+		params.Vulnerabilities = &validations.VulnerabilityCount{
+			ValidateScan:                &validations.ScanCount{Sca: 11, Secrets: 3},
+			ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Applicable: 5, NotApplicable: 4, NotCovered: 1, Undetermined: 1},
+		}
 	}
 	return params
 }

utils/validations/test_mocks.go

@@ -12,6 +12,7 @@ import (
 	"github.com/jfrog/jfrog-cli-core/v2/utils/config"
 	"github.com/jfrog/jfrog-cli-security/utils/results"
 	"github.com/jfrog/jfrog-client-go/artifactory"
+	"github.com/jfrog/jfrog-client-go/xray/services"
 	xscutils "github.com/jfrog/jfrog-client-go/xsc/services/utils"
 	"github.com/owenrumney/go-sarif/v2/sarif"
 	"github.com/stretchr/testify/assert"
@@ -166,3 +167,14 @@ func XrayServer(t *testing.T, params MockServerParams) (*httptest.Server, *confi
 func NewMockJasRuns(runs ...*sarif.Run) []results.ScanResult[[]*sarif.Run] {
 	return []results.ScanResult[[]*sarif.Run]{{Scan: runs}}
 }
+
+func NewMockScaResults(responses ...services.ScanResponse) (converted []results.ScanResult[services.ScanResponse]) {
+	for _, response := range responses {
+		status := 0
+		if response.ScannedStatus == "Failed" {
+			status = 1
+		}
+		converted = append(converted, results.ScanResult[services.ScanResponse]{Scan: response, StatusCode: status})
+	}
+	return
+}

utils/validations/test_validate_sarif.go

@@ -43,57 +43,29 @@ func ValidateCommandSarifOutput(t *testing.T, params ValidationParams) {
 // If Expected is provided, the validation will check if the Actual content matches the expected results.
 // If ExactResultsMatch is true, the validation will check exact values and not only the 'equal or grater' counts / existence of expected attributes. (For Integration tests with JFrog API, ExactResultsMatch should be set to false)
 func ValidateSarifIssuesCount(t *testing.T, params ValidationParams, report *sarif.Report) {
-	var vulnerabilities, violations int
+	actualValues := validationCountActualValues{}
 
 	// SCA
-	scaVulnerabilities, applicableVulnerabilitiesResults, undeterminedVulnerabilitiesResults, notCoveredVulnerabilitiesResults, notApplicableVulnerabilitiesResults, missingContextVulnerabilitiesResults, scaViolations, securityViolations, licenseViolations, applicableViolationsResults, undeterminedViolationsResults, notCoveredViolationsResults, notApplicableViolationsResults, missingContextViolationsResults := countScaResults(report)
-	vulnerabilities += scaVulnerabilities
-	violations += scaViolations
+	actualValues.ScaVulnerabilities, actualValues.ApplicableVulnerabilities, actualValues.UndeterminedVulnerabilities, actualValues.NotCoveredVulnerabilities, actualValues.NotApplicableVulnerabilities, actualValues.MissingContextVulnerabilities, actualValues.ScaViolations, actualValues.SecurityViolations, actualValues.LicenseViolations, actualValues.ApplicableViolations, actualValues.UndeterminedViolations, actualValues.NotCoveredViolations, actualValues.NotApplicableViolations, actualValues.MissingContextViolations = countScaResults(report)
+	actualValues.Vulnerabilities += actualValues.ScaVulnerabilities
+	actualValues.Violations += actualValues.ScaViolations
 
 	// Secrets
-	secretsVulnerabilities, inactiveVulnerabilities, secretsViolations, inactiveViolations := countSecretsResults(report)
-	vulnerabilities += secretsVulnerabilities
-	violations += secretsViolations
+	actualValues.SecretsVulnerabilities, actualValues.InactiveSecretsVulnerabilities, actualValues.SecretsViolations, actualValues.InactiveSecretsViolations = countSecretsResults(report)
+	actualValues.Vulnerabilities += actualValues.SecretsVulnerabilities
+	actualValues.Violations += actualValues.SecretsViolations
 
 	// IAC
-	iacVulnerabilities, iacViolations := countJasResults(sarifutils.GetRunsByToolName(report, IacToolName))
-	vulnerabilities += iacVulnerabilities
-	violations += iacViolations
+	actualValues.IacVulnerabilities, actualValues.IacViolations = countJasResults(sarifutils.GetRunsByToolName(report, IacToolName))
+	actualValues.Vulnerabilities += actualValues.IacVulnerabilities
+	actualValues.Violations += actualValues.IacViolations
 
 	// SAST
-	sastVulnerabilities, sastViolations := countJasResults(sarifutils.GetRunsByToolName(report, SastToolName))
-	vulnerabilities += sastVulnerabilities
-	violations += sastViolations
+	actualValues.SastVulnerabilities, actualValues.SastViolations = countJasResults(sarifutils.GetRunsByToolName(report, SastToolName))
+	actualValues.Vulnerabilities += actualValues.SastVulnerabilities
+	actualValues.Violations += actualValues.SastViolations
 
-	ValidateContent(t, params.ExactResultsMatch,
-		// Total
-		CountValidation[int]{Expected: params.Vulnerabilities, Actual: vulnerabilities, Msg: GetValidationCountErrMsg("vulnerabilities", "sarif report", params.ExactResultsMatch, params.Vulnerabilities, vulnerabilities)},
-		CountValidation[int]{Expected: params.ScaSecurityViolations, Actual: violations, Msg: GetValidationCountErrMsg("violations", "sarif report", params.ExactResultsMatch, params.ScaSecurityViolations, violations)},
-		// JAS Vulnerabilities
-		CountValidation[int]{Expected: params.SastVulnerabilities, Actual: sastVulnerabilities, Msg: GetValidationCountErrMsg("sast vulnerabilities", "sarif report", params.ExactResultsMatch, params.SastVulnerabilities, sastVulnerabilities)},
-		CountValidation[int]{Expected: params.IacVulnerabilities, Actual: iacVulnerabilities, Msg: GetValidationCountErrMsg("Iac vulnerabilities", "sarif report", params.ExactResultsMatch, params.IacVulnerabilities, iacVulnerabilities)},
-		CountValidation[int]{Expected: params.SecretsVulnerabilities, Actual: secretsVulnerabilities, Msg: GetValidationCountErrMsg("secrets vulnerabilities", "sarif report", params.ExactResultsMatch, params.SecretsVulnerabilities, secretsVulnerabilities)},
-		CountValidation[int]{Expected: params.InactiveVulnerabilities, Actual: inactiveVulnerabilities, Msg: GetValidationCountErrMsg("inactive secrets vulnerabilities", "sarif report", params.ExactResultsMatch, params.InactiveVulnerabilities, inactiveVulnerabilities)},
-		// JAS Violations
-		CountValidation[int]{Expected: params.SastViolations, Actual: sastViolations, Msg: GetValidationCountErrMsg("sast violations", "sarif report", params.ExactResultsMatch, params.SastViolations, sastViolations)},
-		CountValidation[int]{Expected: params.IacViolations, Actual: iacViolations, Msg: GetValidationCountErrMsg("Iac violations", "sarif report", params.ExactResultsMatch, params.IacViolations, iacViolations)},
-		CountValidation[int]{Expected: params.SecretsViolations, Actual: secretsViolations, Msg: GetValidationCountErrMsg("secrets violations", "sarif report", params.ExactResultsMatch, params.SecretsViolations, secretsViolations)},
-		CountValidation[int]{Expected: params.InactiveViolations, Actual: inactiveViolations, Msg: GetValidationCountErrMsg("inactive secrets violations", "sarif report", params.ExactResultsMatch, params.InactiveViolations, inactiveViolations)},
-		// SCA Vulnerabilities
-		CountValidation[int]{Expected: params.ApplicableVulnerabilities, Actual: applicableVulnerabilitiesResults, Msg: GetValidationCountErrMsg("applicable vulnerabilities", "sarif report", params.ExactResultsMatch, params.ApplicableVulnerabilities, applicableVulnerabilitiesResults)},
-		CountValidation[int]{Expected: params.UndeterminedVulnerabilities, Actual: undeterminedVulnerabilitiesResults, Msg: GetValidationCountErrMsg("undetermined vulnerabilities", "sarif report", params.ExactResultsMatch, params.UndeterminedVulnerabilities, undeterminedVulnerabilitiesResults)},
-		CountValidation[int]{Expected: params.NotCoveredVulnerabilities, Actual: notCoveredVulnerabilitiesResults, Msg: GetValidationCountErrMsg("not covered vulnerabilities", "sarif report", params.ExactResultsMatch, params.NotCoveredVulnerabilities, notCoveredVulnerabilitiesResults)},
-		CountValidation[int]{Expected: params.NotApplicableVulnerabilities, Actual: notApplicableVulnerabilitiesResults, Msg: GetValidationCountErrMsg("not applicable vulnerabilities", "sarif report", params.ExactResultsMatch, params.NotApplicableVulnerabilities, notApplicableVulnerabilitiesResults)},
-		CountValidation[int]{Expected: params.MissingContextVulnerabilities, Actual: missingContextVulnerabilitiesResults, Msg: GetValidationCountErrMsg("missing context vulnerabilities", "sarif report", params.ExactResultsMatch, params.MissingContextVulnerabilities, missingContextVulnerabilitiesResults)},
-		// SCA Violations
-		CountValidation[int]{Expected: params.ApplicableViolations, Actual: applicableViolationsResults, Msg: GetValidationCountErrMsg("applicable violations", "sarif report", params.ExactResultsMatch, params.ApplicableViolations, applicableViolationsResults)},
-		CountValidation[int]{Expected: params.UndeterminedViolations, Actual: undeterminedViolationsResults, Msg: GetValidationCountErrMsg("undetermined violations", "sarif report", params.ExactResultsMatch, params.UndeterminedViolations, undeterminedViolationsResults)},
-		CountValidation[int]{Expected: params.NotCoveredViolations, Actual: notCoveredViolationsResults, Msg: GetValidationCountErrMsg("not covered violations", "sarif report", params.ExactResultsMatch, params.NotCoveredViolations, notCoveredViolationsResults)},
-		CountValidation[int]{Expected: params.NotApplicableViolations, Actual: notApplicableViolationsResults, Msg: GetValidationCountErrMsg("not applicable violations", "sarif report", params.ExactResultsMatch, params.NotApplicableViolations, notApplicableViolationsResults)},
-		CountValidation[int]{Expected: params.MissingContextViolations, Actual: missingContextViolationsResults, Msg: GetValidationCountErrMsg("missing context violations", "sarif report", params.ExactResultsMatch, params.MissingContextViolations, missingContextViolationsResults)},
-		CountValidation[int]{Expected: params.ScaSecurityViolations, Actual: securityViolations, Msg: GetValidationCountErrMsg("security violations", "sarif report", params.ExactResultsMatch, params.ScaSecurityViolations, securityViolations)},
-		CountValidation[int]{Expected: params.LicenseViolations, Actual: licenseViolations, Msg: GetValidationCountErrMsg("license violations", "sarif report", params.ExactResultsMatch, params.LicenseViolations, licenseViolations)},
-	)
+	ValidateCount(t, "sarif report", params, actualValues)
 }
 
 func countScaResults(report *sarif.Report) (vulnerabilities, applicableVulnerabilitiesResults, undeterminedVulnerabilitiesResults, notCoveredVulnerabilitiesResults, notApplicableVulnerabilitiesResults, missingContextVulnerabilitiesResults, violations, securityViolations, licenseViolations, applicableViolationsResults, undeterminedViolationsResults, notCoveredViolationsResults, notApplicableViolationsResults, missingContextViolationsResults int) {