From 59105890cd865768d129331a63a6dfd62a943f69 Mon Sep 17 00:00:00 2001 From: attiasas Date: Wed, 4 Dec 2024 11:22:00 +0200 Subject: [PATCH] improve test to check sub attribs in validations --- audit_test.go | 204 +++++++-------- commands/audit/audit_test.go | 23 +- scans_test.go | 25 +- utils/results/conversion/convertor_test.go | 57 ++--- utils/validations/test_mocks.go | 12 + utils/validations/test_validate_sarif.go | 56 +--- utils/validations/test_validate_sca.go | 20 +- .../validations/test_validate_simple_json.go | 98 +++---- utils/validations/test_validate_summary.go | 86 +++---- utils/validations/test_validation.go | 240 +++++++++++++++--- xsc_test.go | 13 +- 11 files changed, 445 insertions(+), 389 deletions(-) diff --git a/audit_test.go b/audit_test.go index b3775921..deab9be2 100644 --- a/audit_test.go +++ b/audit_test.go @@ -41,8 +41,10 @@ func TestXrayAuditNpmJson(t *testing.T) { integration.InitAuditJavaScriptTest(t, scangraph.GraphScanMinXrayVersion) output := testAuditNpm(t, string(format.Json), false) validations.VerifyJsonResults(t, output, validations.ValidationParams{ - ScaSecurityViolations: 1, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1}, + Violations: &validations.ViolationCount{ + ValidateType: &validations.ScaViolationCount{Security: 1}, + }, }) } @@ -50,9 +52,10 @@ func TestXrayAuditNpmSimpleJson(t *testing.T) { integration.InitAuditJavaScriptTest(t, scangraph.GraphScanMinXrayVersion) output := testAuditNpm(t, string(format.SimpleJson), true) validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - ScaSecurityViolations: 1, - Vulnerabilities: 1, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1}, + Violations: &validations.ViolationCount{ + ValidateType: &validations.ScaViolationCount{Security: 1}, + }, }) } @@ -76,8 +79,7 @@ func TestXrayAuditConanJson(t *testing.T) { integration.InitAuditCTest(t, scangraph.GraphScanMinXrayVersion) output := testAuditConan(t, string(format.Json), true) validations.VerifyJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 8, - Licenses: 2, + Total: &validations.TotalCount{Licenses: 2, Vulnerabilities: 8}, }) } @@ -85,8 +87,7 @@ func TestXrayAuditConanSimpleJson(t *testing.T) { integration.InitAuditCTest(t, scangraph.GraphScanMinXrayVersion) output := testAuditConan(t, string(format.SimpleJson), true) validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 8, - Licenses: 2, + Total: &validations.TotalCount{Licenses: 2, Vulnerabilities: 8}, }) } @@ -108,8 +109,7 @@ func TestXrayAuditPnpmJson(t *testing.T) { integration.InitAuditJavaScriptTest(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditPnpm(t, string(format.Json)) validations.VerifyJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 1, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1}, }) } @@ -117,8 +117,7 @@ func TestXrayAuditPnpmSimpleJson(t *testing.T) { integration.InitAuditJavaScriptTest(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditPnpm(t, string(format.SimpleJson)) validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 1, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1}, }) } @@ -137,8 +136,7 @@ func TestXrayAuditYarnV2Json(t *testing.T) { testXrayAuditYarn(t, "yarn-v2", func() { output := runXrayAuditYarnWithOutput(t, string(format.Json)) validations.VerifyJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 1, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1}, }) }) } @@ -148,8 +146,7 @@ func TestXrayAuditYarnV2SimpleJson(t *testing.T) { testXrayAuditYarn(t, "yarn-v3", func() { output := runXrayAuditYarnWithOutput(t, string(format.SimpleJson)) validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 1, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1}, }) }) } @@ -159,8 +156,7 @@ func TestXrayAuditYarnV1Json(t *testing.T) { testXrayAuditYarn(t, "yarn-v1", func() { output := runXrayAuditYarnWithOutput(t, string(format.Json)) validations.VerifyJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 1, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1}, }) }) } @@ -183,8 +179,7 @@ func TestXrayAuditYarnV1SimpleJson(t *testing.T) { testXrayAuditYarn(t, "yarn-v1", func() { output := runXrayAuditYarnWithOutput(t, string(format.SimpleJson)) validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 1, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1}, }) }) } @@ -255,8 +250,7 @@ func TestXrayAuditNugetJson(t *testing.T) { func(t *testing.T) { output := testXrayAuditNuget(t, test.projectName, test.format, test.restoreTech) validations.VerifyJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: test.minVulnerabilities, - Licenses: test.minLicences, + Total: &validations.TotalCount{Licenses: test.minLicences, Vulnerabilities: test.minVulnerabilities}, }) }) } @@ -299,8 +293,7 @@ func TestXrayAuditNugetSimpleJson(t *testing.T) { func(t *testing.T) { output := testXrayAuditNuget(t, test.projectName, test.format, test.restoreTech) validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: test.minVulnerabilities, - Licenses: test.minLicences, + Total: &validations.TotalCount{Licenses: test.minLicences, Vulnerabilities: test.minVulnerabilities}, }) }) } @@ -323,8 +316,7 @@ func TestXrayAuditGradleJson(t *testing.T) { integration.InitAuditJavaTest(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditGradle(t, string(format.Json)) validations.VerifyJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 3, - Licenses: 3, + Total: &validations.TotalCount{Licenses: 3, Vulnerabilities: 3}, }) } @@ -332,8 +324,7 @@ func TestXrayAuditGradleSimpleJson(t *testing.T) { integration.InitAuditJavaTest(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditGradle(t, string(format.SimpleJson)) validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 3, - Licenses: 3, + Total: &validations.TotalCount{Licenses: 3, Vulnerabilities: 3}, }) } @@ -349,8 +340,7 @@ func TestXrayAuditMavenJson(t *testing.T) { integration.InitAuditJavaTest(t, scangraph.GraphScanMinXrayVersion) output := testAuditMaven(t, string(format.Json)) validations.VerifyJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 1, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1}, }) } @@ -358,8 +348,7 @@ func TestXrayAuditMavenSimpleJson(t *testing.T) { integration.InitAuditJavaTest(t, scangraph.GraphScanMinXrayVersion) output := testAuditMaven(t, string(format.SimpleJson)) validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 1, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1}, }) } @@ -374,13 +363,18 @@ func testAuditMaven(t *testing.T, format string) string { func TestXrayAuditGoJson(t *testing.T) { integration.InitAuditGoTest(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditGo(t, false, string(format.Json), "simple-project") - validations.VerifyJsonResults(t, output, validations.ValidationParams{Licenses: 1, Vulnerabilities: 4}) + validations.VerifyJsonResults(t, output, validations.ValidationParams{Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 4}}) } func TestXrayAuditGoSimpleJson(t *testing.T) { integration.InitAuditGoTest(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditGo(t, true, string(format.SimpleJson), "simple-project") - validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{Licenses: 3, Vulnerabilities: 4, NotCoveredVulnerabilities: 2, NotApplicableVulnerabilities: 2}) + validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ + Total: &validations.TotalCount{Licenses: 3, Vulnerabilities: 4}, + Vulnerabilities: &validations.VulnerabilityCount{ + ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{NotCovered: 2, NotApplicable: 2}, + }, + }) } func testXrayAuditGo(t *testing.T, noCreds bool, format, project string) string { @@ -425,15 +419,11 @@ func TestXrayAuditMultiProjects(t *testing.T) { output := securityTests.PlatformCli.WithoutCredentials().RunCliCmdWithOutput(t, "audit", "--format="+string(format.SimpleJson), workingDirsFlag) validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - SastVulnerabilities: 1, - IacVulnerabilities: 9, - SecretsVulnerabilities: 6, - - Vulnerabilities: 35, - ApplicableVulnerabilities: 3, - UndeterminedVulnerabilities: 0, - NotCoveredVulnerabilities: 22, - NotApplicableVulnerabilities: 2, + Total: &validations.TotalCount{Vulnerabilities: 43}, + Vulnerabilities: &validations.VulnerabilityCount{ + ValidateScan: &validations.ScanCount{Sca: 27, Sast: 1, Iac: 9, Secrets: 6}, + ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Applicable: 3, NotCovered: 22, NotApplicable: 2}, + }, }) } @@ -441,38 +431,34 @@ func TestXrayAuditPipJson(t *testing.T) { integration.InitAuditPythonTest(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditPip(t, string(format.Json), "") validations.VerifyJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 3, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 3}, }) } func TestXrayAuditCocoapods(t *testing.T) { integration.InitAuditCocoapodsTest(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditCocoapods(t, string(format.Json)) - validations.VerifyJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 1, - }) + validations.VerifyJsonResults(t, output, validations.ValidationParams{Total: &validations.TotalCount{Vulnerabilities: 1}}) } func TestXrayAuditPipSimpleJson(t *testing.T) { integration.InitAuditPythonTest(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditPip(t, string(format.SimpleJson), "") validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 3, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 3}, }) } func TestXrayAuditPipJsonWithRequirementsFile(t *testing.T) { integration.InitAuditPythonTest(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditPip(t, string(format.Json), "requirements.txt") - validations.VerifyJsonResults(t, output, validations.ValidationParams{Vulnerabilities: 2}) + validations.VerifyJsonResults(t, output, validations.ValidationParams{Total: &validations.TotalCount{Vulnerabilities: 2}}) } func TestXrayAuditPipSimpleJsonWithRequirementsFile(t *testing.T) { integration.InitAuditPythonTest(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditPip(t, string(format.SimpleJson), "requirements.txt") - validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{Vulnerabilities: 2}) + validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{Total: &validations.TotalCount{Vulnerabilities: 2}}) } func testXrayAuditPip(t *testing.T, format, requirementsFile string) string { @@ -499,8 +485,7 @@ func TestXrayAuditPipenvJson(t *testing.T) { integration.InitAuditPythonTest(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditPipenv(t, string(format.Json)) validations.VerifyJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 3, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 3}, }) } @@ -508,8 +493,7 @@ func TestXrayAuditPipenvSimpleJson(t *testing.T) { integration.InitAuditPythonTest(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditPipenv(t, string(format.SimpleJson)) validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 3, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 3}, }) } @@ -525,8 +509,7 @@ func TestXrayAuditPoetryJson(t *testing.T) { integration.InitAuditPythonTest(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditPoetry(t, string(format.Json)) validations.VerifyJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 3, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 3}, }) } @@ -534,8 +517,7 @@ func TestXrayAuditPoetrySimpleJson(t *testing.T) { integration.InitAuditPythonTest(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditPoetry(t, string(format.SimpleJson)) validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 3, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 3}, }) } @@ -570,14 +552,14 @@ func TestXrayAuditSastCppFlagSimpleJson(t *testing.T) { name: "withFlag", withFlag: true, expectedResults: validations.ValidationParams{ - Vulnerabilities: 1, - SastVulnerabilities: 1, + Total: &validations.TotalCount{Vulnerabilities: 1}, + Vulnerabilities: &validations.VulnerabilityCount{ValidateScan: &validations.ScanCount{Sast: 1}}, }, }, { name: "withoutFlag", withFlag: false, - expectedResults: validations.ValidationParams{}, + expectedResults: validations.ValidationParams{ExactResultsMatch: true, Total: &validations.TotalCount{}}, }, } for _, tc := range testCase { @@ -593,15 +575,17 @@ func TestXrayAuditSastCSharpFlagSimpleJson(t *testing.T) { integration.InitAuditJasTest(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditJas(t, securityTests.PlatformCli, filepath.Join("package-managers", "dotnet", "dotnet-single"), "3", false, false, true, "") validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 1, - SastVulnerabilities: 1, + Total: &validations.TotalCount{Vulnerabilities: 1}, + Vulnerabilities: &validations.VulnerabilityCount{ValidateScan: &validations.ScanCount{Sast: 1}}, }) } func TestXrayAuditJasMissingContextSimpleJson(t *testing.T) { integration.InitAuditJasTest(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditJas(t, securityTests.PlatformCli, filepath.Join("package-managers", "maven", "missing-context"), "3", false, false, false, "") - validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{MissingContextVulnerabilities: 1}) + validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ + Vulnerabilities: &validations.VulnerabilityCount{ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{MissingContext: 1}}, + }) } func TestXrayAuditNotEntitledForJas(t *testing.T) { @@ -609,7 +593,7 @@ func TestXrayAuditNotEntitledForJas(t *testing.T) { cliToRun, cleanUp := integration.InitTestWithMockCommandOrParams(t, false, getNoJasAuditMockCommand) defer cleanUp() output := testXrayAuditJas(t, cliToRun, filepath.Join("jas", "jas"), "3", false, false, false, "") - validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{Vulnerabilities: 8}) + validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{Total: &validations.TotalCount{Vulnerabilities: 8}}) } func getNoJasAuditMockCommand() components.Command { @@ -632,37 +616,34 @@ func TestXrayAuditJasSimpleJson(t *testing.T) { integration.InitAuditGeneralTests(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditJas(t, securityTests.PlatformCli, filepath.Join("jas", "jas"), "3", false, false, false, "") validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - SastVulnerabilities: 1, - IacVulnerabilities: 9, - SecretsVulnerabilities: 6, - - Vulnerabilities: 8, - ApplicableVulnerabilities: 3, - UndeterminedVulnerabilities: 1, - NotCoveredVulnerabilities: 1, - NotApplicableVulnerabilities: 2, + Total: &validations.TotalCount{Vulnerabilities: 23}, + Vulnerabilities: &validations.VulnerabilityCount{ + ValidateScan: &validations.ScanCount{Sca: 7, Sast: 1, Iac: 9, Secrets: 6}, + ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Applicable: 3, Undetermined: 1, NotCovered: 1, NotApplicable: 2}, + }, }) } func TestXrayAuditJasSimpleJsonWithTokenValidation(t *testing.T) { integration.InitAuditGeneralTests(t, jasutils.DynamicTokenValidationMinXrayVersion) output := testXrayAuditJas(t, securityTests.PlatformCli, filepath.Join("jas", "jas"), "3", true, false, false, "") - validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{Vulnerabilities: 5, InactiveVulnerabilities: 5}) + validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ + Vulnerabilities: &validations.VulnerabilityCount{ + ValidateScan: &validations.ScanCount{Secrets: 5}, + ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Inactive: 5}, + }, + }) } func TestXrayAuditJasSimpleJsonWithOneThread(t *testing.T) { integration.InitAuditGeneralTests(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditJas(t, securityTests.PlatformCli, filepath.Join("jas", "jas"), "1", false, false, false, "") validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - SastVulnerabilities: 1, - IacVulnerabilities: 9, - SecretsVulnerabilities: 6, - - Vulnerabilities: 8, - ApplicableVulnerabilities: 3, - UndeterminedVulnerabilities: 1, - NotCoveredVulnerabilities: 1, - NotApplicableVulnerabilities: 2, + Total: &validations.TotalCount{Vulnerabilities: 23}, + Vulnerabilities: &validations.VulnerabilityCount{ + ValidateScan: &validations.ScanCount{Sca: 7, Sast: 1, Iac: 9, Secrets: 6}, + ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Applicable: 3, Undetermined: 1, NotCovered: 1, NotApplicable: 2}, + }, }) } @@ -670,20 +651,21 @@ func TestXrayAuditJasSimpleJsonWithConfig(t *testing.T) { integration.InitAuditGeneralTests(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditJas(t, securityTests.PlatformCli, filepath.Join("jas", "jas-config"), "3", false, false, false, "") validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - SecretsVulnerabilities: 1, - - Vulnerabilities: 8, - ApplicableVulnerabilities: 3, - UndeterminedVulnerabilities: 1, - NotCoveredVulnerabilities: 1, - NotApplicableVulnerabilities: 2, + Total: &validations.TotalCount{Vulnerabilities: 8}, + Vulnerabilities: &validations.VulnerabilityCount{ + ValidateScan: &validations.ScanCount{Sca: 7, Secrets: 1}, + ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Applicable: 3, Undetermined: 1, NotCovered: 1, NotApplicable: 2}, + }, }) } func TestXrayAuditJasNoViolationsSimpleJson(t *testing.T) { integration.InitAuditGeneralTests(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditJas(t, securityTests.PlatformCli, filepath.Join("package-managers", "npm", "npm"), "3", false, false, false, "") - validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{Vulnerabilities: 1, NotApplicableVulnerabilities: 1}) + validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ + Total: &validations.TotalCount{Vulnerabilities: 1}, + Vulnerabilities: &validations.VulnerabilityCount{ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{NotApplicable: 1}}, + }) } func testXrayAuditJas(t *testing.T, testCli *coreTests.JfrogCli, project string, threads string, validateSecrets bool, validateSastCpp bool, validateSastCSharp bool, customExclusion string) string { @@ -739,7 +721,7 @@ func TestXrayRecursiveScan(t *testing.T) { output := securityTests.PlatformCli.RunCliCmdWithOutput(t, "audit", "--format=json") // We anticipate the identification of five vulnerabilities: four originating from the .NET project and one from the NPM project. - validations.VerifyJsonResults(t, output, validations.ValidationParams{Vulnerabilities: 4}) + validations.VerifyJsonResults(t, output, validations.ValidationParams{Total: &validations.TotalCount{Vulnerabilities: 4}}) var results []services.ScanResponse err = json.Unmarshal([]byte(output), &results) @@ -769,7 +751,7 @@ func TestXrayAuditNotEntitledForJasWithXrayUrl(t *testing.T) { defer cleanUp() output := testXrayAuditJas(t, cliToRun, filepath.Join("jas", "jas"), "3", false, false, false, "") // Verify that scan results are printed - validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{Vulnerabilities: 8}) + validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{Total: &validations.TotalCount{Vulnerabilities: 8}}) // Verify that JAS results are not printed validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{}) } @@ -779,15 +761,11 @@ func TestXrayAuditJasSimpleJsonWithXrayUrl(t *testing.T) { cliToRun := integration.GetTestCli(cli.GetJfrogCliSecurityApp(), true) output := testXrayAuditJas(t, cliToRun, filepath.Join("jas", "jas"), "3", false, false, false, "") validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - SastVulnerabilities: 1, - IacVulnerabilities: 9, - SecretsVulnerabilities: 6, - - Vulnerabilities: 8, - ApplicableVulnerabilities: 3, - UndeterminedVulnerabilities: 1, - NotCoveredVulnerabilities: 1, - NotApplicableVulnerabilities: 2, + Total: &validations.TotalCount{Vulnerabilities: 24}, + Vulnerabilities: &validations.VulnerabilityCount{ + ValidateScan: &validations.ScanCount{Sca: 7, Sast: 2, Iac: 9, Secrets: 6}, + ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Applicable: 3, Undetermined: 1, NotCovered: 1, NotApplicable: 2}, + }, }) } @@ -797,14 +775,10 @@ func TestXrayAuditJasSimpleJsonWithCustomExclusions(t *testing.T) { integration.InitAuditJasTest(t, scangraph.GraphScanMinXrayVersion) output := testXrayAuditJas(t, securityTests.PlatformCli, filepath.Join("jas", "jas"), "3", false, false, false, "non_existing_folder") validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - SastVulnerabilities: 2, - IacVulnerabilities: 9, - SecretsVulnerabilities: 6, - - Vulnerabilities: 8, - ApplicableVulnerabilities: 3, - UndeterminedVulnerabilities: 1, - NotCoveredVulnerabilities: 1, - NotApplicableVulnerabilities: 2, + Total: &validations.TotalCount{Vulnerabilities: 24}, + Vulnerabilities: &validations.VulnerabilityCount{ + ValidateScan: &validations.ScanCount{Sca: 7, Sast: 2, Iac: 9, Secrets: 6}, + ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Applicable: 3, Undetermined: 1, NotCovered: 1, NotApplicable: 2}, + }, }) } diff --git a/commands/audit/audit_test.go b/commands/audit/audit_test.go index 08e3c24a..cd567e89 100644 --- a/commands/audit/audit_test.go +++ b/commands/audit/audit_test.go @@ -439,24 +439,21 @@ func TestAuditWithConfigProfile(t *testing.T) { summary, err := conversion.NewCommandResultsConvertor(conversion.ResultConvertParams{IncludeVulnerabilities: true}).ConvertToSummary(auditResults) assert.NoError(t, err) - var ScaResultsCount int + var scaResultsCount int // When checking Applicability results with ExactResultsMatch = true, the sum of all statuses should equal total Sca results amount. Else, we check the provided Sca issues amount if testcase.expectedCaApplicable > 0 || testcase.expectedCaNotApplicable > 0 || testcase.expectedCaNotCovered > 0 || testcase.expectedCaUndetermined > 0 { - ScaResultsCount = testcase.expectedCaApplicable + testcase.expectedCaNotApplicable + testcase.expectedCaNotCovered + testcase.expectedCaUndetermined + scaResultsCount = testcase.expectedCaApplicable + testcase.expectedCaNotApplicable + testcase.expectedCaNotCovered + testcase.expectedCaUndetermined } else { - ScaResultsCount = testcase.expectedScaIssues + scaResultsCount = testcase.expectedScaIssues } validations.ValidateCommandSummaryOutput(t, validations.ValidationParams{ - Actual: summary, - ExactResultsMatch: true, - Vulnerabilities: testcase.expectedSastIssues + testcase.expectedSecretsIssues + testcase.expectedIacIssues + ScaResultsCount, - SastVulnerabilities: testcase.expectedSastIssues, - SecretsVulnerabilities: testcase.expectedSecretsIssues, - IacVulnerabilities: testcase.expectedIacIssues, - ApplicableVulnerabilities: testcase.expectedCaApplicable, - NotApplicableVulnerabilities: testcase.expectedCaNotApplicable, - NotCoveredVulnerabilities: testcase.expectedCaNotCovered, - UndeterminedVulnerabilities: testcase.expectedCaUndetermined, + Actual: summary, + ExactResultsMatch: true, + Total: &validations.TotalCount{Vulnerabilities: testcase.expectedSastIssues + testcase.expectedSecretsIssues + testcase.expectedIacIssues + scaResultsCount}, + Vulnerabilities: &validations.VulnerabilityCount{ + ValidateScan: &validations.ScanCount{Sca: scaResultsCount, Sast: testcase.expectedSastIssues, Secrets: testcase.expectedSecretsIssues, Iac: testcase.expectedIacIssues}, + ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Applicable: testcase.expectedCaApplicable, NotApplicable: testcase.expectedCaNotApplicable, NotCovered: testcase.expectedCaNotCovered, Undetermined: testcase.expectedCaUndetermined}, + }, }) }) } diff --git a/scans_test.go b/scans_test.go index 5a7bc8d6..3c401e5d 100644 --- a/scans_test.go +++ b/scans_test.go @@ -45,8 +45,7 @@ func TestXrayBinaryScanJson(t *testing.T) { integration.InitScanTest(t, scangraph.GraphScanMinXrayVersion) output := testXrayBinaryScan(t, string(format.Json), false) validations.VerifyJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 1, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1}, }) } @@ -54,9 +53,7 @@ func TestXrayBinaryScanSimpleJson(t *testing.T) { integration.InitScanTest(t, scangraph.GraphScanMinXrayVersion) output := testXrayBinaryScan(t, string(format.SimpleJson), true) validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 1, - ScaSecurityViolations: 1, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1, Violations: 1}, }) } @@ -66,8 +63,7 @@ func TestXrayBinaryScanJsonWithProgress(t *testing.T) { defer callback() output := testXrayBinaryScan(t, string(format.Json), false) validations.VerifyJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 1, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1}, }) } @@ -77,9 +73,7 @@ func TestXrayBinaryScanSimpleJsonWithProgress(t *testing.T) { defer callback() output := testXrayBinaryScan(t, string(format.SimpleJson), true) validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 1, - ScaSecurityViolations: 1, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1, Violations: 1}, }) } @@ -110,8 +104,7 @@ func TestXrayBinaryScanWithBypassArchiveLimits(t *testing.T) { scanArgs = append(scanArgs, "--bypass-archive-limits") output := securityTests.PlatformCli.RunCliCmdWithOutput(t, scanArgs...) validations.VerifyJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 1, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1}, }) } @@ -172,9 +165,11 @@ func runDockerScan(t *testing.T, testCli *coreTests.JfrogCli, imageName, watchNa output := testCli.WithoutCredentials().RunCliCmdWithOutput(t, cmdArgs...) if assert.NotEmpty(t, output) { if validateSecrets { - validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{InactiveVulnerabilities: minInactives}) + validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ + Vulnerabilities: &validations.VulnerabilityCount{ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Inactive: minInactives}}, + }) } else { - validations.VerifyJsonResults(t, output, validations.ValidationParams{Vulnerabilities: minVulnerabilities, Licenses: minLicenses}) + validations.VerifyJsonResults(t, output, validations.ValidationParams{Total: &validations.TotalCount{Vulnerabilities: minVulnerabilities, Licenses: minLicenses}}) } } // Run docker scan on image with watch @@ -184,7 +179,7 @@ func runDockerScan(t *testing.T, testCli *coreTests.JfrogCli, imageName, watchNa cmdArgs = append(cmdArgs, "--watches="+watchName) output = testCli.WithoutCredentials().RunCliCmdWithOutput(t, cmdArgs...) if assert.NotEmpty(t, output) { - validations.VerifyJsonResults(t, output, validations.ValidationParams{ScaSecurityViolations: minViolations}) + validations.VerifyJsonResults(t, output, validations.ValidationParams{Total: &validations.TotalCount{Violations: minViolations}}) } } } diff --git a/utils/results/conversion/convertor_test.go b/utils/results/conversion/convertor_test.go index ce31bf4f..dd12e058 100644 --- a/utils/results/conversion/convertor_test.go +++ b/utils/results/conversion/convertor_test.go @@ -32,19 +32,17 @@ func getAuditValidationParams() validations.ValidationParams { return validations.ValidationParams{ ExactResultsMatch: true, - Vulnerabilities: 19, - ApplicableVulnerabilities: 1, - NotApplicableVulnerabilities: 7, - NotCoveredVulnerabilities: 4, - SastVulnerabilities: 4, - SecretsVulnerabilities: 3, - - Violations: 7, - ScaSecurityViolations: 5, - ApplicableViolations: 1, - NotApplicableViolations: 4, - SastViolations: 1, - SecretsViolations: 1, + Total: &validations.TotalCount{Vulnerabilities: 19, Violations: 7}, + + Vulnerabilities: &validations.VulnerabilityCount{ + ValidateScan: &validations.ScanCount{Sca: 12, Sast: 4, Secrets: 3}, + ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Applicable: 1, NotApplicable: 7, NotCovered: 4}, + }, + + Violations: &validations.ViolationCount{ + ValidateScan: &validations.ScanCount{Sca: 5, Sast: 1, Secrets: 1}, + ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Applicable: 1, NotApplicable: 4}, + }, } } @@ -52,26 +50,25 @@ func getAuditValidationParams() validations.ValidationParams { // We have in the result 2 CVE with 2 impacted components each func getDockerScanValidationParams(unique bool) validations.ValidationParams { params := validations.ValidationParams{ - ExactResultsMatch: true, - SecretsVulnerabilities: 3, - - Violations: 3, - ScaSecurityViolations: 1, - UndeterminedViolations: 1, - SecretsViolations: 2, + ExactResultsMatch: true, + Total: &validations.TotalCount{Violations: 3}, + Violations: &validations.ViolationCount{ + ValidateScan: &validations.ScanCount{Sca: 1, Secrets: 2}, + ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Undetermined: 1}, + }, } if unique { - params.Vulnerabilities = 11 - params.ApplicableVulnerabilities = 3 - params.NotApplicableVulnerabilities = 3 - params.NotCoveredVulnerabilities = 1 - params.UndeterminedVulnerabilities = 1 + params.Total.Vulnerabilities = 11 + params.Vulnerabilities = &validations.VulnerabilityCount{ + ValidateScan: &validations.ScanCount{Sca: 8, Secrets: 3}, + ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Applicable: 3, NotApplicable: 3, NotCovered: 1, Undetermined: 1}, + } } else { - params.Vulnerabilities = 14 - params.ApplicableVulnerabilities = 5 - params.NotApplicableVulnerabilities = 4 - params.NotCoveredVulnerabilities = 1 - params.UndeterminedVulnerabilities = 1 + params.Total.Vulnerabilities = 14 + params.Vulnerabilities = &validations.VulnerabilityCount{ + ValidateScan: &validations.ScanCount{Sca: 11, Secrets: 3}, + ValidateApplicabilityStatus: &validations.ApplicabilityStatusCount{Applicable: 5, NotApplicable: 4, NotCovered: 1, Undetermined: 1}, + } } return params } diff --git a/utils/validations/test_mocks.go b/utils/validations/test_mocks.go index 5fabcb38..29bcd752 100644 --- a/utils/validations/test_mocks.go +++ b/utils/validations/test_mocks.go @@ -12,6 +12,7 @@ import ( "github.com/jfrog/jfrog-cli-core/v2/utils/config" "github.com/jfrog/jfrog-cli-security/utils/results" "github.com/jfrog/jfrog-client-go/artifactory" + "github.com/jfrog/jfrog-client-go/xray/services" xscutils "github.com/jfrog/jfrog-client-go/xsc/services/utils" "github.com/owenrumney/go-sarif/v2/sarif" "github.com/stretchr/testify/assert" @@ -166,3 +167,14 @@ func XrayServer(t *testing.T, params MockServerParams) (*httptest.Server, *confi func NewMockJasRuns(runs ...*sarif.Run) []results.ScanResult[[]*sarif.Run] { return []results.ScanResult[[]*sarif.Run]{{Scan: runs}} } + +func NewMockScaResults(responses ...services.ScanResponse) (converted []results.ScanResult[services.ScanResponse]) { + for _, response := range responses { + status := 0 + if response.ScannedStatus == "Failed" { + status = 1 + } + converted = append(converted, results.ScanResult[services.ScanResponse]{Scan: response, StatusCode: status}) + } + return +} diff --git a/utils/validations/test_validate_sarif.go b/utils/validations/test_validate_sarif.go index 78b2a8da..ace35d83 100644 --- a/utils/validations/test_validate_sarif.go +++ b/utils/validations/test_validate_sarif.go @@ -43,57 +43,29 @@ func ValidateCommandSarifOutput(t *testing.T, params ValidationParams) { // If Expected is provided, the validation will check if the Actual content matches the expected results. // If ExactResultsMatch is true, the validation will check exact values and not only the 'equal or grater' counts / existence of expected attributes. (For Integration tests with JFrog API, ExactResultsMatch should be set to false) func ValidateSarifIssuesCount(t *testing.T, params ValidationParams, report *sarif.Report) { - var vulnerabilities, violations int + actualValues := validationCountActualValues{} // SCA - scaVulnerabilities, applicableVulnerabilitiesResults, undeterminedVulnerabilitiesResults, notCoveredVulnerabilitiesResults, notApplicableVulnerabilitiesResults, missingContextVulnerabilitiesResults, scaViolations, securityViolations, licenseViolations, applicableViolationsResults, undeterminedViolationsResults, notCoveredViolationsResults, notApplicableViolationsResults, missingContextViolationsResults := countScaResults(report) - vulnerabilities += scaVulnerabilities - violations += scaViolations + actualValues.ScaVulnerabilities, actualValues.ApplicableVulnerabilities, actualValues.UndeterminedVulnerabilities, actualValues.NotCoveredVulnerabilities, actualValues.NotApplicableVulnerabilities, actualValues.MissingContextVulnerabilities, actualValues.ScaViolations, actualValues.SecurityViolations, actualValues.LicenseViolations, actualValues.ApplicableViolations, actualValues.UndeterminedViolations, actualValues.NotCoveredViolations, actualValues.NotApplicableViolations, actualValues.MissingContextViolations = countScaResults(report) + actualValues.Vulnerabilities += actualValues.ScaVulnerabilities + actualValues.Violations += actualValues.ScaViolations // Secrets - secretsVulnerabilities, inactiveVulnerabilities, secretsViolations, inactiveViolations := countSecretsResults(report) - vulnerabilities += secretsVulnerabilities - violations += secretsViolations + actualValues.SecretsVulnerabilities, actualValues.InactiveSecretsVulnerabilities, actualValues.SecretsViolations, actualValues.InactiveSecretsViolations = countSecretsResults(report) + actualValues.Vulnerabilities += actualValues.SecretsVulnerabilities + actualValues.Violations += actualValues.SecretsViolations // IAC - iacVulnerabilities, iacViolations := countJasResults(sarifutils.GetRunsByToolName(report, IacToolName)) - vulnerabilities += iacVulnerabilities - violations += iacViolations + actualValues.IacVulnerabilities, actualValues.IacViolations = countJasResults(sarifutils.GetRunsByToolName(report, IacToolName)) + actualValues.Vulnerabilities += actualValues.IacVulnerabilities + actualValues.Violations += actualValues.IacViolations // SAST - sastVulnerabilities, sastViolations := countJasResults(sarifutils.GetRunsByToolName(report, SastToolName)) - vulnerabilities += sastVulnerabilities - violations += sastViolations + actualValues.SastVulnerabilities, actualValues.SastViolations = countJasResults(sarifutils.GetRunsByToolName(report, SastToolName)) + actualValues.Vulnerabilities += actualValues.SastVulnerabilities + actualValues.Violations += actualValues.SastViolations - ValidateContent(t, params.ExactResultsMatch, - // Total - CountValidation[int]{Expected: params.Vulnerabilities, Actual: vulnerabilities, Msg: GetValidationCountErrMsg("vulnerabilities", "sarif report", params.ExactResultsMatch, params.Vulnerabilities, vulnerabilities)}, - CountValidation[int]{Expected: params.ScaSecurityViolations, Actual: violations, Msg: GetValidationCountErrMsg("violations", "sarif report", params.ExactResultsMatch, params.ScaSecurityViolations, violations)}, - // JAS Vulnerabilities - CountValidation[int]{Expected: params.SastVulnerabilities, Actual: sastVulnerabilities, Msg: GetValidationCountErrMsg("sast vulnerabilities", "sarif report", params.ExactResultsMatch, params.SastVulnerabilities, sastVulnerabilities)}, - CountValidation[int]{Expected: params.IacVulnerabilities, Actual: iacVulnerabilities, Msg: GetValidationCountErrMsg("Iac vulnerabilities", "sarif report", params.ExactResultsMatch, params.IacVulnerabilities, iacVulnerabilities)}, - CountValidation[int]{Expected: params.SecretsVulnerabilities, Actual: secretsVulnerabilities, Msg: GetValidationCountErrMsg("secrets vulnerabilities", "sarif report", params.ExactResultsMatch, params.SecretsVulnerabilities, secretsVulnerabilities)}, - CountValidation[int]{Expected: params.InactiveVulnerabilities, Actual: inactiveVulnerabilities, Msg: GetValidationCountErrMsg("inactive secrets vulnerabilities", "sarif report", params.ExactResultsMatch, params.InactiveVulnerabilities, inactiveVulnerabilities)}, - // JAS Violations - CountValidation[int]{Expected: params.SastViolations, Actual: sastViolations, Msg: GetValidationCountErrMsg("sast violations", "sarif report", params.ExactResultsMatch, params.SastViolations, sastViolations)}, - CountValidation[int]{Expected: params.IacViolations, Actual: iacViolations, Msg: GetValidationCountErrMsg("Iac violations", "sarif report", params.ExactResultsMatch, params.IacViolations, iacViolations)}, - CountValidation[int]{Expected: params.SecretsViolations, Actual: secretsViolations, Msg: GetValidationCountErrMsg("secrets violations", "sarif report", params.ExactResultsMatch, params.SecretsViolations, secretsViolations)}, - CountValidation[int]{Expected: params.InactiveViolations, Actual: inactiveViolations, Msg: GetValidationCountErrMsg("inactive secrets violations", "sarif report", params.ExactResultsMatch, params.InactiveViolations, inactiveViolations)}, - // SCA Vulnerabilities - CountValidation[int]{Expected: params.ApplicableVulnerabilities, Actual: applicableVulnerabilitiesResults, Msg: GetValidationCountErrMsg("applicable vulnerabilities", "sarif report", params.ExactResultsMatch, params.ApplicableVulnerabilities, applicableVulnerabilitiesResults)}, - CountValidation[int]{Expected: params.UndeterminedVulnerabilities, Actual: undeterminedVulnerabilitiesResults, Msg: GetValidationCountErrMsg("undetermined vulnerabilities", "sarif report", params.ExactResultsMatch, params.UndeterminedVulnerabilities, undeterminedVulnerabilitiesResults)}, - CountValidation[int]{Expected: params.NotCoveredVulnerabilities, Actual: notCoveredVulnerabilitiesResults, Msg: GetValidationCountErrMsg("not covered vulnerabilities", "sarif report", params.ExactResultsMatch, params.NotCoveredVulnerabilities, notCoveredVulnerabilitiesResults)}, - CountValidation[int]{Expected: params.NotApplicableVulnerabilities, Actual: notApplicableVulnerabilitiesResults, Msg: GetValidationCountErrMsg("not applicable vulnerabilities", "sarif report", params.ExactResultsMatch, params.NotApplicableVulnerabilities, notApplicableVulnerabilitiesResults)}, - CountValidation[int]{Expected: params.MissingContextVulnerabilities, Actual: missingContextVulnerabilitiesResults, Msg: GetValidationCountErrMsg("missing context vulnerabilities", "sarif report", params.ExactResultsMatch, params.MissingContextVulnerabilities, missingContextVulnerabilitiesResults)}, - // SCA Violations - CountValidation[int]{Expected: params.ApplicableViolations, Actual: applicableViolationsResults, Msg: GetValidationCountErrMsg("applicable violations", "sarif report", params.ExactResultsMatch, params.ApplicableViolations, applicableViolationsResults)}, - CountValidation[int]{Expected: params.UndeterminedViolations, Actual: undeterminedViolationsResults, Msg: GetValidationCountErrMsg("undetermined violations", "sarif report", params.ExactResultsMatch, params.UndeterminedViolations, undeterminedViolationsResults)}, - CountValidation[int]{Expected: params.NotCoveredViolations, Actual: notCoveredViolationsResults, Msg: GetValidationCountErrMsg("not covered violations", "sarif report", params.ExactResultsMatch, params.NotCoveredViolations, notCoveredViolationsResults)}, - CountValidation[int]{Expected: params.NotApplicableViolations, Actual: notApplicableViolationsResults, Msg: GetValidationCountErrMsg("not applicable violations", "sarif report", params.ExactResultsMatch, params.NotApplicableViolations, notApplicableViolationsResults)}, - CountValidation[int]{Expected: params.MissingContextViolations, Actual: missingContextViolationsResults, Msg: GetValidationCountErrMsg("missing context violations", "sarif report", params.ExactResultsMatch, params.MissingContextViolations, missingContextViolationsResults)}, - CountValidation[int]{Expected: params.ScaSecurityViolations, Actual: securityViolations, Msg: GetValidationCountErrMsg("security violations", "sarif report", params.ExactResultsMatch, params.ScaSecurityViolations, securityViolations)}, - CountValidation[int]{Expected: params.LicenseViolations, Actual: licenseViolations, Msg: GetValidationCountErrMsg("license violations", "sarif report", params.ExactResultsMatch, params.LicenseViolations, licenseViolations)}, - ) + ValidateCount(t, "sarif report", params, actualValues) } func countScaResults(report *sarif.Report) (vulnerabilities, applicableVulnerabilitiesResults, undeterminedVulnerabilitiesResults, notCoveredVulnerabilitiesResults, notApplicableVulnerabilitiesResults, missingContextVulnerabilitiesResults, violations, securityViolations, licenseViolations, applicableViolationsResults, undeterminedViolationsResults, notCoveredViolationsResults, notApplicableViolationsResults, missingContextViolationsResults int) { diff --git a/utils/validations/test_validate_sca.go b/utils/validations/test_validate_sca.go index ae8e8c00..13c9deb9 100644 --- a/utils/validations/test_validate_sca.go +++ b/utils/validations/test_validate_sca.go @@ -38,10 +38,11 @@ func ValidateCommandJsonOutput(t *testing.T, params ValidationParams) { } func ValidateScanResponseIssuesCount(t *testing.T, params ValidationParams, content ...services.ScanResponse) { - var vulnerabilities, licenses, securityViolations, licenseViolations, operationalViolations int + var vulnerabilities, violations, licenses, securityViolations, licenseViolations, operationalViolations int for _, result := range content { vulnerabilities += len(result.Vulnerabilities) + violations += len(result.Violations) licenses += len(result.Licenses) for _, violation := range result.Violations { switch violation.ViolationType { @@ -55,13 +56,16 @@ func ValidateScanResponseIssuesCount(t *testing.T, params ValidationParams, cont } } - ValidateContent(t, params.ExactResultsMatch, - CountValidation[int]{Expected: params.Vulnerabilities, Actual: vulnerabilities, Msg: GetValidationCountErrMsg("vulnerabilities", "scan responses", params.ExactResultsMatch, params.Vulnerabilities, vulnerabilities)}, - CountValidation[int]{Expected: params.Licenses, Actual: licenses, Msg: GetValidationCountErrMsg("licenses", "scan responses", params.ExactResultsMatch, params.Licenses, licenses)}, - CountValidation[int]{Expected: params.ScaSecurityViolations, Actual: securityViolations, Msg: GetValidationCountErrMsg("security violations", "scan responses", params.ExactResultsMatch, params.ScaSecurityViolations, securityViolations)}, - CountValidation[int]{Expected: params.LicenseViolations, Actual: licenseViolations, Msg: GetValidationCountErrMsg("license violations", "scan responses", params.ExactResultsMatch, params.LicenseViolations, licenseViolations)}, - CountValidation[int]{Expected: params.OperationalViolations, Actual: operationalViolations, Msg: GetValidationCountErrMsg("operational risk violations", "scan responses", params.ExactResultsMatch, params.OperationalViolations, operationalViolations)}, - ) + ValidateTotalCount(t, "json", params.ExactResultsMatch, params.Total, vulnerabilities, violations, licenses) + if params.Violations != nil { + ValidateScaViolationCount(t, "json", params.ExactResultsMatch, params.Violations.ValidateType, securityViolations, licenseViolations, operationalViolations) + if params.Violations.ValidateApplicabilityStatus != nil || params.Violations.ValidateScan != nil { + t.Error("Validate Violations only support ValidateType for JSON output") + } + } + if params.Vulnerabilities != nil { + t.Error("Validate Vulnerabilities is not supported for JSON output") + } } func ValidateScanResponses(t *testing.T, exactMatch bool, expected, actual []services.ScanResponse) { diff --git a/utils/validations/test_validate_simple_json.go b/utils/validations/test_validate_simple_json.go index a4502abd..b15b52b7 100644 --- a/utils/validations/test_validate_simple_json.go +++ b/utils/validations/test_validate_simple_json.go @@ -44,99 +44,73 @@ func ValidateCommandSimpleJsonOutput(t *testing.T, params ValidationParams) { // If Expected is provided, the validation will check if the Actual content matches the expected results. // If ExactResultsMatch is true, the validation will check exact values and not only the 'equal or grater' counts / existence of expected attributes. (For Integration tests with JFrog API, ExactResultsMatch should be set to false) func ValidateSimpleJsonIssuesCount(t *testing.T, params ValidationParams, results formats.SimpleJsonResults) { - var applicableVulnerabilitiesResults, undeterminedVulnerabilitiesResults, notCoveredVulnerabilitiesResults, notApplicableVulnerabilitiesResults, missingContextVulnerabilitiesResults, inactiveSecretsVulnerabilities int - var applicableViolationsResults, undeterminedViolationsResults, notCoveredViolationsResults, notApplicableViolationsResults, missingContextViolationsResults, inactiveSecretsViolations int - // Licenses - licenses := len(results.Licenses) - // Total - vulnerabilities := len(results.Vulnerabilities) + len(results.SecretsVulnerabilities) + len(results.SastVulnerabilities) + len(results.IacsVulnerabilities) - violations := len(results.SecurityViolations) + len(results.LicensesViolations) + len(results.OperationalRiskViolations) + len(results.SecretsViolations) + len(results.SastViolations) + len(results.IacsViolations) - // Jas - sastVulnerabilities := len(results.SastVulnerabilities) - secretsVulnerabilities := len(results.SecretsVulnerabilities) - iacVulnerabilities := len(results.IacsVulnerabilities) - sastViolations := len(results.SastViolations) - secretsViolations := len(results.SecretsViolations) - iacViolations := len(results.IacsViolations) + actualValues := validationCountActualValues{ + // Total + Vulnerabilities: len(results.Vulnerabilities) + len(results.SecretsVulnerabilities) + len(results.SastVulnerabilities) + len(results.IacsVulnerabilities), + Violations: len(results.SecurityViolations) + len(results.LicensesViolations) + len(results.OperationalRiskViolations) + len(results.SecretsViolations) + len(results.SastViolations) + len(results.IacsViolations), + Licenses: len(results.Licenses), + // Jas vulnerabilities + SastVulnerabilities: len(results.SastVulnerabilities), + SecretsVulnerabilities: len(results.SecretsVulnerabilities), + IacVulnerabilities: len(results.IacsVulnerabilities), + // Jas violations + SastViolations: len(results.SastViolations), + SecretsViolations: len(results.SecretsViolations), + IacViolations: len(results.IacsViolations), + // Sca vulnerabilities + ScaVulnerabilities: len(results.Vulnerabilities), + // Sca violations + ScaViolations: len(results.SecurityViolations) + len(results.LicensesViolations) + len(results.OperationalRiskViolations), + SecurityViolations: len(results.SecurityViolations), + LicenseViolations: len(results.LicensesViolations), + OperationalViolations: len(results.OperationalRiskViolations), + } + // Inactive secrets for _, result := range results.SecretsVulnerabilities { if result.Applicability != nil { if result.Applicability.Status == jasutils.Inactive.String() { - inactiveSecretsVulnerabilities += 1 + actualValues.InactiveSecretsVulnerabilities += 1 } } } for _, result := range results.SecretsViolations { if result.Applicability != nil { if result.Applicability.Status == jasutils.Inactive.String() { - inactiveSecretsViolations += 1 + actualValues.InactiveSecretsViolations += 1 } } } - // Sca - securityViolations := len(results.SecurityViolations) - licenseViolations := len(results.LicensesViolations) - opRiskViolations := len(results.OperationalRiskViolations) + // CA status for _, vuln := range results.Vulnerabilities { switch vuln.Applicable { case jasutils.NotApplicable.String(): - notApplicableVulnerabilitiesResults++ + actualValues.NotApplicableVulnerabilities++ case jasutils.Applicable.String(): - applicableVulnerabilitiesResults++ + actualValues.ApplicableVulnerabilities++ case jasutils.NotCovered.String(): - notCoveredVulnerabilitiesResults++ + actualValues.NotCoveredVulnerabilities++ case jasutils.ApplicabilityUndetermined.String(): - undeterminedVulnerabilitiesResults++ + actualValues.UndeterminedVulnerabilities++ case jasutils.MissingContext.String(): - missingContextVulnerabilitiesResults++ + actualValues.MissingContextVulnerabilities++ } } for _, vuln := range results.SecurityViolations { switch vuln.Applicable { case jasutils.NotApplicable.String(): - notApplicableViolationsResults++ + actualValues.NotApplicableViolations++ case jasutils.Applicable.String(): - applicableViolationsResults++ + actualValues.ApplicableViolations++ case jasutils.NotCovered.String(): - notCoveredViolationsResults++ + actualValues.NotCoveredViolations++ case jasutils.ApplicabilityUndetermined.String(): - undeterminedViolationsResults++ + actualValues.UndeterminedViolations++ case jasutils.MissingContext.String(): - missingContextViolationsResults++ + actualValues.MissingContextViolations++ } } - ValidateContent(t, params.ExactResultsMatch, - // Licenses - CountValidation[int]{Expected: params.Licenses, Actual: licenses, Msg: GetValidationCountErrMsg("licenses", "simple-json", params.ExactResultsMatch, params.Licenses, licenses)}, - // Total - CountValidation[int]{Expected: params.Vulnerabilities, Actual: vulnerabilities, Msg: GetValidationCountErrMsg("vulnerabilities", "simple-json", params.ExactResultsMatch, params.Vulnerabilities, vulnerabilities)}, - CountValidation[int]{Expected: params.Violations, Actual: violations, Msg: GetValidationCountErrMsg("violations", "simple-json", params.ExactResultsMatch, params.Violations, violations)}, - // Jas Vulnerabilities - CountValidation[int]{Expected: params.SastVulnerabilities, Actual: sastVulnerabilities, Msg: GetValidationCountErrMsg("sast vulnerabilities", "simple-json", params.ExactResultsMatch, params.SastVulnerabilities, sastVulnerabilities)}, - CountValidation[int]{Expected: params.SecretsVulnerabilities, Actual: secretsVulnerabilities, Msg: GetValidationCountErrMsg("secrets vulnerabilities", "simple-json", params.ExactResultsMatch, params.SecretsVulnerabilities, secretsVulnerabilities)}, - CountValidation[int]{Expected: params.IacVulnerabilities, Actual: iacVulnerabilities, Msg: GetValidationCountErrMsg("IaC vulnerabilities", "simple-json", params.ExactResultsMatch, params.IacVulnerabilities, iacVulnerabilities)}, - CountValidation[int]{Expected: params.InactiveVulnerabilities, Actual: inactiveSecretsVulnerabilities, Msg: GetValidationCountErrMsg("inactive secrets vulnerabilities", "simple-json", params.ExactResultsMatch, params.InactiveVulnerabilities, inactiveSecretsVulnerabilities)}, - // Jas Violation - CountValidation[int]{Expected: params.SastViolations, Actual: sastViolations, Msg: GetValidationCountErrMsg("sast violations", "simple-json", params.ExactResultsMatch, params.SastViolations, sastViolations)}, - CountValidation[int]{Expected: params.SecretsViolations, Actual: secretsViolations, Msg: GetValidationCountErrMsg("secrets violations", "simple-json", params.ExactResultsMatch, params.SecretsViolations, secretsViolations)}, - CountValidation[int]{Expected: params.IacViolations, Actual: iacViolations, Msg: GetValidationCountErrMsg("IaC violations", "simple-json", params.ExactResultsMatch, params.IacViolations, iacViolations)}, - CountValidation[int]{Expected: params.InactiveViolations, Actual: inactiveSecretsViolations, Msg: GetValidationCountErrMsg("inactive secrets violations", "simple-json", params.ExactResultsMatch, params.InactiveViolations, inactiveSecretsViolations)}, - // Sca Vulnerabilities - CountValidation[int]{Expected: params.ApplicableVulnerabilities, Actual: applicableVulnerabilitiesResults, Msg: GetValidationCountErrMsg("applicable vulnerabilities", "simple-json", params.ExactResultsMatch, params.ApplicableVulnerabilities, applicableVulnerabilitiesResults)}, - CountValidation[int]{Expected: params.UndeterminedVulnerabilities, Actual: undeterminedVulnerabilitiesResults, Msg: GetValidationCountErrMsg("undetermined vulnerabilities", "simple-json", params.ExactResultsMatch, params.UndeterminedVulnerabilities, undeterminedVulnerabilitiesResults)}, - CountValidation[int]{Expected: params.NotCoveredVulnerabilities, Actual: notCoveredVulnerabilitiesResults, Msg: GetValidationCountErrMsg("not covered vulnerabilities", "simple-json", params.ExactResultsMatch, params.NotCoveredVulnerabilities, notCoveredVulnerabilitiesResults)}, - CountValidation[int]{Expected: params.NotApplicableVulnerabilities, Actual: notApplicableVulnerabilitiesResults, Msg: GetValidationCountErrMsg("not applicable vulnerabilities", "simple-json", params.ExactResultsMatch, params.NotApplicableVulnerabilities, notApplicableVulnerabilitiesResults)}, - CountValidation[int]{Expected: params.MissingContextVulnerabilities, Actual: missingContextVulnerabilitiesResults, Msg: GetValidationCountErrMsg("missing context vulnerabilities", "simple-json", params.ExactResultsMatch, params.MissingContextVulnerabilities, missingContextVulnerabilitiesResults)}, - // Sca Violations - CountValidation[int]{Expected: params.ApplicableViolations, Actual: applicableViolationsResults, Msg: GetValidationCountErrMsg("applicable violations", "simple-json", params.ExactResultsMatch, params.ApplicableViolations, applicableViolationsResults)}, - CountValidation[int]{Expected: params.UndeterminedViolations, Actual: undeterminedViolationsResults, Msg: GetValidationCountErrMsg("undetermined violations", "simple-json", params.ExactResultsMatch, params.UndeterminedViolations, undeterminedViolationsResults)}, - CountValidation[int]{Expected: params.NotCoveredViolations, Actual: notCoveredViolationsResults, Msg: GetValidationCountErrMsg("not covered violations", "simple-json", params.ExactResultsMatch, params.NotCoveredViolations, notCoveredViolationsResults)}, - CountValidation[int]{Expected: params.NotApplicableViolations, Actual: notApplicableViolationsResults, Msg: GetValidationCountErrMsg("not applicable violations", "simple-json", params.ExactResultsMatch, params.NotApplicableViolations, notApplicableViolationsResults)}, - CountValidation[int]{Expected: params.MissingContextViolations, Actual: missingContextViolationsResults, Msg: GetValidationCountErrMsg("missing context violations", "simple-json", params.ExactResultsMatch, params.MissingContextViolations, missingContextViolationsResults)}, - CountValidation[int]{Expected: params.ScaSecurityViolations, Actual: securityViolations, Msg: GetValidationCountErrMsg("security violations", "simple-json", params.ExactResultsMatch, params.ScaSecurityViolations, securityViolations)}, - CountValidation[int]{Expected: params.LicenseViolations, Actual: licenseViolations, Msg: GetValidationCountErrMsg("license violations", "simple-json", params.ExactResultsMatch, params.LicenseViolations, licenseViolations)}, - CountValidation[int]{Expected: params.OperationalViolations, Actual: opRiskViolations, Msg: GetValidationCountErrMsg("operational risk violations", "simple-json", params.ExactResultsMatch, params.OperationalViolations, opRiskViolations)}, - ) + ValidateCount(t, "simple-json", params, actualValues) } func ValidateSimpleJsonResults(t *testing.T, exactMatch bool, expected, actual formats.SimpleJsonResults) { diff --git a/utils/validations/test_validate_summary.go b/utils/validations/test_validate_summary.go index 14b2eaf5..7b7b90d4 100644 --- a/utils/validations/test_validate_summary.go +++ b/utils/validations/test_validate_summary.go @@ -20,22 +20,27 @@ func ValidateCommandSummaryOutput(t *testing.T, params ValidationParams) { } func ValidateSummaryIssuesCount(t *testing.T, params ValidationParams, results formats.ResultsSummary) { - var vulnerabilities, applicableVulnerabilitiesResults, undeterminedVulnerabilitiesResults, notCoveredVulnerabilitiesResults, notApplicableVulnerabilitiesResults, missingContextVulnerabilitiesResults, sastVulnerabilities, iacVulnerabilities, secretsVulnerabilities int - var violations, securityViolations, licenseViolations, opRiskViolations, applicableViolationsResults, undeterminedViolationsResults, notCoveredViolationsResults, notApplicableViolationsResults, missingContextViolationsResults, sastViolations, iacViolations, secretsViolations int - // Total - vulnerabilities = results.GetTotalVulnerabilities() - violations = results.GetTotalViolations() - // Jas - sastVulnerabilities = results.GetTotalVulnerabilities(formats.SastResult) - secretsVulnerabilities = results.GetTotalVulnerabilities(formats.SecretsResult) - iacVulnerabilities = results.GetTotalVulnerabilities(formats.IacResult) - sastViolations = results.GetTotalViolations(formats.SastResult) - secretsViolations = results.GetTotalViolations(formats.SecretsResult) - iacViolations = results.GetTotalViolations(formats.IacResult) - // Sca - securityViolations = results.GetTotalViolations(formats.ScaSecurityResult) - licenseViolations = results.GetTotalViolations(formats.ScaLicenseResult) - opRiskViolations = results.GetTotalViolations(formats.ScaOperationalResult) + actualValues := validationCountActualValues{ + // Total + Vulnerabilities: results.GetTotalVulnerabilities(), + Violations: results.GetTotalViolations(), + // Jas vulnerabilities + SastVulnerabilities: results.GetTotalVulnerabilities(formats.SastResult), + SecretsVulnerabilities: results.GetTotalVulnerabilities(formats.SecretsResult), + IacVulnerabilities: results.GetTotalVulnerabilities(formats.IacResult), + // Jas violations + SastViolations: results.GetTotalViolations(formats.SastResult), + SecretsViolations: results.GetTotalViolations(formats.SecretsResult), + IacViolations: results.GetTotalViolations(formats.IacResult), + // Sca vulnerabilities + ScaVulnerabilities: results.GetTotalVulnerabilities(formats.ScaSecurityResult), + // Sca violations + ScaViolations: results.GetTotalViolations(formats.ScaSecurityResult, formats.ScaLicenseResult, formats.ScaOperationalResult), + SecurityViolations: results.GetTotalViolations(formats.ScaSecurityResult), + LicenseViolations: results.GetTotalViolations(formats.ScaLicenseResult), + OperationalViolations: results.GetTotalViolations(formats.ScaOperationalResult), + } + // Get applicability status counts for _, scan := range results.Scans { if scan.Vulnerabilities != nil { @@ -44,15 +49,15 @@ func ValidateSummaryIssuesCount(t *testing.T, params ValidationParams, results f for status, count := range counts { switch status { case jasutils.Applicable.String(): - applicableVulnerabilitiesResults += count + actualValues.ApplicableVulnerabilities += count case jasutils.ApplicabilityUndetermined.String(): - undeterminedVulnerabilitiesResults += count + actualValues.UndeterminedVulnerabilities += count case jasutils.NotCovered.String(): - notCoveredVulnerabilitiesResults += count + actualValues.NotCoveredVulnerabilities += count case jasutils.NotApplicable.String(): - notApplicableVulnerabilitiesResults += count + actualValues.NotApplicableVulnerabilities += count case jasutils.MissingContext.String(): - missingContextVulnerabilitiesResults += count + actualValues.MissingContextVulnerabilities += count } } } @@ -64,15 +69,15 @@ func ValidateSummaryIssuesCount(t *testing.T, params ValidationParams, results f for status, count := range counts { switch status { case jasutils.Applicable.String(): - applicableViolationsResults += count + actualValues.ApplicableViolations += count case jasutils.ApplicabilityUndetermined.String(): - undeterminedViolationsResults += count + actualValues.UndeterminedViolations += count case jasutils.NotCovered.String(): - notCoveredViolationsResults += count + actualValues.NotCoveredViolations += count case jasutils.NotApplicable.String(): - notApplicableViolationsResults += count + actualValues.NotApplicableViolations += count case jasutils.MissingContext.String(): - missingContextViolationsResults += count + actualValues.MissingContextViolations += count } } } @@ -80,32 +85,5 @@ func ValidateSummaryIssuesCount(t *testing.T, params ValidationParams, results f } } - ValidateContent(t, params.ExactResultsMatch, - // Total - CountValidation[int]{Expected: params.Vulnerabilities, Actual: vulnerabilities, Msg: GetValidationCountErrMsg("vulnerabilities", "summary", params.ExactResultsMatch, params.Vulnerabilities, vulnerabilities)}, - CountValidation[int]{Expected: params.Violations, Actual: violations, Msg: GetValidationCountErrMsg("violations", "summary", params.ExactResultsMatch, params.Violations, violations)}, - // Jas Vulnerabilities - CountValidation[int]{Expected: params.SastVulnerabilities, Actual: sastVulnerabilities, Msg: GetValidationCountErrMsg("sast vulnerabilities", "summary", params.ExactResultsMatch, params.SastVulnerabilities, sastVulnerabilities)}, - CountValidation[int]{Expected: params.SecretsVulnerabilities, Actual: secretsVulnerabilities, Msg: GetValidationCountErrMsg("secrets vulnerabilities", "summary", params.ExactResultsMatch, params.SecretsVulnerabilities, secretsVulnerabilities)}, - CountValidation[int]{Expected: params.IacVulnerabilities, Actual: iacVulnerabilities, Msg: GetValidationCountErrMsg("IaC vulnerabilities", "summary", params.ExactResultsMatch, params.IacVulnerabilities, iacVulnerabilities)}, - // Jas Violations - CountValidation[int]{Expected: params.SastViolations, Actual: sastViolations, Msg: GetValidationCountErrMsg("sast violations", "summary", params.ExactResultsMatch, params.SastViolations, sastViolations)}, - CountValidation[int]{Expected: params.SecretsViolations, Actual: secretsViolations, Msg: GetValidationCountErrMsg("secrets violations", "summary", params.ExactResultsMatch, params.SecretsViolations, secretsViolations)}, - CountValidation[int]{Expected: params.IacViolations, Actual: iacViolations, Msg: GetValidationCountErrMsg("IaC violations", "summary", params.ExactResultsMatch, params.IacViolations, iacViolations)}, - // Sca vulnerabilities - CountValidation[int]{Expected: params.ApplicableVulnerabilities, Actual: applicableVulnerabilitiesResults, Msg: GetValidationCountErrMsg("applicable vulnerabilities", "summary", params.ExactResultsMatch, params.ApplicableVulnerabilities, applicableVulnerabilitiesResults)}, - CountValidation[int]{Expected: params.UndeterminedVulnerabilities, Actual: undeterminedVulnerabilitiesResults, Msg: GetValidationCountErrMsg("undetermined vulnerabilities", "summary", params.ExactResultsMatch, params.UndeterminedVulnerabilities, undeterminedVulnerabilitiesResults)}, - CountValidation[int]{Expected: params.NotCoveredVulnerabilities, Actual: notCoveredVulnerabilitiesResults, Msg: GetValidationCountErrMsg("not covered vulnerabilities", "summary", params.ExactResultsMatch, params.NotCoveredVulnerabilities, notCoveredVulnerabilitiesResults)}, - CountValidation[int]{Expected: params.NotApplicableVulnerabilities, Actual: notApplicableVulnerabilitiesResults, Msg: GetValidationCountErrMsg("not applicable vulnerabilities", "summary", params.ExactResultsMatch, params.NotApplicableVulnerabilities, notApplicableVulnerabilitiesResults)}, - CountValidation[int]{Expected: params.MissingContextVulnerabilities, Actual: missingContextVulnerabilitiesResults, Msg: GetValidationCountErrMsg("missing context vulnerabilities", "summary", params.ExactResultsMatch, params.MissingContextVulnerabilities, missingContextVulnerabilitiesResults)}, - // Sca violations - CountValidation[int]{Expected: params.ApplicableViolations, Actual: applicableViolationsResults, Msg: GetValidationCountErrMsg("applicable violations", "summary", params.ExactResultsMatch, params.ApplicableViolations, applicableViolationsResults)}, - CountValidation[int]{Expected: params.UndeterminedViolations, Actual: undeterminedViolationsResults, Msg: GetValidationCountErrMsg("undetermined violations", "summary", params.ExactResultsMatch, params.UndeterminedViolations, undeterminedViolationsResults)}, - CountValidation[int]{Expected: params.NotCoveredViolations, Actual: notCoveredViolationsResults, Msg: GetValidationCountErrMsg("not covered violations", "summary", params.ExactResultsMatch, params.NotCoveredViolations, notCoveredViolationsResults)}, - CountValidation[int]{Expected: params.NotApplicableViolations, Actual: notApplicableViolationsResults, Msg: GetValidationCountErrMsg("not applicable violations", "summary", params.ExactResultsMatch, params.NotApplicableViolations, notApplicableViolationsResults)}, - CountValidation[int]{Expected: params.MissingContextViolations, Actual: missingContextViolationsResults, Msg: GetValidationCountErrMsg("missing context violations", "summary", params.ExactResultsMatch, params.MissingContextViolations, missingContextViolationsResults)}, - CountValidation[int]{Expected: params.ScaSecurityViolations, Actual: securityViolations, Msg: GetValidationCountErrMsg("security violations", "summary", params.ExactResultsMatch, params.ScaSecurityViolations, securityViolations)}, - CountValidation[int]{Expected: params.LicenseViolations, Actual: licenseViolations, Msg: GetValidationCountErrMsg("license violations", "summary", params.ExactResultsMatch, params.LicenseViolations, licenseViolations)}, - CountValidation[int]{Expected: params.OperationalViolations, Actual: opRiskViolations, Msg: GetValidationCountErrMsg("operational risk violations", "summary", params.ExactResultsMatch, params.OperationalViolations, opRiskViolations)}, - ) + ValidateCount(t, "summary", params, actualValues) } diff --git a/utils/validations/test_validation.go b/utils/validations/test_validation.go index 4e065f5c..09a7ac26 100644 --- a/utils/validations/test_validation.go +++ b/utils/validations/test_validation.go @@ -28,54 +28,121 @@ type ValidationParams struct { Expected interface{} // If provided, the test will check exact values and not only the minimum values / existence. ExactResultsMatch bool + + // Validate total number of licenses, vulnerabilities and violations + Total *TotalCount + // Validate number of vulnerabilities in different contexts + Vulnerabilities *VulnerabilityCount + // Validate number of violations in different contexts + Violations *ViolationCount + + // Expected number of licenses + // Licenses int + // // Expected number of total vulnerabilities (sca + sast + iac + secrets) + // Vulnerabilities int + // // Expected number of total violations (sca security + sca license + sca operational + sast + iac + secrets) + // Violations int + + // // Expected number of contextual statuses for vulnerabilities (sca/secrets) + // ApplicableVulnerabilities int + // // Expected number of contextual statuses for vulnerabilities (sca/secrets) + // UndeterminedVulnerabilities int + // // Expected number of contextual statuses for vulnerabilities (sca/secrets) + // NotCoveredVulnerabilities int + // // Expected number of contextual statuses for vulnerabilities (sca/secrets) + // NotApplicableVulnerabilities int + // // Expected number of contextual statuses for vulnerabilities (sca/secrets) + // MissingContextVulnerabilities int + // // Expected number of contextual statuses for vulnerabilities (sca/secrets) + // InactiveVulnerabilities int + // // Expected number of contextual statuses for violations (sca/secrets) + // ApplicableViolations int + // // Expected number of contextual statuses for violations (sca/secrets) + // UndeterminedViolations int + // // Expected number of contextual statuses for violations (sca/secrets) + // NotCoveredViolations int + // // Expected number of contextual statuses for violations (sca/secrets) + // NotApplicableViolations int + // // Expected number of contextual statuses for violations (sca/secrets) + // MissingContextViolations int + // // Expected number of contextual statuses for violations (sca/secrets) + // InactiveViolations int + // // Expected number of sca violations by type + // ScaSecurityViolations int + // // Expected number of sca violations by type + // LicenseViolations int + // // Expected number of sca violations by type + // OperationalViolations int + // // Expected number of Jas vulnerabilities + // SastVulnerabilities int + // // Expected number of Jas vulnerabilities + // IacVulnerabilities int + // // Expected number of Jas vulnerabilities + // SecretsVulnerabilities int + // // Expected number of Jas violations + // SastViolations int + // // Expected number of Jas violations + // IacViolations int + // // Expected number of Jas violations + // SecretsViolations int +} + +type TotalCount struct { // Expected number of licenses Licenses int // Expected number of total vulnerabilities (sca + sast + iac + secrets) Vulnerabilities int // Expected number of total violations (sca security + sca license + sca operational + sast + iac + secrets) Violations int - // Expected number of contextual statuses for vulnerabilities (sca/secrets) - ApplicableVulnerabilities int - // Expected number of contextual statuses for vulnerabilities (sca/secrets) - UndeterminedVulnerabilities int - // Expected number of contextual statuses for vulnerabilities (sca/secrets) - NotCoveredVulnerabilities int - // Expected number of contextual statuses for vulnerabilities (sca/secrets) - NotApplicableVulnerabilities int - // Expected number of contextual statuses for vulnerabilities (sca/secrets) - MissingContextVulnerabilities int - // Expected number of contextual statuses for vulnerabilities (sca/secrets) - InactiveVulnerabilities int - // Expected number of contextual statuses for violations (sca/secrets) - ApplicableViolations int - // Expected number of contextual statuses for violations (sca/secrets) - UndeterminedViolations int - // Expected number of contextual statuses for violations (sca/secrets) - NotCoveredViolations int - // Expected number of contextual statuses for violations (sca/secrets) - NotApplicableViolations int - // Expected number of contextual statuses for violations (sca/secrets) - MissingContextViolations int +} + +type ScanCount struct { + // Expected number of Sca issues + Sca int + // Expected number of Jas issues + Sast int + // Expected number of Jas issues + Iac int + // Expected number of Jas issues + Secrets int +} + +type VulnerabilityCount struct { + ValidateScan *ScanCount + ValidateApplicabilityStatus *ApplicabilityStatusCount +} + +type ViolationCount struct { + // Expected number of violations by scan type (SCA/JAS) + ValidateScan *ScanCount // Expected number of contextual statuses for violations (sca/secrets) - InactiveViolations int - // Expected number of sca violations by type - ScaSecurityViolations int - // Expected number of sca violations by type - LicenseViolations int - // Expected number of sca violations by type - OperationalViolations int - // Expected number of Jas vulnerabilities - SastVulnerabilities int - // Expected number of Jas vulnerabilities - IacVulnerabilities int - // Expected number of Jas vulnerabilities - SecretsVulnerabilities int - // Expected number of Jas violations - SastViolations int - // Expected number of Jas violations - IacViolations int - // Expected number of Jas violations - SecretsViolations int + ValidateApplicabilityStatus *ApplicabilityStatusCount + // Expected number of violations by violation type (license, operational, security: SCA+JAS) + ValidateType *ScaViolationCount +} + +type ScaViolationCount struct { + // Expected number of security violations (Sca, JAS) + Security int + // Expected number of license violations + License int + // Expected number of operational violations + Operational int +} + +type ApplicabilityStatusCount struct { + // Expected number of 'Applicable' contextual-analysis statuses for the issues (sca) + Applicable int + // Expected number of 'Undetermined' contextual-analysis statuses for the issues (sca) + Undetermined int + // Expected number of 'NotCovered' contextual-analysis statuses for the issues (sca) + NotCovered int + // Expected number of 'NotApplicable' contextual-analysis statuses for the issues (sca) + NotApplicable int + // Expected number of 'MissingContext' contextual-analysis statuses for the issues (sca) + MissingContext int + // Expected number of 'Inactive' contextual-analysis statuses for the issues (secrets) + Inactive int } // Validation allows to validate/assert a content with expected values. @@ -245,3 +312,94 @@ func ValidateContent(t *testing.T, exactMatch bool, validations ...Validation) b } return validationSuccess } + +type validationCountActualValues struct { + // Total counts + Vulnerabilities, Violations, Licenses int + // Vulnerabilities counts + SastVulnerabilities, SecretsVulnerabilities, IacVulnerabilities, ScaVulnerabilities int + ApplicableVulnerabilities, UndeterminedVulnerabilities, NotCoveredVulnerabilities, NotApplicableVulnerabilities, MissingContextVulnerabilities, InactiveSecretsVulnerabilities int + // Violations counts + SastViolations, SecretsViolations, IacViolations, ScaViolations int + SecurityViolations, LicenseViolations, OperationalViolations int + ApplicableViolations, UndeterminedViolations, NotCoveredViolations, NotApplicableViolations, MissingContextViolations, InactiveSecretsViolations int +} + +func ValidateCount(t *testing.T, outputType string, params ValidationParams, actual validationCountActualValues) { + ValidateTotalCount(t, outputType, params.ExactResultsMatch, params.Total, actual.Vulnerabilities, actual.Violations, actual.Licenses) + ValidateVulnerabilitiesCount(t, outputType, params.ExactResultsMatch, params.Vulnerabilities, actual) + ValidateViolationCount(t, outputType, params.ExactResultsMatch, params.Violations, actual) +} + +func ValidateTotalCount(t *testing.T, outputType string, exactMatch bool, params *TotalCount, vulnerabilities, violations, license int) { + if params == nil { + return + } + ValidateContent(t, exactMatch, + CountValidation[int]{Expected: params.Vulnerabilities, Actual: vulnerabilities, Msg: GetValidationCountErrMsg("vulnerabilities", outputType, exactMatch, params.Vulnerabilities, vulnerabilities)}, + CountValidation[int]{Expected: params.Violations, Actual: violations, Msg: GetValidationCountErrMsg("violations", outputType, exactMatch, params.Violations, violations)}, + CountValidation[int]{Expected: params.Licenses, Actual: license, Msg: GetValidationCountErrMsg("licenses", outputType, exactMatch, params.Licenses, license)}, + ) +} + +func ValidateVulnerabilitiesCount(t *testing.T, outputType string, exactMatch bool, params *VulnerabilityCount, actual validationCountActualValues) { + if params == nil { + return + } + ValidateScanTypeCount(t, outputType, false, exactMatch, params.ValidateScan, actual.ScaVulnerabilities, actual.SastVulnerabilities, actual.SecretsVulnerabilities, actual.IacVulnerabilities) + ValidateApplicabilityStatusCount(t, outputType, false, exactMatch, params.ValidateApplicabilityStatus, actual.ApplicableVulnerabilities, actual.UndeterminedVulnerabilities, actual.NotCoveredVulnerabilities, actual.NotApplicableVulnerabilities, actual.MissingContextVulnerabilities, actual.InactiveSecretsVulnerabilities) +} + +func ValidateViolationCount(t *testing.T, outputType string, exactMatch bool, params *ViolationCount, actual validationCountActualValues) { + if params == nil { + return + } + ValidateScanTypeCount(t, outputType, true, exactMatch, params.ValidateScan, actual.ScaViolations, actual.SastViolations, actual.SecretsViolations, actual.IacViolations) + ValidateApplicabilityStatusCount(t, outputType, true, exactMatch, params.ValidateApplicabilityStatus, actual.ApplicableViolations, actual.UndeterminedViolations, actual.NotCoveredViolations, actual.NotApplicableViolations, actual.MissingContextViolations, actual.InactiveSecretsViolations) + ValidateScaViolationCount(t, outputType, exactMatch, params.ValidateType, actual.SecurityViolations, actual.LicenseViolations, actual.OperationalViolations) +} + +func ValidateScanTypeCount(t *testing.T, outputType string, violation, exactMatch bool, params *ScanCount, scaViolations, sastViolations, secretsViolations, iacViolations int) { + if params == nil { + return + } + suffix := "vulnerabilities" + if violation { + suffix = "violations" + } + ValidateContent(t, exactMatch, + CountValidation[int]{Expected: params.Sast, Actual: sastViolations, Msg: GetValidationCountErrMsg(fmt.Sprintf("sast %s", suffix), outputType, exactMatch, params.Sast, sastViolations)}, + CountValidation[int]{Expected: params.Secrets, Actual: secretsViolations, Msg: GetValidationCountErrMsg(fmt.Sprintf("secrets %s", suffix), outputType, exactMatch, params.Secrets, secretsViolations)}, + CountValidation[int]{Expected: params.Iac, Actual: iacViolations, Msg: GetValidationCountErrMsg(fmt.Sprintf("IaC %s", suffix), outputType, exactMatch, params.Iac, iacViolations)}, + CountValidation[int]{Expected: params.Sca, Actual: scaViolations, Msg: GetValidationCountErrMsg(fmt.Sprintf("Sca %s", suffix), outputType, exactMatch, params.Sca, scaViolations)}, + ) +} + +func ValidateApplicabilityStatusCount(t *testing.T, outputType string, violation, exactMatch bool, params *ApplicabilityStatusCount, applicableResults, undeterminedResults, notCoveredResults, notApplicableResults, missingContextResults, inactiveSecrets int) { + if params == nil { + return + } + suffix := "vulnerabilities" + if violation { + suffix = "violations" + } + ValidateContent(t, exactMatch, + CountValidation[int]{Expected: params.Applicable, Actual: applicableResults, Msg: GetValidationCountErrMsg(fmt.Sprintf("applicable %s", suffix), outputType, exactMatch, params.Applicable, applicableResults)}, + CountValidation[int]{Expected: params.Undetermined, Actual: undeterminedResults, Msg: GetValidationCountErrMsg(fmt.Sprintf("undetermined %s", suffix), outputType, exactMatch, params.Undetermined, undeterminedResults)}, + CountValidation[int]{Expected: params.NotCovered, Actual: notCoveredResults, Msg: GetValidationCountErrMsg(fmt.Sprintf("not covered %s", suffix), outputType, exactMatch, params.NotCovered, notCoveredResults)}, + CountValidation[int]{Expected: params.NotApplicable, Actual: notApplicableResults, Msg: GetValidationCountErrMsg(fmt.Sprintf("not applicable %s", suffix), outputType, exactMatch, params.NotApplicable, notApplicableResults)}, + CountValidation[int]{Expected: params.MissingContext, Actual: missingContextResults, Msg: GetValidationCountErrMsg(fmt.Sprintf("missing context %s", suffix), outputType, exactMatch, params.MissingContext, missingContextResults)}, + CountValidation[int]{Expected: params.Inactive, Actual: inactiveSecrets, Msg: GetValidationCountErrMsg(fmt.Sprintf("inactive secrets %s", suffix), outputType, exactMatch, params.Inactive, inactiveSecrets)}, + ) +} + +func ValidateScaViolationCount(t *testing.T, outputType string, exactMatch bool, params *ScaViolationCount, securityViolations, licenseViolations, operationalViolations int) { + if params == nil { + return + } + ValidateContent(t, exactMatch, + CountValidation[int]{Expected: params.Security, Actual: securityViolations, Msg: GetValidationCountErrMsg("security violations", outputType, exactMatch, params.Security, securityViolations)}, + CountValidation[int]{Expected: params.License, Actual: licenseViolations, Msg: GetValidationCountErrMsg("license violations", outputType, exactMatch, params.License, licenseViolations)}, + CountValidation[int]{Expected: params.Operational, Actual: operationalViolations, Msg: GetValidationCountErrMsg("operational risk violations", outputType, exactMatch, params.Operational, operationalViolations)}, + ) +} diff --git a/xsc_test.go b/xsc_test.go index 2933f44b..7d5264cd 100644 --- a/xsc_test.go +++ b/xsc_test.go @@ -33,8 +33,7 @@ func TestXscAuditNpmJsonWithWatch(t *testing.T) { defer cleanUp() output := testAuditNpm(t, string(format.Json), false) validations.VerifyJsonResults(t, output, validations.ValidationParams{ - ScaSecurityViolations: 1, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Violations: 1}, }) } @@ -43,9 +42,7 @@ func TestXscAuditNpmSimpleJsonWithWatch(t *testing.T) { defer cleanUp() output := testAuditNpm(t, string(format.SimpleJson), true) validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - ScaSecurityViolations: 1, - Vulnerabilities: 1, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Violations: 1, Vulnerabilities: 1}, }) } @@ -54,8 +51,7 @@ func TestXscAuditMavenJson(t *testing.T) { defer cleanUp() output := testAuditMaven(t, string(format.Json)) validations.VerifyJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 1, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1}, }) } @@ -64,8 +60,7 @@ func TestXscAuditMavenSimpleJson(t *testing.T) { defer cleanUp() output := testAuditMaven(t, string(format.SimpleJson)) validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{ - Vulnerabilities: 1, - Licenses: 1, + Total: &validations.TotalCount{Licenses: 1, Vulnerabilities: 1}, }) }