Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move skip tests calls #254

Merged
merged 1 commit into from
Dec 1, 2024
+63 −23
Merged

Move skip tests calls #254

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 59 additions & 22 deletions audit_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ func testAuditNpm(t *testing.T, format string, withVuln bool) string {
}

func TestXrayAuditConanJson(t *testing.T) {
integration.InitAuditCTest(t, scangraph.GraphScanMinXrayVersion)
output := testAuditConan(t, string(format.Json), true)
validations.VerifyJsonResults(t, output, validations.ValidationParams{
Vulnerabilities: 8,
Expand All @@ -81,6 +82,7 @@ func TestXrayAuditConanJson(t *testing.T) {
}

func TestXrayAuditConanSimpleJson(t *testing.T) {
integration.InitAuditCTest(t, scangraph.GraphScanMinXrayVersion)
output := testAuditConan(t, string(format.SimpleJson), true)
validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
Vulnerabilities: 8,
Expand All @@ -89,7 +91,6 @@ func TestXrayAuditConanSimpleJson(t *testing.T) {
}

func testAuditConan(t *testing.T, format string, withVuln bool) string {
integration.InitAuditCTest(t, scangraph.GraphScanMinXrayVersion)
_, cleanUp := securityTestUtils.CreateTestProjectEnvAndChdir(t, filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "projects", "package-managers", "conan"))
defer cleanUp()
// Run conan install before executing jfrog audit
Expand All @@ -104,6 +105,7 @@ func testAuditConan(t *testing.T, format string, withVuln bool) string {
}

func TestXrayAuditPnpmJson(t *testing.T) {
integration.InitAuditJavaScriptTest(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditPnpm(t, string(format.Json))
validations.VerifyJsonResults(t, output, validations.ValidationParams{
Vulnerabilities: 1,
Expand All @@ -112,6 +114,7 @@ func TestXrayAuditPnpmJson(t *testing.T) {
}

func TestXrayAuditPnpmSimpleJson(t *testing.T) {
integration.InitAuditJavaScriptTest(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditPnpm(t, string(format.SimpleJson))
validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
Vulnerabilities: 1,
Expand All @@ -120,7 +123,6 @@ func TestXrayAuditPnpmSimpleJson(t *testing.T) {
}

func testXrayAuditPnpm(t *testing.T, format string) string {
integration.InitAuditJavaScriptTest(t, scangraph.GraphScanMinXrayVersion)
_, cleanUp := securityTestUtils.CreateTestProjectEnvAndChdir(t, filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "projects", "package-managers", "npm", "npm-no-lock"))
defer cleanUp()
// Run pnpm install before executing audit
Expand All @@ -131,6 +133,7 @@ func testXrayAuditPnpm(t *testing.T, format string) string {
}

func TestXrayAuditYarnV2Json(t *testing.T) {
integration.InitAuditJavaScriptTest(t, scangraph.GraphScanMinXrayVersion)
testXrayAuditYarn(t, "yarn-v2", func() {
output := runXrayAuditYarnWithOutput(t, string(format.Json))
validations.VerifyJsonResults(t, output, validations.ValidationParams{
Expand All @@ -141,6 +144,7 @@ func TestXrayAuditYarnV2Json(t *testing.T) {
}

func TestXrayAuditYarnV2SimpleJson(t *testing.T) {
integration.InitAuditJavaScriptTest(t, scangraph.GraphScanMinXrayVersion)
testXrayAuditYarn(t, "yarn-v3", func() {
output := runXrayAuditYarnWithOutput(t, string(format.SimpleJson))
validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
Expand All @@ -151,6 +155,7 @@ func TestXrayAuditYarnV2SimpleJson(t *testing.T) {
}

func TestXrayAuditYarnV1Json(t *testing.T) {
integration.InitAuditJavaScriptTest(t, scangraph.GraphScanMinXrayVersion)
testXrayAuditYarn(t, "yarn-v1", func() {
output := runXrayAuditYarnWithOutput(t, string(format.Json))
validations.VerifyJsonResults(t, output, validations.ValidationParams{
Expand All @@ -161,6 +166,7 @@ func TestXrayAuditYarnV1Json(t *testing.T) {
}

func TestXrayAuditYarnV1JsonWithoutDevDependencies(t *testing.T) {
integration.InitAuditJavaScriptTest(t, scangraph.GraphScanMinXrayVersion)
unsetEnv := clientTests.SetEnvWithCallbackAndAssert(t, "NODE_ENV", "production")
defer unsetEnv()
testXrayAuditYarn(t, "yarn-v1", func() {
Expand All @@ -173,6 +179,7 @@ func TestXrayAuditYarnV1JsonWithoutDevDependencies(t *testing.T) {
}

func TestXrayAuditYarnV1SimpleJson(t *testing.T) {
integration.InitAuditJavaScriptTest(t, scangraph.GraphScanMinXrayVersion)
testXrayAuditYarn(t, "yarn-v1", func() {
output := runXrayAuditYarnWithOutput(t, string(format.SimpleJson))
validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
Expand All @@ -183,7 +190,6 @@ func TestXrayAuditYarnV1SimpleJson(t *testing.T) {
}

func testXrayAuditYarn(t *testing.T, projectDirName string, yarnCmd func()) {
integration.InitAuditJavaScriptTest(t, scangraph.GraphScanMinXrayVersion)
_, cleanUp := securityTestUtils.CreateTestProjectEnvAndChdir(t, filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "projects", "package-managers", "yarn", projectDirName))
defer cleanUp()
// Run yarn install before executing jf audit --yarn. Return error to assert according to test.
Expand All @@ -199,6 +205,7 @@ func runXrayAuditYarnWithOutput(t *testing.T, format string) string {

// Tests NuGet audit by providing simple NuGet project + multi-project NuGet project and asserts any error.
func TestXrayAuditNugetJson(t *testing.T) {
integration.InitAuditCTest(t, scangraph.GraphScanMinXrayVersion)
var testdata = []struct {
projectName string
format string
Expand Down Expand Up @@ -256,6 +263,7 @@ func TestXrayAuditNugetJson(t *testing.T) {
}

func TestXrayAuditNugetSimpleJson(t *testing.T) {
integration.InitAuditCTest(t, scangraph.GraphScanMinXrayVersion)
var testdata = []struct {
projectName string
format string
Expand Down Expand Up @@ -299,7 +307,6 @@ func TestXrayAuditNugetSimpleJson(t *testing.T) {
}

func testXrayAuditNuget(t *testing.T, projectName, format string, restoreTech string) string {
integration.InitAuditCTest(t, scangraph.GraphScanMinXrayVersion)
_, cleanUp := securityTestUtils.CreateTestProjectEnvAndChdir(t, filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "projects", "package-managers", "nuget", projectName))
defer cleanUp()
// Add dummy descriptor file to check that we run only specific audit
Expand All @@ -313,6 +320,7 @@ func testXrayAuditNuget(t *testing.T, projectName, format string, restoreTech st
}

func TestXrayAuditGradleJson(t *testing.T) {
integration.InitAuditJavaTest(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditGradle(t, string(format.Json))
validations.VerifyJsonResults(t, output, validations.ValidationParams{
Vulnerabilities: 3,
Expand All @@ -321,6 +329,7 @@ func TestXrayAuditGradleJson(t *testing.T) {
}

func TestXrayAuditGradleSimpleJson(t *testing.T) {
integration.InitAuditJavaTest(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditGradle(t, string(format.SimpleJson))
validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
Vulnerabilities: 3,
Expand All @@ -329,7 +338,6 @@ func TestXrayAuditGradleSimpleJson(t *testing.T) {
}

func testXrayAuditGradle(t *testing.T, format string) string {
integration.InitAuditJavaTest(t, scangraph.GraphScanMinXrayVersion)
_, cleanUp := securityTestUtils.CreateTestProjectEnvAndChdir(t, filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "projects", "package-managers", "gradle", "gradle"))
defer cleanUp()
// Add dummy descriptor file to check that we run only specific audit
Expand Down Expand Up @@ -364,17 +372,18 @@ func testAuditMaven(t *testing.T, format string) string {
}

func TestXrayAuditGoJson(t *testing.T) {
integration.InitAuditGoTest(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditGo(t, false, string(format.Json), "simple-project")
validations.VerifyJsonResults(t, output, validations.ValidationParams{Licenses: 1, Vulnerabilities: 4})
}

func TestXrayAuditGoSimpleJson(t *testing.T) {
integration.InitAuditGoTest(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditGo(t, true, string(format.SimpleJson), "simple-project")
validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{Licenses: 3, Vulnerabilities: 4, NotCovered: 2, NotApplicable: 2})
}

func testXrayAuditGo(t *testing.T, noCreds bool, format, project string) string {
integration.InitAuditGoTest(t, scangraph.GraphScanMinXrayVersion)
_, cleanUp := securityTestUtils.CreateTestProjectEnvAndChdir(t, filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "projects", "package-managers", "go", project))
defer cleanUp()
// Add dummy descriptor file to check that we run only specific audit
Expand Down Expand Up @@ -429,6 +438,7 @@ func TestXrayAuditMultiProjects(t *testing.T) {
}

func TestXrayAuditPipJson(t *testing.T) {
integration.InitAuditPythonTest(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditPip(t, string(format.Json), "")
validations.VerifyJsonResults(t, output, validations.ValidationParams{
Vulnerabilities: 3,
Expand All @@ -437,13 +447,15 @@ func TestXrayAuditPipJson(t *testing.T) {
}

func TestXrayAuditCocoapods(t *testing.T) {
integration.InitAuditCocoapodsTest(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditCocoapods(t, string(format.Json))
validations.VerifyJsonResults(t, output, validations.ValidationParams{
Vulnerabilities: 1,
})
}

func TestXrayAuditPipSimpleJson(t *testing.T) {
integration.InitAuditPythonTest(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditPip(t, string(format.SimpleJson), "")
validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
Vulnerabilities: 3,
Expand All @@ -452,17 +464,18 @@ func TestXrayAuditPipSimpleJson(t *testing.T) {
}

func TestXrayAuditPipJsonWithRequirementsFile(t *testing.T) {
integration.InitAuditPythonTest(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditPip(t, string(format.Json), "requirements.txt")
validations.VerifyJsonResults(t, output, validations.ValidationParams{Vulnerabilities: 2})
}

func TestXrayAuditPipSimpleJsonWithRequirementsFile(t *testing.T) {
integration.InitAuditPythonTest(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditPip(t, string(format.SimpleJson), "requirements.txt")
validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{Vulnerabilities: 2})
}

func testXrayAuditPip(t *testing.T, format, requirementsFile string) string {
integration.InitAuditPythonTest(t, scangraph.GraphScanMinXrayVersion)
_, cleanUp := securityTestUtils.CreateTestProjectEnvAndChdir(t, filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "projects", "package-managers", "python", "pip", "pip-project"))
defer cleanUp()
// Add dummy descriptor file to check that we run only specific audit
Expand All @@ -475,7 +488,6 @@ func testXrayAuditPip(t *testing.T, format, requirementsFile string) string {
}

func testXrayAuditCocoapods(t *testing.T, format string) string {
integration.InitAuditCocoapodsTest(t, scangraph.GraphScanMinXrayVersion)
_, cleanUp := securityTestUtils.CreateTestProjectEnvAndChdir(t, filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "projects", "package-managers", "cocoapods"))
defer cleanUp()
// Add dummy descriptor file to check that we run only specific audit
Expand All @@ -484,6 +496,7 @@ func testXrayAuditCocoapods(t *testing.T, format string) string {
}

func TestXrayAuditPipenvJson(t *testing.T) {
integration.InitAuditPythonTest(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditPipenv(t, string(format.Json))
validations.VerifyJsonResults(t, output, validations.ValidationParams{
Vulnerabilities: 3,
Expand All @@ -492,6 +505,7 @@ func TestXrayAuditPipenvJson(t *testing.T) {
}

func TestXrayAuditPipenvSimpleJson(t *testing.T) {
integration.InitAuditPythonTest(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditPipenv(t, string(format.SimpleJson))
validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
Vulnerabilities: 3,
Expand All @@ -500,7 +514,6 @@ func TestXrayAuditPipenvSimpleJson(t *testing.T) {
}

func testXrayAuditPipenv(t *testing.T, format string) string {
integration.InitAuditPythonTest(t, scangraph.GraphScanMinXrayVersion)
_, cleanUp := securityTestUtils.CreateTestProjectEnvAndChdir(t, filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "projects", "package-managers", "python", "pipenv", "pipenv-project"))
defer cleanUp()
// Add dummy descriptor file to check that we run only specific audit
Expand All @@ -509,6 +522,7 @@ func testXrayAuditPipenv(t *testing.T, format string) string {
}

func TestXrayAuditPoetryJson(t *testing.T) {
integration.InitAuditPythonTest(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditPoetry(t, string(format.Json))
validations.VerifyJsonResults(t, output, validations.ValidationParams{
Vulnerabilities: 3,
Expand All @@ -517,6 +531,7 @@ func TestXrayAuditPoetryJson(t *testing.T) {
}

func TestXrayAuditPoetrySimpleJson(t *testing.T) {
integration.InitAuditPythonTest(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditPoetry(t, string(format.SimpleJson))
validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
Vulnerabilities: 3,
Expand All @@ -525,7 +540,6 @@ func TestXrayAuditPoetrySimpleJson(t *testing.T) {
}

func testXrayAuditPoetry(t *testing.T, format string) string {
integration.InitAuditPythonTest(t, scangraph.GraphScanMinXrayVersion)
_, cleanUp := securityTestUtils.CreateTestProjectEnvAndChdir(t, filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), "projects", "package-managers", "python", "poetry", "poetry-project"))
defer cleanUp()
// Add dummy descriptor file to check that we run only specific audit
Expand All @@ -546,29 +560,46 @@ func addDummyPackageDescriptor(t *testing.T, hasPackageJson bool) {
// JAS

func TestXrayAuditSastCppFlagSimpleJson(t *testing.T) {
output := testXrayAuditJas(t, securityTests.PlatformCli, filepath.Join("package-managers", "c"), "3", false, true, false, "")
validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
Vulnerabilities: 1,
Sast: 1,
})
integration.InitAuditJasTest(t, scangraph.GraphScanMinXrayVersion)
testCase := []struct {
name string
withFlag bool
expectedResults validations.ValidationParams
}{
{
name: "withFlag",
withFlag: true,
expectedResults: validations.ValidationParams{
Vulnerabilities: 1,
Sast: 1,
},
},
{
name: "withoutFlag",
withFlag: false,
expectedResults: validations.ValidationParams{},
},
}
for _, tc := range testCase {
t.Run(tc.name, func(t *testing.T) {
output := testXrayAuditJas(t, securityTests.PlatformCli, filepath.Join("package-managers", "c"), "3", false, tc.withFlag, false, "")
validations.VerifySimpleJsonResults(t, output, tc.expectedResults)
})
}
}
func TestXrayAuditSastCSharpFlagSimpleJson(t *testing.T) {
// Placeholder until C# Sast is implemented
t.Skip()
integration.InitAuditJasTest(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditJas(t, securityTests.PlatformCli, filepath.Join("package-managers", "dotnet", "dotnet-single"), "3", false, false, true, "")
validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
Vulnerabilities: 1,
Sast: 1,
})
}

func TestXrayAuditWithoutSastCppFlagSimpleJson(t *testing.T) {
output := testXrayAuditJas(t, securityTests.PlatformCli, filepath.Join("package-managers", "c"), "3", false, false, false, "")
// verify no results for Sast
validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{})
}

func TestXrayAuditJasMissingContextSimpleJson(t *testing.T) {
integration.InitAuditJasTest(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditJas(t, securityTests.PlatformCli, filepath.Join("package-managers", "maven", "missing-context"), "3", false, false, false, "")
validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{MissingContext: 1})
}
Expand Down Expand Up @@ -598,6 +629,7 @@ func getNoJasAuditMockCommand() components.Command {
}

func TestXrayAuditJasSimpleJson(t *testing.T) {
integration.InitAuditGeneralTests(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditJas(t, securityTests.PlatformCli, filepath.Join("jas", "jas"), "3", false, false, false, "")
validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
Sast: 1,
Expand All @@ -619,6 +651,7 @@ func TestXrayAuditJasSimpleJsonWithTokenValidation(t *testing.T) {
}

func TestXrayAuditJasSimpleJsonWithOneThread(t *testing.T) {
integration.InitAuditGeneralTests(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditJas(t, securityTests.PlatformCli, filepath.Join("jas", "jas"), "1", false, false, false, "")
validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
Sast: 1,
Expand All @@ -634,6 +667,7 @@ func TestXrayAuditJasSimpleJsonWithOneThread(t *testing.T) {
}

func TestXrayAuditJasSimpleJsonWithConfig(t *testing.T) {
integration.InitAuditGeneralTests(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditJas(t, securityTests.PlatformCli, filepath.Join("jas", "jas-config"), "3", false, false, false, "")
validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
Secrets: 1,
Expand All @@ -647,12 +681,12 @@ func TestXrayAuditJasSimpleJsonWithConfig(t *testing.T) {
}

func TestXrayAuditJasNoViolationsSimpleJson(t *testing.T) {
integration.InitAuditGeneralTests(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditJas(t, securityTests.PlatformCli, filepath.Join("package-managers", "npm", "npm"), "3", false, false, false, "")
validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{Vulnerabilities: 1, NotApplicable: 1})
}

func testXrayAuditJas(t *testing.T, testCli *coreTests.JfrogCli, project string, threads string, validateSecrets bool, validateSastCpp bool, validateSastCSharp bool, customExclusion string) string {
integration.InitAuditGeneralTests(t, scangraph.GraphScanMinXrayVersion)
_, cleanUp := securityTestUtils.CreateTestProjectEnvAndChdir(t, filepath.Join(filepath.FromSlash(securityTests.GetTestResourcesPath()), filepath.Join("projects", project)))
defer cleanUp()
// Configure a new server named "default"
Expand Down Expand Up @@ -730,6 +764,7 @@ func TestAuditOnEmptyProject(t *testing.T) {
// xray-url only - the following tests check the case of adding "xray-url", instead of "url", which is the more common one

func TestXrayAuditNotEntitledForJasWithXrayUrl(t *testing.T) {
integration.InitAuditGeneralTests(t, scangraph.GraphScanMinXrayVersion)
cliToRun, cleanUp := integration.InitTestWithMockCommandOrParams(t, true, getNoJasAuditMockCommand)
defer cleanUp()
output := testXrayAuditJas(t, cliToRun, filepath.Join("jas", "jas"), "3", false, false, false, "")
Expand All @@ -740,6 +775,7 @@ func TestXrayAuditNotEntitledForJasWithXrayUrl(t *testing.T) {
}

func TestXrayAuditJasSimpleJsonWithXrayUrl(t *testing.T) {
integration.InitAuditGeneralTests(t, scangraph.GraphScanMinXrayVersion)
cliToRun := integration.GetTestCli(cli.GetJfrogCliSecurityApp(), true)
output := testXrayAuditJas(t, cliToRun, filepath.Join("jas", "jas"), "3", false, false, false, "")
validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
Expand All @@ -758,6 +794,7 @@ func TestXrayAuditJasSimpleJsonWithXrayUrl(t *testing.T) {
// custom excluded folders

func TestXrayAuditJasSimpleJsonWithCustomExclusions(t *testing.T) {
integration.InitAuditJasTest(t, scangraph.GraphScanMinXrayVersion)
output := testXrayAuditJas(t, securityTests.PlatformCli, filepath.Join("jas", "jas"), "3", false, false, false, "non_existing_folder")
validations.VerifySimpleJsonResults(t, output, validations.ValidationParams{
Sast: 2,
Expand Down
Loading
Loading