feat(scanner): add '--packages-depth' parameter.

The '--packages-depth' parameter allows not to scan the whole dependency tree. For example, if you use 'scan --packages-depth 1' then the scan will be performed on projects and the first level of packages (i.e. the direct dependencies). By default, the value is -1, which means no limit, all packages will be scanned. Signed-off-by: François Barbe <[email protected]>
oss-review-toolkit · Mar 4, 2024 · b28abce · b28abce
1 parent 13b39ef
commit b28abce
Show file tree

Hide file tree

Showing 5 changed files with 730 additions and 4 deletions.
diff --git a/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt b/plugins/commands/scanner/src/main/kotlin/ScannerCommand.kt
@@ -54,6 +54,7 @@ import org.ossreviewtoolkit.plugins.commands.api.utils.configurationGroup
 import org.ossreviewtoolkit.plugins.commands.api.utils.outputGroup
 import org.ossreviewtoolkit.plugins.commands.api.utils.readOrtResult
 import org.ossreviewtoolkit.plugins.commands.api.utils.writeOrtResult
+import org.ossreviewtoolkit.scanner.NO_LIMIT_DEPTH
 import org.ossreviewtoolkit.scanner.ScanStorages
 import org.ossreviewtoolkit.scanner.Scanner
 import org.ossreviewtoolkit.scanner.ScannerWrapperFactory
@@ -117,6 +118,11 @@ class ScannerCommand : OrtCommand(
         help = "A comma-separated list of the package types from the ORT file's analyzer result to limit scans to."
     ).enum<PackageType>().split(",").default(PackageType.entries)
 
+    private val packagesDepth by option(
+        "--packages-depth",
+        help = "The depth in the packages tree to scan, default is -1 (no limit)"
+    ).convert { it.toInt() }.default(NO_LIMIT_DEPTH)
+
     private val skipExcluded by option(
         "--skip-excluded",
         help = "Do not scan excluded projects or packages. Works only with the '--ort-file' parameter."
@@ -231,7 +237,7 @@ class ScannerCommand : OrtCommand(
 
             val ortResult = readOrtResult(input)
             return runBlocking {
-                scanner.scan(ortResult, skipExcluded || ortConfig.scanner.skipExcluded, labels)
+                scanner.scan(ortResult, skipExcluded || ortConfig.scanner.skipExcluded, labels, packagesDepth)
             }
         } finally {
             runBlocking { workingTreeCache.shutdown() }

diff --git a/scanner/src/funTest/assets/scanner-integration-pkgs-level-1-expected-ort-result.yml b/scanner/src/funTest/assets/scanner-integration-pkgs-level-1-expected-ort-result.yml
@@ -0,0 +1,310 @@
+---
+repository:
+  vcs:
+    type: ""
+    url: ""
+    revision: ""
+    path: ""
+  vcs_processed:
+    type: ""
+    url: ""
+    revision: ""
+    path: ""
+  config: {}
+analyzer:
+  start_time: "1970-01-01T00:00:00Z"
+  end_time: "1970-01-01T00:00:00Z"
+  environment:
+    ort_version: "HEAD"
+    java_version: "<REPLACE_JAVA>"
+    os: "<REPLACE_OS>"
+    processors: "<REPLACE_PROCESSORS>"
+    max_memory: "<REPLACE_MAX_MEMORY>"
+    variables: {}
+    tool_versions: {}
+  config:
+    allow_dynamic_versions: false
+    skip_excluded: false
+  result:
+    projects:
+    - id: "Dummy::project:1.0.0"
+      definition_file_path: ""
+      declared_licenses: []
+      declared_licenses_processed: {}
+      vcs:
+        type: ""
+        url: ""
+        revision: ""
+        path: ""
+      vcs_processed:
+        type: ""
+        url: ""
+        revision: ""
+        path: ""
+      homepage_url: ""
+      scopes:
+      - name: "deps"
+        dependencies:
+        - id: "Dummy::pkg3:1.0.0"
+          dependencies:
+          - id: "Dummy::pkg2:1.0.0"
+            dependencies:
+            - id: "Dummy::pkg1:1.0.0"
+              dependencies:
+              - id: "Dummy::pkg0:1.0.0"
+    packages:
+    - id: "Dummy::pkg0:1.0.0"
+      purl: ""
+      declared_licenses: []
+      declared_licenses_processed: {}
+      description: ""
+      homepage_url: ""
+      binary_artifact:
+        url: ""
+        hash:
+          value: ""
+          algorithm: ""
+      source_artifact:
+        url: ""
+        hash:
+          value: ""
+          algorithm: ""
+      vcs:
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-scanner.git"
+        revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
+        path: ""
+      vcs_processed:
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-scanner.git"
+        revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
+        path: ""
+    - id: "Dummy::pkg1:1.0.0"
+      purl: ""
+      declared_licenses: []
+      declared_licenses_processed: {}
+      description: ""
+      homepage_url: ""
+      binary_artifact:
+        url: ""
+        hash:
+          value: ""
+          algorithm: ""
+      source_artifact:
+        url: ""
+        hash:
+          value: ""
+          algorithm: ""
+      vcs:
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-scanner.git"
+        revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
+        path: "pkg1"
+      vcs_processed:
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-scanner.git"
+        revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
+        path: "pkg1"
+    - id: "Dummy::pkg2:1.0.0"
+      purl: ""
+      declared_licenses: []
+      declared_licenses_processed: {}
+      description: ""
+      homepage_url: ""
+      binary_artifact:
+        url: ""
+        hash:
+          value: ""
+          algorithm: ""
+      source_artifact:
+        url: ""
+        hash:
+          value: ""
+          algorithm: ""
+      vcs:
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-scanner.git"
+        revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
+        path: "pkg2"
+      vcs_processed:
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-scanner.git"
+        revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
+        path: "pkg2"
+    - id: "Dummy::pkg3:1.0.0"
+      purl: ""
+      declared_licenses: []
+      declared_licenses_processed: {}
+      description: ""
+      homepage_url: ""
+      binary_artifact:
+        url: ""
+        hash:
+          value: ""
+          algorithm: ""
+      source_artifact:
+        url: ""
+        hash:
+          value: ""
+          algorithm: ""
+      vcs:
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-scanner.git"
+        revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
+        path: "pkg3"
+      vcs_processed:
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-scanner.git"
+        revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
+        path: "pkg3"
+scanner:
+  start_time: "1970-01-01T00:00:00Z"
+  end_time: "1970-01-01T00:00:00Z"
+  environment:
+    ort_version: "HEAD"
+    java_version: "<REPLACE_JAVA>"
+    os: "<REPLACE_OS>"
+    processors: "<REPLACE_PROCESSORS>"
+    max_memory: "<REPLACE_MAX_MEMORY>"
+    variables: {}
+    tool_versions: {}
+  config:
+    skip_concluded: false
+    skip_excluded: false
+    create_missing_archives: false
+    detected_license_mapping:
+      LicenseRef-scancode-agpl-generic-additional-terms: "NOASSERTION"
+      LicenseRef-scancode-free-unknown: "NOASSERTION"
+      LicenseRef-scancode-generic-cla: "NOASSERTION"
+      LicenseRef-scancode-generic-exception: "NOASSERTION"
+      LicenseRef-scancode-generic-export-compliance: "NOASSERTION"
+      LicenseRef-scancode-generic-tos: "NOASSERTION"
+      LicenseRef-scancode-generic-trademark: "NOASSERTION"
+      LicenseRef-scancode-gpl-generic-additional-terms: "NOASSERTION"
+      LicenseRef-scancode-other-copyleft: "NOASSERTION"
+      LicenseRef-scancode-other-permissive: "NOASSERTION"
+      LicenseRef-scancode-patent-disclaimer: "NOASSERTION"
+      LicenseRef-scancode-unknown: "NOASSERTION"
+      LicenseRef-scancode-unknown-license-reference: "NOASSERTION"
+      LicenseRef-scancode-unknown-spdx: "NOASSERTION"
+      LicenseRef-scancode-warranty-disclaimer: "NOASSERTION"
+    ignore_patterns:
+    - "**/*.ort.yml"
+    - "**/*.spdx.yml"
+    - "**/*.spdx.yaml"
+    - "**/*.spdx.json"
+    - "**/META-INF/DEPENDENCIES"
+    - "**/META-INF/DEPENDENCIES.txt"
+    - "**/META-INF/NOTICE"
+    - "**/META-INF/NOTICE.txt"
+  provenances:
+  - id: "Dummy::pkg3:1.0.0"
+    package_provenance:
+      vcs_info:
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-scanner.git"
+        revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
+        path: "pkg3"
+      resolved_revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
+    sub_repositories:
+      pkg3/subrepo:
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-scanner-subrepo.git"
+        revision: "a732695e03efcbd74539208af98c297ee86e49d5"
+        path: ""
+  - id: "Dummy::project:1.0.0"
+    package_provenance_resolution_issue:
+      timestamp: "1970-01-01T00:00:00Z"
+      source: "scanner"
+      message: "IOException: Could not resolve provenance for package 'Dummy::project:1.0.0'\
+        \ for source code origins [VCS, ARTIFACT]."
+      severity: "ERROR"
+  scan_results:
+  - provenance:
+      vcs_info:
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-scanner-subrepo.git"
+        revision: "a732695e03efcbd74539208af98c297ee86e49d5"
+        path: ""
+      resolved_revision: "a732695e03efcbd74539208af98c297ee86e49d5"
+    scanner:
+      name: "Dummy"
+      version: "1.0.0"
+      configuration: ""
+    summary:
+      start_time: "1970-01-01T00:00:00Z"
+      end_time: "1970-01-01T00:00:00Z"
+      licenses:
+      - license: "NOASSERTION"
+        location:
+          path: "LICENSE"
+          start_line: -1
+          end_line: -1
+      - license: "NOASSERTION"
+        location:
+          path: "README"
+          start_line: -1
+          end_line: -1
+      - license: "NOASSERTION"
+        location:
+          path: "pkg-s1/pkg-s1.txt"
+          start_line: -1
+          end_line: -1
+  - provenance:
+      vcs_info:
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-scanner.git"
+        revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
+        path: ""
+      resolved_revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
+    scanner:
+      name: "Dummy"
+      version: "1.0.0"
+      configuration: ""
+    summary:
+      start_time: "1970-01-01T00:00:00Z"
+      end_time: "1970-01-01T00:00:00Z"
+      licenses:
+      - license: "NOASSERTION"
+        location:
+          path: "LICENSE"
+          start_line: -1
+          end_line: -1
+      - license: "NOASSERTION"
+        location:
+          path: "pkg3/pkg3.txt"
+          start_line: -1
+          end_line: -1
+  scanners:
+    Dummy::pkg3:1.0.0:
+    - "Dummy"
+    Dummy::project:1.0.0:
+    - "Dummy"
+  files:
+  - provenance:
+      vcs_info:
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-scanner-subrepo.git"
+        revision: "a732695e03efcbd74539208af98c297ee86e49d5"
+        path: ""
+      resolved_revision: "a732695e03efcbd74539208af98c297ee86e49d5"
+    files:
+    - path: "LICENSE"
+      sha1: "7df059597099bb7dcf25d2a9aedfaf4465f72d8d"
+    - path: "README"
+      sha1: "ae8044f7fce7ee914a853c30c3085895e9be8b9c"
+    - path: "pkg-s1/pkg-s1.txt"
+      sha1: "e5fb17f8f4f4ef0748bb5ba137fd0e091dd5a1f6"
+  - provenance:
+      vcs_info:
+        type: "Git"
+        url: "https://github.com/oss-review-toolkit/ort-test-data-scanner.git"
+        revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
+        path: ""
+      resolved_revision: "97d57bb4795bc41f496e1a8e2c7751cefc7da7ec"
+    files:
+    - path: "pkg3/pkg3.txt"
+      sha1: "859d66be2d153968cdaa8ec7265270c241eea024"
+advisor: null
+evaluator: null
+resolved_configuration: {}