ZAP import accepts types other than "url" (#269)

* ZAP import accepts types other than "url"

config:

```
importUrlsFromFile:
  type: one of 'har', 'modsec2', 'url' (default), 'zap_messages'
  fileName: path to file
```

* fix indentation

* update configVersion schema to 6 to handle importUrlsFromFile change

* re-linting the changes

---------

Co-authored-by: Your Name <[email protected]>

config/config-template-generic-scan.yaml

@@ -7,7 +7,7 @@ config:
   # This value tells RapiDAST what schema should be used to read this configuration.
   # Therefore you should only change it if you update the configuration to a newer schema
   # It is intended to keep backward compatibility (newer RapiDAST running an older config)
-  configVersion: 5
+  configVersion: 6
 
   # (Optional) configure to export scan results to OWASP Defect Dojo
   #defectDojo:

config/config-template-multi-scan.yaml

@@ -10,7 +10,7 @@ config:
   # This value tells RapiDAST what schema should be used to read this configuration.
   # Therefore you should only change it if you update the configuration to a newer schema
   # It is intended to keep backward compatibility (newer RapiDAST running an older config)
-  configVersion: 5
+  configVersion: 6
 
 # `application` contains data related to the application, not to the scans.
 application:

config/config-template-nessus.yaml

@@ -2,7 +2,7 @@ config:
   # WARNING: `configVersion` indicates the schema version of the config file.
   # This value tells RapiDAST what schema should be used to read this configuration.
   # Therefore you should only change it if you update the configuration to a newer schema
-  configVersion: 5
+  configVersion: 6
 
   # all the results of all scanners will be stored under that location
   # base_results_dir: "./results"

config/config-template-trivy-image-scan.yaml

@@ -7,7 +7,7 @@ config:
   # This value tells RapiDAST what schema should be used to read this configuration.
   # Therefore you should only change it if you update the configuration to a newer schema
   # It is intended to keep backward compatibility (newer RapiDAST running an older config)
-  configVersion: 5
+  configVersion: 6
 
 # `application` contains data related to the application, not to the scans.
 application:

config/config-template-trivy-k8s-scan.yaml

@@ -7,7 +7,7 @@ config:
   # This value tells RapiDAST what schema should be used to read this configuration.
   # Therefore you should only change it if you update the configuration to a newer schema
   # It is intended to keep backward compatibility (newer RapiDAST running an older config)
-  configVersion: 5
+  configVersion: 6
 
 # `application` contains data related to the application, not to the scans.
 application:

config/config-template-zap-long.yaml

@@ -10,7 +10,7 @@ config:
   # This value tells RapiDAST what schema should be used to read this configuration.
   # Therefore you should only change it if you update the configuration to a newer schema
   # It is intended to keep backward compatibility (newer RapiDAST running an older config)
-  configVersion: 5
+  configVersion: 6
 
   # all the results of all scanners will be stored under that location
   base_results_dir: "./results"
@@ -118,8 +118,10 @@ scanners:
         apiUrl: "<URL to openAPI>"
         # alternative to apiURL: apiFile: "<local path to openAPI file>"
 
-    # A list of URLs can also be provided, from a text file (1 URL per line)
-    importUrlsFromFile: "<path to import URL>"
+    # A list of URLs can also be provided, type supported: 'har', 'modsec2', 'url' (default), 'zap_messages'
+    importUrlsFromFile:
+      type: "url"
+      fileName: "<path to import URL>"
 
     graphql:
       endpoint: "<URL to GraphQL API endpoint>"

config/config-template-zap-mac.yaml

@@ -10,7 +10,7 @@ config:
   # This value tells RapiDAST what schema should be used to read this configuration.
   # Therefore you should only change it if you update the configuration to a newer schema
   # It is intended to keep backward compatibility (newer RapiDAST running an older config)
-  configVersion: 5
+  configVersion: 6
 
 # `application` contains data related to the application, not to the scans.
 application:

config/config-template-zap-simple.yaml

@@ -10,7 +10,7 @@ config:
   # This value tells RapiDAST what schema should be used to read this configuration.
   # Therefore you should only change it if you update the configuration to a newer schema
   # It is intended to keep backward compatibility (newer RapiDAST running an older config)
-  configVersion: 5
+  configVersion: 6
 
 # `application` contains data related to the application, not to the scans.
 application:

config/config-template-zap-tiny.yaml

@@ -1,5 +1,5 @@
 config:
-  configVersion: 5
+  configVersion: 6
 
 application:
   shortName: "example-1.0"

configmodel/converter.py

@@ -5,7 +5,7 @@
 
 # WARNING: this needs to be incremented everytime a non-compatible change is made in the configuration.
 # A corresponding function also needs to be written
-CURR_CONFIG_VERSION = 5
+CURR_CONFIG_VERSION = 6
 
 
 def config_converter_dispatcher(func):
@@ -48,6 +48,25 @@ def convert_configmodel(conf):
     raise RuntimeError(f"There was an error in converting configuration. No convertion available for version {version}")
 
 
+@convert_configmodel.register(5)
+def convert_from_version_5_to_6(old):
+    """Returns a *copy* of the original rapidast config file, but updated to v6
+    scanner.zap.importUrlsFromFile is now a dictionary, not a string
+    """
+    new = copy.deepcopy(old)
+
+    for key in old.conf["scanners"]:
+        if key.startswith("zap") and old.exists(f"scanners.{key}.importUrlsFromFile"):
+            new.delete(f"scanners.{key}.importUrlsFromFile")  # start from fresh
+            new.set(f"scanners.{key}.importUrlsFromFile.fileName", old.get(f"scanners.{key}.importUrlsFromFile"))
+            new.set(f"scanners.{key}.importUrlsFromFile.type", "url")
+
+    # Finally, set the correct version number
+    new.set("config.configVersion", 6)
+
+    return new
+
+
 @convert_configmodel.register(4)
 def convert_from_version_4_to_5(old):
     """Returns a *copy* of the original rapidast config file, but updated to v5

scanners/zap/zap.py

@@ -376,17 +376,35 @@ def _setup_zap_automation(self):
 
     def _setup_import_urls(self):
         """If importUrlsFromFile exists:
-        prepare an import job for URLs importUrlsFromFile _must_ be an existing file on the host
-        Its content is a text file: a list of GET URLs, each of which will be scanned
-        """
-        job = {"name": "import", "type": "import", "parameters": {"type": "url"}}
+        Prepare a URL import job. All ZAP's import job are supported: 'har', 'modsec2', 'url' (default), 'zap_messages'
+        importUrlsFromFile is a dictionary: { "type": "<type>", "fileName": "<path/to/file>"}
 
-        orig = self.my_conf("importUrlsFromFile")
-        if not orig:
+        The filename of the import will always be copied in the `container_work_dir` as importUrls.txt
+        """
+        if not self.my_conf("importUrlsFromFile"):
+            # no import configured
             return
-        dest = f"{self.container_work_dir}/importUrls.txt"
-        self._include_file(orig, dest)
-        job["parameters"]["fileName"] = dest
+
+        # Basic job config. The `type` parameter will be set later
+        job = {
+            "name": "import",
+            "type": "import",
+            "parameters": {"fileName": f"{self.container_work_dir}/importUrls.txt"},
+        }
+
+        types = ("har", "modsec2", "url", "zap_messages")
+
+        source = ""  # Location of the import file on the host
+
+        source = self.my_conf("importUrlsFromFile.fileName")
+        if not source:
+            raise ValueError("ZAP config error: importUrlsFromFile must have a `fileName` entry")
+        job["parameters"]["type"] = self.my_conf("importUrlsFromFile.type", "url")
+
+        if not job["parameters"]["type"] in types:
+            raise ValueError(f"ZAP config error: importUrlsFromFile.type must be within {types}")
+
+        self._include_file(source, job["parameters"]["fileName"])
         self.automation_config["jobs"].append(job)
 
     def _setup_export_site_tree(self):

tests/configmodel/older-schemas/v5.yaml

@@ -0,0 +1,209 @@
+# This is a verbose configuration template. A lot of value do not need to be present, for most configuration.
+#
+# Author: Red Hat Product Security
+#
+# See "config-template.yaml" for a simpler configuration file.
+# All the values are optional (except `config.configVersion`): if a key is missing, it will mean either "disabled" or a sensible default will be selected
+
+config:
+  # WARNING: `configVersion` indicates the schema version of the config file.
+  # This value tells RapiDAST what schema should be used to read this configuration.
+  # Therefore you should only change it if you update the configuration to a newer schema
+  # It is intended to keep backward compatibility (newer RapiDAST running an older config)
+  configVersion: 5
+
+  # all the results of all scanners will be stored under that location
+  base_results_dir: "./results"
+
+  # In RapiDAST only: should RapiDAST verify certificates
+  # possible values: true [default], false, /path/to/a/PEM/file
+  tls_verify_for_rapidast_downloads: true
+
+  # Import a particular environment, and inject it for each scanner
+  environ:
+    envFile: "path/to/env/file"
+
+  # Export to Google Cloud Storage
+  googleCloudStorage:
+    keyFile: "/path/to/GCS/key"                           # optional: path to the GCS key file (alt.: use GOOGLE_APPLICATION_CREDENTIALS)
+    bucketName: "<name-of-GCS-bucket-to-export-to>"       # Mandatory
+    directory: "<override-of-default-directory>"          # Optional, defaults to `RapiDAST-{app_name}`
+
+
+
+# `application` contains data related to the application, not to the scans.
+application:
+  shortName: "MyApp-1.0"
+  url: "<Mandatory. root URL of the application>"
+
+# `general` is a section that will be applied to all scanners.
+# Any scanner can override a value by creating an entry of the same name in their own configuration
+general:
+
+
+  # remove `proxy` entirely for direct connection
+  proxy:
+    proxyHost: "<hostname>"
+    proxyPort: "<port>"
+
+  # remove `authentication` entirely for unauthenticated connection
+  authentication:
+    type: "oauth2_rtoken"
+    parameters:
+      client_id: "cloud-services"
+      token_endpoint: "<token retrieval URL>"
+      rtoken_from_var: "RTOKEN"     # referring to a env defined in general.environ.envFile
+      #preauth: false  # set to true to pregenerate a token, and stick to it (no refresh)
+    # Other types of authentication:
+    #type: "http_header"
+    #parameters:
+    #  name: "Authorization"
+    #  value: "MySecretHeader"
+    #type: "http_basic"
+    #parameters:
+    #  username: "user"
+    #  password: "mypassw0rd"
+    #type: "cookie"
+    #parameters:
+    #  name: "cookie name"
+    #  value: "cookie value"
+    #
+    # "browser" authentication will use firefox in the background to generate cookies
+    #  - verifyUrl must return an error if the user is not logged in
+    #type: "browser"
+    #parameters:
+    #  username: "user"
+    #  password: "mypassw0rd"
+    #  loginPageUrl: "https://myapp/login"
+    #  verifyUrl: "https://myapp/user/info"
+
+
+  container:
+    # This configures what technology is to be used for RapiDAST to run each scanner.
+    # Currently supported: `podman`  and `none`
+    #   none: Default. RapiDAST runs each scanner in the same host or inside the RapiDAST image container
+    #   podman: RapiDAST orchestrates each scanner on its own using podman
+    # When undefined, relies on rapidast-defaults.yaml, or `none` if nothing is set
+    #type: "none"
+
+  # (Optional) configure to export the results to Defect Dojo.
+  # WARNING: requires an export to be configured: either config.googleCloudStorage or config.defectDojo
+  defectDojoExport:
+    # Parameters contain data that will directly be sent as parameters to DefectDojo's import/reimport endpoints.
+    # For example: commit tag, version, push_to_jira, etc.
+    # See https://demo.defectdojo.org/api/v2/doc/ for a list of possibilities
+    # The minimum set of data is whatever is needed to identify which engagement/test needs to be chosen.
+    # If neither a test ID (`test` parameter), nor product_name and engagement_name were provided, sane default will be attempted:
+    #   - product_name chosen from either application.productName or application.shortName
+    #   - engagement_name:  "RapiDAST" [this way the same engagement will always be chosen, regardless of the scanner]
+    parameters:
+      product_name: "My Product"
+      engagement_name: "RapiDAST"
+      # - or -
+      #engagement: 3   # engagement ID
+      # - or -
+      #test_title: "ZAP"
+      # - or -
+      #test: 5       # test ID, that will force "reimport" mode
+
+      # For additional options, see https://defectdojo.github.io/django-DefectDojo/integrations/importing/
+
+# `scanners' is a section that configures scanning options
+scanners:
+  zap:
+  # define a scan through the ZAP scanner
+    apiScan:
+      target: "<optional, if different from application.url>"
+      apis:
+        apiUrl: "<URL to openAPI>"
+        # alternative to apiURL: apiFile: "<local path to openAPI file>"
+
+    # A list of URLs can also be provided, from a text file (1 URL per line)
+    importUrlsFromFile: "<path to import URL>"
+
+    graphql:
+      endpoint: "<URL to GraphQL API endpoint>"
+      # schemaUrl: ""                       # String: URL pointing to a GraphQL Schema
+      # schemaFile: ""                      # String: Local file path of a GraphQL Schema
+      # maxQueryDepth: 5                    # The maximum query generation depth
+      # lenientMaxQueryDepthEnabled: true   # Whether or not Maximum Query Depth is enforced leniently
+      # maxAdditionalQueryDepth: 5          # The maximum additional query generation depth (used if enforced leniently)
+      # maxArgsDepth: 5                     # The maximum arguments generation depth
+      # optionalArgsEnabled: true           # Whether or not Optional Arguments should be specified
+      # argsType: both                      # Enum [inline, variables, both]: How arguments are specified
+      # querySplitType: leaf                # Enum [leaf, root_field, operation]: The level for which a single query is generated
+      # requestMethod: post_json            # Enum [post_json, post_graphql, get]: The request method
+
+    spider:
+      maxDuration: 0 # in minutes, default: 0 unlimited
+      url: ""        # url to start spidering from, default: application.url set above
+
+    spiderAjax:
+      # The list of parameters: https://www.zaproxy.org/docs/desktop/addons/ajax-spider/automation/
+      #maxCrawlStates: 10   # this may be useful when running in a memory limited environment (default: 0 unlimited)
+      #maxCrawlDepth: 10    # default: unlimited
+      maxDuration: 0 # in minutes, default: 0 unlimited
+      url: ""        # url to start spidering from, default: application.url set above
+      browserId: firefox-headless
+
+    passiveScan:
+      # Optional comma-separated list of passive rules to disable
+      # Use https://www.zaproxy.org/docs/alerts/ to match rule with its ID
+      disabledRules: "2,10015,10024,10027,10054,10096,10109,10112"
+
+    activeScan:
+      # The list of parameters: https://www.zaproxy.org/docs/desktop/addons/ajax-spider/automation/
+      #maxRuleDurationInMins: max scan time for each Rule (default: unlimited)
+      #maxScanDurationInMins: max scan time for the entire scan. Useful for debugging automation
+      #
+      # If no policy is chosen, a default ("API-scan-minimal") will be selected
+      # The list of policies can be found in scanners/zap/policies/
+      policy:  "API-scan-minimal"
+
+    container:
+      parameters:
+        image: "ghcr.io/zaproxy/zaproxy:stable" # for type such as podman
+        #podName: "mypod"  # optional: inject ZAP in an existing Pod
+
+        executable: "zap.sh"  # for Linux
+        #executable: "/Applications/OWASP ZAP.app/Contents/Java/zap.sh"    # for MacOS, when general.container.type is 'none' only
+
+    report:
+      format: ["json"]
+      #format: ["json","html","sarif","xml"]  # default: "json" only
+
+    urls:
+      # Optional, `includes` and `excludes` take a list of regexps.
+      # includes: A URL matching that regexp will be in the scope of scanning, in addition to application.url which is already in scope
+      # excludes: A URL matching that regexp will NOT be in the scope of scanning
+      # Note: The regular expressions MUST match the whole URL.
+      #       e.g.: 'http://example.com/do-not-descend-here/' will actually descend
+
+      #includes:
+      #  - "^https?://example.com:3000/.*$"
+      #excludes:
+      #  - "^https?://example.com:3000/do-not-descend-here/.*$"
+
+    miscOptions:
+      # EnableUI (default: false), requires a compatible runtime (e.g.: `type: none`)
+      enableUI: False
+
+      # Defaults to False, set True to force auto update of ZAP plugins
+      updateAddons: True
+
+      # List (comma-separated string or list) of additional addons to install
+      additionalAddons: "ascanrulesBeta"
+
+      # If set to True and authentication is oauth2_rtoken: manually download schemas (e.g.: openAPI, GraphQL)
+      oauth2ManualDownload: False
+
+      # Overwrite the default port in case it is required. The default port was selected to avoid any collision with other services
+      zapPort: 8080
+
+      # Maximum heap size of the JVM. Default: ¼ of the RAM. acceptable values: [0-9]+[kKmMgG]?
+      # This may be required for large OpenAPI definition
+      memMaxHeap: "6144m"
+
+      overrideConfigs:
+        - formhandler.fields.field(0).fieldId=namespace
+        - formhandler.fields.field(0).value=default