From 2ad5761259347bf381f57250d69ee7b41199e39a Mon Sep 17 00:00:00 2001 From: Cedric Buissart Date: Mon, 16 Dec 2024 08:27:03 +0100 Subject: [PATCH] ZAP import accepts types other than "url" (#269) * ZAP import accepts types other than "url" config: ``` importUrlsFromFile: type: one of 'har', 'modsec2', 'url' (default), 'zap_messages' fileName: path to file ``` * fix indentation * update configVersion schema to 6 to handle importUrlsFromFile change * re-linting the changes --------- Co-authored-by: Your Name --- config/config-template-generic-scan.yaml | 2 +- config/config-template-multi-scan.yaml | 2 +- config/config-template-nessus.yaml | 2 +- config/config-template-trivy-image-scan.yaml | 2 +- config/config-template-trivy-k8s-scan.yaml | 2 +- config/config-template-zap-long.yaml | 8 +- config/config-template-zap-mac.yaml | 2 +- config/config-template-zap-simple.yaml | 2 +- config/config-template-zap-tiny.yaml | 2 +- configmodel/converter.py | 21 +- scanners/zap/zap.py | 36 +++- tests/configmodel/older-schemas/v5.yaml | 209 +++++++++++++++++++ tests/configmodel/test_convert.py | 20 ++ tests/scanners/zap/test_setup.py | 29 ++- 14 files changed, 315 insertions(+), 24 deletions(-) create mode 100644 tests/configmodel/older-schemas/v5.yaml diff --git a/config/config-template-generic-scan.yaml b/config/config-template-generic-scan.yaml index 56740734..8c632a1d 100644 --- a/config/config-template-generic-scan.yaml +++ b/config/config-template-generic-scan.yaml @@ -7,7 +7,7 @@ config: # This value tells RapiDAST what schema should be used to read this configuration. # Therefore you should only change it if you update the configuration to a newer schema # It is intended to keep backward compatibility (newer RapiDAST running an older config) - configVersion: 5 + configVersion: 6 # (Optional) configure to export scan results to OWASP Defect Dojo #defectDojo: diff --git a/config/config-template-multi-scan.yaml b/config/config-template-multi-scan.yaml index 6e8d526f..688e423c 100644 --- a/config/config-template-multi-scan.yaml +++ b/config/config-template-multi-scan.yaml @@ -10,7 +10,7 @@ config: # This value tells RapiDAST what schema should be used to read this configuration. # Therefore you should only change it if you update the configuration to a newer schema # It is intended to keep backward compatibility (newer RapiDAST running an older config) - configVersion: 5 + configVersion: 6 # `application` contains data related to the application, not to the scans. application: diff --git a/config/config-template-nessus.yaml b/config/config-template-nessus.yaml index 39d5997f..bd441009 100644 --- a/config/config-template-nessus.yaml +++ b/config/config-template-nessus.yaml @@ -2,7 +2,7 @@ config: # WARNING: `configVersion` indicates the schema version of the config file. # This value tells RapiDAST what schema should be used to read this configuration. # Therefore you should only change it if you update the configuration to a newer schema - configVersion: 5 + configVersion: 6 # all the results of all scanners will be stored under that location # base_results_dir: "./results" diff --git a/config/config-template-trivy-image-scan.yaml b/config/config-template-trivy-image-scan.yaml index 926113f8..b994dd8f 100644 --- a/config/config-template-trivy-image-scan.yaml +++ b/config/config-template-trivy-image-scan.yaml @@ -7,7 +7,7 @@ config: # This value tells RapiDAST what schema should be used to read this configuration. # Therefore you should only change it if you update the configuration to a newer schema # It is intended to keep backward compatibility (newer RapiDAST running an older config) - configVersion: 5 + configVersion: 6 # `application` contains data related to the application, not to the scans. application: diff --git a/config/config-template-trivy-k8s-scan.yaml b/config/config-template-trivy-k8s-scan.yaml index b9aacaf2..a0ee7820 100644 --- a/config/config-template-trivy-k8s-scan.yaml +++ b/config/config-template-trivy-k8s-scan.yaml @@ -7,7 +7,7 @@ config: # This value tells RapiDAST what schema should be used to read this configuration. # Therefore you should only change it if you update the configuration to a newer schema # It is intended to keep backward compatibility (newer RapiDAST running an older config) - configVersion: 5 + configVersion: 6 # `application` contains data related to the application, not to the scans. application: diff --git a/config/config-template-zap-long.yaml b/config/config-template-zap-long.yaml index b4773952..71f810c3 100644 --- a/config/config-template-zap-long.yaml +++ b/config/config-template-zap-long.yaml @@ -10,7 +10,7 @@ config: # This value tells RapiDAST what schema should be used to read this configuration. # Therefore you should only change it if you update the configuration to a newer schema # It is intended to keep backward compatibility (newer RapiDAST running an older config) - configVersion: 5 + configVersion: 6 # all the results of all scanners will be stored under that location base_results_dir: "./results" @@ -118,8 +118,10 @@ scanners: apiUrl: "" # alternative to apiURL: apiFile: "" - # A list of URLs can also be provided, from a text file (1 URL per line) - importUrlsFromFile: "" + # A list of URLs can also be provided, type supported: 'har', 'modsec2', 'url' (default), 'zap_messages' + importUrlsFromFile: + type: "url" + fileName: "" graphql: endpoint: "" diff --git a/config/config-template-zap-mac.yaml b/config/config-template-zap-mac.yaml index 46c2471b..0b43de20 100644 --- a/config/config-template-zap-mac.yaml +++ b/config/config-template-zap-mac.yaml @@ -10,7 +10,7 @@ config: # This value tells RapiDAST what schema should be used to read this configuration. # Therefore you should only change it if you update the configuration to a newer schema # It is intended to keep backward compatibility (newer RapiDAST running an older config) - configVersion: 5 + configVersion: 6 # `application` contains data related to the application, not to the scans. application: diff --git a/config/config-template-zap-simple.yaml b/config/config-template-zap-simple.yaml index d3718599..c358af39 100644 --- a/config/config-template-zap-simple.yaml +++ b/config/config-template-zap-simple.yaml @@ -10,7 +10,7 @@ config: # This value tells RapiDAST what schema should be used to read this configuration. # Therefore you should only change it if you update the configuration to a newer schema # It is intended to keep backward compatibility (newer RapiDAST running an older config) - configVersion: 5 + configVersion: 6 # `application` contains data related to the application, not to the scans. application: diff --git a/config/config-template-zap-tiny.yaml b/config/config-template-zap-tiny.yaml index ac8ff646..ae1750b8 100644 --- a/config/config-template-zap-tiny.yaml +++ b/config/config-template-zap-tiny.yaml @@ -1,5 +1,5 @@ config: - configVersion: 5 + configVersion: 6 application: shortName: "example-1.0" diff --git a/configmodel/converter.py b/configmodel/converter.py index 8177d888..8de57022 100755 --- a/configmodel/converter.py +++ b/configmodel/converter.py @@ -5,7 +5,7 @@ # WARNING: this needs to be incremented everytime a non-compatible change is made in the configuration. # A corresponding function also needs to be written -CURR_CONFIG_VERSION = 5 +CURR_CONFIG_VERSION = 6 def config_converter_dispatcher(func): @@ -48,6 +48,25 @@ def convert_configmodel(conf): raise RuntimeError(f"There was an error in converting configuration. No convertion available for version {version}") +@convert_configmodel.register(5) +def convert_from_version_5_to_6(old): + """Returns a *copy* of the original rapidast config file, but updated to v6 + scanner.zap.importUrlsFromFile is now a dictionary, not a string + """ + new = copy.deepcopy(old) + + for key in old.conf["scanners"]: + if key.startswith("zap") and old.exists(f"scanners.{key}.importUrlsFromFile"): + new.delete(f"scanners.{key}.importUrlsFromFile") # start from fresh + new.set(f"scanners.{key}.importUrlsFromFile.fileName", old.get(f"scanners.{key}.importUrlsFromFile")) + new.set(f"scanners.{key}.importUrlsFromFile.type", "url") + + # Finally, set the correct version number + new.set("config.configVersion", 6) + + return new + + @convert_configmodel.register(4) def convert_from_version_4_to_5(old): """Returns a *copy* of the original rapidast config file, but updated to v5 diff --git a/scanners/zap/zap.py b/scanners/zap/zap.py index 20f624de..0216bffc 100644 --- a/scanners/zap/zap.py +++ b/scanners/zap/zap.py @@ -376,17 +376,35 @@ def _setup_zap_automation(self): def _setup_import_urls(self): """If importUrlsFromFile exists: - prepare an import job for URLs importUrlsFromFile _must_ be an existing file on the host - Its content is a text file: a list of GET URLs, each of which will be scanned - """ - job = {"name": "import", "type": "import", "parameters": {"type": "url"}} + Prepare a URL import job. All ZAP's import job are supported: 'har', 'modsec2', 'url' (default), 'zap_messages' + importUrlsFromFile is a dictionary: { "type": "", "fileName": ""} - orig = self.my_conf("importUrlsFromFile") - if not orig: + The filename of the import will always be copied in the `container_work_dir` as importUrls.txt + """ + if not self.my_conf("importUrlsFromFile"): + # no import configured return - dest = f"{self.container_work_dir}/importUrls.txt" - self._include_file(orig, dest) - job["parameters"]["fileName"] = dest + + # Basic job config. The `type` parameter will be set later + job = { + "name": "import", + "type": "import", + "parameters": {"fileName": f"{self.container_work_dir}/importUrls.txt"}, + } + + types = ("har", "modsec2", "url", "zap_messages") + + source = "" # Location of the import file on the host + + source = self.my_conf("importUrlsFromFile.fileName") + if not source: + raise ValueError("ZAP config error: importUrlsFromFile must have a `fileName` entry") + job["parameters"]["type"] = self.my_conf("importUrlsFromFile.type", "url") + + if not job["parameters"]["type"] in types: + raise ValueError(f"ZAP config error: importUrlsFromFile.type must be within {types}") + + self._include_file(source, job["parameters"]["fileName"]) self.automation_config["jobs"].append(job) def _setup_export_site_tree(self): diff --git a/tests/configmodel/older-schemas/v5.yaml b/tests/configmodel/older-schemas/v5.yaml new file mode 100644 index 00000000..b4773952 --- /dev/null +++ b/tests/configmodel/older-schemas/v5.yaml @@ -0,0 +1,209 @@ +# This is a verbose configuration template. A lot of value do not need to be present, for most configuration. +# +# Author: Red Hat Product Security +# +# See "config-template.yaml" for a simpler configuration file. +# All the values are optional (except `config.configVersion`): if a key is missing, it will mean either "disabled" or a sensible default will be selected + +config: + # WARNING: `configVersion` indicates the schema version of the config file. + # This value tells RapiDAST what schema should be used to read this configuration. + # Therefore you should only change it if you update the configuration to a newer schema + # It is intended to keep backward compatibility (newer RapiDAST running an older config) + configVersion: 5 + + # all the results of all scanners will be stored under that location + base_results_dir: "./results" + + # In RapiDAST only: should RapiDAST verify certificates + # possible values: true [default], false, /path/to/a/PEM/file + tls_verify_for_rapidast_downloads: true + + # Import a particular environment, and inject it for each scanner + environ: + envFile: "path/to/env/file" + + # Export to Google Cloud Storage + googleCloudStorage: + keyFile: "/path/to/GCS/key" # optional: path to the GCS key file (alt.: use GOOGLE_APPLICATION_CREDENTIALS) + bucketName: "" # Mandatory + directory: "" # Optional, defaults to `RapiDAST-{app_name}` + + + +# `application` contains data related to the application, not to the scans. +application: + shortName: "MyApp-1.0" + url: "" + +# `general` is a section that will be applied to all scanners. +# Any scanner can override a value by creating an entry of the same name in their own configuration +general: + + + # remove `proxy` entirely for direct connection + proxy: + proxyHost: "" + proxyPort: "" + + # remove `authentication` entirely for unauthenticated connection + authentication: + type: "oauth2_rtoken" + parameters: + client_id: "cloud-services" + token_endpoint: "" + rtoken_from_var: "RTOKEN" # referring to a env defined in general.environ.envFile + #preauth: false # set to true to pregenerate a token, and stick to it (no refresh) + # Other types of authentication: + #type: "http_header" + #parameters: + # name: "Authorization" + # value: "MySecretHeader" + #type: "http_basic" + #parameters: + # username: "user" + # password: "mypassw0rd" + #type: "cookie" + #parameters: + # name: "cookie name" + # value: "cookie value" + # + # "browser" authentication will use firefox in the background to generate cookies + # - verifyUrl must return an error if the user is not logged in + #type: "browser" + #parameters: + # username: "user" + # password: "mypassw0rd" + # loginPageUrl: "https://myapp/login" + # verifyUrl: "https://myapp/user/info" + + + container: + # This configures what technology is to be used for RapiDAST to run each scanner. + # Currently supported: `podman` and `none` + # none: Default. RapiDAST runs each scanner in the same host or inside the RapiDAST image container + # podman: RapiDAST orchestrates each scanner on its own using podman + # When undefined, relies on rapidast-defaults.yaml, or `none` if nothing is set + #type: "none" + + # (Optional) configure to export the results to Defect Dojo. + # WARNING: requires an export to be configured: either config.googleCloudStorage or config.defectDojo + defectDojoExport: + # Parameters contain data that will directly be sent as parameters to DefectDojo's import/reimport endpoints. + # For example: commit tag, version, push_to_jira, etc. + # See https://demo.defectdojo.org/api/v2/doc/ for a list of possibilities + # The minimum set of data is whatever is needed to identify which engagement/test needs to be chosen. + # If neither a test ID (`test` parameter), nor product_name and engagement_name were provided, sane default will be attempted: + # - product_name chosen from either application.productName or application.shortName + # - engagement_name: "RapiDAST" [this way the same engagement will always be chosen, regardless of the scanner] + parameters: + product_name: "My Product" + engagement_name: "RapiDAST" + # - or - + #engagement: 3 # engagement ID + # - or - + #test_title: "ZAP" + # - or - + #test: 5 # test ID, that will force "reimport" mode + + # For additional options, see https://defectdojo.github.io/django-DefectDojo/integrations/importing/ + +# `scanners' is a section that configures scanning options +scanners: + zap: + # define a scan through the ZAP scanner + apiScan: + target: "" + apis: + apiUrl: "" + # alternative to apiURL: apiFile: "" + + # A list of URLs can also be provided, from a text file (1 URL per line) + importUrlsFromFile: "" + + graphql: + endpoint: "" + # schemaUrl: "" # String: URL pointing to a GraphQL Schema + # schemaFile: "" # String: Local file path of a GraphQL Schema + # maxQueryDepth: 5 # The maximum query generation depth + # lenientMaxQueryDepthEnabled: true # Whether or not Maximum Query Depth is enforced leniently + # maxAdditionalQueryDepth: 5 # The maximum additional query generation depth (used if enforced leniently) + # maxArgsDepth: 5 # The maximum arguments generation depth + # optionalArgsEnabled: true # Whether or not Optional Arguments should be specified + # argsType: both # Enum [inline, variables, both]: How arguments are specified + # querySplitType: leaf # Enum [leaf, root_field, operation]: The level for which a single query is generated + # requestMethod: post_json # Enum [post_json, post_graphql, get]: The request method + + spider: + maxDuration: 0 # in minutes, default: 0 unlimited + url: "" # url to start spidering from, default: application.url set above + + spiderAjax: + # The list of parameters: https://www.zaproxy.org/docs/desktop/addons/ajax-spider/automation/ + #maxCrawlStates: 10 # this may be useful when running in a memory limited environment (default: 0 unlimited) + #maxCrawlDepth: 10 # default: unlimited + maxDuration: 0 # in minutes, default: 0 unlimited + url: "" # url to start spidering from, default: application.url set above + browserId: firefox-headless + + passiveScan: + # Optional comma-separated list of passive rules to disable + # Use https://www.zaproxy.org/docs/alerts/ to match rule with its ID + disabledRules: "2,10015,10024,10027,10054,10096,10109,10112" + + activeScan: + # The list of parameters: https://www.zaproxy.org/docs/desktop/addons/ajax-spider/automation/ + #maxRuleDurationInMins: max scan time for each Rule (default: unlimited) + #maxScanDurationInMins: max scan time for the entire scan. Useful for debugging automation + # + # If no policy is chosen, a default ("API-scan-minimal") will be selected + # The list of policies can be found in scanners/zap/policies/ + policy: "API-scan-minimal" + + container: + parameters: + image: "ghcr.io/zaproxy/zaproxy:stable" # for type such as podman + #podName: "mypod" # optional: inject ZAP in an existing Pod + + executable: "zap.sh" # for Linux + #executable: "/Applications/OWASP ZAP.app/Contents/Java/zap.sh" # for MacOS, when general.container.type is 'none' only + + report: + format: ["json"] + #format: ["json","html","sarif","xml"] # default: "json" only + + urls: + # Optional, `includes` and `excludes` take a list of regexps. + # includes: A URL matching that regexp will be in the scope of scanning, in addition to application.url which is already in scope + # excludes: A URL matching that regexp will NOT be in the scope of scanning + # Note: The regular expressions MUST match the whole URL. + # e.g.: 'http://example.com/do-not-descend-here/' will actually descend + + #includes: + # - "^https?://example.com:3000/.*$" + #excludes: + # - "^https?://example.com:3000/do-not-descend-here/.*$" + + miscOptions: + # EnableUI (default: false), requires a compatible runtime (e.g.: `type: none`) + enableUI: False + + # Defaults to False, set True to force auto update of ZAP plugins + updateAddons: True + + # List (comma-separated string or list) of additional addons to install + additionalAddons: "ascanrulesBeta" + + # If set to True and authentication is oauth2_rtoken: manually download schemas (e.g.: openAPI, GraphQL) + oauth2ManualDownload: False + + # Overwrite the default port in case it is required. The default port was selected to avoid any collision with other services + zapPort: 8080 + + # Maximum heap size of the JVM. Default: ΒΌ of the RAM. acceptable values: [0-9]+[kKmMgG]? + # This may be required for large OpenAPI definition + memMaxHeap: "6144m" + + overrideConfigs: + - formhandler.fields.field(0).fieldId=namespace + - formhandler.fields.field(0).value=default diff --git a/tests/configmodel/test_convert.py b/tests/configmodel/test_convert.py index 54b996ae..e456d418 100644 --- a/tests/configmodel/test_convert.py +++ b/tests/configmodel/test_convert.py @@ -53,6 +53,26 @@ def generate_config_v4(): raise RuntimeError(f"Unable to load TEST file {path}") from exc +@pytest.fixture(name="config_v5") +def generate_config_v5(): + path = "tests/configmodel/older-schemas/v5.yaml" + try: + with open(path) as file: + return configmodel.RapidastConfigModel(yaml.safe_load(file)) + except yaml.YAMLError as exc: + raise RuntimeError(f"Unable to load TEST file {path}") from exc + + +def test_v5_to_v6(config_v5): + oldconf = config_v5 + newconf = configmodel.converter.convert_from_version_5_to_6(oldconf) + + # Check that new importUrlsFromFile is a dictionary and contains a fileName matching the old importUrlsFromFile + assert newconf.get("scanners.zap.importUrlsFromFile.fileName", "x") == oldconf.get( + "scanners.zap.importUrlsFromFile", "y" + ) + + def test_v4_to_v5(config_v4): oldconf = config_v4 newconf = configmodel.converter.convert_from_version_4_to_5(oldconf) diff --git a/tests/scanners/zap/test_setup.py b/tests/scanners/zap/test_setup.py index e98b007c..b99181e0 100644 --- a/tests/scanners/zap/test_setup.py +++ b/tests/scanners/zap/test_setup.py @@ -206,12 +206,35 @@ def test_setup_authentication_auth_rtoken_preauth(test_config): def test_setup_import_urls(test_config): - # trick: set this very file as import - test_config.set("scanners.zap.importUrlsFromFile", __file__) + # trick: use is the current pytest as import file + # 1- Test importUrlsFromFile, with type "har" + test_config.set("scanners.zap.importUrlsFromFile", {"type": "har", "fileName": __file__}) test_zap = ZapNone(config=test_config) test_zap.setup() - assert Path(test_zap.host_work_dir, "importUrls.txt").is_file() + for item in test_zap.automation_config["jobs"]: + if item["type"] == "import": + assert item["parameters"]["type"] == "har" + break + else: + assert False + + # 2- Test that importUrlsFromFile defaults to "url" type + test_config.set("scanners.zap.importUrlsFromFile", {"fileName": __file__}) + test_zap = ZapNone(config=test_config) + test_zap.setup() + for item in test_zap.automation_config["jobs"]: + if item["type"] == "import": + assert item["parameters"]["type"] == "url" + break + else: + assert False + + # 3- Test that importUrlsFromFile fails if the type is incorrect + test_config.set("scanners.zap.importUrlsFromFile", {"type": "doesntexist", "fileName": __file__}) + test_zap = ZapNone(config=test_config) + with pytest.raises(ValueError) as exc: + test_zap.setup() def test_setup_exclude_urls(test_config):