From 005c3498ada3ea068549c71061d967fb8417ee53 Mon Sep 17 00:00:00 2001 From: Camilo Cota <1499184+ccronca@users.noreply.github.com> Date: Sat, 13 Jul 2024 01:17:05 +0200 Subject: [PATCH] Merge Development to Main after releasing 2.6.0 (#197) * Crawl fail openshift (#187) * Scanners: group temporary dirs into a same parent RapidastScanner._create_temp_dir now gather all the temporary directories under a single one. Easier cleanup * ZAP: change HOME if it is not writable Firefox requires a home directory. When crawling (Ajax spider), in Openshift, Firefox is unable to start if it can't write a ~/.firefox This is an issue in Openshift, where the user is created on the fly and its home directory is '/'. In that case, create a temporary directory, and assign HOME to it. * undo pre-commit change, to run on older pythong * Updated ZAP image url with the latest one (#189) updated ZAP image url with the latest one * updated zap default image url with the latest one in ZapPodman (#190) * require pre-commit for dev (#191) * Export to Google Cloud Storage (#192) * Export to Google Cloud Storage This commit adds a new export. It re-uses the original DefectDojo export. configuration: ```yaml config: googleCloudStorage: keyFile: "" bucketName: "" directory: "" general: defectDojoExport: parameters: # values for defectdojo's import-scan endpoint ``` Note: the generic scanner hasn't been tested yet * remove test description, rephrase example comments * removed some comment refering to DeDo in GCS * reworked defectdojo data to make it optional Latest changes: Now the defectDojoExport is no longer needed: data will still be exported if either googleCloudStorage or defectDojo are set. Note: It is still possible to prevent a particular scan to be exported by setting defectDojoExport: False for that scan (e.g.: RapiDAST runs 2 scans, out of which only 1 should be exported) As explained in the README: if defectDojoExport is missing: product_name will be derived from either application.productName or application.shortName engagement_name will be RapiDAST-{product_name}- * updated the ZAP path in the config template for MacOS since ZAP no longer belong to OWASP (#193) updated the path since ZAP no longer belong to OWASP * Gcs tests (#194) * [unittests] GCS export * Added unittests for exports/gcs Note/todo: ideally, it would be great to test the content of the tarball created. * [gcs unittest] added engagement and product to unittest * Readme updated (#195) readme updated: 1. separated GCS export from DefectDojo 2. removed 'OWASP' in the binary path for MacOS * [ZAP] Ajax spider requires a lot of shared memory (#196) * [ZAP] Ajax spider requires a lot of shared memory The Selenium environment set up by ZAP for the Ajax Spider requires a lot of shared memory (/dev/shm in Linux) This commit does the following: - Update the README troubleshooting section, for when the RapiDAST image is used - In Podman mode: if Ajax is used, automatically ask podman to have 2GB of shared memory - Added corresponding pytest - Fixed `find_context()`, which broke when context was not found (that should happen only in pytest) --------- Co-authored-by: Cedric Buissart Co-authored-by: Jeremy Bonghwan Choi --- README.md | 103 +++++++++++++++------ config/config-template-zap-long.yaml | 65 ++++++------- config/config-template-zap-mac.yaml | 6 +- configmodel/__init__.py | 8 ++ exports/defect_dojo.py | 4 +- exports/google_cloud_storage.py | 96 +++++++++++++++++++ rapidast.py | 26 ++++-- requirements-dev.txt | 1 + requirements.txt | 1 + scanners/__init__.py | 100 +++++++++++++++++++- scanners/generic/generic.py | 35 +------ scanners/zap/zap.py | 53 ++--------- scanners/zap/zap_none.py | 14 +++ scanners/zap/zap_podman.py | 18 +++- tests/exports/test_google_cloud_storage.py | 96 +++++++++++++++++++ tests/scanners/zap/test_setup_podman.py | 12 +++ 16 files changed, 478 insertions(+), 160 deletions(-) create mode 100755 exports/google_cloud_storage.py create mode 100644 tests/exports/test_google_cloud_storage.py diff --git a/README.md b/README.md index cd31638d..4ee5caac 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ RapiDAST supports executing ZAP on the MacOS host directly only. To run RapiDAST on MacOS(See the Configuration section below for more details on configuration): * Set `general.container.type: "none"` or `scanners.zap.container.type: "none"` in the configuration. -* Configure `scanners.zap.container.parameters.executable` to the installation path of the `zap.sh` command, because it is not available in the PATH. Usually, its path is `/Applications/OWASP ZAP.app/Contents/Java/zap.sh` on MacOS. +* Configure `scanners.zap.container.parameters.executable` to the installation path of the `zap.sh` command, because it is not available in the PATH. Usually, its path is `/Applications/ZAP.app/Contents/Java/zap.sh` on MacOS. Example: @@ -39,7 +39,7 @@ scanners: container: type: none parameters: - executable: "/Applications/OWASP ZAP.app/Contents/Java/zap.sh" + executable: "/Applications/ZAP.app/Contents/Java/zap.sh" ``` ## Installation @@ -158,11 +158,30 @@ scanners: In the example above, the ZAP scanner will first run without authentication, and then rerun again with a basic HTTP authentication. The results will be stored in their respective names (i.e.: `zap_unauthenticated` and `zap_authenticated` in the example above). -### DefectDojo integration +### Exporting data to external services + +#### Exporting to Google Cloud Storage + +This simply stores the data as a compressed tarball in a Google Cloud Storage bucket. + +```yaml +config: + # Defect dojo configuration + googleCloudStorage: + keyFile: "/path/to/GCS/key" # optional: path to the GCS key file (alternatively: use GOOGLE_APPLICATION_CREDENTIALS) + bucketName: "" # Mandatory + directory: "" # Optional directory where the credentials have write access, defaults to `RapiDAST-` +``` + +Once this is set, scan results will be exported to the bucket automatically. The tarball file will include: + 1. metadata.json - the file that contains scan_type, uuid and import_data(could be changed later. Currently this comes from the previous DefectDojo integration feature) + 2. scans - the directory that contains scan results + +#### Exporting to DefectDojo RapiDAST supports integration with OWASP DefectDojo which is an open source vulnerability management tool. -#### Preamble: creating DefectDojo user +##### Preamble: creating DefectDojo user RapiDAST needs to be able to authenticate to your DefectDojo instance. However, ideally, it should have the minimum set of permissions, such that it will not be allowed to modify products other than the one(s) it is supposed to. @@ -172,10 +191,10 @@ In order to do that: Then the product, as well as an engagement for that product, must be created in your DefectDojo instance. It would not be advised to give the RapiDAST user an "admin" role and simply set `auto_create_context` to True, as it would be both insecure and accident prone (a typo in the product name would let RapiDAST create a new product) -#### DefectDojo configuration in RapiDAST -##### Authentication -First, RapiDAST needs to be able to authenticate itself to a DefectDojo service. This is a typical configuration: +##### Exporting to Defect Dojo + +RapiDAST will send the results directly to a DefectDojo service. This is a typical configuration: ```yaml config: @@ -197,37 +216,46 @@ Alternatively, the `REQUESTS_CA_BUNDLE` environment variable can be used to sele You can either authenticate using a username/password combination, or a token (make sure it is not expired). In either case, you can use the `_from_var` method described in the previous chapter to avoid hardcoding the value in the configuration. -##### Product/engagement/test +##### Configuration of exported data -Then, RapiDAST needs to know, for each scanner, sufficient information such that it can identify which product/engagement/test to match. -This is configured in the `zap.scanner.defectDojoExport.parameters` entry. See the `import-scan` or `reimport-scan` parameters at https://demo.defectdojo.org/api/v2/doc/ for a list of accepted entries. -Notes: - * `engagement` and `test` refer to identifiers, and should be integers (as opposed to `engagement_name` and `test_title`) - * If a `test` identifier is provided, RapiDAST will reimport the result to that test. The existing test must be compatible (same file schema, such as ZAP Scan, for example) - * If the `product_name` does not exist, the scanner should default to `application.productName`, or `application.shortName` - * Tip: the entries common to all scanners can be added to `general.defectDojoExport.parameters`, while the scanner-dependant entries (e.g.: test identifier) can be set in the scanner's configuration (e.g.: `scanners.zap.defectDojoExport.parameters`) +The data exported follows the Defectdojo methodology of "Product → Engagement → Test" : a test, such as a ZAP scan, belongs to an engagement for a product. +Its configuration is made under the `scanners..defectDojoExport.parameters` configuration entries. As a baseline, parameters from the Defectdojo `import-scan` and `reimport-scan` are accepted. -```yaml -general: - defectDojoExport: - parameters: - productName: "My product" - tags: ["RapiDAST"] +For each scan, the logic applied is the following, in order: +* If a test ID is provided (parameter `test`), this scan will replace the previous one (a "reimport" in Defectdojo) +* If an engagement ID is provided (parameter `engagement`), this scan will be added as a new test in that existing engagement +* If an engagement and a product are given by name (`engagement_name` and `product_name` parameters), this scan will be added for that given engagement for the given product + +In each `defectDojoExport.parameters`, some defaults parameters are applied: +* `product_name`, in order (the first non empty value found): + - `application.productName` + - `application.shortName` (this name should not contain non-printable characters, such as spaces) +* `engagement_name` defaults to `RapiDAST--` +* `scan_type` : filled by the scanner +* `active`: `True` +* `verified`: `False` + +As a reminder: values from `general` are applied to each scanner. + +Here is an example: +```yaml scanners: zap: defectDojoExport: parameters: - test: 34 - endpoint_to_add: "https://qa.myapp.local/" + product_name: "My Product" + engagement_name: "RapiDAST" # or engagement: + #test: ``` +See https://documentation.defectdojo.com/integrations/importing/#api for more information. ## Execution Once you have created a configuration file, you can run a scan with it. -``` -$ rapidast.py --config +```sh +$ rapidast.py --config "" ``` There are more options. @@ -485,7 +513,7 @@ See https://github.com/zaproxy/zaproxy/issues/7703 for additional information. RapiDAST works around this bug, but with little inconvenients (slower because it has to fix itself and download all the plugins) - Verify that the host installation directory is missing its plugins. -e.g., in a MacOS installation, `/Applications/OWASP ZAP.app/Contents/Java/plugin/` will be mostly empty. In particular, no `callhome*.zap` and `network*.zap` file are present. +e.g., in a MacOS installation, `/Applications/ZAP.app/Contents/Java/plugin/` will be mostly empty. In particular, no `callhome*.zap` and `network*.zap` file are present. - Reinstall ZAP, but __DO NOT RUN IT__, as it would delete the plugins. Verify that the directory contains many plugins. - `chown` the installation files to root, so that when running ZAP, the application running as the user does not have sufficient permission to delete its own plugins @@ -535,10 +563,31 @@ com.fasterxml.jackson.dataformat.yaml.JacksonYAMLParseException: The incoming YA at [Source: (StringReader); line: 49813, column: 50] ``` -Solutions: +Solutions: * If you are using a Swagger v2 definition, try converting it to v3 (OpenAPI) * Set a `maxYamlCodePoints` Java proprety with a big value, which can be passed using environment variables (via the `config.environ.envFile` config entry): `_JAVA_OPTIONS=-DmaxYamlCodePoints=99999999` +### ZAP's Ajax Spider failing + +#### Insufficient shared memory + +ZAP's Ajax Spider makes heavy use of shared memory (`/dev/shm/`). When using the RapiDAST image or the ZAP image, the user needs to make sure that sufficient space is available in `/dev/shm/` (in podman, by default, its size is 64MB). A size of 2G would be the minimum recommended. In podman for example, the option would be `--shm-size=2g`. + +ZAP logs that would bring evidence of a lack of shared memory would look like the following: + +``` +2024-07-04 11:21:32,061 [ZAP-AjaxSpiderAuto] WARN SpiderThread - Failed to start browser firefox-headless +com.google.inject.ProvisionException: Unable to provision, see the following errors: + +1) [Guice/ErrorInCustomProvider]: SessionNotCreatedException: Could not start a new session. Response code 500. Message: Failed to decode response from marionette +``` + +Or the following: + +``` +2024-07-04 12:23:28,027 [ZAP-AjaxSpiderAuto] ERROR UncaughtExceptionLogger - Exception in thread "ZAP-AjaxSpiderAuto" +java.lang.OutOfMemoryError: unable to create native thread: possibly out of memory or process/resource limits reached +``` ## Caveats diff --git a/config/config-template-zap-long.yaml b/config/config-template-zap-long.yaml index c650b7c1..53506a65 100644 --- a/config/config-template-zap-long.yaml +++ b/config/config-template-zap-long.yaml @@ -19,16 +19,13 @@ config: environ: envFile: "path/to/env/file" - # (Optional) configure to export scan results to OWASP Defect Dojo - defectDojo: - url: "https://mydefectdojo.example.com/" - # ssl: True|False|/path/to/CA/bundle (default: True). for SSL verification - ssl: True - authorization: - username: "rapidast" - password: "password" - # or - token: "abc" + # Export to Google Cloud Storage + googleCloudStorage: + keyFile: "/path/to/GCS/key" # optional: path to the GCS key file (alt.: use GOOGLE_APPLICATION_CREDENTIALS) + bucketName: "" # Mandatory + directory: "" # Optional, defaults to `RapiDAST-{app_name}` + + # `application` contains data related to the application, not to the scans. application: @@ -76,6 +73,28 @@ general: # When undefined, relies on rapidast-defaults.yaml, or `none` if nothing is set #type: "none" + # (Optional) configure to export the results to Defect Dojo. + # WARNING: requires an export to be configured: either config.googleCloudStorage or config.defectDojo + defectDojoExport: + # Parameters contain data that will directly be sent as parameters to DefectDojo's import/reimport endpoints. + # For example: commit tag, version, push_to_jira, etc. + # See https://demo.defectdojo.org/api/v2/doc/ for a list of possibilities + # The minimum set of data is whatever is needed to identify which engagement/test needs to be chosen. + # If neither a test ID (`test` parameter), nor product_name and engagement_name were provided, sane default will be attempted: + # - product_name chosen from either application.productName or application.shortName + # - engagement_name: "RapiDAST" [this way the same engagement will always be chosen, regardless of the scanner] + parameters: + product_name: "My Product" + engagement_name: "RapiDAST" + # - or - + #engagement: 3 # engagement ID + # - or - + #test_title: "ZAP" + # - or - + #test: 5 # test ID, that will force "reimport" mode + # additional options, see https://demo.defectdojo.org/api/v2/doc/ for list + auto_create_context: False # Optional. set to True to auto-create engagement (requires product_name and engagement_name) + # `scanners' is a section that configures scanning options scanners: zap: @@ -123,7 +142,7 @@ scanners: container: parameters: - image: "docker.io/owasp/zap2docker-stable:latest" # for type such as podman + image: "ghcr.io/zaproxy/zaproxy:stable" # for type such as podman #podName: "mypod" # optional: inject ZAP in an existing Pod executable: "zap.sh" # for Linux @@ -168,27 +187,3 @@ scanners: overrideConfigs: - formhandler.fields.field(0).fieldId=namespace - formhandler.fields.field(0).value=default - - - # (Optional) configure to export scan results to OWASP Defect Dojo. - # `config.defectDojo` must be configured first. - defectDojoExport: - type: "reimport" # choose between: import, reimport, False (disable export). Default (or other content): re-import if test is set - # Parameters contain data that will directly be sent as parameters to DefectDojo's import/reimport endpoints. - # For example: commit tag, version, push_to_jira, etc. - # See https://demo.defectdojo.org/api/v2/doc/ for a list of possibilities - # The minimum set of data is whatever is needed to identify which engagement/test needs to be chosen. - # If neither a test ID (`test` parameter), nor product_name and engagement_name were provided, sane default will be attempted: - # - product_name chosen from either application.productName or application.shortName - # - engagement_name: "RapiDAST" [this way the same engagement will always be chosen, regardless of the scanner] - parameters: - product_name: "My Product" - engagement_name: "RapiDAST" - # - or - - #engagement: 3 # engagement ID - # - or - - #test_title: "ZAP" - # - or - - #test: 5 # test ID, that will force "reimport" mode - # additional options, see https://demo.defectdojo.org/api/v2/doc/ for list - auto_create_context: False # Optional. set to True to auto-create engagement (requires product_name and engagement_name) diff --git a/config/config-template-zap-mac.yaml b/config/config-template-zap-mac.yaml index 12f319c5..46c2471b 100644 --- a/config/config-template-zap-mac.yaml +++ b/config/config-template-zap-mac.yaml @@ -32,7 +32,7 @@ general: # none: Default. RapiDAST runs each scanner in the same host or inside the RapiDAST image container # podman: RapiDAST orchestrates each scanner on its own using podman # When undefined, relies on rapidast-defaults.yaml, or `none` if nothing is set - #type: "none" + type: "none" scanners: zap: @@ -52,8 +52,8 @@ scanners: container: parameters: - image: "docker.io/owasp/zap2docker-stable:latest" # for type such as podman - executable: "/Applications/OWASP ZAP.app/Contents/Java/zap.sh" # for MacOS, when general.container.type is 'none' only, need to download OWASP ZAP https://www.zaproxy.org/download/ + image: "ghcr.io/zaproxy/zaproxy:stable" # for type such as podman + executable: "/Applications/ZAP.app/Contents/Java/zap.sh" # the path to ZAP for MacOS, used when general.container.type is 'none'. Installing ZAP is required from https://www.zaproxy.org/download/ miscOptions: # List (comma-separated string or list) of additional addons to install diff --git a/configmodel/__init__.py b/configmodel/__init__.py index d4859d30..13bb1758 100644 --- a/configmodel/__init__.py +++ b/configmodel/__init__.py @@ -176,6 +176,14 @@ def merge(self, merge, preserve=False, root=None): deep_dict_merge(sub_conf, merge, preserve) + def get_official_app_name(self): + """ Shortcut: + Return a string corresponding to how the application should be called + Based on the configuratoin. + Prefer the full product name, but defer to short name if unavailable + """ + return self.get("application.ProductName") or self.get("application.shortName") + def __repr__(self): return pformat(vars(self), indent=4, width=1) diff --git a/exports/defect_dojo.py b/exports/defect_dojo.py index e1d28aeb..ee13c124 100644 --- a/exports/defect_dojo.py +++ b/exports/defect_dojo.py @@ -159,7 +159,7 @@ def reimport_scan(self, data, filename): ) def import_scan(self, data, filename): - """Import to an existing engagement.""" + """export to an existing engagement, via the `import-scan` endpoint.""" if not data.get("engagement") and not ( data.get("engagement_name") and data.get("product_name") @@ -172,7 +172,7 @@ def import_scan(self, data, filename): f"{self.base_url}/api/v2/import-scan/", data, filename ) - def import_or_reimport_scan(self, data, filename): + def export_scan(self, data, filename): """Decide wether to import or reimport. Based on: - If the data contains a test ID ("test"): it's a reimport - Otherwise import diff --git a/exports/google_cloud_storage.py b/exports/google_cloud_storage.py new file mode 100755 index 00000000..2fe4883f --- /dev/null +++ b/exports/google_cloud_storage.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +import datetime +import json +import logging +import os +import random +import string +import tarfile +import uuid +from io import BytesIO +from io import StringIO + +from google.cloud import storage + + +class GoogleCloudStorage: + """ + Sends the results to a Google Cloud Storage bucket + """ + + def __init__(self, bucket_name, app_name, directory=None, keyfile=None): + if keyfile: + client = storage.Client.from_service_account_json(keyfile) + else: + client = storage.Client() + self.bucket = client.get_bucket(bucket_name) + self.directory = directory or f"RapiDAST-{app_name}" + self.app_name = app_name + + def create_metadata(self, data): + """ + Given a dictionary of key/values corresponding to Defectdojo's `import-scan` parameters, + return a Metadata dictionary + """ + metadata = { + "scan_type": data["scan_type"], + "uuid": str(uuid.uuid1()), + "import_data": data, + } + + return metadata + + def export_scan(self, data, filename): + """ + Send the scan to GCS + + Params: + data: a dictionary of key/values corresponding to Defectdojo's `import-scan` parameters + filename: path to the file containing scan + + """ + if not data or not filename: + # missing data means nothing to do + logging.debug("Insufficient data") + return 1 + + metadata = self.create_metadata(data) + + logging.info( + f"GoogleCloudStorage: sending {filename}. UUID: {metadata['uuid']}" + ) + + # export data as a metadata.json file + json_stream = StringIO() + json.dump(metadata, json_stream) + json_stream = BytesIO(json_stream.getvalue().encode("utf-8")) + json_stream.seek(0) + + # create a tar containing: "scans/" and metadata.json + tar_stream = BytesIO() + with tarfile.open(fileobj=tar_stream, mode="w:gz") as tar: + # add the metadata + info = tarfile.TarInfo(name="metadata.json") + info.size = len(json_stream.getvalue()) + tar.addfile(tarinfo=info, fileobj=json_stream) + + # add the scan + tar.add(name=filename, arcname=f"scans/{os.path.basename(filename)}") + tar_stream.seek(0) + + # generate the blob filename + unique_id = "{}-RapiDAST-{}-{}.tgz".format( # pylint: disable=C0209 + datetime.datetime.now(tz=datetime.timezone.utc).isoformat(), + self.app_name, + "".join( + random.choices( + string.ascii_letters + string.ascii_uppercase + string.digits, k=6 + ) + ), + ) + blob_name = self.directory + "/" + unique_id + + # push to GCS + blob = self.bucket.blob(blob_name) + with blob.open(mode="wb") as dest: + dest.write(tar_stream.getbuffer()) diff --git a/rapidast.py b/rapidast.py index ada81108..5b05ba4e 100755 --- a/rapidast.py +++ b/rapidast.py @@ -14,6 +14,7 @@ import configmodel.converter import scanners from exports.defect_dojo import DefectDojo +from exports.google_cloud_storage import GoogleCloudStorage from utils import add_logging_level pp = pprint.PrettyPrinter(indent=4) @@ -55,7 +56,7 @@ def load_config_file(config_file_location: str): return open(config_file_location, mode="r", encoding="utf-8") -def run_scanner(name, config, args, defect_d): +def run_scanner(name, config, args, scan_exporter): """given the config `config`, runs scanner `name`. Returns: 0 for success @@ -116,10 +117,10 @@ def run_scanner(name, config, args, defect_d): scanner.cleanup() # Part 6: export to defect dojo, if the scanner is compatible - if defect_d and hasattr(scanner, "data_for_defect_dojo"): + if scan_exporter and hasattr(scanner, "data_for_defect_dojo"): logging.info("Exporting results to the Defect Dojo service as configured") - if defect_d.import_or_reimport_scan(*scanner.data_for_defect_dojo()) == 1: + if scan_exporter.export_scan(*scanner.data_for_defect_dojo()) == 1: logging.error("Exporting results to DefectDojo failed") return 1 @@ -191,10 +192,19 @@ def run(): # Do early: load the environment file if one is there load_environment(config) - # Prepare Defect Dojo if configured - defect_d = None - if config.get("config.defectDojo.url"): - defect_d = DefectDojo( + # Prepare an export to Defect Dojo if one is configured. + scan_exporter = None + if config.get("config.googleCloudStorage.bucketName"): + scan_exporter = GoogleCloudStorage( + bucket_name=config.get( + "config.googleCloudStorage.bucketName", "default-bucket-name" + ), + app_name=config.get_official_app_name(), + directory=config.get("config.googleCloudStorage.directory", None), + keyfile=config.get("config.googleCloudStorage.keyFile", None), + ) + elif config.get("config.defectDojo.url"): + scan_exporter = DefectDojo( config.get("config.defectDojo.url"), { "username": config.get( @@ -213,7 +223,7 @@ def run(): for name in config.get("scanners"): logging.info(f"Next scanner: '{name}'") - ret = run_scanner(name, config, args, defect_d) + ret = run_scanner(name, config, args, scan_exporter) if ret == 1: logging.info(f"scanner: '{name}' failed") scan_error_count = scan_error_count + 1 diff --git a/requirements-dev.txt b/requirements-dev.txt index da7022e2..3993dc3e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,3 +3,4 @@ pyyaml pytest >= 7.2.1 black requests +pre-commit == 3.7.1 diff --git a/requirements.txt b/requirements.txt index 0c0111d5..c64861c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ python-dotenv >= 1.0.0 pyyaml >= 6.0 requests >= 2.27.1 +google.cloud.storage >= 2.17.0 diff --git a/scanners/__init__.py b/scanners/__init__.py index 7fadb353..e0733cd8 100644 --- a/scanners/__init__.py +++ b/scanners/__init__.py @@ -1,6 +1,8 @@ +import datetime import importlib import logging import os +import shutil import tempfile from enum import Enum from pprint import pformat @@ -27,6 +29,10 @@ def __init__(self, config, ident): self.config.get("config.results_dir", default="results"), self.ident ) + # When requested to create a temporary file or directory, it will be a subdir of + # this temporary directory + self.main_temp_dir = None + def my_conf(self, path, *args, **kwargs): """Handy shortcut to get the scanner's configuration. Only for within `scanners.` @@ -39,18 +45,102 @@ def set_my_conf(self, path, *args, **kwargs): """ return self.config.set(f"scanners.{self.ident}.{path}", *args, **kwargs) + def _should_export_to_defect_dojo(self): + """Return a truthful value if Defect Dojo export is configured and not disbaled + Returns True if: + - an global export is configured (config.googleCloudStorage or config.defectDojo) + - this particular scanner's export is not explicitely disabled (`defectDojoExport` is not False) + """ + return self.my_conf("defectDojoExport") is not False and ( + self.config.get("config.googleCloudStorage") + or self.config.get("config.defectDojo") + ) + + def _fill_up_data_for_defect_dojo(self, data): + """ + Parent / common code for extracting data for defectdojo. + This code should be called by the scanner's data_for_defect_dojo() + Assumptions: + - data["scan_type"] is already set + """ + # default values + default = { + "product_name": self.config.get_official_app_name(), + "active": True, + "verified": False, + } + for key, value in default.items(): + if key not in data: + data[key] = value + + # lists of configured import parameters + params_root = "defectDojoExport.parameters" + import_params = self.my_conf(params_root, default={}).keys() + + # overload that list onto the defaults + for param in import_params: + data[param] = self.my_conf(f"{params_root}.{param}") + + if data.get("test") is not None: + # A test ID is provided: it takes precedence. + # This is a reimport + # remove unnecessary data + for e in ("product_name", "engagement_name", "engagement"): + if e in data: + del data[e] + elif data.get("engagement") is not None: + # An engagement ID is provided + # remove unnecessary data + for e in ("product_name", "engagement_name"): + if e in data: + del data[e] + else: + # Neither test of engagement IDs provided: make sure there is enough data for import + # A default product name was chosen as part of `self.get_default_defectdojo_data()` + # Generate an engagement name if none are set + if not data.get("engagement_name"): + data[ + "engagement_name" + ] = f"RapiDAST-{data['product_name']}-{datetime.date.today()}" + + return data + def __repr__(self): return pformat(vars(self), indent=4, width=1) def _create_temp_dir(self, name="X"): - """This function simply creates a temporary directory aiming at storing data in transit. - This directory must be manually deleted by the caller during cleanup. - Descendent classes *may* overload this directory (e.g.: if they can't map /tmp) """ - temp_dir = tempfile.mkdtemp(prefix=f"rapidast_{self.ident}_{name}_") - logging.debug(f"Temporary directory created in host: {temp_dir}") + This function creates a temporary directory aiming at storing data used by the scanner + Then create subdirectories based on the name given, under that temporary + Return the full path to that location. + + Example: + -> on first call, `zap._create_temp_dir(name="home")` + will create /tmp/rapidast_zap_RanD0m/home/ + -> on second call, `zap._create_temp_dir(name="work")` + will create /tmp/rapidast_zap_RanD0m/work/ + + The /tmp/rapidast_zap_RanD0m/ directory will be removed in cleanup() + """ + # first call: create a temporary dir + if not self.main_temp_dir: + self.main_temp_dir = tempfile.mkdtemp(prefix=f"rapidast_{self.ident}_") + logging.debug(f"Temporary directory created in host: {self.main_temp_dir}") + + temp_dir = os.path.join(self.main_temp_dir, name) + os.mkdir(temp_dir) return temp_dir + def cleanup(self): + """Generic Scanner cleanup: should be called only via super() inheritance + Deletes the _create_temp_dir() parent directory + """ + if self.main_temp_dir: + logging.debug(f"Deleting temp directories {self.main_temp_dir}") + shutil.rmtree(self.main_temp_dir) + else: + logging.debug("No temporary file to cleanup") + def str_to_scanner(name, method): """Given a scanner ID (name) and a container method, returns the class to be loaded diff --git a/scanners/generic/generic.py b/scanners/generic/generic.py index bd6ff13d..4b96b6cc 100644 --- a/scanners/generic/generic.py +++ b/scanners/generic/generic.py @@ -101,31 +101,9 @@ def data_for_defect_dojo(self): filename = f"{self.results_dir}/{sarif_filename}" logging.debug(f"export {filename} to defect dojo") - # default, mandatory values (which can be overloaded) - data = { - "scan_type": "SARIF", - "active": True, - "verified": False, - } - - # lists of configured import parameters - params_root = "defectDojoExport.parameters" - import_params = self.my_conf(params_root, default={}).keys() - - # overload that list onto the defaults - for param in import_params: - data[param] = self.my_conf(f"{params_root}.{param}") - - if data.get("test") is None: - # No test ID provided, so we need to make sure there is enough info - # But we can't make it default (they should not be filled if there is a test ID - if not data.get("product_name"): - data["product_name"] = self.config.get( - "application.ProductName" - ) or self.config.get("application.shortName") - if not data.get("engagement_name"): - data["engagement_name"] = "RapiDAST" - return data, filename + data = {"scan_type": "SARIF"} + + return (self._fill_up_data_for_defect_dojo(data), filename) ############################################################### # PROTECTED METHODS # @@ -153,13 +131,6 @@ def _add_env(self, key, value=None): # Those are called only from Generic itself # ############################################################### - def _should_export_to_defect_dojo(self): - """Return a truthful value if Defect Dojo export is configured and not disbaled""" - return ( - self.my_conf("defectDojoExport", default=False) is not False - and self.my_conf("defectDojoExport.type", default=False) is not False - ) - ############################################################### # MAGIC METHODS # # Special functions (other than __init__()) # diff --git a/scanners/zap/zap.py b/scanners/zap/zap.py index 3d77ee6f..6162a335 100644 --- a/scanners/zap/zap.py +++ b/scanners/zap/zap.py @@ -98,19 +98,8 @@ def postprocess(self): # log path is like '/tmp/rapidast_*/zap.log' tar.add(log, f"evidences/zap_logs/{log.split('/')[-1]}") - def cleanup(self): - """Generic ZAP cleanup: should be called only via super() inheritance - Deletes home and work directory - """ - - logging.debug( - f"Deleting temp directories {self.host_work_dir} and {self.host_home_dir}" - ) - shutil.rmtree(self.host_work_dir) - shutil.rmtree(self.host_home_dir) - def data_for_defect_dojo(self): - """Return a tuple containing: + """Returns a tuple containing: 1) Metadata for the test (dictionary) 2) Path to the result file (string) For additional info regarding the metadata, see the `import-scan`/`reimport-scan` @@ -125,32 +114,9 @@ def data_for_defect_dojo(self): # the XML report is supposed to have been forcefully added, and expected to exist filename = f"{self.results_dir}/zap-report.xml" - # default, mandatory values (which can be overloaded) - data = { - "scan_type": "ZAP Scan", - "active": True, - "verified": False, - } - - # lists of configured import parameters - params_root = "defectDojoExport.parameters" - import_params = self.my_conf(params_root, default={}).keys() - - # overload that list onto the defaults - for param in import_params: - data[param] = self.my_conf(f"{params_root}.{param}") - - if data.get("test") is None: - # No test ID provided, so we need to make sure there is enough info - # But we can't make it default (they should not be filled if there is a test ID - if not data.get("product_name"): - data["product_name"] = self.config.get( - "application.ProductName" - ) or self.config.get("application.shortName") - if not data.get("engagement_name"): - data["engagement_name"] = "RapiDAST" + data = {"scan_type": "ZAP Scan"} - return data, filename + return (self._fill_up_data_for_defect_dojo(data), filename) def get_update_command(self): """Returns a list of all options required to update ZAP plugins""" @@ -603,13 +569,6 @@ def _construct_report_af(self, report_format): return report_af - def _should_export_to_defect_dojo(self): - """Return a truthful value if Defect Dojo export is configured and not disbaled""" - return ( - self.my_conf("defectDojoExport", default=False) is not False - and self.my_conf("defectDojoExport.type", default=False) is not False - ) - def _setup_report(self): """Adds the report to the job list. This should be called last""" @@ -631,7 +590,7 @@ def _setup_report(self): # DefectDojo requires XML report type if self._should_export_to_defect_dojo(): - logging.debug("ZAP report: ensures XML report for Defect Dojo") + logging.debug("ZAP report: ensures XML report for Export") formats.add("xml") appended = 0 @@ -917,11 +876,11 @@ def ensure_list(entry): except: pass logging.warning( - f"No context matching {context} have ben found in the current Automation Framework configuration.", + f"No context matching {context} have ben found in the current Automation Framework configuration." "It may be missing from default. An empty context is created", ) # something failed: create an empty one and return it - if not automation_config["env"]: + if not automation_config.get("env"): automation_config["env"] = {} if not automation_config["env"].get("contexts"): automation_config["env"]["contexts"] = [] diff --git a/scanners/zap/zap_none.py b/scanners/zap/zap_none.py index 2c687fdb..726cfec9 100644 --- a/scanners/zap/zap_none.py +++ b/scanners/zap/zap_none.py @@ -91,6 +91,9 @@ def setup(self): if self.state != State.ERROR: self.state = State.READY + # Change HOME if needed + self._create_home_if_needed() + def run(self): """If the state is READY, run the final run command on the local machine There is no need to call super() here. @@ -270,3 +273,14 @@ def _check_plugin_status(self): logging.warning( f"ZAP appears to be in a incorrect state. Error: {result.stderr}" ) + + def _create_home_if_needed(self): + """Some tools (most notably: ZAP's Ajax Spider with Firefox) require a writable home directory. + When RapiDAST is run in Openshift, the user's home is /, which is not writable. + In that case, create a temporary directory and redirect $HOME to that directory + """ + # test if HOME is writable. In that case, nothing needs to be done + if os.access(os.environ["HOME"], os.W_OK): + return + os.environ["HOME"] = self._create_temp_dir("home") + logging.debug(f"Replaced HOME directory, to {os.environ['HOME']}") diff --git a/scanners/zap/zap_podman.py b/scanners/zap/zap_podman.py index c86aeeb9..1af27ee6 100644 --- a/scanners/zap/zap_podman.py +++ b/scanners/zap/zap_podman.py @@ -21,7 +21,7 @@ class ZapPodman(Zap): # PRIVATE CONSTANTS # # Accessed by ZapPodman only # ############################################################### - DEFAULT_IMAGE = "docker.io/owasp/zap2docker-stable:latest" + DEFAULT_IMAGE = "ghcr.io/zaproxy/zaproxy:stable" ############################################################### # PROTECTED CONSTANTS # @@ -153,6 +153,22 @@ def cleanup(self): if not self.state == State.ERROR: self.state = State.CLEANEDUP + ############################################################### + # OVERLOADED METHODS # + # Method overloading parent class # + ############################################################### + + def _setup_ajax_spider(self): + """Ajax requires a lot of shared memory""" + + if self.my_conf("spiderAjax", default=False) is False: + return + + self.podman.add_option("--shm-size", "2g") + + # Regular Ajax setup + super()._setup_ajax_spider() + ############################################################### # PROTECTED METHODS # # Accessed by Zap parent only # diff --git a/tests/exports/test_google_cloud_storage.py b/tests/exports/test_google_cloud_storage.py new file mode 100644 index 00000000..f438cf89 --- /dev/null +++ b/tests/exports/test_google_cloud_storage.py @@ -0,0 +1,96 @@ +import pytest + +from unittest.mock import Mock, MagicMock, patch, mock_open + +import datetime + +from exports.google_cloud_storage import GoogleCloudStorage + + + +@patch("exports.google_cloud_storage.storage.Client.from_service_account_json") +def test_GCS_simple_init_keyfile(mock_from_json): + # catching the Client + mock_client = MagicMock() + mock_from_json.return_value = mock_client + + gcs = GoogleCloudStorage("bucket_name", "app_name", "directory_name", "/key/file.json") + + assert gcs.directory == "directory_name" + assert gcs.app_name == "app_name" + mock_from_json.assert_called_once_with("/key/file.json") + mock_client.get_bucket.assert_called_once_with("bucket_name") + +@patch("exports.google_cloud_storage.storage.Client") +def test_GCS_simple_init_no_keyfile(mock_client): + gcs = GoogleCloudStorage("bucket_name", "app_name", "directory_name") + + assert gcs.directory == "directory_name" + assert gcs.app_name == "app_name" + mock_client.assert_called_once_with() + + +@patch("exports.google_cloud_storage.storage.Client") +@patch("exports.google_cloud_storage.uuid") +def test_GCS_create_metadata(mock_uuid, mock_client): + + mock_uuid.uuid1.return_value = 123 + + gcs = GoogleCloudStorage("bucket_name", "app_name", "directory_name") + + import_data = { + "scan_type": "ABC", + "engagement_name": "engagement", + "product_name": "product", + } + + meta = gcs.create_metadata(import_data) + + assert meta["scan_type"] == import_data["scan_type"] + assert meta["uuid"] == "123" + assert meta["import_data"] == import_data + assert meta["import_data"]["engagement_name"] == "engagement" + assert meta["import_data"]["product_name"] == "product" + + +@patch("exports.google_cloud_storage.storage.Client") +@patch("exports.google_cloud_storage.datetime.datetime") +@patch("exports.google_cloud_storage.random.choices") +def test_GCS_export_scan(MockRandom, MockDateTime, MockClient): + # Forcing the random + MockRandom.return_value = "abcdef" + + # Forcing the date + mock_now = MagicMock() + mock_now.isoformat.return_value = '2024-01-31T00:00:00' + MockDateTime.now.return_value = mock_now + + # catching the Client + mock_client = MagicMock() + MockClient.return_value = mock_client + + # catching the bucket + mock_bucket = MagicMock() + mock_client.get_bucket.return_value = mock_bucket + + # catching the blob + mock_blob = MagicMock() + mock_bucket.blob.return_value = mock_blob + + # catching the data written to the blob + mock_open_method = mock_open() + mock_blob.open = mock_open_method + + gcs = GoogleCloudStorage("bucket_name", "app_name", "directory_name") + + import_data = { + "scan_type": "ABC", + "foo": "bar" + } + + # hack: use the pytest file itself as a scan + gcs.export_scan(import_data, __file__) + + mock_bucket.blob.assert_called_once_with("directory_name/2024-01-31T00:00:00-RapiDAST-app_name-abcdef.tgz") + + mock_open_method.assert_called_once_with(mode="wb") diff --git a/tests/scanners/zap/test_setup_podman.py b/tests/scanners/zap/test_setup_podman.py index 1d0feab9..fe1a0bd1 100644 --- a/tests/scanners/zap/test_setup_podman.py +++ b/tests/scanners/zap/test_setup_podman.py @@ -184,6 +184,18 @@ def test_setup_podman_pod_injection(test_config): # Misc tests +def test_podman_handling_ajax(test_config): + test_config.set("scanners.zap.spiderAjax.url", "https://abcdef.jklm") + test_zap = ZapPodman(config=test_config) + # create a fake automation framework: just an empty `jobs` is sufficient + test_zap.automation_config = {"jobs": []} + test_zap._setup_ajax_spider() + + cli = test_zap.podman.get_complete_cli() + i = cli.index("--shm-size") + assert cli[i + 1] == "2g" + + def test_podman_handling_plugins(test_config): test_config.set("scanners.zap.miscOptions.updateAddons", True) test_config.set("scanners.zap.miscOptions.additionalAddons", "pluginA,pluginB")