Enhance the OpenAPI plugin to accept user-provided values for parameters

doc/sphinx/scan-rest-apis.rst

@@ -57,3 +57,80 @@ HTTP requests into ``w3af``:
     requests captured by the proxy. The steps where the user teaches ``w3af``
     about all the API endpoints and parameters is key to the success
     of the security audit.
+
+Providing custom parameter values for API endpoints
+---------------------------------------------------
+
+The ``crawl.open_api`` plugin tries to guess valid values for parameters of API endpoints.
+The plugin takes into account types of parameters and other info from the API specification,
+and then tries its best to find acceptable parameter values.
+But these values may highly depend on the context.
+For example, if an API endpoint takes an numeric user ID as a parameter,
+then most probably the plugin will use a number for this parameter,
+although most likely it's going to be an invalid user ID.
+As a result, the chance that the endpoint will reject such requests is quite high
+because there is no user with such ID. If the plugin knew a valid user ID for testing,
+it might increase chances to catch a vulnerability
+which might exist after that check for a valid user ID.
+
+If users have some knowledge about correct values which may be used for testing,
+they can tell the plugin about them via ``parameter_values_file`` configuration parameter.
+The parameter specifies a path to a YAML config which contains values
+which should be used by the plugin to fill out parameters in HTTP requests.
+
+Here is an example of such a YAML config:
+
+::
+
+    - path: /users/{user-id}
+      parameters:
+      - name: user-id
+        values:
+        - 1234567
+      - name: X-First-Name
+        values:
+        - John
+        - Bill
+      - path: /users
+        parameters:
+        - name: user-id
+          values:
+          - 1234567
+        - name: X-Birth-Date
+          values:
+          - 2000-01-02
+
+The configuration above tells the ``crawl.open_api`` plugin the following:
+
+ * For the ``/users/{user-id}`` endpoint, use ``1234567`` number for ``user-id`` parameter,
+   and ``John`` and ``Bill`` strings for ``X-First-Name`` parameter.
+ * For the ``/users`` endpoint, use ``1234567`` number for ``user-id`` parameter,
+   and ``2000-01-02`` date for ``X-Birth-Date`` parameter.
+
+If a user provides multiple values for parameters, then the plugin tries to enumerate
+all possible combinations of parameters. With the configuration above,
+the plugin is going to generate at least three HTTP requests
+which are going to look like the following:
+
+::
+
+    GET /users/1234567
+    X-First-Name: John
+    ...
+
+    GET /users/1234567
+    X-First-Name: Bill
+    ...
+
+    POST /users?user-id=1234567
+    X-Birth-Date: 200-01-02
+    ...
+
+In this example, we made several assumptions about the API specification for the endpoints:
+
+ * Both ``X-First-Name`` and ``X-Birth-Data`` are headers
+ * ``user-id`` is a parameter in query string for the ``/users`` endpoint.
+ * The ``/users/{user-id}`` endpoint accepts GET requests.
+ * The ``/users`` endpoint accepts POST requests.
+
+Note that the plugin doesn't currently take parameter types into account.

w3af/core/data/options/input_file_option.py

@@ -63,15 +63,8 @@ def set_value(self, value):
 
         validated_value = self.validate(value)
 
-        # I want to make the paths shorter, so we're going to make them
-        # relative, at least in the case where they are inside the cwd
-        current_dir = os.path.abspath(os.curdir)
-        configured_value_dir = os.path.abspath(os.path.dirname(validated_value))
-
-        if configured_value_dir.startswith(current_dir):
-            self._value = os.path.relpath(validated_value)
-        else:
-            self._value = validated_value
+        # I want to make the paths shorter
+        self._value = self.get_relative_path(validated_value)
 
     def get_value_for_profile(self, self_contained=False):
         """
@@ -222,3 +215,16 @@ def encode_b64_data(self, filename):
         data = base64.b64encode(file(filename).read().encode('zlib')).strip()
         return '%s%s' % (self.DATA_PROTO, data)
 
+    @staticmethod
+    def get_relative_path(path):
+        """
+        Tries to make the path shorter. The method is going to make the path
+        relative, at least in the case where it's inside the cwd.
+        """
+        current_dir = os.path.abspath(os.curdir)
+        configured_value_dir = os.path.abspath(os.path.dirname(path))
+
+        if configured_value_dir.startswith(current_dir):
+            return os.path.relpath(path)
+
+        return path

w3af/core/data/options/opt_factory.py

@@ -40,8 +40,9 @@
 
 from w3af.core.data.options.option_types import (
     BOOL, INT, POSITIVE_INT, FLOAT, STRING, URL, IPPORT,
-    LIST, REGEX, COMBO, INPUT_FILE, QUERY_STRING, HEADER,
+    LIST, REGEX, COMBO, INPUT_FILE, YAML_INPUT_FILE, QUERY_STRING, HEADER,
     OUTPUT_FILE, PORT, IP, URL_LIST, FORM_ID_LIST)
+from w3af.core.data.options.yaml_file_option import YamlFileOption
 
 
 def opt_factory(name, default_value, desc, _type, help='', tabid=''):
@@ -64,6 +65,7 @@ def opt_factory(name, default_value, desc, _type, help='', tabid=''):
         REGEX: RegexOption,
         COMBO: ComboOption,
         INPUT_FILE: InputFileOption,
+        YAML_INPUT_FILE: YamlFileOption,
         OUTPUT_FILE: OutputFileOption,
         PORT: PortOption,
         IP: IPOption,

w3af/core/data/options/option_types.py

@@ -37,6 +37,7 @@
 # Files
 OUTPUT_FILE = 'output_file'
 INPUT_FILE = 'input_file'
+YAML_INPUT_FILE = 'yaml_input_file'
 
 # Misc
 BOOL = 'boolean'

w3af/core/data/options/tests/invalid.yaml

@@ -0,0 +1,2 @@
+foo: bar
+invalid: {

w3af/core/data/options/tests/test_yaml_file_option.py

@@ -0,0 +1,57 @@
+"""
+test_yaml_file_option.py
+
+Copyright 2018 Andres Riancho
+
+This file is part of w3af, http://w3af.org/ .
+
+w3af is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation version 2 of the License.
+
+w3af is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with w3af; if not, write to the Free Software
+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+"""
+import os
+import unittest
+
+from w3af import ROOT_PATH
+from w3af.core.controllers.exceptions import BaseFrameworkException
+from w3af.core.data.options.opt_factory import opt_factory
+from w3af.core.data.options.option_types import YAML_INPUT_FILE
+
+
+class TestYamlFileOption(unittest.TestCase):
+
+    VALID_INPUT_YAML_FILE = os.path.relpath(os.path.join(
+        ROOT_PATH, 'core', 'data', 'options', 'tests', 'valid.yaml'))
+    INVALID_INPUT_YAML_FILE = os.path.relpath(os.path.join(
+        ROOT_PATH, 'core', 'data', 'options', 'tests', 'invalid.yaml'))
+
+    def test_valid_yaml_file(self):
+        opt = opt_factory('name', self.VALID_INPUT_YAML_FILE,
+                          'desc', YAML_INPUT_FILE, 'help', 'tab')
+
+        self.assertEqual(opt.get_value(), self.VALID_INPUT_YAML_FILE)
+        self.assertEqual(opt.get_value_for_profile(),
+                         '%ROOT_PATH%/core/data/options/tests/valid.yaml')
+
+        with open(self.VALID_INPUT_YAML_FILE, 'r') as expected:
+            with open(opt.get_value(), 'r') as actual:
+                self.assertEqual(expected.read(), actual.read())
+
+    def test_invalid_yaml_file(self):
+        with self.assertRaises(BaseFrameworkException):
+            opt_factory('name', self.INVALID_INPUT_YAML_FILE,
+                        'desc', YAML_INPUT_FILE, 'help', 'tab')
+
+    def test_file_not_exist(self):
+        with self.assertRaises(BaseFrameworkException):
+            opt_factory('name', 'this/does/not/exist',
+                        'desc', YAML_INPUT_FILE, 'help', 'tab')

w3af/core/data/options/tests/valid.yaml

@@ -0,0 +1,13 @@
+Date: 2001-11-23 15:03:17 -5
+User: ed
+Fatal:
+  Unknown variable "bar"
+Stack:
+  - file: TopClass.py
+    line: 23
+    code: |
+      x = MoreObject("345\n")
+  - file: MoreClass.py
+    line: 58
+    code: |-
+      foo = bar

w3af/core/data/options/yaml_file_option.py

@@ -0,0 +1,95 @@
+"""
+yaml_file_option.py
+
+Copyright 2018 Andres Riancho
+
+This file is part of w3af, http://w3af.org/ .
+
+w3af is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation version 2 of the License.
+
+w3af is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with w3af; if not, write to the Free Software
+Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+"""
+import os
+
+import yaml
+
+from w3af import ROOT_PATH
+from w3af.core.controllers.exceptions import BaseFrameworkException
+from w3af.core.data.options.baseoption import BaseOption
+from w3af.core.data.options.input_file_option import InputFileOption
+from w3af.core.data.options.option_types import YAML_INPUT_FILE
+
+ROOT_PATH_VAR = '%ROOT_PATH%'
+
+
+class YamlFileOption(BaseOption):
+
+    _type = YAML_INPUT_FILE
+
+    def set_value(self, value):
+        """
+        :param value: The value parameter is set by the user interface, which
+        for example sends 'spec/openapi.yaml' or '%ROOT_PATH%/values.yml'.
+
+        If required we replace the %ROOT_PATH% with the right value for this
+        platform.
+        """
+        if value == '':
+            self._value = value
+            return
+
+        validated_value = self.validate(value)
+
+        self._value = InputFileOption.get_relative_path(validated_value)
+
+    def validate(self, value):
+        """
+        Check if a file has a valid Yaml data.
+        The method throws BaseFrameworkException if something goes wrong.
+
+        :param value: User input, which might look like:
+                        * spec/openapi.yaml
+                        * %ROOT_PATH%/values.yml
+        :return: Validated path to show to the user.
+        """
+        filename = value.replace(ROOT_PATH_VAR, ROOT_PATH)
+
+        try:
+            with open(filename, 'r') as content:
+                yaml_data = yaml.load(content)
+                if yaml_data is None:
+                    msg = 'No Yaml loaded from %s.'
+                    raise BaseFrameworkException(msg % value)
+        except IOError, e:
+            msg = 'Could not read file %s, error: %s.'
+            raise BaseFrameworkException(msg % (value, e))
+        except yaml.YAMLError, e:
+            msg = 'Could not parse YAML: %s.'
+            raise BaseFrameworkException(msg % e)
+        except Exception, e:
+            msg = 'Unexpected exception, error: %s.'
+            raise BaseFrameworkException(msg % e)
+
+        # Everything looks fine.
+        return filename
+
+    def get_value_for_profile(self, self_contained=False):
+        """
+        This method is called before saving the option value to the profile file
+
+        :return: A string representation of the path, with the ROOT_PATH
+                 replaced with %ROOT_PATH%. Then when we load a value in
+                 set_value we're going to replace the %ROOT_PATH% with ROOT_PATH
+        """
+        abs_path = os.path.abspath(self._value)
+        return abs_path.replace(ROOT_PATH, ROOT_PATH_VAR)

w3af/core/data/parsers/doc/open_api/main.py

@@ -24,6 +24,8 @@
 
 from yaml import load
 
+from w3af.core.data.parsers.doc.open_api.parameters import ParameterValues
+
 try:
     from yaml import CLoader as Loader, CDumper as Dumper
 except ImportError:
@@ -62,11 +64,25 @@ class OpenAPI(BaseParser):
                 'swagger',
                 'paths')
 
-    def __init__(self, http_response, no_validation=False, discover_fuzzable_headers=True):
+    def __init__(self, http_response,
+                 no_validation=False,
+                 discover_fuzzable_headers=True,
+                 custom_parameter_values=ParameterValues()):
+        """
+        Initialize OpenAPI plugin.
+        :param http_response: An HTTP response with an OpenAPI specification.
+        :param no_validation: Turns on/off validation of the OpenAPI spec.
+        :param discover_fuzzable_headers: Turns on/off discovering HTTP headers
+                                          which are used by API endpoints
+                                          and which can then be used for testing/fuzzing.
+        :param custom_parameter_values: Sets context-specific values for parameters
+                                        used by the API endpoints.
+        """
         super(OpenAPI, self).__init__(http_response)
         self.api_calls = []
         self.no_validation = no_validation
         self.discover_fuzzable_headers = discover_fuzzable_headers
+        self.custom_parameter_values = custom_parameter_values
 
     @staticmethod
     def content_type_match(http_resp):
@@ -144,7 +160,8 @@ def parse(self):
         and stores them in to the fuzzable request
         """
         specification_handler = SpecificationHandler(self.get_http_response(),
-                                                     self.no_validation)
+                                                     self.no_validation,
+                                                     self.custom_parameter_values)
 
         for data in specification_handler.get_api_information():
             try:
@@ -180,7 +197,8 @@ def parse(self):
 
                 self.api_calls.append(fuzzable_request)
 
-    def _should_audit(self, fuzzable_request):
+    @staticmethod
+    def _should_audit(fuzzable_request):
         """
         We want to make sure that w3af doesn't delete all the items from the
         REST API, so we ignore DELETE calls.