feat(testing): simulate multipart file upload

docs/changes/4.0.0.rst

@@ -15,7 +15,6 @@ Changes to Supported Platforms
 - CPython 3.11 is now fully supported. (`#2072 <https://github.com/falconry/falcon/issues/2072>`__)
 - End-of-life Python 3.5 & 3.6 are no longer supported. (`#2074 <https://github.com/falconry/falcon/pull/2074>`__)
 
-
 .. towncrier release notes start
 
 Contributors to this Release

falcon/testing/client.py

@@ -22,18 +22,20 @@
 import datetime as dt
 import inspect
 import json as json_module
+import os
 import time
 from typing import Dict
 from typing import Optional
 from typing import Sequence
 from typing import Union
+from urllib.parse import urlencode
 import warnings
 import wsgiref.validate
 
 from falcon.asgi_spec import ScopeType
 from falcon.constants import COMBINED_METHODS
-from falcon.constants import MEDIA_JSON
-from falcon.errors import CompatibilityError
+from falcon.constants import MEDIA_JSON, MEDIA_URLENCODED
+from falcon.errors import CompatibilityError, HTTPBadRequest
 from falcon.testing import helpers
 from falcon.testing.srmock import StartResponseMock
 from falcon.util import async_to_sync
@@ -95,7 +97,7 @@ class Cookie:
             or ``None`` if not specified.
         max_age (int): The lifetime of the cookie in seconds, or
             ``None`` if not specified.
-        secure (bool): Whether or not the cookie may only only be
+        secure (bool): Whether or not the cookie may only be
             transmitted from the client via HTTPS.
         http_only (bool): Whether or not the cookie may only be
             included in unscripted requests from the client.
@@ -437,6 +439,8 @@ def simulate_request(
     content_type=None,
     body=None,
     json=None,
+    files=None,
+    data=None,
     file_wrapper=None,
     wsgierrors=None,
     params=None,
@@ -528,6 +532,30 @@ def simulate_request(
             overrides `body` and sets the Content-Type header to
             ``'application/json'``, overriding any value specified by either
             the `content_type` or `headers` arguments.
+
+            Note:
+                Can only be used if data and files are null, otherwise an exception
+                is thrown.
+
+        files(dict): same as the  files parameter in requests,
+            dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``)
+            for multipart encoding upload.
+            ``file-tuple``: can be a 2-tuple ``('filename', fileobj)`` or a
+            3-tuple ``('filename', fileobj, 'content_type')``
+            where ``'content-type'`` is a string defining the content
+            type of the given file.
+
+            Note:
+                If both data and json are present, an exception is thrown.
+                To pass additional form-data with files, use data.
+
+        data : list of tuples, dict or (b)string, with additional data
+            to be passed with files (or alone if files is null), to be treated
+            as urlencoded form data.
+
+            Note:
+                If both data and json are present, an exception is thrown.
+
         file_wrapper (callable): Callable that returns an iterable,
             to be used as the value for *wsgi.file_wrapper* in the
             WSGI environ (default: ``None``). This can be used to test
@@ -575,6 +603,8 @@ def simulate_request(
             content_type=content_type,
             body=body,
             json=json,
+            files=files,
+            data=data,
             params=params,
             params_csv=params_csv,
             protocol=protocol,
@@ -598,6 +628,8 @@ def simulate_request(
         headers,
         body,
         json,
+        files,
+        data,
         extras,
     )
 
@@ -622,7 +654,7 @@ def simulate_request(
         # NOTE(vytas): Even given the duct tape nature of overriding
         # arbitrary environ variables, changing the method can potentially
         # be very confusing, particularly when using specialized
-        # simulate_get/post/patch etc methods.
+        # simulate_get/post/patch etc. methods.
         raise ValueError(
             'WSGI environ extras may not override the request method. '
             'Please use the method parameter.'
@@ -651,6 +683,8 @@ async def _simulate_request_asgi(
     content_type=None,
     body=None,
     json=None,
+    files=None,
+    data=None,
     params=None,
     params_csv=True,
     protocol='http',
@@ -736,6 +770,29 @@ async def _simulate_request_asgi(
             overrides `body` and sets the Content-Type header to
             ``'application/json'``, overriding any value specified by either
             the `content_type` or `headers` arguments.
+
+             Note:
+                Can only be used if data and files are null, otherwise an exception
+                is thrown.
+
+        files(dict): same as the  files parameter in requests,
+            dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``)
+            for multipart encoding upload.
+            ``file-tuple``: can be a 2-tuple ``('filename', fileobj)`` or a
+            3-tuple ``('filename', fileobj, 'content_type')``,
+            where ``'content-type'`` is a string defining the content
+            type of the given file.
+
+            Mote:
+                If both files and json are present, an exception is thrown. To pass
+                additional form-data with files, use data.
+
+        data : list of tuples, dict or (b)string with additional data to be passed with
+            files (or alone if files is null), to be treated as urlencoded form data.
+
+            Note:
+                If both data and json are present, an exception is thrown.
+
         host(str): A string to use for the hostname part of the fully
             qualified request URL (default: 'falconframework.org')
         remote_addr (str): A string to use as the remote IP address for the
@@ -774,6 +831,8 @@ async def _simulate_request_asgi(
         headers,
         body,
         json,
+        files,
+        data,
         extras,
     )
 
@@ -2133,8 +2192,163 @@ async def __aexit__(self, exc_type, exc, tb):
         await self._task_req
 
 
+def _prepare_data_fields(data, boundary=None, urlenc=False):
+    """Prepare data  fields for request body.
+
+    Args:
+        data: dict or list of tuples with json data from the request
+
+    Returns: list of 2-tuples (field-name(str), value(bytes))
+
+    """
+    urlresult = []
+    body_part = b''
+    if isinstance(data, (str, bytes)) or hasattr(data, 'read'):
+        fields = list(json_module.loads(data).items())
+    elif isinstance(data, dict):
+        fields = list(data.items())
+    else:
+        fields = list(data)
+
+    # Append data to the other multipart parts
+    for field, val in fields:
+        if isinstance(val, str) or not hasattr(val, '__iter__'):
+            val = [val]
+        # if no files are passed, make urlencoded form
+        if urlenc:
+            for v in val:
+                if v:
+                    urlresult.append(
+                        (
+                            field.encode('utf-8') if isinstance(field, str) else field,
+                            v.encode('utf-8') if isinstance(v, str) else v,
+                        )
+                    )
+        # if files and data are passed, concat data to files body like in requests
+        else:
+            for v in val:
+                body_part += (
+                    f'Content-Disposition: form-data; name={field}; '
+                    f'\r\n\r\n'.encode()
+                )
+                if v:
+                    if not isinstance(v, bytes):
+                        v = str(v)
+                    body_part += v.encode('utf-8') if isinstance(v, str) else v
+                    body_part += b'\r\n--' + boundary.encode() + b'\r\n'
+                else:
+                    body_part += b'\r\n--' + boundary.encode() + b'\r\n'
+
+    return body_part if not urlenc else urlencode(urlresult, doseq=True)
+
+
+def _prepare_files(k, v):
+    """Prepare file attributes for body of request form.
+
+    Args:
+        k: (str), file-name
+        v: fileobj or tuple (filename, data, content_type?)
+
+    Returns: file_name, file_data, file_content_type
+
+    """
+    file_content_type = None
+    if not v:
+        raise ValueError(f'No file provided for {k}')
+    if isinstance(v, (tuple, list)):
+        if len(v) == 2:
+            file_name, file_data = v
+        else:
+            file_name, file_data, file_content_type = v
+        if (
+            len(v) == 3
+            and file_content_type
+            and file_content_type.startswith('multipart/mixed')
+        ):
+            file_data, new_header = _encode_files(json_module.loads(file_data.decode()))
+            file_content_type = 'multipart/mixed; ' + (
+                new_header['Content-Type'].split('; ')[1]
+            )
+    else:
+        # if v is not a tuple or iterable it has to be a filelike obj
+        name = getattr(v, 'name', None)
+        if name and isinstance(name, str) and name[0] != '<' and name[-1] != '>':
+            file_name = os.path.basename(name)
+        else:
+            file_name = k
+        file_data = v
+    if hasattr(file_data, 'read'):
+        file_data = file_data.read()
+    return file_name, file_data, file_content_type
+
+
+def _make_boundary():
+    """
+    Create random boundary to be used in multipart/form-data with files.
+    """
+    boundary = os.urandom(16).hex()
+    return boundary
+
+
+def _encode_files(files, data=None):
+    """Build the body for a multipart/form-data request.
+
+    Will successfully encode files when passed as a dict or a list of
+    tuples. ``data`` fields are added first.
+    The tuples may be 2-tuples (filename, fileobj) or
+    3-tuples (filename, fileobj, contentype).
+    Allows for content_type = ``multipart/mixed`` for submission of nested files
+
+    Returns: (encoded body string, headers dict)
+    """
+    boundary = _make_boundary()
+    body_string = b'--' + boundary.encode() + b'\r\n'
+    header = {'Content-Type': 'multipart/form-data; boundary=' + boundary}
+
+    # Deal with the files tuples
+    if not isinstance(files, (dict, list)):
+        raise ValueError('cannot encode objects that are not 2-tuples')
+    elif isinstance(files, dict):
+        files = list(files.items())
+
+    for (k, v) in files:
+        file_name, file_data, file_content_type = _prepare_files(k, v)
+        if not file_data:
+            continue
+
+        body_string += f'Content-Disposition: form-data; name={k}; '.encode()
+        body_string += (
+            f'filename={file_name}\r\n'.encode() if file_name else '\r\n'.encode()
+        )
+        body_string += (
+            f'Content-Type: {file_content_type or "text/plain"}\r\n\r\n'.encode()
+        )
+        body_string += (
+            file_data.encode('utf-8') if isinstance(file_data, str) else file_data
+        )
+        body_string += b'\r\n--' + boundary.encode() + b'\r\n'
+
+        # Handle whatever json data gets passed along with files
+    if data:
+        body_string += _prepare_data_fields(data, boundary)
+
+    body_string = body_string[:-2] + b'--\r\n'
+
+    return body_string, header
+
+
 def _prepare_sim_args(
-    path, query_string, params, params_csv, content_type, headers, body, json, extras
+    path,
+    query_string,
+    params,
+    params_csv,
+    content_type,
+    headers,
+    body,
+    json,
+    files,
+    data,
+    extras,
 ):
     if not path.startswith('/'):
         raise ValueError("path must start with '/'")
@@ -2163,7 +2377,19 @@ def _prepare_sim_args(
         headers = headers or {}
         headers['Content-Type'] = content_type
 
-    if json is not None:
+    if files or data:
+        if json:
+            raise HTTPBadRequest(
+                description='Cannot process both json and (files or data) args'
+            )
+        elif files:
+            body, headers = _encode_files(files, data)
+        else:
+            body = _prepare_data_fields(data, None, True)
+            headers = headers or {}
+            headers['Content-Type'] = MEDIA_URLENCODED
+
+    elif json is not None:
         body = json_module.dumps(json, ensure_ascii=False)
         headers = headers or {}
         headers['Content-Type'] = MEDIA_JSON

requirements/mintest

@@ -5,3 +5,4 @@ pytest
 pyyaml
 requests
 ujson
+

tests/test_media_multipart.py

@@ -98,7 +98,6 @@
     b'--boundary--\r\n'
 )
 
-
 EXAMPLES = {
     '5b11af82ab65407ba8cdccf37d2a9c4f': EXAMPLE1,
     '---------------------------1574247108204320607285918568': EXAMPLE2,