Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

⚡️ Speed up method HTTPAdapter.request_url by 13% #25

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 7 additions & 17 deletions src/requests/adapters.py
Original file line number Diff line number Diff line change
Expand Up @@ -546,12 +546,9 @@ def close(self):
def request_url(self, request, proxies):
"""Obtain the url to use when making the final request.

If the message is being sent through a HTTP proxy, the full URL has to
be used. Otherwise, we should only use the path portion of the URL.
If the message is being sent through a HTTP proxy, the full URL has to be used. Otherwise, we should only use the path portion of the URL.

This should not be called from user code, and is only exposed for use
when subclassing the
:class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.
This should not be called from user code, and is only exposed for use when subclassing the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.

:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param proxies: A dictionary of schemes or schemes and hosts to proxy URLs.
Expand All @@ -560,20 +557,13 @@ def request_url(self, request, proxies):
proxy = select_proxy(request.url, proxies)
scheme = urlparse(request.url).scheme

is_proxied_http_request = proxy and scheme != "https"
using_socks_proxy = False
if proxy:
if proxy and scheme != "https":
proxy_scheme = urlparse(proxy).scheme.lower()
using_socks_proxy = proxy_scheme.startswith("socks")

if not proxy_scheme.startswith("socks"):
return urldefragauth(request.url)

url = request.path_url
if url.startswith("//"): # Don't confuse urllib3
url = f"/{url.lstrip('/')}"

if is_proxied_http_request and not using_socks_proxy:
url = urldefragauth(request.url)

return url
return f"/{url.lstrip('/')}" if url.startswith("//") else url

def add_headers(self, request, **kwargs):
"""Add any headers needed by the connection. As of v2.0 this does
Expand Down
22 changes: 11 additions & 11 deletions src/requests/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -844,24 +844,25 @@ def select_proxy(url, proxies):
:param url: The url being for the request
:param proxies: A dictionary of schemes or schemes and hosts to proxy URLs
"""
proxies = proxies or {}
if not proxies:
return None # Early return if proxies are not provided

urlparts = urlparse(url)
if urlparts.hostname is None:
if not urlparts.hostname:
return proxies.get(urlparts.scheme, proxies.get("all"))

proxy_keys = [
urlparts.scheme + "://" + urlparts.hostname,
proxy_keys = (
f"{urlparts.scheme}://{urlparts.hostname}",
urlparts.scheme,
"all://" + urlparts.hostname,
f"all://{urlparts.hostname}",
"all",
]
proxy = None
)

for proxy_key in proxy_keys:
if proxy_key in proxies:
proxy = proxies[proxy_key]
break
return proxies[proxy_key] # Early return if proxy found

return proxy
return None # If no proxy is found


def resolve_proxies(request, proxies, trust_env=True):
Expand Down Expand Up @@ -1072,7 +1073,6 @@ def urldefragauth(url):
"""
scheme, netloc, path, params, query, fragment = urlparse(url)

# see func:`prepend_scheme_if_needed`
if not netloc:
netloc, path = path, netloc

Expand Down