Add HTTPS proxy support

Squashed #322

# Conflicts:
#	hyper/contrib.py

Author: KostyaEsmukov

hyper/common/connection.py

@@ -44,8 +44,9 @@ class HTTPConnection(object):
     :param proxy_host: (optional) The proxy to connect to.  This can be an IP
         address or a host name and may include a port.
     :param proxy_port: (optional) The proxy port to connect to. If not provided
-        and one also isn't provided in the ``proxy`` parameter, defaults to
-        8080.
+        and one also isn't provided in the ``proxy_host`` parameter, defaults
+        to 8080.
+    :param proxy_headers: (optional) The headers to send to a proxy.
     """
     def __init__(self,
                  host,
@@ -56,19 +57,21 @@ def __init__(self,
                  ssl_context=None,
                  proxy_host=None,
                  proxy_port=None,
+                 proxy_headers=None,
                  **kwargs):
 
         self._host = host
         self._port = port
         self._h1_kwargs = {
             'secure': secure, 'ssl_context': ssl_context,
             'proxy_host': proxy_host, 'proxy_port': proxy_port,
-            'enable_push': enable_push
+            'proxy_headers': proxy_headers, 'enable_push': enable_push
         }
         self._h2_kwargs = {
             'window_manager': window_manager, 'enable_push': enable_push,
             'secure': secure, 'ssl_context': ssl_context,
-            'proxy_host': proxy_host, 'proxy_port': proxy_port
+            'proxy_host': proxy_host, 'proxy_port': proxy_port,
+            'proxy_headers': proxy_headers
         }
 
         # Add any unexpected kwargs to both dictionaries.

hyper/common/exceptions.py

@@ -71,3 +71,22 @@ class MissingCertFile(Exception):
     The certificate file could not be found.
     """
     pass
+
+
+# Create our own ConnectionError.
+try:  # pragma: no cover
+    ConnectionError = ConnectionError
+except NameError:  # pragma: no cover
+    class ConnectionError(Exception):
+        """
+        An error occurred during connection to a host.
+        """
+
+
+class ProxyError(ConnectionError):
+    """
+    An error occurred during connection to a proxy.
+    """
+    def __init__(self, message, response):
+        self.response = response
+        super(ProxyError, self).__init__(message)

hyper/contrib.py

@@ -9,7 +9,9 @@
     from requests.adapters import HTTPAdapter
     from requests.models import Response
     from requests.structures import CaseInsensitiveDict
-    from requests.utils import get_encoding_from_headers
+    from requests.utils import (
+        get_encoding_from_headers, select_proxy, prepend_scheme_if_needed
+    )
     from requests.cookies import extract_cookies_to_jar
 except ImportError:  # pragma: no cover
     HTTPAdapter = object
@@ -29,7 +31,8 @@ def __init__(self, *args, **kwargs):
         #: A mapping between HTTP netlocs and ``HTTP20Connection`` objects.
         self.connections = {}
 
-    def get_connection(self, host, port, scheme, cert=None, verify=True):
+    def get_connection(self, host, port, scheme, cert=None, verify=True,
+                       proxy=None):
         """
         Gets an appropriate HTTP/2 connection object based on
         host/port/scheme/cert tuples.
@@ -50,29 +53,51 @@ def get_connection(self, host, port, scheme, cert=None, verify=True):
         elif verify is not True:
             ssl_context = init_context(cert_path=verify, cert=cert)
 
+        if proxy:
+            proxy_headers = self.proxy_headers(proxy)
+            proxy_netloc = urlparse(proxy).netloc
+        else:
+            proxy_headers = None
+            proxy_netloc = None
+
+        # We put proxy headers in the connection_key, because
+        # ``proxy_headers`` method might be overridden, so we can't
+        # rely on proxy headers being the same for the same proxies.
+        proxy_headers_key = (frozenset(proxy_headers.items())
+                             if proxy_headers else None)
+        connection_key = (host, port, scheme, cert, verify,
+                          proxy_netloc, proxy_headers_key)
         try:
-            conn = self.connections[(host, port, scheme, cert, verify)]
+            conn = self.connections[connection_key]
         except KeyError:
             conn = HTTPConnection(
                 host,
                 port,
                 secure=secure,
-                ssl_context=ssl_context)
-            self.connections[(host, port, scheme, cert, verify)] = conn
+                ssl_context=ssl_context,
+                proxy_host=proxy_netloc,
+                proxy_headers=proxy_headers)
+            self.connections[connection_key] = conn
 
         return conn
 
-    def send(self, request, stream=False, cert=None, verify=True, **kwargs):
+    def send(self, request, stream=False, cert=None, verify=True, proxies=None,
+             **kwargs):
         """
         Sends a HTTP message to the server.
         """
+        proxy = select_proxy(request.url, proxies)
+        if proxy:
+            proxy = prepend_scheme_if_needed(proxy, 'http')
+
         parsed = urlparse(request.url)
         conn = self.get_connection(
             parsed.hostname,
             parsed.port,
             parsed.scheme,
             cert=cert,
-            verify=verify)
+            verify=verify,
+            proxy=proxy)
 
         # Build the selector.
         selector = parsed.path
@@ -97,7 +122,7 @@ def send(self, request, stream=False, cert=None, verify=True, **kwargs):
     def build_response(self, request, resp):
         """
         Builds a Requests' response object.  This emulates most of the logic of
-        the standard fuction but deals with the lack of the ``.headers``
+        the standard function but deals with the lack of the ``.headers``
         property on the HTTP20Response object.
 
         Additionally, this function builds in a number of features that are

hyper/http11/connection.py

@@ -18,9 +18,11 @@
 from .response import HTTP11Response
 from ..tls import wrap_socket, H2C_PROTOCOL
 from ..common.bufsocket import BufferedSocket
-from ..common.exceptions import TLSUpgrade, HTTPUpgrade
+from ..common.exceptions import TLSUpgrade, HTTPUpgrade, ProxyError
 from ..common.headers import HTTPHeaderMap
-from ..common.util import to_bytestring, to_host_port_tuple, HTTPVersion
+from ..common.util import (
+    to_bytestring, to_host_port_tuple, to_native_string, HTTPVersion
+)
 from ..compat import bytes
 
 # We prefer pycohttpparser to the pure-Python interpretation
@@ -36,6 +38,43 @@
 BODY_FLAT = 2
 
 
+def _create_tunnel(proxy_host, proxy_port, target_host, target_port,
+                   proxy_headers=None):
+    """
+    Sends CONNECT method to a proxy and returns a socket with established
+    connection to the target.
+
+    :returns: socket
+    """
+    conn = HTTP11Connection(proxy_host, proxy_port)
+    conn.request('CONNECT', '%s:%d' % (target_host, target_port),
+                 headers=proxy_headers)
+
+    resp = conn.get_response()
+    if resp.status != 200:
+        raise ProxyError(
+            "Tunnel connection failed: %d %s" %
+            (resp.status, to_native_string(resp.reason)),
+            response=resp
+        )
+    return conn._sock
+
+
+def _headers_to_http_header_map(headers):
+    # TODO turn this to a classmethod of HTTPHeaderMap
+    headers = headers or {}
+    if not isinstance(headers, HTTPHeaderMap):
+        if isinstance(headers, Mapping):
+            headers = HTTPHeaderMap(headers.items())
+        elif isinstance(headers, Iterable):
+            headers = HTTPHeaderMap(headers)
+        else:
+            raise ValueError(
+                'Header argument must be a dictionary or an iterable'
+            )
+    return headers
+
+
 class HTTP11Connection(object):
     """
     An object representing a single HTTP/1.1 connection to a server.
@@ -53,14 +92,16 @@ class HTTP11Connection(object):
     :param proxy_host: (optional) The proxy to connect to.  This can be an IP
         address or a host name and may include a port.
     :param proxy_port: (optional) The proxy port to connect to. If not provided
-        and one also isn't provided in the ``proxy`` parameter,
+        and one also isn't provided in the ``proxy_host`` parameter,
         defaults to 8080.
+    :param proxy_headers: (optional) The headers to send to a proxy.
     """
 
     version = HTTPVersion.http11
 
     def __init__(self, host, port=None, secure=None, ssl_context=None,
-                 proxy_host=None, proxy_port=None, **kwargs):
+                 proxy_host=None, proxy_port=None, proxy_headers=None,
+                 **kwargs):
         if port is None:
             self.host, self.port = to_host_port_tuple(host, default_port=80)
         else:
@@ -83,17 +124,21 @@ def __init__(self, host, port=None, secure=None, ssl_context=None,
         self.ssl_context = ssl_context
         self._sock = None
 
+        # Keep the current request method in order to be able to know
+        # in get_response() what was the request verb.
+        self._current_request_method = None
+
         # Setup proxy details if applicable.
-        if proxy_host:
-            if proxy_port is None:
-                self.proxy_host, self.proxy_port = to_host_port_tuple(
-                    proxy_host, default_port=8080
-                )
-            else:
-                self.proxy_host, self.proxy_port = proxy_host, proxy_port
+        if proxy_host and proxy_port is None:
+            self.proxy_host, self.proxy_port = to_host_port_tuple(
+                proxy_host, default_port=8080
+            )
+        elif proxy_host:
+            self.proxy_host, self.proxy_port = proxy_host, proxy_port
         else:
             self.proxy_host = None
             self.proxy_port = None
+        self.proxy_headers = proxy_headers
 
         #: The size of the in-memory buffer used to store data from the
         #: network. This is used as a performance optimisation. Increase buffer
@@ -113,19 +158,28 @@ def connect(self):
         :returns: Nothing.
         """
         if self._sock is None:
-            if not self.proxy_host:
-                host = self.host
-                port = self.port
-            else:
-                host = self.proxy_host
-                port = self.proxy_port
 
-            sock = socket.create_connection((host, port), 5)
+            if self.proxy_host and self.secure:
+                # Send http CONNECT method to a proxy and acquire the socket
+                sock = _create_tunnel(
+                    self.proxy_host,
+                    self.proxy_port,
+                    self.host,
+                    self.port,
+                    proxy_headers=self.proxy_headers
+                )
+            elif self.proxy_host:
+                # Simple http proxy
+                sock = socket.create_connection(
+                    (self.proxy_host, self.proxy_port),
+                    5
+                )
+            else:
+                sock = socket.create_connection((self.host, self.port), 5)
             proto = None
 
             if self.secure:
-                assert not self.proxy_host, "Proxy with HTTPS not supported."
-                sock, proto = wrap_socket(sock, host, self.ssl_context)
+                sock, proto = wrap_socket(sock, self.host, self.ssl_context)
 
             log.debug("Selected protocol: %s", proto)
             sock = BufferedSocket(sock, self.network_buffer_size)
@@ -154,33 +208,37 @@ def request(self, method, url, body=None, headers=None):
         :returns: Nothing.
         """
 
-        headers = headers or {}
-
         method = to_bytestring(method)
+        is_connect_method = b'CONNECT' == method.upper()
+        self._current_request_method = method
+
+        if self.proxy_host and not self.secure:
+            # As per https://tools.ietf.org/html/rfc2068#section-5.1.2:
+            # The absoluteURI form is required when the request is being made
+            # to a proxy.
+            url = self._absolute_http_url(url)
         url = to_bytestring(url)
 
-        if not isinstance(headers, HTTPHeaderMap):
-            if isinstance(headers, Mapping):
-                headers = HTTPHeaderMap(headers.items())
-            elif isinstance(headers, Iterable):
-                headers = HTTPHeaderMap(headers)
-            else:
-                raise ValueError(
-                    'Header argument must be a dictionary or an iterable'
-                )
+        headers = _headers_to_http_header_map(headers)
+
+        # Append proxy headers.
+        if self.proxy_host and not self.secure:
+            headers.update(
+                _headers_to_http_header_map(self.proxy_headers).items()
+            )
 
         if self._sock is None:
             self.connect()
 
-        if self._send_http_upgrade:
+        if not is_connect_method and self._send_http_upgrade:
             self._add_upgrade_headers(headers)
             self._send_http_upgrade = False
 
         # We may need extra headers.
         if body:
             body_type = self._add_body_headers(headers, body)
 
-        if b'host' not in headers:
+        if not is_connect_method and b'host' not in headers:
             headers[b'host'] = self.host
 
         # Begin by emitting the header block.
@@ -192,13 +250,20 @@ def request(self, method, url, body=None, headers=None):
 
         return
 
+    def _absolute_http_url(self, url):
+        port_part = ':%d' % self.port if self.port != 80 else ''
+        return 'http://%s%s%s' % (self.host, port_part, url)
+
     def get_response(self):
         """
         Returns a response object.
 
         This is an early beta, so the response object is pretty stupid. That's
         ok, we'll fix it later.
         """
+        method = self._current_request_method
+        self._current_request_method = None
+
         headers = HTTPHeaderMap()
 
         response = None
@@ -228,7 +293,8 @@ def get_response(self):
             response.msg.tobytes(),
             headers,
             self._sock,
-            self
+            self,
+            method
         )
 
     def _send_headers(self, method, url, headers):