added SOCKS[45] proxy support

README.md

@@ -11,7 +11,7 @@ See [HTTPClient](http://www.rubydoc.info/gems/httpclient/frames) for documentati
 ## Features
 
 * methods like GET/HEAD/POST/* via HTTP/1.1.
-* HTTPS(SSL), Cookies, proxy, authentication(Digest, NTLM, Basic), etc.
+* HTTPS(SSL), Cookies, (HTTP, socks[45])proxy, authentication(Digest, NTLM, Basic), etc.
 * asynchronous HTTP request, streaming HTTP request.
 * debug mode CLI.
 * by contrast with net/http in standard distribution;
@@ -24,6 +24,7 @@ See [HTTPClient](http://www.rubydoc.info/gems/httpclient/frames) for documentati
   * extensible with filter interface
   * you don't have to care HTTP/1.1 persistent connection
     (httpclient cares instead of you)
+  * socks proxy
 * Not supported now
   * Cache
   * Rather advanced HTTP/1.1 usage such as Range, deflate, etc.

lib/httpclient.rb

@@ -44,9 +44,12 @@
 #     clnt = HTTPClient.new
 #
 # 2. Accessing resources through HTTP proxy.  You can use environment
-#    variable 'http_proxy' or 'HTTP_PROXY' instead.
+#    variable 'http_proxy' or 'HTTP_PROXY' or 'SOCKS_PROXY' instead.
 #
 #     clnt = HTTPClient.new('http://myproxy:8080')
+#     clnt = HTTPClient.new('socks4://myproxy:1080')
+#     clnt = HTTPClient.new('socks5://myproxy:1080')
+#     clnt = HTTPClient.new('socks5://username:password@myproxy:1080')
 #
 # === How to retrieve web resources
 #
@@ -503,8 +506,9 @@ def proxy=(proxy)
       @proxy_auth.reset_challenge
     else
       @proxy = urify(proxy)
-      if @proxy.scheme == nil or @proxy.scheme.downcase != 'http' or
-          @proxy.host == nil or @proxy.port == nil
+      if @proxy.scheme == nil or
+        (@proxy.scheme.downcase != 'http' and !socks?(@proxy)) or
+         @proxy.host == nil or @proxy.port == nil
         raise ArgumentError.new("unsupported proxy #{proxy}")
       end
       @proxy_auth.reset_challenge
@@ -1073,8 +1077,10 @@ def load_environment
       # HTTP_* is used for HTTP header information.  Unlike open-uri, we
       # simply ignore http_proxy in CGI env and use cgi_http_proxy instead.
       self.proxy = getenv('cgi_http_proxy')
-    else
+    elsif ENV.key?('http_proxy')
       self.proxy = getenv('http_proxy')
+    else
+      self.proxy = getenv('socks_proxy')
     end
     # no_proxy
     self.no_proxy = getenv('no_proxy')

lib/httpclient/session.rb

@@ -21,6 +21,7 @@
 require 'httpclient/timeout' # TODO: remove this once we drop 1.8 support
 require 'httpclient/ssl_config'
 require 'httpclient/http'
+require 'httpclient/socks'
 if defined? JRUBY_VERSION
   require 'httpclient/jruby_ssl_socket'
 else
@@ -92,6 +93,8 @@ def inspect # :nodoc:
 
   # Manages sessions for a HTTPClient instance.
   class SessionManager
+    include Util
+
     # Name of this client.  Used for 'User-Agent' header in HTTP request.
     attr_accessor :agent_name
     # Owner of this client.  Used for 'From' header in HTTP request.
@@ -132,6 +135,8 @@ class SessionManager
     def initialize(client)
       @client = client
       @proxy = client.proxy
+      @socks_user = nil
+      @socks_password = nil
 
       @agent_name = nil
       @from = nil
@@ -167,6 +172,10 @@ def proxy=(proxy)
         @proxy = nil
       else
         @proxy = Site.new(proxy)
+        if socks?(proxy)
+          @socks_user = proxy.user
+          @socks_password = proxy.password
+        end
       end
     end
 
@@ -218,6 +227,8 @@ def open(uri, via_proxy = false)
       site = Site.new(uri)
       sess = Session.new(@client, site, @agent_name, @from)
       sess.proxy = via_proxy ? @proxy : nil
+      sess.socks_user = @socks_user
+      sess.socks_password = @socks_password
       sess.socket_sync = @socket_sync
       sess.tcp_keepalive = @tcp_keepalive
       sess.requested_version = @protocol_version if @protocol_version
@@ -448,6 +459,9 @@ class Session
     # Device for dumping log for debugging
     attr_accessor :debug_dev
 
+    attr_accessor :socks_user
+    attr_accessor :socks_password
+
     attr_accessor :connect_timeout
     attr_accessor :connect_retry
     attr_accessor :send_timeout
@@ -469,6 +483,8 @@ def initialize(client, dest, agent_name, from)
       @client = client
       @dest = dest
       @proxy = nil
+      @socks_user = nil
+      @socks_password = nil
       @socket_sync = true
       @tcp_keepalive = false
       @requested_version = nil
@@ -627,6 +643,32 @@ def create_socket(host, port)
       socket
     end
 
+    def via_socks_proxy?
+      @proxy && socks?(@proxy)
+    end
+
+    def via_http_proxy?
+      @proxy && !socks?(@proxy)
+    end
+
+    def create_socks_socket(proxy, dest)
+      if socks4?(proxy)
+        options = {}
+        options = { user: @socks_user } if @socks_user
+        socks_socket = SOCKS4Socket.new(proxy.host, proxy.port, options)
+      elsif socks5?(proxy)
+        options = {}
+        options = { user: @socks_user } if @socks_user
+        options[:password] = @socks_password if @socks_password
+        socks_socket = SOCKS5Socket.new(proxy.host, proxy.port, options)
+      else
+        raise "invalid proxy url #{proxy}"
+      end
+      dest_site = Site.new(dest)
+      opened_socket = socks_socket.open(dest_site.host, dest_site.port, {})
+      opened_socket
+    end
+
     def create_loopback_socket(host, port, str)
       @debug_dev << "! CONNECT TO #{host}:#{port}\n" if @debug_dev
       socket = LoopBackSocket.new(host, port, str)
@@ -751,6 +793,8 @@ def connect
           elsif https?(@dest)
             @socket = SSLSocket.create_socket(self)
             @ssl_peer_cert = @socket.peer_cert
+          elsif socks?(@proxy)
+            @socket = create_socks_socket(@proxy, @dest)
           else
             @socket = create_socket(site.host, site.port)
           end

lib/httpclient/socks.rb

@@ -0,0 +1,213 @@
+# Copyright (c) 2008 Jamis Buck
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the 'Software'), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+require 'socket'
+require 'resolv'
+require 'ipaddr'
+
+#
+# This implementation was borrowed from Net::SSH::Proxy
+#
+class HTTPClient
+  # An Standard SOCKS error.
+  class SOCKSError < SocketError; end
+
+  # Used for reporting proxy connection errors.
+  class SOCKSConnectError < SOCKSError; end
+
+  # Used when the server doesn't recognize the user's credentials.
+  class SOCKSUnauthorizedError < SOCKSError; end
+
+  # An implementation of a SOCKS4 proxy.
+  class SOCKS4Socket
+    # The SOCKS protocol version used by this class
+    VERSION = 4
+
+    # The packet type for connection requests
+    CONNECT = 1
+
+    # The status code for a successful connection
+    GRANTED = 90
+
+    # The proxy's host name or IP address, as given to the constructor.
+    attr_reader :proxy_host
+
+    # The proxy's port number.
+    attr_reader :proxy_port
+
+    # The additional options that were given to the proxy's constructor.
+    attr_reader :options
+
+    # Create a new proxy connection to the given proxy host and port.
+    # Optionally, a :user key may be given to identify the username
+    # with which to authenticate.
+    def initialize(proxy_host, proxy_port = 1080, options = {})
+      @proxy_host = proxy_host
+      @proxy_port = proxy_port
+      @options = options
+    end
+
+    # Return a new socket connected to the given host and port via the
+    # proxy that was requested when the socket factory was instantiated.
+    def open(host, port, connection_options)
+      socket = Socket.tcp(proxy_host, proxy_port, nil, nil,
+                          connect_timeout: connection_options[:timeout])
+      ip_addr = IPAddr.new(Resolv.getaddress(host))
+
+      packet = [
+        VERSION, CONNECT, port.to_i,
+        ip_addr.to_i, options[:user]
+      ].pack('CCnNZ*')
+      socket.send packet, 0
+
+      _version, status, _port, _ip = socket.recv(8).unpack('CCnN')
+      if status != GRANTED
+        socket.close
+        raise SOCKSConnectError, "error connecting to socks proxy (#{status})"
+      end
+
+      socket
+    end
+  end
+
+  # An implementation of a SOCKS5 proxy.
+  class SOCKS5Socket
+    # The SOCKS protocol version used by this class
+    VERSION = 5
+
+    # The SOCKS authentication type for requests without authentication
+    METHOD_NO_AUTH = 0
+
+    # The SOCKS authentication type for requests via username/password
+    METHOD_PASSWD = 2
+
+    # The SOCKS authentication type for when there are no supported
+    # authentication methods.
+    METHOD_NONE = 0xFF
+
+    # The SOCKS packet type for requesting a proxy connection.
+    CMD_CONNECT = 1
+
+    # The SOCKS address type for connections via IP address.
+    ATYP_IPV4 = 1
+
+    # The SOCKS address type for connections via domain name.
+    ATYP_DOMAIN = 3
+
+    # The SOCKS response code for a successful operation.
+    SUCCESS = 0
+
+    # The proxy's host name or IP address
+    attr_reader :proxy_host
+
+    # The proxy's port number
+    attr_reader :proxy_port
+
+    # The map of options given at initialization
+    attr_reader :options
+
+    # Create a new proxy connection to the given proxy host and port.
+    # Optionally, :user and :password options may be given to
+    # identify the username and password with which to authenticate.
+    def initialize(proxy_host, proxy_port = 1080, options = {})
+      @proxy_host = proxy_host
+      @proxy_port = proxy_port
+      @options = options
+    end
+
+    # Return a new socket connected to the given host and port via the
+    # proxy that was requested when the socket factory was instantiated.
+    def open(host, port, connection_options)
+      socket = Socket.tcp(proxy_host, proxy_port, nil, nil,
+                          connect_timeout: connection_options[:timeout])
+
+      methods = [METHOD_NO_AUTH]
+      methods << METHOD_PASSWD if options[:user]
+
+      packet = [VERSION, methods.size, *methods].pack('C*')
+      socket.send packet, 0
+
+      version, method = socket.recv(2).unpack('CC')
+      if version != VERSION
+        socket.close
+        raise SOCKSError, "invalid SOCKS version (#{version})"
+      end
+
+      if method == METHOD_NONE
+        socket.close
+        raise SOCKSError, 'no supported authorization methods'
+      end
+
+      negotiate_password(socket) if method == METHOD_PASSWD
+
+      packet = [VERSION, CMD_CONNECT, 0].pack('C*')
+
+      if host =~ /^(\d+)\.(\d+)\.(\d+)\.(\d+)$/
+        packet << [ATYP_IPV4, $1.to_i, $2.to_i, $3.to_i, $4.to_i].pack('C*')
+      else
+        packet << [ATYP_DOMAIN, host.length, host].pack('CCA*')
+      end
+
+      packet << [port].pack('n')
+      socket.send packet, 0
+
+      _version, reply, = socket.recv(2).unpack('C*')
+      socket.recv(1)
+      address_type = socket.recv(1).getbyte(0)
+      case address_type
+      when 1
+        socket.recv(4) # get four bytes for IPv4 address
+      when 3
+        len = socket.recv(1).getbyte(0)
+        _hostname = socket.recv(len)
+      when 4
+        _ipv6addr _hostname = socket.recv(16)
+      else
+        socket.close
+        raise SOCKSConnectError, 'Illegal response type'
+      end
+      _portnum = socket.recv(2)
+
+      unless reply == SUCCESS
+        socket.close
+        raise SOCKSConnectError, reply.to_s
+      end
+
+      socket
+    end
+
+    private
+
+    # Simple username/password negotiation with the SOCKS5 server.
+    def negotiate_password(socket)
+      packet = [
+        0x01, options[:user].length, options[:user],
+        options[:password].length, options[:password]
+      ].pack('CCA*CA*')
+      socket.send packet, 0
+
+      _version, status = socket.recv(2).unpack('CC')
+
+      return if status == SUCCESS
+      socket.close
+      raise SOCKSUnauthorizedError, 'could not authorize user'
+    end
+  end
+end

lib/httpclient/ssl_socket.rb

@@ -18,9 +18,14 @@ def self.create_socket(session)
         :debug_dev => session.debug_dev
       }
       site = session.proxy || session.dest
-      socket = session.create_socket(site.host, site.port)
+      if session.via_socks_proxy?
+        socket = session.create_socks_socket(session.proxy, session.dest)
+      else
+        socket = session.create_socket(site.host, site.port)
+      end
+
       begin
-        if session.proxy
+        if session.via_http_proxy?
           session.connect_ssl_proxy(socket, Util.urify(session.dest.to_s))
         end
         new(socket, session.dest, session.ssl_config, opts)