Commit 54038bb8 authored by Tiago's avatar Tiago
Browse files

new options: max_response_body_size, max_response_headers, 

and max_response_header_value_size.

These are parse-time enforced thresholds, which when detected, will
immediately raise before resources are further consumed, thereby
avoiding resource exhaustion attacks.

Closes #383
parent fa183311

lib/httpx/connection/http1.rb

+5 −1
Original line number Diff line number Diff line
@@ -24,7 +24,7 @@ module HTTPX 
      @options = options

      @max_concurrent_requests = @options.max_concurrent_requests || MAX_REQUESTS

      @max_requests = @options.max_requests

      @parser = Parser::HTTP1.new(self)

      @parser = Parser::HTTP1.new(self, options.max_response_headers, options.max_response_header_value_size)

      @buffer = buffer

      @version = [1, 1]

      @pending = []
@@ -133,6 +133,10 @@ module HTTPX 
      request.log(color: :yellow) { "-> HEADLINE: #{response.status} HTTP/#{@parser.http_version.join(".")}" }

      request.log(color: :yellow) { response.headers.each.map { |f, v| "-> HEADER: #{f}: #{log_redact_headers(v)}" }.join("\n") }



      if response.content_length && response.content_length > request.options.max_response_body_size

        raise HTTPX::Error, "maximum response body size exceeded"

      end



      request.response = response

      on_complete if response.finished?

    end

lib/httpx/connection/http2.rb

+14 −0
Original line number Diff line number Diff line
@@ -314,7 +314,21 @@ module HTTPX 
      end

      _, status = h.shift

      headers = request.options.headers_class.new(h)



      raise HTTPX::Error, "maximum number of response headers exceeded" if h.size > @options.max_response_headers



      if (max_header_value_size = @options.max_response_header_value_size)

        headers.each do |_, v| # rubocop:disable Style/HashEachMethods

          raise HTTPX::Error, "maximum header value size exceeded" if v.size > max_header_value_size

        end

      end



      response = request.options.response_class.new(request, status, "2.0", headers)



      if response.content_length && response.content_length > request.options.max_response_body_size

        raise HTTPX::Error.new, "maximum response body size exceeded"

      end



      request.response = response

      @streams[request] = stream

lib/httpx/options.rb

+10 −0
Original line number Diff line number Diff line
@@ -81,6 +81,12 @@ module HTTPX 
    #             <tt>:operation_timeout</tt>, <tt>:keep_alive_timeout</tt>,  <tt>:read_timeout</tt>,  <tt>:write_timeout</tt>,

    #             <tt>:request_timeout</tt> and <tt>:total_request_timeout</tt>

    # :headers :: hash of HTTP headers (ex: <tt>{ "x-custom-foo" => "bar" }</tt>)

    # :max_response_body_size :: maximum size (in bytes) that the response body can consume (no threshold by default), after which an

    #                            error is raised.

    # :max_response_headers :: maximum number of header fields that a response can receive, after which an error is raised.

    # :max_response_header_value_size :: maximum size (in bytes) a header value can have (no threshold by default).

    #                                    for cases where the value is broken into multiple header fields (such as "cookie" or "set-cookie"),

    #                                    this is the total aggregated size.

    # :window_size :: number of bytes to read from a socket

    # :buffer_size :: internal read and write buffer size in bytes

    # :body_threshold_size :: maximum size in bytes of response payload that is buffered in memory.
@@ -386,6 +392,7 @@ module HTTPX 
    # number options

    %i[

      max_concurrent_requests max_requests window_size buffer_size

      max_response_body_size max_response_headers max_response_header_value_size

      body_threshold_size debug_level

    ].each do |option|

      class_eval(<<-OUT, __FILE__, __LINE__ + 1)
@@ -552,6 +559,9 @@ module HTTPX 
      :supported_compression_formats => %w[gzip deflate],

      :decompress_response_body => true,

      :compress_request_body => true,

      :max_response_headers => 1000,

      :max_response_header_value_size => nil,

      :max_response_body_size => Float::INFINITY,

      :timeout => {

        connect_timeout: CONNECT_TIMEOUT,

        settings_timeout: SETTINGS_TIMEOUT,

lib/httpx/parser/http1.rb

+8 −2
Original line number Diff line number Diff line
@@ -9,11 +9,13 @@ module HTTPX 


      attr_reader :status_code, :http_version, :headers



      def initialize(observer)

      def initialize(observer, max_headers, max_header_value_size)

        @observer = observer

        @state = :idle

        @buffer = "".b

        @headers = {}

        @max_headers = max_headers

        @max_header_value_size = max_header_value_size

        @content_length = nil

        @_has_trailers = @upgrade = false

      end
@@ -117,7 +119,11 @@ module HTTPX 
          value.strip!

          raise Error, "wrong header format" if value.nil?



          (headers[key.downcase] ||= []) << value

          values = (headers[key.downcase] ||= []) << value



          raise Error, "maximum header value size exceeded" if @max_header_value_size && (values.sum(&:size) > @max_header_value_size)



          raise Error, "maximum number of response headers exceeded" if headers.size > @max_headers

        end

      end

lib/httpx/response.rb

+9 −1
Original line number Diff line number Diff line
@@ -68,7 +68,7 @@ module HTTPX 
      @headers = @options.headers_class.new(headers)

      @body = @options.response_body_class.new(self, @options)

      @finished = complete?

      @content_type = nil

      @content_type = @content_length = nil

    end



    # dupped initialization
@@ -88,6 +88,7 @@ module HTTPX 
    # merges headers defined in +h+ into the response headers.

    def merge_headers(h)

      @headers = @headers.merge(h)

      @content_type = @content_length = nil

    end



    # writes +data+ chunk into the response body.
@@ -103,6 +104,13 @@ module HTTPX 
      @content_type ||= ContentType.new(@headers["content-type"])

    end



    # returns the response content length as advertised in the HTTP Content-Length header value.

    def content_length

      return @content_length if defined?(@content_length)



      @content_length = @headers["content-length"]&.to_i

    end



    # returns whether the response has been fully fetched.

    def finished?

      @finished