httpx backend (#80)

An alternative backend for [httpx](https://gitlab.com/os85/httpx) is
added, built on top of its stream plugin.

Closes #67

down.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "minitest", "~> 5.8"
   spec.add_development_dependency "mocha", "~> 1.5"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "httpx", "~> 0.22", ">= 0.22.2"
   # http 5.0 drop support of ruby 2.3 and 2.4. We still support those versions.
   if RUBY_VERSION >= "2.5"
     spec.add_development_dependency "http", "~> 5.0"

lib/down/httpx.rb

@@ -0,0 +1,175 @@
+# frozen-string-literal: true
+
+require "uri"
+require "tempfile"
+require "httpx"
+
+require "down/backend"
+
+
+module Down
+  # Provides streaming downloads implemented with HTTPX.
+  class Httpx < Backend
+    # Initializes the backend
+
+    USER_AGENT = "Down/#{Down::VERSION}"
+
+    def initialize(**options, &block)
+      @method = options.delete(:method) || :get
+      headers = options.delete(:headers) || {}
+      @client = HTTPX
+          .plugin(:follow_redirects, max_redirects: 2)
+          .plugin(:basic_authentication)
+          .plugin(:stream)
+          .with(
+            headers: { "user-agent": USER_AGENT }.merge(headers),
+            timeout: { connect_timeout: 30, write_timeout: 30, read_timeout: 30 },
+            **options
+          )
+
+      @client = block.call(@client) if block
+    end
+
+
+    # Downlods the remote file to disk. Accepts HTTPX options via a hash or a
+    # block, and some additional options as well.
+    def download(url, max_size: nil, progress_proc: nil, content_length_proc: nil, destination: nil, extension: nil, **options, &block)
+      client = @client
+
+      response = request(client, url, **options, &block)
+
+      content_length = nil
+
+      if response.headers.key?("content-length")
+        content_length = response.headers["content-length"].to_i
+
+        content_length_proc.call(content_length) if content_length_proc
+
+        if max_size && content_length > max_size
+          response.close
+          raise Down::TooLarge, "file is too large (#{content_length/1024/1024}MB, max is #{max_size/1024/1024}MB)"
+        end
+      end
+
+      extname  = extension ? ".#{extension}" : File.extname(response.uri.path)
+      tempfile = Tempfile.new(["down-http", extname], binmode: true)
+
+      stream_body(response) do |chunk|
+        tempfile.write(chunk)
+        chunk.clear # deallocate string
+
+        progress_proc.call(tempfile.size) if progress_proc
+
+        if max_size && tempfile.size > max_size
+          raise Down::TooLarge, "file is too large (#{tempfile.size/1024/1024}MB, max is #{max_size/1024/1024}MB)"
+        end
+      end
+
+      tempfile.open # flush written content
+
+      tempfile.extend DownloadedFile
+      tempfile.url     = response.uri.to_s
+      tempfile.headers = normalize_headers(response.headers.to_h)
+      tempfile.content_type = response.content_type.mime_type
+      tempfile.charset = response.body.encoding
+
+      download_result(tempfile, destination)
+    rescue
+      tempfile.close! if tempfile
+      raise
+    end
+
+    # Starts retrieving the remote file and returns an IO-like object which
+    # downloads the response body on-demand. Accepts HTTP.rb options via a hash
+    # or a block.
+    def open(url, rewindable: true, **options, &block)
+      response = request(@client, url, stream: true, **options, &block)
+      size = response.headers["content-length"]
+      size = size.to_i if size
+      Down::ChunkedIO.new(
+        chunks:     enum_for(:stream_body, response),
+        size:       size,
+        encoding:   response.body.encoding,
+        rewindable: rewindable,
+        data:       {
+          status:   response.status,
+          headers:  normalize_headers(response.headers.to_h),
+          response: response
+        },
+      )
+    end
+
+    private
+
+    # Yields chunks of the response body to the block.
+    def stream_body(response, &block)
+      response.each(&block)
+    rescue => exception
+      request_error!(exception)
+    end
+
+    def request(client, url, method: @method, **options, &block)
+      response = send_request(client, method, url, **options, &block)
+      response.raise_for_status
+      response_error!(response) unless (200..299).include?(response.status)
+      response
+    rescue HTTPX::HTTPError
+      response_error!(response)
+    rescue => error
+      request_error!(error)
+    end
+
+    def send_request(client, method, url, **options, &block)
+      uri = URI(url)
+      client = @client
+      if uri.user || uri.password
+        client = client.basic_auth(uri.user, uri.password)
+        uri.user = uri.password = nil
+      end
+      client = block.call(client) if block
+
+      client.request(method, uri, stream: true, **options)
+    rescue => exception
+      request_error!(exception)
+    end
+
+    # Raises non-sucessful response as a Down::ResponseError.
+    def response_error!(response)
+      args = [response.status.to_s, response]
+
+      case response.status
+      when 300..399 then raise Down::TooManyRedirects, "too many redirects"
+      when 404      then raise Down::NotFound.new(*args)
+      when 400..499 then raise Down::ClientError.new(*args)
+      when 500..599 then raise Down::ServerError.new(*args)
+      else               raise Down::ResponseError.new(*args)
+      end
+    end
+
+    # Re-raise HTTP.rb exceptions as Down::Error exceptions.
+    def request_error!(exception)
+      case exception
+      when URI::Error, HTTPX::UnsupportedSchemeError
+        raise Down::InvalidUrl, exception.message
+      when HTTPX::ConnectionError
+        raise Down::ConnectionError, exception.message
+      when HTTPX::TimeoutError
+        raise Down::TimeoutError, exception.message
+      when OpenSSL::SSL::SSLError
+        raise Down::SSLError, exception.message
+      else
+        raise exception
+      end
+    end
+
+    # Defines some additional attributes for the returned Tempfile.
+    module DownloadedFile
+      attr_accessor :url, :headers, :charset, :content_type
+
+      def original_filename
+        Utils.filename_from_content_disposition(headers["Content-Disposition"]) ||
+        Utils.filename_from_path(URI.parse(url).path)
+      end
+    end
+  end
+end