Replace httpclient with net/http in bosh-monitor

src/Gemfile

@@ -28,7 +28,6 @@ group :production do
 end
 
 group :bat do
- gem 'httpclient'
  gem 'minitar'
  gem 'net-ssh'
 end

src/Gemfile.lock

@@ -34,7 +34,6 @@ PATH
  bosh_common (~> 0.0.0)
  bosh_cpi
  cf-uaa-lib (~> 3.2.1)
- httpclient (~> 2.8.3)
  logging
  membrane (~> 1.1.0)
  nats-pure
@@ -62,7 +61,6 @@ PATH
  async-io
  cf-uaa-lib (~> 3.2.1)
  dogapi (~> 1.45.0)
- httpclient (~> 2.8.3)
  logging
  nats-pure
  net-smtp
@@ -380,7 +378,6 @@ DEPENDENCIES
  bundle-audit
  factory_bot
  fakefs
- httpclient
  io-stream (<= 0.4.0)
  json (~> 2)
  minitar

src/bosh-director/bosh-director.gemspec

@@ -43,7 +43,6 @@ Gem::Specification.new do |spec|
  spec.add_dependency 'bcrypt', '~>3.1.16'
  spec.add_dependency 'bosh_cpi'
  spec.add_dependency 'cf-uaa-lib', '~>3.2.1'
- spec.add_dependency 'httpclient', '~>2.8.3'
  spec.add_dependency 'logging'
  spec.add_dependency 'membrane', '~>1.1.0'
  spec.add_dependency 'nats-pure'

src/bosh-director/lib/bosh/director/config_server/retryable_http_client.rb

@@ -1,3 +1,5 @@
+require 'net/http'
+
 module Bosh::Director::ConfigServer
  class RetryableHTTPClient
  def initialize(http_client)
@@ -25,8 +27,7 @@ def connection_retryable
  Errno::ETIMEDOUT,
  Errno::ECONNRESET,
  ::Timeout::Error,
- ::HTTPClient::TimeoutError,
- ::HTTPClient::KeepAliveDisconnected,
+ Net::HTTPRetriableError,
  OpenSSL::SSL::SSLError,
  ]
  Bosh::Retryable.new({sleep: 0, tries: 3, on: handled_exceptions})

src/bosh-director/lib/bosh/director/config_server/uaa_auth_provider.rb

@@ -1,5 +1,5 @@
 require 'uaa'
-require 'httpclient'
+require 'net/http'
 
 module Bosh::Director::ConfigServer
  class UAAAuthProvider
@@ -55,9 +55,8 @@ def retryable
  Errno::ETIMEDOUT,
  Errno::ECONNRESET,
  ::Timeout::Error,
- ::HTTPClient::TimeoutError,
- ::HTTPClient::KeepAliveDisconnected,
- OpenSSL::SSL::SSLError
+ Net::HTTPRetriableError,
+ OpenSSL::SSL::SSLError,
  ]
  Bosh::Retryable.new({sleep: 0, tries: 3, on: handled_exceptions})
  end

src/bosh-director/spec/unit/config_server/auth_http_client_spec.rb

@@ -1,6 +1,5 @@
 require 'spec_helper'
 require 'rack/test'
-require 'httpclient'
 
 describe Bosh::Director::ConfigServer::AuthHTTPClient do
  subject { Bosh::Director::ConfigServer::AuthHTTPClient.new }

src/bosh-director/spec/unit/config_server/config_server_http_client_spec.rb

@@ -1,5 +1,5 @@
 require 'spec_helper'
-require 'httpclient'
+
 module Bosh::Director::ConfigServer
 
  describe 'compatibility of Enabled and Disabled versions of ConfigServer HTTP Clients' do

src/bosh-director/spec/unit/config_server/retryable_http_client_spec.rb

@@ -1,5 +1,4 @@
 require 'spec_helper'
-require 'httpclient'
 
 describe Bosh::Director::ConfigServer::RetryableHTTPClient do
  subject { Bosh::Director::ConfigServer::RetryableHTTPClient.new(http_client) }
@@ -14,8 +13,7 @@
  Errno::ETIMEDOUT,
  Errno::ECONNRESET,
  Timeout::Error,
- HTTPClient::TimeoutError,
- HTTPClient::KeepAliveDisconnected,
+ Net::HTTPRetriableError,
  OpenSSL::SSL::SSLError,
  ]
  end

src/bosh-director/spec/unit/config_server/uaa_auth_provider_spec.rb

@@ -1,6 +1,5 @@
 require 'spec_helper'
 require 'rack/test'
-require 'httpclient'
 
 describe Bosh::Director::ConfigServer::UAAAuthProvider do
  include Support::UaaHelpers
@@ -145,9 +144,8 @@
  Errno::ETIMEDOUT,
  Errno::ECONNRESET,
  Timeout::Error,
- HTTPClient::TimeoutError,
- HTTPClient::KeepAliveDisconnected,
- OpenSSL::SSL::SSLError
+ Net::HTTPRetriableError,
+ OpenSSL::SSL::SSLError,
  ]
 
  retryable = double("Bosh::Retryable")

src/bosh-monitor/bosh-monitor.gemspec

@@ -32,7 +32,6 @@ Gem::Specification.new do |spec|
  spec.add_dependency 'dogapi', '~> 1.45.0'
  spec.add_dependency 'riemann-client'
  spec.add_dependency 'cf-uaa-lib', '~>3.2.1'
- spec.add_dependency 'httpclient', '~>2.8.3'
 
  spec.add_development_dependency 'async-rspec'
  spec.add_development_dependency 'timecop'

src/bosh-monitor/lib/bosh/monitor/plugins/event_logger.rb

@@ -78,12 +78,12 @@ def auth_provider(director_info)
  def director_info
  return @director_info if @director_info
 
- director_info_url  = @url.dup
+ director_info_url = @url.dup
  director_info_url.path = '/info'
- response = send_http_get_request(director_info_url.to_s)
- return nil if response.status_code != 200
+ body, status = send_http_get_request_synchronous(director_info_url.to_s)
+ return nil if status != 200
 
- @director_info = JSON.parse(response.body)
+ @director_info = JSON.parse(body)
  end
  end
  end

src/bosh-monitor/lib/bosh/monitor/plugins/http_request_helper.rb

@@ -1,6 +1,6 @@
-require 'httpclient'
 require 'async/http'
 require 'async/http/proxy'
+require 'net/http'
 
 module Bosh::Monitor::Plugins
  module HttpRequestHelper
@@ -14,31 +14,45 @@ def send_http_post_request(uri, request)
  process_async_http_request(method: :post, uri: uri, headers: request.fetch(:head, {}), body: request.fetch(:body, nil), proxy: request.fetch(:proxy, nil))
  end
 
- def send_http_get_request(uri, headers = nil)
+ def send_http_get_request_synchronous(uri, headers = nil)
+ parsed_uri = URI.parse(uri.to_s)
+
  # we are interested in response, so send sync request
- logger.debug("Sending GET request to #{uri}")
- cli = sync_client(OpenSSL::SSL::VERIFY_NONE)
- env_proxy = URI.parse(uri.to_s).find_proxy
- cli.proxy = env_proxy unless env_proxy.nil?
+ logger.debug("Sending GET request to #{parsed_uri}")
+
+ net_http = sync_client(parsed_uri, OpenSSL::SSL::VERIFY_NONE)
 
- return cli.get(uri) if headers.nil?
+ response = net_http.get(parsed_uri.request_uri, headers)
 
- cli.get(uri, nil, headers)
+ [response.body, response.code.to_i]
  end
 
- def send_http_post_sync_request(uri, request)
- cli = sync_client
- env_proxy = URI.parse(uri.to_s).find_proxy
- cli.proxy = env_proxy unless env_proxy.nil?
- cli.post(uri, request[:body])
+ def send_http_post_request_synchronous_with_tls_verify_peer(uri, request)
+ parsed_uri = URI.parse(uri.to_s)
+
+ net_http = sync_client(parsed_uri, OpenSSL::SSL::VERIFY_PEER)
+
+ response = net_http.post(parsed_uri.request_uri, request[:body])
+
+ [response.body, response.code.to_i]
  end
 
  private
 
- def sync_client(ssl_verify_mode = OpenSSL::SSL::VERIFY_PEER)
- client = HTTPClient.new
- client.ssl_config.verify_mode = ssl_verify_mode
- client
+ def sync_client(parsed_uri, ssl_verify_mode)
+ net_http = Net::HTTP.new(parsed_uri.host, parsed_uri.port)
+ net_http.use_ssl = (parsed_uri.scheme == 'https')
+ net_http.verify_mode = ssl_verify_mode
+
+ env_proxy = parsed_uri.find_proxy
+ unless env_proxy.nil?
+ net_http.proxy_address = env_proxy.host
+ net_http.proxy_port = env_proxy.port
+ net_http.proxy_user = env_proxy.user
+ net_http.proxy_pass = env_proxy.password
+ end
+
+ net_http
  end
 
  def process_async_http_request(method:, uri:, headers: {}, body: nil, proxy: nil)

src/bosh-monitor/lib/bosh/monitor/plugins/pagerduty.rb

@@ -34,8 +34,13 @@ def process(event)
 
  request[:proxy] = options['http_proxy'] if options['http_proxy']
 
+ # Note this appears to be the only use of `#send_http_post_request_synchronous_with_tls_verify_peer` from git
+ # history it appears that the previous asynchronous implementation (EventMachine) was not able to use
+ # TLS-proxies, so a synchronous call was made from asynchronously.
+ # This should probably be re-implemented to use the asynchronous `HttpRequestHelper#send_http_post_request`
+ # method _however_ that method should be updated to use `OpenSSL::SSL::VERIFY_PEER`.
  Async do
- send_http_post_sync_request(API_URI, request)
+ send_http_post_request_synchronous_with_tls_verify_peer(API_URI, request)
  rescue StandardError => e
  logger.error("Error sending pagerduty event: #{e}")
  end

src/bosh-monitor/lib/bosh/monitor/plugins/resurrector.rb

@@ -116,15 +116,15 @@ def scan_and_fix_already_queued_or_processing?(deployment_name)
  'Content-Type' => 'application/json',
  }
  url.query = URI.encode_www_form({ deployment: deployment_name, state: 'queued,processing', verbose: 2 })
- tasks_response = send_http_get_request(url.to_s, headers)
+ body, status = send_http_get_request_synchronous(url.to_s, headers)
 
  # Getting the current tasks may fail. In a situation where the director is already dealing with lots of scan and fix tasks,
  # we may want to postpone adding another one to the queue to give the director time to deal with the currently scheduled tasks.
  # In the case of the tasks endpoint misbehaving ( status != 200 ) we can safely skip scheduling the the scan and fix in the current iteration.
- # The alerts about missing healthchecks will trigger again some time later (when the director is under less pressure).
- return true if tasks_response.status != 200
+ # The alerts about missing health-checks will trigger again some time later (when the director is under less pressure).
+ return true if status != 200
 
- queued_scan_and_fix = JSON.parse(tasks_response.body).select do |task|
+ queued_scan_and_fix = JSON.parse(body).select do |task|
  task['description'] == 'scan and fix'
  end
 
@@ -142,10 +142,10 @@ def director_info
 
  url = @uri.dup
  url.path = '/info'
- response = send_http_get_request(url.to_s)
- return nil if response.status_code != 200
+ body, status = send_http_get_request_synchronous(url.to_s)
+ return nil if status != 200
 
- @director_info = JSON.parse(response.body)
+ @director_info = JSON.parse(body)
  end
 
  def pretty_str(jobs_to_instances)