Set Kibana timezone to UTC

This script will change the default timezone used by Kibana to display
timestamps. It emits a proper setting directly into the `.kibana`
index stored in elasticsearch.

It will also create a `.kibana` index if one doesn't exist, with the default
settings normally used by Kibana: 1 shard with 1 replica only.

After the release of Kibana 5 we will refactor this script to use Kibana
API: elastic/kibana#6849

Test that the script produces the expected exit codes and hits the expected endpoints.

We run the script under test in a separate process since that is closer to reality.
This requires us to mock http endpoints using a real http server.
We had to upgrade `mimic` and work around a few design issues as the tooling for doing this isn't very good.
We might want to reconsider how we do this sort of testing if we have to do a lot of it.

@combor & @benhyland

Gemfile.lock

@@ -59,26 +59,29 @@ GEM
     cucumber-core (1.4.0)
       gherkin (~> 3.2.0)
     cucumber-wire (0.0.1)
+    daemons (1.2.3)
     diff-lcs (1.2.5)
+    eventmachine (1.2.0.1)
     ffi (1.9.10)
     gherkin (3.2.0)
     hashdiff (0.3.0)
     highline (1.6.21)
     httpclient (2.7.1)
     jmespath (1.1.3)
-    json (1.8.3)
+    json (2.0.1)
     json-minify (0.0.2)
       json (> 0)
     json_pure (1.8.3)
     little-plugger (1.1.4)
     logging (1.8.2)
       little-plugger (>= 1.1.3)
       multi_json (>= 1.8.4)
-    mimic (0.4.3)
+    mimic (0.4.4)
       json
-      plist
+      plist (~> 3.1.0)
       rack
       sinatra
+      thin
     minitar (0.5.4)
     multi_json (1.11.2)
     multi_test (0.1.2)
@@ -90,7 +93,7 @@ GEM
     netaddr (1.5.1)
     parser (2.3.0.7)
       ast (~> 2.2)
-    plist (3.2.0)
+    plist (3.1.0)
     powerpack (0.1.1)
     progressbar (0.9.2)
     rack (1.6.4)
@@ -125,8 +128,12 @@ GEM
       tilt (>= 1.3, < 3)
     sshkey (1.7.0)
     terminal-table (1.4.5)
+    thin (1.7.0)
+      daemons (~> 1.0, >= 1.0.9)
+      eventmachine (~> 1.0, >= 1.0.4)
+      rack (>= 1, < 3)
     thor (0.19.1)
-    tilt (2.0.4)
+    tilt (2.0.5)
     unicode-display_width (1.0.3)
     webmock (1.24.5)
       addressable (>= 2.3.6)
@@ -146,4 +153,4 @@ DEPENDENCIES
   webmock (~> 1.24)
 
 BUNDLED WITH
-   1.11.2
+   1.12.5

concourse/scripts/kibana_set_utc.rb

@@ -0,0 +1,76 @@
+#! /usr/bin/env ruby
+
+require 'net/http'
+require 'json'
+
+def config_uri(es_url)
+  config_url = "#{es_url}/.kibana/config/4.3.1"
+  URI(config_url)
+end
+
+def index_settings_uri(es_url)
+  index_settings_url = "#{es_url}/.kibana"
+  URI(index_settings_url)
+end
+
+def update_es_index(uri, document)
+  send_request(
+    uri: uri,
+    http_method: :Put,
+    body: document,
+    allowed_response_codes: ['200', '201']
+  )
+end
+
+def send_request(uri:, http_method:, allowed_response_codes:, body: nil)
+  raise "Document is not a hash: #{body}" unless body == nil || body.is_a?(Hash)
+  response = nil
+  Net::HTTP.new(uri.host, uri.port).start do |http|
+    method = Net::HTTP.const_get(http_method)
+    headers = body ? { 'Content-Type' => 'application/json' } : {}
+    req = method.new(uri.path, headers)
+    req.body = body.to_json if body
+    response = http.request(req)
+    unless allowed_response_codes.include?(response.code)
+      raise "Unexpected response code: #{response.code}\n#{response.body}"
+    end
+  end
+  response
+end
+
+def need_to_create_index(response)
+  !response["found"]
+end
+
+def need_to_set_timezone(response)
+  !response["_source"] || response["_source"]["dateFormat:tz"] != "UTC"
+end
+
+def set_utc_config
+  es_host = ENV.fetch("ES_HOST")
+  es_port = ENV.fetch("ES_PORT")
+
+  es_url = "http://#{es_host}:#{es_port}"
+
+  config_uri = config_uri(es_url)
+  response = send_request(uri: config_uri, http_method: :Get, allowed_response_codes: ['200', '404'])
+  response_json = JSON.parse(response.body)
+
+  if need_to_create_index(response_json)
+    index_settings = {
+      settings: {
+        index: {
+          number_of_shards: 1,
+          number_of_replicas: 1,
+        }
+      }
+    }
+    update_es_index(index_settings_uri(es_url), index_settings)
+  end
+
+  if need_to_set_timezone(response_json)
+    update_es_index(config_uri(es_url), { "dateFormat:tz" => "UTC" })
+  end
+end
+
+set_utc_config

concourse/scripts/spec/kibana_set_utc_spec.rb

@@ -0,0 +1,116 @@
+require 'mimic'
+
+RSpec.describe "kibana_set_utc.rb", :type => :aruba do
+
+  ES_HOST = "127.0.0.1"
+  ES_PORT = "9200"
+  KIBANA_CONFIG_PATH = "/.kibana/config/4.3.1"
+  KIBANA_INDEX_PATH = "/.kibana"
+
+  Thread.abort_on_exception = true
+
+  def run_set_utc_script
+    run("./kibana_set_utc.rb")
+  end
+
+  before :all do
+    @elasticsearch = Mimic.mimic(:hostname => ES_HOST, :port => ES_PORT)
+    set_environment_variable 'ES_HOST', ES_HOST
+    set_environment_variable 'ES_PORT', ES_PORT
+  end
+
+  after :each do
+    Mimic.reset_all!
+  end
+
+  after :all do
+    Mimic.cleanup!
+  end
+
+  it "creates index and adds utc config if no index exists" do
+
+    @elasticsearch.instance_variable_get(:@app).not_found do
+      # override the not_found handler provided by mimic,
+      # so that we don't emit an empty body instead of the json we want with the 404
+      [404, {}, nil]
+    end
+
+    @elasticsearch.get(KIBANA_CONFIG_PATH).returning('{ "found": false }', 404)
+    @elasticsearch.put(KIBANA_INDEX_PATH).returning('{}', 201)
+    @elasticsearch.put(KIBANA_CONFIG_PATH).returning('{}', 201)
+
+    run_set_utc_script
+
+    expect(last_command_started).to have_exit_status(0)
+    expect(@elasticsearch.received_requests.size).to be(3)
+    expect(@elasticsearch.received_requests).to contain_request('GET', KIBANA_CONFIG_PATH)
+    expect(@elasticsearch.received_requests).to contain_request('PUT', KIBANA_INDEX_PATH)
+    expect(@elasticsearch.received_requests).to contain_request('PUT', KIBANA_CONFIG_PATH)
+  end
+
+  it "adds utc config if index exists but config is not present" do
+    @elasticsearch.get(KIBANA_CONFIG_PATH).returning('{ "found": true }', 200)
+    @elasticsearch.put(KIBANA_CONFIG_PATH).returning('{}', 200)
+
+    run_set_utc_script
+
+    expect(last_command_started).to have_exit_status(0)
+
+    expect(@elasticsearch.received_requests.size).to be(2)
+    expect(@elasticsearch.received_requests).to contain_request('GET', KIBANA_CONFIG_PATH)
+    expect(@elasticsearch.received_requests).to contain_request('PUT', KIBANA_CONFIG_PATH)
+  end
+
+  it "adds utc config if index exists and wrong config is present" do
+    @elasticsearch.get(KIBANA_CONFIG_PATH).returning('{ "found": true, "_source": { "dateFormat:tz": "NOT_UTC" } }', 200)
+    @elasticsearch.put(KIBANA_CONFIG_PATH).returning('{}', 200)
+
+    run_set_utc_script
+
+    expect(last_command_started).to have_exit_status(0)
+
+    expect(@elasticsearch.received_requests.size).to be(2)
+    expect(@elasticsearch.received_requests).to contain_request('GET', KIBANA_CONFIG_PATH)
+    expect(@elasticsearch.received_requests).to contain_request('PUT', KIBANA_CONFIG_PATH)
+  end
+
+  it "reports an error if elastic search is not available" do
+
+    run_set_utc_script
+
+    expect(last_command_started).to have_exit_status(1)
+    expect(@elasticsearch.received_requests.size).to be(0)
+  end
+
+  it "does nothing if index exists and config is already UTC" do
+    @elasticsearch.get(KIBANA_CONFIG_PATH).returning('{ "found": true, "_source": { "dateFormat:tz": "UTC" } }', 200)
+
+    run_set_utc_script
+
+    expect(last_command_started).to have_exit_status(0)
+
+    expect(@elasticsearch.received_requests.size).to be(1)
+    expect(@elasticsearch.received_requests).to contain_request('GET', KIBANA_CONFIG_PATH)
+  end
+
+  it "reports an error if elastic search responds with an unexpected status code" do
+    @elasticsearch.get(KIBANA_CONFIG_PATH).returning('{ "found": true, "_source": { "dateFormat:tz": "UTC" } }', 500)
+
+    run_set_utc_script
+
+    expect(last_command_started).to have_exit_status(1)
+
+    expect(@elasticsearch.received_requests.size).to be(1)
+    expect(@elasticsearch.received_requests).to contain_request('GET', KIBANA_CONFIG_PATH)
+  end
+
+  matcher :contain_request do |expected_method, expected_path|
+    match do |requests|
+      requests.any? do |request|
+        actual_path = request.instance_variable_get(:@path)
+        actual_method = request.instance_variable_get(:@method)
+        actual_method == expected_method && actual_path == expected_path
+      end
+    end
+  end
+end