Speed up tests

Shopify · Nov 8, 2018 · 959649b · 959649b
1 parent dfbc603
commit 959649b
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 22 deletions.
diff --git a/test/integration-serial/serial_task_run_test.rb b/test/integration-serial/serial_task_run_test.rb
@@ -29,21 +29,22 @@ def test_run_without_verify_result_fails_if_pod_was_not_created
     ], in_order: true)
   end
 
-  # We can't ensure that all the metrics are specific test when they are running in parallel
-  # so we run them serially
+  # Run statsd tests in serial because capture_statsd_calls modifies global state in a way
+  # that makes capturing metrics across parrallel runs unreliable
   def test_failure_statsd_metric_emitted
-    deploy_task_template
-    task_runner = build_task_runner
+    bad_ns = "missing"
+    task_runner = build_task_runner(ns: bad_ns)
 
     result = false
     metrics = capture_statsd_calls do
-      result = task_runner.run(run_params.merge(args: ["/not/a/command"]))
+      result = task_runner.run(run_params)
     end
 
     assert_task_run_failure(result)
 
-    metric_tags = metrics.detect { |m| m.tags.include? "namespace:#{@namespace}" }.tags
-    assert_includes metric_tags, "namespace:#{@namespace}"
+    metric = metrics.detect { |m| m.tags.include? "namespace:#{bad_ns}" }
+    assert metric, "No metrics found for this test"
+    metric_tags = metric.tags
     assert_includes metric_tags, "context:#{KubeclientHelper::TEST_CONTEXT}"
     assert_includes metric_tags, "status:failure"
   end
@@ -54,30 +55,32 @@ def test_success_statsd_metric_emitted
 
     result = false
     metrics = capture_statsd_calls do
-      result = task_runner.run(run_params.merge(args: %w(ls)))
+      result = task_runner.run(run_params.merge(verify_result: false))
     end
 
     assert_task_run_success(result)
 
-    metric_tags = metrics.detect { |m| m.tags.include? "namespace:#{@namespace}" }.tags
-    assert_includes metric_tags, "namespace:#{@namespace}"
+    metric = metrics.detect { |m| m.tags.include? "namespace:#{@namespace}" }
+    assert metric, "No metrics found for this test"
+    metric_tags = metric.tags
     assert_includes metric_tags, "context:#{KubeclientHelper::TEST_CONTEXT}"
     assert_includes metric_tags, "status:success"
   end
 
   def test_timedout_statsd_metric_emitted
     deploy_task_template
-    task_runner = build_task_runner(max_watch_seconds: 5)
+    task_runner = build_task_runner(max_watch_seconds: 0)
 
     result = false
     metrics = capture_statsd_calls do
-      result = task_runner.run(run_params.merge(args: ["sleep 600"]))
+      result = task_runner.run(run_params.merge(args: ["sleep 5"]))
     end
 
     assert_task_run_failure(result, :timed_out)
 
-    metric_tags = metrics.detect { |m| m.tags.include? "namespace:#{@namespace}" }.tags
-    assert_includes metric_tags, "namespace:#{@namespace}"
+    metric = metrics.detect { |m| m.tags.include? "namespace:#{@namespace}" }
+    assert metric, "No metrics found for this test"
+    metric_tags = metric.tags
     assert_includes metric_tags, "context:#{KubeclientHelper::TEST_CONTEXT}"
     assert_includes metric_tags, "status:timeout"
   end

diff --git a/test/integration/kubernetes_deploy_test.rb b/test/integration/kubernetes_deploy_test.rb
@@ -1,5 +1,5 @@
 # frozen_string_literal: true
-require 'integration_test_helper'
+require 'test_helper'
 
 class KubernetesDeployTest < KubernetesDeploy::IntegrationTest
   def test_full_hello_cloud_set_deploy_succeeds
@@ -232,7 +232,8 @@ def test_invalid_k8s_spec_that_is_valid_yaml_fails_fast_and_prints_template
       "Template validation failed",
       /Invalid template: ConfigMap-hello-cloud-configmap-data.*yml/,
       "> Error message:",
-      /error validating data: .* (invalid|unknown) field "?myKey"? .*v1.ObjectMeta/,
+      "error validating data: ValidationError(ConfigMap.metadata): \
+unknown field \"myKey\" in io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta",
       "> Template content:",
       "      myKey: uhOh"
     ], in_order: true)
@@ -544,7 +545,7 @@ def test_pruning_of_existing_managed_secrets_when_ejson_file_has_been_deleted
   def test_deploy_result_logging_for_mixed_result_deploy
     subset = ["bad_probe.yml", "init_crash.yml", "missing_volumes.yml", "config_map.yml"]
     result = deploy_fixtures("invalid", subset: subset) do |f|
-      if kube_server_version >= Gem::Version.new("1.10.0") # https://github.com/kubernetes/kubernetes/issues/66135
+      if KUBE_SERVER_VERSION >= Gem::Version.new("1.10.0") # https://github.com/kubernetes/kubernetes/issues/66135
         f["bad_probe.yml"]["Deployment"].first["spec"]["progressDeadlineSeconds"] = 20
       end
     end
@@ -556,7 +557,7 @@ def test_deploy_result_logging_for_mixed_result_deploy
       %r{ConfigMap/test\s+Available}
     ], in_order: true)
 
-    if kube_server_version < Gem::Version.new("1.10.0")
+    if KUBE_SERVER_VERSION < Gem::Version.new("1.10.0")
       start_bad_probe_logs = [
         %r{Deployment/bad-probe: TIMED OUT \(timeout: \d+s\)},
         "Timeout reason: hard deadline for Deployment"
@@ -963,7 +964,7 @@ def test_resource_watcher_reports_failed_after_timeout
       max_watch_seconds: 20
     ) do |f|
       bad_probe = f["bad_probe.yml"]["Deployment"].first
-      if kube_server_version < Gem::Version.new("1.10.0") # https://github.com/kubernetes/kubernetes/issues/66135
+      if KUBE_SERVER_VERSION < Gem::Version.new("1.10.0") # https://github.com/kubernetes/kubernetes/issues/66135
         bad_probe["metadata"]["annotations"]["kubernetes-deploy.shopify.io/timeout-override"] = '5s'
       else
         bad_probe["spec"]["progressDeadlineSeconds"] = 5
@@ -973,7 +974,7 @@ def test_resource_watcher_reports_failed_after_timeout
     end
     assert_deploy_failure(result)
 
-    bad_probe_timeout = if kube_server_version < Gem::Version.new("1.10.0")
+    bad_probe_timeout = if KUBE_SERVER_VERSION < Gem::Version.new("1.10.0")
       "Deployment/bad-probe: TIMED OUT (timeout: 5s)"
     else
       "Deployment/bad-probe: TIMED OUT (progress deadline: 5s)"
@@ -1057,7 +1058,7 @@ def test_raise_on_yaml_missing_kind
   end
 
   def test_hpa_can_be_successful
-    skip if kube_server_version < Gem::Version.new('1.9.0')
+    skip if KUBE_SERVER_VERSION < Gem::Version.new('1.9.0')
     assert_deploy_success(deploy_fixtures("hpa"))
     assert_logs_match_all([
       "Deploying resources:",
@@ -1067,7 +1068,7 @@ def test_hpa_can_be_successful
   end
 
   def test_hpa_can_be_pruned
-    skip if kube_server_version < Gem::Version.new('1.9.0')
+    skip if KUBE_SERVER_VERSION < Gem::Version.new('1.9.0')
     assert_deploy_success(deploy_fixtures("hpa"))
     assert_deploy_success(deploy_fixtures("hpa", subset: ["deployment.yml"]))
     assert_logs_match_all([