From 1c19c68c8f2cd318e2726254c2b8fe77cd9eae20 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Wed, 8 Jul 2020 21:19:24 -0400
Subject: [PATCH 01/34] initial commit

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 api/adaptive_rps/BUILD                        |  22 ++++
 api/adaptive_rps/adaptive_rps.proto           |  65 ++++++++++
 api/adaptive_rps/benchmark_result.proto       |  55 +++++++++
 api/adaptive_rps/metric_spec.proto            |  50 ++++++++
 api/adaptive_rps/metrics_plugin.proto         |  13 ++
 api/adaptive_rps/metrics_plugin_impl.proto    |  15 +++
 api/adaptive_rps/scoring_function.proto       |  14 +++
 api/adaptive_rps/scoring_function_impl.proto  |  32 +++++
 api/adaptive_rps/step_controller.proto        |  13 ++
 api/adaptive_rps/step_controller_impl.proto   |  29 +++++
 .../example_session_spec.textproto            | 116 ++++++++++++++++++
 11 files changed, 424 insertions(+)
 create mode 100644 api/adaptive_rps/BUILD
 create mode 100644 api/adaptive_rps/adaptive_rps.proto
 create mode 100644 api/adaptive_rps/benchmark_result.proto
 create mode 100644 api/adaptive_rps/metric_spec.proto
 create mode 100644 api/adaptive_rps/metrics_plugin.proto
 create mode 100644 api/adaptive_rps/metrics_plugin_impl.proto
 create mode 100644 api/adaptive_rps/scoring_function.proto
 create mode 100644 api/adaptive_rps/scoring_function_impl.proto
 create mode 100644 api/adaptive_rps/step_controller.proto
 create mode 100644 api/adaptive_rps/step_controller_impl.proto
 create mode 100644 samples/adaptive_rps/example_session_spec.textproto

diff --git a/api/adaptive_rps/BUILD b/api/adaptive_rps/BUILD
new file mode 100644
index 000000000..b81508e95
--- /dev/null
+++ b/api/adaptive_rps/BUILD
@@ -0,0 +1,22 @@
+load("@envoy_api//bazel:api_build_system.bzl", "api_cc_py_proto_library")
+
+licenses(["notice"])  # Apache 2
+
+api_cc_py_proto_library(
+    name = "base",
+    srcs = [
+        "adaptive_rps.proto",
+        "benchmark_result.proto",
+        "metric_spec.proto",
+        "metrics_plugin.proto",
+        "metrics_plugin_impl.proto",
+        "scoring_function.proto",
+        "scoring_function_impl.proto",
+        "step_controller.proto",
+        "step_controller_impl.proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "@nighthawk//api/client:base",
+    ],
+)
diff --git a/api/adaptive_rps/adaptive_rps.proto b/api/adaptive_rps/adaptive_rps.proto
new file mode 100644
index 000000000..c4756bd75
--- /dev/null
+++ b/api/adaptive_rps/adaptive_rps.proto
@@ -0,0 +1,65 @@
+syntax = "proto3";
+
+package nighthawk.adaptive_rps;
+
+import "api/adaptive_rps/benchmark_result.proto";
+import "api/adaptive_rps/metrics_plugin.proto";
+import "api/adaptive_rps/metric_spec.proto";
+import "api/adaptive_rps/step_controller.proto";
+import "api/client/options.proto";
+import "google/protobuf/duration.proto";
+import "google/rpc/status.proto";
+import "validate/validate.proto";
+
+// Parameters describing the adjusting and testing stages of an adaptive RPS
+// session, which consists of a series of Nighthawk benchmarks probing for
+// the optimal RPS, followed by a longer benchmark to validate the optimal RPS.
+// RPS calculations are implemented by the selected StepController plugin.
+// Metrics can eb
+message AdaptiveRpsSessionSpec {
+  // Settings for MetricsPlugins that obtain metrics from outside sources.
+  // An entry is required for every plugin referred to by metric_thresholds,
+  // other than the "builtin" plugin.
+  repeated MetricsPluginConfig metrics_plugin_configs = 1;
+  // Metrics and thresholds that determine RPS adjustments.
+  repeated MetricSpecWithThreshold metric_thresholds = 2 [(validate.rules).repeated .min_items = 1];
+  // Metrics that are collected and included in the output but not taken into
+  // account when adjusting the RPS.
+  repeated MetricSpec informational_metric_specs = 3;
+  // A proto describing Nighthawk Service traffic. See
+  // https://github.com/envoyproxy/nighthawk/blob/master/api/client/options.proto
+  //
+  // The adaptive RPS controller will return an error if the |duration|,
+  // |requests_per_second|,  or |open_loop| fields are set within
+  // |nighthawk_traffic_options|; the controller sets these values dynamically.
+  //
+  // All other fields in |nighthawk_traffic_options| are passed through to the
+  // Nighthawk Service.
+  //
+  // Note that |concurrency| in |nighthawk_traffic_options| multiplies the total
+  // RPS sent.
+  nighthawk.client.CommandLineOptions nighthawk_traffic_template = 4;
+  // The duration of each short benchmark during the adjusting stage. Must be
+  // set. O(10 seconds).
+  google.protobuf.Duration measuring_period = 5 [(validate.rules).duration.gt.seconds = 0];
+  // Maximum amount of time the adjusting stage should wait for convergence
+  // before returning an error. O(300 seconds).
+  google.protobuf.Duration convergence_deadline = 6 [(validate.rules).duration.gt.seconds = 0];
+  // The duration of the single benchmark session of the testing stage to
+  // confirm the performance at the RPS found by the adjusting stage. Must be
+  // set.
+  google.protobuf.Duration testing_stage_duration = 7 [(validate.rules).duration.gt.seconds = 0];
+  // Selects and configures a StepController plugin.
+  StepControllerConfig step_controller_config = 8;
+}
+
+// Complete description of an adaptive RPS session, including metric scores
+// for every RPS value attempted during the adjusting stage.
+message AdaptiveRpsSessionOutput {
+  // Overall status of the session with error detail.
+  google.rpc.Status session_status = 1;
+  // Results of each short benchmark performed during the adjusting stage.
+  repeated BenchmarkResult adjusting_stage_results = 2;
+  // Result of the single benchmark of the testing stage.
+  BenchmarkResult testing_stage_result = 3;
+}
diff --git a/api/adaptive_rps/benchmark_result.proto b/api/adaptive_rps/benchmark_result.proto
new file mode 100644
index 000000000..7c09fd35c
--- /dev/null
+++ b/api/adaptive_rps/benchmark_result.proto
@@ -0,0 +1,55 @@
+syntax = "proto3";
+
+package nighthawk.adaptive_rps;
+
+import "api/adaptive_rps/metric_spec.proto";
+import "api/client/output.proto";
+
+// Binary result of checking a metric value against a threshold.
+enum SimpleThresholdStatus {
+  // Value should only occur when the field is unset.
+  UNKNOWN_THRESHOLD_STATUS = 0;
+  // The metric was below an upper threshold or above a lower threshold.
+  WITHIN_THRESHOLD = 1;
+  // The metric was above an upper threshold or below a lower threshold.
+  OUTSIDE_THRESHOLD = 2;
+}
+
+// Encapsulates results of simple and custom threshold checks.
+message ThresholdCheckResult {
+  oneof threshold_check_result {
+    // Binary status from checking the value against a threshold.
+    SimpleThresholdStatus simple_threshold_status = 1;
+    // Score returned by a ScoringFunction plugin. This expresses how close the
+    // metric was to the threshold by an arbitrary formula selected and
+    // configured in the ThresholdSpec, such as a sigmoid curve. The controller
+    // can choose to make larger RPS adjustments when the score is larger, in
+    // order to converge faster.
+    double threshold_score = 2;
+  }
+}
+
+// Records the status of a single metric during a benchmark session.
+message MetricEvaluation {
+  // Original metric specification (plugin name, metric name).
+  MetricSpec metric_spec = 1;
+  // Numerical value of the metric measured during this benchmark session.
+  double metric_value = 2;
+  // Threshold that was used to check this metric value. Not set if the metric
+  // was only informational.
+  ThresholdSpec threshold_spec = 3;
+  // The result of checking this metric value against the configured threshold.
+  // Not set if the metric was only informational.
+  ThresholdCheckResult threshold_check_result = 4;
+}
+
+// Summary of a single Nighthawk Service benchmark session with evaluation
+// results.
+message BenchmarkResult {
+  // Raw Nighthawk Service output. Includes start/end times and full Nighthawk
+  // Service input spec. May contain an error status.
+  nighthawk.client.Output nighthawk_service_output = 1;
+  // Status of all defined adaptive RPS feedback metrics during this benchmark
+  // session. Not present in the event of Nighthawk errors.
+  repeated MetricEvaluation metric_evaluations = 3;
+}
diff --git a/api/adaptive_rps/metric_spec.proto b/api/adaptive_rps/metric_spec.proto
new file mode 100644
index 000000000..249c82ee3
--- /dev/null
+++ b/api/adaptive_rps/metric_spec.proto
@@ -0,0 +1,50 @@
+syntax = "proto3";
+
+package nighthawk.adaptive_rps;
+
+import "api/adaptive_rps/scoring_function.proto";
+import "google/protobuf/wrappers.proto";
+import "validate/validate.proto";
+
+// Identifies a feedback metric.
+message MetricSpec {
+  // Name of the metric to evaluate. For the set of built-in metric names, see
+  // source/adaptive_rps/metrics_plugin_impl.cc.
+  string metric_name = 1 [(validate.rules).string.min_len = 1];
+  // Name of the MetricsPlugin providing the metric ("builtin" for built-in).
+  string metrics_plugin_name = 2 [(validate.rules).string.min_len = 1];
+}
+
+// Specifies a simple upper or lower threshold, or configures a scoring
+// function.
+message ThresholdSpec {
+  // A threshold for evaluating a metric value.
+  oneof threshold_spec {
+    // Simple upper threshold:
+    //   metric <= upper_threshold -> WITHIN_THRESHOLD
+    //   metric > upper_threshold -> OUTSIDE_THRESHOLD
+    google.protobuf.DoubleValue upper_threshold = 1;
+    // Simple lower threshold:
+    //   metric >= lower_threshold -> WITHIN_THRESHOLD
+    //   metric < lower_threshold -> OUTSIDE_THRESHOLD
+    google.protobuf.DoubleValue lower_threshold = 2;
+    // Selection and configuration of a ScoringFunction that gives a numerical
+    // measure of proximity to a threshold according to an arbitrary formula.
+    // 0.0 means the value equals the threshold, positive means the value is
+    // below the threshold so RPS should increase, and negative means the value
+    // is above the threshold so RPS should decrease.
+    ScoringFunctionConfig scoring_function = 3;
+  }
+  // Relative importance of this threshold when calculating adaptive RPS based
+  // on multiple metrics. Must be a positive number if set. If any threshold has
+  // a weight, all must have a weight. If no weights are set, all thresholds
+  // have equal weight.
+  google.protobuf.DoubleValue weight = 4 [(validate.rules).double = {gt: 0.0}];
+}
+
+message MetricSpecWithThreshold {
+  // Identifies a metric to collect and evaluate.
+  MetricSpec metric_spec = 1;
+  // Specifies a threshold for this metric.
+  ThresholdSpec threshold_spec = 2;
+}
diff --git a/api/adaptive_rps/metrics_plugin.proto b/api/adaptive_rps/metrics_plugin.proto
new file mode 100644
index 000000000..2c385bb41
--- /dev/null
+++ b/api/adaptive_rps/metrics_plugin.proto
@@ -0,0 +1,13 @@
+syntax = "proto3";
+
+package nighthawk.adaptive_rps;
+
+import "google/protobuf/any.proto";
+
+// Selects and configures a MetricsPlugin.
+message MetricsPluginConfig {
+  // Name used to locate the plugin in the Envoy registry.
+  string name = 1;
+  // Plugin-specific config such as an external API endpoint and credentials.
+  google.protobuf.Any typed_config = 2;
+}
diff --git a/api/adaptive_rps/metrics_plugin_impl.proto b/api/adaptive_rps/metrics_plugin_impl.proto
new file mode 100644
index 000000000..198ae938f
--- /dev/null
+++ b/api/adaptive_rps/metrics_plugin_impl.proto
@@ -0,0 +1,15 @@
+// Plugin-specific config protos for MetricsPlugin.
+
+syntax = "proto3";
+
+package nighthawk.adaptive_rps;
+
+import "validate/validate.proto";
+
+// Example plugin-specific config proto, representing connection info for an
+// outside source of metrics. Part of an example showing how to create and
+// register a MetricsPlugin.
+message ExampleMetricsPluginConfig {
+  string address = 1;
+  string credentials = 2;
+}
diff --git a/api/adaptive_rps/scoring_function.proto b/api/adaptive_rps/scoring_function.proto
new file mode 100644
index 000000000..88f85ac89
--- /dev/null
+++ b/api/adaptive_rps/scoring_function.proto
@@ -0,0 +1,14 @@
+syntax = "proto3";
+
+package nighthawk.adaptive_rps;
+
+import "google/protobuf/any.proto";
+
+// Configures a ScoringFunction plugin that measures proximity to a threshold
+// according to an arbitrary formula.
+message ScoringFunctionConfig {
+  // Name used to locate the plugin in the Envoy registry.
+  string name = 1;
+  // Plugin-specific config such as thresholds and constants.
+  google.protobuf.Any typed_config = 2;
+}
diff --git a/api/adaptive_rps/scoring_function_impl.proto b/api/adaptive_rps/scoring_function_impl.proto
new file mode 100644
index 000000000..dd5901699
--- /dev/null
+++ b/api/adaptive_rps/scoring_function_impl.proto
@@ -0,0 +1,32 @@
+// Plugin-specific config protos for ScoringFunctions.
+
+syntax = "proto3";
+
+package nighthawk.adaptive_rps;
+
+// Configuration for a linear ScoringFunction that calculates a metric score
+// as k * (threshold - value), where k is a scaling constant. The score is 0.0
+// when the value exactly equals the threshold, positive below the threshold
+// (meaning RPS should increase), and negative above the threshold. The score is
+// proportional to the difference from the threshold.
+message LinearScoringFunctionConfig {
+  // The target value of the metric.
+  double threshold = 1;
+  // Scaling constant: k in k * (threshold - value). Use this in combination
+  // with step controller constants to produce reasonable RPS increments for
+  // reasonable differences from the threshold.
+  double k = 2;
+}
+
+// Configuration for a ScoringFunction that calculates a metric score
+// as 1 - 2 / (1 + exp(-k(value - threshold))), an upside-down sigmoid curve
+// centered on a threshold. The output is 0.0 when the metric equals the
+// threshold, approaches 1.0 for values far below the threshold, and approaches
+// -1.0 for values far above the threshold.
+message SigmoidScoringFunctionConfig {
+  // The target value of the metric.
+  double threshold = 1;
+  // Tuning constant: k in 1 - 2 / (1 + exp(-k(value - threshold))). k should
+  // be around the same size as 1/threshold.
+  double k = 2;
+}
diff --git a/api/adaptive_rps/step_controller.proto b/api/adaptive_rps/step_controller.proto
new file mode 100644
index 000000000..d1acf2dcc
--- /dev/null
+++ b/api/adaptive_rps/step_controller.proto
@@ -0,0 +1,13 @@
+syntax = "proto3";
+
+package nighthawk.adaptive_rps;
+
+import "google/protobuf/any.proto";
+
+// Selects and configures a StepController plugin.
+message StepControllerConfig {
+  // Name used to locate the plugin in the Envoy registry.
+  string name = 1;
+  // Plugin-specific config.
+  google.protobuf.Any typed_config = 2;
+}
diff --git a/api/adaptive_rps/step_controller_impl.proto b/api/adaptive_rps/step_controller_impl.proto
new file mode 100644
index 000000000..e1640225c
--- /dev/null
+++ b/api/adaptive_rps/step_controller_impl.proto
@@ -0,0 +1,29 @@
+// Plugin-specific config protos for StepControllers.
+
+syntax = "proto3";
+
+package nighthawk.adaptive_rps;
+
+import "validate/validate.proto";
+
+// Configuration for a step controller that performs a linear search for the
+// optimal RPS, starting from the minimum RPS and incrementing by a fixed amount
+// until reaching the point where performance declines.
+message LinearSearchStepControllerConfig {
+  // Minimum RPS that should be attempted, regardless of performance. Also used
+  // as the initial RPS.
+  uint32 minimum_rps = 1 [(validate.rules).uint32 = {gt: 0}];
+  // Maximum RPS that should be attempted, regardless of performance.
+  uint32 maximum_rps = 2 [(validate.rules).uint32 = {gt: 0}];
+  // Constant step to be used for all RPS adjustments.
+  uint32 rps_step = 3 [(validate.rules).uint32 = {gt: 0}];
+}
+
+// Configuration for a step controller that performs a binary search for the
+// optimal RPS.
+message BinarySearchStepControllerConfig {
+  // Minimum of the RPS binary search range.
+  uint32 minimum_rps = 1 [(validate.rules).uint32 = {gt: 0}];
+  // Maximum of the RPS binary search range.
+  uint32 maximum_rps = 2 [(validate.rules).uint32 = {gt: 0}];
+}
diff --git a/samples/adaptive_rps/example_session_spec.textproto b/samples/adaptive_rps/example_session_spec.textproto
new file mode 100644
index 000000000..796ac5e8f
--- /dev/null
+++ b/samples/adaptive_rps/example_session_spec.textproto
@@ -0,0 +1,116 @@
+# proto-file: adaptive_rps/adaptive_rps.proto
+# proto-message: AdaptiveRpsSessionSpec
+
+metrics_plugin_configs {
+  name: "example-metrics-plugin"
+  typed_config {
+    [type.googleapis.com/nighthawk.adaptive_rps.ExampleMetricsPluginConfig] {
+      address: "x.y.z"
+      credentials: "key-id-123"
+    }
+  }
+}
+
+# For the full list of built-in metric names, see
+# source/adaptive_rps/metrics_plugin_impl.cc.
+
+metric_thresholds {
+  metric_spec {
+    metric_name: "latency-ns-mean-plus-2stdev"
+    metrics_plugin_name: "builtin"
+  }
+  threshold_spec {
+    scoring_function {
+      name: "sigmoid"
+      typed_config {
+        [type.googleapis.com/nighthawk.adaptive_rps.SigmoidScoringFunctionConfig] {
+          threshold: 50000000.0  # 50 ms
+          k: 0.00000001  # Should be around 1/threshold
+        }
+      }
+    }
+    weight {
+      value: 1.0
+    }
+  }
+}
+
+metric_thresholds {
+  metric_spec {
+    metric_name: "success-rate"
+    metrics_plugin_name: "builtin"
+  }
+  threshold_spec {
+    lower_threshold {
+      value : 0.95
+    }
+    weight {
+      value: 1.0
+    }
+  }
+}
+
+# Informational metrics are recorded in the output but are not taken into account
+# when adjusting the RPS.
+
+informational_metric_specs {
+  metric_name: "attempted-rps"
+  metrics_plugin_name: "builtin"
+}
+
+informational_metric_specs {
+  metric_name: "achieved-rps"
+  metrics_plugin_name: "builtin"
+}
+
+informational_metric_specs {
+  metric_name: "send-rate"
+  metrics_plugin_name: "builtin"
+}
+
+# Metric defined in example metrics plugin:
+informational_metric_specs {
+  metric_name: "example_metric1"
+  metrics_plugin_name: "example-metrics-plugin"
+}
+
+# CommandLineOptions proto describing the traffic to generate, but without
+# requests_per_second, duration, or open_loop, which will all be set dynamically.
+nighthawk_traffic_template {
+  uri {
+    value: "[::1]:12345"
+  }
+}
+
+measuring_period {
+  seconds: 4
+}
+
+convergence_deadline {
+  seconds: 180
+}
+
+testing_stage_duration {
+  seconds: 30
+}
+
+step_controller_config {
+  name: "linear-search"
+  typed_config {
+    [type.googleapis.com/nighthawk.adaptive_rps.LinearSearchStepControllerConfig] {
+      rps_step: 100
+      minimum_rps: 500
+      maximum_rps: 5000
+    }
+  }
+}
+
+# step_controller_config {
+#   name : "binary-search"
+#   typed_config {
+#     [type.googleapis.com/nighthawk.adaptive_rps.BinarySearchStepControllerConfig] {
+#       minimum_rps : 500
+#       maximum_rps : 10000
+#     }
+#   }
+# }

From 70506867655a286e42cec713b69d8ba587bc31d3 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Wed, 8 Jul 2020 21:31:06 -0400
Subject: [PATCH 02/34] fix comments

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 api/adaptive_rps/adaptive_rps.proto | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/api/adaptive_rps/adaptive_rps.proto b/api/adaptive_rps/adaptive_rps.proto
index c4756bd75..bae9473cb 100644
--- a/api/adaptive_rps/adaptive_rps.proto
+++ b/api/adaptive_rps/adaptive_rps.proto
@@ -15,7 +15,8 @@ import "validate/validate.proto";
 // session, which consists of a series of Nighthawk benchmarks probing for
 // the optimal RPS, followed by a longer benchmark to validate the optimal RPS.
 // RPS calculations are implemented by the selected StepController plugin.
-// Metrics can eb
+// Metrics can come from Nighthawk stats and counters or platform-specific data
+// sources via MetricsPlugins.
 message AdaptiveRpsSessionSpec {
   // Settings for MetricsPlugins that obtain metrics from outside sources.
   // An entry is required for every plugin referred to by metric_thresholds,
@@ -40,10 +41,10 @@ message AdaptiveRpsSessionSpec {
   // RPS sent.
   nighthawk.client.CommandLineOptions nighthawk_traffic_template = 4;
   // The duration of each short benchmark during the adjusting stage. Must be
-  // set. O(10 seconds).
+  // set. Reasonable value: 10 seconds.
   google.protobuf.Duration measuring_period = 5 [(validate.rules).duration.gt.seconds = 0];
   // Maximum amount of time the adjusting stage should wait for convergence
-  // before returning an error. O(300 seconds).
+  // before returning an error. Reasonable value: 300 seconds.
   google.protobuf.Duration convergence_deadline = 6 [(validate.rules).duration.gt.seconds = 0];
   // The duration of the single benchmark session of the testing stage to
   // confirm the performance at the RPS found by the adjusting stage. Must be

From 0776563ea70d9b300b2a55ebaaaf067ed3b642a5 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Wed, 8 Jul 2020 21:37:11 -0400
Subject: [PATCH 03/34] fix format

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 api/adaptive_rps/metric_spec.proto | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/api/adaptive_rps/metric_spec.proto b/api/adaptive_rps/metric_spec.proto
index 249c82ee3..60fd7c7c6 100644
--- a/api/adaptive_rps/metric_spec.proto
+++ b/api/adaptive_rps/metric_spec.proto
@@ -28,17 +28,15 @@ message ThresholdSpec {
     //   metric >= lower_threshold -> WITHIN_THRESHOLD
     //   metric < lower_threshold -> OUTSIDE_THRESHOLD
     google.protobuf.DoubleValue lower_threshold = 2;
-    // Selection and configuration of a ScoringFunction that gives a numerical
-    // measure of proximity to a threshold according to an arbitrary formula.
-    // 0.0 means the value equals the threshold, positive means the value is
-    // below the threshold so RPS should increase, and negative means the value
-    // is above the threshold so RPS should decrease.
+    // Selection and configuration of a ScoringFunction that measures proximity
+    // to a threshold. 0.0 means the value equals the threshold, positive means
+    // the value is below the threshold so RPS should increase, and negative
+    // means the value is above the threshold so RPS should decrease.
     ScoringFunctionConfig scoring_function = 3;
   }
   // Relative importance of this threshold when calculating adaptive RPS based
-  // on multiple metrics. Must be a positive number if set. If any threshold has
-  // a weight, all must have a weight. If no weights are set, all thresholds
-  // have equal weight.
+  // on multiple metrics. If any threshold has a weight, all must have a weight.
+  // If no weights are set, all thresholds have equal weight.
   google.protobuf.DoubleValue weight = 4 [(validate.rules).double = {gt: 0.0}];
 }
 

From 16fd8f675a1cc6cb92d9980ccffc8a57a69d89ba Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Thu, 9 Jul 2020 23:27:49 -0400
Subject: [PATCH 04/34] rename adaptive_rps to adaptive_load

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 api/{adaptive_rps => adaptive_load}/BUILD     |  2 +-
 .../adaptive_load.proto}                      | 56 ++++++++++---------
 .../benchmark_result.proto                    | 10 ++--
 .../metric_spec.proto                         | 24 ++++----
 .../metrics_plugin.proto                      |  2 +-
 .../metrics_plugin_impl.proto                 |  2 +-
 .../scoring_function.proto                    |  2 +-
 .../scoring_function_impl.proto               |  6 +-
 api/adaptive_load/step_controller.proto       | 29 ++++++++++
 api/adaptive_load/step_controller_impl.proto  | 38 +++++++++++++
 api/adaptive_rps/step_controller.proto        | 13 -----
 api/adaptive_rps/step_controller_impl.proto   | 29 ----------
 .../example_session_spec.textproto            | 16 +++---
 13 files changed, 131 insertions(+), 98 deletions(-)
 rename api/{adaptive_rps => adaptive_load}/BUILD (94%)
 rename api/{adaptive_rps/adaptive_rps.proto => adaptive_load/adaptive_load.proto} (52%)
 rename api/{adaptive_rps => adaptive_load}/benchmark_result.proto (86%)
 rename api/{adaptive_rps => adaptive_load}/metric_spec.proto (63%)
 rename api/{adaptive_rps => adaptive_load}/metrics_plugin.proto (90%)
 rename api/{adaptive_rps => adaptive_load}/metrics_plugin_impl.proto (91%)
 rename api/{adaptive_rps => adaptive_load}/scoring_function.proto (92%)
 rename api/{adaptive_rps => adaptive_load}/scoring_function_impl.proto (86%)
 create mode 100644 api/adaptive_load/step_controller.proto
 create mode 100644 api/adaptive_load/step_controller_impl.proto
 delete mode 100644 api/adaptive_rps/step_controller.proto
 delete mode 100644 api/adaptive_rps/step_controller_impl.proto

diff --git a/api/adaptive_rps/BUILD b/api/adaptive_load/BUILD
similarity index 94%
rename from api/adaptive_rps/BUILD
rename to api/adaptive_load/BUILD
index b81508e95..e0670c226 100644
--- a/api/adaptive_rps/BUILD
+++ b/api/adaptive_load/BUILD
@@ -5,7 +5,7 @@ licenses(["notice"])  # Apache 2
 api_cc_py_proto_library(
     name = "base",
     srcs = [
-        "adaptive_rps.proto",
+        "adaptive_load.proto",
         "benchmark_result.proto",
         "metric_spec.proto",
         "metrics_plugin.proto",
diff --git a/api/adaptive_rps/adaptive_rps.proto b/api/adaptive_load/adaptive_load.proto
similarity index 52%
rename from api/adaptive_rps/adaptive_rps.proto
rename to api/adaptive_load/adaptive_load.proto
index bae9473cb..915ea4345 100644
--- a/api/adaptive_rps/adaptive_rps.proto
+++ b/api/adaptive_load/adaptive_load.proto
@@ -1,38 +1,41 @@
 syntax = "proto3";
 
-package nighthawk.adaptive_rps;
+package nighthawk.adaptive_load;
 
-import "api/adaptive_rps/benchmark_result.proto";
-import "api/adaptive_rps/metrics_plugin.proto";
-import "api/adaptive_rps/metric_spec.proto";
-import "api/adaptive_rps/step_controller.proto";
+import "api/adaptive_load/benchmark_result.proto";
+import "api/adaptive_load/metrics_plugin.proto";
+import "api/adaptive_load/metric_spec.proto";
+import "api/adaptive_load/step_controller.proto";
 import "api/client/options.proto";
 import "google/protobuf/duration.proto";
 import "google/rpc/status.proto";
 import "validate/validate.proto";
 
-// Parameters describing the adjusting and testing stages of an adaptive RPS
+// Parameters describing the adjusting and testing stages of an adaptive load
 // session, which consists of a series of Nighthawk benchmarks probing for
-// the optimal RPS, followed by a longer benchmark to validate the optimal RPS.
-// RPS calculations are implemented by the selected StepController plugin.
-// Metrics can come from Nighthawk stats and counters or platform-specific data
-// sources via MetricsPlugins.
-message AdaptiveRpsSessionSpec {
+// the optimal load on the system, followed by a longer benchmark to validate
+// the values. Load adjustments are calculated by the selected StepController
+// plugin. Metrics can come from Nighthawk stats and counters or
+// platform-specific data sources via MetricsPlugins.
+message AdaptiveLoadSessionSpec {
   // Settings for MetricsPlugins that obtain metrics from outside sources.
   // An entry is required for every plugin referred to by metric_thresholds,
   // other than the "builtin" plugin.
   repeated MetricsPluginConfig metrics_plugin_configs = 1;
-  // Metrics and thresholds that determine RPS adjustments.
-  repeated MetricSpecWithThreshold metric_thresholds = 2 [(validate.rules).repeated .min_items = 1];
+  // Metrics and thresholds that determine load adjustments.
+  repeated MetricSpecWithThreshold metric_thresholds = 2
+      [ (validate.rules).repeated .min_items = 1 ];
   // Metrics that are collected and included in the output but not taken into
-  // account when adjusting the RPS.
+  // account when adjusting the load.
   repeated MetricSpec informational_metric_specs = 3;
   // A proto describing Nighthawk Service traffic. See
   // https://github.com/envoyproxy/nighthawk/blob/master/api/client/options.proto
   //
-  // The adaptive RPS controller will return an error if the |duration|,
-  // |requests_per_second|,  or |open_loop| fields are set within
-  // |nighthawk_traffic_options|; the controller sets these values dynamically.
+  // The adaptive load controller will return an error if the |duration| or
+  // |open_loop| fields are set within |nighthawk_traffic_options|. The
+  // controller will also be configured to overwrite at least one of the
+  // numerical fields during the search, such as requests_per_second, so any
+  // value of those fields specified here will be ignored.
   //
   // All other fields in |nighthawk_traffic_options| are passed through to the
   // Nighthawk Service.
@@ -42,21 +45,24 @@ message AdaptiveRpsSessionSpec {
   nighthawk.client.CommandLineOptions nighthawk_traffic_template = 4;
   // The duration of each short benchmark during the adjusting stage. Must be
   // set. Reasonable value: 10 seconds.
-  google.protobuf.Duration measuring_period = 5 [(validate.rules).duration.gt.seconds = 0];
+  google.protobuf.Duration measuring_period = 5
+      [ (validate.rules).duration.gt.seconds = 0 ];
   // Maximum amount of time the adjusting stage should wait for convergence
   // before returning an error. Reasonable value: 300 seconds.
-  google.protobuf.Duration convergence_deadline = 6 [(validate.rules).duration.gt.seconds = 0];
+  google.protobuf.Duration convergence_deadline = 6
+      [ (validate.rules).duration.gt.seconds = 0 ];
   // The duration of the single benchmark session of the testing stage to
-  // confirm the performance at the RPS found by the adjusting stage. Must be
-  // set.
-  google.protobuf.Duration testing_stage_duration = 7 [(validate.rules).duration.gt.seconds = 0];
+  // confirm the performance at the level of load found in the adjusting stage.
+  // Must be set.
+  google.protobuf.Duration testing_stage_duration = 7
+      [ (validate.rules).duration.gt.seconds = 0 ];
   // Selects and configures a StepController plugin.
   StepControllerConfig step_controller_config = 8;
 }
 
-// Complete description of an adaptive RPS session, including metric scores
-// for every RPS value attempted during the adjusting stage.
-message AdaptiveRpsSessionOutput {
+// Complete description of an adaptive load session, including metric scores
+// for every degree of load attempted during the adjusting stage.
+message AdaptiveLoadSessionOutput {
   // Overall status of the session with error detail.
   google.rpc.Status session_status = 1;
   // Results of each short benchmark performed during the adjusting stage.
diff --git a/api/adaptive_rps/benchmark_result.proto b/api/adaptive_load/benchmark_result.proto
similarity index 86%
rename from api/adaptive_rps/benchmark_result.proto
rename to api/adaptive_load/benchmark_result.proto
index 7c09fd35c..73811e1fc 100644
--- a/api/adaptive_rps/benchmark_result.proto
+++ b/api/adaptive_load/benchmark_result.proto
@@ -1,8 +1,8 @@
 syntax = "proto3";
 
-package nighthawk.adaptive_rps;
+package nighthawk.adaptive_load;
 
-import "api/adaptive_rps/metric_spec.proto";
+import "api/adaptive_load/metric_spec.proto";
 import "api/client/output.proto";
 
 // Binary result of checking a metric value against a threshold.
@@ -23,7 +23,7 @@ message ThresholdCheckResult {
     // Score returned by a ScoringFunction plugin. This expresses how close the
     // metric was to the threshold by an arbitrary formula selected and
     // configured in the ThresholdSpec, such as a sigmoid curve. The controller
-    // can choose to make larger RPS adjustments when the score is larger, in
+    // can choose to make larger input adjustments when the score is larger, in
     // order to converge faster.
     double threshold_score = 2;
   }
@@ -49,7 +49,7 @@ message BenchmarkResult {
   // Raw Nighthawk Service output. Includes start/end times and full Nighthawk
   // Service input spec. May contain an error status.
   nighthawk.client.Output nighthawk_service_output = 1;
-  // Status of all defined adaptive RPS feedback metrics during this benchmark
-  // session. Not present in the event of Nighthawk errors.
+  // Status of all declared metrics during this benchmark session. Not present
+  // in the event of Nighthawk errors.
   repeated MetricEvaluation metric_evaluations = 3;
 }
diff --git a/api/adaptive_rps/metric_spec.proto b/api/adaptive_load/metric_spec.proto
similarity index 63%
rename from api/adaptive_rps/metric_spec.proto
rename to api/adaptive_load/metric_spec.proto
index 60fd7c7c6..b4dcbd183 100644
--- a/api/adaptive_rps/metric_spec.proto
+++ b/api/adaptive_load/metric_spec.proto
@@ -1,18 +1,18 @@
 syntax = "proto3";
 
-package nighthawk.adaptive_rps;
+package nighthawk.adaptive_load;
 
-import "api/adaptive_rps/scoring_function.proto";
+import "api/adaptive_load/scoring_function.proto";
 import "google/protobuf/wrappers.proto";
 import "validate/validate.proto";
 
 // Identifies a feedback metric.
 message MetricSpec {
   // Name of the metric to evaluate. For the set of built-in metric names, see
-  // source/adaptive_rps/metrics_plugin_impl.cc.
-  string metric_name = 1 [(validate.rules).string.min_len = 1];
+  // source/adaptive_load/metrics_plugin_impl.cc.
+  string metric_name = 1 [ (validate.rules).string.min_len = 1 ];
   // Name of the MetricsPlugin providing the metric ("builtin" for built-in).
-  string metrics_plugin_name = 2 [(validate.rules).string.min_len = 1];
+  string metrics_plugin_name = 2 [ (validate.rules).string.min_len = 1 ];
 }
 
 // Specifies a simple upper or lower threshold, or configures a scoring
@@ -30,14 +30,16 @@ message ThresholdSpec {
     google.protobuf.DoubleValue lower_threshold = 2;
     // Selection and configuration of a ScoringFunction that measures proximity
     // to a threshold. 0.0 means the value equals the threshold, positive means
-    // the value is below the threshold so RPS should increase, and negative
-    // means the value is above the threshold so RPS should decrease.
+    // the value is below the threshold so the input should ramp up, and
+    // negative means the value is above the threshold so input should ramp
+    // down.
     ScoringFunctionConfig scoring_function = 3;
   }
-  // Relative importance of this threshold when calculating adaptive RPS based
-  // on multiple metrics. If any threshold has a weight, all must have a weight.
-  // If no weights are set, all thresholds have equal weight.
-  google.protobuf.DoubleValue weight = 4 [(validate.rules).double = {gt: 0.0}];
+  // Relative importance of this threshold when adjusting based on multiple
+  // metrics. If any threshold has a weight, all must have a weight. If no
+  // weights are set, all thresholds have equal weight.
+  google.protobuf.DoubleValue weight = 4
+      [ (validate.rules).double = {gt : 0.0} ];
 }
 
 message MetricSpecWithThreshold {
diff --git a/api/adaptive_rps/metrics_plugin.proto b/api/adaptive_load/metrics_plugin.proto
similarity index 90%
rename from api/adaptive_rps/metrics_plugin.proto
rename to api/adaptive_load/metrics_plugin.proto
index 2c385bb41..340e99018 100644
--- a/api/adaptive_rps/metrics_plugin.proto
+++ b/api/adaptive_load/metrics_plugin.proto
@@ -1,6 +1,6 @@
 syntax = "proto3";
 
-package nighthawk.adaptive_rps;
+package nighthawk.adaptive_load;
 
 import "google/protobuf/any.proto";
 
diff --git a/api/adaptive_rps/metrics_plugin_impl.proto b/api/adaptive_load/metrics_plugin_impl.proto
similarity index 91%
rename from api/adaptive_rps/metrics_plugin_impl.proto
rename to api/adaptive_load/metrics_plugin_impl.proto
index 198ae938f..f1699be4b 100644
--- a/api/adaptive_rps/metrics_plugin_impl.proto
+++ b/api/adaptive_load/metrics_plugin_impl.proto
@@ -2,7 +2,7 @@
 
 syntax = "proto3";
 
-package nighthawk.adaptive_rps;
+package nighthawk.adaptive_load;
 
 import "validate/validate.proto";
 
diff --git a/api/adaptive_rps/scoring_function.proto b/api/adaptive_load/scoring_function.proto
similarity index 92%
rename from api/adaptive_rps/scoring_function.proto
rename to api/adaptive_load/scoring_function.proto
index 88f85ac89..fbf9b9ad9 100644
--- a/api/adaptive_rps/scoring_function.proto
+++ b/api/adaptive_load/scoring_function.proto
@@ -1,6 +1,6 @@
 syntax = "proto3";
 
-package nighthawk.adaptive_rps;
+package nighthawk.adaptive_load;
 
 import "google/protobuf/any.proto";
 
diff --git a/api/adaptive_rps/scoring_function_impl.proto b/api/adaptive_load/scoring_function_impl.proto
similarity index 86%
rename from api/adaptive_rps/scoring_function_impl.proto
rename to api/adaptive_load/scoring_function_impl.proto
index dd5901699..67726b4ae 100644
--- a/api/adaptive_rps/scoring_function_impl.proto
+++ b/api/adaptive_load/scoring_function_impl.proto
@@ -2,18 +2,18 @@
 
 syntax = "proto3";
 
-package nighthawk.adaptive_rps;
+package nighthawk.adaptive_load;
 
 // Configuration for a linear ScoringFunction that calculates a metric score
 // as k * (threshold - value), where k is a scaling constant. The score is 0.0
 // when the value exactly equals the threshold, positive below the threshold
-// (meaning RPS should increase), and negative above the threshold. The score is
+// (meaning input should ramp up), and negative above the threshold. The score is
 // proportional to the difference from the threshold.
 message LinearScoringFunctionConfig {
   // The target value of the metric.
   double threshold = 1;
   // Scaling constant: k in k * (threshold - value). Use this in combination
-  // with step controller constants to produce reasonable RPS increments for
+  // with step controller constants to produce reasonable input increments for
   // reasonable differences from the threshold.
   double k = 2;
 }
diff --git a/api/adaptive_load/step_controller.proto b/api/adaptive_load/step_controller.proto
new file mode 100644
index 000000000..d03cc19e5
--- /dev/null
+++ b/api/adaptive_load/step_controller.proto
@@ -0,0 +1,29 @@
+syntax = "proto3";
+
+package nighthawk.adaptive_load;
+
+import "google/protobuf/any.proto";
+
+// Identifies a Nighthawk CommandLineOptions proto field that the StepController
+// should vary, in pursuit of the optimal field value. For example, for the
+// basic case of finding the optimal RPS, the NighthawkFieldIdentifier would be
+// REQUESTS_PER_SECOND. Enables the same StepController algorithm to be used to
+// find the optimal RPS, optimal concurrency, etc.
+enum NighthawkFieldSelector {
+  UNKNOWN_FIELD = 0;
+  REQUESTS_PER_SECOND = 1;
+  NUMBER_OF_CONNECTIONS = 2;
+  CONCURRENCY = 3;
+  BURST_SIZE = 4;
+  MAX_PENDING_REQUESTS = 5;
+  MAX_ACTIVE_REQUESTS = 6;
+  MAX_REQUESTS_PER_CONNECTION = 7;
+}
+
+// Selects and configures a StepController plugin.
+message StepControllerConfig {
+  // Name used to locate the plugin in the Envoy registry.
+  string name = 1;
+  // Plugin-specific config.
+  google.protobuf.Any typed_config = 2;
+}
diff --git a/api/adaptive_load/step_controller_impl.proto b/api/adaptive_load/step_controller_impl.proto
new file mode 100644
index 000000000..d0e66e92d
--- /dev/null
+++ b/api/adaptive_load/step_controller_impl.proto
@@ -0,0 +1,38 @@
+// Plugin-specific config protos for StepControllers.
+
+syntax = "proto3";
+
+package nighthawk.adaptive_load;
+
+import "api/adaptive_load/step_controller.proto";
+
+// Configuration for a step controller that performs a linear search for the
+// optimal value of a single Nighthawk input field (e.g. RPS), starting from the
+// minimum value and incrementing by a fixed amount until reaching the point
+// where measured performance declines.
+message LinearSearchStepControllerConfig {
+  // Selects the single Nighthawk Service CommandLineOptions field that this
+  // StepController should vary, in pursuit of the optimal field value.
+  // For example, to find the optimal RPS, set to REQUESTS_PER_SECOND.
+  NighthawkFieldSelector field_selector = 1;
+  // Minimum field value that should be attempted, regardless of performance.
+  // Also used as the initial field value.
+  int64 minimum_field_value = 2;
+  // Maximum field value that should be attempted, regardless of performance.
+  int64 maximum_field_value = 3;
+  // Constant step to be used for all field value adjustments.
+  int64 field_value_step = 4;
+}
+
+// Configuration for a step controller that performs a binary search for the
+// optimal value of a single Nighthawk input field, such as RPS.
+message BinarySearchStepControllerConfig {
+  // Selects the single Nighthawk Service CommandLineOptions field that this
+  // StepController should vary, in pursuit of the optimal field value.
+  // For example, to find the optimal RPS, set to REQUESTS_PER_SECOND.
+  NighthawkFieldSelector field_selector = 1;
+  // Minimum of the field value binary search range.
+  int64 minimum_field_value = 2;
+  // Maximum of the field value binary search range.
+  int64 maximum_field_value = 3;
+}
diff --git a/api/adaptive_rps/step_controller.proto b/api/adaptive_rps/step_controller.proto
deleted file mode 100644
index d1acf2dcc..000000000
--- a/api/adaptive_rps/step_controller.proto
+++ /dev/null
@@ -1,13 +0,0 @@
-syntax = "proto3";
-
-package nighthawk.adaptive_rps;
-
-import "google/protobuf/any.proto";
-
-// Selects and configures a StepController plugin.
-message StepControllerConfig {
-  // Name used to locate the plugin in the Envoy registry.
-  string name = 1;
-  // Plugin-specific config.
-  google.protobuf.Any typed_config = 2;
-}
diff --git a/api/adaptive_rps/step_controller_impl.proto b/api/adaptive_rps/step_controller_impl.proto
deleted file mode 100644
index e1640225c..000000000
--- a/api/adaptive_rps/step_controller_impl.proto
+++ /dev/null
@@ -1,29 +0,0 @@
-// Plugin-specific config protos for StepControllers.
-
-syntax = "proto3";
-
-package nighthawk.adaptive_rps;
-
-import "validate/validate.proto";
-
-// Configuration for a step controller that performs a linear search for the
-// optimal RPS, starting from the minimum RPS and incrementing by a fixed amount
-// until reaching the point where performance declines.
-message LinearSearchStepControllerConfig {
-  // Minimum RPS that should be attempted, regardless of performance. Also used
-  // as the initial RPS.
-  uint32 minimum_rps = 1 [(validate.rules).uint32 = {gt: 0}];
-  // Maximum RPS that should be attempted, regardless of performance.
-  uint32 maximum_rps = 2 [(validate.rules).uint32 = {gt: 0}];
-  // Constant step to be used for all RPS adjustments.
-  uint32 rps_step = 3 [(validate.rules).uint32 = {gt: 0}];
-}
-
-// Configuration for a step controller that performs a binary search for the
-// optimal RPS.
-message BinarySearchStepControllerConfig {
-  // Minimum of the RPS binary search range.
-  uint32 minimum_rps = 1 [(validate.rules).uint32 = {gt: 0}];
-  // Maximum of the RPS binary search range.
-  uint32 maximum_rps = 2 [(validate.rules).uint32 = {gt: 0}];
-}
diff --git a/samples/adaptive_rps/example_session_spec.textproto b/samples/adaptive_rps/example_session_spec.textproto
index 796ac5e8f..e4bbcea26 100644
--- a/samples/adaptive_rps/example_session_spec.textproto
+++ b/samples/adaptive_rps/example_session_spec.textproto
@@ -1,10 +1,10 @@
-# proto-file: adaptive_rps/adaptive_rps.proto
-# proto-message: AdaptiveRpsSessionSpec
+# proto-file: adaptive_load/adaptive_load.proto
+# proto-message: AdaptiveLoadSessionSpec
 
 metrics_plugin_configs {
   name: "example-metrics-plugin"
   typed_config {
-    [type.googleapis.com/nighthawk.adaptive_rps.ExampleMetricsPluginConfig] {
+    [type.googleapis.com/nighthawk.adaptive_load.ExampleMetricsPluginConfig] {
       address: "x.y.z"
       credentials: "key-id-123"
     }
@@ -12,7 +12,7 @@ metrics_plugin_configs {
 }
 
 # For the full list of built-in metric names, see
-# source/adaptive_rps/metrics_plugin_impl.cc.
+# source/adaptive_load/metrics_plugin_impl.cc.
 
 metric_thresholds {
   metric_spec {
@@ -23,7 +23,7 @@ metric_thresholds {
     scoring_function {
       name: "sigmoid"
       typed_config {
-        [type.googleapis.com/nighthawk.adaptive_rps.SigmoidScoringFunctionConfig] {
+        [type.googleapis.com/nighthawk.adaptive_load.SigmoidScoringFunctionConfig] {
           threshold: 50000000.0  # 50 ms
           k: 0.00000001  # Should be around 1/threshold
         }
@@ -51,7 +51,7 @@ metric_thresholds {
 }
 
 # Informational metrics are recorded in the output but are not taken into account
-# when adjusting the RPS.
+# when adjusting the load.
 
 informational_metric_specs {
   metric_name: "attempted-rps"
@@ -97,7 +97,7 @@ testing_stage_duration {
 step_controller_config {
   name: "linear-search"
   typed_config {
-    [type.googleapis.com/nighthawk.adaptive_rps.LinearSearchStepControllerConfig] {
+    [type.googleapis.com/nighthawk.adaptive_load.LinearSearchStepControllerConfig] {
       rps_step: 100
       minimum_rps: 500
       maximum_rps: 5000
@@ -108,7 +108,7 @@ step_controller_config {
 # step_controller_config {
 #   name : "binary-search"
 #   typed_config {
-#     [type.googleapis.com/nighthawk.adaptive_rps.BinarySearchStepControllerConfig] {
+#     [type.googleapis.com/nighthawk.adaptive_load.BinarySearchStepControllerConfig] {
 #       minimum_rps : 500
 #       maximum_rps : 10000
 #     }

From c383010f19ec68371aaa876705e49027f52b0426 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Thu, 9 Jul 2020 23:31:30 -0400
Subject: [PATCH 05/34] add field_selector in example

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 samples/adaptive_rps/example_session_spec.textproto | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/samples/adaptive_rps/example_session_spec.textproto b/samples/adaptive_rps/example_session_spec.textproto
index e4bbcea26..a86fdd808 100644
--- a/samples/adaptive_rps/example_session_spec.textproto
+++ b/samples/adaptive_rps/example_session_spec.textproto
@@ -98,9 +98,10 @@ step_controller_config {
   name: "linear-search"
   typed_config {
     [type.googleapis.com/nighthawk.adaptive_load.LinearSearchStepControllerConfig] {
-      rps_step: 100
-      minimum_rps: 500
-      maximum_rps: 5000
+      field_selector: REQUESTS_PER_SECOND
+      minimum_field_value: 500
+      maximum_field_value: 5000
+      field_value_step: 100
     }
   }
 }
@@ -109,8 +110,9 @@ step_controller_config {
 #   name : "binary-search"
 #   typed_config {
 #     [type.googleapis.com/nighthawk.adaptive_load.BinarySearchStepControllerConfig] {
-#       minimum_rps : 500
-#       maximum_rps : 10000
+#       field_selector: REQUESTS_PER_SECOND
+#       minimum_field_value : 500
+#       maximum_field_value : 10000
 #     }
 #   }
 # }

From 6e1a4832c163aeff92985dc8804ba34397c5e267 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Thu, 9 Jul 2020 23:33:26 -0400
Subject: [PATCH 06/34] fix example comment

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 samples/adaptive_rps/example_session_spec.textproto | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/samples/adaptive_rps/example_session_spec.textproto b/samples/adaptive_rps/example_session_spec.textproto
index a86fdd808..fc3d4faa5 100644
--- a/samples/adaptive_rps/example_session_spec.textproto
+++ b/samples/adaptive_rps/example_session_spec.textproto
@@ -75,7 +75,8 @@ informational_metric_specs {
 }
 
 # CommandLineOptions proto describing the traffic to generate, but without
-# requests_per_second, duration, or open_loop, which will all be set dynamically.
+# duration, open_loop, or the field to be varied (e.g. requests_per_second),
+# which will be managed by the adaptive load controller.
 nighthawk_traffic_template {
   uri {
     value: "[::1]:12345"

From 4ef1140b76b0d11a05ab99b4c29c4413a55dd7fe Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Thu, 9 Jul 2020 23:47:44 -0400
Subject: [PATCH 07/34] fix format

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 api/adaptive_load/adaptive_load.proto | 12 ++++--------
 api/adaptive_load/metric_spec.proto   |  7 +++----
 2 files changed, 7 insertions(+), 12 deletions(-)

diff --git a/api/adaptive_load/adaptive_load.proto b/api/adaptive_load/adaptive_load.proto
index 915ea4345..2c81bf6e9 100644
--- a/api/adaptive_load/adaptive_load.proto
+++ b/api/adaptive_load/adaptive_load.proto
@@ -23,8 +23,7 @@ message AdaptiveLoadSessionSpec {
   // other than the "builtin" plugin.
   repeated MetricsPluginConfig metrics_plugin_configs = 1;
   // Metrics and thresholds that determine load adjustments.
-  repeated MetricSpecWithThreshold metric_thresholds = 2
-      [ (validate.rules).repeated .min_items = 1 ];
+  repeated MetricSpecWithThreshold metric_thresholds = 2 [(validate.rules).repeated .min_items = 1];
   // Metrics that are collected and included in the output but not taken into
   // account when adjusting the load.
   repeated MetricSpec informational_metric_specs = 3;
@@ -45,17 +44,14 @@ message AdaptiveLoadSessionSpec {
   nighthawk.client.CommandLineOptions nighthawk_traffic_template = 4;
   // The duration of each short benchmark during the adjusting stage. Must be
   // set. Reasonable value: 10 seconds.
-  google.protobuf.Duration measuring_period = 5
-      [ (validate.rules).duration.gt.seconds = 0 ];
+  google.protobuf.Duration measuring_period = 5 [(validate.rules).duration.gt.seconds = 0];
   // Maximum amount of time the adjusting stage should wait for convergence
   // before returning an error. Reasonable value: 300 seconds.
-  google.protobuf.Duration convergence_deadline = 6
-      [ (validate.rules).duration.gt.seconds = 0 ];
+  google.protobuf.Duration convergence_deadline = 6 [(validate.rules).duration.gt.seconds = 0];
   // The duration of the single benchmark session of the testing stage to
   // confirm the performance at the level of load found in the adjusting stage.
   // Must be set.
-  google.protobuf.Duration testing_stage_duration = 7
-      [ (validate.rules).duration.gt.seconds = 0 ];
+  google.protobuf.Duration testing_stage_duration = 7 [(validate.rules).duration.gt.seconds = 0];
   // Selects and configures a StepController plugin.
   StepControllerConfig step_controller_config = 8;
 }
diff --git a/api/adaptive_load/metric_spec.proto b/api/adaptive_load/metric_spec.proto
index b4dcbd183..1c9323338 100644
--- a/api/adaptive_load/metric_spec.proto
+++ b/api/adaptive_load/metric_spec.proto
@@ -10,9 +10,9 @@ import "validate/validate.proto";
 message MetricSpec {
   // Name of the metric to evaluate. For the set of built-in metric names, see
   // source/adaptive_load/metrics_plugin_impl.cc.
-  string metric_name = 1 [ (validate.rules).string.min_len = 1 ];
+  string metric_name = 1 [(validate.rules).string.min_len = 1];
   // Name of the MetricsPlugin providing the metric ("builtin" for built-in).
-  string metrics_plugin_name = 2 [ (validate.rules).string.min_len = 1 ];
+  string metrics_plugin_name = 2 [(validate.rules).string.min_len = 1];
 }
 
 // Specifies a simple upper or lower threshold, or configures a scoring
@@ -38,8 +38,7 @@ message ThresholdSpec {
   // Relative importance of this threshold when adjusting based on multiple
   // metrics. If any threshold has a weight, all must have a weight. If no
   // weights are set, all thresholds have equal weight.
-  google.protobuf.DoubleValue weight = 4
-      [ (validate.rules).double = {gt : 0.0} ];
+  google.protobuf.DoubleValue weight = 4 [(validate.rules).double = {gt: 0.0}];
 }
 
 message MetricSpecWithThreshold {

From 4111bf40ce6468dc11592b5a7d82924623fcfd55 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Fri, 10 Jul 2020 12:09:26 -0400
Subject: [PATCH 08/34] add support for fault injection headers

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 api/adaptive_load/step_controller.proto      | 42 +++++++++++++++++---
 api/adaptive_load/step_controller_impl.proto | 26 ++++++------
 2 files changed, 50 insertions(+), 18 deletions(-)

diff --git a/api/adaptive_load/step_controller.proto b/api/adaptive_load/step_controller.proto
index d03cc19e5..a6f7179c1 100644
--- a/api/adaptive_load/step_controller.proto
+++ b/api/adaptive_load/step_controller.proto
@@ -4,12 +4,9 @@ package nighthawk.adaptive_load;
 
 import "google/protobuf/any.proto";
 
-// Identifies a Nighthawk CommandLineOptions proto field that the StepController
-// should vary, in pursuit of the optimal field value. For example, for the
-// basic case of finding the optimal RPS, the NighthawkFieldIdentifier would be
-// REQUESTS_PER_SECOND. Enables the same StepController algorithm to be used to
-// find the optimal RPS, optimal concurrency, etc.
-enum NighthawkFieldSelector {
+// Identifies a simple field or other hard-coded aspect of the Nighthawk
+// CommandLineOptions.
+enum SimpleNighthawkField {
   UNKNOWN_FIELD = 0;
   REQUESTS_PER_SECOND = 1;
   NUMBER_OF_CONNECTIONS = 2;
@@ -18,6 +15,39 @@ enum NighthawkFieldSelector {
   MAX_PENDING_REQUESTS = 5;
   MAX_ACTIVE_REQUESTS = 6;
   MAX_REQUESTS_PER_CONNECTION = 7;
+  // Vary the length of the URI using the form "/aaaaa".
+  URI_LENGTH = 8;
+  // Vary the length of a single header x-nighthawk-adaptive-load with the value
+  // in the form "aaaaa".
+  HEADER_SIZE = 9;
+  // Add a variable number of small headers named x-nighthawk-adaptive-load-# to
+  // the request.
+  HEADER_COUNT = 10;
+  // Vary the size of the request body consisting of "aaaaa" (note: subject to
+  // Nighthawk limits).
+  REQUEST_BODY_SIZE = 12;
+  // Response size requested from a Nighthawk Test Server via a header.
+  RESPONSE_SIZE = 12;
+}
+
+// Specifies any Envoy fault injection filter header that takes a numerical
+// value. See
+// https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/fault_filter.
+message FaultInjectionHeader {
+  // Name of any numeric Envoy fault injection header.
+  string header_name = 1;
+}
+
+// Identifies a some numerical aspect of the Nighthawk CommandLineOptions proto
+// that the StepController should vary, in pursuit of the optimal field value.
+// For example, for the basic case of finding the optimal RPS, the simple_field
+// would be REQUESTS_PER_SECOND. Enables the same StepController algorithm to be
+// used to find the optimal RPS, optimal concurrency, etc.
+message NighthawkInputVariableSpec {
+  oneof selector {
+    SimpleNighthawkField simple_field = 1;
+    FaultInjectionHeader fault_injection_header = 2;
+  }
 }
 
 // Selects and configures a StepController plugin.
diff --git a/api/adaptive_load/step_controller_impl.proto b/api/adaptive_load/step_controller_impl.proto
index d0e66e92d..f6f829ecc 100644
--- a/api/adaptive_load/step_controller_impl.proto
+++ b/api/adaptive_load/step_controller_impl.proto
@@ -7,19 +7,20 @@ package nighthawk.adaptive_load;
 import "api/adaptive_load/step_controller.proto";
 
 // Configuration for a step controller that performs a linear search for the
-// optimal value of a single Nighthawk input field (e.g. RPS), starting from the
-// minimum value and incrementing by a fixed amount until reaching the point
+// optimal value of a single Nighthawk input variable (e.g. RPS), starting from
+// the minimum value and incrementing by a fixed amount until reaching the point
 // where measured performance declines.
 message LinearSearchStepControllerConfig {
-  // Selects the single Nighthawk Service CommandLineOptions field that this
-  // StepController should vary, in pursuit of the optimal field value.
-  // For example, to find the optimal RPS, set to REQUESTS_PER_SECOND.
-  NighthawkFieldSelector field_selector = 1;
+  // Selects the single variable within Nighthawk Service CommandLineOptions
+  // that this StepController should vary, in pursuit of the optimal field
+  // value. For example, to find the optimal RPS, set variable_spec.simple_field
+  // to REQUESTS_PER_SECOND.
+  NighthawkInputVariableSpec variable_spec = 1;
   // Minimum field value that should be attempted, regardless of performance.
   // Also used as the initial field value.
-  int64 minimum_field_value = 2;
+  int64 minimum = 2;
   // Maximum field value that should be attempted, regardless of performance.
-  int64 maximum_field_value = 3;
+  int64 maximum = 3;
   // Constant step to be used for all field value adjustments.
   int64 field_value_step = 4;
 }
@@ -29,10 +30,11 @@ message LinearSearchStepControllerConfig {
 message BinarySearchStepControllerConfig {
   // Selects the single Nighthawk Service CommandLineOptions field that this
   // StepController should vary, in pursuit of the optimal field value.
-  // For example, to find the optimal RPS, set to REQUESTS_PER_SECOND.
-  NighthawkFieldSelector field_selector = 1;
+  // For example, to find the optimal RPS, set variable_spec.simple_field
+  // to REQUESTS_PER_SECOND.
+  NighthawkInputVariableSpec variable_spec = 1;
   // Minimum of the field value binary search range.
-  int64 minimum_field_value = 2;
+  int64 minimum = 2;
   // Maximum of the field value binary search range.
-  int64 maximum_field_value = 3;
+  int64 maximum = 3;
 }

From 871a959e28d01a1d50c8de42ecfeea5dd0487e17 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Fri, 10 Jul 2020 15:58:10 -0400
Subject: [PATCH 09/34] replace linear and binary search with exponential
 search

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 api/adaptive_load/step_controller_impl.proto | 38 +++++++-------------
 1 file changed, 12 insertions(+), 26 deletions(-)

diff --git a/api/adaptive_load/step_controller_impl.proto b/api/adaptive_load/step_controller_impl.proto
index f6f829ecc..8cd624e59 100644
--- a/api/adaptive_load/step_controller_impl.proto
+++ b/api/adaptive_load/step_controller_impl.proto
@@ -6,35 +6,21 @@ package nighthawk.adaptive_load;
 
 import "api/adaptive_load/step_controller.proto";
 
-// Configuration for a step controller that performs a linear search for the
-// optimal value of a single Nighthawk input variable (e.g. RPS), starting from
-// the minimum value and incrementing by a fixed amount until reaching the point
-// where measured performance declines.
-message LinearSearchStepControllerConfig {
+// Configuration for a step controller that performs an exponential search for
+// the optimal value of a single Nighthawk input variable (e.g. RPS).
+// Exponential search starts with the input set to |initial_value| and increases
+// the input by |exponential_factor| until a metric fails the threshold, then
+// performs a binary search with input values between X/2 and X, where X is the
+// input value where the metric failed the threshold.
+message ExponentialSearchStepControllerConfig {
   // Selects the single variable within Nighthawk Service CommandLineOptions
   // that this StepController should vary, in pursuit of the optimal field
   // value. For example, to find the optimal RPS, set variable_spec.simple_field
   // to REQUESTS_PER_SECOND.
   NighthawkInputVariableSpec variable_spec = 1;
-  // Minimum field value that should be attempted, regardless of performance.
-  // Also used as the initial field value.
-  int64 minimum = 2;
-  // Maximum field value that should be attempted, regardless of performance.
-  int64 maximum = 3;
-  // Constant step to be used for all field value adjustments.
-  int64 field_value_step = 4;
-}
-
-// Configuration for a step controller that performs a binary search for the
-// optimal value of a single Nighthawk input field, such as RPS.
-message BinarySearchStepControllerConfig {
-  // Selects the single Nighthawk Service CommandLineOptions field that this
-  // StepController should vary, in pursuit of the optimal field value.
-  // For example, to find the optimal RPS, set variable_spec.simple_field
-  // to REQUESTS_PER_SECOND.
-  NighthawkInputVariableSpec variable_spec = 1;
-  // Minimum of the field value binary search range.
-  int64 minimum = 2;
-  // Maximum of the field value binary search range.
-  int64 maximum = 3;
+  // Initial value of the input variable that should be attempted.
+  int64 initial_value = 2;
+  // Factor to increase the input variable during the exponential phase. Typical
+  // value: 2.0.
+  double exponential_factor = 3;
 }

From 1fd77c194759d96176f81018fda41779e0c10cd5 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Fri, 10 Jul 2020 20:27:36 -0400
Subject: [PATCH 10/34] add InputVariableSetter mechanism

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 api/adaptive_load/input_variable_setter.proto | 18 ++++++++++++++
 .../input_variable_setter_impl.proto          | 18 ++++++++++++++
 api/adaptive_load/step_controller_impl.proto  |  9 ++++---
 .../example_session_spec.textproto            | 24 ++++++-------------
 4 files changed, 47 insertions(+), 22 deletions(-)
 create mode 100644 api/adaptive_load/input_variable_setter.proto
 create mode 100644 api/adaptive_load/input_variable_setter_impl.proto

diff --git a/api/adaptive_load/input_variable_setter.proto b/api/adaptive_load/input_variable_setter.proto
new file mode 100644
index 000000000..69b8e528d
--- /dev/null
+++ b/api/adaptive_load/input_variable_setter.proto
@@ -0,0 +1,18 @@
+syntax = "proto3";
+
+package nighthawk.adaptive_load;
+
+import "google/protobuf/any.proto";
+
+// Selects and configures an InputVariableSetter plugin, which encapsulates
+// knowledge of how to apply a varying numeric input value to a
+// CommandLineOptions proto. For example, RequestsPerSecondInputVariableSetter
+// aka "rps" writes the value to the |requests_per_second| field in
+// CommandLineOptions. Some InputVariableSetter plugins such as "rps" will not
+// define a config proto, in which case typed_config should be unset.
+message InputVariableSetterConfig {
+  // Name used to locate the plugin in the Envoy registry.
+  string name = 1;
+  // Plugin-specific config.
+  google.protobuf.Any typed_config = 2;
+}
diff --git a/api/adaptive_load/input_variable_setter_impl.proto b/api/adaptive_load/input_variable_setter_impl.proto
new file mode 100644
index 000000000..54a790aa9
--- /dev/null
+++ b/api/adaptive_load/input_variable_setter_impl.proto
@@ -0,0 +1,18 @@
+// Plugin-specific config protos for InputVariableSetter plugins.
+
+syntax = "proto3";
+
+package nighthawk.adaptive_load;
+
+// Note: RequestsPerSecondInputVariableSetter aka "rps" does not define a
+// custom config proto.
+
+// Configuration for an InputVariableSetter plugin that sets an HTTP header
+// within CommandLineOptions to a numeric value being varied by a
+// StepController.
+message NumericHttpHeaderInputVariableSetterConfig {
+  // The name of an HTTP header that can be set to a numeric value, such as one
+  // of the headers supported by the Envoy fault injection filter on Nighthawk
+  // Test Server.
+  string header_name = 1;
+}
diff --git a/api/adaptive_load/step_controller_impl.proto b/api/adaptive_load/step_controller_impl.proto
index 8cd624e59..47d57a895 100644
--- a/api/adaptive_load/step_controller_impl.proto
+++ b/api/adaptive_load/step_controller_impl.proto
@@ -13,11 +13,10 @@ import "api/adaptive_load/step_controller.proto";
 // performs a binary search with input values between X/2 and X, where X is the
 // input value where the metric failed the threshold.
 message ExponentialSearchStepControllerConfig {
-  // Selects the single variable within Nighthawk Service CommandLineOptions
-  // that this StepController should vary, in pursuit of the optimal field
-  // value. For example, to find the optimal RPS, set variable_spec.simple_field
-  // to REQUESTS_PER_SECOND.
-  NighthawkInputVariableSpec variable_spec = 1;
+  // Selects a plugin that knows how to apply a numeric value generated by the
+  // StepController within CommandLineOptions. For RPS, use name: "rps", leaving
+  // typed_config unset.
+  InputVariableSetterConfig input_variable_setter = 1;
   // Initial value of the input variable that should be attempted.
   int64 initial_value = 2;
   // Factor to increase the input variable during the exponential phase. Typical
diff --git a/samples/adaptive_rps/example_session_spec.textproto b/samples/adaptive_rps/example_session_spec.textproto
index fc3d4faa5..aecba5cb1 100644
--- a/samples/adaptive_rps/example_session_spec.textproto
+++ b/samples/adaptive_rps/example_session_spec.textproto
@@ -96,24 +96,14 @@ testing_stage_duration {
 }
 
 step_controller_config {
-  name: "linear-search"
+  name: "exponential-search"
   typed_config {
-    [type.googleapis.com/nighthawk.adaptive_load.LinearSearchStepControllerConfig] {
-      field_selector: REQUESTS_PER_SECOND
-      minimum_field_value: 500
-      maximum_field_value: 5000
-      field_value_step: 100
+    [type.googleapis.com/nighthawk.adaptive_load.ExponentialSearchStepControllerConfig] {
+      input_variable_setter {
+        name: "rps"
+      }
+      initial_value: 500
+      exponential_factor: 2.0
     }
   }
 }
-
-# step_controller_config {
-#   name : "binary-search"
-#   typed_config {
-#     [type.googleapis.com/nighthawk.adaptive_load.BinarySearchStepControllerConfig] {
-#       field_selector: REQUESTS_PER_SECOND
-#       minimum_field_value : 500
-#       maximum_field_value : 10000
-#     }
-#   }
-# }

From edc36b248074b93b273b547128fbc88bb8147a19 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Fri, 10 Jul 2020 20:29:22 -0400
Subject: [PATCH 11/34] add input variable setter to build file

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 api/adaptive_load/BUILD | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/api/adaptive_load/BUILD b/api/adaptive_load/BUILD
index e0670c226..5bdb0cac6 100644
--- a/api/adaptive_load/BUILD
+++ b/api/adaptive_load/BUILD
@@ -7,6 +7,8 @@ api_cc_py_proto_library(
     srcs = [
         "adaptive_load.proto",
         "benchmark_result.proto",
+        "input_variable_setter.proto",
+        "input_variable_setter_impl.proto",
         "metric_spec.proto",
         "metrics_plugin.proto",
         "metrics_plugin_impl.proto",

From 4d0364e4e3cc1aa355232941dd0a7d1f6c6a4439 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Fri, 10 Jul 2020 20:33:15 -0400
Subject: [PATCH 12/34] fix syntax errors

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 api/adaptive_load/metrics_plugin_impl.proto  |  2 -
 api/adaptive_load/step_controller.proto      | 46 --------------------
 api/adaptive_load/step_controller_impl.proto |  1 +
 3 files changed, 1 insertion(+), 48 deletions(-)

diff --git a/api/adaptive_load/metrics_plugin_impl.proto b/api/adaptive_load/metrics_plugin_impl.proto
index f1699be4b..772673aea 100644
--- a/api/adaptive_load/metrics_plugin_impl.proto
+++ b/api/adaptive_load/metrics_plugin_impl.proto
@@ -4,8 +4,6 @@ syntax = "proto3";
 
 package nighthawk.adaptive_load;
 
-import "validate/validate.proto";
-
 // Example plugin-specific config proto, representing connection info for an
 // outside source of metrics. Part of an example showing how to create and
 // register a MetricsPlugin.
diff --git a/api/adaptive_load/step_controller.proto b/api/adaptive_load/step_controller.proto
index a6f7179c1..86c274117 100644
--- a/api/adaptive_load/step_controller.proto
+++ b/api/adaptive_load/step_controller.proto
@@ -4,52 +4,6 @@ package nighthawk.adaptive_load;
 
 import "google/protobuf/any.proto";
 
-// Identifies a simple field or other hard-coded aspect of the Nighthawk
-// CommandLineOptions.
-enum SimpleNighthawkField {
-  UNKNOWN_FIELD = 0;
-  REQUESTS_PER_SECOND = 1;
-  NUMBER_OF_CONNECTIONS = 2;
-  CONCURRENCY = 3;
-  BURST_SIZE = 4;
-  MAX_PENDING_REQUESTS = 5;
-  MAX_ACTIVE_REQUESTS = 6;
-  MAX_REQUESTS_PER_CONNECTION = 7;
-  // Vary the length of the URI using the form "/aaaaa".
-  URI_LENGTH = 8;
-  // Vary the length of a single header x-nighthawk-adaptive-load with the value
-  // in the form "aaaaa".
-  HEADER_SIZE = 9;
-  // Add a variable number of small headers named x-nighthawk-adaptive-load-# to
-  // the request.
-  HEADER_COUNT = 10;
-  // Vary the size of the request body consisting of "aaaaa" (note: subject to
-  // Nighthawk limits).
-  REQUEST_BODY_SIZE = 12;
-  // Response size requested from a Nighthawk Test Server via a header.
-  RESPONSE_SIZE = 12;
-}
-
-// Specifies any Envoy fault injection filter header that takes a numerical
-// value. See
-// https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/fault_filter.
-message FaultInjectionHeader {
-  // Name of any numeric Envoy fault injection header.
-  string header_name = 1;
-}
-
-// Identifies a some numerical aspect of the Nighthawk CommandLineOptions proto
-// that the StepController should vary, in pursuit of the optimal field value.
-// For example, for the basic case of finding the optimal RPS, the simple_field
-// would be REQUESTS_PER_SECOND. Enables the same StepController algorithm to be
-// used to find the optimal RPS, optimal concurrency, etc.
-message NighthawkInputVariableSpec {
-  oneof selector {
-    SimpleNighthawkField simple_field = 1;
-    FaultInjectionHeader fault_injection_header = 2;
-  }
-}
-
 // Selects and configures a StepController plugin.
 message StepControllerConfig {
   // Name used to locate the plugin in the Envoy registry.
diff --git a/api/adaptive_load/step_controller_impl.proto b/api/adaptive_load/step_controller_impl.proto
index 47d57a895..e507446da 100644
--- a/api/adaptive_load/step_controller_impl.proto
+++ b/api/adaptive_load/step_controller_impl.proto
@@ -4,6 +4,7 @@ syntax = "proto3";
 
 package nighthawk.adaptive_load;
 
+import "api/adaptive_load/input_variable_setter.proto";
 import "api/adaptive_load/step_controller.proto";
 
 // Configuration for a step controller that performs an exponential search for

From aed6d9464362d850d9886a6c8ca0228f3586ca2c Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Fri, 10 Jul 2020 22:49:39 -0400
Subject: [PATCH 13/34] rename samples/adaptive_rps

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 .../example_session_spec.textproto                                | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename samples/{adaptive_rps => adaptive_load}/example_session_spec.textproto (100%)

diff --git a/samples/adaptive_rps/example_session_spec.textproto b/samples/adaptive_load/example_session_spec.textproto
similarity index 100%
rename from samples/adaptive_rps/example_session_spec.textproto
rename to samples/adaptive_load/example_session_spec.textproto

From d9ae87d165ffdea09f886c6e4721ef4024b1df9b Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Sun, 12 Jul 2020 19:49:52 -0400
Subject: [PATCH 14/34] improve comments, change step controller initial value
 from int64 to double

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 api/adaptive_load/input_variable_setter.proto | 6 +++---
 api/adaptive_load/metric_spec.proto           | 1 +
 api/adaptive_load/metrics_plugin.proto        | 3 ++-
 api/adaptive_load/step_controller.proto       | 4 +++-
 api/adaptive_load/step_controller_impl.proto  | 8 ++++----
 5 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/api/adaptive_load/input_variable_setter.proto b/api/adaptive_load/input_variable_setter.proto
index 69b8e528d..57a84faef 100644
--- a/api/adaptive_load/input_variable_setter.proto
+++ b/api/adaptive_load/input_variable_setter.proto
@@ -7,9 +7,9 @@ import "google/protobuf/any.proto";
 // Selects and configures an InputVariableSetter plugin, which encapsulates
 // knowledge of how to apply a varying numeric input value to a
 // CommandLineOptions proto. For example, RequestsPerSecondInputVariableSetter
-// aka "rps" writes the value to the |requests_per_second| field in
-// CommandLineOptions. Some InputVariableSetter plugins such as "rps" will not
-// define a config proto, in which case typed_config should be unset.
+// aka "rps" writes the value to the CommandLineOptions |requests_per_second|
+// field. Some InputVariableSetter plugins such as "rps" will not define a
+// config proto, in which case typed_config should be unset.
 message InputVariableSetterConfig {
   // Name used to locate the plugin in the Envoy registry.
   string name = 1;
diff --git a/api/adaptive_load/metric_spec.proto b/api/adaptive_load/metric_spec.proto
index 1c9323338..7206abf66 100644
--- a/api/adaptive_load/metric_spec.proto
+++ b/api/adaptive_load/metric_spec.proto
@@ -41,6 +41,7 @@ message ThresholdSpec {
   google.protobuf.DoubleValue weight = 4 [(validate.rules).double = {gt: 0.0}];
 }
 
+// Identifies a feedback metric and specifies a threshold for it.
 message MetricSpecWithThreshold {
   // Identifies a metric to collect and evaluate.
   MetricSpec metric_spec = 1;
diff --git a/api/adaptive_load/metrics_plugin.proto b/api/adaptive_load/metrics_plugin.proto
index 340e99018..e57a9e8b4 100644
--- a/api/adaptive_load/metrics_plugin.proto
+++ b/api/adaptive_load/metrics_plugin.proto
@@ -4,7 +4,8 @@ package nighthawk.adaptive_load;
 
 import "google/protobuf/any.proto";
 
-// Selects and configures a MetricsPlugin.
+// Selects and configures a MetricsPlugin, which connects to a platform-specific
+// system to obtain metrics for the system under test.
 message MetricsPluginConfig {
   // Name used to locate the plugin in the Envoy registry.
   string name = 1;
diff --git a/api/adaptive_load/step_controller.proto b/api/adaptive_load/step_controller.proto
index 86c274117..23c06b32d 100644
--- a/api/adaptive_load/step_controller.proto
+++ b/api/adaptive_load/step_controller.proto
@@ -4,7 +4,9 @@ package nighthawk.adaptive_load;
 
 import "google/protobuf/any.proto";
 
-// Selects and configures a StepController plugin.
+// Selects and configures a StepController plugin, which computes the next load
+// amount (e.g. RPS) and checks for convergence to the optimal load, based on
+// feedback metrics.
 message StepControllerConfig {
   // Name used to locate the plugin in the Envoy registry.
   string name = 1;
diff --git a/api/adaptive_load/step_controller_impl.proto b/api/adaptive_load/step_controller_impl.proto
index e507446da..c3b9f7a0b 100644
--- a/api/adaptive_load/step_controller_impl.proto
+++ b/api/adaptive_load/step_controller_impl.proto
@@ -10,16 +10,16 @@ import "api/adaptive_load/step_controller.proto";
 // Configuration for a step controller that performs an exponential search for
 // the optimal value of a single Nighthawk input variable (e.g. RPS).
 // Exponential search starts with the input set to |initial_value| and increases
-// the input by |exponential_factor| until a metric fails the threshold, then
-// performs a binary search with input values between X/2 and X, where X is the
-// input value where the metric failed the threshold.
+// the input by |exponential_factor| until a metric fails the threshold at some
+// input X, then performs a binary search with input values between
+// (X/exponential_factor) and X.
 message ExponentialSearchStepControllerConfig {
   // Selects a plugin that knows how to apply a numeric value generated by the
   // StepController within CommandLineOptions. For RPS, use name: "rps", leaving
   // typed_config unset.
   InputVariableSetterConfig input_variable_setter = 1;
   // Initial value of the input variable that should be attempted.
-  int64 initial_value = 2;
+  double initial_value = 2;
   // Factor to increase the input variable during the exponential phase. Typical
   // value: 2.0.
   double exponential_factor = 3;

From a05a6f50adb0c65bc4e2f2be9eb059389bad0266 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Mon, 13 Jul 2020 17:31:26 -0400
Subject: [PATCH 15/34] add proto validation rules, fix comments, make rps the
 default input_variable_setter

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 api/adaptive_load/adaptive_load.proto        | 14 +++++++++-----
 api/adaptive_load/metric_spec.proto          |  6 +++---
 api/adaptive_load/step_controller_impl.proto |  4 ++--
 3 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/api/adaptive_load/adaptive_load.proto b/api/adaptive_load/adaptive_load.proto
index 2c81bf6e9..b3d1e5e39 100644
--- a/api/adaptive_load/adaptive_load.proto
+++ b/api/adaptive_load/adaptive_load.proto
@@ -41,19 +41,23 @@ message AdaptiveLoadSessionSpec {
   //
   // Note that |concurrency| in |nighthawk_traffic_options| multiplies the total
   // RPS sent.
-  nighthawk.client.CommandLineOptions nighthawk_traffic_template = 4;
+  nighthawk.client.CommandLineOptions nighthawk_traffic_template = 4
+      [(validate.rules).message.required = true];
   // The duration of each short benchmark during the adjusting stage. Must be
   // set. Reasonable value: 10 seconds.
-  google.protobuf.Duration measuring_period = 5 [(validate.rules).duration.gt.seconds = 0];
+  google.protobuf.Duration measuring_period = 5
+      [(validate.rules).duration.gt.seconds = 0, (validate.rules).duration.required = true];
   // Maximum amount of time the adjusting stage should wait for convergence
   // before returning an error. Reasonable value: 300 seconds.
-  google.protobuf.Duration convergence_deadline = 6 [(validate.rules).duration.gt.seconds = 0];
+  google.protobuf.Duration convergence_deadline = 6
+      [(validate.rules).duration.gt.seconds = 0, (validate.rules).duration.required = true];
   // The duration of the single benchmark session of the testing stage to
   // confirm the performance at the level of load found in the adjusting stage.
   // Must be set.
-  google.protobuf.Duration testing_stage_duration = 7 [(validate.rules).duration.gt.seconds = 0];
+  google.protobuf.Duration testing_stage_duration = 7
+      [(validate.rules).duration.gt.seconds = 0, (validate.rules).duration.required = true];
   // Selects and configures a StepController plugin.
-  StepControllerConfig step_controller_config = 8;
+  StepControllerConfig step_controller_config = 8 [(validate.rules).message.required = true];
 }
 
 // Complete description of an adaptive load session, including metric scores
diff --git a/api/adaptive_load/metric_spec.proto b/api/adaptive_load/metric_spec.proto
index 7206abf66..138da1653 100644
--- a/api/adaptive_load/metric_spec.proto
+++ b/api/adaptive_load/metric_spec.proto
@@ -36,9 +36,9 @@ message ThresholdSpec {
     ScoringFunctionConfig scoring_function = 3;
   }
   // Relative importance of this threshold when adjusting based on multiple
-  // metrics. If any threshold has a weight, all must have a weight. If no
-  // weights are set, all thresholds have equal weight.
-  google.protobuf.DoubleValue weight = 4 [(validate.rules).double = {gt: 0.0}];
+  // metrics.
+  google.protobuf.DoubleValue weight = 4
+      [(validate.rules).double.gt = 0.0, (validate.rules).message.required = true];
 }
 
 // Identifies a feedback metric and specifies a threshold for it.
diff --git a/api/adaptive_load/step_controller_impl.proto b/api/adaptive_load/step_controller_impl.proto
index c3b9f7a0b..0f6ac3b66 100644
--- a/api/adaptive_load/step_controller_impl.proto
+++ b/api/adaptive_load/step_controller_impl.proto
@@ -15,8 +15,8 @@ import "api/adaptive_load/step_controller.proto";
 // (X/exponential_factor) and X.
 message ExponentialSearchStepControllerConfig {
   // Selects a plugin that knows how to apply a numeric value generated by the
-  // StepController within CommandLineOptions. For RPS, use name: "rps", leaving
-  // typed_config unset.
+  // StepController within CommandLineOptions. The default is the "rps" plugin
+  // that sets |requests_per_second| in CommandLineOptions.
   InputVariableSetterConfig input_variable_setter = 1;
   // Initial value of the input variable that should be attempted.
   double initial_value = 2;

From 8cd4d21cf513e23cc98154f4f390a75973454050 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Mon, 13 Jul 2020 17:35:50 -0400
Subject: [PATCH 16/34] fix comment wording

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 api/adaptive_load/step_controller_impl.proto | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/api/adaptive_load/step_controller_impl.proto b/api/adaptive_load/step_controller_impl.proto
index 0f6ac3b66..9cad83fcc 100644
--- a/api/adaptive_load/step_controller_impl.proto
+++ b/api/adaptive_load/step_controller_impl.proto
@@ -15,8 +15,8 @@ import "api/adaptive_load/step_controller.proto";
 // (X/exponential_factor) and X.
 message ExponentialSearchStepControllerConfig {
   // Selects a plugin that knows how to apply a numeric value generated by the
-  // StepController within CommandLineOptions. The default is the "rps" plugin
-  // that sets |requests_per_second| in CommandLineOptions.
+  // StepController within CommandLineOptions. The default is the "rps" plugin,
+  // which sets |requests_per_second| in CommandLineOptions.
   InputVariableSetterConfig input_variable_setter = 1;
   // Initial value of the input variable that should be attempted.
   double initial_value = 2;

From d814a96041ffb006bb5ec24d67a310c75a1e45e1 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Tue, 14 Jul 2020 10:20:51 -0400
Subject: [PATCH 17/34] simplify protos, add defaults, specify required or
 optional

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 api/adaptive_load/BUILD                       |   7 +-
 api/adaptive_load/adaptive_load.proto         |  19 +--
 api/adaptive_load/benchmark_result.proto      |   3 +
 api/adaptive_load/config.proto                |  57 +++++++++
 api/adaptive_load/input_variable_setter.proto |  18 ---
 .../input_variable_setter_impl.proto          |  16 +--
 api/adaptive_load/metric_spec.proto           |  21 ++--
 api/adaptive_load/metrics_plugin.proto        |  14 ---
 api/adaptive_load/metrics_plugin_impl.proto   |  10 +-
 api/adaptive_load/scoring_function.proto      |  14 ---
 api/adaptive_load/scoring_function_impl.proto |  23 ++--
 api/adaptive_load/step_controller.proto       |  15 ---
 api/adaptive_load/step_controller_impl.proto  |  25 ++--
 .../example_session_spec.textproto            | 109 ------------------
 14 files changed, 117 insertions(+), 234 deletions(-)
 create mode 100644 api/adaptive_load/config.proto
 delete mode 100644 api/adaptive_load/input_variable_setter.proto
 delete mode 100644 api/adaptive_load/metrics_plugin.proto
 delete mode 100644 api/adaptive_load/scoring_function.proto
 delete mode 100644 api/adaptive_load/step_controller.proto
 delete mode 100644 samples/adaptive_load/example_session_spec.textproto

diff --git a/api/adaptive_load/BUILD b/api/adaptive_load/BUILD
index 5bdb0cac6..d5eb3156c 100644
--- a/api/adaptive_load/BUILD
+++ b/api/adaptive_load/BUILD
@@ -3,18 +3,15 @@ load("@envoy_api//bazel:api_build_system.bzl", "api_cc_py_proto_library")
 licenses(["notice"])  # Apache 2
 
 api_cc_py_proto_library(
-    name = "base",
+    name = "adaptive_load_proto",
     srcs = [
         "adaptive_load.proto",
         "benchmark_result.proto",
-        "input_variable_setter.proto",
+        "config.proto",
         "input_variable_setter_impl.proto",
         "metric_spec.proto",
-        "metrics_plugin.proto",
         "metrics_plugin_impl.proto",
-        "scoring_function.proto",
         "scoring_function_impl.proto",
-        "step_controller.proto",
         "step_controller_impl.proto",
     ],
     visibility = ["//visibility:public"],
diff --git a/api/adaptive_load/adaptive_load.proto b/api/adaptive_load/adaptive_load.proto
index b3d1e5e39..90bd20619 100644
--- a/api/adaptive_load/adaptive_load.proto
+++ b/api/adaptive_load/adaptive_load.proto
@@ -1,3 +1,5 @@
+// Top-level session spec and output protos for the adaptive load controller.
+
 syntax = "proto3";
 
 package nighthawk.adaptive_load;
@@ -20,12 +22,13 @@ import "validate/validate.proto";
 message AdaptiveLoadSessionSpec {
   // Settings for MetricsPlugins that obtain metrics from outside sources.
   // An entry is required for every plugin referred to by metric_thresholds,
-  // other than the "builtin" plugin.
+  // other than the "builtin" plugin. Optional.
   repeated MetricsPluginConfig metrics_plugin_configs = 1;
-  // Metrics and thresholds that determine load adjustments.
+  // Metrics and thresholds that determine load adjustments. Required.
   repeated MetricSpecWithThreshold metric_thresholds = 2 [(validate.rules).repeated .min_items = 1];
   // Metrics that are collected and included in the output but not taken into
-  // account when adjusting the load.
+  // account when adjusting the load. May be used for debugging or
+  // visualization. Optional.
   repeated MetricSpec informational_metric_specs = 3;
   // A proto describing Nighthawk Service traffic. See
   // https://github.com/envoyproxy/nighthawk/blob/master/api/client/options.proto
@@ -41,22 +44,24 @@ message AdaptiveLoadSessionSpec {
   //
   // Note that |concurrency| in |nighthawk_traffic_options| multiplies the total
   // RPS sent.
+  //
+  // Required.
   nighthawk.client.CommandLineOptions nighthawk_traffic_template = 4
       [(validate.rules).message.required = true];
   // The duration of each short benchmark during the adjusting stage. Must be
-  // set. Reasonable value: 10 seconds.
+  // set. Optional, default 10 seconds.
   google.protobuf.Duration measuring_period = 5
       [(validate.rules).duration.gt.seconds = 0, (validate.rules).duration.required = true];
   // Maximum amount of time the adjusting stage should wait for convergence
-  // before returning an error. Reasonable value: 300 seconds.
+  // before returning an error. Optional, default 300 seconds.
   google.protobuf.Duration convergence_deadline = 6
       [(validate.rules).duration.gt.seconds = 0, (validate.rules).duration.required = true];
   // The duration of the single benchmark session of the testing stage to
   // confirm the performance at the level of load found in the adjusting stage.
-  // Must be set.
+  // Required.
   google.protobuf.Duration testing_stage_duration = 7
       [(validate.rules).duration.gt.seconds = 0, (validate.rules).duration.required = true];
-  // Selects and configures a StepController plugin.
+  // Selects and configures a StepController plugin. Required.
   StepControllerConfig step_controller_config = 8 [(validate.rules).message.required = true];
 }
 
diff --git a/api/adaptive_load/benchmark_result.proto b/api/adaptive_load/benchmark_result.proto
index 73811e1fc..694f394d4 100644
--- a/api/adaptive_load/benchmark_result.proto
+++ b/api/adaptive_load/benchmark_result.proto
@@ -1,3 +1,6 @@
+// Protos describing the results of running a single Nighthawk benchmark and
+// scoring the resulting metrics against thresholds.
+
 syntax = "proto3";
 
 package nighthawk.adaptive_load;
diff --git a/api/adaptive_load/config.proto b/api/adaptive_load/config.proto
new file mode 100644
index 000000000..f8ba7e26f
--- /dev/null
+++ b/api/adaptive_load/config.proto
@@ -0,0 +1,57 @@
+// Protos for selecting and configuring plugins for the adaptive load
+// controller infrastructure.
+
+syntax = "proto3";
+
+package nighthawk.adaptive_load;
+
+import "google/protobuf/any.proto";
+
+// Selects and configures a StepController plugin, which computes the next load
+// amount (e.g. RPS) and checks for convergence to the optimal load, based on
+// feedback metrics.
+//
+// For available plugin names, see step_controller_impl.proto.
+message StepControllerConfig {
+  // Name used to locate the plugin in the Envoy registry. Required.
+  string name = 1;
+  // Plugin-specific config. Required.
+  google.protobuf.Any typed_config = 2;
+}
+
+// Selects and configures a MetricsPlugin, which connects to a platform-specific
+// system to obtain metrics for the system under test.
+//
+// For available plugin names, see metrics_plugin_impl.proto.
+message MetricsPluginConfig {
+  // Name used to locate the plugin in the Envoy registry. Required.
+  string name = 1;
+  // Plugin-specific config such as an external API endpoint and credentials.
+  // Required.
+  google.protobuf.Any typed_config = 2;
+}
+
+// Selects and configures a ScoringFunction plugin, which measures proximity to
+// a threshold according to an arbitrary formula.
+//
+// For available plugin names, see scoring_function_impl.proto.
+message ScoringFunctionConfig {
+  // Name used to locate the plugin in the Envoy registry. Required.
+  string name = 1;
+  // Plugin-specific config such as thresholds and constants. Required.
+  google.protobuf.Any typed_config = 2;
+}
+
+// Selects and configures an InputVariableSetter plugin, which encapsulates
+// knowledge of how to apply a varying numeric input value to a
+// CommandLineOptions proto. For example, RequestsPerSecondInputVariableSetter
+// aka "rps" writes the value to the CommandLineOptions |requests_per_second|
+// field.
+//
+// For available plugin names, see input_variable_setter_impl.proto.
+message InputVariableSetterConfig {
+  // Name used to locate the plugin in the Envoy registry. Required.
+  string name = 1;
+  // Plugin-specific config. Required.
+  google.protobuf.Any typed_config = 2;
+}
diff --git a/api/adaptive_load/input_variable_setter.proto b/api/adaptive_load/input_variable_setter.proto
deleted file mode 100644
index 57a84faef..000000000
--- a/api/adaptive_load/input_variable_setter.proto
+++ /dev/null
@@ -1,18 +0,0 @@
-syntax = "proto3";
-
-package nighthawk.adaptive_load;
-
-import "google/protobuf/any.proto";
-
-// Selects and configures an InputVariableSetter plugin, which encapsulates
-// knowledge of how to apply a varying numeric input value to a
-// CommandLineOptions proto. For example, RequestsPerSecondInputVariableSetter
-// aka "rps" writes the value to the CommandLineOptions |requests_per_second|
-// field. Some InputVariableSetter plugins such as "rps" will not define a
-// config proto, in which case typed_config should be unset.
-message InputVariableSetterConfig {
-  // Name used to locate the plugin in the Envoy registry.
-  string name = 1;
-  // Plugin-specific config.
-  google.protobuf.Any typed_config = 2;
-}
diff --git a/api/adaptive_load/input_variable_setter_impl.proto b/api/adaptive_load/input_variable_setter_impl.proto
index 54a790aa9..138c23603 100644
--- a/api/adaptive_load/input_variable_setter_impl.proto
+++ b/api/adaptive_load/input_variable_setter_impl.proto
@@ -4,15 +4,9 @@ syntax = "proto3";
 
 package nighthawk.adaptive_load;
 
-// Note: RequestsPerSecondInputVariableSetter aka "rps" does not define a
-// custom config proto.
-
-// Configuration for an InputVariableSetter plugin that sets an HTTP header
-// within CommandLineOptions to a numeric value being varied by a
-// StepController.
-message NumericHttpHeaderInputVariableSetterConfig {
-  // The name of an HTTP header that can be set to a numeric value, such as one
-  // of the headers supported by the Envoy fault injection filter on Nighthawk
-  // Test Server.
-  string header_name = 1;
+// Configuration for RequestsPerSecondInputVariableSetter (plugin name: "rps")
+// that sets |requests_per_second| within CommandLineOptions to a numeric value
+// being varied by a StepController.
+message RequestsPerSecondInputVariableSetterConfig {
+  // This plugin does not need any configuration.
 }
diff --git a/api/adaptive_load/metric_spec.proto b/api/adaptive_load/metric_spec.proto
index 138da1653..6643c3123 100644
--- a/api/adaptive_load/metric_spec.proto
+++ b/api/adaptive_load/metric_spec.proto
@@ -1,3 +1,5 @@
+// Protos for identifying metrics and specifying thresholds.
+
 syntax = "proto3";
 
 package nighthawk.adaptive_load;
@@ -9,16 +11,16 @@ import "validate/validate.proto";
 // Identifies a feedback metric.
 message MetricSpec {
   // Name of the metric to evaluate. For the set of built-in metric names, see
-  // source/adaptive_load/metrics_plugin_impl.cc.
+  // source/adaptive_load/metrics_plugin_impl.cc. Required.
   string metric_name = 1 [(validate.rules).string.min_len = 1];
-  // Name of the MetricsPlugin providing the metric ("builtin" for built-in).
+  // Name of the MetricsPlugin providing the metric ("builtin" for built-in). Required.
   string metrics_plugin_name = 2 [(validate.rules).string.min_len = 1];
 }
 
 // Specifies a simple upper or lower threshold, or configures a scoring
 // function.
 message ThresholdSpec {
-  // A threshold for evaluating a metric value.
+  // A threshold for evaluating a metric value. Required.
   oneof threshold_spec {
     // Simple upper threshold:
     //   metric <= upper_threshold -> WITHIN_THRESHOLD
@@ -36,15 +38,14 @@ message ThresholdSpec {
     ScoringFunctionConfig scoring_function = 3;
   }
   // Relative importance of this threshold when adjusting based on multiple
-  // metrics.
-  google.protobuf.DoubleValue weight = 4
-      [(validate.rules).double.gt = 0.0, (validate.rules).message.required = true];
+  // metrics. Optional, default 1.0.
+  google.protobuf.DoubleValue weight = 4 [(validate.rules).double.gt = 0.0];
 }
 
 // Identifies a feedback metric and specifies a threshold for it.
 message MetricSpecWithThreshold {
-  // Identifies a metric to collect and evaluate.
-  MetricSpec metric_spec = 1;
-  // Specifies a threshold for this metric.
-  ThresholdSpec threshold_spec = 2;
+  // Identifies a metric to collect and evaluate. Required.
+  MetricSpec metric_spec = 1 [(validate.rules).message.required = true];
+  // Specifies a threshold for this metric. Required.
+  ThresholdSpec threshold_spec = 2 [(validate.rules).message.required = true];
 }
diff --git a/api/adaptive_load/metrics_plugin.proto b/api/adaptive_load/metrics_plugin.proto
deleted file mode 100644
index e57a9e8b4..000000000
--- a/api/adaptive_load/metrics_plugin.proto
+++ /dev/null
@@ -1,14 +0,0 @@
-syntax = "proto3";
-
-package nighthawk.adaptive_load;
-
-import "google/protobuf/any.proto";
-
-// Selects and configures a MetricsPlugin, which connects to a platform-specific
-// system to obtain metrics for the system under test.
-message MetricsPluginConfig {
-  // Name used to locate the plugin in the Envoy registry.
-  string name = 1;
-  // Plugin-specific config such as an external API endpoint and credentials.
-  google.protobuf.Any typed_config = 2;
-}
diff --git a/api/adaptive_load/metrics_plugin_impl.proto b/api/adaptive_load/metrics_plugin_impl.proto
index 772673aea..638365d88 100644
--- a/api/adaptive_load/metrics_plugin_impl.proto
+++ b/api/adaptive_load/metrics_plugin_impl.proto
@@ -1,13 +1,7 @@
-// Plugin-specific config protos for MetricsPlugin.
+// Plugin-specific config protos for MetricsPlugin plugins.
 
 syntax = "proto3";
 
 package nighthawk.adaptive_load;
 
-// Example plugin-specific config proto, representing connection info for an
-// outside source of metrics. Part of an example showing how to create and
-// register a MetricsPlugin.
-message ExampleMetricsPluginConfig {
-  string address = 1;
-  string credentials = 2;
-}
+// Plugin-specific config protos for MetricsPlugins that ship with Nighthawk should go here.
\ No newline at end of file
diff --git a/api/adaptive_load/scoring_function.proto b/api/adaptive_load/scoring_function.proto
deleted file mode 100644
index fbf9b9ad9..000000000
--- a/api/adaptive_load/scoring_function.proto
+++ /dev/null
@@ -1,14 +0,0 @@
-syntax = "proto3";
-
-package nighthawk.adaptive_load;
-
-import "google/protobuf/any.proto";
-
-// Configures a ScoringFunction plugin that measures proximity to a threshold
-// according to an arbitrary formula.
-message ScoringFunctionConfig {
-  // Name used to locate the plugin in the Envoy registry.
-  string name = 1;
-  // Plugin-specific config such as thresholds and constants.
-  google.protobuf.Any typed_config = 2;
-}
diff --git a/api/adaptive_load/scoring_function_impl.proto b/api/adaptive_load/scoring_function_impl.proto
index 67726b4ae..707bcaa93 100644
--- a/api/adaptive_load/scoring_function_impl.proto
+++ b/api/adaptive_load/scoring_function_impl.proto
@@ -1,14 +1,15 @@
-// Plugin-specific config protos for ScoringFunctions.
+// Plugin-specific config protos for ScoringFunction plugins.
 
 syntax = "proto3";
 
 package nighthawk.adaptive_load;
 
-// Configuration for a linear ScoringFunction that calculates a metric score
-// as k * (threshold - value), where k is a scaling constant. The score is 0.0
-// when the value exactly equals the threshold, positive below the threshold
-// (meaning input should ramp up), and negative above the threshold. The score is
-// proportional to the difference from the threshold.
+// Configuration for LinearScoringFunction (plugin name: "linear") that
+// calculates a metric score as k * (threshold - value), where k is a scaling
+// constant. The score is 0.0 when the value exactly equals the threshold,
+// positive below the threshold (meaning input should ramp up), and negative
+// above the threshold. The score is proportional to the difference from the
+// threshold.
 message LinearScoringFunctionConfig {
   // The target value of the metric.
   double threshold = 1;
@@ -18,11 +19,11 @@ message LinearScoringFunctionConfig {
   double k = 2;
 }
 
-// Configuration for a ScoringFunction that calculates a metric score
-// as 1 - 2 / (1 + exp(-k(value - threshold))), an upside-down sigmoid curve
-// centered on a threshold. The output is 0.0 when the metric equals the
-// threshold, approaches 1.0 for values far below the threshold, and approaches
-// -1.0 for values far above the threshold.
+// Configuration for SigmoidScoringFunction (plugin name: "sigmoid") that
+// calculates a metric score as 1 - 2 / (1 + exp(-k(value - threshold))), an
+// upside-down sigmoid curve centered on a threshold. The output is 0.0 when the
+// metric equals the threshold, approaches 1.0 for values far below the
+// threshold, and approaches -1.0 for values far above the threshold.
 message SigmoidScoringFunctionConfig {
   // The target value of the metric.
   double threshold = 1;
diff --git a/api/adaptive_load/step_controller.proto b/api/adaptive_load/step_controller.proto
deleted file mode 100644
index 23c06b32d..000000000
--- a/api/adaptive_load/step_controller.proto
+++ /dev/null
@@ -1,15 +0,0 @@
-syntax = "proto3";
-
-package nighthawk.adaptive_load;
-
-import "google/protobuf/any.proto";
-
-// Selects and configures a StepController plugin, which computes the next load
-// amount (e.g. RPS) and checks for convergence to the optimal load, based on
-// feedback metrics.
-message StepControllerConfig {
-  // Name used to locate the plugin in the Envoy registry.
-  string name = 1;
-  // Plugin-specific config.
-  google.protobuf.Any typed_config = 2;
-}
diff --git a/api/adaptive_load/step_controller_impl.proto b/api/adaptive_load/step_controller_impl.proto
index 9cad83fcc..5189946ac 100644
--- a/api/adaptive_load/step_controller_impl.proto
+++ b/api/adaptive_load/step_controller_impl.proto
@@ -1,4 +1,4 @@
-// Plugin-specific config protos for StepControllers.
+// Plugin-specific config protos for StepController plugins.
 
 syntax = "proto3";
 
@@ -7,20 +7,21 @@ package nighthawk.adaptive_load;
 import "api/adaptive_load/input_variable_setter.proto";
 import "api/adaptive_load/step_controller.proto";
 
-// Configuration for a step controller that performs an exponential search for
-// the optimal value of a single Nighthawk input variable (e.g. RPS).
-// Exponential search starts with the input set to |initial_value| and increases
-// the input by |exponential_factor| until a metric fails the threshold at some
-// input X, then performs a binary search with input values between
-// (X/exponential_factor) and X.
+// Configuration for ExponentialSearchStepController (plugin name:
+// "exponential-search") that performs an exponential search for the optimal
+// value of a single Nighthawk input variable (e.g. RPS). Exponential search
+// starts with the input set to |initial_value| and increases the input by
+// |exponential_factor| until a metric fails the threshold at some input X, then
+// performs a binary search with input values between (X/exponential_factor) and
+// X.
 message ExponentialSearchStepControllerConfig {
   // Selects a plugin that knows how to apply a numeric value generated by the
-  // StepController within CommandLineOptions. The default is the "rps" plugin,
-  // which sets |requests_per_second| in CommandLineOptions.
+  // StepController within CommandLineOptions. Optional, defaults to "rps"
+  // plugin, which sets |requests_per_second| in CommandLineOptions.
   InputVariableSetterConfig input_variable_setter = 1;
-  // Initial value of the input variable that should be attempted.
+  // Initial value of the input variable that should be attempted. Required.
   double initial_value = 2;
-  // Factor to increase the input variable during the exponential phase. Typical
-  // value: 2.0.
+  // Factor to increase the input variable during the exponential phase.
+  // Optional, default 2.0.
   double exponential_factor = 3;
 }
diff --git a/samples/adaptive_load/example_session_spec.textproto b/samples/adaptive_load/example_session_spec.textproto
deleted file mode 100644
index aecba5cb1..000000000
--- a/samples/adaptive_load/example_session_spec.textproto
+++ /dev/null
@@ -1,109 +0,0 @@
-# proto-file: adaptive_load/adaptive_load.proto
-# proto-message: AdaptiveLoadSessionSpec
-
-metrics_plugin_configs {
-  name: "example-metrics-plugin"
-  typed_config {
-    [type.googleapis.com/nighthawk.adaptive_load.ExampleMetricsPluginConfig] {
-      address: "x.y.z"
-      credentials: "key-id-123"
-    }
-  }
-}
-
-# For the full list of built-in metric names, see
-# source/adaptive_load/metrics_plugin_impl.cc.
-
-metric_thresholds {
-  metric_spec {
-    metric_name: "latency-ns-mean-plus-2stdev"
-    metrics_plugin_name: "builtin"
-  }
-  threshold_spec {
-    scoring_function {
-      name: "sigmoid"
-      typed_config {
-        [type.googleapis.com/nighthawk.adaptive_load.SigmoidScoringFunctionConfig] {
-          threshold: 50000000.0  # 50 ms
-          k: 0.00000001  # Should be around 1/threshold
-        }
-      }
-    }
-    weight {
-      value: 1.0
-    }
-  }
-}
-
-metric_thresholds {
-  metric_spec {
-    metric_name: "success-rate"
-    metrics_plugin_name: "builtin"
-  }
-  threshold_spec {
-    lower_threshold {
-      value : 0.95
-    }
-    weight {
-      value: 1.0
-    }
-  }
-}
-
-# Informational metrics are recorded in the output but are not taken into account
-# when adjusting the load.
-
-informational_metric_specs {
-  metric_name: "attempted-rps"
-  metrics_plugin_name: "builtin"
-}
-
-informational_metric_specs {
-  metric_name: "achieved-rps"
-  metrics_plugin_name: "builtin"
-}
-
-informational_metric_specs {
-  metric_name: "send-rate"
-  metrics_plugin_name: "builtin"
-}
-
-# Metric defined in example metrics plugin:
-informational_metric_specs {
-  metric_name: "example_metric1"
-  metrics_plugin_name: "example-metrics-plugin"
-}
-
-# CommandLineOptions proto describing the traffic to generate, but without
-# duration, open_loop, or the field to be varied (e.g. requests_per_second),
-# which will be managed by the adaptive load controller.
-nighthawk_traffic_template {
-  uri {
-    value: "[::1]:12345"
-  }
-}
-
-measuring_period {
-  seconds: 4
-}
-
-convergence_deadline {
-  seconds: 180
-}
-
-testing_stage_duration {
-  seconds: 30
-}
-
-step_controller_config {
-  name: "exponential-search"
-  typed_config {
-    [type.googleapis.com/nighthawk.adaptive_load.ExponentialSearchStepControllerConfig] {
-      input_variable_setter {
-        name: "rps"
-      }
-      initial_value: 500
-      exponential_factor: 2.0
-    }
-  }
-}

From 5f5a885f00b61527fe1c53023bad6bdfa3177214 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Tue, 14 Jul 2020 10:37:03 -0400
Subject: [PATCH 18/34] add missing newline

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 api/adaptive_load/metrics_plugin_impl.proto   | 2 +-
 api/adaptive_load/scoring_function_impl.proto | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/api/adaptive_load/metrics_plugin_impl.proto b/api/adaptive_load/metrics_plugin_impl.proto
index 638365d88..e9b1149e8 100644
--- a/api/adaptive_load/metrics_plugin_impl.proto
+++ b/api/adaptive_load/metrics_plugin_impl.proto
@@ -4,4 +4,4 @@ syntax = "proto3";
 
 package nighthawk.adaptive_load;
 
-// Plugin-specific config protos for MetricsPlugins that ship with Nighthawk should go here.
\ No newline at end of file
+// Plugin-specific config protos for MetricsPlugins that ship with Nighthawk should go here.
diff --git a/api/adaptive_load/scoring_function_impl.proto b/api/adaptive_load/scoring_function_impl.proto
index 707bcaa93..46e151ed9 100644
--- a/api/adaptive_load/scoring_function_impl.proto
+++ b/api/adaptive_load/scoring_function_impl.proto
@@ -11,11 +11,11 @@ package nighthawk.adaptive_load;
 // above the threshold. The score is proportional to the difference from the
 // threshold.
 message LinearScoringFunctionConfig {
-  // The target value of the metric.
+  // The target value of the metric. Required.
   double threshold = 1;
   // Scaling constant: k in k * (threshold - value). Use this in combination
   // with step controller constants to produce reasonable input increments for
-  // reasonable differences from the threshold.
+  // reasonable differences from the threshold. Required.
   double k = 2;
 }
 
@@ -25,9 +25,9 @@ message LinearScoringFunctionConfig {
 // metric equals the threshold, approaches 1.0 for values far below the
 // threshold, and approaches -1.0 for values far above the threshold.
 message SigmoidScoringFunctionConfig {
-  // The target value of the metric.
+  // The target value of the metric. Required.
   double threshold = 1;
   // Tuning constant: k in 1 - 2 / (1 + exp(-k(value - threshold))). k should
-  // be around the same size as 1/threshold.
+  // be around the same size as 1/threshold. Required.
   double k = 2;
 }

From 7e20a78f66626425cf2f208e1ac3ab8c16e6302e Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Tue, 14 Jul 2020 18:51:42 -0400
Subject: [PATCH 19/34] Kick CI

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>

From 904826753c744ddd27e8985ffd2a575aaa0f9997 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Wed, 15 Jul 2020 14:33:20 -0400
Subject: [PATCH 20/34] simplify protos

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 api/adaptive_load/adaptive_load.proto         | 32 ++++++++------
 api/adaptive_load/benchmark_result.proto      | 32 +++-----------
 api/adaptive_load/metric_spec.proto           | 42 +++++++------------
 api/adaptive_load/scoring_function_impl.proto | 11 +++++
 api/adaptive_load/step_controller_impl.proto  | 10 ++---
 5 files changed, 58 insertions(+), 69 deletions(-)

diff --git a/api/adaptive_load/adaptive_load.proto b/api/adaptive_load/adaptive_load.proto
index 90bd20619..b916aea54 100644
--- a/api/adaptive_load/adaptive_load.proto
+++ b/api/adaptive_load/adaptive_load.proto
@@ -5,9 +5,8 @@ syntax = "proto3";
 package nighthawk.adaptive_load;
 
 import "api/adaptive_load/benchmark_result.proto";
-import "api/adaptive_load/metrics_plugin.proto";
+import "api/adaptive_load/config.proto";
 import "api/adaptive_load/metric_spec.proto";
-import "api/adaptive_load/step_controller.proto";
 import "api/client/options.proto";
 import "google/protobuf/duration.proto";
 import "google/rpc/status.proto";
@@ -24,8 +23,10 @@ message AdaptiveLoadSessionSpec {
   // An entry is required for every plugin referred to by metric_thresholds,
   // other than the "builtin" plugin. Optional.
   repeated MetricsPluginConfig metrics_plugin_configs = 1;
-  // Metrics and thresholds that determine load adjustments. Required.
-  repeated MetricSpecWithThreshold metric_thresholds = 2 [(validate.rules).repeated .min_items = 1];
+  // Metrics and thresholds that determine load adjustments. The order of
+  // metrics is not significant. Required.
+  repeated MetricSpecWithThreshold metric_thresholds = 2
+      [ (validate.rules).repeated .min_items = 1 ];
   // Metrics that are collected and included in the output but not taken into
   // account when adjusting the load. May be used for debugging or
   // visualization. Optional.
@@ -47,22 +48,29 @@ message AdaptiveLoadSessionSpec {
   //
   // Required.
   nighthawk.client.CommandLineOptions nighthawk_traffic_template = 4
-      [(validate.rules).message.required = true];
+      [ (validate.rules).message.required = true ];
   // The duration of each short benchmark during the adjusting stage. Must be
   // set. Optional, default 10 seconds.
-  google.protobuf.Duration measuring_period = 5
-      [(validate.rules).duration.gt.seconds = 0, (validate.rules).duration.required = true];
+  google.protobuf.Duration measuring_period = 5 [
+    (validate.rules).duration.gt.seconds = 0,
+    (validate.rules).duration.required = true
+  ];
   // Maximum amount of time the adjusting stage should wait for convergence
   // before returning an error. Optional, default 300 seconds.
-  google.protobuf.Duration convergence_deadline = 6
-      [(validate.rules).duration.gt.seconds = 0, (validate.rules).duration.required = true];
+  google.protobuf.Duration convergence_deadline = 6 [
+    (validate.rules).duration.gt.seconds = 0,
+    (validate.rules).duration.required = true
+  ];
   // The duration of the single benchmark session of the testing stage to
   // confirm the performance at the level of load found in the adjusting stage.
   // Required.
-  google.protobuf.Duration testing_stage_duration = 7
-      [(validate.rules).duration.gt.seconds = 0, (validate.rules).duration.required = true];
+  google.protobuf.Duration testing_stage_duration = 7 [
+    (validate.rules).duration.gt.seconds = 0,
+    (validate.rules).duration.required = true
+  ];
   // Selects and configures a StepController plugin. Required.
-  StepControllerConfig step_controller_config = 8 [(validate.rules).message.required = true];
+  StepControllerConfig step_controller_config = 8
+      [ (validate.rules).message.required = true ];
 }
 
 // Complete description of an adaptive load session, including metric scores
diff --git a/api/adaptive_load/benchmark_result.proto b/api/adaptive_load/benchmark_result.proto
index 694f394d4..54ed93d19 100644
--- a/api/adaptive_load/benchmark_result.proto
+++ b/api/adaptive_load/benchmark_result.proto
@@ -8,30 +8,6 @@ package nighthawk.adaptive_load;
 import "api/adaptive_load/metric_spec.proto";
 import "api/client/output.proto";
 
-// Binary result of checking a metric value against a threshold.
-enum SimpleThresholdStatus {
-  // Value should only occur when the field is unset.
-  UNKNOWN_THRESHOLD_STATUS = 0;
-  // The metric was below an upper threshold or above a lower threshold.
-  WITHIN_THRESHOLD = 1;
-  // The metric was above an upper threshold or below a lower threshold.
-  OUTSIDE_THRESHOLD = 2;
-}
-
-// Encapsulates results of simple and custom threshold checks.
-message ThresholdCheckResult {
-  oneof threshold_check_result {
-    // Binary status from checking the value against a threshold.
-    SimpleThresholdStatus simple_threshold_status = 1;
-    // Score returned by a ScoringFunction plugin. This expresses how close the
-    // metric was to the threshold by an arbitrary formula selected and
-    // configured in the ThresholdSpec, such as a sigmoid curve. The controller
-    // can choose to make larger input adjustments when the score is larger, in
-    // order to converge faster.
-    double threshold_score = 2;
-  }
-}
-
 // Records the status of a single metric during a benchmark session.
 message MetricEvaluation {
   // Original metric specification (plugin name, metric name).
@@ -41,9 +17,13 @@ message MetricEvaluation {
   // Threshold that was used to check this metric value. Not set if the metric
   // was only informational.
   ThresholdSpec threshold_spec = 3;
-  // The result of checking this metric value against the configured threshold.
+  // Score returned by a ScoringFunction plugin. This expresses how close the
+  // metric was to the threshold by an arbitrary formula selected and
+  // configured in the ThresholdSpec, such as a sigmoid curve. The controller
+  // can choose to make larger input adjustments when the score is larger, in
+  // order to converge faster.
   // Not set if the metric was only informational.
-  ThresholdCheckResult threshold_check_result = 4;
+  double threshold_score = 4;
 }
 
 // Summary of a single Nighthawk Service benchmark session with evaluation
diff --git a/api/adaptive_load/metric_spec.proto b/api/adaptive_load/metric_spec.proto
index 6643c3123..ff32b7782 100644
--- a/api/adaptive_load/metric_spec.proto
+++ b/api/adaptive_load/metric_spec.proto
@@ -4,7 +4,7 @@ syntax = "proto3";
 
 package nighthawk.adaptive_load;
 
-import "api/adaptive_load/scoring_function.proto";
+import "api/adaptive_load/config.proto";
 import "google/protobuf/wrappers.proto";
 import "validate/validate.proto";
 
@@ -12,40 +12,30 @@ import "validate/validate.proto";
 message MetricSpec {
   // Name of the metric to evaluate. For the set of built-in metric names, see
   // source/adaptive_load/metrics_plugin_impl.cc. Required.
-  string metric_name = 1 [(validate.rules).string.min_len = 1];
-  // Name of the MetricsPlugin providing the metric ("builtin" for built-in). Required.
-  string metrics_plugin_name = 2 [(validate.rules).string.min_len = 1];
+  string metric_name = 1 [ (validate.rules).string.min_len = 1 ];
+  // Name of the MetricsPlugin providing the metric ("builtin" for built-in).
+  // Required.
+  string metrics_plugin_name = 2 [ (validate.rules).string.min_len = 1 ];
 }
 
-// Specifies a simple upper or lower threshold, or configures a scoring
-// function.
+// Specifies how to score a metric against a threshold.
 message ThresholdSpec {
-  // A threshold for evaluating a metric value. Required.
-  oneof threshold_spec {
-    // Simple upper threshold:
-    //   metric <= upper_threshold -> WITHIN_THRESHOLD
-    //   metric > upper_threshold -> OUTSIDE_THRESHOLD
-    google.protobuf.DoubleValue upper_threshold = 1;
-    // Simple lower threshold:
-    //   metric >= lower_threshold -> WITHIN_THRESHOLD
-    //   metric < lower_threshold -> OUTSIDE_THRESHOLD
-    google.protobuf.DoubleValue lower_threshold = 2;
-    // Selection and configuration of a ScoringFunction that measures proximity
-    // to a threshold. 0.0 means the value equals the threshold, positive means
-    // the value is below the threshold so the input should ramp up, and
-    // negative means the value is above the threshold so input should ramp
-    // down.
-    ScoringFunctionConfig scoring_function = 3;
-  }
+  // Selection and configuration of a ScoringFunction that measures proximity
+  // to a threshold. 0.0 means the value equals the threshold, positive means
+  // the value is within the threshold so the input should ramp up, and
+  // negative means the value is outside the threshold so input should ramp
+  // down.
+  ScoringFunctionConfig scoring_function = 1
+      [ (validate.rules).message.required = true ];
   // Relative importance of this threshold when adjusting based on multiple
   // metrics. Optional, default 1.0.
-  google.protobuf.DoubleValue weight = 4 [(validate.rules).double.gt = 0.0];
+  google.protobuf.DoubleValue weight = 2 [ (validate.rules).double.gt = 0.0 ];
 }
 
 // Identifies a feedback metric and specifies a threshold for it.
 message MetricSpecWithThreshold {
   // Identifies a metric to collect and evaluate. Required.
-  MetricSpec metric_spec = 1 [(validate.rules).message.required = true];
+  MetricSpec metric_spec = 1 [ (validate.rules).message.required = true ];
   // Specifies a threshold for this metric. Required.
-  ThresholdSpec threshold_spec = 2 [(validate.rules).message.required = true];
+  ThresholdSpec threshold_spec = 2 [ (validate.rules).message.required = true ];
 }
diff --git a/api/adaptive_load/scoring_function_impl.proto b/api/adaptive_load/scoring_function_impl.proto
index 46e151ed9..ca22d9d3e 100644
--- a/api/adaptive_load/scoring_function_impl.proto
+++ b/api/adaptive_load/scoring_function_impl.proto
@@ -4,6 +4,17 @@ syntax = "proto3";
 
 package nighthawk.adaptive_load;
 
+import "google/protobuf/wrappers.proto";
+
+// Configuration for BinaryScoringFunction (plugin name: "binary") that is 1.0
+// when the value is within thresholds and -1.0 otherwise.
+message BinaryScoringFunctionConfig {
+  // The minimum allowed value of the metric. Optional, default -infinity.
+  google.protobuf.DoubleValue lower_threshold = 1;
+  // The maximum allowed value of the metric. Optional, default infinity.
+  google.protobuf.DoubleValue upper_threshold = 2;
+}
+
 // Configuration for LinearScoringFunction (plugin name: "linear") that
 // calculates a metric score as k * (threshold - value), where k is a scaling
 // constant. The score is 0.0 when the value exactly equals the threshold,
diff --git a/api/adaptive_load/step_controller_impl.proto b/api/adaptive_load/step_controller_impl.proto
index 5189946ac..7c138a215 100644
--- a/api/adaptive_load/step_controller_impl.proto
+++ b/api/adaptive_load/step_controller_impl.proto
@@ -4,16 +4,16 @@ syntax = "proto3";
 
 package nighthawk.adaptive_load;
 
-import "api/adaptive_load/input_variable_setter.proto";
-import "api/adaptive_load/step_controller.proto";
+import "api/adaptive_load/config.proto";
 
 // Configuration for ExponentialSearchStepController (plugin name:
 // "exponential-search") that performs an exponential search for the optimal
 // value of a single Nighthawk input variable (e.g. RPS). Exponential search
 // starts with the input set to |initial_value| and increases the input by
-// |exponential_factor| until a metric fails the threshold at some input X, then
-// performs a binary search with input values between (X/exponential_factor) and
-// X.
+// |exponential_factor| until the metric goes outside thresholds at some input
+// value X (i.e. the sign of the score becomes negative); then it performs a
+// binary search with input values between (X/exponential_factor) and X for the
+// highest input value for which the metric is within thresholds.
 message ExponentialSearchStepControllerConfig {
   // Selects a plugin that knows how to apply a numeric value generated by the
   // StepController within CommandLineOptions. Optional, defaults to "rps"

From 306c0ec8e2bbc40fa1ff8a31e63fc7f19078a034 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Wed, 15 Jul 2020 15:06:00 -0400
Subject: [PATCH 21/34] fix format

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 api/adaptive_load/adaptive_load.proto | 26 +++++++++-----------------
 api/adaptive_load/metric_spec.proto   | 13 ++++++-------
 2 files changed, 15 insertions(+), 24 deletions(-)

diff --git a/api/adaptive_load/adaptive_load.proto b/api/adaptive_load/adaptive_load.proto
index b916aea54..a6f59c339 100644
--- a/api/adaptive_load/adaptive_load.proto
+++ b/api/adaptive_load/adaptive_load.proto
@@ -25,8 +25,7 @@ message AdaptiveLoadSessionSpec {
   repeated MetricsPluginConfig metrics_plugin_configs = 1;
   // Metrics and thresholds that determine load adjustments. The order of
   // metrics is not significant. Required.
-  repeated MetricSpecWithThreshold metric_thresholds = 2
-      [ (validate.rules).repeated .min_items = 1 ];
+  repeated MetricSpecWithThreshold metric_thresholds = 2 [(validate.rules).repeated .min_items = 1];
   // Metrics that are collected and included in the output but not taken into
   // account when adjusting the load. May be used for debugging or
   // visualization. Optional.
@@ -48,29 +47,22 @@ message AdaptiveLoadSessionSpec {
   //
   // Required.
   nighthawk.client.CommandLineOptions nighthawk_traffic_template = 4
-      [ (validate.rules).message.required = true ];
+      [(validate.rules).message.required = true];
   // The duration of each short benchmark during the adjusting stage. Must be
   // set. Optional, default 10 seconds.
-  google.protobuf.Duration measuring_period = 5 [
-    (validate.rules).duration.gt.seconds = 0,
-    (validate.rules).duration.required = true
-  ];
+  google.protobuf.Duration measuring_period = 5
+      [(validate.rules).duration.gt.seconds = 0, (validate.rules).duration.required = true];
   // Maximum amount of time the adjusting stage should wait for convergence
   // before returning an error. Optional, default 300 seconds.
-  google.protobuf.Duration convergence_deadline = 6 [
-    (validate.rules).duration.gt.seconds = 0,
-    (validate.rules).duration.required = true
-  ];
+  google.protobuf.Duration convergence_deadline = 6
+      [(validate.rules).duration.gt.seconds = 0, (validate.rules).duration.required = true];
   // The duration of the single benchmark session of the testing stage to
   // confirm the performance at the level of load found in the adjusting stage.
   // Required.
-  google.protobuf.Duration testing_stage_duration = 7 [
-    (validate.rules).duration.gt.seconds = 0,
-    (validate.rules).duration.required = true
-  ];
+  google.protobuf.Duration testing_stage_duration = 7
+      [(validate.rules).duration.gt.seconds = 0, (validate.rules).duration.required = true];
   // Selects and configures a StepController plugin. Required.
-  StepControllerConfig step_controller_config = 8
-      [ (validate.rules).message.required = true ];
+  StepControllerConfig step_controller_config = 8 [(validate.rules).message.required = true];
 }
 
 // Complete description of an adaptive load session, including metric scores
diff --git a/api/adaptive_load/metric_spec.proto b/api/adaptive_load/metric_spec.proto
index ff32b7782..5de8864a4 100644
--- a/api/adaptive_load/metric_spec.proto
+++ b/api/adaptive_load/metric_spec.proto
@@ -12,10 +12,10 @@ import "validate/validate.proto";
 message MetricSpec {
   // Name of the metric to evaluate. For the set of built-in metric names, see
   // source/adaptive_load/metrics_plugin_impl.cc. Required.
-  string metric_name = 1 [ (validate.rules).string.min_len = 1 ];
+  string metric_name = 1 [(validate.rules).string.min_len = 1];
   // Name of the MetricsPlugin providing the metric ("builtin" for built-in).
   // Required.
-  string metrics_plugin_name = 2 [ (validate.rules).string.min_len = 1 ];
+  string metrics_plugin_name = 2 [(validate.rules).string.min_len = 1];
 }
 
 // Specifies how to score a metric against a threshold.
@@ -25,17 +25,16 @@ message ThresholdSpec {
   // the value is within the threshold so the input should ramp up, and
   // negative means the value is outside the threshold so input should ramp
   // down.
-  ScoringFunctionConfig scoring_function = 1
-      [ (validate.rules).message.required = true ];
+  ScoringFunctionConfig scoring_function = 1 [(validate.rules).message.required = true];
   // Relative importance of this threshold when adjusting based on multiple
   // metrics. Optional, default 1.0.
-  google.protobuf.DoubleValue weight = 2 [ (validate.rules).double.gt = 0.0 ];
+  google.protobuf.DoubleValue weight = 2 [(validate.rules).double.gt = 0.0];
 }
 
 // Identifies a feedback metric and specifies a threshold for it.
 message MetricSpecWithThreshold {
   // Identifies a metric to collect and evaluate. Required.
-  MetricSpec metric_spec = 1 [ (validate.rules).message.required = true ];
+  MetricSpec metric_spec = 1 [(validate.rules).message.required = true];
   // Specifies a threshold for this metric. Required.
-  ThresholdSpec threshold_spec = 2 [ (validate.rules).message.required = true ];
+  ThresholdSpec threshold_spec = 2 [(validate.rules).message.required = true];
 }

From d33f5434fa18ef1d977b4041661ef679b7dc7fee Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Wed, 15 Jul 2020 15:45:25 -0400
Subject: [PATCH 22/34] fix some optional field comments and rules

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 api/adaptive_load/adaptive_load.proto | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/api/adaptive_load/adaptive_load.proto b/api/adaptive_load/adaptive_load.proto
index a6f59c339..1a4e6da0f 100644
--- a/api/adaptive_load/adaptive_load.proto
+++ b/api/adaptive_load/adaptive_load.proto
@@ -48,14 +48,11 @@ message AdaptiveLoadSessionSpec {
   // Required.
   nighthawk.client.CommandLineOptions nighthawk_traffic_template = 4
       [(validate.rules).message.required = true];
-  // The duration of each short benchmark during the adjusting stage. Must be
-  // set. Optional, default 10 seconds.
-  google.protobuf.Duration measuring_period = 5
-      [(validate.rules).duration.gt.seconds = 0, (validate.rules).duration.required = true];
+  // The duration of each short benchmark during the adjusting stage. Optional, default 10 seconds.
+  google.protobuf.Duration measuring_period = 5 [(validate.rules).duration.gt.seconds = 0];
   // Maximum amount of time the adjusting stage should wait for convergence
   // before returning an error. Optional, default 300 seconds.
-  google.protobuf.Duration convergence_deadline = 6
-      [(validate.rules).duration.gt.seconds = 0, (validate.rules).duration.required = true];
+  google.protobuf.Duration convergence_deadline = 6 [(validate.rules).duration.gt.seconds = 0];
   // The duration of the single benchmark session of the testing stage to
   // confirm the performance at the level of load found in the adjusting stage.
   // Required.

From 677b78359748a24b72ce492bcec1fde66bb61c7b Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Sun, 19 Jul 2020 19:40:42 -0400
Subject: [PATCH 23/34] add Nighthawk status field in BenchmarkResult as nested
 nighthawk.client.Output turns out not to include the status

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 api/adaptive_load/benchmark_result.proto | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/api/adaptive_load/benchmark_result.proto b/api/adaptive_load/benchmark_result.proto
index 54ed93d19..89630fc72 100644
--- a/api/adaptive_load/benchmark_result.proto
+++ b/api/adaptive_load/benchmark_result.proto
@@ -7,6 +7,7 @@ package nighthawk.adaptive_load;
 
 import "api/adaptive_load/metric_spec.proto";
 import "api/client/output.proto";
+import "google/rpc/status.proto";
 
 // Records the status of a single metric during a benchmark session.
 message MetricEvaluation {
@@ -30,9 +31,11 @@ message MetricEvaluation {
 // results.
 message BenchmarkResult {
   // Raw Nighthawk Service output. Includes start/end times and full Nighthawk
-  // Service input spec. May contain an error status.
+  // Service input spec.
   nighthawk.client.Output nighthawk_service_output = 1;
+  // Status of this Nighthawk Service benchmark session.
+  google.rpc.Status status = 2;
   // Status of all declared metrics during this benchmark session. Not present
-  // in the event of Nighthawk errors.
+  // in the event of Nighthawk Service errors.
   repeated MetricEvaluation metric_evaluations = 3;
 }

From cefb3663f7b1b5def2d5e12393214f4750043f04 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Tue, 21 Jul 2020 20:33:22 -0400
Subject: [PATCH 24/34] switch to standard Envoy plugin config proto, add
 prefix to internal plugin names, log thresholds only once per session

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 api/adaptive_load/BUILD                       |  2 +-
 api/adaptive_load/adaptive_load.proto         | 12 ++--
 api/adaptive_load/benchmark_result.proto      | 15 ++---
 api/adaptive_load/config.proto                | 57 -------------------
 .../input_variable_setter_impl.proto          |  2 +-
 api/adaptive_load/metric_spec.proto           |  7 ++-
 api/adaptive_load/scoring_function_impl.proto |  6 +-
 api/adaptive_load/step_controller_impl.proto  |  6 +-
 8 files changed, 26 insertions(+), 81 deletions(-)
 delete mode 100644 api/adaptive_load/config.proto

diff --git a/api/adaptive_load/BUILD b/api/adaptive_load/BUILD
index d5eb3156c..009bbea75 100644
--- a/api/adaptive_load/BUILD
+++ b/api/adaptive_load/BUILD
@@ -7,7 +7,6 @@ api_cc_py_proto_library(
     srcs = [
         "adaptive_load.proto",
         "benchmark_result.proto",
-        "config.proto",
         "input_variable_setter_impl.proto",
         "metric_spec.proto",
         "metrics_plugin_impl.proto",
@@ -16,6 +15,7 @@ api_cc_py_proto_library(
     ],
     visibility = ["//visibility:public"],
     deps = [
+        "@envoy_api//envoy/config/core/v3:pkg",
         "@nighthawk//api/client:base",
     ],
 )
diff --git a/api/adaptive_load/adaptive_load.proto b/api/adaptive_load/adaptive_load.proto
index 1a4e6da0f..6ffcf1d69 100644
--- a/api/adaptive_load/adaptive_load.proto
+++ b/api/adaptive_load/adaptive_load.proto
@@ -5,9 +5,9 @@ syntax = "proto3";
 package nighthawk.adaptive_load;
 
 import "api/adaptive_load/benchmark_result.proto";
-import "api/adaptive_load/config.proto";
 import "api/adaptive_load/metric_spec.proto";
 import "api/client/options.proto";
+import "envoy/config/core/v3/extension.proto";
 import "google/protobuf/duration.proto";
 import "google/rpc/status.proto";
 import "validate/validate.proto";
@@ -21,8 +21,8 @@ import "validate/validate.proto";
 message AdaptiveLoadSessionSpec {
   // Settings for MetricsPlugins that obtain metrics from outside sources.
   // An entry is required for every plugin referred to by metric_thresholds,
-  // other than the "builtin" plugin. Optional.
-  repeated MetricsPluginConfig metrics_plugin_configs = 1;
+  // other than the "nighthawk.builtin" plugin. Optional.
+  repeated envoy.config.core.v3.TypedExtensionConfig metrics_plugin_configs = 1;
   // Metrics and thresholds that determine load adjustments. The order of
   // metrics is not significant. Required.
   repeated MetricSpecWithThreshold metric_thresholds = 2 [(validate.rules).repeated .min_items = 1];
@@ -59,7 +59,8 @@ message AdaptiveLoadSessionSpec {
   google.protobuf.Duration testing_stage_duration = 7
       [(validate.rules).duration.gt.seconds = 0, (validate.rules).duration.required = true];
   // Selects and configures a StepController plugin. Required.
-  StepControllerConfig step_controller_config = 8 [(validate.rules).message.required = true];
+  envoy.config.core.v3.TypedExtensionConfig step_controller_config = 8
+      [(validate.rules).message.required = true];
 }
 
 // Complete description of an adaptive load session, including metric scores
@@ -71,4 +72,7 @@ message AdaptiveLoadSessionOutput {
   repeated BenchmarkResult adjusting_stage_results = 2;
   // Result of the single benchmark of the testing stage.
   BenchmarkResult testing_stage_result = 3;
+  // Metrics and thresholds that were used to determine load adjustments, as referenced in the
+  // BenchmarkResults.
+  repeated MetricSpecWithThreshold metric_thresholds = 4;
 }
diff --git a/api/adaptive_load/benchmark_result.proto b/api/adaptive_load/benchmark_result.proto
index 89630fc72..8f2dd73de 100644
--- a/api/adaptive_load/benchmark_result.proto
+++ b/api/adaptive_load/benchmark_result.proto
@@ -5,26 +5,23 @@ syntax = "proto3";
 
 package nighthawk.adaptive_load;
 
-import "api/adaptive_load/metric_spec.proto";
 import "api/client/output.proto";
 import "google/rpc/status.proto";
 
 // Records the status of a single metric during a benchmark session.
 message MetricEvaluation {
-  // Original metric specification (plugin name, metric name).
-  MetricSpec metric_spec = 1;
+  // Identifier for the metric that was evaluated (<plugin name>/<metric name>).
+  string metric_id = 1;
   // Numerical value of the metric measured during this benchmark session.
   double metric_value = 2;
-  // Threshold that was used to check this metric value. Not set if the metric
-  // was only informational.
-  ThresholdSpec threshold_spec = 3;
   // Score returned by a ScoringFunction plugin. This expresses how close the
   // metric was to the threshold by an arbitrary formula selected and
   // configured in the ThresholdSpec, such as a sigmoid curve. The controller
   // can choose to make larger input adjustments when the score is larger, in
-  // order to converge faster.
-  // Not set if the metric was only informational.
-  double threshold_score = 4;
+  // order to converge faster. Not set if the metric was only informational.
+  double threshold_score = 3;
+  // Configured weight of the metric. 0.0 if the metric was only informational.
+  double weight = 4;
 }
 
 // Summary of a single Nighthawk Service benchmark session with evaluation
diff --git a/api/adaptive_load/config.proto b/api/adaptive_load/config.proto
deleted file mode 100644
index f8ba7e26f..000000000
--- a/api/adaptive_load/config.proto
+++ /dev/null
@@ -1,57 +0,0 @@
-// Protos for selecting and configuring plugins for the adaptive load
-// controller infrastructure.
-
-syntax = "proto3";
-
-package nighthawk.adaptive_load;
-
-import "google/protobuf/any.proto";
-
-// Selects and configures a StepController plugin, which computes the next load
-// amount (e.g. RPS) and checks for convergence to the optimal load, based on
-// feedback metrics.
-//
-// For available plugin names, see step_controller_impl.proto.
-message StepControllerConfig {
-  // Name used to locate the plugin in the Envoy registry. Required.
-  string name = 1;
-  // Plugin-specific config. Required.
-  google.protobuf.Any typed_config = 2;
-}
-
-// Selects and configures a MetricsPlugin, which connects to a platform-specific
-// system to obtain metrics for the system under test.
-//
-// For available plugin names, see metrics_plugin_impl.proto.
-message MetricsPluginConfig {
-  // Name used to locate the plugin in the Envoy registry. Required.
-  string name = 1;
-  // Plugin-specific config such as an external API endpoint and credentials.
-  // Required.
-  google.protobuf.Any typed_config = 2;
-}
-
-// Selects and configures a ScoringFunction plugin, which measures proximity to
-// a threshold according to an arbitrary formula.
-//
-// For available plugin names, see scoring_function_impl.proto.
-message ScoringFunctionConfig {
-  // Name used to locate the plugin in the Envoy registry. Required.
-  string name = 1;
-  // Plugin-specific config such as thresholds and constants. Required.
-  google.protobuf.Any typed_config = 2;
-}
-
-// Selects and configures an InputVariableSetter plugin, which encapsulates
-// knowledge of how to apply a varying numeric input value to a
-// CommandLineOptions proto. For example, RequestsPerSecondInputVariableSetter
-// aka "rps" writes the value to the CommandLineOptions |requests_per_second|
-// field.
-//
-// For available plugin names, see input_variable_setter_impl.proto.
-message InputVariableSetterConfig {
-  // Name used to locate the plugin in the Envoy registry. Required.
-  string name = 1;
-  // Plugin-specific config. Required.
-  google.protobuf.Any typed_config = 2;
-}
diff --git a/api/adaptive_load/input_variable_setter_impl.proto b/api/adaptive_load/input_variable_setter_impl.proto
index 138c23603..5f421fa6d 100644
--- a/api/adaptive_load/input_variable_setter_impl.proto
+++ b/api/adaptive_load/input_variable_setter_impl.proto
@@ -4,7 +4,7 @@ syntax = "proto3";
 
 package nighthawk.adaptive_load;
 
-// Configuration for RequestsPerSecondInputVariableSetter (plugin name: "rps")
+// Configuration for RequestsPerSecondInputVariableSetter (plugin name: "nighthawk.rps")
 // that sets |requests_per_second| within CommandLineOptions to a numeric value
 // being varied by a StepController.
 message RequestsPerSecondInputVariableSetterConfig {
diff --git a/api/adaptive_load/metric_spec.proto b/api/adaptive_load/metric_spec.proto
index 5de8864a4..75f73a795 100644
--- a/api/adaptive_load/metric_spec.proto
+++ b/api/adaptive_load/metric_spec.proto
@@ -4,7 +4,7 @@ syntax = "proto3";
 
 package nighthawk.adaptive_load;
 
-import "api/adaptive_load/config.proto";
+import "envoy/config/core/v3/extension.proto";
 import "google/protobuf/wrappers.proto";
 import "validate/validate.proto";
 
@@ -13,7 +13,7 @@ message MetricSpec {
   // Name of the metric to evaluate. For the set of built-in metric names, see
   // source/adaptive_load/metrics_plugin_impl.cc. Required.
   string metric_name = 1 [(validate.rules).string.min_len = 1];
-  // Name of the MetricsPlugin providing the metric ("builtin" for built-in).
+  // Name of the MetricsPlugin providing the metric ("nighthawk.builtin" for built-in).
   // Required.
   string metrics_plugin_name = 2 [(validate.rules).string.min_len = 1];
 }
@@ -25,7 +25,8 @@ message ThresholdSpec {
   // the value is within the threshold so the input should ramp up, and
   // negative means the value is outside the threshold so input should ramp
   // down.
-  ScoringFunctionConfig scoring_function = 1 [(validate.rules).message.required = true];
+  envoy.config.core.v3.TypedExtensionConfig scoring_function = 1
+      [(validate.rules).message.required = true];
   // Relative importance of this threshold when adjusting based on multiple
   // metrics. Optional, default 1.0.
   google.protobuf.DoubleValue weight = 2 [(validate.rules).double.gt = 0.0];
diff --git a/api/adaptive_load/scoring_function_impl.proto b/api/adaptive_load/scoring_function_impl.proto
index ca22d9d3e..77b1505fa 100644
--- a/api/adaptive_load/scoring_function_impl.proto
+++ b/api/adaptive_load/scoring_function_impl.proto
@@ -6,7 +6,7 @@ package nighthawk.adaptive_load;
 
 import "google/protobuf/wrappers.proto";
 
-// Configuration for BinaryScoringFunction (plugin name: "binary") that is 1.0
+// Configuration for BinaryScoringFunction (plugin name: "nighthawk.binary") that is 1.0
 // when the value is within thresholds and -1.0 otherwise.
 message BinaryScoringFunctionConfig {
   // The minimum allowed value of the metric. Optional, default -infinity.
@@ -15,7 +15,7 @@ message BinaryScoringFunctionConfig {
   google.protobuf.DoubleValue upper_threshold = 2;
 }
 
-// Configuration for LinearScoringFunction (plugin name: "linear") that
+// Configuration for LinearScoringFunction (plugin name: "nighthawk.linear") that
 // calculates a metric score as k * (threshold - value), where k is a scaling
 // constant. The score is 0.0 when the value exactly equals the threshold,
 // positive below the threshold (meaning input should ramp up), and negative
@@ -30,7 +30,7 @@ message LinearScoringFunctionConfig {
   double k = 2;
 }
 
-// Configuration for SigmoidScoringFunction (plugin name: "sigmoid") that
+// Configuration for SigmoidScoringFunction (plugin name: "nighthawk.sigmoid") that
 // calculates a metric score as 1 - 2 / (1 + exp(-k(value - threshold))), an
 // upside-down sigmoid curve centered on a threshold. The output is 0.0 when the
 // metric equals the threshold, approaches 1.0 for values far below the
diff --git a/api/adaptive_load/step_controller_impl.proto b/api/adaptive_load/step_controller_impl.proto
index 7c138a215..35b24029d 100644
--- a/api/adaptive_load/step_controller_impl.proto
+++ b/api/adaptive_load/step_controller_impl.proto
@@ -4,10 +4,10 @@ syntax = "proto3";
 
 package nighthawk.adaptive_load;
 
-import "api/adaptive_load/config.proto";
+import "envoy/config/core/v3/extension.proto";
 
 // Configuration for ExponentialSearchStepController (plugin name:
-// "exponential-search") that performs an exponential search for the optimal
+// "nighthawk.exponential-search") that performs an exponential search for the optimal
 // value of a single Nighthawk input variable (e.g. RPS). Exponential search
 // starts with the input set to |initial_value| and increases the input by
 // |exponential_factor| until the metric goes outside thresholds at some input
@@ -18,7 +18,7 @@ message ExponentialSearchStepControllerConfig {
   // Selects a plugin that knows how to apply a numeric value generated by the
   // StepController within CommandLineOptions. Optional, defaults to "rps"
   // plugin, which sets |requests_per_second| in CommandLineOptions.
-  InputVariableSetterConfig input_variable_setter = 1;
+  envoy.config.core.v3.TypedExtensionConfig input_variable_setter = 1;
   // Initial value of the input variable that should be attempted. Required.
   double initial_value = 2;
   // Factor to increase the input variable during the exponential phase.

From 54630513def1bf306ad5023c9e7612b9034048e8 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Tue, 21 Jul 2020 21:14:19 -0400
Subject: [PATCH 25/34] create headers

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 include/nighthawk/adaptive_load/BUILD         | 24 +++++++++
 .../adaptive_load/adaptive_load_controller.h  | 26 ++++++++++
 .../adaptive_load/input_variable_setter.h     | 44 ++++++++++++++++
 .../nighthawk/adaptive_load/metrics_plugin.h  | 36 +++++++++++++
 .../adaptive_load/scoring_function.h          | 35 +++++++++++++
 .../nighthawk/adaptive_load/step_controller.h | 50 +++++++++++++++++++
 6 files changed, 215 insertions(+)
 create mode 100644 include/nighthawk/adaptive_load/BUILD
 create mode 100644 include/nighthawk/adaptive_load/adaptive_load_controller.h
 create mode 100644 include/nighthawk/adaptive_load/input_variable_setter.h
 create mode 100644 include/nighthawk/adaptive_load/metrics_plugin.h
 create mode 100644 include/nighthawk/adaptive_load/scoring_function.h
 create mode 100644 include/nighthawk/adaptive_load/step_controller.h

diff --git a/include/nighthawk/adaptive_load/BUILD b/include/nighthawk/adaptive_load/BUILD
new file mode 100644
index 000000000..e8454beec
--- /dev/null
+++ b/include/nighthawk/adaptive_load/BUILD
@@ -0,0 +1,24 @@
+load(
+    "@envoy//bazel:envoy_build_system.bzl",
+    "envoy_basic_cc_library",
+    "envoy_package",
+)
+
+licenses(["notice"])  # Apache 2
+
+envoy_package()
+
+envoy_basic_cc_library(
+    name = "adaptive_load_includes",
+    hdrs = [
+        "adaptive_load_controller.h",
+        "input_variable_setter.h",
+        "metrics_plugin.h",
+        "scoring_function.h",
+        "step_controller.h",
+    ],
+    include_prefix = "nighthawk/adaptive_load",
+    deps = [
+        "//api/adaptive_load:adaptive_load_proto_cc_proto",
+    ],
+)
diff --git a/include/nighthawk/adaptive_load/adaptive_load_controller.h b/include/nighthawk/adaptive_load/adaptive_load_controller.h
new file mode 100644
index 000000000..fb0d839b9
--- /dev/null
+++ b/include/nighthawk/adaptive_load/adaptive_load_controller.h
@@ -0,0 +1,26 @@
+#pragma once
+
+#include "envoy/common/time.h"
+
+#include "api/adaptive_load/adaptive_load.pb.h"
+#include "api/client/service.grpc.pb.h"
+
+namespace Nighthawk {
+namespace AdaptiveLoad {
+
+// Performs an adaptive load session defined by |spec| using the Nighthawk Service at
+// |nighthawk_service_stub|. The adaptive load session consists of the Adjusting Stage and the
+// Testing Stage. Adjusting Stage: Runs a series of short benchmarks, checks metrics according to
+// MetricSpecs, and adjusts load up or down based on the result; returns an error if convergence is
+// not detected before the deadline in the spec. Load adjustments and convergence detection are
+// computed by a StepController plugin. Metric values are obtained through MetricsPlugins. Testing
+// Stage: When the optimal load is found, runs one long benchmark to validate it. Progress messages
+// are written to |diagnostic_ostream| such as std::cerr or a std::ostringstream. |time_source| can
+// be an Envoy::Event::RealTimeSystem constructed from scratch. NO_CHECK_FORMAT(real_time)
+nighthawk::adaptive_load::AdaptiveLoadSessionOutput PerformAdaptiveLoadSession(
+    nighthawk::client::NighthawkService::StubInterface* nighthawk_service_stub,
+    const nighthawk::adaptive_load::AdaptiveLoadSessionSpec& spec, std::ostream& diagnostic_ostream,
+    Envoy::TimeSource& time_source);
+
+} // namespace AdaptiveLoad
+} // namespace Nighthawk
diff --git a/include/nighthawk/adaptive_load/input_variable_setter.h b/include/nighthawk/adaptive_load/input_variable_setter.h
new file mode 100644
index 000000000..0a1692541
--- /dev/null
+++ b/include/nighthawk/adaptive_load/input_variable_setter.h
@@ -0,0 +1,44 @@
+#pragma once
+
+#include "envoy/common/pure.h"
+#include "envoy/config/typed_config.h"
+
+#include "external/envoy/source/common/protobuf/protobuf.h"
+
+#include "envoy/config/core/v3/base.pb.h"
+#include "api/client/options.pb.h"
+
+namespace Nighthawk {
+namespace AdaptiveLoad {
+
+// An interface for plugins that apply a StepController-computed input value to a CommandLineOptions
+// proto. This may entail setting a numeric proto field directly, setting the value in a header, or
+// otherwise manipulating the proto to reflect the number.
+//
+// See source/adaptive_load/input_variable_setter_impl.h for example plugins.
+class InputVariableSetter {
+public:
+  virtual ~InputVariableSetter() = default;
+  // Applies the numeric input value to the |CommandLineOptions| object.
+  virtual void SetInputVariable(nighthawk::client::CommandLineOptions* command_line_options,
+                                double input_value) PURE;
+};
+
+using InputVariableSetterPtr = std::unique_ptr<InputVariableSetter>;
+
+// A factory that must be implemented for each InputVariableSetter plugin. It instantiates the
+// specific InputVariableSetter class after unpacking the optional plugin-specific config proto.
+class InputVariableSetterConfigFactory : public Envoy::Config::TypedFactory {
+public:
+  ~InputVariableSetterConfigFactory() override = default;
+  std::string category() const override { return "nighthawk.input_variable_setter"; }
+  // Instantiates the specific InputVariableSetter class. Casts |message| to Any, unpacks it to the
+  // plugin-specific proto, and passes the strongly typed proto to the constructor.
+  // If the plugin does not have a config proto, the constructor should not take an argument, and
+  // createInputVariableSetter() should ignore |message|.
+  virtual InputVariableSetterPtr
+  createInputVariableSetter(const Envoy::Protobuf::Message& message) PURE;
+};
+
+} // namespace AdaptiveLoad
+} // namespace Nighthawk
diff --git a/include/nighthawk/adaptive_load/metrics_plugin.h b/include/nighthawk/adaptive_load/metrics_plugin.h
new file mode 100644
index 000000000..46390d419
--- /dev/null
+++ b/include/nighthawk/adaptive_load/metrics_plugin.h
@@ -0,0 +1,36 @@
+#pragma once
+
+#include "envoy/common/pure.h"
+#include "envoy/config/typed_config.h"
+
+#include "envoy/config/core/v3/base.pb.h"
+
+namespace Nighthawk {
+namespace AdaptiveLoad {
+
+// An interface for plugins that retrieve platform-specific metrics from outside data sources.
+// Connection info is passed via a plugin-specific config proto.
+class MetricsPlugin {
+public:
+  virtual ~MetricsPlugin() = default;
+  // Obtain the numeric metric with the given name, usually by querying an outside system.
+  virtual double GetMetricByName(const std::string& metric_name) PURE;
+  // All metric names implemented by this plugin, for use in input validation.
+  virtual const std::vector<std::string> GetAllSupportedMetricNames() const PURE;
+};
+
+using MetricsPluginPtr = std::unique_ptr<MetricsPlugin>;
+
+// A factory that must be implemented for each MetricsPlugin. It instantiates the specific
+// MetricsPlugin class after unpacking the plugin-specific config proto.
+class MetricsPluginConfigFactory : public Envoy::Config::TypedFactory {
+public:
+  ~MetricsPluginConfigFactory() override = default;
+  std::string category() const override { return "nighthawk.metrics_plugin"; }
+  // Instantiates the specific MetricsPlugin class. Casts |message| to Any, unpacks it to the
+  // plugin-specific proto, and passes the strongly typed proto to the constructor.
+  virtual MetricsPluginPtr createMetricsPlugin(const Envoy::Protobuf::Message& message) PURE;
+};
+
+} // namespace AdaptiveLoad
+} // namespace Nighthawk
diff --git a/include/nighthawk/adaptive_load/scoring_function.h b/include/nighthawk/adaptive_load/scoring_function.h
new file mode 100644
index 000000000..258267938
--- /dev/null
+++ b/include/nighthawk/adaptive_load/scoring_function.h
@@ -0,0 +1,35 @@
+#pragma once
+
+#include "envoy/common/pure.h"
+#include "envoy/config/typed_config.h"
+
+#include "envoy/config/core/v3/base.pb.h"
+
+namespace Nighthawk {
+namespace AdaptiveLoad {
+
+// An interface for custom functions that measure a metric relative to a threshold.
+class ScoringFunction {
+public:
+  virtual ~ScoringFunction() = default;
+  // Returns a score of 0.0 if the metric is exactly the threshold, a positive score if the metric
+  // is below the threshold and load should be increased, and a negative score if the metric is
+  // above the threshold and load should be decreased.
+  virtual double EvaluateMetric(double value) const PURE;
+};
+
+using ScoringFunctionPtr = std::unique_ptr<ScoringFunction>;
+
+// A factory that must be implemented for each ScoringFunction plugin. It instantiates the
+// specific ScoringFunction class after unpacking the plugin-specific config proto.
+class ScoringFunctionConfigFactory : public Envoy::Config::TypedFactory {
+public:
+  ~ScoringFunctionConfigFactory() override = default;
+  std::string category() const override { return "nighthawk.scoring_function"; }
+  // Instantiates the specific ScoringFunction class. Casts |message| to Any, unpacks it to
+  // the plugin-specific proto, and passes the strongly typed proto to the constructor.
+  virtual ScoringFunctionPtr createScoringFunction(const Envoy::Protobuf::Message& message) PURE;
+};
+
+} // namespace AdaptiveLoad
+} // namespace Nighthawk
diff --git a/include/nighthawk/adaptive_load/step_controller.h b/include/nighthawk/adaptive_load/step_controller.h
new file mode 100644
index 000000000..d3a51c953
--- /dev/null
+++ b/include/nighthawk/adaptive_load/step_controller.h
@@ -0,0 +1,50 @@
+#pragma once
+
+#include "envoy/common/pure.h"
+#include "envoy/config/typed_config.h"
+
+#include "external/envoy/source/common/protobuf/protobuf.h"
+
+#include "api/adaptive_load/benchmark_result.pb.h"
+#include "envoy/config/core/v3/base.pb.h"
+
+namespace Nighthawk {
+namespace AdaptiveLoad {
+
+// An interface for custom StepControllers that compute load adjustments and check for convergence.
+class StepController {
+public:
+  virtual ~StepController() = default;
+  // Returns the current CommandLineOptions load specification that the StepController recommends.
+  virtual nighthawk::client::CommandLineOptions GetCurrentCommandLineOptions() const PURE;
+  // Reports if the search for the optimal load has converged, based on the StepController's
+  // internal state variables.
+  virtual bool IsConverged() const PURE;
+  // Reports if the algorithm has determined it can never succeed as configured, e.g. because
+  // metrics were outside thresholds at input values throughout the configured search range.
+  // If returning true, sets |doom_reason| to an explanation of why it can never succeed; otherwise
+  // does not touch |doom_reason|.
+  virtual bool IsDoomed(std::string* doom_reason) const PURE;
+  // Reports the result of the latest Nighthawk benchmark to the StepController so that the
+  // StepController can add data to its history (if any), recompute any internal state, and
+  // recompute its load recommendation.
+  virtual void UpdateAndRecompute(const nighthawk::adaptive_load::BenchmarkResult& result) PURE;
+};
+
+using StepControllerPtr = std::unique_ptr<StepController>;
+
+// A factory that must be implemented for each StepController plugin. It instantiates the
+// specific StepController class after unpacking the plugin-specific config proto.
+class StepControllerConfigFactory : public Envoy::Config::TypedFactory {
+public:
+  ~StepControllerConfigFactory() override = default;
+  std::string category() const override { return "nighthawk.step_controller"; }
+  // Instantiates the specific StepController class. Casts |message| to Any, unpacks it to the
+  // plugin-specific proto, and passes the strongly typed proto to the constructor.
+  virtual StepControllerPtr createStepController(
+      const Envoy::Protobuf::Message& message,
+      const nighthawk::client::CommandLineOptions& command_line_options_template) PURE;
+};
+
+} // namespace AdaptiveLoad
+} // namespace Nighthawk

From 46e0e2576af399bb1251d1336f223773200b4aa0 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Tue, 21 Jul 2020 21:42:51 -0400
Subject: [PATCH 26/34] fix format

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 include/nighthawk/adaptive_load/input_variable_setter.h | 2 +-
 include/nighthawk/adaptive_load/metrics_plugin.h        | 3 +--
 include/nighthawk/adaptive_load/scoring_function.h      | 3 +--
 include/nighthawk/adaptive_load/step_controller.h       | 2 +-
 4 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/include/nighthawk/adaptive_load/input_variable_setter.h b/include/nighthawk/adaptive_load/input_variable_setter.h
index 0a1692541..7d623fed2 100644
--- a/include/nighthawk/adaptive_load/input_variable_setter.h
+++ b/include/nighthawk/adaptive_load/input_variable_setter.h
@@ -1,11 +1,11 @@
 #pragma once
 
 #include "envoy/common/pure.h"
+#include "envoy/config/core/v3/base.pb.h"
 #include "envoy/config/typed_config.h"
 
 #include "external/envoy/source/common/protobuf/protobuf.h"
 
-#include "envoy/config/core/v3/base.pb.h"
 #include "api/client/options.pb.h"
 
 namespace Nighthawk {
diff --git a/include/nighthawk/adaptive_load/metrics_plugin.h b/include/nighthawk/adaptive_load/metrics_plugin.h
index 46390d419..798bb9f03 100644
--- a/include/nighthawk/adaptive_load/metrics_plugin.h
+++ b/include/nighthawk/adaptive_load/metrics_plugin.h
@@ -1,9 +1,8 @@
 #pragma once
 
 #include "envoy/common/pure.h"
-#include "envoy/config/typed_config.h"
-
 #include "envoy/config/core/v3/base.pb.h"
+#include "envoy/config/typed_config.h"
 
 namespace Nighthawk {
 namespace AdaptiveLoad {
diff --git a/include/nighthawk/adaptive_load/scoring_function.h b/include/nighthawk/adaptive_load/scoring_function.h
index 258267938..bf13ddcf4 100644
--- a/include/nighthawk/adaptive_load/scoring_function.h
+++ b/include/nighthawk/adaptive_load/scoring_function.h
@@ -1,9 +1,8 @@
 #pragma once
 
 #include "envoy/common/pure.h"
-#include "envoy/config/typed_config.h"
-
 #include "envoy/config/core/v3/base.pb.h"
+#include "envoy/config/typed_config.h"
 
 namespace Nighthawk {
 namespace AdaptiveLoad {
diff --git a/include/nighthawk/adaptive_load/step_controller.h b/include/nighthawk/adaptive_load/step_controller.h
index d3a51c953..85384a773 100644
--- a/include/nighthawk/adaptive_load/step_controller.h
+++ b/include/nighthawk/adaptive_load/step_controller.h
@@ -1,12 +1,12 @@
 #pragma once
 
 #include "envoy/common/pure.h"
+#include "envoy/config/core/v3/base.pb.h"
 #include "envoy/config/typed_config.h"
 
 #include "external/envoy/source/common/protobuf/protobuf.h"
 
 #include "api/adaptive_load/benchmark_result.pb.h"
-#include "envoy/config/core/v3/base.pb.h"
 
 namespace Nighthawk {
 namespace AdaptiveLoad {

From f6346429934b3b7d404d14d65ddeadbee54219fc Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Wed, 22 Jul 2020 13:50:40 -0400
Subject: [PATCH 27/34] use docstring format

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 .../adaptive_load/adaptive_load_controller.h  | 31 ++++++---
 .../adaptive_load/input_variable_setter.h     | 38 +++++++----
 .../nighthawk/adaptive_load/metrics_plugin.h  | 37 ++++++++---
 .../adaptive_load/scoring_function.h          | 35 +++++++---
 .../nighthawk/adaptive_load/step_controller.h | 66 ++++++++++++++-----
 5 files changed, 155 insertions(+), 52 deletions(-)

diff --git a/include/nighthawk/adaptive_load/adaptive_load_controller.h b/include/nighthawk/adaptive_load/adaptive_load_controller.h
index fb0d839b9..7c8078195 100644
--- a/include/nighthawk/adaptive_load/adaptive_load_controller.h
+++ b/include/nighthawk/adaptive_load/adaptive_load_controller.h
@@ -8,15 +8,28 @@
 namespace Nighthawk {
 namespace AdaptiveLoad {
 
-// Performs an adaptive load session defined by |spec| using the Nighthawk Service at
-// |nighthawk_service_stub|. The adaptive load session consists of the Adjusting Stage and the
-// Testing Stage. Adjusting Stage: Runs a series of short benchmarks, checks metrics according to
-// MetricSpecs, and adjusts load up or down based on the result; returns an error if convergence is
-// not detected before the deadline in the spec. Load adjustments and convergence detection are
-// computed by a StepController plugin. Metric values are obtained through MetricsPlugins. Testing
-// Stage: When the optimal load is found, runs one long benchmark to validate it. Progress messages
-// are written to |diagnostic_ostream| such as std::cerr or a std::ostringstream. |time_source| can
-// be an Envoy::Event::RealTimeSystem constructed from scratch. NO_CHECK_FORMAT(real_time)
+/**
+ * Performs an adaptive load session consisting of the Adjusting Stage and the
+ * Testing Stage. Adjusting Stage: Runs a series of short benchmarks, checks metrics according to
+ * MetricSpecs, and adjusts load up or down based on the result; returns an error if convergence is
+ * not detected before the deadline in the spec. Load adjustments and convergence detection are
+ * computed by a StepController plugin. Metric values are obtained through MetricsPlugins. Testing
+ * Stage: When the optimal load is found, runs one long benchmark to validate it.
+ *
+ * @param nighthawk_service_stub A Nighthawk Service gRPC stub.
+ * @param spec A proto that defines all aspects of the adaptive load session, including metrics,
+ * threshold, duration of adjusting stage benchmarks, and underlying Nighthawk traffic parameters.
+ * @param diagnostic_ostream A place to write progress messages, such as std::cerr to write to the
+ * console, a std::ostringstream to store the message in memory, or a std::ostream adapter around a
+ * custom logging system.
+ * @param time_source An abstraction of the system clock. Normally, just construct an
+ * Envoy::Event::RealTimeSystem and pass it. NO_CHECK_FORMAT(real_time). If calling from an
+ * Envoy-based process, there may be an existing TimeSource or TimeSystem to use. If calling
+ * from a test, pass a fake TimeSource.
+ *
+ * @return AdaptiveLoadSessionOutput a proto logging the result of all traffic attempted and all
+ * corresponding metric values and scores.
+ */
 nighthawk::adaptive_load::AdaptiveLoadSessionOutput PerformAdaptiveLoadSession(
     nighthawk::client::NighthawkService::StubInterface* nighthawk_service_stub,
     const nighthawk::adaptive_load::AdaptiveLoadSessionSpec& spec, std::ostream& diagnostic_ostream,
diff --git a/include/nighthawk/adaptive_load/input_variable_setter.h b/include/nighthawk/adaptive_load/input_variable_setter.h
index 7d623fed2..115c4e08e 100644
--- a/include/nighthawk/adaptive_load/input_variable_setter.h
+++ b/include/nighthawk/adaptive_load/input_variable_setter.h
@@ -11,31 +11,45 @@
 namespace Nighthawk {
 namespace AdaptiveLoad {
 
-// An interface for plugins that apply a StepController-computed input value to a CommandLineOptions
-// proto. This may entail setting a numeric proto field directly, setting the value in a header, or
-// otherwise manipulating the proto to reflect the number.
-//
-// See source/adaptive_load/input_variable_setter_impl.h for example plugins.
+/**
+ * An interface for plugins that apply a StepController-computed input value to a CommandLineOptions
+ * proto. This may entail setting a numeric proto field directly, setting the value in a header, or
+ * otherwise manipulating the proto to reflect the number.
+ *
+ * See source/adaptive_load/input_variable_setter_impl.h for example plugins.
+ */
 class InputVariableSetter {
 public:
   virtual ~InputVariableSetter() = default;
-  // Applies the numeric input value to the |CommandLineOptions| object.
+  /**
+   * Applies the numeric input value to the CommandLineOptions proto.
+   *
+   * @param command_line_options A Nighthawk Service CommandLineOptions load spec template with most
+   * fields already filled.
+   * @param input_value A value generated by a StepController to be applied to the next load spec.
+   */
   virtual void SetInputVariable(nighthawk::client::CommandLineOptions* command_line_options,
                                 double input_value) PURE;
 };
 
 using InputVariableSetterPtr = std::unique_ptr<InputVariableSetter>;
 
-// A factory that must be implemented for each InputVariableSetter plugin. It instantiates the
-// specific InputVariableSetter class after unpacking the optional plugin-specific config proto.
+/**
+ * A factory that must be implemented for each InputVariableSetter plugin. It instantiates the
+ * specific InputVariableSetter class after unpacking the plugin-specific config proto.
+ */
 class InputVariableSetterConfigFactory : public Envoy::Config::TypedFactory {
 public:
   ~InputVariableSetterConfigFactory() override = default;
   std::string category() const override { return "nighthawk.input_variable_setter"; }
-  // Instantiates the specific InputVariableSetter class. Casts |message| to Any, unpacks it to the
-  // plugin-specific proto, and passes the strongly typed proto to the constructor.
-  // If the plugin does not have a config proto, the constructor should not take an argument, and
-  // createInputVariableSetter() should ignore |message|.
+  /**
+   * Instantiates the specific InputVariableSetter class. Casts |message| to Any, unpacks it to the
+   * plugin-specific proto, and passes the strongly typed proto to the plugin constructor.
+   *
+   * @param message Any typed_config proto taken from the TypedExtensionConfig.
+   *
+   * @return InputVariableSetterPtr Pointer to the new plugin instance.
+   */
   virtual InputVariableSetterPtr
   createInputVariableSetter(const Envoy::Protobuf::Message& message) PURE;
 };
diff --git a/include/nighthawk/adaptive_load/metrics_plugin.h b/include/nighthawk/adaptive_load/metrics_plugin.h
index 798bb9f03..11413896d 100644
--- a/include/nighthawk/adaptive_load/metrics_plugin.h
+++ b/include/nighthawk/adaptive_load/metrics_plugin.h
@@ -7,27 +7,48 @@
 namespace Nighthawk {
 namespace AdaptiveLoad {
 
-// An interface for plugins that retrieve platform-specific metrics from outside data sources.
-// Connection info is passed via a plugin-specific config proto.
+/**
+ * An interface for plugins that retrieve platform-specific metrics from outside data sources.
+ * Connection info is passed via a plugin-specific config proto.
+ */
 class MetricsPlugin {
 public:
   virtual ~MetricsPlugin() = default;
-  // Obtain the numeric metric with the given name, usually by querying an outside system.
+  /**
+   * Obtains the numeric metric with the given name, usually by querying an outside system.
+   *
+   * @param metric_name The name of the metric to retrieve. Must be supported by the plugin.
+   *
+   * @return double The metric value, or 0.0 if the metric was unsupported or unavailable.
+   */
   virtual double GetMetricByName(const std::string& metric_name) PURE;
-  // All metric names implemented by this plugin, for use in input validation.
+  /**
+   * All metric names implemented by this plugin, for use in input validation.
+   *
+   * @return const std::vector<std::string> List of metric names that can be queried from this
+   * plugin.
+   */
   virtual const std::vector<std::string> GetAllSupportedMetricNames() const PURE;
 };
 
 using MetricsPluginPtr = std::unique_ptr<MetricsPlugin>;
 
-// A factory that must be implemented for each MetricsPlugin. It instantiates the specific
-// MetricsPlugin class after unpacking the plugin-specific config proto.
+/**
+ * A factory that must be implemented for each MetricsPlugin. It instantiates the specific
+ * MetricsPlugin class after unpacking the plugin-specific config proto.
+ */
 class MetricsPluginConfigFactory : public Envoy::Config::TypedFactory {
 public:
   ~MetricsPluginConfigFactory() override = default;
   std::string category() const override { return "nighthawk.metrics_plugin"; }
-  // Instantiates the specific MetricsPlugin class. Casts |message| to Any, unpacks it to the
-  // plugin-specific proto, and passes the strongly typed proto to the constructor.
+  /**
+   * Instantiates the specific MetricsPlugin class. Casts |message| to Any, unpacks it to the
+   * plugin-specific proto, and passes the strongly typed proto to the plugin constructor.
+   *
+   * @param message Any typed_config proto taken from the TypedExtensionConfig.
+   *
+   * @return MetricsPluginPtr Pointer to the new plugin instance.
+   */
   virtual MetricsPluginPtr createMetricsPlugin(const Envoy::Protobuf::Message& message) PURE;
 };
 
diff --git a/include/nighthawk/adaptive_load/scoring_function.h b/include/nighthawk/adaptive_load/scoring_function.h
index bf13ddcf4..be9bf74d9 100644
--- a/include/nighthawk/adaptive_load/scoring_function.h
+++ b/include/nighthawk/adaptive_load/scoring_function.h
@@ -7,26 +7,45 @@
 namespace Nighthawk {
 namespace AdaptiveLoad {
 
-// An interface for custom functions that measure a metric relative to a threshold.
+/**
+ * An interface for custom functions that score a metric relative to a threshold.
+ *
+ * See source/adaptive_load/scoring_function_impl.h for example plugins.
+ */
 class ScoringFunction {
 public:
   virtual ~ScoringFunction() = default;
-  // Returns a score of 0.0 if the metric is exactly the threshold, a positive score if the metric
-  // is below the threshold and load should be increased, and a negative score if the metric is
-  // above the threshold and load should be decreased.
+  /**
+   * Returns a score of 0.0 if the metric is exactly the threshold, a positive score if the metric
+   * is below the threshold and load should be increased, and a negative score if the metric is
+   * above the threshold and load should be decreased. The magnitude of the value is determined in a
+   * plugin-specific way, based on thresholds and other configuration.
+   *
+   * @param value The measurement to be scored.
+   *
+   * @return double The score of the measurement according to the formula of this plugin.
+   */
   virtual double EvaluateMetric(double value) const PURE;
 };
 
 using ScoringFunctionPtr = std::unique_ptr<ScoringFunction>;
 
-// A factory that must be implemented for each ScoringFunction plugin. It instantiates the
-// specific ScoringFunction class after unpacking the plugin-specific config proto.
+/**
+ * A factory that must be implemented for each ScoringFunction plugin. It instantiates the
+ * specific ScoringFunction class after unpacking the plugin-specific config proto.
+ */
 class ScoringFunctionConfigFactory : public Envoy::Config::TypedFactory {
 public:
   ~ScoringFunctionConfigFactory() override = default;
   std::string category() const override { return "nighthawk.scoring_function"; }
-  // Instantiates the specific ScoringFunction class. Casts |message| to Any, unpacks it to
-  // the plugin-specific proto, and passes the strongly typed proto to the constructor.
+  /**
+   * Instantiates the specific ScoringFunction class. Casts |message| to Any, unpacks it to the
+   * plugin-specific proto, and passes the strongly typed proto to the plugin constructor.
+   *
+   * @param message Any typed_config proto taken from the TypedExtensionConfig.
+   *
+   * @return ScoringFunctionPtr Pointer to the new plugin instance.
+   */
   virtual ScoringFunctionPtr createScoringFunction(const Envoy::Protobuf::Message& message) PURE;
 };
 
diff --git a/include/nighthawk/adaptive_load/step_controller.h b/include/nighthawk/adaptive_load/step_controller.h
index 85384a773..b85c96df1 100644
--- a/include/nighthawk/adaptive_load/step_controller.h
+++ b/include/nighthawk/adaptive_load/step_controller.h
@@ -11,36 +11,72 @@
 namespace Nighthawk {
 namespace AdaptiveLoad {
 
-// An interface for custom StepControllers that compute load adjustments and check for convergence.
+/**
+ * An interface for StepControllers that compute load adjustments and check for convergence.
+ *
+ * See source/adaptive_load/step_controller_impl.h for example plugins.
+ */
 class StepController {
 public:
   virtual ~StepController() = default;
-  // Returns the current CommandLineOptions load specification that the StepController recommends.
+  /**
+   * Returns the current CommandLineOptions load specification that the StepController recommends.
+   *
+   * @return CommandLineOptions The final product after applying all computed load variables via
+   * InputVariableSetter plugins to the stored CommandLineOptions template.
+   */
   virtual nighthawk::client::CommandLineOptions GetCurrentCommandLineOptions() const PURE;
-  // Reports if the search for the optimal load has converged, based on the StepController's
-  // internal state variables.
+  /**
+   * Reports if the search for the optimal load has converged, based on the StepController's
+   * internal state variables.
+   *
+   * @return bool Whether the load has converged.
+   */
   virtual bool IsConverged() const PURE;
-  // Reports if the algorithm has determined it can never succeed as configured, e.g. because
-  // metrics were outside thresholds at input values throughout the configured search range.
-  // If returning true, sets |doom_reason| to an explanation of why it can never succeed; otherwise
-  // does not touch |doom_reason|.
+  /** Reports if the algorithm has determined it can never succeed as configured, e.g. because
+   * metrics were outside thresholds at input values throughout the configured search range.
+   *
+   * @param doom_reason Pointer to a string to write the reason for being doomed. If returning true,
+   * an explanation of why success is impossible is written here; otherwise this string is not
+   * touched.
+   *
+   * @return bool true if the controller has determined convergence is impossible.
+   */
   virtual bool IsDoomed(std::string* doom_reason) const PURE;
-  // Reports the result of the latest Nighthawk benchmark to the StepController so that the
-  // StepController can add data to its history (if any), recompute any internal state, and
-  // recompute its load recommendation.
+  /**
+   * Reports the result of the latest Nighthawk benchmark to the StepController so that the
+   * StepController can add data to its history (if any), recompute any internal state, and
+   * recompute its load recommendation.
+   *
+   * @param result The result of running Nighthawk Service, calling any MetricsPlugins, and scoring
+   * all metrics against configured thresholds. Some StepFunction plugins will store this value in a
+   * history internally.
+   */
   virtual void UpdateAndRecompute(const nighthawk::adaptive_load::BenchmarkResult& result) PURE;
 };
 
 using StepControllerPtr = std::unique_ptr<StepController>;
 
-// A factory that must be implemented for each StepController plugin. It instantiates the
-// specific StepController class after unpacking the plugin-specific config proto.
+/**
+ * A factory that must be implemented for each StepController plugin. It instantiates the
+ * specific StepController class after unpacking the plugin-specific config proto.
+ */
 class StepControllerConfigFactory : public Envoy::Config::TypedFactory {
 public:
   ~StepControllerConfigFactory() override = default;
   std::string category() const override { return "nighthawk.step_controller"; }
-  // Instantiates the specific StepController class. Casts |message| to Any, unpacks it to the
-  // plugin-specific proto, and passes the strongly typed proto to the constructor.
+  /**
+   * Instantiates the specific StepController class. Casts |message| to Any, unpacks it to the
+   * plugin-specific proto, and passes the strongly typed proto to the plugin constructor.
+   *
+   * @param message Any typed_config proto taken from the TypedExtensionConfig.
+   * @param command_line_options_template A partially filled CommandLineOptions describing all
+   * aspects of the traffic not managed by this StepController. While running, this StepController
+   * will be asked repeatedly for a fully formed CommandLineOptions with some variables filled in
+   * dynamically, and this proto template is the basis for all such protos.
+   *
+   * @return StepControllerPtr Pointer to the new plugin instance.
+   */
   virtual StepControllerPtr createStepController(
       const Envoy::Protobuf::Message& message,
       const nighthawk::client::CommandLineOptions& command_line_options_template) PURE;

From 3c39faa2fc65016273e247cda4202cecd0a1f9d9 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Thu, 23 Jul 2020 09:27:55 -0400
Subject: [PATCH 28/34] fix typos in comments

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 include/nighthawk/adaptive_load/step_controller.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/include/nighthawk/adaptive_load/step_controller.h b/include/nighthawk/adaptive_load/step_controller.h
index b85c96df1..969e8e472 100644
--- a/include/nighthawk/adaptive_load/step_controller.h
+++ b/include/nighthawk/adaptive_load/step_controller.h
@@ -48,9 +48,9 @@ class StepController {
    * StepController can add data to its history (if any), recompute any internal state, and
    * recompute its load recommendation.
    *
-   * @param result The result of running Nighthawk Service, calling any MetricsPlugins, and scoring
-   * all metrics against configured thresholds. Some StepFunction plugins will store this value in a
-   * history internally.
+   * @param result The result of running a benchmark with Nighthawk Service, calling any
+   * MetricsPlugins, and scoring all metrics against configured thresholds. Some StepController
+   * plugins will store this value in a history internally.
    */
   virtual void UpdateAndRecompute(const nighthawk::adaptive_load::BenchmarkResult& result) PURE;
 };

From b9c8f2b15e7ad2c42e46cb29ee8ecc88659b0f63 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Thu, 23 Jul 2020 21:17:24 -0400
Subject: [PATCH 29/34] split build target, get rid of ostream, change
 InputValueSetter to use a reference, add file comments

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 include/nighthawk/adaptive_load/BUILD         | 42 ++++++++++++++++++-
 .../adaptive_load/adaptive_load_controller.h  |  9 ++--
 .../adaptive_load/input_variable_setter.h     |  4 +-
 .../nighthawk/adaptive_load/metrics_plugin.h  |  2 +
 .../adaptive_load/scoring_function.h          |  2 +
 .../nighthawk/adaptive_load/step_controller.h |  2 +
 6 files changed, 53 insertions(+), 8 deletions(-)

diff --git a/include/nighthawk/adaptive_load/BUILD b/include/nighthawk/adaptive_load/BUILD
index e8454beec..35260b9c7 100644
--- a/include/nighthawk/adaptive_load/BUILD
+++ b/include/nighthawk/adaptive_load/BUILD
@@ -9,12 +9,52 @@ licenses(["notice"])  # Apache 2
 envoy_package()
 
 envoy_basic_cc_library(
-    name = "adaptive_load_includes",
+    name = "adaptive_load_controller_include",
     hdrs = [
         "adaptive_load_controller.h",
+    ],
+    include_prefix = "nighthawk/adaptive_load",
+    deps = [
+        "//api/adaptive_load:adaptive_load_proto_cc_proto",
+    ],
+)
+
+envoy_basic_cc_library(
+    name = "input_variable_setter_include",
+    hdrs = [
         "input_variable_setter.h",
+    ],
+    include_prefix = "nighthawk/adaptive_load",
+    deps = [
+        "//api/adaptive_load:adaptive_load_proto_cc_proto",
+    ],
+)
+
+envoy_basic_cc_library(
+    name = "metrics_plugin_include",
+    hdrs = [
         "metrics_plugin.h",
+    ],
+    include_prefix = "nighthawk/adaptive_load",
+    deps = [
+        "//api/adaptive_load:adaptive_load_proto_cc_proto",
+    ],
+)
+
+envoy_basic_cc_library(
+    name = "scoring_function_include",
+    hdrs = [
         "scoring_function.h",
+    ],
+    include_prefix = "nighthawk/adaptive_load",
+    deps = [
+        "//api/adaptive_load:adaptive_load_proto_cc_proto",
+    ],
+)
+
+envoy_basic_cc_library(
+    name = "step_controller_include",
+    hdrs = [
         "step_controller.h",
     ],
     include_prefix = "nighthawk/adaptive_load",
diff --git a/include/nighthawk/adaptive_load/adaptive_load_controller.h b/include/nighthawk/adaptive_load/adaptive_load_controller.h
index 7c8078195..e90be5f6b 100644
--- a/include/nighthawk/adaptive_load/adaptive_load_controller.h
+++ b/include/nighthawk/adaptive_load/adaptive_load_controller.h
@@ -19,21 +19,18 @@ namespace AdaptiveLoad {
  * @param nighthawk_service_stub A Nighthawk Service gRPC stub.
  * @param spec A proto that defines all aspects of the adaptive load session, including metrics,
  * threshold, duration of adjusting stage benchmarks, and underlying Nighthawk traffic parameters.
- * @param diagnostic_ostream A place to write progress messages, such as std::cerr to write to the
- * console, a std::ostringstream to store the message in memory, or a std::ostream adapter around a
- * custom logging system.
  * @param time_source An abstraction of the system clock. Normally, just construct an
  * Envoy::Event::RealTimeSystem and pass it. NO_CHECK_FORMAT(real_time). If calling from an
  * Envoy-based process, there may be an existing TimeSource or TimeSystem to use. If calling
  * from a test, pass a fake TimeSource.
  *
  * @return AdaptiveLoadSessionOutput a proto logging the result of all traffic attempted and all
- * corresponding metric values and scores.
+ * corresponding metric values and scores. Any errors that occur will be recorded in the
+ * |session_status| field.
  */
 nighthawk::adaptive_load::AdaptiveLoadSessionOutput PerformAdaptiveLoadSession(
     nighthawk::client::NighthawkService::StubInterface* nighthawk_service_stub,
-    const nighthawk::adaptive_load::AdaptiveLoadSessionSpec& spec, std::ostream& diagnostic_ostream,
-    Envoy::TimeSource& time_source);
+    const nighthawk::adaptive_load::AdaptiveLoadSessionSpec& spec, Envoy::TimeSource& time_source);
 
 } // namespace AdaptiveLoad
 } // namespace Nighthawk
diff --git a/include/nighthawk/adaptive_load/input_variable_setter.h b/include/nighthawk/adaptive_load/input_variable_setter.h
index 115c4e08e..cae0b0dae 100644
--- a/include/nighthawk/adaptive_load/input_variable_setter.h
+++ b/include/nighthawk/adaptive_load/input_variable_setter.h
@@ -1,3 +1,5 @@
+// Interfaces for InputVariableSetter plugins and plugin factories.
+
 #pragma once
 
 #include "envoy/common/pure.h"
@@ -28,7 +30,7 @@ class InputVariableSetter {
    * fields already filled.
    * @param input_value A value generated by a StepController to be applied to the next load spec.
    */
-  virtual void SetInputVariable(nighthawk::client::CommandLineOptions* command_line_options,
+  virtual void SetInputVariable(nighthawk::client::CommandLineOptions& command_line_options,
                                 double input_value) PURE;
 };
 
diff --git a/include/nighthawk/adaptive_load/metrics_plugin.h b/include/nighthawk/adaptive_load/metrics_plugin.h
index 11413896d..0262bc773 100644
--- a/include/nighthawk/adaptive_load/metrics_plugin.h
+++ b/include/nighthawk/adaptive_load/metrics_plugin.h
@@ -1,3 +1,5 @@
+// Interfaces for MetricsPlugin plugins and plugin factories.
+
 #pragma once
 
 #include "envoy/common/pure.h"
diff --git a/include/nighthawk/adaptive_load/scoring_function.h b/include/nighthawk/adaptive_load/scoring_function.h
index be9bf74d9..079ecde35 100644
--- a/include/nighthawk/adaptive_load/scoring_function.h
+++ b/include/nighthawk/adaptive_load/scoring_function.h
@@ -1,3 +1,5 @@
+// Interfaces for ScoringFunction plugins and plugin factories.
+
 #pragma once
 
 #include "envoy/common/pure.h"
diff --git a/include/nighthawk/adaptive_load/step_controller.h b/include/nighthawk/adaptive_load/step_controller.h
index 969e8e472..15e4a3978 100644
--- a/include/nighthawk/adaptive_load/step_controller.h
+++ b/include/nighthawk/adaptive_load/step_controller.h
@@ -1,3 +1,5 @@
+// Interfaces for StepController plugins and plugin factories.
+
 #pragma once
 
 #include "envoy/common/pure.h"

From 5fc4db47fcf45da183ce45ea9d7d375c7e9a5051 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Sat, 25 Jul 2020 22:52:24 -0400
Subject: [PATCH 30/34] remove nested namespace, remove redundant _include in
 target names

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 include/nighthawk/adaptive_load/BUILD                  | 10 +++++-----
 .../nighthawk/adaptive_load/adaptive_load_controller.h |  2 --
 .../nighthawk/adaptive_load/input_variable_setter.h    |  2 --
 include/nighthawk/adaptive_load/metrics_plugin.h       |  2 --
 include/nighthawk/adaptive_load/scoring_function.h     |  2 --
 include/nighthawk/adaptive_load/step_controller.h      |  2 --
 6 files changed, 5 insertions(+), 15 deletions(-)

diff --git a/include/nighthawk/adaptive_load/BUILD b/include/nighthawk/adaptive_load/BUILD
index 35260b9c7..caf45f17c 100644
--- a/include/nighthawk/adaptive_load/BUILD
+++ b/include/nighthawk/adaptive_load/BUILD
@@ -9,7 +9,7 @@ licenses(["notice"])  # Apache 2
 envoy_package()
 
 envoy_basic_cc_library(
-    name = "adaptive_load_controller_include",
+    name = "adaptive_load_controller",
     hdrs = [
         "adaptive_load_controller.h",
     ],
@@ -20,7 +20,7 @@ envoy_basic_cc_library(
 )
 
 envoy_basic_cc_library(
-    name = "input_variable_setter_include",
+    name = "input_variable_setter",
     hdrs = [
         "input_variable_setter.h",
     ],
@@ -31,7 +31,7 @@ envoy_basic_cc_library(
 )
 
 envoy_basic_cc_library(
-    name = "metrics_plugin_include",
+    name = "metrics_plugin",
     hdrs = [
         "metrics_plugin.h",
     ],
@@ -42,7 +42,7 @@ envoy_basic_cc_library(
 )
 
 envoy_basic_cc_library(
-    name = "scoring_function_include",
+    name = "scoring_function",
     hdrs = [
         "scoring_function.h",
     ],
@@ -53,7 +53,7 @@ envoy_basic_cc_library(
 )
 
 envoy_basic_cc_library(
-    name = "step_controller_include",
+    name = "step_controller",
     hdrs = [
         "step_controller.h",
     ],
diff --git a/include/nighthawk/adaptive_load/adaptive_load_controller.h b/include/nighthawk/adaptive_load/adaptive_load_controller.h
index e90be5f6b..6ede4e315 100644
--- a/include/nighthawk/adaptive_load/adaptive_load_controller.h
+++ b/include/nighthawk/adaptive_load/adaptive_load_controller.h
@@ -6,7 +6,6 @@
 #include "api/client/service.grpc.pb.h"
 
 namespace Nighthawk {
-namespace AdaptiveLoad {
 
 /**
  * Performs an adaptive load session consisting of the Adjusting Stage and the
@@ -32,5 +31,4 @@ nighthawk::adaptive_load::AdaptiveLoadSessionOutput PerformAdaptiveLoadSession(
     nighthawk::client::NighthawkService::StubInterface* nighthawk_service_stub,
     const nighthawk::adaptive_load::AdaptiveLoadSessionSpec& spec, Envoy::TimeSource& time_source);
 
-} // namespace AdaptiveLoad
 } // namespace Nighthawk
diff --git a/include/nighthawk/adaptive_load/input_variable_setter.h b/include/nighthawk/adaptive_load/input_variable_setter.h
index cae0b0dae..8513c56d7 100644
--- a/include/nighthawk/adaptive_load/input_variable_setter.h
+++ b/include/nighthawk/adaptive_load/input_variable_setter.h
@@ -11,7 +11,6 @@
 #include "api/client/options.pb.h"
 
 namespace Nighthawk {
-namespace AdaptiveLoad {
 
 /**
  * An interface for plugins that apply a StepController-computed input value to a CommandLineOptions
@@ -56,5 +55,4 @@ class InputVariableSetterConfigFactory : public Envoy::Config::TypedFactory {
   createInputVariableSetter(const Envoy::Protobuf::Message& message) PURE;
 };
 
-} // namespace AdaptiveLoad
 } // namespace Nighthawk
diff --git a/include/nighthawk/adaptive_load/metrics_plugin.h b/include/nighthawk/adaptive_load/metrics_plugin.h
index 0262bc773..8fe10d985 100644
--- a/include/nighthawk/adaptive_load/metrics_plugin.h
+++ b/include/nighthawk/adaptive_load/metrics_plugin.h
@@ -7,7 +7,6 @@
 #include "envoy/config/typed_config.h"
 
 namespace Nighthawk {
-namespace AdaptiveLoad {
 
 /**
  * An interface for plugins that retrieve platform-specific metrics from outside data sources.
@@ -54,5 +53,4 @@ class MetricsPluginConfigFactory : public Envoy::Config::TypedFactory {
   virtual MetricsPluginPtr createMetricsPlugin(const Envoy::Protobuf::Message& message) PURE;
 };
 
-} // namespace AdaptiveLoad
 } // namespace Nighthawk
diff --git a/include/nighthawk/adaptive_load/scoring_function.h b/include/nighthawk/adaptive_load/scoring_function.h
index 079ecde35..2b01535fc 100644
--- a/include/nighthawk/adaptive_load/scoring_function.h
+++ b/include/nighthawk/adaptive_load/scoring_function.h
@@ -7,7 +7,6 @@
 #include "envoy/config/typed_config.h"
 
 namespace Nighthawk {
-namespace AdaptiveLoad {
 
 /**
  * An interface for custom functions that score a metric relative to a threshold.
@@ -51,5 +50,4 @@ class ScoringFunctionConfigFactory : public Envoy::Config::TypedFactory {
   virtual ScoringFunctionPtr createScoringFunction(const Envoy::Protobuf::Message& message) PURE;
 };
 
-} // namespace AdaptiveLoad
 } // namespace Nighthawk
diff --git a/include/nighthawk/adaptive_load/step_controller.h b/include/nighthawk/adaptive_load/step_controller.h
index 15e4a3978..07223c985 100644
--- a/include/nighthawk/adaptive_load/step_controller.h
+++ b/include/nighthawk/adaptive_load/step_controller.h
@@ -11,7 +11,6 @@
 #include "api/adaptive_load/benchmark_result.pb.h"
 
 namespace Nighthawk {
-namespace AdaptiveLoad {
 
 /**
  * An interface for StepControllers that compute load adjustments and check for convergence.
@@ -84,5 +83,4 @@ class StepControllerConfigFactory : public Envoy::Config::TypedFactory {
       const nighthawk::client::CommandLineOptions& command_line_options_template) PURE;
 };
 
-} // namespace AdaptiveLoad
 } // namespace Nighthawk

From 7e6a810085f5511e5713b63a5e82f35d36798ca8 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Sun, 28 Mar 2021 21:02:27 -0400
Subject: [PATCH 31/34] log troubleshooting tips on URI and failure predicate
 errors; add failure predicate for stream_resets

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 source/client/options_impl.cc |  1 +
 source/client/process_impl.cc | 24 ++++++++++++++++++++++--
 test/options_test.cc          |  3 +++
 3 files changed, 26 insertions(+), 2 deletions(-)

diff --git a/source/client/options_impl.cc b/source/client/options_impl.cc
index 5019df334..548d1c6c2 100644
--- a/source/client/options_impl.cc
+++ b/source/client/options_impl.cc
@@ -672,6 +672,7 @@ void OptionsImpl::setNonTrivialDefaults() {
   failure_predicates_["benchmark.http_4xx"] = 0;
   failure_predicates_["benchmark.http_5xx"] = 0;
   failure_predicates_["benchmark.pool_connection_failure"] = 0;
+  failure_predicates_["benchmark.stream_resets"] = 0;
   // Also, fail fast when a remote request source is specified that we can't connect to or otherwise
   // fails.
   failure_predicates_["requestsource.upstream_rq_5xx"] = 0;
diff --git a/source/client/process_impl.cc b/source/client/process_impl.cc
index 69318a662..d186d8345 100644
--- a/source/client/process_impl.cc
+++ b/source/client/process_impl.cc
@@ -598,7 +598,23 @@ bool ProcessImpl::runInternal(OutputCollector& collector, const std::vector<UriP
   StatisticFactoryImpl statistic_factory(options_);
   collector.addResult("global", mergeWorkerStatistics(workers_), counters,
                       total_execution_duration / workers_.size(), first_acquisition_time);
-  return counters.find("sequencer.failed_terminations") == counters.end();
+  if (counters.find("sequencer.failed_terminations") == counters.end()) {
+    return true;
+  } else {
+    ENVOY_LOG(
+        error,
+        "Terminated early because of a failure predicate. Check the output for problematic counter "
+        "values. The default Nighthawk failure predicates report failure if (1) Nighthawk could "
+        "not connect to the target (see 'benchmark.pool_connection_failure' counter; check the "
+        "address and port number, and try explicitly setting --address-family v4 or v6, especially "
+        "when using DNS; instead of localhost try 127.0.0.1 or ::1 explicitly), (2) the protocol "
+        "was not supported by the target (see 'benchmark.stream_resets' counter; check http/https "
+        "in the URI, --h2), (3) the target returned a 4xx or 5xx HTTP response code (see "
+        "'benchmark.http_4xx' and 'benchmark.http_5xx' counters; check the URI path and the server "
+        "config), or (4) a custom gRPC RequestSource failed. --failure-predicate can be used to "
+        "relax expectations.");
+    return false;
+  }
 }
 
 bool ProcessImpl::run(OutputCollector& collector) {
@@ -632,7 +648,11 @@ bool ProcessImpl::run(OutputCollector& collector) {
       tracing_uri->resolve(*dispatcher_,
                            Utility::translateFamilyOptionString(options_.addressFamily()));
     }
-  } catch (const UriException&) {
+  } catch (const UriException& ex) {
+    ENVOY_LOG(error,
+              "URI exception (for example, malformed URI syntax, bad "
+              "MultiTarget path, unresolvable host DNS): {}",
+              ex.what());
     return false;
   }
 
diff --git a/test/options_test.cc b/test/options_test.cc
index c0b847fd1..73a06ef92 100644
--- a/test/options_test.cc
+++ b/test/options_test.cc
@@ -297,6 +297,7 @@ TEST_P(RequestSourcePluginTestFixture, CreatesOptionsImplWithRequestSourceConfig
   // comparison below.
   EXPECT_EQ(1, command->mutable_failure_predicates()->erase("benchmark.http_4xx"));
   EXPECT_EQ(1, command->mutable_failure_predicates()->erase("benchmark.http_5xx"));
+  EXPECT_EQ(1, command->mutable_failure_predicates()->erase("benchmark.stream_resets"));
   EXPECT_EQ(1, command->mutable_failure_predicates()->erase("requestsource.upstream_rq_5xx"));
 
   // TODO(#433)
@@ -438,6 +439,7 @@ TEST_F(OptionsImplTest, TlsContext) {
   // comparison below.
   EXPECT_EQ(1, cmd->mutable_failure_predicates()->erase("benchmark.http_4xx"));
   EXPECT_EQ(1, cmd->mutable_failure_predicates()->erase("benchmark.http_5xx"));
+  EXPECT_EQ(1, cmd->mutable_failure_predicates()->erase("benchmark.stream_resets"));
   EXPECT_EQ(1, cmd->mutable_failure_predicates()->erase("requestsource.upstream_rq_5xx"));
   // TODO(#433)
   OptionsImpl options_from_proto(*cmd);
@@ -500,6 +502,7 @@ TEST_F(OptionsImplTest, MultiTarget) {
   // textual comparison below.
   EXPECT_EQ(1, cmd->mutable_failure_predicates()->erase("benchmark.http_4xx"));
   EXPECT_EQ(1, cmd->mutable_failure_predicates()->erase("benchmark.http_5xx"));
+  EXPECT_EQ(1, cmd->mutable_failure_predicates()->erase("benchmark.stream_resets"));
   EXPECT_EQ(1, cmd->mutable_failure_predicates()->erase("requestsource.upstream_rq_5xx"));
   // TODO(#433)
   OptionsImpl options_from_proto(*cmd);

From 8f163b4602a07c99798fc0e5bd544891f29badcc Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Sun, 28 Mar 2021 21:13:44 -0400
Subject: [PATCH 32/34] mention Nighthawk Service logs in Error 13 message

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 source/client/service_impl.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/client/service_impl.cc b/source/client/service_impl.cc
index 36554e096..0a90c02c1 100644
--- a/source/client/service_impl.cc
+++ b/source/client/service_impl.cc
@@ -42,7 +42,7 @@ void ServiceImpl::handleExecutionRequest(const nighthawk::client::ExecutionReque
     response.mutable_error_detail()->set_code(grpc::StatusCode::INTERNAL);
     // TODO(https://github.com/envoyproxy/nighthawk/issues/181): wire through error descriptions, so
     // we can do better here.
-    response.mutable_error_detail()->set_message("Unknown failure");
+    response.mutable_error_detail()->set_message("Unknown failure. See Nighthawk Service logs.");
   }
   *(response.mutable_output()) = output_collector.toProto();
   process.shutdown();

From 39a68adf9f9c40eaa9730d794070a6442a8894e1 Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Sun, 28 Mar 2021 21:22:27 -0400
Subject: [PATCH 33/34] return tips in Nighthawk Service response

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 source/client/service_impl.cc | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/source/client/service_impl.cc b/source/client/service_impl.cc
index 0a90c02c1..31b965105 100644
--- a/source/client/service_impl.cc
+++ b/source/client/service_impl.cc
@@ -42,7 +42,18 @@ void ServiceImpl::handleExecutionRequest(const nighthawk::client::ExecutionReque
     response.mutable_error_detail()->set_code(grpc::StatusCode::INTERNAL);
     // TODO(https://github.com/envoyproxy/nighthawk/issues/181): wire through error descriptions, so
     // we can do better here.
-    response.mutable_error_detail()->set_message("Unknown failure. See Nighthawk Service logs.");
+    response.mutable_error_detail()->set_message(
+        "Unknown failure. See Nighthawk Service logs. Make sure the URI is well formed and the DNS "
+        "name resolves (if applicable). Check the output for problematic counter values. The "
+        "default Nighthawk failure predicates report failure if (1) Nighthawk could not connect to "
+        "the target (see 'benchmark.pool_connection_failure' counter; check the address and port "
+        "number, and try explicitly setting --address-family v4 or v6, especially when using DNS; "
+        "instead of localhost try 127.0.0.1 or ::1 explicitly), (2) the protocol was not supported "
+        "by the target (see 'benchmark.stream_resets' counter; check http/https in the URI, --h2), "
+        "(3) the target returned a 4xx or 5xx HTTP response code (see 'benchmark.http_4xx' and "
+        "'benchmark.http_5xx' counters; check the URI path and the server config), or (4) a custom "
+        "gRPC RequestSource failed. To relax expectations, set explicit failure predicates in the "
+        "benchmark request.");
   }
   *(response.mutable_output()) = output_collector.toProto();
   process.shutdown();

From 5520728ef1bdfd2422dd2b352186511e609db69e Mon Sep 17 00:00:00 2001
From: eric846 <56563761+eric846@users.noreply.github.com>
Date: Thu, 1 Apr 2021 11:13:00 -0400
Subject: [PATCH 34/34] split large log text into small error and large info

Signed-off-by: eric846 <56563761+eric846@users.noreply.github.com>
---
 source/client/process_impl.cc | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/source/client/process_impl.cc b/source/client/process_impl.cc
index d186d8345..40c5859eb 100644
--- a/source/client/process_impl.cc
+++ b/source/client/process_impl.cc
@@ -601,18 +601,18 @@ bool ProcessImpl::runInternal(OutputCollector& collector, const std::vector<UriP
   if (counters.find("sequencer.failed_terminations") == counters.end()) {
     return true;
   } else {
+    ENVOY_LOG(error, "Terminated early because of a failure predicate.");
     ENVOY_LOG(
-        error,
-        "Terminated early because of a failure predicate. Check the output for problematic counter "
-        "values. The default Nighthawk failure predicates report failure if (1) Nighthawk could "
-        "not connect to the target (see 'benchmark.pool_connection_failure' counter; check the "
-        "address and port number, and try explicitly setting --address-family v4 or v6, especially "
-        "when using DNS; instead of localhost try 127.0.0.1 or ::1 explicitly), (2) the protocol "
-        "was not supported by the target (see 'benchmark.stream_resets' counter; check http/https "
-        "in the URI, --h2), (3) the target returned a 4xx or 5xx HTTP response code (see "
-        "'benchmark.http_4xx' and 'benchmark.http_5xx' counters; check the URI path and the server "
-        "config), or (4) a custom gRPC RequestSource failed. --failure-predicate can be used to "
-        "relax expectations.");
+        info,
+        "Check the output for problematic counter values. The default Nighthawk failure predicates "
+        "report failure if (1) Nighthawk could not connect to the target (see "
+        "'benchmark.pool_connection_failure' counter; check the address and port number, and try "
+        "explicitly setting --address-family v4 or v6, especially when using DNS; instead of "
+        "localhost try 127.0.0.1 or ::1 explicitly), (2) the protocol was not supported by the "
+        "target (see 'benchmark.stream_resets' counter; check http/https in the URI, --h2), (3) "
+        "the target returned a 4xx or 5xx HTTP response code (see 'benchmark.http_4xx' and "
+        "'benchmark.http_5xx' counters; check the URI path and the server config), or (4) a custom "
+        "gRPC RequestSource failed. --failure-predicate can be used to relax expectations.");
     return false;
   }
 }