From 3a37205c446dc9946ada9a449ff999877cafbcf6 Mon Sep 17 00:00:00 2001 From: Evan Krall Date: Wed, 11 Sep 2024 11:25:34 -0700 Subject: [PATCH 1/5] Remove unused forecasting.py and test_forecasting.py --- .../paasta_tools.autoscaling.forecasting.rst | 7 -- .../generated/paasta_tools.autoscaling.rst | 1 - paasta_tools/autoscaling/forecasting.py | 106 ------------------ tests/autoscaling/test_forecasting.py | 72 ------------ 4 files changed, 186 deletions(-) delete mode 100644 docs/source/generated/paasta_tools.autoscaling.forecasting.rst delete mode 100644 paasta_tools/autoscaling/forecasting.py delete mode 100644 tests/autoscaling/test_forecasting.py diff --git a/docs/source/generated/paasta_tools.autoscaling.forecasting.rst b/docs/source/generated/paasta_tools.autoscaling.forecasting.rst deleted file mode 100644 index f6a122733c..0000000000 --- a/docs/source/generated/paasta_tools.autoscaling.forecasting.rst +++ /dev/null @@ -1,7 +0,0 @@ -paasta\_tools.autoscaling.forecasting module -============================================ - -.. automodule:: paasta_tools.autoscaling.forecasting - :members: - :undoc-members: - :show-inheritance: diff --git a/docs/source/generated/paasta_tools.autoscaling.rst b/docs/source/generated/paasta_tools.autoscaling.rst index 45be7d2296..a8e885f073 100644 --- a/docs/source/generated/paasta_tools.autoscaling.rst +++ b/docs/source/generated/paasta_tools.autoscaling.rst @@ -7,7 +7,6 @@ Submodules .. toctree:: paasta_tools.autoscaling.autoscaling_service_lib - paasta_tools.autoscaling.forecasting paasta_tools.autoscaling.max_all_k8s_services paasta_tools.autoscaling.pause_service_autoscaler paasta_tools.autoscaling.utils diff --git a/paasta_tools/autoscaling/forecasting.py b/paasta_tools/autoscaling/forecasting.py deleted file mode 100644 index da3573e98b..0000000000 --- a/paasta_tools/autoscaling/forecasting.py +++ /dev/null @@ -1,106 +0,0 @@ -from paasta_tools.autoscaling.utils import get_autoscaling_component -from paasta_tools.autoscaling.utils import register_autoscaling_component -from paasta_tools.long_running_service_tools import ( - DEFAULT_UWSGI_AUTOSCALING_MOVING_AVERAGE_WINDOW, -) - - -FORECAST_POLICY_KEY = "forecast_policy" - - -def get_forecast_policy(name): - """ - Returns a forecast policy matching the given name. Only used by decision policies that try to forecast load, like - the proportional decision policy. - """ - return get_autoscaling_component(name, FORECAST_POLICY_KEY) - - -@register_autoscaling_component("current", FORECAST_POLICY_KEY) -def current_value_forecast_policy(historical_load, **kwargs): - """A prediction policy that assumes that the value any time in the future will be the same as the current value. - - :param historical_load: a list of (timestamp, value)s, where timestamp is a unix timestamp and value is load. - """ - return historical_load[-1][1] - - -def window_historical_load(historical_load, window_begin, window_end): - """Filter historical_load down to just the datapoints lying between times window_begin and window_end, inclusive.""" - filtered = [] - for timestamp, value in historical_load: - if timestamp >= window_begin and timestamp <= window_end: - filtered.append((timestamp, value)) - return filtered - - -def trailing_window_historical_load(historical_load, window_size): - window_end, _ = historical_load[-1] - window_begin = window_end - window_size - return window_historical_load(historical_load, window_begin, window_end) - - -@register_autoscaling_component("moving_average", FORECAST_POLICY_KEY) -def moving_average_forecast_policy( - historical_load, - moving_average_window_seconds=DEFAULT_UWSGI_AUTOSCALING_MOVING_AVERAGE_WINDOW, - **kwargs, -): - """Does a simple average of all historical load data points within the moving average window. Weights all data - points within the window equally.""" - - windowed_data = trailing_window_historical_load( - historical_load, moving_average_window_seconds - ) - windowed_values = [value for timestamp, value in windowed_data] - return sum(windowed_values) / len(windowed_values) - - -@register_autoscaling_component("linreg", FORECAST_POLICY_KEY) -def linreg_forecast_policy( - historical_load, - linreg_window_seconds, - linreg_extrapolation_seconds, - linreg_default_slope=0, - **kwargs, -): - """Does a linear regression on the load data within the last linreg_window_seconds. For every time delta in - linreg_extrapolation_seconds, forecasts the value at that time delta from now, and returns the maximum of these - predicted values. (With linear extrapolation, it doesn't make sense to forecast at more than two points, as the max - load will always be at the first or last time delta.) - - :param linreg_window_seconds: Consider all data from this many seconds ago until now. - :param linreg_extrapolation_seconds: A list of floats representing a number of seconds in the future at which to - predict the load. The highest prediction will be returned. - :param linreg_default_slope: If there is only one data point within the window, the equation for slope is undefined, - so we use this value (expressed in load/second) for prediction instead. Default is - 0. - - """ - - window = trailing_window_historical_load(historical_load, linreg_window_seconds) - - loads = [load for timestamp, load in window] - times = [timestamp for timestamp, load in window] - - mean_time = sum(times) / len(times) - mean_load = sum(loads) / len(loads) - - if len(window) > 1: - slope = sum((t - mean_time) * (l - mean_load) for t, l in window) / sum( - (t - mean_time) ** 2 for t in times - ) - else: - slope = linreg_default_slope - - intercept = mean_load - slope * mean_time - - def predict(timestamp): - return slope * timestamp + intercept - - if isinstance(linreg_extrapolation_seconds, (int, float)): - linreg_extrapolation_seconds = [linreg_extrapolation_seconds] - - now, _ = historical_load[-1] - forecasted_values = [predict(now + delta) for delta in linreg_extrapolation_seconds] - return max(forecasted_values) diff --git a/tests/autoscaling/test_forecasting.py b/tests/autoscaling/test_forecasting.py deleted file mode 100644 index e0a3fed1d6..0000000000 --- a/tests/autoscaling/test_forecasting.py +++ /dev/null @@ -1,72 +0,0 @@ -from paasta_tools.autoscaling import forecasting - - -def test_moving_average_forecast_policy(): - historical_load = [ - (1, 100), - (2, 120), - (3, 140), - (4, 160), - (5, 180), - (6, 200), - (7, 220), - ] - - assert 170 == forecasting.moving_average_forecast_policy( - historical_load, moving_average_window_seconds=5 - ) - assert 220 == forecasting.moving_average_forecast_policy( - historical_load, moving_average_window_seconds=0.5 - ) - - -def test_linreg_forecast_policy(): - historical_load = [ - (1, 100), - (2, 120), - (3, 140), - (4, 160), - (5, 180), - (6, 200), - (7, 220), - ] - - assert 220 == forecasting.linreg_forecast_policy( - historical_load, linreg_window_seconds=7, linreg_extrapolation_seconds=0 - ) - assert 1000 == forecasting.linreg_forecast_policy( - historical_load, linreg_window_seconds=7, linreg_extrapolation_seconds=39 - ) - - # We should handle the case where there's only 1 data point within the window. - assert 220 == forecasting.linreg_forecast_policy( - historical_load, linreg_window_seconds=0, linreg_extrapolation_seconds=0 - ) - assert 220 == forecasting.linreg_forecast_policy( - historical_load, linreg_window_seconds=0, linreg_extrapolation_seconds=10 - ) - assert 1000 == forecasting.linreg_forecast_policy( - historical_load, - linreg_window_seconds=0, - linreg_extrapolation_seconds=78, - linreg_default_slope=10, - ) - - historical_load_2 = [ - (1, 100), - (2, 100), - (3, 100), - (4, 100), - (5, 100), - (6, 100), - (1, 100), - (2, 200), - (3, 300), - (4, 400), - (5, 500), - (6, 600), - ] - - assert 350 == forecasting.linreg_forecast_policy( - historical_load_2, linreg_window_seconds=7, linreg_extrapolation_seconds=0 - ) From e1f3c16b5589969f2f97af1412a5012340deb485 Mon Sep 17 00:00:00 2001 From: Evan Krall Date: Wed, 11 Sep 2024 11:29:39 -0700 Subject: [PATCH 2/5] Remove unused autoscaling_component stuff, now that forecast_policy is gone. PAASTA-18298 --- paasta_tools/autoscaling/utils.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/paasta_tools/autoscaling/utils.py b/paasta_tools/autoscaling/utils.py index 3b5e18ba86..879324ddaf 100644 --- a/paasta_tools/autoscaling/utils.py +++ b/paasta_tools/autoscaling/utils.py @@ -12,35 +12,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from collections import defaultdict -from typing import Callable -from typing import Dict from typing import List from typing import Optional from typing import TypedDict -_autoscaling_components: Dict[str, Dict[str, Callable]] = defaultdict(dict) - - -def register_autoscaling_component(name, method_type): - def outer(autoscaling_method): - _autoscaling_components[method_type][name] = autoscaling_method - return autoscaling_method - - return outer - - -def get_autoscaling_component(name, method_type): - return _autoscaling_components[method_type][name] - - class MetricsProviderDict(TypedDict, total=False): type: str decision_policy: str setpoint: float desired_active_requests_per_replica: int - forecast_policy: Optional[str] moving_average_window_seconds: Optional[int] use_resource_metrics: bool prometheus_adapter_config: Optional[dict] From 078beef4582be5f8ff46d5167457796c2cefe42f Mon Sep 17 00:00:00 2001 From: Evan Krall Date: Wed, 11 Sep 2024 11:34:14 -0700 Subject: [PATCH 3/5] Remove references to forecast_policy from test_kubernetes_tools.py --- tests/test_kubernetes_tools.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_kubernetes_tools.py b/tests/test_kubernetes_tools.py index 93fb8fa72b..a4444627a3 100644 --- a/tests/test_kubernetes_tools.py +++ b/tests/test_kubernetes_tools.py @@ -2402,7 +2402,6 @@ def test_get_autoscaling_metric_spec_uwsgi_prometheus( { "type": METRICS_PROVIDER_UWSGI, "setpoint": 0.4, - "forecast_policy": "moving_average", "moving_average_window_seconds": 300, } ] @@ -2486,7 +2485,6 @@ def test_get_autoscaling_metric_spec_gunicorn_prometheus( { "type": METRICS_PROVIDER_GUNICORN, "setpoint": 0.5, - "forecast_policy": "moving_average", "moving_average_window_seconds": 300, } ] From 0d6fc13c495a005e27a28fef2acb568389806d4b Mon Sep 17 00:00:00 2001 From: Evan Krall Date: Wed, 11 Sep 2024 11:34:32 -0700 Subject: [PATCH 4/5] Remove forecast_policy from autoscaling_schema. PAASTA-18298 --- paasta_tools/cli/schemas/autoscaling_schema.json | 6 ------ 1 file changed, 6 deletions(-) diff --git a/paasta_tools/cli/schemas/autoscaling_schema.json b/paasta_tools/cli/schemas/autoscaling_schema.json index d24f311562..4169b9a9f7 100644 --- a/paasta_tools/cli/schemas/autoscaling_schema.json +++ b/paasta_tools/cli/schemas/autoscaling_schema.json @@ -25,12 +25,6 @@ "max_instances_alert_threshold": { "type": "number" }, - "forecast_policy": { - "enum": [ - "moving_average", - "current" - ] - }, "moving_average_window_seconds": { "type": "integer" }, From f0277b568d57ff5450ab10f11775cd46893ed248 Mon Sep 17 00:00:00 2001 From: Evan Krall Date: Wed, 11 Sep 2024 12:02:27 -0700 Subject: [PATCH 5/5] Remove references to decision_policy: bespoke, and update some docs around decision_policy to be a little more accurate. --- docs/source/autoscaling.rst | 29 +++++++++++----------- docs/source/yelpsoa_configs.rst | 5 ++-- paasta_tools/kubernetes_tools.py | 5 +++- paasta_tools/long_running_service_tools.py | 3 ++- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/docs/source/autoscaling.rst b/docs/source/autoscaling.rst index d4d10ab35e..7b9e9c3ee1 100644 --- a/docs/source/autoscaling.rst +++ b/docs/source/autoscaling.rst @@ -67,8 +67,11 @@ The currently available metrics providers are: :uwsgi: With the ``uwsgi`` metrics provider, Paasta will configure your pods to be scraped from your uWSGI master via its `stats server `_. + Setpoint refers to the worker utilization, which is the percentage of workers that are busy. We currently only support uwsgi stats on port 8889, and Prometheus will attempt to scrape that port. + You can specify ``moving_average_window_seconds`` (default ``1800``, or 30 minutes) to adjust how long of a time period your worker utilization is averaged over: set a smaller value to autoscale more quickly, or set a larger value to ignore spikes. + .. note:: If you have configured your service to use a non-default stats port (8889), PaaSTA will not scale your service correctly! @@ -78,15 +81,21 @@ The currently available metrics providers are: With the ``gunicorn`` metrics provider, Paasta will configure your pods to run an additional container with the `statsd_exporter `_ image. This sidecar will listen on port 9117 and receive stats from the gunicorn service. The ``statsd_exporter`` will translate the stats into Prometheus format, which Prometheus will scrape. + You can specify ``moving_average_window_seconds`` (default ``1800``, or 30 minutes) to adjust how long of a time period your worker utilization is averaged over: set a smaller value to autoscale more quickly, or set a larger value to ignore spikes. + :active-requests: With the ``active-requests`` metrics provider, Paasta will use Envoy metrics to scale your service based on the amount of incoming traffic. Note that, instead of using ``setpoint``, the active requests provider looks at the ``desired_active_requests_per_replica`` field of the autoscaling configuration to determine how to scale. + You can specify ``moving_average_window_seconds`` (default ``1800``, or 30 minutes) to adjust how long of a time period the number of active requests is averaged over: set a smaller value to autoscale more quickly, or set a larger value to ignore spikes. + :piscina: This metrics provider is only valid for the Yelp-internal server-side-rendering (SSR) service. With the ``piscina`` metrics provider, Paasta will scale your SSR instance based on how many Piscina workers are busy. + You can specify ``moving_average_window_seconds`` (default ``1800``, or 30 minutes) to adjust how long of a time period your worker utilization is averaged over: set a smaller value to autoscale more quickly, or set a larger value to ignore spikes. + :arbitrary_promql: The ``arbitrary_promql`` metrics provider allows you to specify any Prometheus query you want using the `Prometheus query language (promql) `. The autoscaler will attempt @@ -99,25 +108,17 @@ The currently available metrics providers are: Decision policies ^^^^^^^^^^^^^^^^^ -The currently available decicion policies are: - -:proportional: - (This is the default policy.) - Uses a simple proportional model to decide the correct number of instances - to scale to, i.e. if load is 110% of the setpoint, scales up by 10%. - - Extra parameters: - - :moving_average_window_seconds: - The number of seconds to load data points over in order to calculate the average. - Defaults to 1800s (30m). - Currently, this is only supported for ``metrics_provider: uwsgi``. - :bespoke: Allows a service author to implement their own autoscaling. This policy results in no HPA being configured. An external process should periodically decide how many replicas this service needs to run, and use the Paasta API to tell Paasta to scale. See the :ref:`How to create a custom (bespoke) autoscaling method` section for details. + This is most commonly used by the Kew autoscaler. + +:Anything other value: + The default autoscaling method. + Paasta will configure a Kubernetes HPA to scale the service based on the metrics providers and setpoints. + Using multiple metrics providers -------------------------------- diff --git a/docs/source/yelpsoa_configs.rst b/docs/source/yelpsoa_configs.rst index 6068f36f64..1aade1783c 100644 --- a/docs/source/yelpsoa_configs.rst +++ b/docs/source/yelpsoa_configs.rst @@ -393,8 +393,9 @@ instance MAY have: * ``type``: Which method the autoscaler will use to determine a service's utilization. Should be ``cpu``, ``uwsgi``, ``active-reqeusts``, ``piscina``, ``gunicorn``, or ``arbitrary_promql``. - * ``decision_policy``: Which method the autoscaler will use to determine when to autoscale a service. - Should be ``proportional`` or ``bespoke``. + * ``decision_policy``: If you want to autoscale with a separate system which calls ``paasta autoscale`` (such as the Kew autoscaler), rather than an HPA, then set this to ``bespoke`` on the first ``metrics_provider`` entry. + Otherwise, leave this unset. + (It does not make sense to have more than one ``metrics_provider`` with ``decision_policy: bespoke``.) * ``setpoint``: The target utilization (as measured by your ``metrics_provider``) that the autoscaler will try to achieve. Default value is 0.8. diff --git a/paasta_tools/kubernetes_tools.py b/paasta_tools/kubernetes_tools.py index 9f8730c7a0..e6fdf1cc31 100644 --- a/paasta_tools/kubernetes_tools.py +++ b/paasta_tools/kubernetes_tools.py @@ -888,7 +888,10 @@ def get_autoscaling_metric_spec( return None autoscaling_params = self.get_autoscaling_params() - if autoscaling_params["metrics_providers"][0]["decision_policy"] == "bespoke": + if ( + autoscaling_params["metrics_providers"][0].get("decision_policy", "") + == "bespoke" + ): return None min_replicas = self.get_min_instances() diff --git a/paasta_tools/long_running_service_tools.py b/paasta_tools/long_running_service_tools.py index 3106481202..4468623128 100644 --- a/paasta_tools/long_running_service_tools.py +++ b/paasta_tools/long_running_service_tools.py @@ -37,6 +37,8 @@ DEFAULT_AUTOSCALING_SETPOINT = 0.8 DEFAULT_DESIRED_ACTIVE_REQUESTS_PER_REPLICA = 1 + +# If you change any of these, make sure to update docs/source/autoscaling.rst DEFAULT_ACTIVE_REQUESTS_AUTOSCALING_MOVING_AVERAGE_WINDOW = 1800 DEFAULT_UWSGI_AUTOSCALING_MOVING_AVERAGE_WINDOW = 1800 DEFAULT_PISCINA_AUTOSCALING_MOVING_AVERAGE_WINDOW = 1800 @@ -345,7 +347,6 @@ def limit_instance_count(self, instances: int) -> int: def get_autoscaling_params(self) -> AutoscalingParamsDict: default_provider_params: MetricsProviderDict = { "type": METRICS_PROVIDER_CPU, - "decision_policy": "proportional", "setpoint": DEFAULT_AUTOSCALING_SETPOINT, }