From f994d5ddf24048b7b1006722ab4f705776877bef Mon Sep 17 00:00:00 2001 From: Abraham Chavez Date: Wed, 14 Aug 2024 15:06:04 -0700 Subject: [PATCH] [a 3/3] Announce maintenance in stable deployments (#5979) --- lambdas/service/app.py | 16 ++++ src/azul/maintenance.py | 173 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 187 insertions(+), 2 deletions(-) diff --git a/lambdas/service/app.py b/lambdas/service/app.py index 405652c806..cac2f920c0 100644 --- a/lambdas/service/app.py +++ b/lambdas/service/app.py @@ -73,6 +73,9 @@ from azul.logging import ( configure_app_logging, ) +from azul.maintenance import ( + MaintenanceController, +) from azul.openapi import ( application_json, format_description as fd, @@ -328,6 +331,10 @@ def manifest_controller(self) -> ManifestController: return self._service_controller(ManifestController, manifest_url_func=manifest_url_func) + @cached_property + def maintenance_controller(self) -> MaintenanceController: + return MaintenanceController() + def _service_controller(self, controller_cls: Type[C], **kwargs) -> C: file_url_func: FileUrlFunc = self.file_url return self._controller(controller_cls, @@ -870,6 +877,15 @@ def get_integrations(): body=json.dumps(body)) +@app.route( + '/maintenance/schedule', + methods=['GET'], + cors=True +) +def get_maintenance_schedule(): + return app.maintenance_controller.get_maintenance_schedule() + + @app.route( '/index/catalogs', methods=['GET'], diff --git a/src/azul/maintenance.py b/src/azul/maintenance.py index ded9cac205..8b0de50a72 100644 --- a/src/azul/maintenance.py +++ b/src/azul/maintenance.py @@ -7,13 +7,14 @@ Enum, auto, ) +import json from operator import ( attrgetter, ) from typing import ( Iterator, - Sequence, Self, + Sequence, ) import attrs @@ -23,14 +24,22 @@ ) from azul import ( + CatalogName, JSON, + cached_property, + config, reject, require, - CatalogName, ) from azul.collections import ( adict, ) +from azul.deployment import ( + aws, +) +from azul.service.storage_service import ( + StorageObjectNotFound, +) from azul.time import ( format_dcp2_datetime, parse_dcp2_datetime, @@ -165,6 +174,13 @@ def pending_events(self) -> list[MaintenanceEvent]: return self.events[i:] return [] + def past_events(self) -> list[MaintenanceEvent]: + return [ + e + for e in self.events + if e.actual_end is not None and e.actual_start is not None + ] + def active_event(self) -> MaintenanceEvent | None: return only(self._active_events()) @@ -193,6 +209,14 @@ def add_event(self, event: MaintenanceEvent): self.events = events raise + def adjust_event(self, additional_duration: timedelta): + event = self.active_event() + reject(event is None, 'No active event') + event.planned_duration += additional_duration + self._heal(event, iter(self.pending_events())) + assert self.active_event() is not None + return event + def cancel_event(self, index: int) -> MaintenanceEvent: event = self.pending_events()[index] self.events.remove(event) @@ -224,3 +248,148 @@ def _heal(self, next_event.planned_start = event.end event = next_event self.validate() + + +class MaintenanceService: + + @property + def bucket(self): + return aws.shared_bucket + + @property + def object_key(self): + return f'azul/{config.deployment_stage}/azul.json' + + @cached_property + def client(self): + return aws.s3 + + @property + def _create_empty_schedule(self): + schedule = { + "maintenance": { + "schedule": { + "events": [] + } + } + } + self.client.put_object(Bucket=self.bucket, + Key=self.object_key, + Body=json.dumps(schedule).encode()) + return schedule + + @property + def _get_schedule(self) -> JSON: + try: + response = self.client.get_object(Bucket=self.bucket, + Key=self.object_key) + except self.client.exceptions.NoSuchKey: + raise StorageObjectNotFound + else: + return json.loads(response['Body'].read()) + + def get_maintenance_schedule(self, + all: bool = False, + schedule: MaintenanceSchedule | None = None) -> JSON: + """ + Display's the schedule, of active and pending events + """ + if schedule is None: + schedule = self.get_schedule + active_event = schedule.active_event() + active_event = {} if active_event is None else {'active': active_event.to_json()} + events = schedule.pending_events() + if events is not None: + events = {'pending': list(pe.to_json() for pe in events)} + else: + events = {} + completed = {} + if all: + completed = {'completed': [pe.to_json() for pe in schedule.past_events()]} + + events = active_event | events | completed + return events + + @property + def get_schedule(self) -> MaintenanceSchedule: + schedule = self._get_schedule + schedule = MaintenanceSchedule.from_json(schedule['maintenance']['schedule']) + schedule.validate() + return schedule + + def put_schedule(self, schedule: MaintenanceSchedule): + schedule = schedule.to_json() + return self.client.put_object(Bucket=self.bucket, + Key=self.object_key, + Body=json.dumps({ + "maintenance": { + "schedule": schedule + } + }).encode()) + + def provision_event(self, + planned_start: str, + planned_duration: int, + description: str, + partial: list[str] | None = None, + degraded: list[str] | None = None, + unavailable: list[str] | None = None) -> MaintenanceEvent: + partial = [{ + 'kind': 'partial_responses', + 'affected_catalogs': partial + }] if partial is not None else [] + degraded = [{ + 'kind': 'degraded_performance', + 'affected_catalogs': degraded + }] if degraded is not None else [] + unavailable = [{ + 'kind': 'service_unavailable', + 'affected_catalogs': unavailable + }] if unavailable is not None else [] + impacts = [*partial, *degraded, *unavailable] + return MaintenanceEvent.from_json({ + 'planned_start': planned_start, + 'planned_duration': planned_duration, + 'description': description, + 'impacts': impacts + }) + + def add_event(self, event: MaintenanceEvent) -> JSON: + schedule = self.get_schedule + schedule.add_event(event) + self.put_schedule(schedule) + return self.get_maintenance_schedule(schedule=schedule) + + def cancel_event(self, index: int) -> JSON: + schedule = self.get_schedule + event = schedule.cancel_event(index) + self.put_schedule(schedule) + return event.to_json() + + def start_event(self) -> JSON: + schedule = self.get_schedule + event = schedule.start_event() + self.put_schedule(schedule) + return event.to_json() + + def end_event(self) -> JSON: + schedule = self.get_schedule + event = schedule.end_event() + self.put_schedule(schedule) + return event.to_json() + + def adjust_event(self, additional_duration: timedelta) -> JSON: + schedule = self.get_schedule + event = schedule.adjust_event(additional_duration) + self.put_schedule(schedule) + return event.to_json() + + +class MaintenanceController: + + @cached_property + def service(self) -> MaintenanceService: + return MaintenanceService() + + def get_maintenance_schedule(self): + return self.service.get_maintenance_schedule()