From 36083d92bb84b278ce16de817a45bfd6035bde8d Mon Sep 17 00:00:00 2001 From: Hannes Schmidt Date: Tue, 12 Mar 2024 17:01:54 -0700 Subject: [PATCH 1/3] [1/2] Announce maintenance in stable deployments (#5979) Initial patch by Hannes --- src/azul/maintenance.py | 225 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 src/azul/maintenance.py diff --git a/src/azul/maintenance.py b/src/azul/maintenance.py new file mode 100644 index 0000000000..4e56ed9270 --- /dev/null +++ b/src/azul/maintenance.py @@ -0,0 +1,225 @@ +from datetime import ( + UTC, + datetime, + timedelta, +) +from enum import ( + Enum, + auto, +) +import json +from operator import ( + attrgetter, +) +import sys +from typing import ( + Iterator, + Sequence, +) + +import attrs +from more_itertools import ( + flatten, + only, +) + +from azul import ( + JSON, + reject, + require, +) +from azul.collections import ( + adict, +) +from azul.time import ( + format_dcp2_datetime, + parse_dcp2_datetime, +) + + +class MaintenanceImpactKind(Enum): + partial_responses = auto() + degraded_performance = auto() + service_unavailable = auto() + + +@attrs.define +class MaintenanceImpact: + kind: MaintenanceImpactKind + affected_catalogs: list[str] + + @classmethod + def from_json(cls, impact: JSON): + return cls(kind=MaintenanceImpactKind[impact['kind']], + affected_catalogs=impact['affected_catalogs']) + + def to_json(self) -> JSON: + return dict(kind=self.kind.name, + affected_catalogs=self.affected_catalogs) + + def validate(self): + require(all(isinstance(c, str) and c for c in self.affected_catalogs), + 'Invalid catalog name/pattern') + require(all({0: True, 1: c[-1] == '*'}.get(c.count('*'), False) + for c in self.affected_catalogs), + 'Invalid catalog pattern') + + +@attrs.define +class MaintenanceEvent: + planned_start: datetime + planned_duration: timedelta + description: str + impacts: list[MaintenanceImpact] + actual_start: datetime | None + actual_end: datetime | None + + @classmethod + def from_json(cls, event: JSON): + return cls(planned_start=cls._parse_datetime(event['planned_start']), + planned_duration=timedelta(seconds=event['planned_duration']), + description=event['description'], + impacts=list(map(MaintenanceImpact.from_json, event['impacts'])), + actual_start=cls._parse_datetime(event.get('actual_start')), + actual_end=cls._parse_datetime(event.get('actual_end'))) + + def to_json(self) -> JSON: + result = adict(planned_start=self._format_datetime(self.planned_start), + planned_duration=int(self.planned_duration.total_seconds()), + description=self.description, + impacts=[i.to_json() for i in self.impacts], + actual_start=self._format_datetime(self.actual_start), + actual_end=self._format_datetime(self.actual_end)) + return result + + @classmethod + def _parse_datetime(cls, value: str | None) -> datetime | None: + return None if value is None else parse_dcp2_datetime(value) + + @classmethod + def _format_datetime(cls, value: datetime | None) -> str | None: + return None if value is None else format_dcp2_datetime(value) + + @property + def start(self): + return self.actual_start or self.planned_start + + @property + def end(self): + return self.actual_end or self.start + self.planned_duration + + def validate(self): + require(isinstance(self.planned_start, datetime), + 'No planned start', self.planned_start) + require(self.planned_start.tzinfo == UTC) + require(isinstance(self.description, str) and self.description, + 'Invalid description', self.description) + for impact in self.impacts: + impact.validate() + reject(self.actual_end is not None and self.actual_start is None, + 'Event has end but no start') + require(self.start < self.end, + 'Event start is not before end') + + +@attrs.define +class MaintenanceSchedule: + events: list[MaintenanceEvent] + + @classmethod + def from_json(cls, schedule: JSON): + return cls(events=list(map(MaintenanceEvent.from_json, schedule['events']))) + + def to_json(self) -> JSON: + return dict(events=[e.to_json() for e in self.events]) + + def validate(self): + for e in self.events: + e.validate() + starts = set(e.start for e in self.events) + require(len(starts) == len(self.events), + 'Start times are not distinct') + # Since starts are distinct, we'll never need the end as a tie breaker + intervals = [(e.start, e.end) for e in self.events] + require(intervals == sorted(intervals), + 'Events are not sorted by start time') + values = list(flatten(intervals)) + require(values == sorted(values), + 'Events overlap', values) + reject(len(self._active_events()) > 1, + 'More than one active event') + require(all(e.actual_start is None for e in self.pending_events()), + 'Active event mixed in with pending ones') + + def pending_events(self) -> list[MaintenanceEvent]: + """ + Returns a list of pending events in this schedule. The elements in the + returned list can be modified until another method is invoked on this schedule. The + modifications will be reflected in ``self.events`` but the caller is + responsible for ensuring they don't invalidate this schedule. + """ + events = enumerate(self.events) + for i, e in events: + if e.actual_start is None: + return self.events[i:] + return [] + + def active_event(self) -> MaintenanceEvent | None: + return only(self._active_events()) + + def _active_events(self) -> Sequence[MaintenanceEvent]: + return [ + e + for e in self.events + if e.actual_start is not None and e.actual_end is None + ] + + def _now(self): + return datetime.now(UTC) + + def add_event(self, event: MaintenanceEvent): + """ + Add the given event to this schedule unless doing so would invalidate + this schedule. + """ + events = self.events + try: + self.events = events.copy() + self.events.append(event) + self.events.sort(key=attrgetter('start')) + self.validate() + except BaseException: + self.events = events + raise + + def cancel_event(self, index: int) -> MaintenanceEvent: + event = self.pending_events()[index] + self.events.remove(event) + self.validate() + return event + + def start_event(self) -> MaintenanceEvent: + pending = iter(self.pending_events()) + event = next(pending, None) + reject(event is None, 'No events pending to be started') + event.actual_start = self._now() + self._heal(event, pending) + assert event == self.active_event() + return event + + def end_event(self) -> MaintenanceEvent: + event = self.active_event() + reject(event is None, 'No active event') + event.actual_end = self._now() + self._heal(event, iter(self.pending_events())) + assert self.active_event() is None + return event + + def _heal(self, + event: MaintenanceEvent, + pending: Iterator[MaintenanceEvent]): + for next_event in pending: + if next_event.planned_start < event.end: + next_event.planned_start = event.end + event = next_event + self.validate() From 91bca25d7963311a80e72e632520223769152f36 Mon Sep 17 00:00:00 2001 From: Abraham Chavez Date: Wed, 21 Aug 2024 12:41:57 -0700 Subject: [PATCH 2/3] [a 2/2] Announce maintenance in stable deployments (#5979) --- lambdas/service/app.py | 41 ++++++++++ lambdas/service/openapi.json | 13 +++ scripts/manage_maintenance.py | 142 ++++++++++++++++++++++++++++++++ src/azul/maintenance.py | 148 +++++++++++++++++++++++++++++++--- test/test_doctests.py | 1 + 5 files changed, 336 insertions(+), 9 deletions(-) create mode 100644 scripts/manage_maintenance.py diff --git a/lambdas/service/app.py b/lambdas/service/app.py index 405652c806..3334b120fb 100644 --- a/lambdas/service/app.py +++ b/lambdas/service/app.py @@ -73,6 +73,9 @@ from azul.logging import ( configure_app_logging, ) +from azul.maintenance import ( + MaintenanceService, +) from azul.openapi import ( application_json, format_description as fd, @@ -870,6 +873,44 @@ def get_integrations(): body=json.dumps(body)) +@app.route( + '/maintenance/schedule', + methods=['GET'], + cors=True, + method_spec={ + 'summary': 'A maintenance schedule as an JSON object', + 'tags': ['Auxiliary'], + 'responses': { + '200': { + 'description': fd(''' + This object may be hanceforth refered to as "the schedule" + or `schedule`. + The `start` time of an event is its `actual_start` if set, + or its `planned_start` otherwise. The `end` time of an event + is its `actual_end` if set, or its `start` plus + `planned_duration` otherwise. All events in the schedule are + sorted by their `start` time. No two events have the same + `start` time. Each event defines an interval + `[e.start, e.end)` and there is no overlap between these + intervals. + + A pending event is one where `actual_start` is absent. An + active event is one where `actual_start` is present but + `actual_end` is absent. There can be at most one active + event. + ''') + } + } + } +) +def get_maintenance_schedule(): + service = MaintenanceService() + schedule = service.get_schedule.to_json() + return Response(status_code=200, + headers={'content-type': 'application/json'}, + body=json.dumps(schedule)) + + @app.route( '/index/catalogs', methods=['GET'], diff --git a/lambdas/service/openapi.json b/lambdas/service/openapi.json index 2e75a0f1da..0c9de5a5a2 100644 --- a/lambdas/service/openapi.json +++ b/lambdas/service/openapi.json @@ -653,6 +653,19 @@ } } }, + "/maintenance/schedule": { + "get": { + "summary": "A maintenance schedule as an JSON object", + "tags": [ + "Auxiliary" + ], + "responses": { + "200": { + "description": "\nThis object may be hanceforth refered to as \"the schedule\"\nor `schedule`.\nThe `start` time of an event is its `actual_start` if set,\nor its `planned_start` otherwise. The `end` time of an event\nis its `actual_end` if set, or its `start` plus\n`planned_duration` otherwise. All events in the schedule are\nsorted by their `start` time. No two events have the same\n`start` time. Each event defines an interval\n`[e.start, e.end)` and there is no overlap between these\n intervals.\n\n A pending event is one where `actual_start` is absent. An\n active event is one where `actual_start` is present but\n `actual_end` is absent. There can be at most one active\n event.\n" + } + } + } + }, "/index/catalogs": { "get": { "summary": "List all available catalogs.", diff --git a/scripts/manage_maintenance.py b/scripts/manage_maintenance.py new file mode 100644 index 0000000000..6654f4e57c --- /dev/null +++ b/scripts/manage_maintenance.py @@ -0,0 +1,142 @@ +""" +This is a command line utility for managing announcement of maintenance events. +Reads the JSON from designated bucket, deserializes the model from it, validates +the model, applies an action to it, serializes the model back to JSON and +finally uploads it back to the bucket where the service exposes it. The service +must also validate the model before returning it. +""" +import argparse +from datetime import ( + timedelta, +) +import json +import re +import sys + +from azul import ( + require, +) +from azul.args import ( + AzulArgumentHelpFormatter, +) +from azul.maintenance import ( + MaintenanceService, +) + + +def parse_duration(duration: str) -> timedelta: + """ + >>> parse_duration('1d') + datetime.timedelta(days=1) + >>> parse_duration('24 hours') + datetime.timedelta(days=1) + >>> parse_duration('.5 Days 12 hours') + datetime.timedelta(days=1) + + >>> parse_duration('2h20Min') + datetime.timedelta(seconds=8400) + >>> parse_duration('1 H 80m') + datetime.timedelta(seconds=8400) + >>> parse_duration('140 Minutes') + datetime.timedelta(seconds=8400) + + >>> parse_duration('2 Days 3hours 4min 5 secs') + datetime.timedelta(days=2, seconds=11045) + >>> parse_duration('1d 25h') + datetime.timedelta(days=2, seconds=3600) + >>> parse_duration('1m30s') + datetime.timedelta(seconds=90) + + >>> parse_duration('Bad foo') + Traceback (most recent call last): + ... + azul.RequirementError: Try a duration such as "2d 6hrs", "1.5 Days", or "15m" + """ + + pattern = r'(\d*\.?\d+)\s*(d|h|m|s)' + matches = re.findall(pattern, duration.lower()) + require(bool(matches), 'Try a duration such as "2d 6hrs", "1.5 Days", or "15m"') + time_delta = {'days': 0, 'hours': 0, 'minutes': 0, 'seconds': 0} + for value, unit in matches: + value = float(value) + match unit: + case 'd': + time_delta['days'] += value + case 'h': + time_delta['hours'] += value + case 'm': + time_delta['minutes'] += value + case 's': + time_delta['seconds'] += value + return timedelta(**time_delta) + + +def main(args: list[str]): + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=AzulArgumentHelpFormatter) + subparsers = parser.add_subparsers(dest="command") + list_parser = subparsers.add_parser("list", help="List events in JSON form") + list_parser.add_argument("--all", action="store_true", + help="Include completed events") + add_parser = subparsers.add_parser("add", help="Schedule an event") + add_parser.add_argument("--start", required=True, + help="Event start time (ISO format)") + add_parser.add_argument("--duration", required=True, + help="Event duration (e.g., '1h30m', '2d')") + add_parser.add_argument("--description", required=True, + help="Event description") + add_parser.add_argument("--partial-responses", nargs="+", + help="Catalog names for partial responses") + add_parser.add_argument("--degraded-performance", nargs="+", + help="Catalog names for degraded performance") + add_parser.add_argument("--service-unavailable", nargs="+", + help="Catalog names for service unavailability") + cancel_parser = subparsers.add_parser("cancel", + help="Cancel a pending event") + cancel_parser.add_argument("--index", type=int, required=True, + help="Index of the event to cancel") + subparsers.add_parser("start", help="Activate a pending event") + subparsers.add_parser("end", help="Complete the active event") + adjust_parser = subparsers.add_parser("adjust", + help="Modify the active event") + adjust_parser.add_argument("--duration", required=True, + help="New event duration (e.g., '1h30m', '2d')") + + args = parser.parse_args(args) + + service = MaintenanceService() + + if args.command == "list": + events = service.get_schedule + if args.all: + events = events.to_json() + else: + active = events.active_event() + active = {} if active is None else {'active': active.to_json()} + pending = events.pending_events() + pending = {'pending': list(pe.to_json() for pe in pending)} + events = active | pending + elif args.command == "add": + duration = int(parse_duration(args.duration).total_seconds()) + events = service.provision_event(planned_start=args.start, + planned_duration=duration, + description=args.description, + partial=args.partial_responses, + degraded=args.degraded_performance, + unavailable=args.service_unavailable) + events = service.add_event(events) + elif args.command == "cancel": + events = service.cancel_event(args.index) + elif args.command == "start": + events = service.start_event() + elif args.command == "end": + events = service.end_event() + elif args.command == "adjust": + events = service.adjust_event(parse_duration(args.duration)) + else: + assert False, 'Invalid command' + print(json.dumps(events, indent=4)) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/src/azul/maintenance.py b/src/azul/maintenance.py index 4e56ed9270..77de486e86 100644 --- a/src/azul/maintenance.py +++ b/src/azul/maintenance.py @@ -11,9 +11,9 @@ from operator import ( attrgetter, ) -import sys from typing import ( Iterator, + Self, Sequence, ) @@ -24,13 +24,22 @@ ) from azul import ( + CatalogName, JSON, + cached_property, + config, reject, require, ) from azul.collections import ( adict, ) +from azul.deployment import ( + aws, +) +from azul.service.storage_service import ( + StorageObjectNotFound, +) from azul.time import ( format_dcp2_datetime, parse_dcp2_datetime, @@ -46,7 +55,7 @@ class MaintenanceImpactKind(Enum): @attrs.define class MaintenanceImpact: kind: MaintenanceImpactKind - affected_catalogs: list[str] + affected_catalogs: list[CatalogName] @classmethod def from_json(cls, impact: JSON): @@ -58,8 +67,9 @@ def to_json(self) -> JSON: affected_catalogs=self.affected_catalogs) def validate(self): - require(all(isinstance(c, str) and c for c in self.affected_catalogs), - 'Invalid catalog name/pattern') + require(all( + isinstance(c, CatalogName) and c for c in self.affected_catalogs), + 'Invalid catalog name/pattern') require(all({0: True, 1: c[-1] == '*'}.get(c.count('*'), False) for c in self.affected_catalogs), 'Invalid catalog pattern') @@ -75,7 +85,7 @@ class MaintenanceEvent: actual_end: datetime | None @classmethod - def from_json(cls, event: JSON): + def from_json(cls, event: JSON) -> Self: return cls(planned_start=cls._parse_datetime(event['planned_start']), planned_duration=timedelta(seconds=event['planned_duration']), description=event['description'], @@ -139,7 +149,7 @@ def validate(self): starts = set(e.start for e in self.events) require(len(starts) == len(self.events), 'Start times are not distinct') - # Since starts are distinct, we'll never need the end as a tie breaker + # Since starts are distinct, we'll never need the end as a tie-breaker intervals = [(e.start, e.end) for e in self.events] require(intervals == sorted(intervals), 'Events are not sorted by start time') @@ -154,9 +164,9 @@ def validate(self): def pending_events(self) -> list[MaintenanceEvent]: """ Returns a list of pending events in this schedule. The elements in the - returned list can be modified until another method is invoked on this schedule. The - modifications will be reflected in ``self.events`` but the caller is - responsible for ensuring they don't invalidate this schedule. + returned list can be modified until another method is invoked on this + schedule. The modifications will be reflected in ``self.events`` but the + caller is responsible for ensuring they don't invalidate this schedule. """ events = enumerate(self.events) for i, e in events: @@ -164,6 +174,13 @@ def pending_events(self) -> list[MaintenanceEvent]: return self.events[i:] return [] + def past_events(self) -> list[MaintenanceEvent]: + return [ + e + for e in self.events + if e.actual_end is not None and e.actual_start is not None + ] + def active_event(self) -> MaintenanceEvent | None: return only(self._active_events()) @@ -192,6 +209,14 @@ def add_event(self, event: MaintenanceEvent): self.events = events raise + def adjust_event(self, additional_duration: timedelta): + event = self.active_event() + reject(event is None, 'No active event') + event.planned_duration += additional_duration + self._heal(event, iter(self.pending_events())) + assert self.active_event() is not None + return event + def cancel_event(self, index: int) -> MaintenanceEvent: event = self.pending_events()[index] self.events.remove(event) @@ -223,3 +248,108 @@ def _heal(self, next_event.planned_start = event.end event = next_event self.validate() + + +class MaintenanceService: + + @property + def bucket(self): + return aws.shared_bucket + + @property + def object_key(self): + return f'azul/{config.deployment_stage}/azul.json' + + @cached_property + def client(self): + return aws.s3 + + @property + def _get_schedule(self) -> JSON: + try: + response = self.client.get_object(Bucket=self.bucket, + Key=self.object_key) + except self.client.exceptions.NoSuchKey: + raise StorageObjectNotFound + else: + return json.loads(response['Body'].read()) + + @property + def get_schedule(self) -> MaintenanceSchedule: + schedule = self._get_schedule + schedule = MaintenanceSchedule.from_json(schedule['maintenance']['schedule']) + schedule.validate() + return schedule + + def put_schedule(self, schedule: MaintenanceSchedule): + schedule = schedule.to_json() + return self.client.put_object(Bucket=self.bucket, + Key=self.object_key, + Body=json.dumps({ + "maintenance": { + "schedule": schedule + } + }).encode()) + + def provision_event(self, + planned_start: str, + planned_duration: int, + description: str, + partial: list[str] | None = None, + degraded: list[str] | None = None, + unavailable: list[str] | None = None) -> MaintenanceEvent: + """ + Uses the given inmput parametes to provision a new MaintenanceEvent. + This new MaintenanceEvent object can then be added as an event to an + existing schedule. It is primarily used by `add_event` to create and add + events to the maintenance schedule. + """ + partial = [{ + 'kind': 'partial_responses', + 'affected_catalogs': partial + }] if partial is not None else [] + degraded = [{ + 'kind': 'degraded_performance', + 'affected_catalogs': degraded + }] if degraded is not None else [] + unavailable = [{ + 'kind': 'service_unavailable', + 'affected_catalogs': unavailable + }] if unavailable is not None else [] + impacts = [*partial, *degraded, *unavailable] + return MaintenanceEvent.from_json({ + 'planned_start': planned_start, + 'planned_duration': planned_duration, + 'description': description, + 'impacts': impacts + }) + + def add_event(self, event: MaintenanceEvent) -> JSON: + schedule = self.get_schedule + schedule.add_event(event) + self.put_schedule(schedule) + return schedule.to_json() + + def cancel_event(self, index: int) -> JSON: + schedule = self.get_schedule + event = schedule.cancel_event(index) + self.put_schedule(schedule) + return event.to_json() + + def start_event(self) -> JSON: + schedule = self.get_schedule + event = schedule.start_event() + self.put_schedule(schedule) + return event.to_json() + + def end_event(self) -> JSON: + schedule = self.get_schedule + event = schedule.end_event() + self.put_schedule(schedule) + return event.to_json() + + def adjust_event(self, additional_duration: timedelta) -> JSON: + schedule = self.get_schedule + event = schedule.adjust_event(additional_duration) + self.put_schedule(schedule) + return event.to_json() diff --git a/test/test_doctests.py b/test/test_doctests.py index c4bbe4023e..36b658683a 100644 --- a/test/test_doctests.py +++ b/test/test_doctests.py @@ -104,6 +104,7 @@ def load_tests(_loader, tests, _ignore): load_script('can_bundle'), load_script('envhook'), load_script('export_environment'), + load_script('manage_maintenance'), load_module(root + '/.flake8/azul_flake8.py', 'azul_flake8'), load_module(root + '/.github/workflows/schedule.py', 'schedule'), test_tagging, From 9af2251d46d7d0ddf7fa29dc65e9f842f26df18b Mon Sep 17 00:00:00 2001 From: Abraham Chavez Date: Mon, 26 Aug 2024 09:27:33 -0700 Subject: [PATCH 3/3] fixup! [a 2/2] Announce maintenance in stable deployments (#5979) --- lambdas/service/app.py | 2 +- lambdas/service/openapi.json | 2 +- scripts/manage_maintenance.py | 72 +++++++++++++++++------------------ src/azul/maintenance.py | 25 ++++++------ 4 files changed, 50 insertions(+), 51 deletions(-) diff --git a/lambdas/service/app.py b/lambdas/service/app.py index 3334b120fb..ff8b92d547 100644 --- a/lambdas/service/app.py +++ b/lambdas/service/app.py @@ -235,7 +235,7 @@ # changes and reset the minor version to zero. Otherwise, increment only # the minor version for backwards compatible changes. A backwards # compatible change is one that does not require updates to clients. - 'version': '9.1' + 'version': '9.2' }, 'tags': [ { diff --git a/lambdas/service/openapi.json b/lambdas/service/openapi.json index 0c9de5a5a2..84e62f00da 100644 --- a/lambdas/service/openapi.json +++ b/lambdas/service/openapi.json @@ -3,7 +3,7 @@ "info": { "title": "azul_service", "description": "\n# Overview\n\nAzul is a REST web service for querying metadata associated with\nboth experimental and analysis data from a data repository. In order\nto deliver response times that make it suitable for interactive use\ncases, the set of metadata properties that it exposes for sorting,\nfiltering, and aggregation is limited. Azul provides a uniform view\nof the metadata over a range of diverse schemas, effectively\nshielding clients from changes in the schemas as they occur over\ntime. It does so, however, at the expense of detail in the set of\nmetadata properties it exposes and in the accuracy with which it\naggregates them.\n\nAzul denormalizes and aggregates metadata into several different\nindices for selected entity types. Metadata entities can be queried\nusing the [Index](#operations-tag-Index) endpoints.\n\nA set of indices forms a catalog. There is a default catalog called\n`dcp2` which will be used unless a\ndifferent catalog name is specified using the `catalog` query\nparameter. Metadata from different catalogs is completely\nindependent: a response obtained by querying one catalog does not\nnecessarily correlate to a response obtained by querying another\none. Two catalogs can contain metadata from the same sources or\ndifferent sources. It is only guaranteed that the body of a\nresponse by any given endpoint adheres to one schema,\nindependently of which catalog was specified in the request.\n\nAzul provides the ability to download data and metadata via the\n[Manifests](#operations-tag-Manifests) endpoints. The\n`curl` format manifests can be used to\ndownload data files. Other formats provide various views of the\nmetadata. Manifests can be generated for a selection of files using\nfilters. These filters are interchangeable with the filters used by\nthe [Index](#operations-tag-Index) endpoints.\n\nAzul also provides a [summary](#operations-Index-get_index_summary)\nview of indexed data.\n\n## Data model\n\nAny index, when queried, returns a JSON array of hits. Each hit\nrepresents a metadata entity. Nested in each hit is a summary of the\nproperties of entities associated with the hit. An entity is\nassociated either by a direct edge in the original metadata graph,\nor indirectly as a series of edges. The nested properties are\ngrouped by the type of the associated entity. The properties of all\ndata files associated with a particular sample, for example, are\nlisted under `hits[*].files` in a `/index/samples` response. It is\nimportant to note that while each _hit_ represents a discrete\nentity, the properties nested within that hit are the result of an\naggregation over potentially many associated entities.\n\nTo illustrate this, consider a data file that is part of two\nprojects (a project is a group of related experiments, typically by\none laboratory, institution or consortium). Querying the `files`\nindex for this file yields a hit looking something like:\n\n```\n{\n \"projects\": [\n {\n \"projectTitle\": \"Project One\"\n \"laboratory\": ...,\n ...\n },\n {\n \"projectTitle\": \"Project Two\"\n \"laboratory\": ...,\n ...\n }\n ],\n \"files\": [\n {\n \"format\": \"pdf\",\n \"name\": \"Team description.pdf\",\n ...\n }\n ]\n}\n```\n\nThis example hit contains two kinds of nested entities (a hit in an\nactual Azul response will contain more): There are the two projects\nentities, and the file itself. These nested entities contain\nselected metadata properties extracted in a consistent way. This\nmakes filtering and sorting simple.\n\nAlso notice that there is only one file. When querying a particular\nindex, the corresponding entity will always be a singleton like\nthis.\n", - "version": "9.1" + "version": "9.2" }, "tags": [ { diff --git a/scripts/manage_maintenance.py b/scripts/manage_maintenance.py index 6654f4e57c..19ec8003a2 100644 --- a/scripts/manage_maintenance.py +++ b/scripts/manage_maintenance.py @@ -74,39 +74,39 @@ def parse_duration(duration: str) -> timedelta: def main(args: list[str]): parser = argparse.ArgumentParser(description=__doc__, formatter_class=AzulArgumentHelpFormatter) - subparsers = parser.add_subparsers(dest="command") - list_parser = subparsers.add_parser("list", help="List events in JSON form") - list_parser.add_argument("--all", action="store_true", - help="Include completed events") - add_parser = subparsers.add_parser("add", help="Schedule an event") - add_parser.add_argument("--start", required=True, - help="Event start time (ISO format)") - add_parser.add_argument("--duration", required=True, - help="Event duration (e.g., '1h30m', '2d')") - add_parser.add_argument("--description", required=True, - help="Event description") - add_parser.add_argument("--partial-responses", nargs="+", - help="Catalog names for partial responses") - add_parser.add_argument("--degraded-performance", nargs="+", - help="Catalog names for degraded performance") - add_parser.add_argument("--service-unavailable", nargs="+", - help="Catalog names for service unavailability") - cancel_parser = subparsers.add_parser("cancel", - help="Cancel a pending event") - cancel_parser.add_argument("--index", type=int, required=True, - help="Index of the event to cancel") - subparsers.add_parser("start", help="Activate a pending event") - subparsers.add_parser("end", help="Complete the active event") - adjust_parser = subparsers.add_parser("adjust", - help="Modify the active event") - adjust_parser.add_argument("--duration", required=True, - help="New event duration (e.g., '1h30m', '2d')") + subparsers = parser.add_subparsers(dest='command') + list_parser = subparsers.add_parser('list', help='List events in JSON form') + list_parser.add_argument('--all', action='store_true', + help='Include completed events') + add_parser = subparsers.add_parser('add', help='Schedule an event') + add_parser.add_argument('--start', required=True, + help='Event start time (ISO format)') + add_parser.add_argument('--duration', required=True, + help='Event duration (e.g., "1h30m", "2d")') + add_parser.add_argument('--description', required=True, + help='Event description') + add_parser.add_argument('--partial-responses', nargs='+', + help='Catalog names for partial responses') + add_parser.add_argument('--degraded-performance', nargs='+', + help='Catalog names for degraded performance') + add_parser.add_argument('--service-unavailable', nargs='+', + help='Catalog names for service unavailability') + cancel_parser = subparsers.add_parser('cancel', + help='Cancel a pending event') + cancel_parser.add_argument('--index', type=int, required=True, + help='Index of the event to cancel') + subparsers.add_parser('start', help='Activate a pending event') + subparsers.add_parser('end', help='Complete the active event') + adjust_parser = subparsers.add_parser('adjust', + help='Modify the active event') + adjust_parser.add_argument('--duration', required=True, + help='New event duration (e.g., "1h30m", "2d")') args = parser.parse_args(args) service = MaintenanceService() - if args.command == "list": + if args.command == 'list': events = service.get_schedule if args.all: events = events.to_json() @@ -114,9 +114,9 @@ def main(args: list[str]): active = events.active_event() active = {} if active is None else {'active': active.to_json()} pending = events.pending_events() - pending = {'pending': list(pe.to_json() for pe in pending)} + pending = {'pending': [pe.to_json() for pe in pending]} events = active | pending - elif args.command == "add": + elif args.command == 'add': duration = int(parse_duration(args.duration).total_seconds()) events = service.provision_event(planned_start=args.start, planned_duration=duration, @@ -125,18 +125,18 @@ def main(args: list[str]): degraded=args.degraded_performance, unavailable=args.service_unavailable) events = service.add_event(events) - elif args.command == "cancel": + elif args.command == 'cancel': events = service.cancel_event(args.index) - elif args.command == "start": + elif args.command == 'start': events = service.start_event() - elif args.command == "end": + elif args.command == 'end': events = service.end_event() - elif args.command == "adjust": + elif args.command == 'adjust': events = service.adjust_event(parse_duration(args.duration)) else: - assert False, 'Invalid command' + raise argparse.ArgumentError(subparsers, 'Invalid command') print(json.dumps(events, indent=4)) -if __name__ == "__main__": +if __name__ == '__main__': main(sys.argv[1:]) diff --git a/src/azul/maintenance.py b/src/azul/maintenance.py index 77de486e86..080b57c832 100644 --- a/src/azul/maintenance.py +++ b/src/azul/maintenance.py @@ -58,7 +58,7 @@ class MaintenanceImpact: affected_catalogs: list[CatalogName] @classmethod - def from_json(cls, impact: JSON): + def from_json(cls, impact: JSON) -> Self: return cls(kind=MaintenanceImpactKind[impact['kind']], affected_catalogs=impact['affected_catalogs']) @@ -137,7 +137,7 @@ class MaintenanceSchedule: events: list[MaintenanceEvent] @classmethod - def from_json(cls, schedule: JSON): + def from_json(cls, schedule: JSON) -> Self: return cls(events=list(map(MaintenanceEvent.from_json, schedule['events']))) def to_json(self) -> JSON: @@ -168,8 +168,7 @@ def pending_events(self) -> list[MaintenanceEvent]: schedule. The modifications will be reflected in ``self.events`` but the caller is responsible for ensuring they don't invalidate this schedule. """ - events = enumerate(self.events) - for i, e in events: + for i, e in enumerate(self.events): if e.actual_start is None: return self.events[i:] return [] @@ -209,7 +208,7 @@ def add_event(self, event: MaintenanceEvent): self.events = events raise - def adjust_event(self, additional_duration: timedelta): + def adjust_event(self, additional_duration: timedelta) -> MaintenanceEvent: event = self.active_event() reject(event is None, 'No active event') event.planned_duration += additional_duration @@ -283,13 +282,13 @@ def get_schedule(self) -> MaintenanceSchedule: def put_schedule(self, schedule: MaintenanceSchedule): schedule = schedule.to_json() - return self.client.put_object(Bucket=self.bucket, - Key=self.object_key, - Body=json.dumps({ - "maintenance": { - "schedule": schedule - } - }).encode()) + self.client.put_object(Bucket=self.bucket, + Key=self.object_key, + Body=json.dumps({ + "maintenance": { + "schedule": schedule + } + }).encode()) def provision_event(self, planned_start: str, @@ -299,7 +298,7 @@ def provision_event(self, degraded: list[str] | None = None, unavailable: list[str] | None = None) -> MaintenanceEvent: """ - Uses the given inmput parametes to provision a new MaintenanceEvent. + Uses the given input parameters to provision a new MaintenanceEvent. This new MaintenanceEvent object can then be added as an event to an existing schedule. It is primarily used by `add_event` to create and add events to the maintenance schedule.