Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cache cost results #2894

Merged
merged 25 commits into from
Nov 21, 2022
Merged
Changes from 24 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
521df46
handle cost management api throtlling
guybartal Nov 16, 2022
a268a7e
add cost api error examples in schema
guybartal Nov 16, 2022
e144c5d
bump api version + changelog
guybartal Nov 16, 2022
f44b213
Merge branch 'main' into guybartal-handle-cost-api-throtlling
guybartal Nov 16, 2022
da54a4b
bump api version
guybartal Nov 16, 2022
2347813
remove mock + bump ui version
guybartal Nov 16, 2022
4e20696
Merge branch 'main' into guybartal-handle-cost-api-throtlling
guybartal Nov 17, 2022
b5fa357
cache cost result
guybartal Nov 20, 2022
8700d4c
fix typo api_app/resources/strings.py
guybartal Nov 20, 2022
9d1191a
fix spelling
guybartal Nov 20, 2022
b7ec080
fix spelling
guybartal Nov 20, 2022
fefe1c0
add space in ui/app/src/components/root/RootLayout.tsx
guybartal Nov 20, 2022
d04a076
Merge branch 'main' into guybartal-handle-cost-api-throtlling
guybartal Nov 20, 2022
c8094bc
Merge branch 'guybartal-handle-cost-api-throtlling' into guybartal/ca…
guybartal Nov 20, 2022
37f7270
add remarks and types to cost service
guybartal Nov 20, 2022
1bd7541
a more elegant way of implementing singleton
guybartal Nov 20, 2022
eb8e938
remove unused import
guybartal Nov 20, 2022
0d80bdd
Merge branch 'main' into guybartal/cache-cost-results
guybartal Nov 21, 2022
e037853
raise HTTPException with 500 instead of rethrow
guybartal Nov 21, 2022
f2f6303
adding error message
guybartal Nov 21, 2022
898eead
convert starlette JSONResponse to fastapi.responses.JSONResponse
guybartal Nov 21, 2022
d31e133
uppercase API
guybartal Nov 21, 2022
e153b80
add factory method to cost service
guybartal Nov 21, 2022
ef5a0ab
bump api version
guybartal Nov 21, 2022
4cc6d4d
fix unittest for cached cost service
guybartal Nov 21, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion api_app/_version.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.5.16"
__version__ = "0.5.17"
14 changes: 7 additions & 7 deletions api_app/api/routes/costs.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from datetime import datetime
from dateutil.relativedelta import relativedelta
from fastapi import APIRouter, Depends, Query, HTTPException, status
from fastapi.responses import JSONResponse
import logging
from typing import Optional

@@ -16,8 +17,7 @@
from models.domain.costs import CostReport, GranularityEnum, WorkspaceCostReport
from resources import strings
from services.authentication import get_current_admin_user, get_current_workspace_owner_or_tre_admin
from services.cost_service import CostService, ServiceUnavailable, SubscriptionNotSupported, TooManyRequests, WorkspaceDoesNotExist
from starlette.responses import JSONResponse
from services.cost_service import CostService, ServiceUnavailable, SubscriptionNotSupported, TooManyRequests, WorkspaceDoesNotExist, cost_service_factory

costs_core_router = APIRouter(dependencies=[Depends(get_current_admin_user)])
costs_workspace_router = APIRouter(dependencies=[Depends(get_current_workspace_owner_or_tre_admin)])
@@ -54,7 +54,7 @@ def __init__(
responses=get_cost_report_responses())
async def costs(
params: CostsQueryParams = Depends(),
cost_service=Depends(CostService),
cost_service: CostService = Depends(cost_service_factory),
workspace_repo=Depends(get_repository(WorkspaceRepository)),
shared_services_repo=Depends(get_repository(SharedServiceRepository))) -> CostReport:

@@ -80,15 +80,15 @@ async def costs(
}}, status_code=503, headers={"Retry-After": str(e.retry_after)})
except Exception as e:
logging.error("Failed to query Azure TRE costs", exc_info=e)
raise e
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=strings.API_GET_COSTS_INTERNAL_SERVER_ERROR)


@costs_workspace_router.get("/workspaces/{workspace_id}/costs", response_model=WorkspaceCostReport,
name=strings.API_GET_WORKSPACE_COSTS,
dependencies=[Depends(get_current_workspace_owner_or_tre_admin)],
responses=get_workspace_cost_report_responses())
async def workspace_costs(workspace_id: UUID4, params: CostsQueryParams = Depends(),
cost_service=Depends(CostService),
cost_service: CostService = Depends(cost_service_factory),
workspace_repo=Depends(get_repository(WorkspaceRepository)),
workspace_services_repo=Depends(get_repository(WorkspaceServiceRepository)),
user_resource_repo=Depends(get_repository(UserResourceRepository))) -> WorkspaceCostReport:
@@ -117,5 +117,5 @@ async def workspace_costs(workspace_id: UUID4, params: CostsQueryParams = Depend
"retry-after": str(e.retry_after)
}}, status_code=503, headers={"Retry-After": str(e.retry_after)})
except Exception as e:
logging.error("Failed to query Azure TRE workspace costs", exc_info=e)
raise e
logging.error("Failed to query Azure TRE costs", exc_info=e)
raise HTTPException(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=strings.API_GET_COSTS_INTERNAL_SERVER_ERROR)
LizaShak marked this conversation as resolved.
Show resolved Hide resolved
1 change: 1 addition & 0 deletions api_app/main.py
Original file line number Diff line number Diff line change
@@ -87,5 +87,6 @@ async def watch_deployment_status() -> None:
async def update_airlock_request_status() -> None:
await receive_step_result_message_and_update_status(app)


if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=8000)
8 changes: 4 additions & 4 deletions api_app/models/schemas/costs.py
Original file line number Diff line number Diff line change
@@ -38,7 +38,7 @@ def get_cost_report_responses():
"example": {
"error": {
"code": "429",
"message": "Too many requests to Azure cost management api. Please retry.",
"message": "Too many requests to Azure cost management API. Please retry.",
"retry-after": "30"
}
}
@@ -52,7 +52,7 @@ def get_cost_report_responses():
"example": {
"error": {
"code": "503",
"message": "Azure cost management api is temporarly unavaiable. Please retry.",
"message": "Azure cost management API is temporarly unavaiable. Please retry.",
"retry-after": "30"
}
}
@@ -98,7 +98,7 @@ def get_workspace_cost_report_responses():
"example": {
"error": {
"code": "429",
"message": "Too many requests to Azure cost management api. Please retry.",
"message": "Too many requests to Azure cost management API. Please retry.",
"retry-after": "30"
}
}
@@ -112,7 +112,7 @@ def get_workspace_cost_report_responses():
"example": {
"error": {
"code": "503",
"message": "Azure cost management api is temporarly unavaiable. Please retry.",
"message": "Azure cost management API is temporarly unavaiable. Please retry.",
"retry-after": "30"
}
}
5 changes: 3 additions & 2 deletions api_app/resources/strings.py
Original file line number Diff line number Diff line change
@@ -72,8 +72,9 @@
API_GET_COSTS_TO_DATE_NEED_TO_BE_LATER_THEN_FROM_DATE = "to_date needs to be later than from_date"
API_GET_COSTS_FROM_DATE_NEED_TO_BE_BEFORE_TO_DATE = "from_date needs to be before to_date"
API_GET_COSTS_SUBSCRIPTION_NOT_SUPPORTED = "Azure subscription doesn't support cost management"
API_GET_COSTS_TOO_MANY_REQUESTS = "Too many requests to Azure cost management api. Please retry."
API_GET_COSTS_SERVICE_UNAVAILABLE = "Azure cost management api is temporarily unavailable. Please retry."
API_GET_COSTS_TOO_MANY_REQUESTS = "Too many requests to Azure cost management API. Please retry."
API_GET_COSTS_SERVICE_UNAVAILABLE = "Azure cost management API is temporarily unavailable. Please retry."
API_GET_COSTS_INTERNAL_SERVER_ERROR = "Failed to query Azure TRE costs."


# State store status
80 changes: 75 additions & 5 deletions api_app/services/cost_service.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from datetime import datetime, date
from datetime import datetime, date, timedelta
from enum import Enum
from typing import Dict, Optional
from functools import lru_cache
from typing import Dict, Optional, Union
import pandas as pd
import logging

@@ -63,9 +64,22 @@ def __init__(self, retry_after: int, *args: object) -> None:
self.retry_after = retry_after


class CostCacheItem():
"""Holds cost qery result and time to leave for storing in cache"""
result: QueryResult
ttl: datetime

def __init__(self, item: QueryResult, ttl: datetime) -> None:
self.result = item
self.ttl = ttl


# make sure CostService is singleton
@lru_cache(maxsize=None)
martinpeck marked this conversation as resolved.
Show resolved Hide resolved
class CostService:
scope: str
client: CostManagementClient
cache: Dict[str, CostCacheItem]
TRE_ID_TAG: str = "tre_id"
TRE_CORE_SERVICE_ID_TAG: str = "tre_core_service_id"
TRE_WORKSPACE_ID_TAG: str = "tre_workspace_id"
@@ -76,18 +90,62 @@ class CostService:
RATE_LIMIT_RETRY_AFTER_HEADER_KEY: str = "x-ms-ratelimit-microsoft.costmanagement-entity-retry-after"
SERVICE_UNAVAILABLE_RETRY_AFTER_HEADER_KEY: str = "Retry-After"

def __init__(self):
def __init__(self) -> None:
self.scope = "/subscriptions/{}".format(config.SUBSCRIPTION_ID)
self.client = CostManagementClient(credential=credentials.get_credential())
self.resource_client = ResourceManagementClient(credentials.get_credential(), config.SUBSCRIPTION_ID)
self.cache = {}

def get_cached_result(self, key: str) -> Union[QueryResult, None]:
"""Returns cached item result.

Args:
key (str): key of the cached item in cache.
Returns:
result (Union[QueryResult, None]): cost query result or None if not found or expired.
"""
cached_item: CostCacheItem = self.cache.get(key, None)

# return None if key doesn't exist
if cached_item is None:
return None

# return None if key expired
if (datetime.now() > cached_item.ttl):
# remove expired cache item
self.cache.pop(key)
return None

return cached_item.result

def clear_expired_cache_items(self) -> None:
"""Clears all expired cache items."""
expired_keys = [key for key in self.cache.keys() if datetime.now() > self.cache[key].ttl]
for key in expired_keys:
self.cache.pop(key)

def cache_result(self, key: str, result: QueryResult, timedelta: timedelta) -> None:
"""Add cost result to cache.

Args:
key (str) : key of the cached item in cache.
result (QueryResult) : cost query result to cache.
"""
self.cache[key] = CostCacheItem(result, datetime.now() + timedelta)
self.clear_expired_cache_items()

def query_tre_costs(self, tre_id, granularity: GranularityEnum, from_date: datetime, to_date: datetime,
workspace_repo: WorkspaceRepository,
shared_services_repo: SharedServiceRepository) -> CostReport:

resource_groups_dict = self.get_resource_groups_by_tag(self.TRE_ID_TAG, tre_id)

query_result = self.query_costs(CostService.TRE_ID_TAG, tre_id, granularity, from_date, to_date, list(resource_groups_dict.keys()))
cache_key = f"{CostService.TRE_ID_TAG}_{tre_id}_granularity{granularity}_from_date{from_date}_to_date{to_date}_rgs{'_'.join(list(resource_groups_dict.keys()))}"
query_result = self.get_cached_result(cache_key)

if query_result is None:
query_result = self.query_costs(CostService.TRE_ID_TAG, tre_id, granularity, from_date, to_date, list(resource_groups_dict.keys()))
self.cache_result(cache_key, query_result, timedelta(hours=2))

summerized_result = self.summerize_untagged(query_result, granularity, resource_groups_dict)

@@ -112,7 +170,14 @@ def query_tre_workspace_costs(self, workspace_id: str, granularity: GranularityE
user_resource_repo) -> WorkspaceCostReport:

resource_groups_dict = self.get_resource_groups_by_tag(self.TRE_WORKSPACE_ID_TAG, workspace_id)
query_result = self.query_costs(CostService.TRE_WORKSPACE_ID_TAG, workspace_id, granularity, from_date, to_date, list(resource_groups_dict.keys()))

cache_key = f"{CostService.TRE_WORKSPACE_ID_TAG}_{workspace_id}_granularity{granularity}_from_date{from_date}_to_date{to_date}_rgs{'_'.join(list(resource_groups_dict.keys()))}"
query_result = self.get_cached_result(cache_key)

if query_result is None:
query_result = self.query_costs(CostService.TRE_WORKSPACE_ID_TAG, workspace_id, granularity, from_date, to_date, list(resource_groups_dict.keys()))
self.cache_result(cache_key, query_result, timedelta(hours=2))

summerized_result = self.summerize_untagged(query_result, granularity, resource_groups_dict)
query_result_dict = self.__query_result_to_dict(summerized_result, granularity)

@@ -330,3 +395,8 @@ def __query_result_to_dict(self, query_result: list, granularity: GranularityEnu

def __parse_cost_management_date_value(self, date_value: int):
return datetime.strptime(str(date_value), "%Y%m%d").date()


@lru_cache(maxsize=None)
def cost_service_factory() -> CostService:
return CostService()