From 35038b1eb795be6b4ae517cb9ab1df3f70046d47 Mon Sep 17 00:00:00 2001 From: Vijay Chakravarty Date: Tue, 17 Sep 2024 11:01:07 +0200 Subject: [PATCH 1/2] Fix #8661 changing local account usage to tape recall usage in quota report --- src/python/RucioUtils.py | 12 +++++++----- src/script/Monitor/ReportRecallQuota.py | 14 ++++---------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/python/RucioUtils.py b/src/python/RucioUtils.py index bfb9de9211..1ee1ae410f 100644 --- a/src/python/RucioUtils.py +++ b/src/python/RucioUtils.py @@ -114,13 +114,15 @@ def getRuleQuota(rucioClient=None, ruleId=None): size = sum(file['bytes'] for file in files) return size - def getTapeRecallUsage(rucioClient=None, account=None): - """ size of ongoing tape recalls for this account """ + """ size of ongoing tape recalls for this account (if provided) or by activity """ activity = 'Analysis TapeRecall' - rucioAccount = account - rules = rucioClient.list_replication_rules( - filters={'account': rucioAccount, 'activity': activity}) + filters = {'activity': activity} + + if account is not None: + filters['account'] = account + + rules = rucioClient.list_replication_rules(filters=filters) usage = sum(getRuleQuota(rucioClient, rule['id']) for rule in rules\ if rule['state'] in ['REPLICATING', 'STUCK', 'SUSPENDED']) # in Bytes return usage diff --git a/src/script/Monitor/ReportRecallQuota.py b/src/script/Monitor/ReportRecallQuota.py index 5403ee668e..129b486b50 100644 --- a/src/script/Monitor/ReportRecallQuota.py +++ b/src/script/Monitor/ReportRecallQuota.py @@ -10,6 +10,7 @@ import requests from requests.auth import HTTPBasicAuth +from python.RucioUtils import getTapeRecallUsage FMT = "%Y-%m-%dT%H:%M:%S%z" WORKDIR = '/data/srv/monit/' @@ -29,18 +30,11 @@ def readpwd(): def createQuotaReport(rucioClient=None, account=None): """ create a dictionary with the quota report to be sent to MONIT - even if we do not report usage at single RSE's now, let's collect that info as well - returns {'rse1':bytes, 'rse':bytes,..., 'totalTB':TBypte} + we do not report usage at single RSE's now, we don't collect that info either + returns {'totalTB':TBypte} """ - - usageGenerator = rucioClient.get_local_account_usage(account=account) - totalBytes = 0 + totalBytes = getTapeRecallUsage(rucioClient=rucioClient,account=None) report = {} - for usage in usageGenerator: - rse = usage['rse'] - used = usage['bytes'] - report[rse] = used // 1e12 - totalBytes += used totalTB = totalBytes // 1e12 report['totalTB'] = totalTB return report From 424e72e34759e2d5f17bb3964fa2a6db44a1145b Mon Sep 17 00:00:00 2001 From: Vijay Chakravarty Date: Tue, 17 Sep 2024 12:17:06 +0200 Subject: [PATCH 2/2] fix #8661 fixing pylint warnings and removing a comment --- src/script/Monitor/ReportRecallQuota.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/script/Monitor/ReportRecallQuota.py b/src/script/Monitor/ReportRecallQuota.py index 129b486b50..62f2b3f0fd 100644 --- a/src/script/Monitor/ReportRecallQuota.py +++ b/src/script/Monitor/ReportRecallQuota.py @@ -22,7 +22,7 @@ def readpwd(): """ Reads password from disk """ - with open(f"/data/certs/monit.d/MONIT-CRAB.json", encoding='utf-8') as f: + with open("/data/certs/monit.d/MONIT-CRAB.json", encoding='utf-8') as f: credentials = json.load(f) return credentials["url"], credentials["username"], credentials["password"] MONITURL, MONITUSER, MONITPWD = readpwd() @@ -30,7 +30,6 @@ def readpwd(): def createQuotaReport(rucioClient=None, account=None): """ create a dictionary with the quota report to be sent to MONIT - we do not report usage at single RSE's now, we don't collect that info either returns {'totalTB':TBypte} """ totalBytes = getTapeRecallUsage(rucioClient=rucioClient,account=None) @@ -68,13 +67,13 @@ def send_and_check(document, should_fail=False): assert ((response.status_code in [200]) != should_fail), \ msg -def main(logger): +def main(log): from rucio.client import Client rucioClient = Client( creds={"client_cert": "/data/certs/robotcert.pem", "client_key": "/data/certs/robotkey.pem"}, auth_type='x509', ) - logger.info("rucio client initialized: %s %s", rucioClient.ping(), rucioClient.whoami() ) + log.info("rucio client initialized: %s %s", rucioClient.ping(), rucioClient.whoami() ) # prepare a JSON to be sent to MONIT jsonDoc = {'producer': MONITUSER, 'type': 'reportrecallquota', 'hostname': gethostname()}