From 010cd6de21463266274a25047c50962325c8c9ee Mon Sep 17 00:00:00 2001 From: takishida <38262981+takishida@users.noreply.github.com> Date: Tue, 15 Oct 2024 08:45:37 -0700 Subject: [PATCH] Add pagination in `icurl()` to support large amount of objects (#174) * feat: Add pagination in icurl() to handle 100K+ objects * test: Add pytest for icurl --- aci-preupgrade-validation-script.py | 43 ++++++--- tests/conftest.py | 100 +++++++++++++++----- tests/test_icurl.py | 141 ++++++++++++++++++++++++++++ 3 files changed, 251 insertions(+), 33 deletions(-) create mode 100644 tests/test_icurl.py diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 62e26ea..096130e 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -1044,16 +1044,7 @@ def print_result(title, result, msg='', prints(output) -def icurl(apitype, query): - if apitype not in ['class', 'mo']: - print('invalid API type - %s' % apitype) - return [] - uri = 'http://127.0.0.1:7777/api/{}/{}'.format(apitype, query) - cmd = ['icurl', '-gs', uri] - logging.info('cmd = ' + ' '.join(cmd)) - response = subprocess.check_output(cmd) - logging.debug('response: ' + str(response)) - imdata = json.loads(response)['imdata'] +def _icurl_error_handler(imdata): if imdata and "error" in imdata[0]: if "not found in class" in imdata[0]['error']['attributes']['text']: raise OldVerPropNotFound('cversion does not have requested property') @@ -1063,8 +1054,36 @@ def icurl(apitype, query): raise OldVerClassNotFound('cversion does not have requested class') else: raise Exception('API call failed! Check debug log') - else: - return imdata + + +def _icurl(apitype, query, page=0, page_size=100000): + if apitype not in ['class', 'mo']: + print('invalid API type - %s' % apitype) + return [] + pre = '&' if '?' in query else '?' + query += '{}page={}&page-size={}'.format(pre, page, page_size) + uri = 'http://127.0.0.1:7777/api/{}/{}'.format(apitype, query) + cmd = ['icurl', '-gs', uri] + logging.info('cmd = ' + ' '.join(cmd)) + response = subprocess.check_output(cmd) + logging.debug('response: ' + str(response)) + data = json.loads(response) + _icurl_error_handler(data['imdata']) + return data + + +def icurl(apitype, query, page_size=100000): + total_imdata = [] + total_cnt = 999999 + page = 0 + while total_cnt > len(total_imdata): + data = _icurl(apitype, query, page, page_size) + if not data['imdata']: + break + total_imdata += data['imdata'] + total_cnt = int(data['totalCount']) + page += 1 + return total_imdata def get_credentials(): diff --git a/tests/conftest.py b/tests/conftest.py index 650b588..c535516 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,19 +16,65 @@ @pytest.fixture def icurl_outputs(): """Allows each test function to simulate the responses of icurl() with a - test data in the form of { key: value }. + test data in the form of { query: test_data }. where: - key = icurl query parameter such as `topSystem.json` - value = An expected result of icurl(). Should be the list under - `imdata` in the JSON response from APIC. + query = icurl query parameter such as `topSystem.json` + test_data = An expected result of icurl(). + This can take different forms in the test data. See below for details. If a single check performs multiple icurl quries, provide test data as shown below: { - "query1": "result1", - "query2": "result2", + "query1": "test_data1", + "query2": "test_data2", } + + Examples) + + Option 1 - test_data is the content of imdata: + { + "object_class1.json?filter1=xxx&filter2=yyy": [], + "object_class2.json": [{"object_class": {"attributes": {}}}], + } + + Option 2 - test_data is the whole response of an API query: + { + "object_class1.json?filter1=xxx&filter2=yyy": { + "totalCount": "0", + "imdata": [], + }, + "object_class2.json": { + "totalCount": "1", + "imdata": [{"object_class": {"attributes": {}}}], + } + } + + Option 3 - test_data is the bundle of API queries with multiple pages: + { + "object_class1.json?filter1=xxx&filter2=yyy": [ + { + "totalCount": "0", + "imdata": [], + } + ], + "object_class2.json": [ + { + "totalCount": "199000", + "imdata": [ + {"object_class": {"attributes": {...}}}, + ... + ], + }, + { + "totalCount": "199000", + "imdata": [ + {"object_class": {"attributes": {...}}}, + ... + ], + }, + ] + } """ return { "object_class1.json?filter1=xxx&filter2=yyy": [], @@ -38,21 +84,33 @@ def icurl_outputs(): @pytest.fixture def mock_icurl(monkeypatch, icurl_outputs): - def _mock_icurl(apitype, query): - if icurl_outputs.get(query) is None: + def _mock_icurl(apitype, query, page=0, page_size=100000): + output = icurl_outputs.get(query) + if output is None: log.error("Query `%s` not found in test data", query) - - imdata = icurl_outputs.get(query, []) - if imdata and "error" in imdata[0]: - if "not found in class" in imdata[0]['error']['attributes']['text']: - raise script.OldVerPropNotFound('cversion does not have requested property') - elif "unresolved class for" in imdata[0]['error']['attributes']['text']: - raise script.OldVerClassNotFound('cversion does not have requested class') - elif "not found" in imdata[0]['error']['attributes']['text']: - raise script.OldVerClassNotFound('cversion does not have requested class') + data = {"totalCount": "0", "imdata": []} + elif isinstance(output, list): + # icurl_outputs option 1 - output is imdata which is empty + if not output: + data = {"totalCount": "0", "imdata": []} + # icurl_outputs option 1 - output is imdata + elif output[0].get("totalCount") is None: + data = {"totalCount": str(len(output)), "imdata": output} + # icurl_outputs option 3 - output is each page of icurl + elif len(output) > page: + data = output[page] + # icurl_outputs option 3 - output is each page of icurl + # page after the last page which is empty + else: + data = {"totalCount": output[0]["totalCount"], "imdata": []} + # icurl_outputs option 2 - output is full response of icurl without pages + elif isinstance(output, dict): + if page == 0: + data = output else: - raise Exception('API call failed! Check debug log') - else: - return imdata + data = {"totalCount": output["totalCount"], "imdata": []} + + script._icurl_error_handler(data['imdata']) + return data - monkeypatch.setattr(script, "icurl", _mock_icurl) + monkeypatch.setattr(script, "_icurl", _mock_icurl) diff --git a/tests/test_icurl.py b/tests/test_icurl.py new file mode 100644 index 0000000..68417a1 --- /dev/null +++ b/tests/test_icurl.py @@ -0,0 +1,141 @@ +import pytest +import importlib + +script = importlib.import_module("aci-preupgrade-validation-script") + +# icurl queries +fabricNodePEps = "fabricNodePEp.json" + +data = [ + { + "fabricNodePEp": { + "attributes": { + "dn": "uni/fabric/protpol/expgep-101-103/nodepep-101", + "id": "101", + } + } + }, + { + "fabricNodePEp": { + "attributes": { + "dn": "uni/fabric/protpol/expgep-204-206/nodepep-206", + "id": "206", + } + } + }, + { + "fabricNodePEp": { + "attributes": { + "dn": "uni/fabric/protpol/expgep-101-103/nodepep-103", + "id": "103", + } + } + }, + { + "fabricNodePEp": { + "attributes": { + "dn": "uni/fabric/protpol/expgep-204-206/nodepep-204", + "id": "204", + } + } + }, +] +long_data = data * 25000 # 100K entries +long_data_all = long_data * 2 + data + + +@pytest.mark.parametrize( + "apitype, query, icurl_outputs, expected_result", + [ + # option 1: test_data is imdata + ( + "class", + "fabricNodePEp.json", + {"fabricNodePEp.json": data}, + data, + ), + # option 2: test_data is the whole response of an API query (totalCount + imdata) + ( + "class", + "fabricNodePEp.json", + {"fabricNodePEp.json": {"totalCount": str(len(data)), "imdata": data}}, + data, + ), + # option 3: test_data is the bundle of API queries with multiple pages + ( + "class", + "fabricNodePEp.json", + { + "fabricNodePEp.json": [ + { # page 0 + "totalCount": str(len(long_data_all)), + "imdata": long_data, + }, + { # page 1 + "totalCount": str(len(long_data_all)), + "imdata": long_data, + }, + { # page 2 + "totalCount": str(len(long_data_all)), + "imdata": data, + }, + ] + }, + long_data_all, + ), + ], +) +def test_icurl(mock_icurl, apitype, query, expected_result): + assert script.icurl(apitype, query) == expected_result + + +@pytest.mark.parametrize( + "imdata, expected_exception", + [ + # /api/class/faultInfo.json?query-target-filter=eq(faultInst.cod,"F2109") + ( + [ + { + "error": { + "attributes": { + "code": "121", + "text": "Prop 'cod' not found in class 'faultInst' property table", + } + } + } + ], + script.OldVerPropNotFound, + ), + # /api/class/faultInf.json?query-target-filter=eq(faultInst.code,"F2109") + ( + [ + { + "error": { + "attributes": { + "code": "400", + "text": "Request failed, unresolved class for faultInf", + } + } + } + ], + script.OldVerClassNotFound, + ), + # /api/class/faultInfo.json?query-target-filter=eq(faultIns.code,"F2109") + ( + [ + { + "error": { + "attributes": { + "code": "122", + "text": "class faultIns not found", + } + } + } + ], + script.OldVerClassNotFound, + ), + ], +) +def test_icurl_error_handler(imdata, expected_exception): + with pytest.raises(expected_exception): + script._icurl_error_handler(imdata)