Skip to content

Commit

Permalink
Add pagination in icurl() to support large amount of objects (#174)
Browse files Browse the repository at this point in the history 
* feat: Add pagination in icurl() to handle 100K+ objects

* test: Add pytest for icurl
  • Loading branch information
@takishida
takishida authored Oct 15, 2024
1 parent 910b7dc commit 010cd6d
Show file tree
Hide file tree
Showing 3 changed files with 251 additions and 33 deletions.
43 changes: 31 additions & 12 deletions aci-preupgrade-validation-script.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1044,16 +1044,7 @@ def print_result(title, result, msg='',
prints(output)


def icurl(apitype, query):
if apitype not in ['class', 'mo']:
print('invalid API type - %s' % apitype)
return []
uri = 'http://127.0.0.1:7777/api/{}/{}'.format(apitype, query)
cmd = ['icurl', '-gs', uri]
logging.info('cmd = ' + ' '.join(cmd))
response = subprocess.check_output(cmd)
logging.debug('response: ' + str(response))
imdata = json.loads(response)['imdata']
def _icurl_error_handler(imdata):
if imdata and "error" in imdata[0]:
if "not found in class" in imdata[0]['error']['attributes']['text']:
raise OldVerPropNotFound('cversion does not have requested property')
Expand All @@ -1063,8 +1054,36 @@ def icurl(apitype, query):
raise OldVerClassNotFound('cversion does not have requested class')
else:
raise Exception('API call failed! Check debug log')
else:
return imdata


def _icurl(apitype, query, page=0, page_size=100000):
if apitype not in ['class', 'mo']:
print('invalid API type - %s' % apitype)
return []
pre = '&' if '?' in query else '?'
query += '{}page={}&page-size={}'.format(pre, page, page_size)
uri = 'http://127.0.0.1:7777/api/{}/{}'.format(apitype, query)
cmd = ['icurl', '-gs', uri]
logging.info('cmd = ' + ' '.join(cmd))
response = subprocess.check_output(cmd)
logging.debug('response: ' + str(response))
data = json.loads(response)
_icurl_error_handler(data['imdata'])
return data


def icurl(apitype, query, page_size=100000):
total_imdata = []
total_cnt = 999999
page = 0
while total_cnt > len(total_imdata):
data = _icurl(apitype, query, page, page_size)
if not data['imdata']:
break
total_imdata += data['imdata']
total_cnt = int(data['totalCount'])
page += 1
return total_imdata


def get_credentials():
Expand Down
100 changes: 79 additions & 21 deletions tests/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,65 @@
@pytest.fixture
def icurl_outputs():
"""Allows each test function to simulate the responses of icurl() with a
test data in the form of { key: value }.
test data in the form of { query: test_data }.
where:
key = icurl query parameter such as `topSystem.json`
value = An expected result of icurl(). Should be the list under
`imdata` in the JSON response from APIC.
query = icurl query parameter such as `topSystem.json`
test_data = An expected result of icurl().
This can take different forms in the test data. See below for details.
If a single check performs multiple icurl quries, provide test data as
shown below:
{
"query1": "result1",
"query2": "result2",
"query1": "test_data1",
"query2": "test_data2",
}
Examples)
Option 1 - test_data is the content of imdata:
{
"object_class1.json?filter1=xxx&filter2=yyy": [],
"object_class2.json": [{"object_class": {"attributes": {}}}],
}
Option 2 - test_data is the whole response of an API query:
{
"object_class1.json?filter1=xxx&filter2=yyy": {
"totalCount": "0",
"imdata": [],
},
"object_class2.json": {
"totalCount": "1",
"imdata": [{"object_class": {"attributes": {}}}],
}
}
Option 3 - test_data is the bundle of API queries with multiple pages:
{
"object_class1.json?filter1=xxx&filter2=yyy": [
{
"totalCount": "0",
"imdata": [],
}
],
"object_class2.json": [
{
"totalCount": "199000",
"imdata": [
{"object_class": {"attributes": {...}}},
...
],
},
{
"totalCount": "199000",
"imdata": [
{"object_class": {"attributes": {...}}},
...
],
},
]
}
"""
return {
"object_class1.json?filter1=xxx&filter2=yyy": [],
Expand All @@ -38,21 +84,33 @@ def icurl_outputs():

@pytest.fixture
def mock_icurl(monkeypatch, icurl_outputs):
def _mock_icurl(apitype, query):
if icurl_outputs.get(query) is None:
def _mock_icurl(apitype, query, page=0, page_size=100000):
output = icurl_outputs.get(query)
if output is None:
log.error("Query `%s` not found in test data", query)

imdata = icurl_outputs.get(query, [])
if imdata and "error" in imdata[0]:
if "not found in class" in imdata[0]['error']['attributes']['text']:
raise script.OldVerPropNotFound('cversion does not have requested property')
elif "unresolved class for" in imdata[0]['error']['attributes']['text']:
raise script.OldVerClassNotFound('cversion does not have requested class')
elif "not found" in imdata[0]['error']['attributes']['text']:
raise script.OldVerClassNotFound('cversion does not have requested class')
data = {"totalCount": "0", "imdata": []}
elif isinstance(output, list):
# icurl_outputs option 1 - output is imdata which is empty
if not output:
data = {"totalCount": "0", "imdata": []}
# icurl_outputs option 1 - output is imdata
elif output[0].get("totalCount") is None:
data = {"totalCount": str(len(output)), "imdata": output}
# icurl_outputs option 3 - output is each page of icurl
elif len(output) > page:
data = output[page]
# icurl_outputs option 3 - output is each page of icurl
# page after the last page which is empty
else:
data = {"totalCount": output[0]["totalCount"], "imdata": []}
# icurl_outputs option 2 - output is full response of icurl without pages
elif isinstance(output, dict):
if page == 0:
data = output
else:
raise Exception('API call failed! Check debug log')
else:
return imdata
data = {"totalCount": output["totalCount"], "imdata": []}

script._icurl_error_handler(data['imdata'])
return data

monkeypatch.setattr(script, "icurl", _mock_icurl)
monkeypatch.setattr(script, "_icurl", _mock_icurl)
141 changes: 141 additions & 0 deletions tests/test_icurl.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
import pytest
import importlib

script = importlib.import_module("aci-preupgrade-validation-script")

# icurl queries
fabricNodePEps = "fabricNodePEp.json"

data = [
{
"fabricNodePEp": {
"attributes": {
"dn": "uni/fabric/protpol/expgep-101-103/nodepep-101",
"id": "101",
}
}
},
{
"fabricNodePEp": {
"attributes": {
"dn": "uni/fabric/protpol/expgep-204-206/nodepep-206",
"id": "206",
}
}
},
{
"fabricNodePEp": {
"attributes": {
"dn": "uni/fabric/protpol/expgep-101-103/nodepep-103",
"id": "103",
}
}
},
{
"fabricNodePEp": {
"attributes": {
"dn": "uni/fabric/protpol/expgep-204-206/nodepep-204",
"id": "204",
}
}
},
]
long_data = data * 25000 # 100K entries
long_data_all = long_data * 2 + data


@pytest.mark.parametrize(
"apitype, query, icurl_outputs, expected_result",
[
# option 1: test_data is imdata
(
"class",
"fabricNodePEp.json",
{"fabricNodePEp.json": data},
data,
),
# option 2: test_data is the whole response of an API query (totalCount + imdata)
(
"class",
"fabricNodePEp.json",
{"fabricNodePEp.json": {"totalCount": str(len(data)), "imdata": data}},
data,
),
# option 3: test_data is the bundle of API queries with multiple pages
(
"class",
"fabricNodePEp.json",
{
"fabricNodePEp.json": [
{ # page 0
"totalCount": str(len(long_data_all)),
"imdata": long_data,
},
{ # page 1
"totalCount": str(len(long_data_all)),
"imdata": long_data,
},
{ # page 2
"totalCount": str(len(long_data_all)),
"imdata": data,
},
]
},
long_data_all,
),
],
)
def test_icurl(mock_icurl, apitype, query, expected_result):
assert script.icurl(apitype, query) == expected_result


@pytest.mark.parametrize(
"imdata, expected_exception",
[
# /api/class/faultInfo.json?query-target-filter=eq(faultInst.cod,"F2109")
(
[
{
"error": {
"attributes": {
"code": "121",
"text": "Prop 'cod' not found in class 'faultInst' property table",
}
}
}
],
script.OldVerPropNotFound,
),
# /api/class/faultInf.json?query-target-filter=eq(faultInst.code,"F2109")
(
[
{
"error": {
"attributes": {
"code": "400",
"text": "Request failed, unresolved class for faultInf",
}
}
}
],
script.OldVerClassNotFound,
),
# /api/class/faultInfo.json?query-target-filter=eq(faultIns.code,"F2109")
(
[
{
"error": {
"attributes": {
"code": "122",
"text": "class faultIns not found",
}
}
}
],
script.OldVerClassNotFound,
),
],
)
def test_icurl_error_handler(imdata, expected_exception):
with pytest.raises(expected_exception):
script._icurl_error_handler(imdata)

0 comments on commit 010cd6d

Please sign in to comment.