From 4d6b398453357c404021312e04d8ae8cf3b2b9fd Mon Sep 17 00:00:00 2001 From: Svein Seldal Date: Sun, 14 Apr 2024 00:19:10 +0200 Subject: [PATCH] Fix handling of DOMAIN * Fix UI editing of DOMAIN values * Unicode DOMAIN doesn't work * Add `odg cat` to odg as alias to `odg list` * Print string length on `odg list` output * Fixed encoding handling in nosis/XML * Test suite update to reflect changes --- src/objdictgen/__main__.py | 4 +- src/objdictgen/node.py | 5 +- src/objdictgen/nodemanager.py | 8 +- src/objdictgen/nosis.py | 4 +- tests/conftest.py | 18 +- tests/od/domain.json | 56 +++- tests/od/domain.od | 160 +++++++++- tests/od/legacy-domain.od | 158 +++++++++- tests/od/legacy-strings.od | 159 +++++----- tests/od/strings.json | 101 ++---- tests/od/strings.od | 556 ++++++++++++++++++++++++++++++++++ tests/od/unicode.json | 29 -- tests/od/unicode.od | 65 ---- tests/test_knownfailures.py | 6 +- tests/test_nodemanager.py | 110 ++++++- tests/test_objdictgen.py | 4 +- tests/test_odcompare.py | 37 ++- tests/test_odg.py | 6 + 18 files changed, 1180 insertions(+), 306 deletions(-) create mode 100644 tests/od/strings.od delete mode 100644 tests/od/unicode.json delete mode 100644 tests/od/unicode.od diff --git a/src/objdictgen/__main__.py b/src/objdictgen/__main__.py index 63a2735..95cfadb 100644 --- a/src/objdictgen/__main__.py +++ b/src/objdictgen/__main__.py @@ -201,7 +201,7 @@ def main(debugopts: DebugOpts, args: Sequence[str]|None = None): # -- LIST -- subp = subparser.add_parser('list', help=""" List - """) + """, aliases=['cat']) subp.add_argument('od', nargs="+", help="Object dictionary") subp.add_argument('-i', '--index', action="append", help="Specify parameter index to show") subp.add_argument('--all', action="store_true", @@ -325,7 +325,7 @@ def main(debugopts: DebugOpts, args: Sequence[str]|None = None): # -- LIST command -- - elif opts.command == "list": + elif opts.command in ("list", "cat"): for n, name in enumerate(opts.od): diff --git a/src/objdictgen/node.py b/src/objdictgen/node.py index 210f78f..4db87be 100644 --- a/src/objdictgen/node.py +++ b/src/objdictgen/node.py @@ -1094,7 +1094,10 @@ def GetPrintParams(self, keys=None, short=False, compact=False, unused=False, ve # Special formatting on value if isinstance(value, str): - value = '"' + value + '"' + length = len(value) + if typename == 'DOMAIN': + value = value.encode('unicode_escape').decode() + value = '"' + value + f'" ({length})' elif i and index_range and index_range.name in ('rpdom', 'tpdom'): # FIXME: In PDO mappings, the value is ints assert isinstance(value, int) diff --git a/src/objdictgen/nodemanager.py b/src/objdictgen/nodemanager.py index c3ea9dd..6a08076 100644 --- a/src/objdictgen/nodemanager.py +++ b/src/objdictgen/nodemanager.py @@ -697,7 +697,9 @@ def SetCurrentEntry(self, index: int, subindex: int, value: str, name: str, edit # Might fail with binascii.Error if hex is malformed if len(value) % 2 != 0: value = "0" + value - bvalue = codecs.decode(value, 'hex_codec').decode() + # The latin-1 encoding supports using 0x80-0xFF as values + # FIXME: Doesn't work with unicode + bvalue = codecs.decode(value, 'hex_codec').decode('latin-1') node.SetEntry(index, subindex, bvalue) elif editor == "dcf": node.SetEntry(index, subindex, value) @@ -1047,7 +1049,9 @@ def GetNodeEntryValues(self, node: Node, index: int) -> tuple[list[dict], list[d editor["value"] = "dcf" else: editor["value"] = "domain" - dic["value"] = codecs.encode(dic["value"].encode(), 'hex_codec').decode() + # The latin-1 encoding supports using 0x80-0xFF as values + # FIXME: Doesn't work with unicode + dic["value"] = codecs.encode(dic["value"].encode('latin-1'), 'hex_codec').decode().upper() elif dic["type"] == "BOOLEAN": editor["value"] = "bool" dic["value"] = maps.BOOL_TYPE[dic["value"]] diff --git a/src/objdictgen/nosis.py b/src/objdictgen/nosis.py index 5f597e9..fe59f3c 100644 --- a/src/objdictgen/nosis.py +++ b/src/objdictgen/nosis.py @@ -198,7 +198,7 @@ def safe_string(s: str, isattr: bool = True) -> str: if isattr: # for others, use Python style escapes - return repr(s)[1:-1] # without the extra single-quotes + return s.encode('unicode_escape').decode('utf-8') return s @@ -279,7 +279,7 @@ def xmldump(filehandle: io.TextIOWrapper|None, py_obj: object, id_tag = f' id="{objid}"' if objid is not None else "" - fh.write(f""" + fh.write(f""" """) diff --git a/tests/conftest.py b/tests/conftest.py index 204e108..9bed6dc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -29,14 +29,15 @@ ] # Files to exclude from py2 legacy testing -PY2_OD_EXCLUDE = [ - ODDIR / "unicode.json", - ODDIR / "unicode.od", +PY2_OD_EXCLUDE: list[Path] = [ ] # Files to exclude in EDS testing -PY2_EDS_EXCLUDE = [ - ODDIR / "legacy-strings.od", # The legacy tool silently crash on this input +PY2_EDS_EXCLUDE: list[Path] = [ +] + +# Files to exclude in pickle testing +PY2_PICKLE_EXCLUDE: list[Path] = [ ] # Equivalent files that should compare as equal @@ -377,6 +378,9 @@ def py2_pickle(odfile, py2, wd_session): if not odfile.exists(): pytest.skip(f"File not found: {odfile.rel_to_wd()}") + if odfile in PY2_PICKLE_EXCLUDE: + pytest.skip(f"File {odfile.rel_to_wd()} is excluded from py2 testing") + tmpod = odfile.stem shutil.copy(odfile, tmpod + '.od') @@ -398,7 +402,9 @@ def py2_pickle(odfile, py2, wd_session): # Load the pickled data with open(tmpod + '.pickle', 'rb') as f: - data = pickle.load(f, encoding='utf-8') + # It seems unicode_escape is needed to be able to encode low and high + # ascii characters as well as unicode characters + data = pickle.load(f, encoding='unicode_escape') return odfile, data diff --git a/tests/od/domain.json b/tests/od/domain.json index 701ff82..bcf59a3 100644 --- a/tests/od/domain.json +++ b/tests/od/domain.json @@ -3,7 +3,7 @@ "$version": "1", "$description": "Canfestival object dictionary data", "$tool": "odg 3.4", - "$date": "2024-04-10T22:48:13.333406", + "$date": "2024-04-13T23:37:31.909712", "name": "domain", "description": "", "type": "master", @@ -12,17 +12,65 @@ "dictionary": [ { "index": "0x2000", // 8192 - "name": "Domain", + "name": "Extended", "struct": "var", "group": "user", "mandatory": false, "sub": [ { - "name": "Domain", + "name": "Extended", "type": "DOMAIN", // 15 "access": "rw", "pdo": true, - "value": "\u0002@ABC\u0001" + "value": "\u0000\u0011\"3DUfw\u0088\u0099\u00aa\u00bb\u00cc\u00dd\u00ee\u00ff" + } + ] + }, + { + "index": "0x2001", // 8193 + "name": "Bytes", + "struct": "var", + "group": "user", + "mandatory": false, + "sub": [ + { + "name": "Bytes", + "type": "DOMAIN", // 15 + "access": "rw", + "pdo": true, + "value": "\u00e2\u009c\u0093" + } + ] + }, + { + "index": "0x2002", // 8194 + "name": "Utf8", + "struct": "var", + "group": "user", + "mandatory": false, + "sub": [ + { + "name": "Utf8", + "type": "DOMAIN", // 15 + "access": "rw", + "pdo": true, + "value": "✓" + } + ] + }, + { + "index": "0x2003", // 8195 + "name": "Unicode", + "struct": "var", + "group": "user", + "mandatory": false, + "sub": [ + { + "name": "Unicode", + "type": "DOMAIN", // 15 + "access": "rw", + "pdo": true, + "value": "\u2713" } ] } diff --git a/tests/od/domain.od b/tests/od/domain.od index 7d4e54b..201871c 100644 --- a/tests/od/domain.od +++ b/tests/od/domain.od @@ -1,31 +1,83 @@ - + - - + + - + - + + + + + + + + + + + + + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + @@ -40,23 +92,103 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + - + diff --git a/tests/od/legacy-domain.od b/tests/od/legacy-domain.od index 74f2482..8806a3f 100644 --- a/tests/od/legacy-domain.od +++ b/tests/od/legacy-domain.od @@ -1,31 +1,163 @@ - - + + - + - + + + + + + + + + ✓ + + + + \u2713 - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + Extended + + + + + + + Extended + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Bytes + + + + + + + Bytes + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Utf8 + + + + + + + Utf8 + + + + + + + + + + - - + + @@ -40,14 +172,14 @@ - Domain + Unicode - Domain + Unicode @@ -56,7 +188,7 @@ - + diff --git a/tests/od/legacy-strings.od b/tests/od/legacy-strings.od index b5fc561..96c0f0f 100644 --- a/tests/od/legacy-strings.od +++ b/tests/od/legacy-strings.od @@ -1,21 +1,21 @@ - - + + Complicated strings with unicode - - - - - + + + + + - + abcd abcø abc✓ @@ -23,63 +23,48 @@ - + - - + + + + - - - - - - + + abcd abcø abc✓ - - - - - - - - - - - - - - - - - - + + + abcd + abcø + abc✓ - + - + - + - + - - + + @@ -111,15 +96,15 @@ - + - - + + @@ -151,15 +136,15 @@ - + - - + + @@ -177,7 +162,7 @@ - + @@ -195,7 +180,7 @@ Plain - + @@ -213,7 +198,7 @@ Latin1 - + @@ -245,15 +230,15 @@ - + - - + + @@ -271,7 +256,7 @@ - + @@ -289,7 +274,7 @@ Plain - + @@ -307,7 +292,7 @@ Latin1 - + @@ -325,6 +310,42 @@ Unicode + + + + + + + + + + + + + + + + Extended + + + + + + + + + + + + + + + + + + Bytes + + @@ -339,15 +360,15 @@ - + - - + + @@ -365,7 +386,7 @@ - + @@ -383,7 +404,7 @@ Plain - + @@ -401,7 +422,7 @@ Latin1 - + @@ -433,15 +454,15 @@ - + - - + + @@ -459,7 +480,7 @@ - + @@ -477,7 +498,7 @@ Plain - + @@ -495,7 +516,7 @@ Latin1 - + @@ -526,7 +547,7 @@ - + diff --git a/tests/od/strings.json b/tests/od/strings.json index 45393f9..ea7cf06 100644 --- a/tests/od/strings.json +++ b/tests/od/strings.json @@ -3,7 +3,7 @@ "$version": "1", "$description": "Canfestival object dictionary data", "$tool": "odg 3.4", - "$date": "2024-02-28T01:14:57.869658", + "$date": "2024-04-13T22:20:40.007581", "name": "a", "description": "Complicated strings with unicode", "type": "master", @@ -15,6 +15,7 @@ "index": "0x2000", // 8192 "name": "brod", "struct": "var", + "group": "user", "mandatory": false, "sub": [ { @@ -30,6 +31,7 @@ "index": "0x2001", // 8193 "name": "br\u00f8d", "struct": "var", + "group": "user", "mandatory": false, "sub": [ { @@ -45,6 +47,7 @@ "index": "0x2002", // 8194 "name": "OCTET_STRING", "struct": "record", + "group": "user", "mandatory": false, "sub": [ { @@ -80,6 +83,7 @@ "index": "0x2003", // 8195 "name": "DOMAIN", "struct": "record", + "group": "user", "mandatory": false, "sub": [ { @@ -100,14 +104,28 @@ "type": "DOMAIN", // 15 "access": "rw", "pdo": true, - "value": "abc\u00f8" + "value": "abc\u00c3\u00b8" }, { "name": "Unicode", "type": "DOMAIN", // 15 "access": "rw", "pdo": true, - "value": "abc\u2713" + "value": "abc\u00e2\u009c\u0093" + }, + { + "name": "Extended", + "type": "DOMAIN", // 15 + "access": "rw", + "pdo": true, + "value": "\u0000\u0011\"3DUfw\u0088\u0099\u00aa\u00bb\u00cc\u00dd\u00ee\u00ff" + }, + { + "name": "Bytes", + "type": "DOMAIN", // 15 + "access": "rw", + "pdo": true, + "value": "\u00e2\u009c\u0093" } ] }, @@ -115,6 +133,7 @@ "index": "0x2004", // 8196 "name": "UNICODE_STRING", "struct": "record", + "group": "user", "mandatory": false, "sub": [ { @@ -150,6 +169,7 @@ "index": "0x2006", // 8198 "name": "VISIBLE_STRING", "struct": "record", + "group": "user", "mandatory": false, "sub": [ { @@ -180,81 +200,6 @@ "value": "abc\u2713" } ] - }, - { - "index": "0x1000", // 4096 - "name": "Device Type", - "struct": "var", - "group": "built-in", - "mandatory": true, - "sub": [ - { - "name": "Device Type", - "type": "UNSIGNED32", // 7 - "access": "ro", - "pdo": false, - "value": 0 - } - ] - }, - { - "index": "0x1001", // 4097 - "name": "Error Register", - "struct": "var", - "group": "built-in", - "mandatory": true, - "sub": [ - { - "name": "Error Register", - "type": "UNSIGNED8", // 5 - "access": "ro", - "pdo": true, - "value": 0 - } - ] - }, - { - "index": "0x1018", // 4120 - "name": "Identity", - "struct": "record", - "group": "built-in", - "mandatory": true, - "sub": [ - { - "name": "Number of Entries", - "type": "UNSIGNED8", // 5 - "access": "ro", - "pdo": false - }, - { - "name": "Vendor ID", - "type": "UNSIGNED32", // 7 - "access": "ro", - "pdo": false, - "value": 0 - }, - { - "name": "Product Code", - "type": "UNSIGNED32", // 7 - "access": "ro", - "pdo": false, - "value": 0 - }, - { - "name": "Revision Number", - "type": "UNSIGNED32", // 7 - "access": "ro", - "pdo": false, - "value": 0 - }, - { - "name": "Serial Number", - "type": "UNSIGNED32", // 7 - "access": "ro", - "pdo": false, - "value": 0 - } - ] } ] } \ No newline at end of file diff --git a/tests/od/strings.od b/tests/od/strings.od new file mode 100644 index 0000000..80bd65e --- /dev/null +++ b/tests/od/strings.od @@ -0,0 +1,556 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/od/unicode.json b/tests/od/unicode.json deleted file mode 100644 index 5960f76..0000000 --- a/tests/od/unicode.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "$id": "od data", - "$version": "1", - "$description": "Canfestival object dictionary data", - "$tool": "odg 3.2", - "$date": "2022-11-11T15:48:49.623000", - "name": "Unicode", - "description": "Unicode test", - "type": "master", - "id": 0, - "profile": "None", - "dictionary": [ - { - "index": "0x2000", // 8192 - "name": "Unicode", - "struct": "var", - "mandatory": false, - "sub": [ - { - "name": "Unicode", - "type": "UNICODE_STRING", // 11 - "access": "rw", - "pdo": true, - "value": "OK✓" - } - ] - } - ] -} \ No newline at end of file diff --git a/tests/od/unicode.od b/tests/od/unicode.od deleted file mode 100644 index fa17f27..0000000 --- a/tests/od/unicode.od +++ /dev/null @@ -1,65 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/tests/test_knownfailures.py b/tests/test_knownfailures.py index ba44f76..6916e91 100644 --- a/tests/test_knownfailures.py +++ b/tests/test_knownfailures.py @@ -4,7 +4,7 @@ @pytest.mark.parametrize("suffix", ['od', 'json']) -def test_edsfail_null(wd, odpath, suffix): +def test_fail_eds_null(wd, odpath, suffix): """ EDS export of null.od fails because it contains no data. This is possibly a bug, or EDS does not support empty EDS. @@ -20,10 +20,10 @@ def test_edsfail_null(wd, odpath, suffix): @pytest.mark.parametrize("suffix", ['od', 'json']) -def test_cexportfail_unicode(wd, odpath, suffix): +def test_fail_cexport_unicode(wd, odpath, suffix): """ C-export does not support UNICODE yet. """ - fa = odpath / 'unicode' + fa = odpath / 'strings' m0 = Node.LoadFile(fa + '.' + suffix) diff --git a/tests/test_nodemanager.py b/tests/test_nodemanager.py index a5f999c..88f201f 100644 --- a/tests/test_nodemanager.py +++ b/tests/test_nodemanager.py @@ -1,30 +1,116 @@ +import pytest +from pprint import pprint from objdictgen.nodemanager import NodeManager +from objdictgen.maps import OD +from objdictgen.jsonod import node_todict +def test_nodemanager_createnewnode(): -def test_nodemanager_create_master(): + nm = NodeManager() - m1 = NodeManager() - m1.CreateNewNode( + nm.CreateNewNode( name="TestMaster", id=0x00, type="master", description="Longer description", - profile="None", filepath=None, nmt="Heartbeat", + profile="None", filepath="", nmt="Heartbeat", options=["DS302", "GenSYNC", "Emergency"] ) - m1.CloseCurrent() + nm.BufferCurrentNode() + node = nm.current + assert node.Name == "TestMaster" + assert node.ID == 0x00 + assert node.Type == "master" + assert node.Description == "Longer description" + assert node.ProfileName == "None" -def test_nodemanager_create_slave(): + nm.CloseCurrent() - m1 = NodeManager() - m1.CreateNewNode( + # FIXME: This doesn't work, it doesn't raise an error. The class doesn't + # seem to unset the current node when closing it. + # with pytest.raises(AttributeError): + # node = nm.current + + nm = NodeManager() + nm.CreateNewNode( name="TestSlave", id=0x01, type="slave", description="Longer description", - profile="None", filepath=None, nmt="Heartbeat", + profile="None", filepath="", nmt="Heartbeat", options=["DS302", "GenSYNC", "Emergency"] ) - m1.CloseCurrent() + + node = nm.current + assert node.Name == "TestSlave" + assert node.ID == 0x01 + assert node.Type == "slave" + assert node.Description == "Longer description" + assert node.ProfileName == "None" + + nm.CloseCurrent() def test_nodemanager_load(odpath): - m1 = NodeManager() - m1.OpenFileInCurrent(odpath / 'master.od') + nm = NodeManager() + nm.OpenFileInCurrent(odpath / 'master.od') + + +def test_nodemanager_addmapvariabletocurrent(odpath): + + nm = NodeManager() + nm.OpenFileInCurrent(odpath / 'master.od') + + nm.AddMapVariableToCurrent(0x2000, "A", OD.VAR, 0) + + node = nm.current + assert node.IsEntry(0x2000) + + entry = node.GetIndexEntry(0x2000) + object = entry['object'] + assert object['name'] == "A" + assert object['struct'] == OD.VAR + + +def test_nodemanager_setcurrententry(odpath): + + nm = NodeManager() + nm.OpenFileInCurrent(odpath / 'master.od') + + index = 0x2000 + subindex = 0 + nm.AddMapVariableToCurrent(index, "A", OD.VAR, 0) + + node = nm.current + def getv(index): + entry = node.GetIndexEntry(index) + # from pprint import pprint + # pprint(entry) + object = entry['object'] + return entry['dictionary'], object['values'][0] + + # == DOMAIN tests == + nm.SetCurrentEntry(index, subindex, "DOMAIN", "type", "type") + val, obj = getv(index) + assert obj['type'] == node.GetTypeIndex("DOMAIN") + + for inv, outv in [ + ("00", "\x00"), + ("11", "\x11"), + ("AA", "\xAA"), + ("CC", "\xCC"), + ("FF", "\xFF"), + ("E29C93", "\xE2\x9C\x93"), # Is a unicode checkmark + ]: + nm.SetCurrentEntry(index, subindex, inv, "value", "domain") + val, obj = getv(index) + assert val == outv + + data, editors = nm.GetNodeEntryValues(node, index) + for d in data: + if d['type'] != 'DOMAIN': + continue + assert d['value'] == inv + + nd, _ = node_todict(node, rich=False) + for jobj in nd['dictionary']: + if jobj['index'] != index: + continue + pprint(jobj) diff --git a/tests/test_objdictgen.py b/tests/test_objdictgen.py index 9876855..4a09ec7 100644 --- a/tests/test_objdictgen.py +++ b/tests/test_objdictgen.py @@ -9,7 +9,7 @@ def test_objdictgen_run(odjsoneds, mocker, wd): od = odjsoneds tmpod = od.stem - if tmpod in ('legacy-strings', 'strings', 'unicode'): + if tmpod in ('legacy-strings', 'strings'): pytest.xfail("UNICODE_STRINGS is not supported in C") mocker.patch("sys.argv", [ @@ -28,7 +28,7 @@ def test_objdictgen_py2_compare(py2_cfile, mocker, wd, fn): od, py2od = py2_cfile tmpod = od.stem - if tmpod in ('legacy-strings'): + if tmpod in ('strings', 'legacy-strings', 'domain'): pytest.xfail("UNICODE_STRINGS is not supported in C") mocker.patch("sys.argv", [ diff --git a/tests/test_odcompare.py b/tests/test_odcompare.py index 2823f61..2a9199b 100644 --- a/tests/test_odcompare.py +++ b/tests/test_odcompare.py @@ -5,6 +5,7 @@ import pytest from objdictgen import Node +from objdictgen.nosis import unsafe_string def shave_dict(a, b): @@ -61,6 +62,21 @@ def test_odload_py2_compare(py2_pickle, wd): # Load the OD m1 = Node.LoadFile(od) + # Special handling for string handling in py2. Py3 puts everything in OD + # as value attributes, which py2 reads as strings. A py2 string does not + # support unicode, so the data will be coming over as raw bytes in the + # string. If they differ, attempt to convert the py2 string to utf-8 + def convert_to_utf8(index: int): + """Convert the string to utf-8 if it is different from the py2 data""" + a = m1.Dictionary[index] + b = py2data["Dictionary"][index] + if a != b: + py2data["Dictionary"][index] = b.encode('latin-1').decode('utf-8') + + # # Convert the known string that are encoded differently in py2 + if od.stem == 'domain': + convert_to_utf8(8194) + a, b = shave_dict(py2data, m1.__dict__) assert a == b @@ -128,7 +144,7 @@ def test_cexport(odjsoneds, wd, fn): od = odjsoneds tmpod = od.stem - if tmpod in ('strings', 'legacy-strings', 'unicode'): + if tmpod in ('strings', 'legacy-strings'): pytest.xfail("UNICODE_STRINGS is not supported in C") m0 = Node.LoadFile(od) @@ -149,7 +165,7 @@ def test_cexport_py2_compare(py2_cfile, wd, fn): od, py2od = py2_cfile tmpod = od.stem - if tmpod in ('strings', 'legacy-strings'): + if tmpod in ('strings', 'legacy-strings', 'domain'): pytest.xfail("UNICODE_STRINGS is not supported in C") m0 = Node.LoadFile(od) @@ -380,8 +396,6 @@ def test_save_with_profile(odpath, oddut, suffix, wd, profile): assert a == b - - @pytest.mark.parametrize("suffix", ['od', 'json']) def test_equiv_compare(odpath, equiv_files, suffix): """ Test reading the od and compare it with the corresponding json file @@ -397,5 +411,20 @@ def test_equiv_compare(odpath, equiv_files, suffix): m1 = Node.LoadFile(oda) m2 = Node.LoadFile(odb) + # Special handling for string handling in py2. Py3 puts everything in OD + # as value attributes, which py2 reads as strings. A py2 string does not + # support unicode, so the data will be coming over as raw bytes in the + # string. If they differ, attempt to convert the py2 string to utf-8 + def convert_to_utf8(index: int): + """Convert the string to utf-8 if it is different from the py2 data""" + a = m1.Dictionary[index] + b = m2.Dictionary[index] + if a != b: + m2.Dictionary[index] = unsafe_string(b, True) + + # Convert the known string that are encoded differently in py2 + if oda.stem == 'domain': + convert_to_utf8(8195) + a, b = shave_equal(m1, m2, ignore=('Description', 'IndexOrder')) assert a == b diff --git a/tests/test_odg.py b/tests/test_odg.py index 8cec23b..2d14071 100644 --- a/tests/test_odg.py +++ b/tests/test_odg.py @@ -35,6 +35,12 @@ def test_odg_compare(odpath, equiv_files, suffix): if not oda.exists(): pytest.skip(f"No {oda.rel_to_wd()} file") + # Due to well-known differences between py2 and p3 handling + # we skip the domain comparison + excludes = ('legacy-domain',) + if oda.stem in excludes or odb.stem in excludes: + pytest.skip("py2 and py3 are by design different and can't be compared with this OD") + main(( 'compare', '-D', str(oda),