Fix handling of DOMAIN

* Fix UI editing of DOMAIN values * Unicode DOMAIN doesn't work * Add `odg cat` to odg as alias to `odg list` * Print string length on `odg list` output * Fixed encoding handling in nosis/XML * Test suite update to reflect changes
Laerdal · Apr 13, 2024 · 4d6b398 · 4d6b398
1 parent e7aeb5f
commit 4d6b398
Show file tree

Hide file tree

Showing 18 changed files with 1,180 additions and 306 deletions.
diff --git a/src/objdictgen/__main__.py b/src/objdictgen/__main__.py
@@ -201,7 +201,7 @@ def main(debugopts: DebugOpts, args: Sequence[str]|None = None):
     # -- LIST --
     subp = subparser.add_parser('list', help="""
         List
-    """)
+    """, aliases=['cat'])
     subp.add_argument('od', nargs="+", help="Object dictionary")
     subp.add_argument('-i', '--index', action="append", help="Specify parameter index to show")
     subp.add_argument('--all', action="store_true",
@@ -325,7 +325,7 @@ def main(debugopts: DebugOpts, args: Sequence[str]|None = None):
 
 
     # -- LIST command --
-    elif opts.command == "list":
+    elif opts.command in ("list", "cat"):
 
         for n, name in enumerate(opts.od):
 

diff --git a/src/objdictgen/node.py b/src/objdictgen/node.py
@@ -1094,7 +1094,10 @@ def GetPrintParams(self, keys=None, short=False, compact=False, unused=False, ve
 
                 # Special formatting on value
                 if isinstance(value, str):
-                    value = '"' + value + '"'
+                    length = len(value)
+                    if typename == 'DOMAIN':
+                        value = value.encode('unicode_escape').decode()
+                    value = '"' + value + f'"  ({length})'
                 elif i and index_range and index_range.name in ('rpdom', 'tpdom'):
                     # FIXME: In PDO mappings, the value is ints
                     assert isinstance(value, int)

diff --git a/src/objdictgen/nodemanager.py b/src/objdictgen/nodemanager.py
@@ -697,7 +697,9 @@ def SetCurrentEntry(self, index: int, subindex: int, value: str, name: str, edit
                     # Might fail with binascii.Error if hex is malformed
                     if len(value) % 2 != 0:
                         value = "0" + value
-                    bvalue = codecs.decode(value, 'hex_codec').decode()
+                    # The latin-1 encoding supports using 0x80-0xFF as values
+                    # FIXME: Doesn't work with unicode
+                    bvalue = codecs.decode(value, 'hex_codec').decode('latin-1')
                     node.SetEntry(index, subindex, bvalue)
                 elif editor == "dcf":
                     node.SetEntry(index, subindex, value)
@@ -1047,7 +1049,9 @@ def GetNodeEntryValues(self, node: Node, index: int) -> tuple[list[dict], list[d
                                 editor["value"] = "dcf"
                             else:
                                 editor["value"] = "domain"
-                            dic["value"] = codecs.encode(dic["value"].encode(), 'hex_codec').decode()
+                            # The latin-1 encoding supports using 0x80-0xFF as values
+                            # FIXME: Doesn't work with unicode
+                            dic["value"] = codecs.encode(dic["value"].encode('latin-1'), 'hex_codec').decode().upper()
                         elif dic["type"] == "BOOLEAN":
                             editor["value"] = "bool"
                             dic["value"] = maps.BOOL_TYPE[dic["value"]]

diff --git a/src/objdictgen/nosis.py b/src/objdictgen/nosis.py
@@ -198,7 +198,7 @@ def safe_string(s: str, isattr: bool = True) -> str:
 
     if isattr:
         # for others, use Python style escapes
-        return repr(s)[1:-1]  # without the extra single-quotes
+        return s.encode('unicode_escape').decode('utf-8')
 
     return s
 
@@ -279,7 +279,7 @@ def xmldump(filehandle: io.TextIOWrapper|None, py_obj: object,
 
     id_tag = f' id="{objid}"' if objid is not None else ""
 
-    fh.write(f"""<?xml version="1.0"?>
+    fh.write(f"""<?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE PyObject SYSTEM "PyObjects.dtd">
 <PyObject module="{module}" class="{klass_tag}"{id_tag}>
 """)

diff --git a/tests/conftest.py b/tests/conftest.py
@@ -29,14 +29,15 @@
 ]
 
 # Files to exclude from py2 legacy testing
-PY2_OD_EXCLUDE = [
-    ODDIR / "unicode.json",
-    ODDIR / "unicode.od",
+PY2_OD_EXCLUDE: list[Path] = [
 ]
 
 # Files to exclude in EDS testing
-PY2_EDS_EXCLUDE = [
-    ODDIR / "legacy-strings.od",  # The legacy tool silently crash on this input
+PY2_EDS_EXCLUDE: list[Path] = [
+]
+
+# Files to exclude in pickle testing
+PY2_PICKLE_EXCLUDE: list[Path] = [
 ]
 
 # Equivalent files that should compare as equal
@@ -377,6 +378,9 @@ def py2_pickle(odfile, py2, wd_session):
     if not odfile.exists():
         pytest.skip(f"File not found: {odfile.rel_to_wd()}")
 
+    if odfile in PY2_PICKLE_EXCLUDE:
+        pytest.skip(f"File {odfile.rel_to_wd()} is excluded from py2 testing")
+
     tmpod = odfile.stem
 
     shutil.copy(odfile, tmpod + '.od')
@@ -398,7 +402,9 @@ def py2_pickle(odfile, py2, wd_session):
 
     # Load the pickled data
     with open(tmpod + '.pickle', 'rb') as f:
-        data = pickle.load(f, encoding='utf-8')
+        # It seems unicode_escape is needed to be able to encode low and high
+        # ascii characters as well as unicode characters
+        data = pickle.load(f, encoding='unicode_escape')
 
     return odfile, data
 

diff --git a/tests/od/domain.json b/tests/od/domain.json
@@ -3,7 +3,7 @@
   "$version": "1",
   "$description": "Canfestival object dictionary data",
   "$tool": "odg 3.4",
-  "$date": "2024-04-10T22:48:13.333406",
+  "$date": "2024-04-13T23:37:31.909712",
   "name": "domain",
   "description": "",
   "type": "master",
@@ -12,17 +12,65 @@
   "dictionary": [
     {
       "index": "0x2000",  // 8192
-      "name": "Domain",
+      "name": "Extended",
       "struct": "var",
       "group": "user",
       "mandatory": false,
       "sub": [
         {
-          "name": "Domain",
+          "name": "Extended",
           "type": "DOMAIN",  // 15
           "access": "rw",
           "pdo": true,
-          "value": "\u0002@ABC\u0001"
+          "value": "\u0000\u0011\"3DUfw\u0088\u0099\u00aa\u00bb\u00cc\u00dd\u00ee\u00ff"
+        }
+      ]
+    },
+    {
+      "index": "0x2001",  // 8193
+      "name": "Bytes",
+      "struct": "var",
+      "group": "user",
+      "mandatory": false,
+      "sub": [
+        {
+          "name": "Bytes",
+          "type": "DOMAIN",  // 15
+          "access": "rw",
+          "pdo": true,
+          "value": "\u00e2\u009c\u0093"
+        }
+      ]
+    },
+    {
+      "index": "0x2002",  // 8194
+      "name": "Utf8",
+      "struct": "var",
+      "group": "user",
+      "mandatory": false,
+      "sub": [
+        {
+          "name": "Utf8",
+          "type": "DOMAIN",  // 15
+          "access": "rw",
+          "pdo": true,
+          "value": "✓"
+        }
+      ]
+    },
+    {
+      "index": "0x2003",  // 8195
+      "name": "Unicode",
+      "struct": "var",
+      "group": "user",
+      "mandatory": false,
+      "sub": [
+        {
+          "name": "Unicode",
+          "type": "DOMAIN",  // 15
+          "access": "rw",
+          "pdo": true,
+          "value": "\u2713"
         }
       ]
     }

diff --git a/tests/od/domain.od b/tests/od/domain.od
@@ -1,31 +1,83 @@
-<?xml version="1.0"?>
+<?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE PyObject SYSTEM "PyObjects.dtd">
-<PyObject module="node" class="Node" id="2217354715472">
-<attr name="Profile" type="dict" id="2217369299472">
+<PyObject module="node" class="Node" id="2273352183440">
+<attr name="Profile" type="dict" id="2273365816464">
 </attr>
 <attr name="Description" type="string" value="" />
-<attr name="Dictionary" type="dict" id="2217369505408">
+<attr name="Dictionary" type="dict" id="2273373045312">
   <entry>
     <key type="numeric" value="8192" />
-    <val type="string" value="\x02@ABC\x01" />
+    <val type="string" value="\x00\x11&quot;3DUfw\x88\x99\xaa\xbb\xcc\xdd\xee\xff" />
+  </entry>
+  <entry>
+    <key type="numeric" value="8193" />
+    <val type="string" value="\xe2\x9c\x93" />
+  </entry>
+  <entry>
+    <key type="numeric" value="8194" />
+    <val type="string" value="✓" />
+  </entry>
+  <entry>
+    <key type="numeric" value="8195" />
+    <val type="string" value="\u2713" />
   </entry>
 </attr>
-<attr name="SpecificMenu" type="list" id="2217369263296">
+<attr name="SpecificMenu" type="list" id="2273373059712">
 </attr>
-<attr name="ParamsDictionary" type="dict" id="2217369504960">
+<attr name="ParamsDictionary" type="dict" id="2273373045248">
 </attr>
-<attr name="UserMapping" type="dict" id="2217355415632">
+<attr name="UserMapping" type="dict" id="2273373059408">
   <entry>
     <key type="numeric" value="8192" />
-    <val type="dict" id="2217369504832">
+    <val type="dict" id="2273373010688">
+      <entry>
+        <key type="string" value="need" />
+        <val type="False" value="" />
+      </entry>
+      <entry>
+        <key type="string" value="values" />
+        <val type="list" id="2273373059456">
+          <item type="dict" id="2273373010624">
+            <entry>
+              <key type="string" value="access" />
+              <val type="string" value="rw" />
+            </entry>
+            <entry>
+              <key type="string" value="pdo" />
+              <val type="True" value="" />
+            </entry>
+            <entry>
+              <key type="string" value="type" />
+              <val type="numeric" value="15" />
+            </entry>
+            <entry>
+              <key type="string" value="name" />
+              <val type="string" value="Extended" />
+            </entry>
+          </item>
+        </val>
+      </entry>
+      <entry>
+        <key type="string" value="name" />
+        <val type="string" value="Extended" />
+      </entry>
+      <entry>
+        <key type="string" value="struct" />
+        <val type="numeric" value="1" />
+      </entry>
+    </val>
+  </entry>
+  <entry>
+    <key type="numeric" value="8193" />
+    <val type="dict" id="2273373009792">
       <entry>
         <key type="string" value="need" />
         <val type="False" value="" />
       </entry>
       <entry>
         <key type="string" value="values" />
-        <val type="list" id="2217369261952">
-          <item type="dict" id="2217352937216">
+        <val type="list" id="2273373059520">
+          <item type="dict" id="2273373009728">
             <entry>
               <key type="string" value="access" />
               <val type="string" value="rw" />
@@ -40,23 +92,103 @@
             </entry>
             <entry>
               <key type="string" value="name" />
-              <val type="string" value="Domain" />
+              <val type="string" value="Bytes" />
+            </entry>
+          </item>
+        </val>
+      </entry>
+      <entry>
+        <key type="string" value="name" />
+        <val type="string" value="Bytes" />
+      </entry>
+      <entry>
+        <key type="string" value="struct" />
+        <val type="numeric" value="1" />
+      </entry>
+    </val>
+  </entry>
+  <entry>
+    <key type="numeric" value="8194" />
+    <val type="dict" id="2273373009152">
+      <entry>
+        <key type="string" value="need" />
+        <val type="False" value="" />
+      </entry>
+      <entry>
+        <key type="string" value="name" />
+        <val type="string" value="Utf8" />
+      </entry>
+      <entry>
+        <key type="string" value="struct" />
+        <val type="numeric" value="1" />
+      </entry>
+      <entry>
+        <key type="string" value="values" />
+        <val type="list" id="2273373060800">
+          <item type="dict" id="2273373009088">
+            <entry>
+              <key type="string" value="name" />
+              <val type="string" value="Utf8" />
+            </entry>
+            <entry>
+              <key type="string" value="type" />
+              <val type="numeric" value="15" />
+            </entry>
+            <entry>
+              <key type="string" value="access" />
+              <val type="string" value="rw" />
+            </entry>
+            <entry>
+              <key type="string" value="pdo" />
+              <val type="True" value="" />
             </entry>
           </item>
         </val>
       </entry>
+    </val>
+  </entry>
+  <entry>
+    <key type="numeric" value="8195" />
+    <val type="dict" id="2273373049792">
+      <entry>
+        <key type="string" value="need" />
+        <val type="False" value="" />
+      </entry>
       <entry>
         <key type="string" value="name" />
-        <val type="string" value="Domain" />
+        <val type="string" value="Unicode" />
       </entry>
       <entry>
         <key type="string" value="struct" />
         <val type="numeric" value="1" />
       </entry>
+      <entry>
+        <key type="string" value="values" />
+        <val type="list" id="2273373156800">
+          <item type="dict" id="2273373048448">
+            <entry>
+              <key type="string" value="name" />
+              <val type="string" value="Unicode" />
+            </entry>
+            <entry>
+              <key type="string" value="type" />
+              <val type="numeric" value="15" />
+            </entry>
+            <entry>
+              <key type="string" value="access" />
+              <val type="string" value="rw" />
+            </entry>
+            <entry>
+              <key type="string" value="pdo" />
+              <val type="True" value="" />
+            </entry>
+          </item>
+        </val>
+      </entry>
     </val>
   </entry>
 </attr>
-<attr name="DS302" type="dict" id="2217369297168">
+<attr name="DS302" type="dict" id="2273373058960">
 </attr>
 <attr name="ProfileName" type="string" value="None" />
 <attr name="Type" type="string" value="master" />