From 8bb152f79f2de3ed58051a01f57b4a2ae265e7a1 Mon Sep 17 00:00:00 2001
From: Chris Jefferson <caj21@st-andrews.ac.uk>
Date: Wed, 10 May 2023 21:47:44 +0800
Subject: [PATCH] Ensure all lists of lists are marked as rectangular tables
 when appropriate

---
 src/plist.c               | 39 +++++++++++++++++++++++----------------
 tst/testinstall/table.tst | 31 +++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+), 16 deletions(-)
 create mode 100644 tst/testinstall/table.tst

diff --git a/src/plist.c b/src/plist.c
index 92bc926345d..5355446d73c 100644
--- a/src/plist.c
+++ b/src/plist.c
@@ -287,20 +287,22 @@ static Int KTNumPlist(Obj list, Obj * famfirst)
         }
         isHom   = 1;
         areMut  = IS_MUTABLE_OBJ(elm);
-        if ( ktnumFirst >= T_PLIST_HOM ||
-             ( ktnumFirst == 0 && IS_HOMOG_LIST( elm) )) {
 
-          // entry is a homogeneous list, so this might be a table
-          isTable = 1;
-
-          // also check for rectangularity, unless this would be expensive
-          if (IS_PLIST(elm))
-            {
+        // if entry is a homogeneous list this might be a table or list
+        if (ktnumFirst >= T_PLIST_HOM) {
+            isTable = 1;
+            isRect = 1;
+            len = LEN_PLIST(elm);
+        }
+        else if (ktnumFirst == 0 && IS_HOMOG_LIST(elm)) {
+            isTable = 1;
+            // Only handle small lists as rectangular
+            if(IS_SMALL_LIST(elm)) {
               isRect = 1;
-              len = LEN_PLIST(elm);
+              len = LEN_LIST(elm);
             }
-
         }
+
         if (!testing) CLEAR_OBJ_FLAG(list, OBJ_FLAG_TESTING);
     }
 
@@ -342,16 +344,21 @@ static Int KTNumPlist(Obj list, Obj * famfirst)
                     isRect = 0;
                 }
                 if ( isTable ) {
-                    // IS_PLIST first, as it is much cheaper
-                    if (!(IS_PLIST(elm) || IS_LIST(elm))) {
-                        isTable = 0;
-                        isRect = 0;
+                    // Check IS_PLIST first, as it is much cheaper
+                    if (IS_PLIST(elm)) {
+                        if (isRect && LEN_PLIST(elm) != len) {
+                            isRect = 0;
+                        }
                     }
-                    if ( isRect ) {
-                        if ( !(IS_PLIST(elm) && LEN_PLIST(elm) == len) ) {
+                    else if (IS_SMALL_LIST(elm)) {
+                        if (isRect && LEN_LIST(elm) != len) {
                             isRect = 0;
                         }
                     }
+                    else {
+                        isTable = 0;
+                        isRect = 0;
+                    }
                 }
             }
             areMut  = (areMut || IS_MUTABLE_OBJ(elm));
diff --git a/tst/testinstall/table.tst b/tst/testinstall/table.tst
new file mode 100644
index 00000000000..7eaaa34d057
--- /dev/null
+++ b/tst/testinstall/table.tst
@@ -0,0 +1,31 @@
+#@local u,v, check
+gap> START_TEST("table.tst");
+gap> check := {m} -> [IsTable(m), IsRectangularTable(m), HasIsRectangularTable(m) ];
+function( m ) ... end
+gap> check( [ ]);
+[ false, false, false ]
+gap> check( [ [] ]);
+[ false, false, false ]
+gap> check( [ [], [] ]);
+[ false, false, false ]
+gap> check([ [1,2], [3,4] ]);
+[ true, true, true ]
+gap> check( [ [1,2], [3]]);
+[ true, false, false ]
+gap> check( [ "ab", "cd" ]);
+[ true, true, true ]
+gap> check( [ ['a', 'b'], "cd"]);
+[ true, true, true ]
+gap> check( [ "ab", ['a', 'b']]);
+[ true, true, true ]
+gap> check( [ [1,2], "cd"]);
+[ false, false, false ]
+gap> check( [ [1,2], 3]);
+[ false, false, false ]
+gap> check(InfiniteListOfNames("a"));
+[ false, false, true ]
+gap> check([InfiniteListOfNames("a")]);
+[ true, true, true ]
+gap> check([ ["a","b"], InfiniteListOfNames("a")]);
+[ false, false, false ]
+gap> STOP_TEST( "table.tst", 1);