Merge branch 'main' into sm64-tests

.github/workflows/unittests.yml

@@ -53,7 +53,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install pytest "pytest-subtests<0.14.0" pytest-xdist
+        pip install pytest pytest-subtests pytest-xdist
         python ModuleUpdate.py --yes --force --append "WebHostLib/requirements.txt"
         python Launcher.py --update_settings  # make sure host.yaml exists for tests
     - name: Unittests

MultiServer.py

@@ -1914,7 +1914,7 @@ async def process_client_cmd(ctx: Context, client: Client, args: dict):
             hint = ctx.get_hint(client.team, player, location)
             if not hint:
                 return  # Ignored safely
-            if hint.receiving_player != client.slot:
+            if client.slot not in ctx.slot_set(hint.receiving_player):
                 await ctx.send_msgs(client,
                                     [{'cmd': 'InvalidPacket', "type": "arguments", "text": 'UpdateHint: No Permission',
                                       "original_cmd": cmd}])

NetUtils.py

@@ -232,7 +232,7 @@ def _handle_text(self, node: JSONMessagePart):
 
     def _handle_player_id(self, node: JSONMessagePart):
         player = int(node["text"])
-        node["color"] = 'magenta' if player == self.ctx.slot else 'yellow'
+        node["color"] = 'magenta' if self.ctx.slot_concerns_self(player) else 'yellow'
         node["text"] = self.ctx.player_names[player]
         return self._handle_color(node)
 
@@ -410,6 +410,8 @@ def get_checked(self, state: typing.Dict[typing.Tuple[int, int], typing.Set[int]
         checked = state[team, slot]
         if not checked:
             # This optimizes the case where everyone connects to a fresh game at the same time.
+            if slot not in self:
+                raise KeyError(slot)
             return []
         return [location_id for
                 location_id in self[slot] if

Options.py

@@ -754,7 +754,7 @@ def __init__(self, value: int) -> None:
         elif value > self.range_end and value not in self.special_range_names.values():
             raise Exception(f"{value} is higher than maximum {self.range_end} for option {self.__class__.__name__} " +
                             f"and is also not one of the supported named special values: {self.special_range_names}")
-        
+
         # See docstring
         for key in self.special_range_names:
             if key != key.lower():
@@ -1180,7 +1180,7 @@ def __len__(self) -> int:
 class Accessibility(Choice):
     """
     Set rules for reachability of your items/locations.
-    
+
     **Full:** ensure everything can be reached and acquired.
 
     **Minimal:** ensure what is needed to reach your goal can be acquired.
@@ -1198,7 +1198,7 @@ class Accessibility(Choice):
 class ItemsAccessibility(Accessibility):
     """
     Set rules for reachability of your items/locations.
-    
+
     **Full:** ensure everything can be reached and acquired.
 
     **Minimal:** ensure what is needed to reach your goal can be acquired.
@@ -1249,12 +1249,16 @@ class CommonOptions(metaclass=OptionsMetaProperty):
     progression_balancing: ProgressionBalancing
     accessibility: Accessibility
 
-    def as_dict(self, *option_names: str, casing: str = "snake") -> typing.Dict[str, typing.Any]:
+    def as_dict(self,
+                *option_names: str,
+                casing: typing.Literal["snake", "camel", "pascal", "kebab"] = "snake",
+                toggles_as_bools: bool = False) -> typing.Dict[str, typing.Any]:
         """
         Returns a dictionary of [str, Option.value]
 
         :param option_names: names of the options to return
         :param casing: case of the keys to return. Supports `snake`, `camel`, `pascal`, `kebab`
+        :param toggles_as_bools: whether toggle options should be output as bools instead of strings
         """
         assert option_names, "options.as_dict() was used without any option names."
         option_results = {}
@@ -1276,6 +1280,8 @@ def as_dict(self, *option_names: str, casing: str = "snake") -> typing.Dict[str,
                 value = getattr(self, option_name).value
                 if isinstance(value, set):
                     value = sorted(value)
+                elif toggles_as_bools and issubclass(type(self).type_hints[option_name], Toggle):
+                    value = bool(value)
                 option_results[display_name] = value
             else:
                 raise ValueError(f"{option_name} not found in {tuple(type(self).type_hints)}")

_speedups.pyx

@@ -69,6 +69,14 @@ cdef struct IndexEntry:
     size_t count
 
 
+if TYPE_CHECKING:
+    State = Dict[Tuple[int, int], Set[int]]
+else:
+    State = Union[Tuple[int, int], Set[int], defaultdict]
+
+T = TypeVar('T')
+
+
 @cython.auto_pickle(False)
 cdef class LocationStore:
     """Compact store for locations and their items in a MultiServer"""
@@ -137,10 +145,16 @@ cdef class LocationStore:
             warnings.warn("Game has no locations")
 
         # allocate the arrays and invalidate index (0xff...)
-        self.entries = <LocationEntry*>self._mem.alloc(count, sizeof(LocationEntry))
+        if count:
+            # leaving entries as NULL if there are none, makes potential memory errors more visible
+            self.entries = <LocationEntry*>self._mem.alloc(count, sizeof(LocationEntry))
         self.sender_index = <IndexEntry*>self._mem.alloc(max_sender + 1, sizeof(IndexEntry))
         self._raw_proxies = <PyObject**>self._mem.alloc(max_sender + 1, sizeof(PyObject*))
 
+        assert (not self.entries) == (not count)
+        assert self.sender_index
+        assert self._raw_proxies
+
         # build entries and index
         cdef size_t i = 0
         for sender, locations in sorted(locations_dict.items()):
@@ -190,8 +204,6 @@ cdef class LocationStore:
             raise KeyError(key)
         return <object>self._raw_proxies[key]
 
-    T = TypeVar('T')
-
     def get(self, key: int, default: T) -> Union[PlayerLocationProxy, T]:
         # calling into self.__getitem__ here is slow, but this is not used in MultiServer
         try:
@@ -246,12 +258,11 @@ cdef class LocationStore:
                         all_locations[sender].add(entry.location)
         return all_locations
 
-    if TYPE_CHECKING:
-        State = Dict[Tuple[int, int], Set[int]]
-    else:
-        State = Union[Tuple[int, int], Set[int], defaultdict]
-
     def get_checked(self, state: State, team: int, slot: int) -> List[int]:
+        cdef ap_player_t sender = slot
+        if sender < 0 or sender >= self.sender_index_size:
+            raise KeyError(slot)
+
         # This used to validate checks actually exist. A remnant from the past.
         # If the order of locations becomes relevant at some point, we could not do sorted(set), so leaving it.
         cdef set checked = state[team, slot]
@@ -263,7 +274,6 @@ cdef class LocationStore:
 
         # Unless the set is close to empty, it's cheaper to use the python set directly, so we do that.
         cdef LocationEntry* entry
-        cdef ap_player_t sender = slot
         cdef size_t start = self.sender_index[sender].start
         cdef size_t count = self.sender_index[sender].count
         return [entry.location for
@@ -273,9 +283,11 @@ cdef class LocationStore:
     def get_missing(self, state: State, team: int, slot: int) -> List[int]:
         cdef LocationEntry* entry
         cdef ap_player_t sender = slot
+        if sender < 0 or sender >= self.sender_index_size:
+            raise KeyError(slot)
+        cdef set checked = state[team, slot]
         cdef size_t start = self.sender_index[sender].start
         cdef size_t count = self.sender_index[sender].count
-        cdef set checked = state[team, slot]
         if not len(checked):
             # Skip `in` if none have been checked.
             # This optimizes the case where everyone connects to a fresh game at the same time.
@@ -290,9 +302,11 @@ cdef class LocationStore:
     def get_remaining(self, state: State, team: int, slot: int) -> List[Tuple[int, int]]:
         cdef LocationEntry* entry
         cdef ap_player_t sender = slot
+        if sender < 0 or sender >= self.sender_index_size:
+            raise KeyError(slot)
+        cdef set checked = state[team, slot]
         cdef size_t start = self.sender_index[sender].start
         cdef size_t count = self.sender_index[sender].count
-        cdef set checked = state[team, slot]
         return sorted([(entry.receiver, entry.item) for
                         entry in self.entries[start:start+count] if
                         entry.location not in checked])
@@ -328,7 +342,8 @@ cdef class PlayerLocationProxy:
         cdef LocationEntry* entry = NULL
         # binary search
         cdef size_t l = self._store.sender_index[self._player].start
-        cdef size_t r = l + self._store.sender_index[self._player].count
+        cdef size_t e = l + self._store.sender_index[self._player].count
+        cdef size_t r = e
         cdef size_t m
         while l < r:
             m = (l + r) // 2
@@ -337,7 +352,7 @@ cdef class PlayerLocationProxy:
                 l = m + 1
             else:
                 r = m
-        if entry:  # count != 0
+        if l < e:
             entry = self._store.entries + l
             if entry.location == loc:
                 return entry
@@ -349,8 +364,6 @@ cdef class PlayerLocationProxy:
             return entry.item, entry.receiver, entry.flags
         raise KeyError(f"No location {key} for player {self._player}")
 
-    T = TypeVar('T')
-
     def get(self, key: int, default: T) -> Union[Tuple[int, int, int], T]:
         cdef LocationEntry* entry = self._get(key)
         if entry:

_speedups.pyxbld

@@ -3,8 +3,16 @@ import os
 
 def make_ext(modname, pyxfilename):
     from distutils.extension import Extension
-    return Extension(name=modname,
-                     sources=[pyxfilename],
-                     depends=["intset.h"],
-                     include_dirs=[os.getcwd()],
-                     language="c")
+    return Extension(
+        name=modname,
+        sources=[pyxfilename],
+        depends=["intset.h"],
+        include_dirs=[os.getcwd()],
+        language="c",
+        # to enable ASAN and debug build:
+        # extra_compile_args=["-fsanitize=address", "-UNDEBUG", "-Og", "-g"],
+        # extra_objects=["-fsanitize=address"],
+        # NOTE: we can not put -lasan at the front of link args, so needs to be run with
+        #       LD_PRELOAD=/usr/lib/libasan.so ASAN_OPTIONS=detect_leaks=0 path/to/exe
+        # NOTE: this can't find everything unless libpython and cymem are also built with ASAN
+    )

kvui.py

@@ -371,7 +371,7 @@ def on_touch_down(self, touch):
                     if self.hint["status"] == HintStatus.HINT_FOUND:
                         return
                     ctx = App.get_running_app().ctx
-                    if ctx.slot == self.hint["receiving_player"]:  # If this player owns this hint
+                    if ctx.slot_concerns_self(self.hint["receiving_player"]):  # If this player owns this hint
                         # open a dropdown
                         self.dropdown.open(self.ids["status"])
                 elif self.selected:
@@ -800,7 +800,7 @@ def refresh_hints(self, hints):
             hint_status_node = self.parser.handle_node({"type": "color",
                                                         "color": status_colors.get(hint["status"], "red"),
                                                         "text": status_names.get(hint["status"], "Unknown")})
-            if hint["status"] != HintStatus.HINT_FOUND and hint["receiving_player"] == ctx.slot:
+            if hint["status"] != HintStatus.HINT_FOUND and ctx.slot_concerns_self(hint["receiving_player"]):
                 hint_status_node = f"[u]{hint_status_node}[/u]"
             data.append({
                 "receiving": {"text": self.parser.handle_node({"type": "player_id", "text": hint["receiving_player"]})},

test/netutils/test_location_store.py

@@ -115,25 +115,56 @@ def test_find_item(self) -> None:
         def test_get_for_player(self) -> None:
             self.assertEqual(self.store.get_for_player(3), {4: {9}})
             self.assertEqual(self.store.get_for_player(1), {1: {13}, 2: {22, 23}})
+            self.assertEqual(self.store.get_for_player(9999), {})
 
         def test_get_checked(self) -> None:
             self.assertEqual(self.store.get_checked(full_state, 0, 1), [11, 12, 13])
             self.assertEqual(self.store.get_checked(one_state, 0, 1), [12])
             self.assertEqual(self.store.get_checked(empty_state, 0, 1), [])
             self.assertEqual(self.store.get_checked(full_state, 0, 3), [9])
 
+        def test_get_checked_exception(self) -> None:
+            with self.assertRaises(KeyError):
+                self.store.get_checked(empty_state, 0, 9999)
+            bad_state = {(0, 6): {1}}
+            with self.assertRaises(KeyError):
+                self.store.get_checked(bad_state, 0, 6)
+            bad_state = {(0, 9999): set()}
+            with self.assertRaises(KeyError):
+                self.store.get_checked(bad_state, 0, 9999)
+
         def test_get_missing(self) -> None:
             self.assertEqual(self.store.get_missing(full_state, 0, 1), [])
             self.assertEqual(self.store.get_missing(one_state, 0, 1), [11, 13])
             self.assertEqual(self.store.get_missing(empty_state, 0, 1), [11, 12, 13])
             self.assertEqual(self.store.get_missing(empty_state, 0, 3), [9])
 
+        def test_get_missing_exception(self) -> None:
+            with self.assertRaises(KeyError):
+                self.store.get_missing(empty_state, 0, 9999)
+            bad_state = {(0, 6): {1}}
+            with self.assertRaises(KeyError):
+                self.store.get_missing(bad_state, 0, 6)
+            bad_state = {(0, 9999): set()}
+            with self.assertRaises(KeyError):
+                self.store.get_missing(bad_state, 0, 9999)
+
         def test_get_remaining(self) -> None:
             self.assertEqual(self.store.get_remaining(full_state, 0, 1), [])
             self.assertEqual(self.store.get_remaining(one_state, 0, 1), [(1, 13), (2, 21)])
             self.assertEqual(self.store.get_remaining(empty_state, 0, 1), [(1, 13), (2, 21), (2, 22)])
             self.assertEqual(self.store.get_remaining(empty_state, 0, 3), [(4, 99)])
 
+        def test_get_remaining_exception(self) -> None:
+            with self.assertRaises(KeyError):
+                self.store.get_remaining(empty_state, 0, 9999)
+            bad_state = {(0, 6): {1}}
+            with self.assertRaises(KeyError):
+                self.store.get_missing(bad_state, 0, 6)
+            bad_state = {(0, 9999): set()}
+            with self.assertRaises(KeyError):
+                self.store.get_remaining(bad_state, 0, 9999)
+
         def test_location_set_intersection(self) -> None:
             locations = {10, 11, 12}
             locations.intersection_update(self.store[1])
@@ -181,6 +212,16 @@ def test_no_locations(self) -> None:
                 })
                 self.assertEqual(len(store), 1)
                 self.assertEqual(len(store[1]), 0)
+                self.assertEqual(sorted(store.find_item(set(), 1)), [])
+                self.assertEqual(sorted(store.find_item({1}, 1)), [])
+                self.assertEqual(sorted(store.find_item({1, 2}, 1)), [])
+                self.assertEqual(store.get_for_player(1), {})
+                self.assertEqual(store.get_checked(empty_state, 0, 1), [])
+                self.assertEqual(store.get_checked(full_state, 0, 1), [])
+                self.assertEqual(store.get_missing(empty_state, 0, 1), [])
+                self.assertEqual(store.get_missing(full_state, 0, 1), [])
+                self.assertEqual(store.get_remaining(empty_state, 0, 1), [])
+                self.assertEqual(store.get_remaining(full_state, 0, 1), [])
 
         def test_no_locations_for_1(self) -> None:
             store = self.type({

worlds/lingo/test/TestDatafile.py

@@ -1,7 +1,7 @@
 import os
 import unittest
 
-from ..static_logic import HASHES
+from ..static_logic import HASHES, PANELS_BY_ROOM
 from ..utils.pickle_static_data import hash_file
 
 
@@ -14,3 +14,8 @@ def test_check_hashes(self) -> None:
                          "LL1.yaml hash does not match generated.dat. Please regenerate using 'python worlds/lingo/utils/pickle_static_data.py'")
         self.assertEqual(ids_file_hash, HASHES["ids.yaml"],
                          "ids.yaml hash does not match generated.dat. Please regenerate using 'python worlds/lingo/utils/pickle_static_data.py'")
+
+    def test_panel_doors_are_set(self) -> None:
+        # This panel is defined earlier in the file than the panel door, so we want to check that the panel door is
+        # correctly applied.
+        self.assertNotEqual(PANELS_BY_ROOM["Outside The Agreeable"]["FIVE (1)"].panel_door, None)

worlds/lingo/utils/pickle_static_data.py

@@ -111,6 +111,16 @@ def load_static_data(ll1_path, ids_path):
     with open(ll1_path, "r") as file:
         config = Utils.parse_yaml(file)
 
+        # We have to process all panel doors first so that panels can see what panel doors they're in even if they're
+        # defined earlier in the file than the panel door.
+        for room_name, room_data in config.items():
+            if "panel_doors" in room_data:
+                PANEL_DOORS_BY_ROOM[room_name] = dict()
+
+                for panel_door_name, panel_door_data in room_data["panel_doors"].items():
+                    process_panel_door(room_name, panel_door_name, panel_door_data)
+
+        # Process the rest of the room.
         for room_name, room_data in config.items():
             process_room(room_name, room_data)
 
@@ -515,12 +525,6 @@ def process_room(room_name, room_data):
         for source_room, doors in room_data["entrances"].items():
             process_entrance(source_room, doors, room_obj)
 
-    if "panel_doors" in room_data:
-        PANEL_DOORS_BY_ROOM[room_name] = dict()
-
-        for panel_door_name, panel_door_data in room_data["panel_doors"].items():
-            process_panel_door(room_name, panel_door_name, panel_door_data)
-
     if "panels" in room_data:
         PANELS_BY_ROOM[room_name] = dict()