Merge branch 'main' into plando_items_rewrite

.github/workflows/scan-build.yml

@@ -40,10 +40,10 @@ jobs:
       run: |
         wget https://apt.llvm.org/llvm.sh
         chmod +x ./llvm.sh
-        sudo ./llvm.sh 17
+        sudo ./llvm.sh 19
     - name: Install scan-build command
       run: |
-        sudo apt install clang-tools-17
+        sudo apt install clang-tools-19
     - name: Get a recent python
       uses: actions/setup-python@v5
       with:
@@ -56,7 +56,7 @@ jobs:
     - name: scan-build
       run: |
         source venv/bin/activate
-        scan-build-17 --status-bugs -o scan-build-reports -disable-checker deadcode.DeadStores python setup.py build -y
+        scan-build-19 --status-bugs -o scan-build-reports -disable-checker deadcode.DeadStores python setup.py build -y
     - name: Store report
       if: failure()
       uses: actions/upload-artifact@v4

MultiServer.py

@@ -1914,7 +1914,7 @@ async def process_client_cmd(ctx: Context, client: Client, args: dict):
             hint = ctx.get_hint(client.team, player, location)
             if not hint:
                 return  # Ignored safely
-            if hint.receiving_player != client.slot:
+            if client.slot not in ctx.slot_set(hint.receiving_player):
                 await ctx.send_msgs(client,
                                     [{'cmd': 'InvalidPacket', "type": "arguments", "text": 'UpdateHint: No Permission',
                                       "original_cmd": cmd}])

NetUtils.py

@@ -232,7 +232,7 @@ def _handle_text(self, node: JSONMessagePart):
 
     def _handle_player_id(self, node: JSONMessagePart):
         player = int(node["text"])
-        node["color"] = 'magenta' if player == self.ctx.slot else 'yellow'
+        node["color"] = 'magenta' if self.ctx.slot_concerns_self(player) else 'yellow'
         node["text"] = self.ctx.player_names[player]
         return self._handle_color(node)
 
@@ -410,6 +410,8 @@ def get_checked(self, state: typing.Dict[typing.Tuple[int, int], typing.Set[int]
         checked = state[team, slot]
         if not checked:
             # This optimizes the case where everyone connects to a fresh game at the same time.
+            if slot not in self:
+                raise KeyError(slot)
             return []
         return [location_id for
                 location_id in self[slot] if

Options.py

@@ -760,7 +760,7 @@ def __init__(self, value: int) -> None:
         elif value > self.range_end and value not in self.special_range_names.values():
             raise Exception(f"{value} is higher than maximum {self.range_end} for option {self.__class__.__name__} " +
                             f"and is also not one of the supported named special values: {self.special_range_names}")
-        
+
         # See docstring
         for key in self.special_range_names:
             if key != key.lower():
@@ -1186,7 +1186,7 @@ def __len__(self) -> int:
 class Accessibility(Choice):
     """
     Set rules for reachability of your items/locations.
-    
+
     **Full:** ensure everything can be reached and acquired.
 
     **Minimal:** ensure what is needed to reach your goal can be acquired.
@@ -1204,7 +1204,7 @@ class Accessibility(Choice):
 class ItemsAccessibility(Accessibility):
     """
     Set rules for reachability of your items/locations.
-    
+
     **Full:** ensure everything can be reached and acquired.
 
     **Minimal:** ensure what is needed to reach your goal can be acquired.
@@ -1255,12 +1255,16 @@ class CommonOptions(metaclass=OptionsMetaProperty):
     progression_balancing: ProgressionBalancing
     accessibility: Accessibility
 
-    def as_dict(self, *option_names: str, casing: str = "snake") -> typing.Dict[str, typing.Any]:
+    def as_dict(self,
+                *option_names: str,
+                casing: typing.Literal["snake", "camel", "pascal", "kebab"] = "snake",
+                toggles_as_bools: bool = False) -> typing.Dict[str, typing.Any]:
         """
         Returns a dictionary of [str, Option.value]
 
         :param option_names: names of the options to return
         :param casing: case of the keys to return. Supports `snake`, `camel`, `pascal`, `kebab`
+        :param toggles_as_bools: whether toggle options should be output as bools instead of strings
         """
         assert option_names, "options.as_dict() was used without any option names."
         option_results = {}
@@ -1282,6 +1286,8 @@ def as_dict(self, *option_names: str, casing: str = "snake") -> typing.Dict[str,
                 value = getattr(self, option_name).value
                 if isinstance(value, set):
                     value = sorted(value)
+                elif toggles_as_bools and issubclass(type(self).type_hints[option_name], Toggle):
+                    value = bool(value)
                 option_results[display_name] = value
             else:
                 raise ValueError(f"{option_name} not found in {tuple(type(self).type_hints)}")

README.md

@@ -77,6 +77,7 @@ Currently, the following games are supported:
 * Mega Man 2
 * Yacht Dice
 * Faxanadu
+* Saving Princess
 
 For setup and instructions check out our [tutorials page](https://archipelago.gg/tutorial/).
 Downloads can be found at [Releases](https://github.com/ArchipelagoMW/Archipelago/releases), including compiled

Utils.py

@@ -557,7 +557,7 @@ def _cleanup():
     import platform
     logging.info(
         f"Archipelago ({__version__}) logging initialized"
-        f" on {platform.platform()}"
+        f" on {platform.platform()} process {os.getpid()}"
         f" running Python {sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}"
         f"{' (frozen)' if is_frozen() else ''}"
     )

WebHost.py

@@ -34,7 +34,7 @@ def get_app() -> "Flask":
         app.config.from_file(configpath, yaml.safe_load)
         logging.info(f"Updated config from {configpath}")
     # inside get_app() so it's usable in systems like gunicorn, which do not run WebHost.py, but import it.
-    parser = argparse.ArgumentParser()
+    parser = argparse.ArgumentParser(allow_abbrev=False)
     parser.add_argument('--config_override', default=None,
                         help="Path to yaml config file that overrules config.yaml.")
     args = parser.parse_known_args()[0]

WebHostLib/__init__.py

@@ -39,6 +39,8 @@
 app.config["JOB_THRESHOLD"] = 1
 # after what time in seconds should generation be aborted, freeing the queue slot. Can be set to None to disable.
 app.config["JOB_TIME"] = 600
+# memory limit for generator processes in bytes
+app.config["GENERATOR_MEMORY_LIMIT"] = 4294967296
 app.config['SESSION_PERMANENT'] = True
 
 # waitress uses one thread for I/O, these are for processing of views that then get sent

WebHostLib/autolauncher.py

@@ -6,6 +6,7 @@
 import typing
 from datetime import timedelta, datetime
 from threading import Event, Thread
+from typing import Any
 from uuid import UUID
 
 from pony.orm import db_session, select, commit
@@ -53,7 +54,21 @@ def launch_generator(pool: multiprocessing.pool.Pool, generation: Generation):
         generation.state = STATE_STARTED
 
 
-def init_db(pony_config: dict):
+def init_generator(config: dict[str, Any]) -> None:
+    try:
+        import resource
+    except ModuleNotFoundError:
+        pass  # unix only module
+    else:
+        # set soft limit for memory to from config (default 4GiB)
+        soft_limit = config["GENERATOR_MEMORY_LIMIT"]
+        old_limit, hard_limit = resource.getrlimit(resource.RLIMIT_AS)
+        if soft_limit != old_limit:
+            resource.setrlimit(resource.RLIMIT_AS, (soft_limit, hard_limit))
+            logging.debug(f"Changed AS mem limit {old_limit} -> {soft_limit}")
+        del resource, soft_limit, hard_limit
+
+    pony_config = config["PONY"]
     db.bind(**pony_config)
     db.generate_mapping()
 
@@ -105,8 +120,8 @@ def keep_running():
         try:
             with Locker("autogen"):
 
-                with multiprocessing.Pool(config["GENERATORS"], initializer=init_db,
-                                          initargs=(config["PONY"],), maxtasksperchild=10) as generator_pool:
+                with multiprocessing.Pool(config["GENERATORS"], initializer=init_generator,
+                                          initargs=(config,), maxtasksperchild=10) as generator_pool:
                     with db_session:
                         to_start = select(generation for generation in Generation if generation.state == STATE_STARTED)
 

WebHostLib/templates/hostRoom.html

@@ -178,8 +178,15 @@
               })
               .then(text => new DOMParser().parseFromString(text, 'text/html'))
               .then(newDocument => {
-                let el = newDocument.getElementById("host-room-info");
-                document.getElementById("host-room-info").innerHTML = el.innerHTML;
+                ["host-room-info", "slots-table"].forEach(function(id) {
+                  const newEl = newDocument.getElementById(id);
+                  const oldEl = document.getElementById(id);
+                  if (oldEl && newEl) {
+                    oldEl.innerHTML = newEl.innerHTML;
+                  } else if (newEl) {
+                    console.warn(`Did not find element to replace for ${id}`)
+                  }
+                });
             });
           }
 

WebHostLib/templates/macros.html

@@ -8,7 +8,7 @@
 {%- endmacro %}
 {% macro list_patches_room(room) %}
     {% if room.seed.slots %}
-        <table>
+        <table id="slots-table">
             <thead>
                 <tr>
                     <th>Id</th>

_speedups.pyx

@@ -69,6 +69,14 @@ cdef struct IndexEntry:
     size_t count
 
 
+if TYPE_CHECKING:
+    State = Dict[Tuple[int, int], Set[int]]
+else:
+    State = Union[Tuple[int, int], Set[int], defaultdict]
+
+T = TypeVar('T')
+
+
 @cython.auto_pickle(False)
 cdef class LocationStore:
     """Compact store for locations and their items in a MultiServer"""
@@ -137,10 +145,16 @@ cdef class LocationStore:
             warnings.warn("Game has no locations")
 
         # allocate the arrays and invalidate index (0xff...)
-        self.entries = <LocationEntry*>self._mem.alloc(count, sizeof(LocationEntry))
+        if count:
+            # leaving entries as NULL if there are none, makes potential memory errors more visible
+            self.entries = <LocationEntry*>self._mem.alloc(count, sizeof(LocationEntry))
         self.sender_index = <IndexEntry*>self._mem.alloc(max_sender + 1, sizeof(IndexEntry))
         self._raw_proxies = <PyObject**>self._mem.alloc(max_sender + 1, sizeof(PyObject*))
 
+        assert (not self.entries) == (not count)
+        assert self.sender_index
+        assert self._raw_proxies
+
         # build entries and index
         cdef size_t i = 0
         for sender, locations in sorted(locations_dict.items()):
@@ -190,8 +204,6 @@ cdef class LocationStore:
             raise KeyError(key)
         return <object>self._raw_proxies[key]
 
-    T = TypeVar('T')
-
     def get(self, key: int, default: T) -> Union[PlayerLocationProxy, T]:
         # calling into self.__getitem__ here is slow, but this is not used in MultiServer
         try:
@@ -246,12 +258,11 @@ cdef class LocationStore:
                         all_locations[sender].add(entry.location)
         return all_locations
 
-    if TYPE_CHECKING:
-        State = Dict[Tuple[int, int], Set[int]]
-    else:
-        State = Union[Tuple[int, int], Set[int], defaultdict]
-
     def get_checked(self, state: State, team: int, slot: int) -> List[int]:
+        cdef ap_player_t sender = slot
+        if sender < 0 or sender >= self.sender_index_size:
+            raise KeyError(slot)
+
         # This used to validate checks actually exist. A remnant from the past.
         # If the order of locations becomes relevant at some point, we could not do sorted(set), so leaving it.
         cdef set checked = state[team, slot]
@@ -263,7 +274,6 @@ cdef class LocationStore:
 
         # Unless the set is close to empty, it's cheaper to use the python set directly, so we do that.
         cdef LocationEntry* entry
-        cdef ap_player_t sender = slot
         cdef size_t start = self.sender_index[sender].start
         cdef size_t count = self.sender_index[sender].count
         return [entry.location for
@@ -273,9 +283,11 @@ cdef class LocationStore:
     def get_missing(self, state: State, team: int, slot: int) -> List[int]:
         cdef LocationEntry* entry
         cdef ap_player_t sender = slot
+        if sender < 0 or sender >= self.sender_index_size:
+            raise KeyError(slot)
+        cdef set checked = state[team, slot]
         cdef size_t start = self.sender_index[sender].start
         cdef size_t count = self.sender_index[sender].count
-        cdef set checked = state[team, slot]
         if not len(checked):
             # Skip `in` if none have been checked.
             # This optimizes the case where everyone connects to a fresh game at the same time.
@@ -290,9 +302,11 @@ cdef class LocationStore:
     def get_remaining(self, state: State, team: int, slot: int) -> List[Tuple[int, int]]:
         cdef LocationEntry* entry
         cdef ap_player_t sender = slot
+        if sender < 0 or sender >= self.sender_index_size:
+            raise KeyError(slot)
+        cdef set checked = state[team, slot]
         cdef size_t start = self.sender_index[sender].start
         cdef size_t count = self.sender_index[sender].count
-        cdef set checked = state[team, slot]
         return sorted([(entry.receiver, entry.item) for
                         entry in self.entries[start:start+count] if
                         entry.location not in checked])
@@ -328,7 +342,8 @@ cdef class PlayerLocationProxy:
         cdef LocationEntry* entry = NULL
         # binary search
         cdef size_t l = self._store.sender_index[self._player].start
-        cdef size_t r = l + self._store.sender_index[self._player].count
+        cdef size_t e = l + self._store.sender_index[self._player].count
+        cdef size_t r = e
         cdef size_t m
         while l < r:
             m = (l + r) // 2
@@ -337,7 +352,7 @@ cdef class PlayerLocationProxy:
                 l = m + 1
             else:
                 r = m
-        if entry:  # count != 0
+        if l < e:
             entry = self._store.entries + l
             if entry.location == loc:
                 return entry
@@ -349,8 +364,6 @@ cdef class PlayerLocationProxy:
             return entry.item, entry.receiver, entry.flags
         raise KeyError(f"No location {key} for player {self._player}")
 
-    T = TypeVar('T')
-
     def get(self, key: int, default: T) -> Union[Tuple[int, int, int], T]:
         cdef LocationEntry* entry = self._get(key)
         if entry:

_speedups.pyxbld

@@ -3,8 +3,16 @@ import os
 
 def make_ext(modname, pyxfilename):
     from distutils.extension import Extension
-    return Extension(name=modname,
-                     sources=[pyxfilename],
-                     depends=["intset.h"],
-                     include_dirs=[os.getcwd()],
-                     language="c")
+    return Extension(
+        name=modname,
+        sources=[pyxfilename],
+        depends=["intset.h"],
+        include_dirs=[os.getcwd()],
+        language="c",
+        # to enable ASAN and debug build:
+        # extra_compile_args=["-fsanitize=address", "-UNDEBUG", "-Og", "-g"],
+        # extra_objects=["-fsanitize=address"],
+        # NOTE: we can not put -lasan at the front of link args, so needs to be run with
+        #       LD_PRELOAD=/usr/lib/libasan.so ASAN_OPTIONS=detect_leaks=0 path/to/exe
+        # NOTE: this can't find everything unless libpython and cymem are also built with ASAN
+    )

docs/CODEOWNERS

@@ -142,6 +142,9 @@
 # Risk of Rain 2
 /worlds/ror2/ @kindasneaki
 
+# Saving Princess
+/worlds/saving_princess/ @LeonarthCG
+
 # Shivers
 /worlds/shivers/ @GodlFire
 

docs/webhost configuration sample.yaml

@@ -27,8 +27,14 @@
 # If you wish to deploy, uncomment the following line and set it to something not easily guessable.
 # SECRET_KEY: "Your secret key here"
 
-# TODO
-#JOB_THRESHOLD: 2
+# Slot limit to post a generation to Generator process pool instead of rolling directly in WebHost process
+#JOB_THRESHOLD: 1
+
+# After what time in seconds should generation be aborted, freeing the queue slot. Can be set to None to disable.
+#JOB_TIME: 600
+
+# Memory limit for Generator processes in bytes, -1 for unlimited. Currently only works on Linux.
+#GENERATOR_MEMORY_LIMIT: 4294967296
 
 # waitress uses one thread for I/O, these are for processing of view that get sent
 #WAITRESS_THREADS: 10

kvui.py

@@ -371,7 +371,7 @@ def on_touch_down(self, touch):
                     if self.hint["status"] == HintStatus.HINT_FOUND:
                         return
                     ctx = App.get_running_app().ctx
-                    if ctx.slot == self.hint["receiving_player"]:  # If this player owns this hint
+                    if ctx.slot_concerns_self(self.hint["receiving_player"]):  # If this player owns this hint
                         # open a dropdown
                         self.dropdown.open(self.ids["status"])
                 elif self.selected:
@@ -800,7 +800,7 @@ def refresh_hints(self, hints):
             hint_status_node = self.parser.handle_node({"type": "color",
                                                         "color": status_colors.get(hint["status"], "red"),
                                                         "text": status_names.get(hint["status"], "Unknown")})
-            if hint["status"] != HintStatus.HINT_FOUND and hint["receiving_player"] == ctx.slot:
+            if hint["status"] != HintStatus.HINT_FOUND and ctx.slot_concerns_self(hint["receiving_player"]):
                 hint_status_node = f"[u]{hint_status_node}[/u]"
             data.append({
                 "receiving": {"text": self.parser.handle_node({"type": "player_id", "text": hint["receiving_player"]})},