Merge remote-tracking branch 'upstream/main' into k64cs

.github/workflows/codeql-analysis.yml

@@ -47,7 +47,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -58,7 +58,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+      uses: github/codeql-action/autobuild@v3
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -72,4 +72,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/unittests.yml

@@ -89,4 +89,4 @@ jobs:
       run: |
         source venv/bin/activate
         export PYTHONPATH=$(pwd)
-        python test/hosting/__main__.py
+        timeout 600 python test/hosting/__main__.py

BaseClasses.py

@@ -1264,6 +1264,10 @@ def useful(self) -> bool:
     def trap(self) -> bool:
         return ItemClassification.trap in self.classification
 
+    @property
+    def excludable(self) -> bool:
+        return not (self.advancement or self.useful)
+
     @property
     def flags(self) -> int:
         return self.classification.as_flag()

CommonClient.py

@@ -710,6 +710,11 @@ def run_gui(self):
 
     def run_cli(self):
         if sys.stdin:
+            if sys.stdin.fileno() != 0:
+                from multiprocessing import parent_process
+                if parent_process():
+                    return  # ignore MultiProcessing pipe
+
             # steam overlay breaks when starting console_loop
             if 'gameoverlayrenderer' in os.environ.get('LD_PRELOAD', ''):
                 logger.info("Skipping terminal input, due to conflicting Steam Overlay detected. Please use GUI only.")

Generate.py

@@ -110,7 +110,7 @@ def main(args=None) -> Tuple[argparse.Namespace, int]:
     player_files = {}
     for file in os.scandir(args.player_files_path):
         fname = file.name
-        if file.is_file() and not fname.startswith(".") and \
+        if file.is_file() and not fname.startswith(".") and not fname.lower().endswith(".ini") and \
                 os.path.join(args.player_files_path, fname) not in {args.meta_file_path, args.weights_file_path}:
             path = os.path.join(args.player_files_path, fname)
             try:
@@ -453,6 +453,10 @@ def roll_settings(weights: dict, plando_options: PlandoOptions = PlandoOptions.b
             raise Exception(f"Option {option_key} has to be in a game's section, not on its own.")
 
     ret.game = get_choice("game", weights)
+    if not isinstance(ret.game, str):
+        if ret.game is None:
+            raise Exception('"game" not specified')
+        raise Exception(f"Invalid game: {ret.game}")
     if ret.game not in AutoWorldRegister.world_types:
         from worlds import failed_world_loads
         picks = Utils.get_fuzzy_results(ret.game, list(AutoWorldRegister.world_types) + failed_world_loads, limit=1)[0]

Launcher.py

@@ -22,16 +22,15 @@
 from shutil import which
 from typing import Callable, Optional, Sequence, Tuple, Union
 
-import Utils
-import settings
-from worlds.LauncherComponents import Component, components, Type, SuffixIdentifier, icon_paths
-
 if __name__ == "__main__":
     import ModuleUpdate
     ModuleUpdate.update()
 
-from Utils import is_frozen, user_path, local_path, init_logging, open_filename, messagebox, \
-    is_windows, is_macos, is_linux
+import settings
+import Utils
+from Utils import (init_logging, is_frozen, is_linux, is_macos, is_windows, local_path, messagebox, open_filename,
+                   user_path)
+from worlds.LauncherComponents import Component, components, icon_paths, SuffixIdentifier, Type
 
 
 def open_host_yaml():
@@ -104,6 +103,7 @@ def update_settings():
     Component("Open host.yaml", func=open_host_yaml),
     Component("Open Patch", func=open_patch),
     Component("Generate Template Options", func=generate_yamls),
+    Component("Archipelago Website", func=lambda: webbrowser.open("https://archipelago.gg/")),
     Component("Discord Server", icon="discord", func=lambda: webbrowser.open("https://discord.gg/8Z65BR2")),
     Component("Unrated/18+ Discord Server", icon="discord", func=lambda: webbrowser.open("https://discord.gg/fqvNCCRsu4")),
     Component("Browse Files", func=browse_files),
@@ -181,6 +181,11 @@ def update_label(self, dt):
                 App.get_running_app().stop()
                 Window.close()
 
+        def _stop(self, *largs):
+            # see run_gui Launcher _stop comment for details
+            self.root_window.close()
+            super()._stop(*largs)
+
     Popup().run()
 
 

MultiServer.py

@@ -727,15 +727,15 @@ def notify_hints(self, team: int, hints: typing.List[NetUtils.Hint], only_new: b
             if not hint.local and data not in concerns[hint.finding_player]:
                 concerns[hint.finding_player].append(data)
             # remember hints in all cases
-            if not hint.found:
-                # since hints are bidirectional, finding player and receiving player,
-                # we can check once if hint already exists
-                if hint not in self.hints[team, hint.finding_player]:
-                    self.hints[team, hint.finding_player].add(hint)
-                    new_hint_events.add(hint.finding_player)
-                    for player in self.slot_set(hint.receiving_player):
-                        self.hints[team, player].add(hint)
-                        new_hint_events.add(player)
+
+            # since hints are bidirectional, finding player and receiving player,
+            # we can check once if hint already exists
+            if hint not in self.hints[team, hint.finding_player]:
+                self.hints[team, hint.finding_player].add(hint)
+                new_hint_events.add(hint.finding_player)
+                for player in self.slot_set(hint.receiving_player):
+                    self.hints[team, player].add(hint)
+                    new_hint_events.add(player)
 
             self.logger.info("Notice (Team #%d): %s" % (team + 1, format_hint(self, team, hint)))
         for slot in new_hint_events:
@@ -1960,8 +1960,10 @@ def _cmd_status(self, tag: str = "") -> bool:
 
     def _cmd_exit(self) -> bool:
         """Shutdown the server"""
-        self.ctx.server.ws_server.close()
-        self.ctx.exit_event.set()
+        try:
+            self.ctx.server.ws_server.close()
+        finally:
+            self.ctx.exit_event.set()
         return True
 
     @mark_raw

Options.py

@@ -15,7 +15,7 @@
 from schema import And, Optional, Or, Schema
 from typing_extensions import Self
 
-from Utils import get_fuzzy_results, is_iterable_except_str, output_path
+from Utils import get_file_safe_name, get_fuzzy_results, is_iterable_except_str, output_path
 
 if typing.TYPE_CHECKING:
     from BaseClasses import MultiWorld, PlandoOptions
@@ -1531,7 +1531,7 @@ def yaml_dump_scalar(scalar) -> str:
 
             del file_data
 
-            with open(os.path.join(target_folder, game_name + ".yaml"), "w", encoding="utf-8-sig") as f:
+            with open(os.path.join(target_folder, get_file_safe_name(game_name) + ".yaml"), "w", encoding="utf-8-sig") as f:
                 f.write(res)
 
 

Utils.py

@@ -18,6 +18,7 @@
 
 from argparse import Namespace
 from settings import Settings, get_settings
+from time import sleep
 from typing import BinaryIO, Coroutine, Optional, Set, Dict, Any, Union
 from typing_extensions import TypeGuard
 from yaml import load, load_all, dump
@@ -568,6 +569,8 @@ def queuer():
             else:
                 if text:
                     queue.put_nowait(text)
+                else:
+                    sleep(0.01)  # non-blocking stream
 
     from threading import Thread
     thread = Thread(target=queuer, name=f"Stream handler for {stream.name}", daemon=True)

WebHost.py

@@ -12,6 +12,7 @@
 # in case app gets imported by something like gunicorn
 import Utils
 import settings
+from Utils import get_file_safe_name
 
 if typing.TYPE_CHECKING:
     from flask import Flask
@@ -71,7 +72,7 @@ def create_ordered_tutorials_file() -> typing.List[typing.Dict[str, typing.Any]]
     shutil.rmtree(base_target_path, ignore_errors=True)
     for game, world in worlds.items():
         # copy files from world's docs folder to the generated folder
-        target_path = os.path.join(base_target_path, game)
+        target_path = os.path.join(base_target_path, get_file_safe_name(game))
         os.makedirs(target_path, exist_ok=True)
 
         if world.zip_path:

WebHostLib/__init__.py

@@ -9,7 +9,7 @@
 from pony.flask import Pony
 from werkzeug.routing import BaseConverter
 
-from Utils import title_sorted
+from Utils import title_sorted, get_file_safe_name
 
 UPLOAD_FOLDER = os.path.relpath('uploads')
 LOGS_FOLDER = os.path.relpath('logs')
@@ -20,6 +20,7 @@
 
 app.jinja_env.filters['any'] = any
 app.jinja_env.filters['all'] = all
+app.jinja_env.filters['get_file_safe_name'] = get_file_safe_name
 
 app.config["SELFHOST"] = True  # application process is in charge of running the websites
 app.config["GENERATORS"] = 8  # maximum concurrent world gens

WebHostLib/requirements.txt

@@ -1,5 +1,5 @@
 flask>=3.0.3
-werkzeug>=3.0.4
+werkzeug>=3.0.6
 pony>=0.7.19
 waitress>=3.0.0
 Flask-Caching>=2.3.0

WebHostLib/templates/gameInfo.html

@@ -11,7 +11,7 @@
 
 {% block body %}
     {% include 'header/'+theme+'Header.html' %}
-    <div id="game-info" class="markdown" data-lang="{{ lang }}" data-game="{{ game }}">
+    <div id="game-info" class="markdown" data-lang="{{ lang }}" data-game="{{ game | get_file_safe_name }}">
         <!-- Populated my JS / MD -->
     </div>
 {% endblock %}

WebHostLib/templates/genericTracker.html

@@ -98,6 +98,8 @@
                             <td>
                                 {% if hint.finding_player == player %}
                                     <b>{{ player_names_with_alias[(team, hint.finding_player)] }}</b>
+                                {% elif get_slot_info(team, hint.finding_player).type == 2 %}
+                                    <i>{{ player_names_with_alias[(team, hint.finding_player)] }}</i>
                                 {% else %}
                                     <a href="{{ url_for("get_player_tracker", tracker=room.tracker, tracked_team=team, tracked_player=hint.finding_player) }}">
                                         {{ player_names_with_alias[(team, hint.finding_player)] }}
@@ -107,6 +109,8 @@
                             <td>
                                 {% if hint.receiving_player == player %}
                                     <b>{{ player_names_with_alias[(team, hint.receiving_player)] }}</b>
+                                {% elif get_slot_info(team, hint.receiving_player).type == 2 %}
+                                    <i>{{ player_names_with_alias[(team, hint.receiving_player)] }}</i>
                                 {% else %}
                                     <a href="{{ url_for("get_player_tracker", tracker=room.tracker, tracked_team=team, tracked_player=hint.receiving_player) }}">
                                         {{ player_names_with_alias[(team, hint.receiving_player)] }}

WebHostLib/templates/multitrackerHintTable.html

@@ -21,8 +21,20 @@
                     )
                 -%}
                     <tr>
-                        <td>{{ player_names_with_alias[(team, hint.finding_player)] }}</td>
-                        <td>{{ player_names_with_alias[(team, hint.receiving_player)] }}</td>
+                        <td>
+                            {% if get_slot_info(team, hint.finding_player).type == 2 %}
+                                <i>{{ player_names_with_alias[(team, hint.finding_player)] }}</i>
+                            {% else %}
+                                {{ player_names_with_alias[(team, hint.finding_player)] }}
+                            {% endif %}
+                        </td>
+                        <td>
+                            {% if get_slot_info(team, hint.receiving_player).type == 2 %}
+                                <i>{{ player_names_with_alias[(team, hint.receiving_player)] }}</i>
+                            {% else %}
+                                {{ player_names_with_alias[(team, hint.receiving_player)] }}
+                            {% endif %}
+                        </td>
                         <td>{{ item_id_to_name[games[(team, hint.receiving_player)]][hint.item] }}</td>
                         <td>{{ location_id_to_name[games[(team, hint.finding_player)]][hint.location] }}</td>
                         <td>{{ games[(team, hint.finding_player)] }}</td>

WebHostLib/templates/playerOptions/playerOptions.html

@@ -42,7 +42,7 @@ <h1>Player Options</h1>
             A list of all games you have generated can be found on the <a href="/user-content">User Content Page</a>.
             <br />
             You may also download the
-            <a href="/static/generated/configs/{{ world_name }}.yaml">template file for this game</a>.
+            <a href="/static/generated/configs/{{ world_name | get_file_safe_name }}.yaml">template file for this game</a>.
         </p>
 
         <form id="options-form" method="post" enctype="application/x-www-form-urlencoded" action="generate-yaml">

WebHostLib/templates/tutorial.html

@@ -11,7 +11,7 @@
 {% endblock %}
 
 {% block body %}
-    <div id="tutorial-wrapper" class="markdown" data-game="{{ game }}" data-file="{{ file }}" data-lang="{{ lang }}">
+    <div id="tutorial-wrapper" class="markdown" data-game="{{ game | get_file_safe_name }}" data-file="{{ file | get_file_safe_name }}" data-lang="{{ lang }}">
         <!-- Content generated by JavaScript -->
     </div>
 {%  endblock %}

WebHostLib/tracker.py

@@ -5,7 +5,7 @@
 from uuid import UUID
 from email.utils import parsedate_to_datetime
 
-from flask import render_template, make_response, Response, request
+from flask import make_response, render_template, request, Request, Response
 from werkzeug.exceptions import abort
 
 from MultiServer import Context, get_saving_second
@@ -298,17 +298,25 @@ def get_spheres(self) -> List[List[int]]:
         return self._multidata.get("spheres", [])
 
 
-def _process_if_request_valid(incoming_request, room: Optional[Room]) -> Optional[Response]:
+def _process_if_request_valid(incoming_request: Request, room: Optional[Room]) -> Optional[Response]:
     if not room:
         abort(404)
 
-    if_modified = incoming_request.headers.get("If-Modified-Since", None)
-    if if_modified:
-        if_modified = parsedate_to_datetime(if_modified)
+    if_modified_str: Optional[str] = incoming_request.headers.get("If-Modified-Since", None)
+    if if_modified_str:
+        if_modified = parsedate_to_datetime(if_modified_str)
+        if if_modified.tzinfo is None:
+            abort(400)  # standard requires "GMT" timezone
+        # database may use datetime.utcnow(), which is timezone-naive. convert to timezone-aware.
+        last_activity = room.last_activity
+        if last_activity.tzinfo is None:
+            last_activity = room.last_activity.replace(tzinfo=datetime.timezone.utc)
         # if_modified has less precision than last_activity, so we bring them to same precision
-        if if_modified >= room.last_activity.replace(microsecond=0):
+        if if_modified >= last_activity.replace(microsecond=0):
             return make_response("",  304)
 
+    return None
+
 
 @app.route("/tracker/<suuid:tracker>/<int:tracked_team>/<int:tracked_player>")
 def get_player_tracker(tracker: UUID, tracked_team: int, tracked_player: int, generic: bool = False) -> Response:
@@ -415,6 +423,7 @@ def render_generic_tracker(tracker_data: TrackerData, team: int, player: int) ->
         template_name_or_list="genericTracker.html",
         game_specific_tracker=game in _player_trackers,
         room=tracker_data.room,
+        get_slot_info=tracker_data.get_slot_info,
         team=team,
         player=player,
         player_name=tracker_data.get_room_long_player_names()[team, player],
@@ -438,6 +447,7 @@ def render_generic_multiworld_tracker(tracker_data: TrackerData, enabled_tracker
         enabled_trackers=enabled_trackers,
         current_tracker="Generic",
         room=tracker_data.room,
+        get_slot_info=tracker_data.get_slot_info,
         all_slots=tracker_data.get_all_slots(),
         room_players=tracker_data.get_all_players(),
         locations=tracker_data.get_room_locations(),
@@ -489,7 +499,7 @@ def render_Factorio_multiworld_tracker(tracker_data: TrackerData, enabled_tracke
             (team, player): collections.Counter({
                 tracker_data.item_id_to_name["Factorio"][item_id]: count
                 for item_id, count in tracker_data.get_player_inventory_counts(team, player).items()
-            }) for team, players in tracker_data.get_all_slots().items() for player in players
+            }) for team, players in tracker_data.get_all_players().items() for player in players
             if tracker_data.get_player_game(team, player) == "Factorio"
         }
 
@@ -498,6 +508,7 @@ def render_Factorio_multiworld_tracker(tracker_data: TrackerData, enabled_tracke
             enabled_trackers=enabled_trackers,
             current_tracker="Factorio",
             room=tracker_data.room,
+            get_slot_info=tracker_data.get_slot_info,
             all_slots=tracker_data.get_all_slots(),
             room_players=tracker_data.get_all_players(),
             locations=tracker_data.get_room_locations(),
@@ -630,6 +641,7 @@ def render_ALinkToThePast_multiworld_tracker(tracker_data: TrackerData, enabled_
             enabled_trackers=enabled_trackers,
             current_tracker="A Link to the Past",
             room=tracker_data.room,
+            get_slot_info=tracker_data.get_slot_info,
             all_slots=tracker_data.get_all_slots(),
             room_players=tracker_data.get_all_players(),
             locations=tracker_data.get_room_locations(),

data/options.yaml

@@ -28,9 +28,9 @@
 name: Player{number}
 
 # Used to describe your yaml. Useful if you have multiple files.
-description: Default {{ game }} Template
+description: {{ yaml_dump("Default %s Template" % game) }}
 
-game: {{ game }}
+game: {{ yaml_dump(game) }}
 requires:
   version: {{ __version__ }} # Version of Archipelago required for this yaml to work as expected.
 
@@ -44,7 +44,7 @@ requires:
     {%- endfor -%}
 {% endmacro %}
 
-{{ game }}:
+{{ yaml_dump(game) }}:
   {%- for group_name, group_options in option_groups.items() %}
   # {{ group_name }}