Skip to content

Commit

Permalink
Merge branch 'main' into test_world_determinism
Browse files Browse the repository at this point in the history
  • Loading branch information
alwaysintreble authored Nov 25, 2024
2 parents 00507ad + a650e90 commit a3b6408
Show file tree
Hide file tree
Showing 218 changed files with 1,995 additions and 1,018 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/codeql-analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ jobs:

# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
Expand All @@ -58,7 +58,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v2
uses: github/codeql-action/autobuild@v3

# ℹ️ Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
Expand All @@ -72,4 +72,4 @@ jobs:
# make release

- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2
uses: github/codeql-action/analyze@v3
2 changes: 1 addition & 1 deletion .github/workflows/unittests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -89,4 +89,4 @@ jobs:
run: |
source venv/bin/activate
export PYTHONPATH=$(pwd)
python test/hosting/__main__.py
timeout 600 python test/hosting/__main__.py
6 changes: 5 additions & 1 deletion BaseClasses.py
Original file line number Diff line number Diff line change
Expand Up @@ -341,7 +341,7 @@ def find_common_pool(players: Set[int], shared_pool: Set[str]) -> Tuple[
new_item.classification |= classifications[item_name]
new_itempool.append(new_item)

region = Region("Menu", group_id, self, "ItemLink")
region = Region(group["world"].origin_region_name, group_id, self, "ItemLink")
self.regions.append(region)
locations = region.locations
# ensure that progression items are linked first, then non-progression
Expand Down Expand Up @@ -1264,6 +1264,10 @@ def useful(self) -> bool:
def trap(self) -> bool:
return ItemClassification.trap in self.classification

@property
def excludable(self) -> bool:
return not (self.advancement or self.useful)

@property
def flags(self) -> int:
return self.classification.as_flag()
Expand Down
5 changes: 5 additions & 0 deletions CommonClient.py
Original file line number Diff line number Diff line change
Expand Up @@ -710,6 +710,11 @@ def run_gui(self):

def run_cli(self):
if sys.stdin:
if sys.stdin.fileno() != 0:
from multiprocessing import parent_process
if parent_process():
return # ignore MultiProcessing pipe

# steam overlay breaks when starting console_loop
if 'gameoverlayrenderer' in os.environ.get('LD_PRELOAD', ''):
logger.info("Skipping terminal input, due to conflicting Steam Overlay detected. Please use GUI only.")
Expand Down
6 changes: 5 additions & 1 deletion Generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ def main(args=None) -> Tuple[argparse.Namespace, int]:
player_files = {}
for file in os.scandir(args.player_files_path):
fname = file.name
if file.is_file() and not fname.startswith(".") and \
if file.is_file() and not fname.startswith(".") and not fname.lower().endswith(".ini") and \
os.path.join(args.player_files_path, fname) not in {args.meta_file_path, args.weights_file_path}:
path = os.path.join(args.player_files_path, fname)
try:
Expand Down Expand Up @@ -453,6 +453,10 @@ def roll_settings(weights: dict, plando_options: PlandoOptions = PlandoOptions.b
raise Exception(f"Option {option_key} has to be in a game's section, not on its own.")

ret.game = get_choice("game", weights)
if not isinstance(ret.game, str):
if ret.game is None:
raise Exception('"game" not specified')
raise Exception(f"Invalid game: {ret.game}")
if ret.game not in AutoWorldRegister.world_types:
from worlds import failed_world_loads
picks = Utils.get_fuzzy_results(ret.game, list(AutoWorldRegister.world_types) + failed_world_loads, limit=1)[0]
Expand Down
19 changes: 12 additions & 7 deletions Launcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,16 +22,15 @@
from shutil import which
from typing import Callable, Optional, Sequence, Tuple, Union

import Utils
import settings
from worlds.LauncherComponents import Component, components, Type, SuffixIdentifier, icon_paths

if __name__ == "__main__":
import ModuleUpdate
ModuleUpdate.update()

from Utils import is_frozen, user_path, local_path, init_logging, open_filename, messagebox, \
is_windows, is_macos, is_linux
import settings
import Utils
from Utils import (init_logging, is_frozen, is_linux, is_macos, is_windows, local_path, messagebox, open_filename,
user_path)
from worlds.LauncherComponents import Component, components, icon_paths, SuffixIdentifier, Type


def open_host_yaml():
Expand Down Expand Up @@ -104,6 +103,7 @@ def update_settings():
Component("Open host.yaml", func=open_host_yaml),
Component("Open Patch", func=open_patch),
Component("Generate Template Options", func=generate_yamls),
Component("Archipelago Website", func=lambda: webbrowser.open("https://archipelago.gg/")),
Component("Discord Server", icon="discord", func=lambda: webbrowser.open("https://discord.gg/8Z65BR2")),
Component("Unrated/18+ Discord Server", icon="discord", func=lambda: webbrowser.open("https://discord.gg/fqvNCCRsu4")),
Component("Browse Files", func=browse_files),
Expand Down Expand Up @@ -181,6 +181,11 @@ def update_label(self, dt):
App.get_running_app().stop()
Window.close()

def _stop(self, *largs):
# see run_gui Launcher _stop comment for details
self.root_window.close()
super()._stop(*largs)

Popup().run()


Expand Down Expand Up @@ -254,7 +259,7 @@ class Launcher(App):
_client_layout: Optional[ScrollBox] = None

def __init__(self, ctx=None):
self.title = self.base_title
self.title = self.base_title + " " + Utils.__version__
self.ctx = ctx
self.icon = r"data/icon.png"
super().__init__()
Expand Down
24 changes: 13 additions & 11 deletions MultiServer.py
Original file line number Diff line number Diff line change
Expand Up @@ -727,15 +727,15 @@ def notify_hints(self, team: int, hints: typing.List[NetUtils.Hint], only_new: b
if not hint.local and data not in concerns[hint.finding_player]:
concerns[hint.finding_player].append(data)
# remember hints in all cases
if not hint.found:
# since hints are bidirectional, finding player and receiving player,
# we can check once if hint already exists
if hint not in self.hints[team, hint.finding_player]:
self.hints[team, hint.finding_player].add(hint)
new_hint_events.add(hint.finding_player)
for player in self.slot_set(hint.receiving_player):
self.hints[team, player].add(hint)
new_hint_events.add(player)

# since hints are bidirectional, finding player and receiving player,
# we can check once if hint already exists
if hint not in self.hints[team, hint.finding_player]:
self.hints[team, hint.finding_player].add(hint)
new_hint_events.add(hint.finding_player)
for player in self.slot_set(hint.receiving_player):
self.hints[team, player].add(hint)
new_hint_events.add(player)

self.logger.info("Notice (Team #%d): %s" % (team + 1, format_hint(self, team, hint)))
for slot in new_hint_events:
Expand Down Expand Up @@ -1960,8 +1960,10 @@ def _cmd_status(self, tag: str = "") -> bool:

def _cmd_exit(self) -> bool:
"""Shutdown the server"""
self.ctx.server.ws_server.close()
self.ctx.exit_event.set()
try:
self.ctx.server.ws_server.close()
finally:
self.ctx.exit_event.set()
return True

@mark_raw
Expand Down
4 changes: 2 additions & 2 deletions Options.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from schema import And, Optional, Or, Schema
from typing_extensions import Self

from Utils import get_fuzzy_results, is_iterable_except_str, output_path
from Utils import get_file_safe_name, get_fuzzy_results, is_iterable_except_str, output_path

if typing.TYPE_CHECKING:
from BaseClasses import MultiWorld, PlandoOptions
Expand Down Expand Up @@ -1531,7 +1531,7 @@ def yaml_dump_scalar(scalar) -> str:

del file_data

with open(os.path.join(target_folder, game_name + ".yaml"), "w", encoding="utf-8-sig") as f:
with open(os.path.join(target_folder, get_file_safe_name(game_name) + ".yaml"), "w", encoding="utf-8-sig") as f:
f.write(res)


Expand Down
16 changes: 14 additions & 2 deletions SNIClient.py
Original file line number Diff line number Diff line change
Expand Up @@ -633,7 +633,13 @@ async def game_watcher(ctx: SNIContext) -> None:
if not ctx.client_handler:
continue

rom_validated = await ctx.client_handler.validate_rom(ctx)
try:
rom_validated = await ctx.client_handler.validate_rom(ctx)
except Exception as e:
snes_logger.error(f"An error occurred, see logs for details: {e}")
text_file_logger = logging.getLogger()
text_file_logger.exception(e)
rom_validated = False

if not rom_validated or (ctx.auth and ctx.auth != ctx.rom):
snes_logger.warning("ROM change detected, please reconnect to the multiworld server")
Expand All @@ -649,7 +655,13 @@ async def game_watcher(ctx: SNIContext) -> None:

perf_counter = time.perf_counter()

await ctx.client_handler.game_watcher(ctx)
try:
await ctx.client_handler.game_watcher(ctx)
except Exception as e:
snes_logger.error(f"An error occurred, see logs for details: {e}")
text_file_logger = logging.getLogger()
text_file_logger.exception(e)
await snes_disconnect(ctx)


async def run_game(romfile: str) -> None:
Expand Down
45 changes: 38 additions & 7 deletions Utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

from argparse import Namespace
from settings import Settings, get_settings
from time import sleep
from typing import BinaryIO, Coroutine, Optional, Set, Dict, Any, Union
from typing_extensions import TypeGuard
from yaml import load, load_all, dump
Expand All @@ -31,6 +32,7 @@
import tkinter
import pathlib
from BaseClasses import Region
import multiprocessing


def tuplize_version(version: str) -> Version:
Expand Down Expand Up @@ -567,6 +569,8 @@ def queuer():
else:
if text:
queue.put_nowait(text)
else:
sleep(0.01) # non-blocking stream

from threading import Thread
thread = Thread(target=queuer, name=f"Stream handler for {stream.name}", daemon=True)
Expand Down Expand Up @@ -664,6 +668,19 @@ def get_input_text_from_response(text: str, command: str) -> typing.Optional[str
return None


def is_kivy_running() -> bool:
if "kivy" in sys.modules:
from kivy.app import App
return App.get_running_app() is not None
return False


def _mp_open_filename(res: "multiprocessing.Queue[typing.Optional[str]]", *args: Any) -> None:
if is_kivy_running():
raise RuntimeError("kivy should not be running in multiprocess")
res.put(open_filename(*args))


def open_filename(title: str, filetypes: typing.Iterable[typing.Tuple[str, typing.Iterable[str]]], suggest: str = "") \
-> typing.Optional[str]:
logging.info(f"Opening file input dialog for {title}.")
Expand Down Expand Up @@ -693,6 +710,13 @@ def run(*args: str):
f'This attempt was made because open_filename was used for "{title}".')
raise e
else:
if is_macos and is_kivy_running():
# on macOS, mixing kivy and tk does not work, so spawn a new process
# FIXME: performance of this is pretty bad, and we should (also) look into alternatives
from multiprocessing import Process, Queue
res: "Queue[typing.Optional[str]]" = Queue()
Process(target=_mp_open_filename, args=(res, title, filetypes, suggest)).start()
return res.get()
try:
root = tkinter.Tk()
except tkinter.TclError:
Expand All @@ -702,6 +726,12 @@ def run(*args: str):
initialfile=suggest or None)


def _mp_open_directory(res: "multiprocessing.Queue[typing.Optional[str]]", *args: Any) -> None:
if is_kivy_running():
raise RuntimeError("kivy should not be running in multiprocess")
res.put(open_directory(*args))


def open_directory(title: str, suggest: str = "") -> typing.Optional[str]:
def run(*args: str):
return subprocess.run(args, capture_output=True, text=True).stdout.split("\n", 1)[0] or None
Expand All @@ -725,9 +755,16 @@ def run(*args: str):
import tkinter.filedialog
except Exception as e:
logging.error('Could not load tkinter, which is likely not installed. '
f'This attempt was made because open_filename was used for "{title}".')
f'This attempt was made because open_directory was used for "{title}".')
raise e
else:
if is_macos and is_kivy_running():
# on macOS, mixing kivy and tk does not work, so spawn a new process
# FIXME: performance of this is pretty bad, and we should (also) look into alternatives
from multiprocessing import Process, Queue
res: "Queue[typing.Optional[str]]" = Queue()
Process(target=_mp_open_directory, args=(res, title, suggest)).start()
return res.get()
try:
root = tkinter.Tk()
except tkinter.TclError:
Expand All @@ -740,12 +777,6 @@ def messagebox(title: str, text: str, error: bool = False) -> None:
def run(*args: str):
return subprocess.run(args, capture_output=True, text=True).stdout.split("\n", 1)[0] or None

def is_kivy_running():
if "kivy" in sys.modules:
from kivy.app import App
return App.get_running_app() is not None
return False

if is_kivy_running():
from kvui import MessageBox
MessageBox(title, text, error).open()
Expand Down
3 changes: 2 additions & 1 deletion WebHost.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
# in case app gets imported by something like gunicorn
import Utils
import settings
from Utils import get_file_safe_name

if typing.TYPE_CHECKING:
from flask import Flask
Expand Down Expand Up @@ -71,7 +72,7 @@ def create_ordered_tutorials_file() -> typing.List[typing.Dict[str, typing.Any]]
shutil.rmtree(base_target_path, ignore_errors=True)
for game, world in worlds.items():
# copy files from world's docs folder to the generated folder
target_path = os.path.join(base_target_path, game)
target_path = os.path.join(base_target_path, get_file_safe_name(game))
os.makedirs(target_path, exist_ok=True)

if world.zip_path:
Expand Down
3 changes: 2 additions & 1 deletion WebHostLib/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from pony.flask import Pony
from werkzeug.routing import BaseConverter

from Utils import title_sorted
from Utils import title_sorted, get_file_safe_name

UPLOAD_FOLDER = os.path.relpath('uploads')
LOGS_FOLDER = os.path.relpath('logs')
Expand All @@ -20,6 +20,7 @@

app.jinja_env.filters['any'] = any
app.jinja_env.filters['all'] = all
app.jinja_env.filters['get_file_safe_name'] = get_file_safe_name

app.config["SELFHOST"] = True # application process is in charge of running the websites
app.config["GENERATORS"] = 8 # maximum concurrent world gens
Expand Down
16 changes: 14 additions & 2 deletions WebHostLib/misc.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,13 @@ def faq(lang: str):
return render_template(
"markdown_document.html",
title="Frequently Asked Questions",
html_from_markdown=markdown.markdown(document, extensions=["mdx_breakless_lists"]),
html_from_markdown=markdown.markdown(
document,
extensions=["toc", "mdx_breakless_lists"],
extension_configs={
"toc": {"anchorlink": True}
}
),
)


Expand All @@ -90,7 +96,13 @@ def glossary(lang: str):
return render_template(
"markdown_document.html",
title="Glossary",
html_from_markdown=markdown.markdown(document, extensions=["mdx_breakless_lists"]),
html_from_markdown=markdown.markdown(
document,
extensions=["toc", "mdx_breakless_lists"],
extension_configs={
"toc": {"anchorlink": True}
}
),
)


Expand Down
2 changes: 1 addition & 1 deletion WebHostLib/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
flask>=3.0.3
werkzeug>=3.0.4
werkzeug>=3.0.6
pony>=0.7.19
waitress>=3.0.0
Flask-Caching>=2.3.0
Expand Down
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Binary file modified WebHostLib/static/static/backgrounds/cliffs/grass/cliff-left.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Binary file modified WebHostLib/static/static/backgrounds/cliffs/grass/cliff-top.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file added WebHostLib/static/static/backgrounds/dirt.webp
Binary file not shown.
Binary file modified WebHostLib/static/static/backgrounds/footer/footer-0001.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Binary file modified WebHostLib/static/static/backgrounds/footer/footer-0002.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Binary file modified WebHostLib/static/static/backgrounds/footer/footer-0003.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Binary file modified WebHostLib/static/static/backgrounds/footer/footer-0004.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Binary file modified WebHostLib/static/static/backgrounds/footer/footer-0005.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Binary file not shown.
Binary file added WebHostLib/static/static/backgrounds/grass.webp
Binary file not shown.
Binary file modified WebHostLib/static/static/backgrounds/header/dirt-header.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Binary file modified WebHostLib/static/static/backgrounds/header/grass-header.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Binary file modified WebHostLib/static/static/backgrounds/header/ocean-header.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Binary file modified WebHostLib/static/static/backgrounds/header/stone-header.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Binary file added WebHostLib/static/static/backgrounds/ice.webp
Binary file not shown.
Binary file not shown.
Binary file modified WebHostLib/static/static/backgrounds/ocean.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added WebHostLib/static/static/backgrounds/ocean.webp
Binary file not shown.
Binary file not shown.
Binary file added WebHostLib/static/static/backgrounds/stone.webp
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file modified WebHostLib/static/static/button-images/island-button-a.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Binary file modified WebHostLib/static/static/button-images/island-button-b.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Binary file modified WebHostLib/static/static/button-images/island-button-c.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Binary file not shown.
Binary file modified WebHostLib/static/static/decorations/island-a.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Binary file modified WebHostLib/static/static/decorations/island-b.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file not shown.
Binary file modified WebHostLib/static/static/decorations/island-c.png
Binary file not shown.
Binary file not shown.
Binary file not shown.
Loading

0 comments on commit a3b6408

Please sign in to comment.