several improvements to dependency handling

pyproject.toml

@@ -22,6 +22,7 @@ dependencies = [
  "construct>=2.10.0",
  "lzokay",
  "nod>=1.7.0",
+ "typing-extensions>=4.0.0"
 ]
 dynamic = ["version"]
 

src/retro_data_structures/asset_manager.py

@@ -21,7 +21,7 @@
  Resource,
  resolve_asset_id,
 )
-from retro_data_structures.exceptions import UnknownAssetId
+from retro_data_structures.exceptions import DependenciesHandledElsewhere, UnknownAssetId
 from retro_data_structures.formats import Dgrp, dependency_cheating
 from retro_data_structures.formats.audio_group import Agsc, Atbl
 from retro_data_structures.formats.pak import Pak
@@ -116,7 +116,7 @@ class AssetManager:
  _modified_resources: dict[AssetId, RawResource | None]
  _in_memory_paks: dict[str, Pak]
  _custom_asset_ids: dict[str, AssetId]
- _audio_group_dependency: Dgrp | None = None
+ _audio_group_dependency: tuple[Dgrp, ...] | None = None
 
  _cached_dependencies: dict[AssetId, tuple[Dependency, ...]]
  _cached_ancs_per_char_dependencies: defaultdict[AssetId, dict[int, tuple[Dependency, ...]]]
@@ -364,15 +364,9 @@ def ensure_present(self, pak_name: str, asset_id: NameOrAssetId):
  if pak_name not in self._paks_for_asset_id[asset_id]:
  self._ensured_asset_ids[pak_name].add(asset_id)
 
- # Ensure the asset's dependencies are present as well
- for dep in self.get_dependencies_for_asset(asset_id):
- if dep.id == asset_id:
- continue
- self.ensure_present(pak_name, dep.id)
-
  def get_pak(self, pak_name: str) -> Pak:
  if pak_name not in self._ensured_asset_ids:
- raise ValueError(f"Unknown pak_name: {pak_name}")
+ raise ValueError(f"Unknown pak_name: {pak_name}. Known names: {tuple(self._ensured_asset_ids.keys())}")
 
  if pak_name not in self._in_memory_paks:
  logger.info("Reading %s", pak_name)
@@ -408,6 +402,10 @@ def _get_dependencies_for_asset(self, asset_id: NameOrAssetId, must_exist: bool,
  elif formats.has_resource_type(asset_type):
  if self.get_asset_format(asset_id).has_dependencies(self.target_game):
  deps = tuple(self.get_parsed_asset(asset_id).dependencies_for())
+ deps += tuple(self.target_game.special_ancs_dependencies(asset_id))
+
+ else:
+ logger.warning(f"Potential missing assets for {asset_type} {asset_id}")
 
  logger.debug(f"Adding {asset_id:#8x} deps to cache...")
  dep_cache[asset_id] = deps
@@ -417,7 +415,11 @@ def _get_dependencies_for_asset(self, asset_id: NameOrAssetId, must_exist: bool,
 
  def get_dependencies_for_asset(self, asset_id: NameOrAssetId, *, must_exist: bool = False) -> Iterator[Dependency]:
  override = asset_id in self.target_game.mlvl_dependencies_to_ignore
- for it in self._get_dependencies_for_asset(asset_id, must_exist):
+ try:
+ deps = self._get_dependencies_for_asset(asset_id, must_exist)
+ except DependenciesHandledElsewhere:
+ return
+ for it in deps:
  yield Dependency(it.type, it.id, it.exclude_for_mlvl or override)
 
  def get_dependencies_for_ancs(self, asset_id: NameOrAssetId, char_index: int | None = None):
@@ -477,12 +479,17 @@ def get_audio_group_dependency(self, sound_id: int) -> Iterator[Dependency]:
  return
 
  if self._audio_group_dependency is None:
- # audio_groups_single_player_DGRP
- self._audio_group_dependency = self.get_file(0x31CB5ADB)
+ self._audio_group_dependency = tuple(
+ self.get_file(asset, Dgrp)
+ for asset in self.target_game.audio_group_dependencies()
+ )
 
  dep = Dependency("AGSC", agsc, False)
- if dep in self._audio_group_dependency.direct_dependencies:
- yield Dependency("AGSC", agsc, True)
+ if any(
+ (dep in deps.direct_dependencies)
+ for deps in self._audio_group_dependency
+ ):
+ return
  else:
  yield dep
 

src/retro_data_structures/base_resource.py

@@ -1,9 +1,13 @@
 from __future__ import annotations
 
+import logging
 import typing
 import uuid
 
 from construct import Construct, Container
+from typing_extensions import Self
+
+from retro_data_structures.exceptions import DependenciesHandledElsewhere
 
 if typing.TYPE_CHECKING:
  from retro_data_structures.asset_manager import AssetManager
@@ -18,6 +22,13 @@ class Dependency(typing.NamedTuple):
  type: AssetType
  id: AssetId
  exclude_for_mlvl: bool = False
+ can_duplicate: bool = False
+
+ def __repr__(self):
+ s = f"Dep {self.type} 0x{self.id:08X}"
+ if self.exclude_for_mlvl:
+ s += " (non-MLVL)"
+ return s
 
 
 class BaseResource:
@@ -40,7 +51,7 @@ def resource_type(cls) -> AssetType:
 
  @classmethod
  def parse(cls, data: bytes, target_game: Game,
- asset_manager: AssetManager | None = None) -> BaseResource:
+ asset_manager: AssetManager | None = None) -> Self:
  return cls(cls.construct_class(target_game).parse(data, target_game=target_game),
  target_game, asset_manager)
 
@@ -58,6 +69,13 @@ def has_dependencies(cls, target_game: Game) -> bool:
  except (KeyError, AttributeError):
  return True
 
+ except DependenciesHandledElsewhere:
+ return False
+
+ except NotImplementedError:
+ logging.warning("Potential missing dependencies for %s", cls.resource_type())
+ return False
+
  def dependencies_for(self) -> typing.Iterator[Dependency]:
  raise NotImplementedError()
 
@@ -66,18 +84,34 @@ def raw(self) -> Container:
  return self._raw
 
 
+class AssetId32(int):
+ def __repr__(self) -> str:
+ return f"{self:#010x}"
+
+
+class AssetId64(int):
+ def __repr__(self) -> str:
+ return f"{self:#018x}"
+
+
 def resolve_asset_id(game: Game, value: NameOrAssetId) -> AssetId:
  if isinstance(value, str):
  value = game.hash_asset_id(value)
 
- if game.uses_guid_as_asset_id and isinstance(value, int):
- return uuid.UUID(int=value)
+ if isinstance(value, int):
+ if game.uses_guid_as_asset_id:
+ return uuid.UUID(int=value)
+ elif game.uses_asset_id_32:
+ return AssetId32(value)
+ elif game.uses_asset_id_64:
+ return AssetId64(value)
 
  return value
 
 
 class RawResource(typing.NamedTuple):
  type: AssetType
  data: bytes
+ compressed: bool = False
 
 Resource = RawResource | BaseResource

src/retro_data_structures/exceptions.py

@@ -1,5 +1,9 @@
+from __future__ import annotations
 
-from retro_data_structures.base_resource import AssetId
+import typing
+
+if typing.TYPE_CHECKING:
+ from retro_data_structures.base_resource import AssetId
 
 
 def format_asset_id(asset_id: AssetId) -> str:
@@ -23,3 +27,7 @@ def __init__(self, asset_id, reason: str):
  super().__init__(f"Unable to decode asset id {format_asset_id(asset_id)}: {reason}")
  self.asset_id = asset_id
  self.reason = reason
+
+
+class DependenciesHandledElsewhere(Exception):
+ pass

src/retro_data_structures/formats/__init__.py

@@ -31,7 +31,8 @@
 )
 from retro_data_structures.formats.evnt import EVNT, Evnt
 from retro_data_structures.formats.mapa import MAPA, Mapa
-from retro_data_structures.formats.mapw import Mapw
+from retro_data_structures.formats.mapu import MAPU, Mapu
+from retro_data_structures.formats.mapw import MAPW, Mapw
 from retro_data_structures.formats.mlvl import MLVL, Mlvl
 from retro_data_structures.formats.mrea import MREA, Mrea
 from retro_data_structures.formats.msbt import Msbt
@@ -54,6 +55,8 @@
  "DGRP": DGRP,
  "EVNT": EVNT,
  "MAPA": MAPA,
+ "MAPW": MAPW,
+ "MAPU": MAPU,
  "MLVL": MLVL,
  "MREA": MREA,
  "PART": PART,
@@ -84,6 +87,7 @@
  "DGRP": Dgrp,
  "EVNT": Evnt,
  "MAPA": Mapa,
+ "MAPU": Mapu,
  "MAPW": Mapw,
  "MLVL": Mlvl,
  "MREA": Mrea,

src/retro_data_structures/formats/ancs.py

@@ -213,12 +213,14 @@ def construct_class(cls, target_game: Game) -> construct.Construct:
  return ANCS
 
  def ancs_dependencies_for(self, char_index: int | None) -> typing.Iterator[Dependency]:
+ def char_anims(char) -> typing.Iterator[tuple[int, str]]:
+ for anim_name in char.animation_names:
+ yield next((i, a) for i, a in enumerate(self.raw.animation_set.animations)
+ if a.name == anim_name.name)
  def char_deps(char):
  yield from char_dependencies_for(char, self.asset_manager)
 
- for anim_name in char.animation_names:
- anim_index, anim = next((i, a) for i, a in enumerate(self.raw.animation_set.animations)
- if a.name == anim_name.name)
+ for anim_index, anim in char_anims(char):
  yield from meta_animation.dependencies_for(anim.meta, self.asset_manager)
 
  if self.raw.animation_set.animation_resources is not None:

src/retro_data_structures/formats/audio_group.py

@@ -141,6 +141,9 @@ def construct_class(cls, target_game: Game) -> Construct:
  def resource_type(cls) -> AssetType:
  return "ATBL"
 
+ def dependencies_for(self) -> typing.Iterator[Dependency]:
+ yield from []
+
 
 class Agsc(BaseResource):
  @classmethod

src/retro_data_structures/formats/dependency_cheating.py

@@ -7,12 +7,13 @@
 import construct
 
 from retro_data_structures.base_resource import AssetType, Dependency, RawResource
-from retro_data_structures.common_types import AssetId32, String
+from retro_data_structures.common_types import AssetId32, FourCC, String
 from retro_data_structures.construct_extensions.alignment import AlignTo
 from retro_data_structures.data_section import DataSection
 from retro_data_structures.exceptions import UnknownAssetId
 from retro_data_structures.formats import Part, effect_script
 from retro_data_structures.formats.hier import Hier
+from retro_data_structures.formats.tree import Tree
 from retro_data_structures.game_check import Game
 
 if typing.TYPE_CHECKING:
@@ -42,28 +43,42 @@ def _cheat(stream: bytes, asset_manager: AssetManager) -> typing.Iterator[Depend
 
 
 def csng_dependencies(asset: RawResource, asset_manager: AssetManager) -> typing.Iterator[Dependency]:
- yield from asset_manager.get_dependencies_for_asset(_csng.parse(asset))
+ yield from asset_manager.get_dependencies_for_asset(_csng.parse(asset.data))
 
 
 def dumb_dependencies(asset: RawResource, asset_manager: AssetManager) -> typing.Iterator[Dependency]:
- hier = Hier.parse(asset.data, asset_manager.target_game, asset_manager)
- yield from hier.dependencies_for()
+ try:
+ magic = FourCC.parse(asset.data)
+ except Exception:
+ raise UnableToCheatError()
+
+ if magic == "HIER":
+ hier = Hier.parse(asset.data, asset_manager.target_game, asset_manager)
+ yield from hier.dependencies_for()
+ elif magic == "TREE":
+ tree = Tree.parse(asset.data, asset_manager.target_game, asset_manager)
+ yield from tree.dependencies_for()
+ else:
+ raise UnableToCheatError()
 
 
 _frme = construct.FocusedSeq(
  "deps",
  construct.Int32ub,
  "deps" / construct.PrefixedArray(
  construct.Int32ub,
- construct.Int32ub
+ construct.Struct(
+ type=FourCC,
+ id=construct.Int32ub
+ )
  )
 )
 
 
 def frme_dependencies(asset: RawResource, asset_manager: AssetManager) -> typing.Iterator[Dependency]:
  for dep in _frme.parse(asset.data):
  try:
- yield from asset_manager.get_dependencies_for_asset(dep)
+ yield from asset_manager.get_dependencies_for_asset(dep.id)
  except UnknownAssetId:
  raise UnableToCheatError()
 
@@ -233,6 +248,10 @@ def effect_dependencies(asset: RawResource, asset_manager: AssetManager) -> typi
  raise UnableToCheatError()
 
 
+def no_dependencies(asset: RawResource, asset_manager: AssetManager) -> typing.Iterator[Dependency]:
+ yield from []
+
+
 _FORMATS_TO_CHEAT = {
  "CSNG": csng_dependencies,
  "DUMB": dumb_dependencies,
@@ -243,6 +262,12 @@ def effect_dependencies(asset: RawResource, asset_manager: AssetManager) -> typi
  "FONT": font_dependencies,
  "CMDL": cmdl_dependencies,
  "PART": effect_dependencies,
+ "AFSM": no_dependencies,
+ "DCLN": no_dependencies,
+ "STLC": no_dependencies,
+ "PATH": no_dependencies,
+ "EGMC": no_dependencies,
+ "PTLA": no_dependencies,
 }
 
 

src/retro_data_structures/formats/mapu.py

@@ -35,5 +35,3 @@ def construct_class(cls, target_game: Game) -> Construct:
 
  def dependencies_for(self) -> typing.Iterator[Dependency]:
  yield from self.asset_manager.get_dependencies_for_asset(self.raw.hexagon_mapa)
- for world in self.raw.worlds:
- yield from self.asset_manager.get_dependencies_for_asset(world.mlvl)

src/retro_data_structures/formats/mlvl.py

@@ -235,7 +235,22 @@ def construct_class(cls, target_game: Game) -> construct.Construct:
  return MLVL
 
  def dependencies_for(self) -> typing.Iterator[Dependency]:
- raise NotImplementedError()
+ for area in self.areas:
+ area.build_mlvl_dependencies(False)
+ yield from area.dependencies_for()
+
+ mlvl_deps = [
+ self._raw.world_name_id,
+ self._raw.world_save_info_id,
+ self._raw.default_skybox_id
+ ]
+ if self.asset_manager.target_game == Game.ECHOES:
+ mlvl_deps.append(self._raw.dark_world_name_id)
+ if self.asset_manager.target_game <= Game.CORRUPTION:
+ mlvl_deps.append(self._raw.world_map_id)
+
+ for dep in mlvl_deps:
+ yield from self.asset_manager.get_dependencies_for_asset(dep)
 
  def __repr__(self) -> str:
  try: