In worker, populate small batches of game statistics at a time

ark_nova_stats/BUILD.bazel

@@ -6,6 +6,7 @@ py_library(
  visibility = ["//ark_nova_stats:__subpackages__"],
  deps = [
  ":config_py",
+ "//ark_nova_stats/bga_log_parser:exceptions",
  "//ark_nova_stats/bga_log_parser:game_log",
  "@py_deps//sqlalchemy",
  ],

ark_nova_stats/api/migrations/versions/deca8621de11_create_game_statistics_table.py

@@ -22,7 +22,7 @@ def upgrade():
  "game_statistics",
  sa.Column("id", sa.Integer, primary_key=True, autoincrement=True),
  sa.Column("bga_table_id", sa.Integer, nullable=False),
- sa.Column("user_id", sa.Integer, nullable=False),
+ sa.Column("bga_user_id", sa.Integer, nullable=False),
  sa.Column("created_at", sa.DateTime, default=now, nullable=False),
  # statistics. see PlayerStats in protobuf.
  sa.Column("score", sa.Integer, nullable=False),
@@ -31,7 +31,7 @@ def upgrade():
  sa.Column("starting_position", sa.Integer, nullable=False),
  sa.Column("turns", sa.Integer, nullable=False),
  sa.Column("breaks_triggered", sa.Integer, nullable=False),
- sa.Column("triggered_end", sa.Integer, nullable=False),
+ sa.Column("triggered_end", sa.Boolean, nullable=False),
  sa.Column("map_id", sa.Integer, nullable=False),
  sa.Column("appeal", sa.Integer, nullable=False),
  sa.Column("conservation", sa.Integer, nullable=False),
@@ -60,7 +60,7 @@ def upgrade():
  sa.Column("cards_discarded", sa.Integer, nullable=False),
  sa.Column("played_sponsors", sa.Integer, nullable=False),
  sa.Column("played_animals", sa.Integer, nullable=False),
- sa.Column("release_animals", sa.Integer, nullable=False),
+ sa.Column("released_animals", sa.Integer, nullable=False),
  sa.Column("association_workers", sa.Integer, nullable=False),
  sa.Column("association_donations", sa.Integer, nullable=False),
  sa.Column("association_reputation_actions", sa.Integer, nullable=False),
@@ -75,7 +75,7 @@ def upgrade():
  sa.Column("built_unique_buildings", sa.Integer, nullable=False),
  sa.Column("hexes_covered", sa.Integer, nullable=False),
  sa.Column("hexes_empty", sa.Integer, nullable=False),
- sa.Column("upgraded_action_cards", sa.Boolean, nullable=False),
+ sa.Column("upgraded_action_cards", sa.Integer, nullable=False),
  sa.Column("upgraded_animals", sa.Boolean, nullable=False),
  sa.Column("upgraded_build", sa.Boolean, nullable=False),
  sa.Column("upgraded_cards", sa.Boolean, nullable=False),

ark_nova_stats/models.py

@@ -5,6 +5,7 @@
 from sqlalchemy import ForeignKey, Select, desc, func, select
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
+from ark_nova_stats.bga_log_parser.exceptions import StatsNotSetError
 from ark_nova_stats.bga_log_parser.game_log import GameLog as ParsedGameLog
 from ark_nova_stats.config import db
 
@@ -127,77 +128,81 @@ def create_card_and_plays(
  def create_game_statistics(
  self, parsed_logs: ParsedGameLog
  ) -> Generator[db.Model, None, None]:
- for s in parsed_logs.parse_game_stats().player_stats:
- yield GameStatistics(
- bga_table_id=parsed_logs.table_id,
- bga_user_id=s.player_id,
- score=s.score,
- rank=s.rank,
- thinking_time=s.thinking_time,
- starting_position=s.starting_position,
- turns=s.turns,
- breaks_triggered=s.breaks_triggered,
- triggered_end=s.triggered_end,
- map_id=s.map_id,
- appeal=s.appeal,
- conservation=s.conservation,
- reputation=s.reputation,
- actions_build=s.actions_build,
- actions_animals=s.actions_animals,
- actions_cards=s.actions_cards,
- actions_association=s.actions_association,
- actions_sponsors=s.actions_sponsors,
- x_tokens_gained=s.x_tokens_gained,
- x_actions=s.x_actions,
- x_tokens_used=s.x_tokens_used,
- money_gained=s.money_gained,
- money_gained_through_income=s.money_gained_through_income,
- money_spent_on_animals=s.money_spent_on_animals,
- money_spent_on_enclosures=s.money_spent_on_enclosures,
- money_spent_on_donations=s.money_spent_on_donations,
- money_spent_on_playing_cards_from_reputation_range=s.money_spent_on_playing_cards_from_reputation_range,
- cards_drawn_from_deck=s.cards_drawn_from_deck,
- cards_drawn_from_reputation_range=s.cards_drawn_from_reputation_range,
- cards_snapped=s.cards_snapped,
- cards_discarded=s.cards_discarded,
- played_sponsors=s.played_sponsors,
- played_animals=s.played_animals,
- release_animals=s.release_animals,
- association_workers=s.association_workers,
- association_donations=s.association_donations,
- association_reputation_actions=s.association_reputation_actions,
- association_partner_zoo_actions=s.association_partner_zoo_actions,
- association_university_actions=s.association_university_actions,
- association_conservation_project_actions=s.association_conservation_project_actions,
- built_enclosures=s.built_enclosures,
- built_kiosks=s.built_kiosks,
- built_pavilions=s.built_pavilions,
- built_unique_buildings=s.built_unique_buildings,
- hexes_covered=s.hexes_covered,
- hexes_empty=s.hexes_empty,
- upgraded_action_cards=s.upgraded_action_cards,
- upgraded_animals=s.upgraded_animals,
- upgraded_build=s.upgraded_build,
- upgraded_cards=s.upgraded_cards,
- upgraded_sponsors=s.upgraded_sponsors,
- upgraded_association=s.upgraded_association,
- icons_africa=s.icons_africa,
- icons_europe=s.icons_europe,
- icons_asia=s.icons_asia,
- icons_australia=s.icons_australia,
- icons_americas=s.icons_americas,
- icons_bird=s.icons_bird,
- icons_predator=s.icons_predator,
- icons_herbivore=s.icons_herbivore,
- icons_bear=s.icons_bear,
- icons_reptile=s.icons_reptile,
- icons_primate=s.icons_primate,
- icons_petting_zoo=s.icons_petting_zoo,
- icons_sea_animal=s.icons_sea_animal,
- icons_water=s.icons_water,
- icons_rock=s.icons_rock,
- icons_science=s.icons_science,
- )
+ try:
+ for s in parsed_logs.parse_game_stats().player_stats:
+ yield GameStatistics(
+ bga_table_id=parsed_logs.table_id,
+ bga_user_id=s.player_id,
+ score=s.score,
+ rank=s.rank,
+ thinking_time=s.thinking_time,
+ starting_position=s.starting_position,
+ turns=s.turns,
+ breaks_triggered=s.breaks_triggered,
+ triggered_end=s.triggered_end,
+ map_id=s.map_id,
+ appeal=s.appeal,
+ conservation=s.conservation,
+ reputation=s.reputation,
+ actions_build=s.actions_build,
+ actions_animals=s.actions_animals,
+ actions_cards=s.actions_cards,
+ actions_association=s.actions_association,
+ actions_sponsors=s.actions_sponsors,
+ x_tokens_gained=s.x_tokens_gained,
+ x_actions=s.x_actions,
+ x_tokens_used=s.x_tokens_used,
+ money_gained=s.money_gained,
+ money_gained_through_income=s.money_gained_through_income,
+ money_spent_on_animals=s.money_spent_on_animals,
+ money_spent_on_enclosures=s.money_spent_on_enclosures,
+ money_spent_on_donations=s.money_spent_on_donations,
+ money_spent_on_playing_cards_from_reputation_range=s.money_spent_on_playing_cards_from_reputation_range,
+ cards_drawn_from_deck=s.cards_drawn_from_deck,
+ cards_drawn_from_reputation_range=s.cards_drawn_from_reputation_range,
+ cards_snapped=s.cards_snapped,
+ cards_discarded=s.cards_discarded,
+ played_sponsors=s.played_sponsors,
+ played_animals=s.played_animals,
+ released_animals=s.released_animals,
+ association_workers=s.association_workers,
+ association_donations=s.association_donations,
+ association_reputation_actions=s.association_reputation_actions,
+ association_partner_zoo_actions=s.association_partner_zoo_actions,
+ association_university_actions=s.association_university_actions,
+ association_conservation_project_actions=s.association_conservation_project_actions,
+ built_enclosures=s.built_enclosures,
+ built_kiosks=s.built_kiosks,
+ built_pavilions=s.built_pavilions,
+ built_unique_buildings=s.built_unique_buildings,
+ hexes_covered=s.hexes_covered,
+ hexes_empty=s.hexes_empty,
+ upgraded_action_cards=s.upgraded_action_cards,
+ upgraded_animals=s.upgraded_animals,
+ upgraded_build=s.upgraded_build,
+ upgraded_cards=s.upgraded_cards,
+ upgraded_sponsors=s.upgraded_sponsors,
+ upgraded_association=s.upgraded_association,
+ icons_africa=s.icons_africa,
+ icons_europe=s.icons_europe,
+ icons_asia=s.icons_asia,
+ icons_australia=s.icons_australia,
+ icons_americas=s.icons_americas,
+ icons_bird=s.icons_bird,
+ icons_predator=s.icons_predator,
+ icons_herbivore=s.icons_herbivore,
+ icons_bear=s.icons_bear,
+ icons_reptile=s.icons_reptile,
+ icons_primate=s.icons_primate,
+ icons_petting_zoo=s.icons_petting_zoo,
+ icons_sea_animal=s.icons_sea_animal,
+ icons_water=s.icons_water,
+ icons_rock=s.icons_rock,
+ icons_science=s.icons_science,
+ )
+ except StatsNotSetError:
+ # Stats are not set on this replay; bail.
+ return
 
 
 class User(db.Model):
@@ -442,7 +447,7 @@ class GameStatistics(db.Model):
  starting_position: Mapped[int]
  turns: Mapped[int]
  breaks_triggered: Mapped[int]
- triggered_end: Mapped[int]
+ triggered_end: Mapped[bool]
  map_id: Mapped[int]
  appeal: Mapped[int]
  conservation: Mapped[int]
@@ -467,7 +472,7 @@ class GameStatistics(db.Model):
  cards_discarded: Mapped[int]
  played_sponsors: Mapped[int]
  played_animals: Mapped[int]
- release_animals: Mapped[int]
+ released_animals: Mapped[int]
  association_workers: Mapped[int]
  association_donations: Mapped[int]
  association_reputation_actions: Mapped[int]
@@ -480,7 +485,7 @@ class GameStatistics(db.Model):
  built_unique_buildings: Mapped[int]
  hexes_covered: Mapped[int]
  hexes_empty: Mapped[int]
- upgraded_action_cards: Mapped[bool]
+ upgraded_action_cards: Mapped[int]
  upgraded_animals: Mapped[bool]
  upgraded_build: Mapped[bool]
  upgraded_cards: Mapped[bool]

ark_nova_stats/worker/app.py

@@ -9,14 +9,20 @@
 
 from ark_nova_stats.bga_log_parser.game_log import GameLog as BGAGameLog
 from ark_nova_stats.config import app, db
-from ark_nova_stats.models import Card, CardPlay, GameLog, GameLogArchive
+from ark_nova_stats.models import (
+ Card,
+ CardPlay,
+ GameLog,
+ GameLogArchive,
+ GameStatistics,
+)
 from ark_nova_stats.worker.archives import (
  BGAWithELOArchiveCreator,
  RawBGALogArchiveCreator,
  TopLevelStatsCsvArchiveCreator,
 )
 
-max_delay = 12 * 60 * 60
+max_delay = 10
 
 logging.basicConfig(
  format="[%(asctime)s][%(levelname)s] %(message)s", level=logging.INFO
@@ -148,6 +154,20 @@ def populate_game_log_start_end() -> None:
  logger.info(f"Done populating {updated} game log starts & ends!")
 
 
+def populate_game_statistics() -> None:
+ logger.info(f"Populating game statistics.")
+ updated = 0
+ for game_log in GameLog.query.outerjoin(GameStatistics).where(GameStatistics.bga_table_id == None).limit(25).yield_per(10): # type: ignore
+ parsed_log = BGAGameLog(**json.loads(game_log.log))
+ for s in game_log.create_game_statistics(parsed_log):
+ db.session.add(s)
+ updated += 1
+
+ logger.info(f"Committing {updated} populated game statistics.")
+ db.session.commit()
+ logger.info(f"Done populating {updated} game statistics!")
+
+
 API_SECRET_KEY = os.getenv("API_WORKER_SECRET")
 
 
@@ -160,6 +180,7 @@ def main() -> int:
  archive_logs_to_tigris(tigris_client)
  # populate_card_play_actions()
  # populate_game_log_start_end()
+ populate_game_statistics()
  delay = (start + max_delay) - time.time()
  if delay > 0:
  time.sleep(delay)