diff --git a/server/fishtest/__init__.py b/server/fishtest/__init__.py index 028c3496b..33ad3cc50 100644 --- a/server/fishtest/__init__.py +++ b/server/fishtest/__init__.py @@ -142,6 +142,7 @@ def group_finder(username, request): config.add_route("api_get_elo", "/api/get_elo/{id}") config.add_route("api_actions", "/api/actions") config.add_route("api_calc_elo", "/api/calc_elo") + config.add_route("api_serialize_run", "/api/serialize_run/{id}") config.scan() return config.make_wsgi_app() diff --git a/server/fishtest/api.py b/server/fishtest/api.py index e8bc58529..09dd80d6c 100644 --- a/server/fishtest/api.py +++ b/server/fishtest/api.py @@ -7,7 +7,7 @@ from fishtest.schemas import api_access_schema, api_schema, gzip_data from fishtest.stats.stat_util import SPRT_elo, get_elo -from fishtest.util import worker_name +from fishtest.util import serialize, worker_name from pyramid.httpexceptions import ( HTTPBadRequest, HTTPFound, @@ -90,7 +90,9 @@ def __init__(self, request): def handle_error(self, error, exception=HTTPBadRequest): if error != "": - full_url = self.request.route_url(self.request.matched_route.name) + full_url = self.request.route_url( + self.request.matched_route.name, **self.request.matchdict + ) api = urlparse(full_url).path error = f"{api}: {error}" print(error, flush=True) @@ -290,6 +292,25 @@ def finished_runs(self): finished[str(run["_id"])] = run return finished + @view_config(route_name="api_serialize_run") + def serialize_run(self): + self.__t0 = datetime.now(timezone.utc) + try: + run_id = self.request.matchdict["id"] + except Exception as e: + self.handle_error(str(e)) + try: + run = self.request.rundb.get_run(run_id) + except Exception as e: + self.handle_error(str(e)) + if run is None: + self.handle_error("Run does not exist") + try: + s = serialize(run) + except Exception as e: + self.handle_error(str(e)) + return Response(body=s, content_type="application/octet-stream") + @view_config(route_name="api_actions") def actions(self): try: diff --git a/server/fishtest/rundb.py b/server/fishtest/rundb.py index ba136bda7..74199ba82 100644 --- a/server/fishtest/rundb.py +++ b/server/fishtest/rundb.py @@ -12,6 +12,7 @@ from datetime import datetime, timedelta, timezone import fishtest.stats.stat_util +import requests from bson.binary import Binary from bson.codec_options import CodecOptions from bson.objectid import ObjectId @@ -22,6 +23,7 @@ from fishtest.util import ( GeneratorAsFileReader, crash_or_time, + deserialize, estimate_game_duration, format_bounds, format_results, @@ -381,7 +383,14 @@ def get_run(self, r_id): } return run else: - return self.runs.find_one({"_id": ObjectId(r_id)}) + result = requests.get(f"http://localhost/api/serialize_run/{r_id}") + try: + result.raise_for_status() + run = deserialize(result.content) + except Exception as e: + print(f"Failed to deserialize {r_id}: {str(e)}", flush=True) + return None + return run def start_timer(self): self.timer = threading.Timer(1.0, self.flush_buffers) diff --git a/server/fishtest/util.py b/server/fishtest/util.py index b7ea591af..c4b2bcc6c 100644 --- a/server/fishtest/util.py +++ b/server/fishtest/util.py @@ -1,3 +1,5 @@ +import base64 +import gzip import hashlib import math import re @@ -6,6 +8,7 @@ from email.mime.text import MIMEText from functools import cache +import bson import fishtest.stats.stat_util import numpy import scipy.stats @@ -512,3 +515,15 @@ def get_hash(s): if h: return int(h.group(1)) return 0 + + +def serialize(run): + b = bson.encode(run) + g = gzip.compress(b) + return g + + +def deserialize(g): + b = gzip.decompress(g) + run = bson.decode(b) + return run