From f243562ad11444662f2762339f6c708fc1bbc4b1 Mon Sep 17 00:00:00 2001 From: Jerry Date: Fri, 12 May 2023 20:22:25 -0700 Subject: [PATCH] Add utxo and datum cache in Ogmios context Reducing duplicated calls to Ogmios and Kupo --- poetry.lock | 16 ++++++++++-- pycardano/backend/ogmios.py | 49 ++++++++++++++++++++++++++++++++++--- pycardano/plutus.py | 2 +- pyproject.toml | 1 + 4 files changed, 61 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 6a202000..53b35dcd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "alabaster" @@ -127,6 +127,18 @@ files = [ [package.dependencies] requests = "*" +[[package]] +name = "cachetools" +version = "5.3.0" +description = "Extensible memoizing collections and decorators" +category = "main" +optional = false +python-versions = "~=3.7" +files = [ + {file = "cachetools-5.3.0-py3-none-any.whl", hash = "sha256:429e1a1e845c008ea6c85aa35d4b98b65d6a9763eeef3e37e92728a12d1de9d4"}, + {file = "cachetools-5.3.0.tar.gz", hash = "sha256:13dfddc7b8df938c21a940dfa6557ce6e94a2f1cdfa58eb90c805721d58f2c14"}, +] + [[package]] name = "cbor2" version = "5.4.6" @@ -1692,4 +1704,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.7" -content-hash = "242468ff5a03dfe8983c3e990a044b500c4c60ad878f165e056b5e6bfa5c0fcc" +content-hash = "6d227be4b5f57497cf7b45d18e4e666faf87b53b0936f340be9521861d238868" diff --git a/pycardano/backend/ogmios.py b/pycardano/backend/ogmios.py index 29aa4c41..1e3e2678 100644 --- a/pycardano/backend/ogmios.py +++ b/pycardano/backend/ogmios.py @@ -1,5 +1,6 @@ import json import time +from copy import deepcopy from datetime import datetime, timezone from enum import Enum from typing import Any, Dict, List, Optional, Tuple, Union @@ -7,6 +8,7 @@ import cbor2 import requests import websocket +from cachetools import Cache, LRUCache, TTLCache from pycardano.address import Address from pycardano.backend.base import ( @@ -49,6 +51,8 @@ class OgmiosChainContext(ChainContext): _last_chain_tip_fetch: float _genesis_param: Optional[GenesisParameters] _protocol_param: Optional[ProtocolParameters] + _utxo_cache: Cache + _datum_cache: Cache def __init__( self, @@ -57,6 +61,8 @@ def __init__( compact_result=True, kupo_url=None, refetch_chain_tip_interval: Optional[float] = None, + utxo_cache_size: int = 10000, + datum_cache_size: int = 10000, ): self._ws_url = ws_url self._network = network @@ -77,6 +83,11 @@ def __init__( / self.genesis_param.active_slots_coefficient ) + self._utxo_cache = TTLCache( + ttl=self._refetch_chain_tip_interval, maxsize=utxo_cache_size + ) + self._datum_cache = LRUCache(maxsize=datum_cache_size) + def _request(self, method: OgmiosQueryType, args: JsonDict) -> Any: ws = websocket.WebSocket() ws.connect(self._ws_url) @@ -253,13 +264,45 @@ def _utxos(self, address: str) -> List[UTxO]: Returns: List[UTxO]: A list of UTxOs. """ + if (self.last_block_slot, address) in self._utxo_cache: + return deepcopy(self._utxo_cache[(self.last_block_slot, address)]) + if self._kupo_url: utxos = self._utxos_kupo(address) else: utxos = self._utxos_ogmios(address) + self._utxo_cache[(self.last_block_slot, address)] = deepcopy(utxos) + return utxos + def _get_datum_from_kupo(self, datum_hash: str) -> Optional[RawCBOR]: + """Get datum from Kupo. + + Args: + datum_hash (str): A datum hash. + + Returns: + Optional[RawCBOR]: A datum. + """ + datum = self._datum_cache.get(datum_hash, None) + + if datum is not None or (datum is None and not self._is_chain_tip_updated()): + return datum + + if self._kupo_url is None: + raise AssertionError( + "kupo_url object attribute has not been assigned properly." + ) + + kupo_datum_url = self._kupo_url + "/datums/" + datum_hash + datum_result = requests.get(kupo_datum_url).json() + if datum_result and datum_result["datum"] != datum_hash: + datum = RawCBOR(bytes.fromhex(datum_result["datum"])) + + self._datum_cache[datum_hash] = datum + return datum + def _utxos_kupo(self, address: str) -> List[UTxO]: """Get all UTxOs associated with an address with Kupo. Since UTxO querying will be deprecated from Ogmios in next @@ -313,10 +356,8 @@ def _utxos_kupo(self, address: str) -> List[UTxO]: else None ) if datum_hash and result.get("datum_type", "inline"): - kupo_datum_url = self._kupo_url + "/datums/" + result["datum_hash"] - datum_result = requests.get(kupo_datum_url).json() - if datum_result and datum_result["datum"] != datum_hash: - datum = RawCBOR(bytes.fromhex(datum_result["datum"])) + datum = self._get_datum_from_kupo(result["datum_hash"]) + if datum is not None: datum_hash = None if not result["value"]["assets"]: diff --git a/pycardano/plutus.py b/pycardano/plutus.py index 85d05cb2..2fc7f0a4 100644 --- a/pycardano/plutus.py +++ b/pycardano/plutus.py @@ -670,7 +670,7 @@ def __deepcopy__(self, memo): return self.__class__.from_cbor(self.to_cbor_hex()) -@dataclass +@dataclass(repr=False) class RawPlutusData(CBORSerializable): data: CBORTag diff --git a/pyproject.toml b/pyproject.toml index b8cf5c9e..e57f6393 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ mnemonic = "^0.20" ECPy = "^1.2.5" frozendict = "^2.3.8" frozenlist = "^1.3.3" +cachetools = "^5.3.0" [tool.poetry.dev-dependencies] Sphinx = "^4.3.2"