From 162918e8e2d47e3b9abaa0e8e2d621d011b7c36b Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Thu, 28 Nov 2024 12:24:37 +0200 Subject: [PATCH 1/9] CERT 7649 Allow dynamic price feed providers --- Quorum/apis/git_api/git_manager.py | 2 +- Quorum/apis/price_feeds/__init__.py | 3 +- Quorum/apis/price_feeds/chainlink_api.py | 16 ++-- Quorum/apis/price_feeds/chronicle_api.py | 13 +-- .../apis/price_feeds/price_feed_providers.py | 9 +++ Quorum/check_proposal.py | 2 +- Quorum/checks/feed_price.py | 80 ++++++++++++------- Quorum/checks/global_variables.py | 1 - Quorum/config.py | 8 +- Quorum/{repos.json => ground_truth.json} | 3 +- README.md | 12 +-- setup.py | 2 +- 12 files changed, 94 insertions(+), 57 deletions(-) create mode 100644 Quorum/apis/price_feeds/price_feed_providers.py rename Quorum/{repos.json => ground_truth.json} (85%) diff --git a/Quorum/apis/git_api/git_manager.py b/Quorum/apis/git_api/git_manager.py index 4bcd96a..76270c1 100644 --- a/Quorum/apis/git_api/git_manager.py +++ b/Quorum/apis/git_api/git_manager.py @@ -40,7 +40,7 @@ def _load_repos_from_file(self) -> tuple[dict[str, str], dict[str, str]]: tuple[dict[str, str], dict[str, str]]: 2 dictionaries mapping repository names to their URLs. The first dictionary contains the repos to diff against. The second dictionary is the verification repo. """ - with open(config.REPOS_PATH) as f: + with open(config.GROUND_TRUTH_PATH) as f: repos_data = json.load(f) # Normalize the customer name to handle case differences diff --git a/Quorum/apis/price_feeds/__init__.py b/Quorum/apis/price_feeds/__init__.py index 4438771..b1c4b1d 100644 --- a/Quorum/apis/price_feeds/__init__.py +++ b/Quorum/apis/price_feeds/__init__.py @@ -1,4 +1,5 @@ from .chainlink_api import ChainLinkAPI from .chronicle_api import ChronicleAPI +from .price_feed_providers import PriceFeedProvider -all = [ChainLinkAPI, ChronicleAPI] \ No newline at end of file +all = [ChainLinkAPI, ChronicleAPI, PriceFeedProvider] \ No newline at end of file diff --git a/Quorum/apis/price_feeds/chainlink_api.py b/Quorum/apis/price_feeds/chainlink_api.py index 644e695..f4bf5d4 100644 --- a/Quorum/apis/price_feeds/chainlink_api.py +++ b/Quorum/apis/price_feeds/chainlink_api.py @@ -81,21 +81,18 @@ def __init__(self) -> None: price feed data for different blockchain networks. """ self.session = requests.Session() - self.memory: dict[Chain, list[PriceFeedData]] = {} + self.memory: dict[Chain, dict[str, PriceFeedData]] = {} - def get_price_feeds_info(self, chain: Chain) -> list[PriceFeedData]: + def get_price_feeds_info(self, chain: Chain) -> dict[str, PriceFeedData]: """ Fetches the price feeds information from the Chainlink API for a specified blockchain network. - - This method first checks if the data for the given chain is already cached in memory. If not, it makes an HTTP - request to the Chainlink API to fetch the data, parses the JSON response into a list of PriceFeedData objects, - and stores it in the cache. + The method fetches the price feed data from the Chainlink API and stores it in the memory cache. Args: chain (Chain): The blockchain network to fetch price feeds for. Returns: - list[PriceFeedData]: A list of PriceFeedData objects containing the price feeds information. + dict[str, PriceFeedData]: A dictionary mapping the contract address of the price feed to the PriceFeedData object. Raises: requests.HTTPError: If the HTTP request to the Chainlink API fails. @@ -108,6 +105,9 @@ def get_price_feeds_info(self, chain: Chain) -> list[PriceFeedData]: response = self.session.get(url) response.raise_for_status() - self.memory[chain] = [PriceFeedData(**feed) for feed in response.json()] + chain_link_price_feeds = [PriceFeedData(**feed) for feed in response.json()] + chain_link_price_feeds = {feed.contractAddress: feed for feed in chain_link_price_feeds} + chain_link_price_feeds.update({feed.proxyAddress: feed for feed in chain_link_price_feeds if feed.proxyAddress}) + self.memory[chain] = chain_link_price_feeds return self.memory[chain] diff --git a/Quorum/apis/price_feeds/chronicle_api.py b/Quorum/apis/price_feeds/chronicle_api.py index 5fb6217..996ec16 100644 --- a/Quorum/apis/price_feeds/chronicle_api.py +++ b/Quorum/apis/price_feeds/chronicle_api.py @@ -16,7 +16,7 @@ class ChronicleAPI(metaclass=Singleton): def __init__(self): self.session = requests.Session() - self.memory = defaultdict(list) + self.memory: dict[Chain, dict[str, dict]] = {} self.__pairs = self.__process_pairs() def __process_pairs(self) -> dict[str, list[str]]: @@ -33,7 +33,7 @@ def __process_pairs(self) -> dict[str, list[str]]: result[p["blockchain"]].append(p["pair"]) return result - def get_price_feeds_info(self, chain: Chain) -> list[dict]: + def get_price_feeds_info(self, chain: Chain) -> dict[str, dict]: """ Get price feed data for a given blockchain network. @@ -43,13 +43,14 @@ def get_price_feeds_info(self, chain: Chain) -> list[dict]: chain (Chain): The blockchain network to fetch price feed data for. Returns: - dict: The price feed data for the specified chain. + dict[str, dict]: A dictionary mapping the contract address of the price feed to the price feed data. """ if chain not in self.memory: pairs = self.__pairs.get(chain) if not pairs: - return [] + return {} + chronicle_price_feeds = [] for pair in pairs: response = self.session.get( f"https://chroniclelabs.org/api/median/info/{pair}/{chain.value}/?testnet=false" @@ -58,6 +59,8 @@ def get_price_feeds_info(self, chain: Chain) -> list[dict]: data = response.json() for pair_info in data: - self.memory[chain].append(pair_info) + chronicle_price_feeds.append(pair_info) + + self.memory[chain] = {feed.get("address"): feed for feed in chronicle_price_feeds} return self.memory[chain] diff --git a/Quorum/apis/price_feeds/price_feed_providers.py b/Quorum/apis/price_feeds/price_feed_providers.py new file mode 100644 index 0000000..17cb354 --- /dev/null +++ b/Quorum/apis/price_feeds/price_feed_providers.py @@ -0,0 +1,9 @@ +from enum import StrEnum + + +class PriceFeedProvider(StrEnum): + """ + Enumeration for supported price feed providers. + """ + CHAINLINK = 'Chainlink' + CHRONICLE = 'Chronicle' diff --git a/Quorum/check_proposal.py b/Quorum/check_proposal.py index 99e1eff..c0fd4e9 100644 --- a/Quorum/check_proposal.py +++ b/Quorum/check_proposal.py @@ -74,7 +74,7 @@ def proposals_check(customer: str, chain_name: str, proposal_addresses: list[str Checks.GlobalVariableCheck(customer, chain, proposal_address, missing_files).check_global_variables() # Feed price check - Checks.FeedPriceCheck(customer, chain, proposal_address, missing_files).verify_feed_price() + Checks.FeedPriceCheck(customer, chain, proposal_address, missing_files).verify_price_feed() # New listing check Checks.NewListingCheck(customer, chain, proposal_address, missing_files).new_listing_check() diff --git a/Quorum/checks/feed_price.py b/Quorum/checks/feed_price.py index 1e0d19d..30fc48a 100644 --- a/Quorum/checks/feed_price.py +++ b/Quorum/checks/feed_price.py @@ -1,10 +1,12 @@ from pathlib import Path import re +import json -from Quorum.apis.price_feeds import ChainLinkAPI, ChronicleAPI +from Quorum.apis.price_feeds import ChainLinkAPI, ChronicleAPI, PriceFeedProvider from Quorum.utils.chain_enum import Chain from Quorum.checks.check import Check from Quorum.apis.block_explorers.source_code import SourceCode +import Quorum.config as config import Quorum.utils.pretty_printer as pp @@ -26,22 +28,57 @@ def __init__(self, customer: str, chain: Chain, proposal_address: str, source_co source_codes (list[SourceCode]): A list of source code objects containing the Solidity contracts to be checked. """ super().__init__(customer, chain, proposal_address, source_codes) - self.chainlink_api = ChainLinkAPI() - self.chronicle_api = ChronicleAPI() - self.address_pattern = r'0x[a-fA-F0-9]{40}' + + # load providers price feeds + self.providers_to_price_feed = self.__fetch_price_feed_providers() + + def __fetch_price_feed_providers(self) -> dict[PriceFeedProvider, dict]: + """ + Load the price feed providers from the ground truth file. + + Returns: + dict[PriceFeedProvider, dict]: A dictionary mapping the price feed provider to the price feed data. + """ + with open(config.GROUND_TRUTH_PATH) as f: + providers: list = json.load(f).get(self.customer).get("price_feed_providers", []) + + # Map providers to price feeds + providers_to_price_feed = {} + for provider in providers: + if provider == PriceFeedProvider.CHAINLINK: + providers_to_price_feed[provider] = ChainLinkAPI().get_price_feeds_info(self.chain) + elif provider == PriceFeedProvider.CHRONICLE: + providers_to_price_feed[provider] = ChronicleAPI().get_price_feeds_info(self.chain) + else: + pp.pretty_print(f"Unknown price feed provider: {provider}", pp.Colors.FAILURE) + providers.remove(provider) - # Retrieve price feeds from Chainlink API and map them by contract address - chain_link_price_feeds = self.chainlink_api.get_price_feeds_info(self.chain) - self.chain_link_price_feeds = {feed.contractAddress: feed for feed in chain_link_price_feeds} - self.chain_link_price_feeds.update({feed.proxyAddress: feed for feed in chain_link_price_feeds if feed.proxyAddress}) + return providers_to_price_feed - # Retrieve price feeds from Chronical API and map them by contract address - chronicle_price_feeds = self.chronicle_api.get_price_feeds_info(self.chain) - self.chronicle_price_feeds_dict = {feed.get("address"): feed for feed in chronicle_price_feeds} + def __check_price_feed_address(self, address: str) -> dict | None: + """ + Check if the given address is present in the price feed providers. + + Args: + address (str): The address to be checked. - - def verify_feed_price(self) -> None: + Returns: + dict | None: The price feed data if the address is found, otherwise None. + """ + for provider, price_feeds in self.providers_to_price_feed.items(): + if address in price_feeds: + feed = price_feeds[address] + pp.pretty_print( + f"Found {address} on {provider.name}\nname:{feed.get('pair')}", + pp.Colors.SUCCESS + ) + return feed + + pp.pretty_print(f"Address {address} not found in any price feed provider", pp.Colors.INFO) + return None + + def verify_price_feed(self) -> None: """ Verifies the price feed addresses in the source code against official Chainlink or Chronical data. @@ -57,23 +94,10 @@ def verify_feed_price(self) -> None: contract_text = '\n'.join(source_code.file_content) addresses = re.findall(self.address_pattern, contract_text) for address in addresses: - if address in self.chain_link_price_feeds: - feed = self.chain_link_price_feeds[address] - pp.pretty_print( - f"Found {address} on Chainlink\nname:{feed.name} Decimals:{feed.decimals}", - pp.Colors.SUCCESS - ) - verified_variables.append(feed.dict()) - - elif address in self.chronicle_price_feeds_dict: - feed = self.chronicle_price_feeds_dict[address] - pp.pretty_print( - f"Found {address} on Chronicle\nname:{feed.get('pair')}", - pp.Colors.SUCCESS - ) + if feed := self.__check_price_feed_address(address): verified_variables.append(feed) if verified_variables: self._write_to_file(verified_sources_path, verified_variables) else: - pp.pretty_print(f"No address related to chain link or chronicle found in {Path(source_code.file_name).stem}", pp.Colors.INFO) \ No newline at end of file + pp.pretty_print(f"No address related to chain link or chronicle found in {Path(source_code.file_name).stem}", pp.Colors.INFO) diff --git a/Quorum/checks/global_variables.py b/Quorum/checks/global_variables.py index 3ac1759..df7db06 100644 --- a/Quorum/checks/global_variables.py +++ b/Quorum/checks/global_variables.py @@ -64,7 +64,6 @@ def __check_immutable(self, variables: list[dict], source_code: list[str]) -> li """ return [v for v in variables if v.get("mutability") != "immutable"] - def __process_results(self, source_code_to_violated_variables: dict[str, list[dict]]): """ Processes the results of the global variable checks and prints them to the console. diff --git a/Quorum/config.py b/Quorum/config.py index 32ad0e7..6323c85 100644 --- a/Quorum/config.py +++ b/Quorum/config.py @@ -12,8 +12,8 @@ if not MAIN_PATH.exists(): MAIN_PATH.mkdir(parents=True) -REPOS_PATH = MAIN_PATH / "repos.json" -DEFAULT_REPOS = Path(__file__).parent / "repos.json" +GROUND_TRUTH_PATH = MAIN_PATH / "ground_truth.json" +DEFAULT_REPOS = Path(__file__).parent / "ground_truth.json" -if not REPOS_PATH.exists(): - shutil.copy(DEFAULT_REPOS, REPOS_PATH) \ No newline at end of file +if not GROUND_TRUTH_PATH.exists(): + shutil.copy(DEFAULT_REPOS, GROUND_TRUTH_PATH) \ No newline at end of file diff --git a/Quorum/repos.json b/Quorum/ground_truth.json similarity index 85% rename from Quorum/repos.json rename to Quorum/ground_truth.json index b803164..d685b3e 100644 --- a/Quorum/repos.json +++ b/Quorum/ground_truth.json @@ -7,6 +7,7 @@ "https://github.com/bgd-labs/aave-address-book", "https://github.com/aave-dao/aave-v3-origin" ], - "review_repo": "https://github.com/bgd-labs/aave-proposals-v3" + "review_repo": "https://github.com/bgd-labs/aave-proposals-v3", + "price_feed_providers": ["Chainlink"] } } \ No newline at end of file diff --git a/README.md b/README.md index fd3b9ca..dc69f25 100644 --- a/README.md +++ b/README.md @@ -152,9 +152,9 @@ CheckProposal --config path/to/config.json ## Configuration -The `repos.json` file defines the repositories for each customer. It should be located under the `QUORUM_PATH`. If not found, a default `repos.json` configuration will be created. +The `ground_truth.json` file defines the repositories for each customer. It should be located under the `QUORUM_PATH`. If not found, a default `ground_truth.json` configuration will be created. -Example `repos.json`: +Example `ground_truth.json`: ```json { @@ -181,7 +181,7 @@ Quorum generates and organizes artifacts in a structured manner under the `QUORU ``` QUORUM_PATH/ -├── repos.json +├── ground_truth.json ├── CustomerName/ | ├── modules/ | │ ├── repository1/ @@ -218,7 +218,7 @@ QUORUM_PATH/ - **execution.json**: This file stores the configuration and results of the last execution, including details like which proposals were checked and any findings or issues encountered. - - **repos.json**: A configuration file specifying the repositories to be managed for the customer. This file can be customized to include the URLs of the repositories related to the customer. + - **ground_truth.json**: A configuration file specifying the repositories to be managed for the customer. This file can be customized to include the URLs of the repositories related to the customer. ### Example @@ -243,10 +243,10 @@ Aave/ │ ├── aave-helpers/ │ ├── aave-v3-origin/ ├── execution.json -├── repos.json +├── ground_truth.json ``` -In this example, each proposal address under the `checks/` directory contains diff files that highlight the differences between the local and fetched source codes, as well as global variable check results. The `modules/` directory contains the repositories relevant to the customer "Aave," and the `execution.json` and `repos.json` files hold metadata and configuration details. +In this example, each proposal address under the `checks/` directory contains diff files that highlight the differences between the local and fetched source codes, as well as global variable check results. The `modules/` directory contains the repositories relevant to the customer "Aave," and the `execution.json` and `ground_truth.json` files hold metadata and configuration details. ## License diff --git a/setup.py b/setup.py index dcf6eda..79fb089 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ def read_version() -> str: install_requires=read_requirements(), include_package_data=True, package_data={ - '': ['repos.json'], + '': ['ground_truth.json'], }, entry_points={ "console_scripts": [ From 8552c77b5a2255bc1de2a843efb1c27be17ec308 Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Thu, 28 Nov 2024 12:26:34 +0200 Subject: [PATCH 2/9] Address comments --- Quorum/checks/feed_price.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Quorum/checks/feed_price.py b/Quorum/checks/feed_price.py index 30fc48a..6956aac 100644 --- a/Quorum/checks/feed_price.py +++ b/Quorum/checks/feed_price.py @@ -99,5 +99,3 @@ def verify_price_feed(self) -> None: if verified_variables: self._write_to_file(verified_sources_path, verified_variables) - else: - pp.pretty_print(f"No address related to chain link or chronicle found in {Path(source_code.file_name).stem}", pp.Colors.INFO) From 7e938bb9d487f584988a78c1cdb37dd33ef5068c Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Thu, 28 Nov 2024 14:39:43 +0200 Subject: [PATCH 3/9] Fix --- Quorum/apis/price_feeds/__init__.py | 4 +- Quorum/apis/price_feeds/chainlink_api.py | 49 ++----------------- Quorum/apis/price_feeds/chronicle_api.py | 8 +-- .../apis/price_feeds/price_feed_providers.py | 9 ---- Quorum/apis/price_feeds/price_feed_utils.py | 21 ++++++++ Quorum/checks/feed_price.py | 14 ++++-- 6 files changed, 40 insertions(+), 65 deletions(-) delete mode 100644 Quorum/apis/price_feeds/price_feed_providers.py create mode 100644 Quorum/apis/price_feeds/price_feed_utils.py diff --git a/Quorum/apis/price_feeds/__init__.py b/Quorum/apis/price_feeds/__init__.py index b1c4b1d..b770d93 100644 --- a/Quorum/apis/price_feeds/__init__.py +++ b/Quorum/apis/price_feeds/__init__.py @@ -1,5 +1,5 @@ from .chainlink_api import ChainLinkAPI from .chronicle_api import ChronicleAPI -from .price_feed_providers import PriceFeedProvider +from .price_feed_utils import PriceFeedProvider, PriceFeedData -all = [ChainLinkAPI, ChronicleAPI, PriceFeedProvider] \ No newline at end of file +all = [ChainLinkAPI, ChronicleAPI, PriceFeedProvider, PriceFeedData] \ No newline at end of file diff --git a/Quorum/apis/price_feeds/chainlink_api.py b/Quorum/apis/price_feeds/chainlink_api.py index f4bf5d4..bd6f8f4 100644 --- a/Quorum/apis/price_feeds/chainlink_api.py +++ b/Quorum/apis/price_feeds/chainlink_api.py @@ -1,51 +1,8 @@ import requests -from pydantic import BaseModel -from typing import Optional from Quorum.utils.chain_enum import Chain from Quorum.utils.singleton import Singleton - -class Docs(BaseModel): - assetClass: Optional[str] = None - assetName: Optional[str] = None - baseAsset: Optional[str] = None - baseAssetClic: Optional[str] = None - blockchainName: Optional[str] = None - clicProductName: Optional[str] = None - deliveryChannelCode: Optional[str] = None - feedType: Optional[str] = None - hidden: Optional[bool] = None - marketHours: Optional[str] = None - productSubType: Optional[str] = None - productType: Optional[str] = None - productTypeCode: Optional[str] = None - quoteAsset: Optional[str] = None - quoteAssetClic: Optional[str] = None - - -class PriceFeedData(BaseModel): - compareOffchain: Optional[str] = None - contractAddress: str - contractType: Optional[str] = None - contractVersion: Optional[int] = None - decimalPlaces: Optional[int] = None - ens: Optional[str] = None - formatDecimalPlaces: Optional[int] = None - healthPrice: Optional[str] = None - heartbeat: Optional[int] = None - history: Optional[str | bool] = None - multiply: Optional[str] = None - name: Optional[str] = None - pair: Optional[list[Optional[str]]] = None - path: Optional[str] = None - proxyAddress: Optional[str] = None - threshold: Optional[float] = None - valuePrefix: Optional[str] = None - assetName: Optional[str] = None - feedCategory: Optional[str] = None - feedType: Optional[str] = None - docs: Optional[Docs] = None - decimals: Optional[int] = None +from .price_feed_utils import PriceFeedData class ChainLinkAPI(metaclass=Singleton): @@ -106,8 +63,8 @@ def get_price_feeds_info(self, chain: Chain) -> dict[str, PriceFeedData]: response = self.session.get(url) response.raise_for_status() chain_link_price_feeds = [PriceFeedData(**feed) for feed in response.json()] - chain_link_price_feeds = {feed.contractAddress: feed for feed in chain_link_price_feeds} - chain_link_price_feeds.update({feed.proxyAddress: feed for feed in chain_link_price_feeds if feed.proxyAddress}) + chain_link_price_feeds = {feed.address: feed for feed in chain_link_price_feeds} + chain_link_price_feeds.update({feed.proxy_address: feed for feed in chain_link_price_feeds.values() if feed.proxy_address}) self.memory[chain] = chain_link_price_feeds return self.memory[chain] diff --git a/Quorum/apis/price_feeds/chronicle_api.py b/Quorum/apis/price_feeds/chronicle_api.py index 996ec16..7d9a35a 100644 --- a/Quorum/apis/price_feeds/chronicle_api.py +++ b/Quorum/apis/price_feeds/chronicle_api.py @@ -3,6 +3,8 @@ from Quorum.utils.chain_enum import Chain from Quorum.utils.singleton import Singleton +from .price_feed_utils import PriceFeedData + class ChronicleAPI(metaclass=Singleton): """ @@ -50,7 +52,7 @@ def get_price_feeds_info(self, chain: Chain) -> dict[str, dict]: if not pairs: return {} - chronicle_price_feeds = [] + chronicle_price_feeds: list[PriceFeedData] = [] for pair in pairs: response = self.session.get( f"https://chroniclelabs.org/api/median/info/{pair}/{chain.value}/?testnet=false" @@ -59,8 +61,8 @@ def get_price_feeds_info(self, chain: Chain) -> dict[str, dict]: data = response.json() for pair_info in data: - chronicle_price_feeds.append(pair_info) + chronicle_price_feeds.append(PriceFeedData(**pair_info)) - self.memory[chain] = {feed.get("address"): feed for feed in chronicle_price_feeds} + self.memory[chain] = {feed.address: feed for feed in chronicle_price_feeds} return self.memory[chain] diff --git a/Quorum/apis/price_feeds/price_feed_providers.py b/Quorum/apis/price_feeds/price_feed_providers.py deleted file mode 100644 index 17cb354..0000000 --- a/Quorum/apis/price_feeds/price_feed_providers.py +++ /dev/null @@ -1,9 +0,0 @@ -from enum import StrEnum - - -class PriceFeedProvider(StrEnum): - """ - Enumeration for supported price feed providers. - """ - CHAINLINK = 'Chainlink' - CHRONICLE = 'Chronicle' diff --git a/Quorum/apis/price_feeds/price_feed_utils.py b/Quorum/apis/price_feeds/price_feed_utils.py new file mode 100644 index 0000000..bda0688 --- /dev/null +++ b/Quorum/apis/price_feeds/price_feed_utils.py @@ -0,0 +1,21 @@ +from enum import StrEnum +from typing import Optional +from pydantic import BaseModel, Field + +class PriceFeedProvider(StrEnum): + """ + Enumeration for supported price feed providers. + """ + CHAINLINK = 'Chainlink' + CHRONICLE = 'Chronicle' + + +class PriceFeedData(BaseModel): + name: Optional[str] + pair: Optional[str | list] + address: str = Field(..., alias='contractAddress') + proxy_address: Optional[str] = Field(None, alias='proxyAddress') + + class Config: + allow_population_by_field_name = True # Allows population using field names + extra = 'ignore' # Ignores extra fields not defined in the model diff --git a/Quorum/checks/feed_price.py b/Quorum/checks/feed_price.py index 6956aac..6382021 100644 --- a/Quorum/checks/feed_price.py +++ b/Quorum/checks/feed_price.py @@ -2,7 +2,7 @@ import re import json -from Quorum.apis.price_feeds import ChainLinkAPI, ChronicleAPI, PriceFeedProvider +from Quorum.apis.price_feeds import ChainLinkAPI, ChronicleAPI, PriceFeedProvider, PriceFeedData from Quorum.utils.chain_enum import Chain from Quorum.checks.check import Check from Quorum.apis.block_explorers.source_code import SourceCode @@ -41,8 +41,12 @@ def __fetch_price_feed_providers(self) -> dict[PriceFeedProvider, dict]: dict[PriceFeedProvider, dict]: A dictionary mapping the price feed provider to the price feed data. """ with open(config.GROUND_TRUTH_PATH) as f: - providers: list = json.load(f).get(self.customer).get("price_feed_providers", []) + providers: list = json.load(f).get(self.customer, {}).get("price_feed_providers", []) + if not providers: + pp.pretty_print(f"No price feed providers found for {self.customer}", pp.Colors.FAILURE) + return {} + # Map providers to price feeds providers_to_price_feed = {} for provider in providers: @@ -68,12 +72,12 @@ def __check_price_feed_address(self, address: str) -> dict | None: """ for provider, price_feeds in self.providers_to_price_feed.items(): if address in price_feeds: - feed = price_feeds[address] + feed: PriceFeedData = price_feeds[address] pp.pretty_print( - f"Found {address} on {provider.name}\nname:{feed.get('pair')}", + f"Found {address} on {provider}\nname:{feed.name if feed.name else feed.pair}", pp.Colors.SUCCESS ) - return feed + return feed.dict() pp.pretty_print(f"Address {address} not found in any price feed provider", pp.Colors.INFO) return None From 7de0a7a523c9406a6f4a6f6d3d0e98570b4b8b06 Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Thu, 28 Nov 2024 17:20:53 +0200 Subject: [PATCH 4/9] New design + add cache to prevent api dealy --- .gitignore | 3 +- Quorum/apis/git_api/git_manager.py | 17 +++--- Quorum/apis/price_feeds/__init__.py | 4 +- Quorum/apis/price_feeds/chainlink_api.py | 47 ++++++---------- Quorum/apis/price_feeds/chronicle_api.py | 52 +++++++++--------- Quorum/apis/price_feeds/price_feed_utils.py | 54 ++++++++++++++++++- Quorum/check_proposal.py | 21 +++++--- Quorum/checks/__init__.py | 2 +- .../checks/{feed_price.py => price_feed.py} | 47 ++++++++-------- Quorum/config.py | 2 +- Quorum/utils/config_loader.py | 39 ++++++++++++++ Quorum/utils/singleton.py | 7 ++- 12 files changed, 191 insertions(+), 104 deletions(-) rename Quorum/checks/{feed_price.py => price_feed.py} (68%) create mode 100644 Quorum/utils/config_loader.py diff --git a/.gitignore b/.gitignore index c133d90..0f5087b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ **/local* build/ *.egg-info -CustomerClones/ \ No newline at end of file +CustomerClones/ +**cache** diff --git a/Quorum/apis/git_api/git_manager.py b/Quorum/apis/git_api/git_manager.py index 76270c1..5085eee 100644 --- a/Quorum/apis/git_api/git_manager.py +++ b/Quorum/apis/git_api/git_manager.py @@ -1,11 +1,9 @@ -import json from pathlib import Path from git import Repo import Quorum.config as config import Quorum.utils.pretty_printer as pp - class GitManager: """ A class to manage Git repositories for a specific customer. @@ -15,12 +13,13 @@ class GitManager: repos (dict): A dictionary mapping repository names to their URLs. """ - def __init__(self, customer: str) -> None: + def __init__(self, customer: str, gt_config: dict[str, any]) -> None: """ Initialize the GitManager with the given customer name and load the repository URLs. Args: customer (str): The name or identifier of the customer. + gt_config (dict[str, any]): The ground truth configuration data. """ self.customer = customer @@ -30,22 +29,22 @@ def __init__(self, customer: str) -> None: self.review_module_path = config.MAIN_PATH / self.customer / "review_module" self.review_module_path.mkdir(parents=True, exist_ok=True) - self.repos, self.review_repo = self._load_repos_from_file() + self.repos, self.review_repo = self._load_repos_from_file(gt_config) - def _load_repos_from_file(self) -> tuple[dict[str, str], dict[str, str]]: + def _load_repos_from_file(self, gt_config: dict[str, any]) -> tuple[dict[str, str], dict[str, str]]: """ Load repository URLs from the JSON file for the given customer. + Args: + gt_config (dict[str, any]): The ground truth configuration data. + Returns: tuple[dict[str, str], dict[str, str]]: 2 dictionaries mapping repository names to their URLs. The first dictionary contains the repos to diff against. The second dictionary is the verification repo. """ - with open(config.GROUND_TRUTH_PATH) as f: - repos_data = json.load(f) - # Normalize the customer name to handle case differences normalized_customer = self.customer.lower() - customer_repos = next((repos for key, repos in repos_data.items() if key.lower() == normalized_customer), None) + customer_repos = next((repos for key, repos in gt_config.items() if key.lower() == normalized_customer), None) if customer_repos is None: return {}, {} diff --git a/Quorum/apis/price_feeds/__init__.py b/Quorum/apis/price_feeds/__init__.py index b770d93..6378276 100644 --- a/Quorum/apis/price_feeds/__init__.py +++ b/Quorum/apis/price_feeds/__init__.py @@ -1,5 +1,5 @@ from .chainlink_api import ChainLinkAPI from .chronicle_api import ChronicleAPI -from .price_feed_utils import PriceFeedProvider, PriceFeedData +from .price_feed_utils import PriceFeedData, PriceFeedProvider, PriceFeedProviderBase -all = [ChainLinkAPI, ChronicleAPI, PriceFeedProvider, PriceFeedData] \ No newline at end of file +all = [ChainLinkAPI, ChronicleAPI, PriceFeedData, PriceFeedProvider, PriceFeedProviderBase] \ No newline at end of file diff --git a/Quorum/apis/price_feeds/chainlink_api.py b/Quorum/apis/price_feeds/chainlink_api.py index bd6f8f4..b19f0dd 100644 --- a/Quorum/apis/price_feeds/chainlink_api.py +++ b/Quorum/apis/price_feeds/chainlink_api.py @@ -1,11 +1,9 @@ -import requests - from Quorum.utils.chain_enum import Chain -from Quorum.utils.singleton import Singleton -from .price_feed_utils import PriceFeedData +from Quorum.utils.singleton import SingletonABCMeta +from .price_feed_utils import PriceFeedData, PriceFeedProviderBase, PriceFeedProvider -class ChainLinkAPI(metaclass=Singleton): +class ChainLinkAPI(PriceFeedProviderBase, metaclass=SingletonABCMeta): """ ChainLinkAPI is a class designed to interact with the Chainlink data feed API. It fetches and stores price feed data for various blockchain networks supported by Chainlink. @@ -29,22 +27,11 @@ class ChainLinkAPI(metaclass=Singleton): Chain.SCR: "https://reference-data-directory.vercel.app/feeds-ethereum-mainnet-scroll-1.json", Chain.ZK: "https://reference-data-directory.vercel.app/feeds-ethereum-mainnet-zksync-1.json" } - - def __init__(self) -> None: - """ - Initialize the ChainLinkAPI instance. - - Creates an HTTP session for managing requests and initializes an in-memory cache to store - price feed data for different blockchain networks. - """ - self.session = requests.Session() - self.memory: dict[Chain, dict[str, PriceFeedData]] = {} - def get_price_feeds_info(self, chain: Chain) -> dict[str, PriceFeedData]: + def _get_price_feeds_info(self, chain: Chain) -> dict[str, PriceFeedData]: """ - Fetches the price feeds information from the Chainlink API for a specified blockchain network. + Get price feed data for a given blockchain network. - The method fetches the price feed data from the Chainlink API and stores it in the memory cache. Args: chain (Chain): The blockchain network to fetch price feeds for. @@ -55,16 +42,16 @@ def get_price_feeds_info(self, chain: Chain) -> dict[str, PriceFeedData]: requests.HTTPError: If the HTTP request to the Chainlink API fails. KeyError: If the specified chain is not supported. """ - if chain not in self.memory: - url = self.chain_mapping.get(chain) - if not url: - raise KeyError(f"Chain {chain.name} is not supported.") - - response = self.session.get(url) - response.raise_for_status() - chain_link_price_feeds = [PriceFeedData(**feed) for feed in response.json()] - chain_link_price_feeds = {feed.address: feed for feed in chain_link_price_feeds} - chain_link_price_feeds.update({feed.proxy_address: feed for feed in chain_link_price_feeds.values() if feed.proxy_address}) - self.memory[chain] = chain_link_price_feeds + url = self.chain_mapping.get(chain) + if not url: + raise KeyError(f"Chain {chain.name} is not supported.") - return self.memory[chain] + response = self.session.get(url) + response.raise_for_status() + chain_link_price_feeds = [PriceFeedData(**feed) for feed in response.json()] + chain_link_price_feeds = {feed.address: feed for feed in chain_link_price_feeds} + chain_link_price_feeds.update({feed.proxy_address: feed for feed in chain_link_price_feeds.values() if feed.proxy_address}) + return chain_link_price_feeds + + def get_name(self) -> PriceFeedProvider: + return "Chainlink" diff --git a/Quorum/apis/price_feeds/chronicle_api.py b/Quorum/apis/price_feeds/chronicle_api.py index 7d9a35a..50a0031 100644 --- a/Quorum/apis/price_feeds/chronicle_api.py +++ b/Quorum/apis/price_feeds/chronicle_api.py @@ -1,12 +1,13 @@ import requests from collections import defaultdict +import json from Quorum.utils.chain_enum import Chain -from Quorum.utils.singleton import Singleton -from .price_feed_utils import PriceFeedData +from Quorum.utils.singleton import SingletonABCMeta +from .price_feed_utils import PriceFeedData, PriceFeedProviderBase, PriceFeedProvider -class ChronicleAPI(metaclass=Singleton): +class ChronicleAPI(PriceFeedProviderBase, metaclass=SingletonABCMeta): """ ChronicleAPI is a class designed to interact with the Chronicle data feed API. It fetches and stores price feed data for various blockchain networks supported by Chronicle. @@ -17,8 +18,7 @@ class ChronicleAPI(metaclass=Singleton): """ def __init__(self): - self.session = requests.Session() - self.memory: dict[Chain, dict[str, dict]] = {} + super().__init__() self.__pairs = self.__process_pairs() def __process_pairs(self) -> dict[str, list[str]]: @@ -35,34 +35,34 @@ def __process_pairs(self) -> dict[str, list[str]]: result[p["blockchain"]].append(p["pair"]) return result - def get_price_feeds_info(self, chain: Chain) -> dict[str, dict]: + def _get_price_feeds_info(self, chain: Chain) -> dict[str, PriceFeedData]: """ Get price feed data for a given blockchain network. - This method retrieves price feed data from the cache if it has been fetched previously. - Args: - chain (Chain): The blockchain network to fetch price feed data for. + chain (Chain): The blockchain network to fetch price feeds for. Returns: - dict[str, dict]: A dictionary mapping the contract address of the price feed to the price feed data. + dict[str, PriceFeedData]: A dictionary mapping the contract address of the price feed to the PriceFeedData object. """ - if chain not in self.memory: - pairs = self.__pairs.get(chain) - if not pairs: - return {} + + pairs = self.__pairs.get(chain) + if not pairs: + return {} + + chronicle_price_feeds: list[PriceFeedData] = [] + for pair in pairs: + response = self.session.get( + f"https://chroniclelabs.org/api/median/info/{pair}/{chain.value}/?testnet=false" + ) + response.raise_for_status() + data = response.json() - chronicle_price_feeds: list[PriceFeedData] = [] - for pair in pairs: - response = self.session.get( - f"https://chroniclelabs.org/api/median/info/{pair}/{chain.value}/?testnet=false" - ) - response.raise_for_status() - data = response.json() + for pair_info in data: + chronicle_price_feeds.append(PriceFeedData(**pair_info)) - for pair_info in data: - chronicle_price_feeds.append(PriceFeedData(**pair_info)) - - self.memory[chain] = {feed.address: feed for feed in chronicle_price_feeds} + return {feed.address: feed for feed in chronicle_price_feeds} + - return self.memory[chain] + def get_name(self) -> PriceFeedProvider: + return "Chronicle" diff --git a/Quorum/apis/price_feeds/price_feed_utils.py b/Quorum/apis/price_feeds/price_feed_utils.py index bda0688..3c31061 100644 --- a/Quorum/apis/price_feeds/price_feed_utils.py +++ b/Quorum/apis/price_feeds/price_feed_utils.py @@ -1,6 +1,12 @@ +from abc import ABC, abstractmethod from enum import StrEnum from typing import Optional from pydantic import BaseModel, Field +from pathlib import Path +import json +import requests + +from Quorum.utils.chain_enum import Chain class PriceFeedProvider(StrEnum): """ @@ -9,7 +15,6 @@ class PriceFeedProvider(StrEnum): CHAINLINK = 'Chainlink' CHRONICLE = 'Chronicle' - class PriceFeedData(BaseModel): name: Optional[str] pair: Optional[str | list] @@ -19,3 +24,50 @@ class PriceFeedData(BaseModel): class Config: allow_population_by_field_name = True # Allows population using field names extra = 'ignore' # Ignores extra fields not defined in the model + + +class PriceFeedProviderBase(ABC): + """ + PriceFeedProviderBase is an abstract base class for price feed providers. + It defines the interface for fetching price feed data for different blockchain networks. + + Attributes: + cache_dir (Path): The directory path to store the fetched price feed data. + """ + + def __init__(self): + super().__init__() + self.cache_dir = Path(f"{__file__.removesuffix('.py')}/cache/{self.get_name()}") + self.cache_dir.mkdir(parents=True, exist_ok=True) + self.session = requests.Session() + self.memory: dict[Chain, dict[str, PriceFeedData]] = {} + + def get_feeds(self, chain: Chain) -> dict[str, PriceFeedData]: + """ + Get price feed data for a given blockchain network from the cache. + + Args: + chain (Chain): The blockchain network to fetch price feed data for. + + Returns: + dict[str, PriceFeedData]: A dictionary mapping the contract address of the price feed to the price feed data. + """ + cache_file = self.cache_dir / f"{chain.value}.json" + if cache_file.exists(): + with open(cache_file, 'r') as file: + data: dict = json.load(file) + self.memory[chain] = {addr: PriceFeedData(**feed) for addr, feed in data.items()} + else: + if chain not in self.memory: + self.memory[chain] = self._get_price_feeds_info(chain) + with open(cache_file, 'w') as file: + json.dump({addr: feed.dict() for addr, feed in self.memory[chain].items()}, file) + return self.memory[chain] + + @abstractmethod + def _get_price_feeds_info(self, chain: Chain) -> dict[str, PriceFeedData]: + pass + + @abstractmethod + def get_name(self) -> PriceFeedProvider: + pass diff --git a/Quorum/check_proposal.py b/Quorum/check_proposal.py index c0fd4e9..f49e519 100644 --- a/Quorum/check_proposal.py +++ b/Quorum/check_proposal.py @@ -3,10 +3,12 @@ from typing import Any, Optional from Quorum.utils.chain_enum import Chain -import Quorum.utils.pretty_printer as pp from Quorum.apis.git_api.git_manager import GitManager +from Quorum.apis.price_feeds.price_feed_utils import PriceFeedProviderBase from Quorum.apis.block_explorers.chains_api import ChainAPI import Quorum.checks as Checks +import Quorum.utils.pretty_printer as pp +import Quorum.utils.config_loader as ConfigLoader def parse_args() -> tuple[Optional[str], Optional[str], Optional[str], Optional[str]]: @@ -45,7 +47,7 @@ def load_config(config_path: str) -> dict[str, Any] | None: pp.pretty_print(f"Failed to parse given config file {config_path}:\n{e}", pp.Colors.FAILURE) -def proposals_check(customer: str, chain_name: str, proposal_addresses: list[str]) -> None: +def proposals_check(customer: str, chain_name: str, proposal_addresses: list[str], providers: list[PriceFeedProviderBase]) -> None: """ Check and compare source code files for given proposals. @@ -55,6 +57,7 @@ def proposals_check(customer: str, chain_name: str, proposal_addresses: list[str customer (str): The customer name or identifier. chain_name (str): The blockchain chain name. proposal_addresses (list[str]): List of proposal addresses. + providers (list[PriceFeedProviderInterface]): List of price feed providers. """ chain = Chain[chain_name.upper()] api = ChainAPI(chain) @@ -74,7 +77,7 @@ def proposals_check(customer: str, chain_name: str, proposal_addresses: list[str Checks.GlobalVariableCheck(customer, chain, proposal_address, missing_files).check_global_variables() # Feed price check - Checks.FeedPriceCheck(customer, chain, proposal_address, missing_files).verify_price_feed() + Checks.PriceFeedCheck(customer, chain, proposal_address, missing_files, providers).verify_price_feed() # New listing check Checks.NewListingCheck(customer, chain, proposal_address, missing_files).new_listing_check() @@ -91,17 +94,21 @@ def main() -> None: if config_data: # Multi-task mode using JSON configuration for customer, chain_info in config_data.items(): - GitManager(customer).clone_or_update() + ground_truth_config = ConfigLoader.load_customer_config(customer) + GitManager(customer, ground_truth_config).clone_or_update() + price_feed_providers = ground_truth_config.get("price_feed_providers", []) for chain_name, proposals in chain_info.items(): if proposals["Proposals"]: - proposals_check(customer, chain_name, proposals["Proposals"]) + proposals_check(customer, chain_name, proposals["Proposals"], price_feed_providers) else: # Single-task mode using command line arguments if not (customer and chain_name and proposal_address): raise ValueError("Customer, chain, and proposal_address must be specified if not using a config file.") - GitManager(customer).clone_or_update() - proposals_check(customer, chain_name, [proposal_address]) + ground_truth_config = ConfigLoader.load_customer_config(customer) + GitManager(customer, ground_truth_config).clone_or_update() + price_feed_providers = ground_truth_config.get("price_feed_providers", []) + proposals_check(customer, chain_name, [proposal_address], price_feed_providers) if __name__ == "__main__": diff --git a/Quorum/checks/__init__.py b/Quorum/checks/__init__.py index 679ef78..e146df8 100644 --- a/Quorum/checks/__init__.py +++ b/Quorum/checks/__init__.py @@ -1,7 +1,7 @@ from .diff import DiffCheck from .review_diff import ReviewDiffCheck from .global_variables import GlobalVariableCheck -from .feed_price import FeedPriceCheck +from .price_feed import PriceFeedCheck from .new_listing import NewListingCheck all = ["DiffCheck", "ReviewDiffCheck", "GlobalVariableCheck", "FeedPriceCheck", "NewListingCheck"] \ No newline at end of file diff --git a/Quorum/checks/feed_price.py b/Quorum/checks/price_feed.py similarity index 68% rename from Quorum/checks/feed_price.py rename to Quorum/checks/price_feed.py index 6382021..321ac7e 100644 --- a/Quorum/checks/feed_price.py +++ b/Quorum/checks/price_feed.py @@ -2,23 +2,29 @@ import re import json -from Quorum.apis.price_feeds import ChainLinkAPI, ChronicleAPI, PriceFeedProvider, PriceFeedData +from Quorum.apis.price_feeds import PriceFeedProvider, PriceFeedData, PriceFeedProviderBase from Quorum.utils.chain_enum import Chain from Quorum.checks.check import Check from Quorum.apis.block_explorers.source_code import SourceCode -import Quorum.config as config import Quorum.utils.pretty_printer as pp -class FeedPriceCheck(Check): +class PriceFeedCheck(Check): """ - The VerifyFeedPrice class is responsible for verifying the price feed addresses in the source code + The PriceFeedCheck class is responsible for verifying the price feed addresses in the source code against official Chainlink or Chronical data. """ - def __init__(self, customer: str, chain: Chain, proposal_address: str, source_codes: list[SourceCode]) -> None: + def __init__( + self, + customer: str, + chain: Chain, + proposal_address: str, + source_codes: list[SourceCode], + providers: list[PriceFeedProviderBase] + ) -> None: """ - Initializes the VerifyFeedPrice object with customer information, proposal address, + Initializes the PriceFeedCheck object with customer information, proposal address, and source codes to be checked. Args: @@ -26,39 +32,30 @@ def __init__(self, customer: str, chain: Chain, proposal_address: str, source_co chain (Chain): The blockchain network to verify the price feeds against. proposal_address (str): The address of the proposal being verified. source_codes (list[SourceCode]): A list of source code objects containing the Solidity contracts to be checked. + providers (list[PriceFeedProviderInterface]): A list of price feed providers to be used for verification. """ super().__init__(customer, chain, proposal_address, source_codes) self.address_pattern = r'0x[a-fA-F0-9]{40}' # load providers price feeds - self.providers_to_price_feed = self.__fetch_price_feed_providers() + self.providers_to_price_feed = self.__fetch_price_feed_data(providers) - def __fetch_price_feed_providers(self) -> dict[PriceFeedProvider, dict]: + def __fetch_price_feed_data(self, providers: list[PriceFeedProviderBase]) -> dict[PriceFeedProvider, dict]: """ Load the price feed providers from the ground truth file. + Args: + providers (list[PriceFeedProviderInterface]): A list of price feed providers + Returns: - dict[PriceFeedProvider, dict]: A dictionary mapping the price feed provider to the price feed data. + dict[str, dict]: A dictionary mapping the price feed provider to the price feed data. """ - with open(config.GROUND_TRUTH_PATH) as f: - providers: list = json.load(f).get(self.customer, {}).get("price_feed_providers", []) - if not providers: pp.pretty_print(f"No price feed providers found for {self.customer}", pp.Colors.FAILURE) return {} - - # Map providers to price feeds - providers_to_price_feed = {} - for provider in providers: - if provider == PriceFeedProvider.CHAINLINK: - providers_to_price_feed[provider] = ChainLinkAPI().get_price_feeds_info(self.chain) - elif provider == PriceFeedProvider.CHRONICLE: - providers_to_price_feed[provider] = ChronicleAPI().get_price_feeds_info(self.chain) - else: - pp.pretty_print(f"Unknown price feed provider: {provider}", pp.Colors.FAILURE) - providers.remove(provider) - - return providers_to_price_feed + return { + provider.get_name(): provider.get_feeds(self.chain) for provider in providers + } def __check_price_feed_address(self, address: str) -> dict | None: """ diff --git a/Quorum/config.py b/Quorum/config.py index 6323c85..15c3d37 100644 --- a/Quorum/config.py +++ b/Quorum/config.py @@ -16,4 +16,4 @@ DEFAULT_REPOS = Path(__file__).parent / "ground_truth.json" if not GROUND_TRUTH_PATH.exists(): - shutil.copy(DEFAULT_REPOS, GROUND_TRUTH_PATH) \ No newline at end of file + shutil.copy(DEFAULT_REPOS, GROUND_TRUTH_PATH) diff --git a/Quorum/utils/config_loader.py b/Quorum/utils/config_loader.py new file mode 100644 index 0000000..2b8dfff --- /dev/null +++ b/Quorum/utils/config_loader.py @@ -0,0 +1,39 @@ +import json +from typing import Dict +import Quorum.config as config +import Quorum.utils.pretty_printer as pp +from Quorum.apis.price_feeds import PriceFeedProvider, ChainLinkAPI, ChronicleAPI + +SUPPORTED_PROVIDERS = set(PriceFeedProvider.__members__.values()) + +with open(config.GROUND_TRUTH_PATH) as f: + config_data = json.load(f) + +def load_customer_config(customer: str) -> Dict[str, any]: + """ + Load the customer ground truth configuration data from the ground truth file, + and validate the price feed providers. + + Args: + customer (str): The name or identifier of the customer. + + Returns: + Dict[str, any]: The customer configuration data. + """ + customer_config = config_data.get(customer, {}) + providers = customer_config.get("price_feed_providers", []) + unsupported = set(providers) - SUPPORTED_PROVIDERS + if unsupported: + pp.pretty_print(f"Unsupported providers for {customer}: {', '.join(unsupported)}", pp.Colors.FAILURE) + providers = list(set(providers) & SUPPORTED_PROVIDERS) + customer_config["price_feed_providers"] = providers + + # Replace the provider names with the actual API objects + for i, provider in enumerate(providers): + if provider == PriceFeedProvider.CHAINLINK: + providers[i] = ChainLinkAPI() + elif provider == PriceFeedProvider.CHRONICLE: + providers[i] = ChronicleAPI() + + customer_config["price_feed_providers"] = providers + return customer_config diff --git a/Quorum/utils/singleton.py b/Quorum/utils/singleton.py index 8a09d95..d539318 100644 --- a/Quorum/utils/singleton.py +++ b/Quorum/utils/singleton.py @@ -1,6 +1,11 @@ +from abc import ABCMeta + class Singleton(type): _instances = {} def __call__(cls, *args, **kwargs): if cls not in cls._instances: cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) - return cls._instances[cls] \ No newline at end of file + return cls._instances[cls] + +class SingletonABCMeta(Singleton, ABCMeta): + pass From b6b011624ef70fe93d262babf6f87855bef08d0a Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Thu, 28 Nov 2024 17:37:14 +0200 Subject: [PATCH 5/9] Address comments --- Quorum/apis/price_feeds/chainlink_api.py | 8 ++++---- Quorum/apis/price_feeds/chronicle_api.py | 8 ++++---- Quorum/apis/price_feeds/price_feed_utils.py | 2 +- Quorum/utils/singleton.py | 18 +++++++----------- 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/Quorum/apis/price_feeds/chainlink_api.py b/Quorum/apis/price_feeds/chainlink_api.py index b19f0dd..40e4624 100644 --- a/Quorum/apis/price_feeds/chainlink_api.py +++ b/Quorum/apis/price_feeds/chainlink_api.py @@ -1,9 +1,9 @@ from Quorum.utils.chain_enum import Chain -from Quorum.utils.singleton import SingletonABCMeta +from Quorum.utils.singleton import singleton from .price_feed_utils import PriceFeedData, PriceFeedProviderBase, PriceFeedProvider - -class ChainLinkAPI(PriceFeedProviderBase, metaclass=SingletonABCMeta): +@singleton +class ChainLinkAPI(PriceFeedProviderBase): """ ChainLinkAPI is a class designed to interact with the Chainlink data feed API. It fetches and stores price feed data for various blockchain networks supported by Chainlink. @@ -54,4 +54,4 @@ def _get_price_feeds_info(self, chain: Chain) -> dict[str, PriceFeedData]: return chain_link_price_feeds def get_name(self) -> PriceFeedProvider: - return "Chainlink" + return PriceFeedProvider.CHAINLINK diff --git a/Quorum/apis/price_feeds/chronicle_api.py b/Quorum/apis/price_feeds/chronicle_api.py index 50a0031..b46043d 100644 --- a/Quorum/apis/price_feeds/chronicle_api.py +++ b/Quorum/apis/price_feeds/chronicle_api.py @@ -1,13 +1,13 @@ import requests from collections import defaultdict -import json from Quorum.utils.chain_enum import Chain -from Quorum.utils.singleton import SingletonABCMeta +from Quorum.utils.singleton import singleton from .price_feed_utils import PriceFeedData, PriceFeedProviderBase, PriceFeedProvider -class ChronicleAPI(PriceFeedProviderBase, metaclass=SingletonABCMeta): +@singleton +class ChronicleAPI(PriceFeedProviderBase): """ ChronicleAPI is a class designed to interact with the Chronicle data feed API. It fetches and stores price feed data for various blockchain networks supported by Chronicle. @@ -65,4 +65,4 @@ def _get_price_feeds_info(self, chain: Chain) -> dict[str, PriceFeedData]: def get_name(self) -> PriceFeedProvider: - return "Chronicle" + return PriceFeedProvider.CHRONICLE diff --git a/Quorum/apis/price_feeds/price_feed_utils.py b/Quorum/apis/price_feeds/price_feed_utils.py index 3c31061..a7bb73b 100644 --- a/Quorum/apis/price_feeds/price_feed_utils.py +++ b/Quorum/apis/price_feeds/price_feed_utils.py @@ -37,7 +37,7 @@ class PriceFeedProviderBase(ABC): def __init__(self): super().__init__() - self.cache_dir = Path(f"{__file__.removesuffix('.py')}/cache/{self.get_name()}") + self.cache_dir = Path(__file__).parent / "cache" / self.get_name().value self.cache_dir.mkdir(parents=True, exist_ok=True) self.session = requests.Session() self.memory: dict[Chain, dict[str, PriceFeedData]] = {} diff --git a/Quorum/utils/singleton.py b/Quorum/utils/singleton.py index d539318..0d29736 100644 --- a/Quorum/utils/singleton.py +++ b/Quorum/utils/singleton.py @@ -1,11 +1,7 @@ -from abc import ABCMeta - -class Singleton(type): - _instances = {} - def __call__(cls, *args, **kwargs): - if cls not in cls._instances: - cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) - return cls._instances[cls] - -class SingletonABCMeta(Singleton, ABCMeta): - pass +def singleton(cls): + instances = {} + def wrapper(*args, **kwargs): + if cls not in instances: + instances[cls] = cls(*args, **kwargs) + return instances[cls] + return wrapper \ No newline at end of file From 87f3dd32db5d7b65f8008a49e3feb1d74f96b6c2 Mon Sep 17 00:00:00 2001 From: nivcertora Date: Thu, 28 Nov 2024 16:12:31 +0000 Subject: [PATCH 6/9] Auto change version. --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 85dcb05..22ea403 100644 --- a/version +++ b/version @@ -1 +1 @@ -20241110.154442.714477 +20241128.161231.832485 From 1441b95098c8d1f56dbfcc4263c3e87a4e8b3414 Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Thu, 28 Nov 2024 19:24:36 +0200 Subject: [PATCH 7/9] Update Readme --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dc69f25..c4e4100 100644 --- a/README.md +++ b/README.md @@ -166,12 +166,14 @@ Example `ground_truth.json`: "https://github.com/bgd-labs/aave-address-book", "https://github.com/aave-dao/aave-v3-origin" ], - "review_repo": "https://github.com/bgd-labs/aave-proposals-v3" + "review_repo": "https://github.com/bgd-labs/aave-proposals-v3", + "price_feed_providers": ["Chainlink"] } } ``` -This configuration is used by the tool to manage the repositories. +This configuration is used by the tool to manage the ground truth information for each customer. The `dev_repos` array contains the URLs of the repositories associated with the customer. The `review_repo` field specifies the repository to compare against when checking proposals. The `price_feed_providers` array lists the feed price providers to check against. + ## Artifacts Structure From e7bd5f0dc7bd41c17b21c3d0bdd11fc1dc3eb81f Mon Sep 17 00:00:00 2001 From: niv vaknin Date: Thu, 28 Nov 2024 19:24:51 +0200 Subject: [PATCH 8/9] Address comments --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index c4e4100..2519722 100644 --- a/README.md +++ b/README.md @@ -174,6 +174,9 @@ Example `ground_truth.json`: This configuration is used by the tool to manage the ground truth information for each customer. The `dev_repos` array contains the URLs of the repositories associated with the customer. The `review_repo` field specifies the repository to compare against when checking proposals. The `price_feed_providers` array lists the feed price providers to check against. +current supported price feed providers are: +- Chainlink +- Chronicle ## Artifacts Structure From b4a2bbb92982c5dd9af8bb789007f6f90fb1c0df Mon Sep 17 00:00:00 2001 From: nivcertora Date: Thu, 28 Nov 2024 17:27:20 +0000 Subject: [PATCH 9/9] Auto change version. --- version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version b/version index 22ea403..7717179 100644 --- a/version +++ b/version @@ -1 +1 @@ -20241128.161231.832485 +20241128.172720.408365