New design + add cache to prevent api dealy

.gitignore

@@ -4,4 +4,5 @@
 **/local*
 build/
 *.egg-info
-CustomerClones/
+CustomerClones/
+**cache**

Quorum/apis/git_api/git_manager.py

@@ -1,11 +1,9 @@
-import json
 from pathlib import Path
 from git import Repo
 
 import Quorum.config as config
 import Quorum.utils.pretty_printer as pp
 
-
 class GitManager:
     """
     A class to manage Git repositories for a specific customer.
@@ -15,12 +13,13 @@ class GitManager:
         repos (dict): A dictionary mapping repository names to their URLs.
     """
 
-    def __init__(self, customer: str) -> None:
+    def __init__(self, customer: str, gt_config: dict[str, any]) -> None:
         """
         Initialize the GitManager with the given customer name and load the repository URLs.
 
         Args:
             customer (str): The name or identifier of the customer.
+            gt_config (dict[str, any]): The ground truth configuration data.
         """
         self.customer = customer
 
@@ -30,22 +29,22 @@ def __init__(self, customer: str) -> None:
         self.review_module_path = config.MAIN_PATH / self.customer / "review_module"
         self.review_module_path.mkdir(parents=True, exist_ok=True)
 
-        self.repos, self.review_repo = self._load_repos_from_file()
+        self.repos, self.review_repo = self._load_repos_from_file(gt_config)
 
-    def _load_repos_from_file(self) -> tuple[dict[str, str], dict[str, str]]:
+    def _load_repos_from_file(self, gt_config: dict[str, any]) -> tuple[dict[str, str], dict[str, str]]:
         """
         Load repository URLs from the JSON file for the given customer.
 
+        Args:
+            gt_config (dict[str, any]): The ground truth configuration data.
+
         Returns:
             tuple[dict[str, str], dict[str, str]]: 2 dictionaries mapping repository names to their URLs.
                 The first dictionary contains the repos to diff against. The second dictionary is the verification repo.
         """
-        with open(config.GROUND_TRUTH_PATH) as f:
-            repos_data = json.load(f)
-
         # Normalize the customer name to handle case differences
         normalized_customer = self.customer.lower()
-        customer_repos = next((repos for key, repos in repos_data.items() if key.lower() == normalized_customer), None)
+        customer_repos = next((repos for key, repos in gt_config.items() if key.lower() == normalized_customer), None)
         if customer_repos is None:
             return {}, {}
 

Quorum/apis/price_feeds/__init__.py

@@ -1,5 +1,5 @@
 from .chainlink_api import ChainLinkAPI
 from .chronicle_api import ChronicleAPI
-from .price_feed_utils import PriceFeedProvider, PriceFeedData
+from .price_feed_utils import PriceFeedData, PriceFeedProvider, PriceFeedProviderBase
 
-all = [ChainLinkAPI, ChronicleAPI, PriceFeedProvider, PriceFeedData]
+all = [ChainLinkAPI, ChronicleAPI, PriceFeedData, PriceFeedProvider, PriceFeedProviderBase]

Quorum/apis/price_feeds/chainlink_api.py

@@ -1,11 +1,9 @@
-import requests
-
 from Quorum.utils.chain_enum import Chain
-from Quorum.utils.singleton import Singleton
-from .price_feed_utils import PriceFeedData
+from Quorum.utils.singleton import SingletonABCMeta
+from .price_feed_utils import PriceFeedData, PriceFeedProviderBase, PriceFeedProvider
 
 
-class ChainLinkAPI(metaclass=Singleton):
+class ChainLinkAPI(PriceFeedProviderBase, metaclass=SingletonABCMeta):
     """
     ChainLinkAPI is a class designed to interact with the Chainlink data feed API.
     It fetches and stores price feed data for various blockchain networks supported by Chainlink.
@@ -29,22 +27,11 @@ class ChainLinkAPI(metaclass=Singleton):
         Chain.SCR: "https://reference-data-directory.vercel.app/feeds-ethereum-mainnet-scroll-1.json",
         Chain.ZK: "https://reference-data-directory.vercel.app/feeds-ethereum-mainnet-zksync-1.json"
     }
-
-    def __init__(self) -> None:
-        """
-        Initialize the ChainLinkAPI instance.
-        
-        Creates an HTTP session for managing requests and initializes an in-memory cache to store
-        price feed data for different blockchain networks.
-        """
-        self.session = requests.Session()
-        self.memory: dict[Chain, dict[str, PriceFeedData]] = {}
 
-    def get_price_feeds_info(self, chain: Chain) -> dict[str, PriceFeedData]:
+    def _get_price_feeds_info(self, chain: Chain) -> dict[str, PriceFeedData]:
         """
-        Fetches the price feeds information from the Chainlink API for a specified blockchain network.
+        Get price feed data for a given blockchain network.
 
-        The method fetches the price feed data from the Chainlink API and stores it in the memory cache.
         Args:
             chain (Chain): The blockchain network to fetch price feeds for.
 
@@ -55,16 +42,16 @@ def get_price_feeds_info(self, chain: Chain) -> dict[str, PriceFeedData]:
             requests.HTTPError: If the HTTP request to the Chainlink API fails.
             KeyError: If the specified chain is not supported.
         """
-        if chain not in self.memory:
-            url = self.chain_mapping.get(chain)
-            if not url:
-                raise KeyError(f"Chain {chain.name} is not supported.")
-
-            response = self.session.get(url)
-            response.raise_for_status()
-            chain_link_price_feeds = [PriceFeedData(**feed) for feed in response.json()]
-            chain_link_price_feeds = {feed.address: feed for feed in chain_link_price_feeds}
-            chain_link_price_feeds.update({feed.proxy_address: feed for feed in chain_link_price_feeds.values() if feed.proxy_address})
-            self.memory[chain] = chain_link_price_feeds
+        url = self.chain_mapping.get(chain)
+        if not url:
+            raise KeyError(f"Chain {chain.name} is not supported.")
 
-        return self.memory[chain]
+        response = self.session.get(url)
+        response.raise_for_status()
+        chain_link_price_feeds = [PriceFeedData(**feed) for feed in response.json()]
+        chain_link_price_feeds = {feed.address: feed for feed in chain_link_price_feeds}
+        chain_link_price_feeds.update({feed.proxy_address: feed for feed in chain_link_price_feeds.values() if feed.proxy_address})
+        return chain_link_price_feeds
+
+    def get_name(self) -> PriceFeedProvider:
+        return "Chainlink"

Quorum/apis/price_feeds/chronicle_api.py

@@ -1,12 +1,13 @@
 import requests
 from collections import defaultdict
+import json
 
 from Quorum.utils.chain_enum import Chain
-from Quorum.utils.singleton import Singleton
-from .price_feed_utils import PriceFeedData
+from Quorum.utils.singleton import SingletonABCMeta
+from .price_feed_utils import PriceFeedData, PriceFeedProviderBase, PriceFeedProvider
 
 
-class ChronicleAPI(metaclass=Singleton):
+class ChronicleAPI(PriceFeedProviderBase, metaclass=SingletonABCMeta):
     """
     ChronicleAPI is a class designed to interact with the Chronicle data feed API.
     It fetches and stores price feed data for various blockchain networks supported by Chronicle.
@@ -17,8 +18,7 @@ class ChronicleAPI(metaclass=Singleton):
     """
 
     def __init__(self):
-        self.session = requests.Session()
-        self.memory: dict[Chain, dict[str, dict]] = {}
+        super().__init__()
         self.__pairs = self.__process_pairs()
 
     def __process_pairs(self) -> dict[str, list[str]]:
@@ -35,34 +35,34 @@ def __process_pairs(self) -> dict[str, list[str]]:
             result[p["blockchain"]].append(p["pair"])
         return result
 
-    def get_price_feeds_info(self, chain: Chain) -> dict[str, dict]:
+    def _get_price_feeds_info(self, chain: Chain) -> dict[str, PriceFeedData]:
         """
         Get price feed data for a given blockchain network.
 
-        This method retrieves price feed data from the cache if it has been fetched previously.
-
         Args:
-            chain (Chain): The blockchain network to fetch price feed data for.
+            chain (Chain): The blockchain network to fetch price feeds for.
 
         Returns:
-            dict[str, dict]: A dictionary mapping the contract address of the price feed to the price feed data.
+            dict[str, PriceFeedData]: A dictionary mapping the contract address of the price feed to the PriceFeedData object.
         """
-        if chain not in self.memory:
-            pairs = self.__pairs.get(chain)
-            if not pairs:
-                return {}
+
+        pairs = self.__pairs.get(chain)
+        if not pairs:
+            return {}
+
+        chronicle_price_feeds: list[PriceFeedData] = []
+        for pair in pairs:
+            response = self.session.get(
+                f"https://chroniclelabs.org/api/median/info/{pair}/{chain.value}/?testnet=false"
+            )
+            response.raise_for_status()
+            data = response.json()
 
-            chronicle_price_feeds: list[PriceFeedData] = []
-            for pair in pairs:
-                response = self.session.get(
-                    f"https://chroniclelabs.org/api/median/info/{pair}/{chain.value}/?testnet=false"
-                )
-                response.raise_for_status()
-                data = response.json()
+            for pair_info in data:
+                chronicle_price_feeds.append(PriceFeedData(**pair_info))
 
-                for pair_info in data:
-                    chronicle_price_feeds.append(PriceFeedData(**pair_info))
-
-            self.memory[chain] = {feed.address: feed for feed in chronicle_price_feeds}
+        return {feed.address: feed for feed in chronicle_price_feeds}
+
 
-        return self.memory[chain]
+    def get_name(self) -> PriceFeedProvider:
+        return "Chronicle"

Quorum/apis/price_feeds/price_feed_utils.py

@@ -1,6 +1,12 @@
+from abc import ABC, abstractmethod
 from enum import StrEnum
 from typing import Optional
 from pydantic import BaseModel, Field
+from pathlib import Path
+import json
+import requests
+
+from Quorum.utils.chain_enum import Chain
 
 class PriceFeedProvider(StrEnum):
     """
@@ -9,7 +15,6 @@ class PriceFeedProvider(StrEnum):
     CHAINLINK = 'Chainlink'
     CHRONICLE = 'Chronicle'
 
-
 class PriceFeedData(BaseModel):
     name: Optional[str]
     pair: Optional[str | list]
@@ -19,3 +24,50 @@ class PriceFeedData(BaseModel):
     class Config:
         allow_population_by_field_name = True  # Allows population using field names
         extra = 'ignore'  # Ignores extra fields not defined in the model
+
+
+class PriceFeedProviderBase(ABC):
+    """
+    PriceFeedProviderBase is an abstract base class for price feed providers.
+    It defines the interface for fetching price feed data for different blockchain networks.
+
+    Attributes:
+        cache_dir (Path): The directory path to store the fetched price feed data.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self.cache_dir = Path(f"{__file__.removesuffix('.py')}/cache/{self.get_name()}")
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+        self.session = requests.Session()
+        self.memory: dict[Chain, dict[str, PriceFeedData]] = {}
+
+    def get_feeds(self, chain: Chain) -> dict[str, PriceFeedData]:
+        """
+        Get price feed data for a given blockchain network from the cache.
+
+        Args:
+            chain (Chain): The blockchain network to fetch price feed data for.
+
+        Returns:
+            dict[str, PriceFeedData]: A dictionary mapping the contract address of the price feed to the price feed data.
+        """
+        cache_file = self.cache_dir / f"{chain.value}.json"
+        if cache_file.exists():
+            with open(cache_file, 'r') as file:
+                data: dict = json.load(file)
+            self.memory[chain] = {addr: PriceFeedData(**feed) for addr, feed in data.items()}
+        else:
+            if chain not in self.memory:
+                self.memory[chain] = self._get_price_feeds_info(chain)
+                with open(cache_file, 'w') as file:
+                    json.dump({addr: feed.dict() for addr, feed in self.memory[chain].items()}, file)
+        return self.memory[chain]
+
+    @abstractmethod
+    def _get_price_feeds_info(self, chain: Chain) -> dict[str, PriceFeedData]:
+        pass
+
+    @abstractmethod
+    def get_name(self) -> PriceFeedProvider:
+        pass

Quorum/check_proposal.py

@@ -3,10 +3,12 @@
 from typing import Any, Optional
 
 from Quorum.utils.chain_enum import Chain
-import Quorum.utils.pretty_printer as pp
 from Quorum.apis.git_api.git_manager import GitManager
+from Quorum.apis.price_feeds.price_feed_utils import PriceFeedProviderBase
 from Quorum.apis.block_explorers.chains_api import ChainAPI
 import Quorum.checks as Checks
+import Quorum.utils.pretty_printer as pp
+import Quorum.utils.config_loader as ConfigLoader
 
 
 def parse_args() -> tuple[Optional[str], Optional[str], Optional[str], Optional[str]]:
@@ -45,7 +47,7 @@ def load_config(config_path: str) -> dict[str, Any] | None:
         pp.pretty_print(f"Failed to parse given config file {config_path}:\n{e}", pp.Colors.FAILURE)
 
 
-def proposals_check(customer: str, chain_name: str, proposal_addresses: list[str]) -> None:
+def proposals_check(customer: str, chain_name: str, proposal_addresses: list[str], providers: list[PriceFeedProviderBase]) -> None:
     """
     Check and compare source code files for given proposals.
 
@@ -55,6 +57,7 @@ def proposals_check(customer: str, chain_name: str, proposal_addresses: list[str
         customer (str): The customer name or identifier.
         chain_name (str): The blockchain chain name.
         proposal_addresses (list[str]): List of proposal addresses.
+        providers (list[PriceFeedProviderInterface]): List of price feed providers.
     """
     chain = Chain[chain_name.upper()]
     api = ChainAPI(chain)
@@ -74,7 +77,7 @@ def proposals_check(customer: str, chain_name: str, proposal_addresses: list[str
         Checks.GlobalVariableCheck(customer, chain, proposal_address, missing_files).check_global_variables()
 
         # Feed price check
-        Checks.FeedPriceCheck(customer, chain, proposal_address, missing_files).verify_price_feed()
+        Checks.PriceFeedCheck(customer, chain, proposal_address, missing_files, providers).verify_price_feed()
 
         # New listing check
         Checks.NewListingCheck(customer, chain, proposal_address, missing_files).new_listing_check()
@@ -91,17 +94,21 @@ def main() -> None:
     if config_data:
         # Multi-task mode using JSON configuration
         for customer, chain_info in config_data.items():
-            GitManager(customer).clone_or_update()
+            ground_truth_config = ConfigLoader.load_customer_config(customer)
+            GitManager(customer, ground_truth_config).clone_or_update()
+            price_feed_providers = ground_truth_config.get("price_feed_providers", [])
             for chain_name, proposals in chain_info.items():
                 if proposals["Proposals"]: 
-                    proposals_check(customer, chain_name, proposals["Proposals"])
+                    proposals_check(customer, chain_name, proposals["Proposals"], price_feed_providers)
     else:
         # Single-task mode using command line arguments
         if not (customer and chain_name and proposal_address):
             raise ValueError("Customer, chain, and proposal_address must be specified if not using a config file.")
 
-        GitManager(customer).clone_or_update()    
-        proposals_check(customer, chain_name, [proposal_address])
+        ground_truth_config = ConfigLoader.load_customer_config(customer)
+        GitManager(customer, ground_truth_config).clone_or_update()
+        price_feed_providers = ground_truth_config.get("price_feed_providers", [])    
+        proposals_check(customer, chain_name, [proposal_address], price_feed_providers)
 
 
 if __name__ == "__main__":

Quorum/checks/__init__.py

@@ -1,7 +1,7 @@
 from .diff import DiffCheck
 from .review_diff import ReviewDiffCheck
 from .global_variables import GlobalVariableCheck
-from .feed_price import FeedPriceCheck
+from .price_feed import PriceFeedCheck
 from .new_listing import NewListingCheck
 
 all = ["DiffCheck", "ReviewDiffCheck", "GlobalVariableCheck", "FeedPriceCheck", "NewListingCheck"]