Merge pull request #15 from Solratic/feature/proxy-for-dataset

- feat(installer): add download_from_url function

decodex/installer/__init__.py

@@ -1,6 +1,8 @@
+from .installer import download_from_url
 from .installer import download_github_file
 
 
 __all__ = [
  "download_github_file",
+ "download_from_url",
 ]

decodex/installer/callbacks.py

@@ -134,7 +134,7 @@ def post_download(src_path: str, save_path: str, **config):
  hash_object = hashlib.new(hash_algorithm)
 
  with Path(src_path).open("rb") as f:
- while chunk := f.read(4096):
+ for chunk in iter(lambda: f.read(4096), b""):
  hash_object.update(chunk)
 
  checksum = hash_object.hexdigest()

decodex/installer/installer.py

@@ -1,6 +1,8 @@
+import os
 import pathlib
 import tempfile
 import warnings
+from typing import Dict
 
 import requests
 from tqdm import tqdm
@@ -28,15 +30,22 @@ def _get_github_url(save_path: str, org: str, repo: str, branch: str, path: str,
  )
 
 
-def _download_from_url(url: str, save_path: str, verify_ssl: bool) -> None:
- response = requests.get(url=url, verify=verify_ssl, stream=True)
- file_size = int(response.headers.get("content-length", 0))
- with tqdm(total=file_size, unit="iB", unit_scale=True) as pbar:
- with open(save_path, "wb") as f:
- for chunk in response.iter_content(chunk_size=1024):
- if chunk:
- f.write(chunk)
- pbar.update(len(chunk))
+def download_from_url(url: str, save_path: str, verify_ssl: bool, retry_with_proxy: bool = True) -> None:
+ try:
+ response = requests.get(url=url, verify=verify_ssl, stream=True)
+ file_size = int(response.headers.get("content-length", 0))
+ with tqdm(total=file_size, unit="iB", unit_scale=True) as pbar:
+ with open(save_path, "wb") as f:
+ for chunk in response.iter_content(chunk_size=1024):
+ if chunk:
+ f.write(chunk)
+ pbar.update(len(chunk))
+ except Exception as e:
+ if not retry_with_proxy:
+ raise e
+ proxy = os.getenv("PROXY_URL", None)
+ if proxy:
+ download_from_url(url, save_path, verify_ssl, retry_with_proxy=False)
 
 
 def download_github_file(
@@ -63,7 +72,7 @@ def download_github_file(
 
  try:
  src_path = tempfile.mktemp() if use_tempfile else save_path
- _download_from_url(url, src_path, verify_ssl)
+ download_from_url(url, src_path, verify_ssl)
  except Exception as e:
  print(f"Error downloading file: {e}")
  return

decodex/translate/translate.py

@@ -117,27 +117,31 @@ def install(self):
  from decodex import installer
  from decodex import constant
 
- installer.download_github_file(
- save_path=str(constant.DECODEX_DIR.joinpath(self.chain, "tags.json")),
- org="brianleect",
- repo="etherscan-labels",
- branch="main",
- path="data/etherscan/combined/combinedAllLabels.json",
- is_lfs=False,
- verify_ssl=False,
- use_tempfile=False,
- )
+ tags_path = constant.DECODEX_DIR.joinpath(self.chain, "tags.json")
+ if not tags_path.exists():
+ installer.download_github_file(
+ save_path=str(tags_path),
+ org="brianleect",
+ repo="etherscan-labels",
+ branch="main",
+ path="data/etherscan/combined/combinedAllLabels.json",
+ is_lfs=False,
+ verify_ssl=False,
+ use_tempfile=False,
+ )
 
- installer.download_github_file(
- save_path=str(constant.DECODEX_DIR.joinpath(self.chain, "signatures.csv")),
- org="Solratic",
- repo="function-signature-registry",
- branch="main",
- path="data/ethereum/func_sign.csv.gz",
- is_lfs=True,
- verify_ssl=False,
- use_tempfile=True,
- )
+ signature_path = constant.DECODEX_DIR.joinpath(self.chain, "signatures.csv")
+ if not signature_path.exists():
+ installer.download_github_file(
+ save_path=str(signature_path),
+ org="Solratic",
+ repo="function-signature-registry",
+ branch="main",
+ path="data/ethereum/func_sign.csv.gz",
+ is_lfs=True,
+ verify_ssl=False,
+ use_tempfile=True,
+ )
 
  def translate(self, txhash: str, *, max_workers: int = 10) -> TaggedTx:
  tx: Tx = self.searcher.get_tx(txhash)