Merge branch 'feat/indexer-cleaning-script' of github.com:Synthetixio…

…/data into feat/indexer-cleaning-script

Makefile

@@ -14,8 +14,8 @@ extract:
 	docker compose run extractors python main.py configs/arbitrum_sepolia.yaml
 
 index:
-	docker compose run indexer --network_name base_mainnet --config_name synthetix-v3
-	docker compose run indexer --network_name arbitrum_mainnet --config_name synthetix-v3
+	docker compose run indexer --network_name base_mainnet --protocol_name synthetix
+	docker compose run indexer --network_name arbitrum_mainnet --protocol_name synthetix
 
 synths:
 	docker compose run transformer python scripts/get_synths.py

docker-compose.yml

@@ -32,7 +32,7 @@ services:
       - .env
     environment:
       NETWORK_NAME: base_mainnet
-      CONFIG_NAME: synthetix-v3
+      PROTOCOL_NAME: synthetix
     volumes:
       - ./parquet-data:/parquet-data
 

indexers/.dockerignore

@@ -0,0 +1,11 @@
+.git
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.DS_Store
+.env
+.venv
+venv
+env
+node_modules

indexers/Dockerfile

@@ -1,20 +1,22 @@
 FROM node:16-alpine
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 
 WORKDIR /app
 
 COPY package*.json ./
-COPY requirements.txt .
 COPY patches/ ./patches/
 
-RUN apk add --no-cache python3 py3-pip
-RUN apk add --no-cache build-base && npm ci && apk del build-base
+RUN apk add --no-cache python3 python3-dev py3-pip clang cmake build-base
 
-COPY . .
+RUN npm ci
 
-RUN chmod +x entrypoint.sh
+COPY pyproject.toml uv.lock ./
+RUN uv sync --frozen
 
-RUN pip install --upgrade pip
-RUN pip install --no-cache-dir -r requirements.txt
+RUN apk del build-base
 
+COPY . .
+
+RUN chmod +x entrypoint.sh
 
-ENTRYPOINT ["./entrypoint.sh"]
+CMD ["./entrypoint.sh"]

indexers/entrypoint.sh

@@ -4,7 +4,7 @@
 set -e
 
 # Get contract data from SDK and generate squidgen.yaml and squid.yaml
-python3 main.py --network_name $NETWORK_NAME --config_name $CONFIG_NAME "$@"
+uv run main.py --network_name $NETWORK_NAME --protocol_name $PROTOCOL_NAME "$@"
 
 # Generate squid processor
 npm run generate:processor

indexers/main.py

@@ -21,7 +21,7 @@ def create_squidgen_config(
     network_name,
     contracts_info,
     block_range,
-    config_name,
+    protocol_name,
     rate_limit=10,
 ):
     config = {
@@ -30,7 +30,7 @@ def create_squidgen_config(
         "chain": {"url": rpc_url, "rateLimit": rate_limit},
         "target": {
             "type": "parquet",
-            "path": f"/parquet-data/indexed-raw/{network_name}/{config_name}",
+            "path": f"/parquet-data/indexers/raw/{network_name}/{protocol_name}",
         },
         "contracts": [],
     }
@@ -83,9 +83,9 @@ def load_network_config(path):
     )
     parser.add_argument("--network_name", type=str, help="Network name", required=True)
     parser.add_argument(
-        "--config_name",
+        "--protocol_name",
         type=str,
-        help="Name of the configuration to use",
+        help="Name of the protocol to index",
         required=True,
     )
     parser.add_argument(
@@ -96,7 +96,7 @@ def load_network_config(path):
     args = parser.parse_args()
 
     network_name = args.network_name
-    config_name = args.config_name
+    protocol_name = args.protocol_name
     contract_names = args.contract_names
 
     # Get contract names
@@ -117,7 +117,7 @@ def load_network_config(path):
     archive_url = network_params.get("archive_url", "None")
 
     # Load custom config
-    custom_config = config_file["configs"][config_name]
+    custom_config = config_file["configs"][protocol_name]
 
     # Initialize Synthetix SDK (with optional Cannon config)
     if "cannon_config" in custom_config:
@@ -147,9 +147,6 @@ def load_network_config(path):
     if "contracts_from_sdk" in custom_config:
         contracts_from_sdk = custom_config["contracts_from_sdk"]
         for contract in contracts_from_sdk:
-            if contract_names is not None:
-                if contract["name"] not in parsed_contract_names:
-                    continue
             name = contract["name"]
             package = contract["package"]
             contract_data = snx.contracts[package][name]
@@ -160,11 +157,9 @@ def load_network_config(path):
     elif "contracts_from_abi" in custom_config:
         contracts_from_abi = custom_config["contracts_from_abi"]
         for contract in contracts_from_abi:
-            if contract_names is not None:
-                if contract["name"] not in parsed_contract_names:
-                    continue
             name = contract["name"]
-            with open(f"{path}/abi/{name}.json", "r") as file:
+            abi_name = contract["abi"]
+            with open(f"{path}/{abi_name}", "r") as file:
                 abi = json.load(file)
             save_abi(abi, name)
             contracts.append({"name": name, "address": contract["address"]})
@@ -180,7 +175,7 @@ def load_network_config(path):
         network_name,
         contracts,
         block_range,
-        config_name,
+        protocol_name,
         rate_limit,
     )
     write_yaml(squidgen_config, "squidgen.yaml")

indexers/networks/arbitrum_mainnet/network_config.yaml

@@ -4,7 +4,7 @@ network:
   archive_url: "https://v2.archive.subsquid.io/network/arbitrum-one"
 
 configs:
-  synthetix-v3:
+  synthetix:
     range:
       from: 218000000
       to: latest
@@ -20,11 +20,11 @@ configs:
       - name: PerpsAccountProxy
         package: perpsFactory
 
-  curve_usdx:
+  curve:
     range:
       from: 236000000
       to: latest
     contracts_from_abi:
-      - name: CurveUSDx
+      - name: CurveUsdx
         address: '0x096A8865367686290639bc50bF8D85C0110d9Fea'
-        abi: ./abi/CurveUSDx.json
+        abi: ./abi/CurvePool.json

indexers/networks/base_mainnet/network_config.yaml

@@ -4,7 +4,7 @@ network:
   archive_url: "https://v2.archive.subsquid.io/network/base-mainnet"
 
 configs:
-  synthetix-v3:
+  synthetix:
     range:
       from: 7500000
       to: latest

indexers/pyproject.toml

@@ -0,0 +1,13 @@
+[project]
+name = "indexers"
+version = "0.1.0"
+description = "A blockchain indexer"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "fastparquet>=2024.11.0",
+    "pandas>=2.2.3",
+    "python-dotenv>=1.0.1",
+    "pyyaml>=6.0.2",
+    "synthetix>=0.1.21",
+]

indexers/scripts/clean.py

@@ -5,8 +5,8 @@
 
 
 def clean_parquet_files(network_name: str, protocol: str):
-    source_base = f"parquet-data/indexers/raw/{network_name}/{protocol}"
-    target_base = f"parquet-data/indexers/clean/{network_name}/{protocol}"
+    source_base = f"/parquet-data/indexers/raw/{network_name}/{protocol}"
+    target_base = f"/parquet-data/indexers/clean/{network_name}/{protocol}"
 
     protocol_path = Path(source_base)
     if not protocol_path.exists():