Merge pull request #29 from hummingbot/feat/add_download_candles_page

Feat/add download candles page

Makefile

@@ -10,7 +10,7 @@ env_remove:
 	conda env remove -n dashboard
 
 env_create:
-	conda env create -f environment.yml
+	conda env create -f environment_conda.yml
 
 docker_build:
 	docker build -t dashboard:latest .

README.md

@@ -34,6 +34,14 @@ These files are located in the `/data` folder in Hummingbot, and are named `<str
 
 Easily compare various DeFi protocols based on their market capitalization and total value locked, using DeFiLlama data.
 
+### 🗂 Candles Downloader
+
+Download historical exchange data as OHLVC candles. Supports multiple trading pairs and custom time ranges/intervals.
+
+Current Hummingbot connectors supported:
+* `binance`
+* `binance_perpetual`
+
 ### 📋 Data
 
 Reference data for the various apps this collection.
@@ -71,6 +79,13 @@ ln -s /path/to/hummingbot/data data
 
 For more info about Streamlit installation, see the instructions located at https://docs.streamlit.io/library/get-started/installation.
 
+## Updates
+
+To update the `dashboard` environment for new dependencies, run:
+```
+conda env update -f environment_conda.yml
+```
+
 ## Contributions
 
 We welcome new data apps, bug fixes, and improvements by the community!

constants.py

@@ -0,0 +1,2 @@
+CANDLES_DATA_PATH = "data/candles"
+DOWNLOAD_CANDLES_CONFIG_YML = "hummingbot_files/scripts_configs/data_downloader_config.yml"

environment_conda.yml

@@ -14,4 +14,5 @@ dependencies:
       - glom
       - defillama
       - statsmodels
-      - pandas_ta
+      - pandas_ta
+      - pyyaml

hummingbot_files/compose_files/data-downloader-compose.yml

@@ -0,0 +1,22 @@
+version: "3.9"
+services:
+  bot:
+    container_name: data_downloader
+    image: dardonacci/hummingbot:development
+    volumes:
+      - "../../data/candles:/home/hummingbot/data"
+      - "../conf:/home/hummingbot/conf"
+      - "../conf/connectors:/home/hummingbot/conf/connectors"
+    environment:
+      - CONFIG_PASSWORD=a
+      - CONFIG_FILE_NAME=download_candles.py
+    env_file:
+      - ../scripts_configs/data_downloader_config.yml
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "10m"
+        max-file: 5
+    tty: true
+    stdin_open: true
+    network_mode: host

hummingbot_files/conf/.password_verification

@@ -0,0 +1 @@
+7b2263727970746f223a207b22636970686572223a20226165732d3132382d637472222c2022636970686572706172616d73223a207b226976223a20223864336365306436393461623131396334363135663935366464653839363063227d2c202263697068657274657874223a20223836333266323430613563306131623665353664222c20226b6466223a202270626b646632222c20226b6466706172616d73223a207b2263223a20313030303030302c2022646b6c656e223a2033322c2022707266223a2022686d61632d736861323536222c202273616c74223a20226566373330376531636464373964376132303338323534656139343433663930227d2c20226d6163223a202266393439383534613530633138363633386363353962336133363665633962353333386633613964373266636635343066313034333361353431636232306438227d2c202276657273696f6e223a20337d

hummingbot_files/conf/conf_client.yml

@@ -0,0 +1,194 @@
+####################################
+###   client_config_map config   ###
+####################################
+
+instance_id: 039758736d451914503a45ff596e168902d62557
+
+log_level: INFO
+
+debug_console: false
+
+strategy_report_interval: 900.0
+
+logger_override_whitelist:
+- hummingbot.strategy.arbitrage
+- hummingbot.strategy.cross_exchange_market_making
+- conf
+
+log_file_path: /home/hummingbot/logs
+
+kill_switch_mode: {}
+
+# What to auto-fill in the prompt after each import command (start/config)
+autofill_import: disabled
+
+telegram_mode: {}
+
+# MQTT Bridge configuration.
+mqtt_bridge:
+  mqtt_host: localhost
+  mqtt_port: 1883
+  mqtt_username: ''
+  mqtt_password: ''
+  mqtt_namespace: hbot
+  mqtt_ssl: false
+  mqtt_logger: true
+  mqtt_notifier: true
+  mqtt_commands: true
+  mqtt_events: true
+  mqtt_external_events: true
+  mqtt_autostart: false
+
+# Error log sharing
+send_error_logs: true
+
+# Can store the previous strategy ran for quick retrieval.
+previous_strategy: null
+
+# Advanced database options, currently supports SQLAlchemy's included dialects
+# Reference: https://docs.sqlalchemy.org/en/13/dialects/
+# To use an instance of SQLite DB the required configuration is 
+#   db_engine: sqlite
+# To use a DBMS the required configuration is
+#   db_host: 127.0.0.1
+#   db_port: 3306
+#   db_username: username
+#   db_password: password
+#   db_name: dbname
+db_mode:
+  db_engine: sqlite
+
+pmm_script_mode: {}
+
+# Balance Limit Configurations
+# e.g. Setting USDT and BTC limits on Binance.
+# balance_asset_limit:
+#   binance:
+#     BTC: 0.1
+#     USDT: 1000
+balance_asset_limit:
+  bybit_testnet: {}
+  lbank: {}
+  binance_us: {}
+  crypto_com: {}
+  ascend_ex_paper_trade: {}
+  hotbit: {}
+  gate_io_paper_trade: {}
+  bitmex_testnet: {}
+  ndax_testnet: {}
+  huobi: {}
+  probit_kr: {}
+  altmarkets: {}
+  hitbtc: {}
+  foxbit: {}
+  ascend_ex: {}
+  binance: {}
+  okx: {}
+  ciex: {}
+  bitmex: {}
+  bitfinex: {}
+  probit: {}
+  kraken: {}
+  kucoin: {}
+  bitmart: {}
+  bybit: {}
+  bittrex: {}
+  btc_markets: {}
+  mock_paper_exchange: {}
+  kucoin_paper_trade: {}
+  ndax: {}
+  loopring: {}
+  mexc: {}
+  whitebit: {}
+  coinbase_pro: {}
+  binance_paper_trade: {}
+  gate_io: {}
+
+# Fixed gas price (in Gwei) for Ethereum transactions
+manual_gas_price: 50.0
+
+# Gateway API Configurations
+# default host to only use localhost
+# Port need to match the final installation port for Gateway
+gateway:
+  gateway_api_host: localhost
+  gateway_api_port: '15888'
+
+certs_path: /home/hummingbot/certs
+
+# Whether to enable aggregated order and trade data collection
+anonymized_metrics_mode:
+  anonymized_metrics_interval_min: 15.0
+
+# Command Shortcuts
+# Define abbreviations for often used commands
+# or batch grouped commands together
+command_shortcuts:
+- command: spreads
+  help: Set bid and ask spread
+  arguments:
+  - Bid Spread
+  - Ask Spread
+  output:
+  - config bid_spread $1
+  - config ask_spread $2
+
+# A source for rate oracle, currently ascend_ex, binance, coin_gecko, kucoin, gate_io
+rate_oracle_source:
+  name: binance
+
+# A universal token which to display tokens values in, e.g. USD,EUR,BTC
+global_token:
+  global_token_name: USD
+  global_token_symbol: $
+
+# Percentage of API rate limits (on any exchange and any end point) allocated to this bot instance.
+# Enter 50 to indicate 50%. E.g. if the API rate limit is 100 calls per second, and you allocate 
+# 50% to this setting, the bot will have a maximum (limit) of 50 calls per second
+rate_limits_share_pct: 100.0
+
+commands_timeout:
+  create_command_timeout: 10.0
+  other_commands_timeout: 30.0
+
+# Tabulate table format style (https://github.com/astanin/python-tabulate#table-format)
+tables_format: psql
+
+paper_trade:
+  paper_trade_exchanges:
+  - binance
+  - kucoin
+  - ascend_ex
+  - gate_io
+  paper_trade_account_balance:
+    BTC: 1.0
+    USDT: 1000.0
+    ONE: 1000.0
+    USDQ: 1000.0
+    TUSD: 1000.0
+    ETH: 10.0
+    WETH: 10.0
+    USDC: 1000.0
+    DAI: 1000.0
+
+color:
+  top_pane: '#000000'
+  bottom_pane: '#000000'
+  output_pane: '#262626'
+  input_pane: '#1C1C1C'
+  logs_pane: '#121212'
+  terminal_primary: '#5FFFD7'
+  primary_label: '#5FFFD7'
+  secondary_label: '#FFFFFF'
+  success_label: '#5FFFD7'
+  warning_label: '#FFFF00'
+  info_label: '#5FD7FF'
+  error_label: '#FF0000'
+  gold_label: '#FFD700'
+  silver_label: '#C0C0C0'
+  bronze_label: '#CD7F32'
+
+# The tick size is the frequency with which the clock notifies the time iterators by calling the
+# c_tick() method, that means for example that if the tick size is 1, the logic of the strategy 
+# will run every second.
+tick_size: 1.0