From 92e5987e2a90c4329296e2fa8859c513457530b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Andr=C3=A9s=20Marino=20Rojas?= <47573394+Marinovsky@users.noreply.github.com> Date: Mon, 11 Sep 2023 10:59:02 -0500 Subject: [PATCH] Add support for different datafeed in cloud (#359) * Fix bug and add unit test After debugging the logs from the strategies deployed in the cloud, it was found the configuration name used by the API to use QC + IB data feed. Therefore, only the method which returned the data handler name was changed. * Modify get_price_data_handler() to rely on json * Solve bugs * Make get_price_data_handler() more generic * Fix bug * update version of modules json file * Update README.md * Add support for interactive mode * Add _configure_data_feed() method --- README.md | 8 +++- lean/commands/cloud/live/deploy.py | 19 +++++++- lean/models/__init__.py | 2 +- lean/models/brokerages/cloud/__init__.py | 17 ++++++- .../brokerages/cloud/cloud_brokerage.py | 10 +++- .../cloud/live/test_cloud_live_commands.py | 47 ++++++++++++++++++- 6 files changed, 94 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a3024c2c..53059f9e 100644 --- a/README.md +++ b/README.md @@ -269,12 +269,16 @@ Options: Weekly restart UTC time (hh:mm:ss). Each week on Sunday your algorithm is restarted at this time, and will require 2FA verification. This is required by Interactive Brokers. Use this option explicitly to override the default value. - --ib-data-feed BOOLEAN Whether the Interactive Brokers price data feed must be used instead of the - QuantConnect price data feed + --ib-data-feed [QuantConnect|Interactive Brokers|QuantConnect + InteractiveBrokers] + The data feed to use. These are the available ones: Interactive Brokers price data + feed, QuantConnect price data feed or QuantConnect + InteractiveBrokers price data feed. --tradier-account-id TEXT Your Tradier account id --tradier-access-token TEXT Your Tradier access token --tradier-environment [live|paper] Whether the developer sandbox should be used + --tradier-data-feed [QuantConnect|Tradier Brokerage] + The data feed to use. These are the available ones: QuantConnect price data feed or + Tradier Brokerage data feed. --oanda-account-id TEXT Your OANDA account id --oanda-access-token TEXT Your OANDA API token --oanda-environment [Practice|Trade] diff --git a/lean/commands/cloud/live/deploy.py b/lean/commands/cloud/live/deploy.py index be68cfcb..640680e2 100644 --- a/lean/commands/cloud/live/deploy.py +++ b/lean/commands/cloud/live/deploy.py @@ -25,7 +25,7 @@ from lean.models.brokerages.cloud.cloud_brokerage import CloudBrokerage from lean.models.configuration import InternalInputUserInput from lean.models.click_options import options_from_json, get_configs_for_options -from lean.models.brokerages.cloud import all_cloud_brokerages +from lean.models.brokerages.cloud import all_cloud_brokerages, cloud_brokerage_data_feeds from lean.commands.cloud.live.live import live from lean.components.util.live_utils import get_last_portfolio_cash_holdings, configure_initial_cash_balance, configure_initial_holdings,\ _configure_initial_cash_interactively, _configure_initial_holdings_interactively @@ -113,6 +113,20 @@ def _configure_brokerage(lean_config: Dict[str, Any], logger: Logger, user_provi user_provided_options, hide_input=not show_secrets) +def _configure_data_feed(brokerage: CloudBrokerage, logger: Logger) -> None: + """Configures the data feed to use based on the brokerage given. + + :param brokerage: the cloud brokerage + :param logger: the logger to use + """ + if len(cloud_brokerage_data_feeds[brokerage]) != 0: + data_feed_selected = logger.prompt_list("Select a data feed", [ + Option(id=data_feed, label=data_feed) for data_feed in cloud_brokerage_data_feeds[brokerage] + ], multiple=False) + data_feed_property_name = [name for name in brokerage.get_required_properties([InternalInputUserInput]) if ("data-feed" in name)] + data_feed_property_name = data_feed_property_name[0] if len(data_feed_property_name) != 0 else "" + brokerage.update_value_for_given_config(data_feed_property_name, data_feed_selected) + def _configure_live_node(logger: Logger, api_client: APIClient, cloud_project: QCProject) -> QCNode: """Interactively configures the live node to use. @@ -251,7 +265,7 @@ def deploy(project: str, ensure_options(essential_properties) essential_properties_value = {brokerage_instance.convert_variable_to_lean_key(prop) : kwargs[prop] for prop in essential_properties} brokerage_instance.update_configs(essential_properties_value) - # now required properties can be fetched as per data provider from esssential properties + # now required properties can be fetched as per data provider from essential properties required_properties = [brokerage_instance.convert_lean_key_to_variable(prop) for prop in brokerage_instance.get_required_properties([InternalInputUserInput])] ensure_options(required_properties) required_properties_value = {brokerage_instance.convert_variable_to_lean_key(prop) : kwargs[prop] for prop in required_properties} @@ -308,6 +322,7 @@ def deploy(project: str, else: lean_config = container.lean_config_manager.get_lean_config() brokerage_instance = _configure_brokerage(lean_config, logger, kwargs, show_secrets=show_secrets) + _configure_data_feed(brokerage_instance, logger) live_node = _configure_live_node(logger, api_client, cloud_project) notify_order_events, notify_insights, notify_methods = _configure_notifications(logger) auto_restart = _configure_auto_restart(logger) diff --git a/lean/models/__init__.py b/lean/models/__init__.py index c824d28a..2eba3c18 100644 --- a/lean/models/__init__.py +++ b/lean/models/__init__.py @@ -17,7 +17,7 @@ from time import time json_modules = {} -file_name = "modules-1.11.json" +file_name = "modules-1.12.json" directory = Path(__file__).parent file_path = directory.parent / file_name diff --git a/lean/models/brokerages/cloud/__init__.py b/lean/models/brokerages/cloud/__init__.py index b1e641de..72867060 100644 --- a/lean/models/brokerages/cloud/__init__.py +++ b/lean/models/brokerages/cloud/__init__.py @@ -13,13 +13,28 @@ from lean.models.brokerages.cloud.cloud_brokerage import CloudBrokerage from lean.models import json_modules -from typing import List +from typing import Dict, Type, List +from lean.models.brokerages.local.data_feed import DataFeed all_cloud_brokerages: List[CloudBrokerage] = [] +all_cloud_data_feeds: List[DataFeed] = [] +cloud_brokerage_data_feeds: Dict[Type[CloudBrokerage], + List[Type[DataFeed]]] = {} for json_module in json_modules: if "cloud-brokerage" in json_module["type"]: all_cloud_brokerages.append(CloudBrokerage(json_module)) + if "data-queue-handler" in json_module["type"]: + all_cloud_data_feeds.append((DataFeed(json_module))) + +for cloud_brokerage in all_cloud_brokerages: + data_feed_property_found = False + for x in cloud_brokerage.get_all_input_configs(): + if "data-feed" in x.__getattribute__("_id"): + data_feed_property_found = True + cloud_brokerage_data_feeds[cloud_brokerage] = x.__getattribute__("_choices") + if not data_feed_property_found: + cloud_brokerage_data_feeds[cloud_brokerage] = [] [PaperTradingBrokerage] = [ cloud_brokerage for cloud_brokerage in all_cloud_brokerages if cloud_brokerage._id == "QuantConnectBrokerage"] diff --git a/lean/models/brokerages/cloud/cloud_brokerage.py b/lean/models/brokerages/cloud/cloud_brokerage.py index e306238b..fba89d04 100644 --- a/lean/models/brokerages/cloud/cloud_brokerage.py +++ b/lean/models/brokerages/cloud/cloud_brokerage.py @@ -79,6 +79,12 @@ def get_price_data_handler(self) -> str: :return: the value to assign to the "dataHandler" property of the live/create API endpoint """ # TODO: Handle this case with json conditions - if self.get_name() == "Interactive Brokers": - return "InteractiveBrokersHandler" if self.get_config_value_from_name("ib-data-feed") else "QuantConnectHandler" + property_name = [name for name in self.get_required_properties([InternalInputUserInput]) if ("data-feed" in name)] + property_name = property_name[0] if len(property_name) != 0 else "" + brokerage_name = self.get_name().replace(" ", "") + if property_name != "": + if "QuantConnect +" in self.get_config_value_from_name(property_name): + return "quantconnecthandler+" + brokerage_name.lower() + "handler" + else: + return self.get_config_value_from_name(property_name).replace(" ", "") + "Handler" return "QuantConnectHandler" diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index da87f486..f53cc574 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -94,6 +94,49 @@ def test_cloud_live_deploy() -> None: mock.ANY, mock.ANY) +def test_cloud_live_deploy_with_ib_using_hybrid_datafeed() -> None: + create_fake_lean_cli_directory() + + api_client = mock.Mock() + api_client.nodes.get_all.return_value = create_qc_nodes() + api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} + container.api_client = api_client + + cloud_project_manager = mock.Mock() + container.cloud_project_manager = cloud_project_manager + + cloud_runner = mock.Mock() + container.cloud_runner = cloud_runner + + result = CliRunner().invoke(lean, ["cloud", "live", "Python Project", "--brokerage", "Interactive Brokers", "--node", "live", + "--auto-restart", "yes", "--notify-order-events", "no", "--notify-insights", "no", + "--ib-data-feed", "QuantConnect + InteractiveBrokers", "--ib-user-name", "test_user", + "--ib-account", "DU2366417", "--ib-password", "test_password"]) + + assert result.exit_code == 0 + assert "Data provider: quantconnecthandler+interactivebrokershandler" in result.output.split("\n") + +def test_cloud_live_deploy_with_tradier_using_tradier_datafeed() -> None: + create_fake_lean_cli_directory() + + api_client = mock.Mock() + api_client.nodes.get_all.return_value = create_qc_nodes() + container.api_client = api_client + + cloud_project_manager = mock.Mock() + container.cloud_project_manager = cloud_project_manager + + cloud_runner = mock.Mock() + container.cloud_runner = cloud_runner + + result = CliRunner().invoke(lean, ["cloud", "live", "Python Project", "--brokerage", "Tradier", "--node", "live", + "--auto-restart", "yes", "--notify-order-events", "no", "--notify-insights", "no", + "--tradier-data-feed", "Tradier Brokerage", "--tradier-account-id", "123", + "--tradier-access-token", "456", "--tradier-environment", "paper"]) + + assert result.exit_code == 0 + assert "Data provider: TradierBrokerage" in result.output.split("\n") + @pytest.mark.parametrize("notice_method,configs", [("emails", "customAddress:customSubject"), ("emails", "customAddress1:customSubject1,customAddress2:customSubject2"), ("webhooks", "customAddress:header1=value1"), @@ -287,7 +330,9 @@ def test_cloud_live_deploy_with_live_holdings(brokerage: str, holdings: str) -> if brokerage == "Trading Technologies": options.extend(["--live-cash-balance", "USD:100"]) elif brokerage == "Interactive Brokers": - options.extend(["--ib-data-feed", "no"]) + options.extend(["--ib-data-feed", "QuantConnect"]) + elif brokerage == "Tradier": + options.extend(["--tradier-data-feed", "QuantConnect"]) result = CliRunner().invoke(lean, ["cloud", "live", "Python Project", "--brokerage", brokerage, "--live-holdings", holdings, "--node", "live", "--auto-restart", "yes", "--notify-order-events", "no",