From 5d0444dd991bcf9aaff3d6c07ed4e3b3ae011dee Mon Sep 17 00:00:00 2001 From: Ronit Jain Date: Wed, 23 Aug 2023 00:04:45 +0530 Subject: [PATCH] add local object store commands --- lean/commands/cloud/__init__.py | 3 +- lean/commands/cloud/object_store/__init__.py | 24 +++++++++ lean/commands/cloud/object_store/delete.py | 29 ++++++++++ lean/commands/cloud/object_store/get.py | 53 +++++++++++++++++++ lean/commands/cloud/object_store/list.py | 43 +++++++++++++++ .../cloud/object_store/object_store.py | 23 ++++++++ lean/commands/cloud/object_store/set.py | 34 ++++++++++++ lean/commands/object_store/delete.py | 13 ++--- lean/commands/object_store/get.py | 34 ++---------- lean/commands/object_store/list.py | 27 ++-------- lean/commands/object_store/object_store.py | 2 +- lean/commands/object_store/set.py | 22 +++----- lean/components/util/object_store_helper.py | 26 +++++++++ 13 files changed, 257 insertions(+), 76 deletions(-) create mode 100644 lean/commands/cloud/object_store/__init__.py create mode 100644 lean/commands/cloud/object_store/delete.py create mode 100644 lean/commands/cloud/object_store/get.py create mode 100644 lean/commands/cloud/object_store/list.py create mode 100644 lean/commands/cloud/object_store/object_store.py create mode 100644 lean/commands/cloud/object_store/set.py create mode 100644 lean/components/util/object_store_helper.py diff --git a/lean/commands/cloud/__init__.py b/lean/commands/cloud/__init__.py index 63caef70..af5532bf 100644 --- a/lean/commands/cloud/__init__.py +++ b/lean/commands/cloud/__init__.py @@ -19,7 +19,7 @@ from lean.commands.cloud.pull import pull from lean.commands.cloud.push import push from lean.commands.cloud.status import status - +from lean.commands.cloud.object_store import object_store @group() def cloud() -> None: @@ -35,3 +35,4 @@ def cloud() -> None: cloud.add_command(optimize) cloud.add_command(live) cloud.add_command(status) +cloud.add_command(object_store) diff --git a/lean/commands/cloud/object_store/__init__.py b/lean/commands/cloud/object_store/__init__.py new file mode 100644 index 00000000..8d4e7297 --- /dev/null +++ b/lean/commands/cloud/object_store/__init__.py @@ -0,0 +1,24 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from lean.commands.cloud.object_store.object_store import object_store +from lean.commands.cloud.object_store.get import get +from lean.commands.cloud.object_store.set import set +from lean.commands.cloud.object_store.list import list +from lean.commands.cloud.object_store.delete import delete + +object_store.add_command(get) +object_store.add_command(set) +object_store.add_command(list) +object_store.add_command(delete) + diff --git a/lean/commands/cloud/object_store/delete.py b/lean/commands/cloud/object_store/delete.py new file mode 100644 index 00000000..4d2633f2 --- /dev/null +++ b/lean/commands/cloud/object_store/delete.py @@ -0,0 +1,29 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from click import command, argument + +from lean.click import LeanCommand +from lean.container import container + + +@command(cls=LeanCommand) +@argument("key", type=str) +def delete(key: str) -> str: + """ + Delete a value from the organization's cloud object store. + + """ + organization_id = container.organization_manager.try_get_working_organization_id() + api_client = container.api_client + api_client.object_store.delete(key, organization_id) \ No newline at end of file diff --git a/lean/commands/cloud/object_store/get.py b/lean/commands/cloud/object_store/get.py new file mode 100644 index 00000000..8cc17981 --- /dev/null +++ b/lean/commands/cloud/object_store/get.py @@ -0,0 +1,53 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from click import argument +from lean.commands.cloud.object_store import object_store +from lean.click import LeanCommand +from lean.container import container + + +@object_store.command(cls=LeanCommand, name="get", aliases=["ls"]) +@argument("key", type=str) +def get(key: str) -> str: + """ + Get a value from the organization's cloud object store. + + """ + organization_id = container.organization_manager.try_get_working_organization_id() + api_client = container.api_client + logger = container.logger + data = api_client.object_store.get(key, organization_id) + + try: + headers = ["size", "modified", "preview", "key"] + display_headers = ["Bytes", "Modified", "Preview", "Filename"] + row = [str(data["metadata"].get(header, "")) for header in headers] + all_rows = [display_headers] + [row] + column_widths = [max(len(row[i]) for row in all_rows) for i in range(len(all_rows[0]))] + + logger.info(" ".join(header.ljust(width) for header, width in zip(display_headers, column_widths))) + + bytes = str(data["metadata"].get('size', "")) + modified = data["metadata"].get('modified', "") + preview = data["metadata"].get('preview', "")[:10].rstrip() + filename = data["metadata"].get('key', "") + + values = [bytes, modified, preview, filename] + logger.info(" ".join(value.ljust(width) for value, width in zip(values, column_widths))) + except KeyError as e: + logger.error(f"Key {key} not found.") + except Exception as e: + logger.error(f"Error: {e}") + + diff --git a/lean/commands/cloud/object_store/list.py b/lean/commands/cloud/object_store/list.py new file mode 100644 index 00000000..459ad36d --- /dev/null +++ b/lean/commands/cloud/object_store/list.py @@ -0,0 +1,43 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from click import command, argument + +from lean.click import LeanCommand +from lean.container import container + + +@command(cls=LeanCommand) +@argument("root-key", type=str) +def list(root_key: str) -> str: + """ + List all values for the given root key in the organization's cloud object store. + + """ + organization_id = container.organization_manager.try_get_working_organization_id() + api_client = container.api_client + logger = container.logger + data = api_client.object_store.list(root_key, organization_id) + + try: + headers = ["key", "size", "folder", "name"] + display_headers = ["Key", "Bytes", "Folder", "Filename"] + rows = [[str(obj.get(header, "")) for header in headers] for obj in data['objects']] + all_rows = [display_headers] + rows + column_widths = [max(len(row[i]) for row in all_rows) for i in range(len(all_rows[0]))] + for row in all_rows: + logger.info(" ".join(value.ljust(width) for value, width in zip(row, column_widths))) + except KeyError as e: + logger.error(f"Key {root_key} not found.") + except Exception as e: + logger.error(f"Error: {e}") \ No newline at end of file diff --git a/lean/commands/cloud/object_store/object_store.py b/lean/commands/cloud/object_store/object_store.py new file mode 100644 index 00000000..e0d08877 --- /dev/null +++ b/lean/commands/cloud/object_store/object_store.py @@ -0,0 +1,23 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from click import group +from lean.components.util.click_aliased_command_group import AliasedCommandGroup + +@group(cls=AliasedCommandGroup, invoke_without_command=True) +def object_store() -> None: + """Interact with the Organization's Cloud Object Store.""" + # This method is intentionally empty + # It is used as the command group for all `lean object-store ` commands + pass + diff --git a/lean/commands/cloud/object_store/set.py b/lean/commands/cloud/object_store/set.py new file mode 100644 index 00000000..553f2ce1 --- /dev/null +++ b/lean/commands/cloud/object_store/set.py @@ -0,0 +1,34 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from click import command, argument +from pathlib import Path +from lean.click import LeanCommand, PathParameter +from lean.container import container + + +@command(cls=LeanCommand) +@argument("key", type=str) +@argument("path", type= PathParameter(exists=True, file_okay=True, dir_okay=False)) +def set(key: str, path: Path) -> None: + """Sets the data to the given key in the organization's cloud object store. + + :param key: The key to set the data to. + :param path: Path to the file containing the object data. + """ + organization_id = container.organization_manager.try_get_working_organization_id() + container.logger.info(f"Setting object {key} in organization {organization_id}") + api_client = container.api_client + with open(path, 'r', encoding='utf-8') as f: + objectData = f.read() + api_client.object_store.set(key, objectData, organization_id) \ No newline at end of file diff --git a/lean/commands/object_store/delete.py b/lean/commands/object_store/delete.py index 3c6cf8c8..77d0b56f 100644 --- a/lean/commands/object_store/delete.py +++ b/lean/commands/object_store/delete.py @@ -11,19 +11,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -from click import command, argument - +from click import command from lean.click import LeanCommand from lean.container import container +from lean.components.util.object_store_helper import open_storage_directory_in_explorer @command(cls=LeanCommand) -@argument("key", type=str) -def delete(key: str) -> str: +def delete() -> str: """ - Delete a value from the organization's object store. + Opens the local storage directory in the file explorer. """ - organization_id = container.organization_manager.try_get_working_organization_id() - api_client = container.api_client - api_client.object_store.delete(key, organization_id) \ No newline at end of file + open_storage_directory_in_explorer(container.lean_config_manager) \ No newline at end of file diff --git a/lean/commands/object_store/get.py b/lean/commands/object_store/get.py index 5a1fce58..dd8dbc9b 100644 --- a/lean/commands/object_store/get.py +++ b/lean/commands/object_store/get.py @@ -11,43 +11,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -from click import argument -from lean.commands.object_store import object_store from lean.click import LeanCommand from lean.container import container +from lean.components.util.object_store_helper import open_storage_directory_in_explorer +from lean.commands.object_store import object_store @object_store.command(cls=LeanCommand, name="get", aliases=["ls"]) -@argument("key", type=str) -def get(key: str) -> str: +def get() -> str: """ - Get a value from the organization's object store. + Opens the local storage directory in the file explorer. """ - organization_id = container.organization_manager.try_get_working_organization_id() - api_client = container.api_client - logger = container.logger - data = api_client.object_store.get(key, organization_id) - - try: - headers = ["size", "modified", "preview", "key"] - display_headers = ["Bytes", "Modified", "Preview", "Filename"] - row = [str(data["metadata"].get(header, "")) for header in headers] - all_rows = [display_headers] + [row] - column_widths = [max(len(row[i]) for row in all_rows) for i in range(len(all_rows[0]))] - - logger.info(" ".join(header.ljust(width) for header, width in zip(display_headers, column_widths))) + open_storage_directory_in_explorer(container.lean_config_manager) - bytes = str(data["metadata"].get('size', "")) - modified = data["metadata"].get('modified', "") - preview = data["metadata"].get('preview', "")[:10].rstrip() - filename = data["metadata"].get('key', "") - - values = [bytes, modified, preview, filename] - logger.info(" ".join(value.ljust(width) for value, width in zip(values, column_widths))) - except KeyError as e: - logger.error(f"Key {key} not found.") - except Exception as e: - logger.error(f"Error: {e}") diff --git a/lean/commands/object_store/list.py b/lean/commands/object_store/list.py index 17f4b0e4..b15aa4a8 100644 --- a/lean/commands/object_store/list.py +++ b/lean/commands/object_store/list.py @@ -11,33 +11,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -from click import command, argument - +from click import command from lean.click import LeanCommand from lean.container import container +from lean.components.util.object_store_helper import open_storage_directory_in_explorer @command(cls=LeanCommand) -@argument("root-key", type=str) -def list(root_key: str) -> str: +def list() -> str: """ - List all values for the given root key in the organization's object store. + Opens the local storage directory in the file explorer. """ - organization_id = container.organization_manager.try_get_working_organization_id() - api_client = container.api_client - logger = container.logger - data = api_client.object_store.list(root_key, organization_id) - - try: - headers = ["key", "size", "folder", "name"] - display_headers = ["Key", "Bytes", "Folder", "Filename"] - rows = [[str(obj.get(header, "")) for header in headers] for obj in data['objects']] - all_rows = [display_headers] + rows - column_widths = [max(len(row[i]) for row in all_rows) for i in range(len(all_rows[0]))] - for row in all_rows: - logger.info(" ".join(value.ljust(width) for value, width in zip(row, column_widths))) - except KeyError as e: - logger.error(f"Key {root_key} not found.") - except Exception as e: - logger.error(f"Error: {e}") \ No newline at end of file + open_storage_directory_in_explorer(container.lean_config_manager) \ No newline at end of file diff --git a/lean/commands/object_store/object_store.py b/lean/commands/object_store/object_store.py index 8c0128ce..cd228874 100644 --- a/lean/commands/object_store/object_store.py +++ b/lean/commands/object_store/object_store.py @@ -16,7 +16,7 @@ @group(cls=AliasedCommandGroup, invoke_without_command=True) def object_store() -> None: - """Interact with the Organization's Object Store.""" + """Interact with the Organization's Local Object Store.""" # This method is intentionally empty # It is used as the command group for all `lean object-store ` commands pass diff --git a/lean/commands/object_store/set.py b/lean/commands/object_store/set.py index 620f69e8..0fd7e2c0 100644 --- a/lean/commands/object_store/set.py +++ b/lean/commands/object_store/set.py @@ -11,24 +11,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -from click import command, argument -from pathlib import Path -from lean.click import LeanCommand, PathParameter +from click import command +from lean.click import LeanCommand from lean.container import container +from lean.components.util.object_store_helper import open_storage_directory_in_explorer @command(cls=LeanCommand) -@argument("key", type=str) -@argument("path", type= PathParameter(exists=True, file_okay=True, dir_okay=False)) -def set(key: str, path: Path) -> None: - """Sets the data to the given key in the organization's object store. +def set() -> None: + """ + Opens the local storage directory in the file explorer. - :param key: The key to set the data to. - :param path: Path to the file containing the object data. """ - organization_id = container.organization_manager.try_get_working_organization_id() - container.logger.info(f"Setting object {key} in organization {organization_id}") - api_client = container.api_client - with open(path, 'r', encoding='utf-8') as f: - objectData = f.read() - api_client.object_store.set(key, objectData, organization_id) \ No newline at end of file + open_storage_directory_in_explorer(container.lean_config_manager) \ No newline at end of file diff --git a/lean/components/util/object_store_helper.py b/lean/components/util/object_store_helper.py new file mode 100644 index 00000000..1375ab57 --- /dev/null +++ b/lean/components/util/object_store_helper.py @@ -0,0 +1,26 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import webbrowser +from lean.components.config.lean_config_manager import LeanConfigManager + +def open_storage_directory_in_explorer(lean_config_manager: LeanConfigManager): + """Opens the storage directory in the file explorer.""" + global_storage_directory_path = lean_config_manager.get_cli_root_directory() / "storage" + if not global_storage_directory_path.exists(): + global_storage_directory_path.mkdir(parents=True) + open_file_explorer(str(global_storage_directory_path)) + +def open_file_explorer(directory_path: str): + """Opens the given directory in the file explorer.""" + webbrowser.open('file:///' + directory_path) \ No newline at end of file