From c5aa9d33eeb6ecf4465809389bae96cb7516aed0 Mon Sep 17 00:00:00 2001 From: Brett Simons Date: Fri, 4 Jan 2019 15:37:25 -0800 Subject: [PATCH 1/3] Fixed Client.cluster.list_applications function to return [aztk.models.Task] since it was previously throwing errors. --- .../client/base/helpers/list_applications.py | 21 +++++++------------ aztk/spark/client/base/operations.py | 4 ++-- aztk/spark/client/cluster/operations.py | 11 ++++++++++ 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/aztk/spark/client/base/helpers/list_applications.py b/aztk/spark/client/base/helpers/list_applications.py index 8945a455..9337ef3c 100644 --- a/aztk/spark/client/base/helpers/list_applications.py +++ b/aztk/spark/client/base/helpers/list_applications.py @@ -2,22 +2,17 @@ from azure.batch.models import BatchErrorException from aztk import error -from aztk.spark import models +from aztk.spark.models import SchedulingTarget from aztk.utils import helpers -def _list_applications(core_operations, id): - # info about the app - scheduling_target = core_operations.get_cluster_configuration(id).scheduling_target - if scheduling_target is not models.SchedulingTarget.Any: - return models.Application(core_operations.list_applications(id)) - - recent_run_job = core_operations.get_recent_job(id) - return core_operations.list_batch_tasks(id=recent_run_job.id) - - -def list_applications(core_operations, id): +def list_applications(core_operations, cluster_id): try: - return models.Application(_list_applications(core_operations, id)) + scheduling_target = core_operations.get_cluster_configuration(cluster_id).scheduling_target + if scheduling_target is not SchedulingTarget.Any: + task = core_operations.list_task_table_entries(cluster_id) + else: + task = core_operations.list_batch_tasks(cluster_id) + return task except BatchErrorException as e: raise error.AztkError(helpers.format_batch_exception(e)) diff --git a/aztk/spark/client/base/operations.py b/aztk/spark/client/base/operations.py index d2d428b1..622834c3 100644 --- a/aztk/spark/client/base/operations.py +++ b/aztk/spark/client/base/operations.py @@ -72,7 +72,7 @@ def _generate_application_task(self, core_base_operations, container_id, applica return generate_application_task.generate_application_task(core_base_operations, container_id, application, remote) - def list_applications(self, id): + def _list_applications(self, core_base_operations, id): """Get information on a submitted application Args: @@ -82,4 +82,4 @@ def list_applications(self, id): Returns: :obj:`aztk.spark.models.Application`: object representing that state and output of an application """ - return list_applications.list_applications(self, id) + return list_applications.list_applications(core_base_operations, id) diff --git a/aztk/spark/client/cluster/operations.py b/aztk/spark/client/cluster/operations.py index 01ad6d95..3066d81f 100644 --- a/aztk/spark/client/cluster/operations.py +++ b/aztk/spark/client/cluster/operations.py @@ -117,6 +117,17 @@ def get_application_state(self, id: str, application_name: str): """ return get_application_state.get_application_state(self._core_cluster_operations, id, application_name) + def list_applications(self, id: str): + """Get all tasks that have been submitted to the cluster + + Args: + id (:obj:`str`): the name of the cluster the tasks belong to + + Returns: + :obj:`[aztk.models.Task]`: list of aztk tasks + """ + return self._list_applications(self._core_cluster_operations, id) + def run(self, id: str, command: str, host=False, internal: bool = False, timeout=None): """Run a bash command on every node in the cluster From ec77fd81c62477c87cfd836bc861f50a716ef4b7 Mon Sep 17 00:00:00 2001 From: Brett Simons Date: Fri, 4 Jan 2019 15:40:44 -0800 Subject: [PATCH 2/3] Updated documentation for the base _list_applications function. --- aztk/spark/client/base/operations.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/aztk/spark/client/base/operations.py b/aztk/spark/client/base/operations.py index 622834c3..2964eed9 100644 --- a/aztk/spark/client/base/operations.py +++ b/aztk/spark/client/base/operations.py @@ -73,13 +73,12 @@ def _generate_application_task(self, core_base_operations, container_id, applica remote) def _list_applications(self, core_base_operations, id): - """Get information on a submitted application + """Get information on tasks submitted to a cluster Args: - id (:obj:`str`): the name of the job the application was submitted to - application_name (:obj:`str`): the name of the application to get + id (:obj:`str`): the name of the cluster the tasks belong to Returns: - :obj:`aztk.spark.models.Application`: object representing that state and output of an application + :obj:`[aztk.models.Task]`: list of aztk tasks """ return list_applications.list_applications(core_base_operations, id) From 337cbbe5b967211962588cf198dcd5dd9213f31e Mon Sep 17 00:00:00 2001 From: Brett Simons Date: Wed, 9 Jan 2019 13:51:52 -0800 Subject: [PATCH 3/3] Updated to return a list of aztk.spark.models.Application instances instead of the raw tasks as would be expected by the name. Also updated documentation to reflect this change. --- aztk/spark/client/base/helpers/list_applications.py | 8 ++++---- aztk/spark/client/base/operations.py | 2 +- aztk/spark/client/cluster/operations.py | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/aztk/spark/client/base/helpers/list_applications.py b/aztk/spark/client/base/helpers/list_applications.py index 9337ef3c..c6af2534 100644 --- a/aztk/spark/client/base/helpers/list_applications.py +++ b/aztk/spark/client/base/helpers/list_applications.py @@ -2,7 +2,7 @@ from azure.batch.models import BatchErrorException from aztk import error -from aztk.spark.models import SchedulingTarget +from aztk.spark.models import Application, SchedulingTarget from aztk.utils import helpers @@ -10,9 +10,9 @@ def list_applications(core_operations, cluster_id): try: scheduling_target = core_operations.get_cluster_configuration(cluster_id).scheduling_target if scheduling_target is not SchedulingTarget.Any: - task = core_operations.list_task_table_entries(cluster_id) + tasks = core_operations.list_task_table_entries(cluster_id) else: - task = core_operations.list_batch_tasks(cluster_id) - return task + tasks = core_operations.list_batch_tasks(cluster_id) + return [Application(task) for task in tasks] except BatchErrorException as e: raise error.AztkError(helpers.format_batch_exception(e)) diff --git a/aztk/spark/client/base/operations.py b/aztk/spark/client/base/operations.py index 2964eed9..0d95016b 100644 --- a/aztk/spark/client/base/operations.py +++ b/aztk/spark/client/base/operations.py @@ -79,6 +79,6 @@ def _list_applications(self, core_base_operations, id): id (:obj:`str`): the name of the cluster the tasks belong to Returns: - :obj:`[aztk.models.Task]`: list of aztk tasks + :obj:`[aztk.spark.models.Application]`: list of aztk applications """ return list_applications.list_applications(core_base_operations, id) diff --git a/aztk/spark/client/cluster/operations.py b/aztk/spark/client/cluster/operations.py index 3066d81f..72dc3620 100644 --- a/aztk/spark/client/cluster/operations.py +++ b/aztk/spark/client/cluster/operations.py @@ -124,7 +124,7 @@ def list_applications(self, id: str): id (:obj:`str`): the name of the cluster the tasks belong to Returns: - :obj:`[aztk.models.Task]`: list of aztk tasks + :obj:`[aztk.spark.models.Application]`: list of aztk applications """ return self._list_applications(self._core_cluster_operations, id)