From 2e5f1b6bbd2512fc3361e899a0889c52343833bf Mon Sep 17 00:00:00 2001 From: Anshuman Goel Date: Thu, 15 Oct 2020 10:47:45 -0700 Subject: [PATCH 1/8] modifying reset --- src/cli/onefuzz/api.py | 176 +++++++++++++++++++++++++++++++---------- 1 file changed, 135 insertions(+), 41 deletions(-) diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index f3f0f3e289..7a78662899 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -1290,70 +1290,164 @@ def config( return data - def reset(self, everything: bool = False) -> None: - """ - Resets onefuzz. Stops all jobs, notifications, and repro VMs. - Specifying 'everything' will delete all containers, pools, and managed - scalesets. - """ - message = ["Confirm stopping *all* running jobs"] - if everything: - message += ["AND deleting *ALL* fuzzing related data"] - message += ["(specify y or n): "] - + def _user_confirmation(self, message: str) -> bool: answer: Optional[str] = None while answer not in ["y", "n"]: answer = input(" ".join(message)).strip() if answer == "n": self.logger.info("not stopping") - return - - self.logger.info("stopping all interactions") + return False + return True + def _delete_jobs(self): for job in self.jobs.list(): self.logger.info("stopping job %s", job.job_id) self.jobs.delete(job.job_id) + def _delete_tasks(self): for task in self.tasks.list(): self.logger.info("stopping task %s", task.task_id) self.tasks.delete(task.task_id) + def _delete_notifications(self): for notification in self.notifications.list(): self.logger.info("stopping notification %s", notification.notification_id) self.notifications.delete(notification.notification_id) + def _delete_repros(self): for vm in self.repro.list(): self.repro.delete(str(vm.vm_id)) + def _delete_pools(self): + for pool in self.pools.list(): + self.logger.info("stopping pool: %s", pool.name) + self.pools.shutdown(pool.name, now=True) + + def _delete_scalesets(self): + for scaleset in self.scalesets.list(): + self.logger.info("stopping scaleset: %s", scaleset.scaleset_id) + self.scalesets.shutdown(scaleset.scaleset_id, now=True) + + def _delete_containers(self): + for container in self.containers.list(): + if ( + container.metadata + and "container_type" in container.metadata + and container.metadata["container_type"] + in [ + "analysis", + "coverage", + "crashes", + "inputs", + "no_repro", + "readonly_inputs", + "reports", + "setup", + "unique_reports", + ] + ): + self.logger.info("removing container: %s", container.name) + self.containers.delete(container.name) + + def _delete_all_running_jobs(self): + self._delete_jobs() + self._delete_tasks() + self._delete_notifications() + self._delete_repros() + + def _delete_fuzzing_data(self): + self._delete_scalesets() + self._delete_pools() + self._delete_containers() + + def reset( + self, + *, + containers: bool = False, + everything: bool = False, + jobs: bool = False, + notifications: bool = False, + pools: bool = False, + repros: bool = False, + scalesets: bool = False, + tasks: bool = False, + yes: bool = False, + ) -> None: + """ + Resets onefuzz. Stops all jobs, notifications, and repro VMs. + Specifying 'everything' will delete all containers, pools, and managed + scalesets. + + :param bool containers: Delete all the containers. + :param bool everything: Delete all containers, pools and managed scalesets. + :param bool jobs: Delete all jobs. + :param bool notifications: Delete all notifications. + :param bool pools: Delete all pools. + :param bool repros: Delete all repro vms. + :param bool scalesets: Delete all managed scalesets. + :param bool tasks: Delete all tasks. + :param bool yes: Ignoring to specify "y" in prompt. + """ + message = ["Confirm stopping *all* running jobs"] if everything: - for pool in self.pools.list(): - self.logger.info("stopping pool: %s", pool.name) - self.pools.shutdown(pool.name, now=True) - - for scaleset in self.scalesets.list(): - self.logger.info("stopping scaleset: %s", scaleset.scaleset_id) - self.scalesets.shutdown(scaleset.scaleset_id, now=True) - - for container in self.containers.list(): - if ( - container.metadata - and "container_type" in container.metadata - and container.metadata["container_type"] - in [ - "analysis", - "coverage", - "crashes", - "inputs", - "no_repro", - "readonly_inputs", - "reports", - "setup", - "unique_reports", - ] - ): - self.logger.info("removing container: %s", container.name) - self.containers.delete(container.name) + message += ["AND deleting *ALL* fuzzing related data"] + message += ["(specify y or n): "] + + delete_options = [ + jobs, + tasks, + notifications, + pools, + scalesets, + repros, + containers, + ] + arguments = [everything] + arguments.extend(delete_options) + if not any(arguments): + if not yes and not self._user_confirmation(message): + return + self._delete_all_running_jobs() + return + + if everything: + if not yes and not self._user_confirmation(message): + return + self._delete_all_running_jobs() + self._delete_fuzzing_data() + return + + to_delete = [] + for k, _ in locals().items(): + if k in delete_options: + to_delete.append(k) + print(to_delete) + message = ["Confirm stopping %s running jobs" % (", ".join(to_delete))] + message += ["(specify y or n): "] + if not yes and not self._user_confirmation(message): + return + + if jobs: + self._delete_jobs() + + if tasks: + self._delete_tasks() + + if notifications: + self._delete_notifications() + + if repros: + self._delete_repros() + + if pools: + self._delete_pools() + + if scalesets: + self._delete_scalesets() + + if containers: + self._delete_containers() from .debug import Debug # noqa: E402 From c5c35e3cdbfedf5bbe2c7e39086161596ce45aac Mon Sep 17 00:00:00 2001 From: Anshuman Goel Date: Thu, 15 Oct 2020 11:06:47 -0700 Subject: [PATCH 2/8] pipeline fixes --- src/cli/onefuzz/api.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index 7a78662899..4010a83172 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -1290,7 +1290,7 @@ def config( return data - def _user_confirmation(self, message: str) -> bool: + def _user_confirmation(self, message: List[str]) -> bool: answer: Optional[str] = None while answer not in ["y", "n"]: answer = input(" ".join(message)).strip() @@ -1422,7 +1422,6 @@ def reset( for k, _ in locals().items(): if k in delete_options: to_delete.append(k) - print(to_delete) message = ["Confirm stopping %s running jobs" % (", ".join(to_delete))] message += ["(specify y or n): "] if not yes and not self._user_confirmation(message): From 90f1bca0f3469d09e26e12e4ec9911b2c2c69d58 Mon Sep 17 00:00:00 2001 From: Anshuman Goel Date: Thu, 15 Oct 2020 11:13:09 -0700 Subject: [PATCH 3/8] lint fixes --- src/cli/onefuzz/api.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index 4010a83172..86b5c3723a 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -1300,36 +1300,36 @@ def _user_confirmation(self, message: List[str]) -> bool: return False return True - def _delete_jobs(self): + def _delete_jobs(self) -> None: for job in self.jobs.list(): self.logger.info("stopping job %s", job.job_id) self.jobs.delete(job.job_id) - def _delete_tasks(self): + def _delete_tasks(self) -> None: for task in self.tasks.list(): self.logger.info("stopping task %s", task.task_id) self.tasks.delete(task.task_id) - def _delete_notifications(self): + def _delete_notifications(self) -> None: for notification in self.notifications.list(): self.logger.info("stopping notification %s", notification.notification_id) self.notifications.delete(notification.notification_id) - def _delete_repros(self): + def _delete_repros(self) -> None: for vm in self.repro.list(): self.repro.delete(str(vm.vm_id)) - def _delete_pools(self): + def _delete_pools(self) -> None: for pool in self.pools.list(): self.logger.info("stopping pool: %s", pool.name) self.pools.shutdown(pool.name, now=True) - def _delete_scalesets(self): + def _delete_scalesets(self) -> None: for scaleset in self.scalesets.list(): self.logger.info("stopping scaleset: %s", scaleset.scaleset_id) self.scalesets.shutdown(scaleset.scaleset_id, now=True) - def _delete_containers(self): + def _delete_containers(self) -> None: for container in self.containers.list(): if ( container.metadata @@ -1350,20 +1350,19 @@ def _delete_containers(self): self.logger.info("removing container: %s", container.name) self.containers.delete(container.name) - def _delete_all_running_jobs(self): + def _delete_all_running_jobs(self) -> None: self._delete_jobs() self._delete_tasks() self._delete_notifications() self._delete_repros() - def _delete_fuzzing_data(self): + def _delete_fuzzing_data(self) -> None: self._delete_scalesets() self._delete_pools() self._delete_containers() def reset( self, - *, containers: bool = False, everything: bool = False, jobs: bool = False, From 43021276ebbe28d48b06869a4894920e6eb86bd2 Mon Sep 17 00:00:00 2001 From: Anshuman Goel Date: Thu, 15 Oct 2020 11:48:17 -0700 Subject: [PATCH 4/8] fixing iteration bug --- src/cli/onefuzz/api.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index 86b5c3723a..fe4318a53c 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -1418,10 +1418,19 @@ def reset( return to_delete = [] - for k, _ in locals().items(): - if k in delete_options: + arguments = { + "jobs", + "tasks", + "notifications", + "pools", + "scalesets", + "repros", + "containers", + } + for k, v in locals().items(): + if k in arguments and v: to_delete.append(k) - message = ["Confirm stopping %s running jobs" % (", ".join(to_delete))] + message = ["Confirm stopping %s " % (", ".join(to_delete))] message += ["(specify y or n): "] if not yes and not self._user_confirmation(message): return From c7c173679153d3b29641b8a8f58550a3b29096b1 Mon Sep 17 00:00:00 2001 From: Anshuman Goel Date: Thu, 15 Oct 2020 11:50:12 -0700 Subject: [PATCH 5/8] lint fix --- src/cli/onefuzz/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index fe4318a53c..7304358f36 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -1418,7 +1418,7 @@ def reset( return to_delete = [] - arguments = { + argument_str = { "jobs", "tasks", "notifications", @@ -1428,7 +1428,7 @@ def reset( "containers", } for k, v in locals().items(): - if k in arguments and v: + if k in argument_str and v: to_delete.append(k) message = ["Confirm stopping %s " % (", ".join(to_delete))] message += ["(specify y or n): "] From 5bdee756708634b7605c709f2a6a89ff3c13e0a4 Mon Sep 17 00:00:00 2001 From: Anshuman Goel Date: Mon, 19 Oct 2020 08:51:14 -0700 Subject: [PATCH 6/8] pr comments --- src/cli/onefuzz/api.py | 144 ++++++++++++++++------------------------- 1 file changed, 55 insertions(+), 89 deletions(-) diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index 902b30b340..a72a961eb6 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -1300,67 +1300,6 @@ def _user_confirmation(self, message: List[str]) -> bool: return False return True - def _delete_jobs(self) -> None: - for job in self.jobs.list(): - self.logger.info("stopping job %s", job.job_id) - self.jobs.delete(job.job_id) - - def _delete_tasks(self) -> None: - for task in self.tasks.list(): - self.logger.info("stopping task %s", task.task_id) - self.tasks.delete(task.task_id) - - def _delete_notifications(self) -> None: - for notification in self.notifications.list(): - self.logger.info("stopping notification %s", notification.notification_id) - self.notifications.delete(notification.notification_id) - - def _delete_repros(self) -> None: - for vm in self.repro.list(): - self.repro.delete(str(vm.vm_id)) - - def _delete_pools(self) -> None: - for pool in self.pools.list(): - self.logger.info("stopping pool: %s", pool.name) - self.pools.shutdown(pool.name, now=True) - - def _delete_scalesets(self) -> None: - for scaleset in self.scalesets.list(): - self.logger.info("stopping scaleset: %s", scaleset.scaleset_id) - self.scalesets.shutdown(scaleset.scaleset_id, now=True) - - def _delete_containers(self) -> None: - for container in self.containers.list(): - if ( - container.metadata - and "container_type" in container.metadata - and container.metadata["container_type"] - in [ - "analysis", - "coverage", - "crashes", - "inputs", - "no_repro", - "readonly_inputs", - "reports", - "setup", - "unique_reports", - ] - ): - self.logger.info("removing container: %s", container.name) - self.containers.delete(container.name) - - def _delete_all_running_jobs(self) -> None: - self._delete_jobs() - self._delete_tasks() - self._delete_notifications() - self._delete_repros() - - def _delete_fuzzing_data(self) -> None: - self._delete_scalesets() - self._delete_pools() - self._delete_containers() - def reset( self, containers: bool = False, @@ -1380,42 +1319,38 @@ def reset( :param bool containers: Delete all the containers. :param bool everything: Delete all containers, pools and managed scalesets. - :param bool jobs: Delete all jobs. - :param bool notifications: Delete all notifications. + :param bool jobs: Stop all jobs. + :param bool notifications: Stop all notifications. :param bool pools: Delete all pools. :param bool repros: Delete all repro vms. :param bool scalesets: Delete all managed scalesets. - :param bool tasks: Delete all tasks. + :param bool tasks: Stop all tasks. :param bool yes: Ignoring to specify "y" in prompt. """ - message = ["Confirm stopping *all* running jobs"] - if everything: - message += ["AND deleting *ALL* fuzzing related data"] - message += ["(specify y or n): "] - delete_options = [ + containers, jobs, - tasks, - notifications, pools, - scalesets, + notifications, repros, - containers, + scalesets, + tasks, ] arguments = [everything] arguments.extend(delete_options) if not any(arguments): - if not yes and not self._user_confirmation(message): - return - self._delete_all_running_jobs() - return + jobs, notifications, repros, tasks = True, True, True, True if everything: - if not yes and not self._user_confirmation(message): - return - self._delete_all_running_jobs() - self._delete_fuzzing_data() - return + containers, jobs, pools, notifications, repros, scalesets, tasks = ( + True, + True, + True, + True, + True, + True, + True, + ) to_delete = [] argument_str = { @@ -1436,25 +1371,56 @@ def reset( return if jobs: - self._delete_jobs() + for job in self.jobs.list(): + self.logger.info("stopping job %s", job.job_id) + self.jobs.delete(job.job_id) if tasks: - self._delete_tasks() + for task in self.tasks.list(): + self.logger.info("stopping task %s", task.task_id) + self.tasks.delete(task.task_id) if notifications: - self._delete_notifications() + for notification in self.notifications.list(): + self.logger.info( + "stopping notification %s", notification.notification_id + ) + self.notifications.delete(notification.notification_id) if repros: - self._delete_repros() + for vm in self.repro.list(): + self.repro.delete(str(vm.vm_id)) if pools: - self._delete_pools() + for pool in self.pools.list(): + self.logger.info("stopping pool: %s", pool.name) + self.pools.shutdown(pool.name, now=True) if scalesets: - self._delete_scalesets() + for scaleset in self.scalesets.list(): + self.logger.info("stopping scaleset: %s", scaleset.scaleset_id) + self.scalesets.shutdown(scaleset.scaleset_id, now=True) if containers: - self._delete_containers() + for container in self.containers.list(): + if ( + container.metadata + and "container_type" in container.metadata + and container.metadata["container_type"] + in [ + "analysis", + "coverage", + "crashes", + "inputs", + "no_repro", + "readonly_inputs", + "reports", + "setup", + "unique_reports", + ] + ): + self.logger.info("removing container: %s", container.name) + self.containers.delete(container.name) from .debug import Debug # noqa: E402 From c2dfeeb10dc24010a148643d6f9fb0c1dbd7f766 Mon Sep 17 00:00:00 2001 From: Anshuman Goel Date: Mon, 19 Oct 2020 08:57:29 -0700 Subject: [PATCH 7/8] avoid deleteing only containers --- src/cli/onefuzz/api.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index a72a961eb6..7a42a3c802 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -1352,6 +1352,10 @@ def reset( True, ) + if containers and not (tasks or jobs): + print("Use --tasks with containers") + return + to_delete = [] argument_str = { "jobs", From 7421071e2b2bedfcae85831c6a6e76ac227f853f Mon Sep 17 00:00:00 2001 From: Anshuman Goel Date: Mon, 19 Oct 2020 09:39:41 -0700 Subject: [PATCH 8/8] too complex fn --- src/cli/onefuzz/api.py | 118 +++++++++++++++++++++++------------------ 1 file changed, 66 insertions(+), 52 deletions(-) diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index 7a42a3c802..ac67eb0bd8 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -1300,6 +1300,68 @@ def _user_confirmation(self, message: List[str]) -> bool: return False return True + def _delete_components( + self, + containers: bool = False, + jobs: bool = False, + notifications: bool = False, + pools: bool = False, + repros: bool = False, + scalesets: bool = False, + tasks: bool = False, + ) -> None: + if jobs: + for job in self.jobs.list(): + self.logger.info("stopping job %s", job.job_id) + self.jobs.delete(job.job_id) + + if tasks: + for task in self.tasks.list(): + self.logger.info("stopping task %s", task.task_id) + self.tasks.delete(task.task_id) + + if notifications: + for notification in self.notifications.list(): + self.logger.info( + "stopping notification %s", notification.notification_id + ) + self.notifications.delete(notification.notification_id) + + if repros: + for vm in self.repro.list(): + self.repro.delete(str(vm.vm_id)) + + if pools: + for pool in self.pools.list(): + self.logger.info("stopping pool: %s", pool.name) + self.pools.shutdown(pool.name, now=True) + + if scalesets: + for scaleset in self.scalesets.list(): + self.logger.info("stopping scaleset: %s", scaleset.scaleset_id) + self.scalesets.shutdown(scaleset.scaleset_id, now=True) + + if containers: + for container in self.containers.list(): + if ( + container.metadata + and "container_type" in container.metadata + and container.metadata["container_type"] + in [ + "analysis", + "coverage", + "crashes", + "inputs", + "no_repro", + "readonly_inputs", + "reports", + "setup", + "unique_reports", + ] + ): + self.logger.info("removing container: %s", container.name) + self.containers.delete(container.name) + def reset( self, containers: bool = False, @@ -1353,7 +1415,7 @@ def reset( ) if containers and not (tasks or jobs): - print("Use --tasks with containers") + print("Use --jobs or --tasks with containers") return to_delete = [] @@ -1374,57 +1436,9 @@ def reset( if not yes and not self._user_confirmation(message): return - if jobs: - for job in self.jobs.list(): - self.logger.info("stopping job %s", job.job_id) - self.jobs.delete(job.job_id) - - if tasks: - for task in self.tasks.list(): - self.logger.info("stopping task %s", task.task_id) - self.tasks.delete(task.task_id) - - if notifications: - for notification in self.notifications.list(): - self.logger.info( - "stopping notification %s", notification.notification_id - ) - self.notifications.delete(notification.notification_id) - - if repros: - for vm in self.repro.list(): - self.repro.delete(str(vm.vm_id)) - - if pools: - for pool in self.pools.list(): - self.logger.info("stopping pool: %s", pool.name) - self.pools.shutdown(pool.name, now=True) - - if scalesets: - for scaleset in self.scalesets.list(): - self.logger.info("stopping scaleset: %s", scaleset.scaleset_id) - self.scalesets.shutdown(scaleset.scaleset_id, now=True) - - if containers: - for container in self.containers.list(): - if ( - container.metadata - and "container_type" in container.metadata - and container.metadata["container_type"] - in [ - "analysis", - "coverage", - "crashes", - "inputs", - "no_repro", - "readonly_inputs", - "reports", - "setup", - "unique_reports", - ] - ): - self.logger.info("removing container: %s", container.name) - self.containers.delete(container.name) + self._delete_components( + containers, jobs, pools, notifications, repros, scalesets, tasks + ) from .debug import Debug # noqa: E402