From c83d17007b751fcfb9460bd5204cfd63ae3aa2b0 Mon Sep 17 00:00:00 2001 From: AetherBreeze <26798666+AetherBreeze@users.noreply.github.com> Date: Thu, 13 Jan 2022 04:41:14 -0500 Subject: [PATCH] Add several flags to task_create CLI (#4119) * Add serveral flags to CLI & generalize CLI code definition.py: - Add use_zip_chunks, start_frame, stop_frame, and chunk_size as CLI flags - Rename --bug to --bug_tracker to be consistent with website and API - No longer include optional kwargs in all requests core.py: - Add a single loop to handle all kwargs for extensibility - Lump optional task_create args into kwargs for brevity * Add --bug as alias for --bug_tracker - Add back old --bug flag for backwards compatability * Update licence header Update licence header for core.py * Remove redundant parameter assignment - Remove hardcoded `image_quality` default value, as this is already handled by the argparse default value. * CHANGELOG entry for new CLI flags * CLI linting fixes - Several linting fixes for CLI * Fix CLI test - Re-add default image_quality in case task_create is called manually * Resolve changelog merge conflict * Update CHANGELOG.md Co-authored-by: Andrey Zhavoronkov --- CHANGELOG.md | 1 + utils/cli/core/core.py | 33 +++++----- utils/cli/core/definition.py | 116 +++++++++++++++++++++-------------- utils/cli/tests/test_cli.py | 1 - 4 files changed, 85 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3a2c9723db..750298858f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Support for uploading manifest with any name () - Added information about OpenVINO toolkit to login page () - Support for working with ellipses () +- Add several flags to task creation CLI () ### Changed - Users don't have access to a task object anymore if they are assigneed only on some jobs of the task () diff --git a/utils/cli/core/core.py b/utils/cli/core/core.py index ebfc36e3b6f..8d724a221e8 100644 --- a/utils/cli/core/core.py +++ b/utils/cli/core/core.py @@ -34,19 +34,15 @@ def tasks_data(self, task_id, resource_type, resources, **kwargs): data = {'remote_files[{}]'.format(i): f for i, f in enumerate(resources)} elif resource_type == ResourceType.SHARE: data = {'server_files[{}]'.format(i): f for i, f in enumerate(resources)} - data['image_quality'] = 50 + data['image_quality'] = 70 ## capture additional kwargs - if 'image_quality' in kwargs: - data['image_quality'] = kwargs.get('image_quality') - if 'frame_step' in kwargs: + for flag in ['chunk_size', 'copy_data', 'image_quality', 'sorting_method', + 'start_frame', 'stop_frame', 'use_cache', 'use_zip_chunks']: + if kwargs.get(flag) is not None: + data[flag] = kwargs.get(flag) + if kwargs.get('frame_step') is not None: data['frame_filter'] = f"step={kwargs.get('frame_step')}" - if 'copy_data' in kwargs: - data['copy_data'] = kwargs.get('copy_data') - if 'use_cache' in kwargs: - data['use_cache'] = kwargs.get('use_cache') - if 'sorting_method' in kwargs: - data['sorting_method'] = kwargs.get('sorting_method') response = self.session.post(url, data=data, files=files) response.raise_for_status() @@ -76,25 +72,24 @@ def tasks_list(self, use_json_output, **kwargs): response.raise_for_status() return output - def tasks_create(self, name, labels, overlap, segment_size, bug, resource_type, resources, + def tasks_create(self, name, labels, resource_type, resources, annotation_path='', annotation_format='CVAT XML 1.1', completion_verification_period=20, git_completion_verification_period=2, dataset_repository_url='', - project_id=None, lfs=False, **kwargs): """ Create a new task with the given name and labels JSON and add the files to it. """ url = self.api.tasks - labels = [] if project_id is not None else labels + labels = [] if kwargs.get('project_id') is not None else labels data = {'name': name, - 'labels': labels, - 'overlap': overlap, - 'segment_size': segment_size, - 'bug_tracker': bug, + 'labels': labels } - if project_id: - data.update({'project_id': project_id}) + + for flag in ['bug_tracker', 'overlap', 'project_id', 'segment_size']: + if kwargs.get(flag) is not None: + data[flag] = kwargs.get(flag) + response = self.session.post(url, json=data) response.raise_for_status() response_json = response.json() diff --git a/utils/cli/core/definition.py b/utils/cli/core/definition.py index e2fc937aa9e..b75c19fc9ca 100644 --- a/utils/cli/core/definition.py +++ b/utils/cli/core/definition.py @@ -1,3 +1,5 @@ +# Copyright (C) 2021 Intel Corporation +# # SPDX-License-Identifier: MIT import argparse import getpass @@ -106,36 +108,6 @@ def argparse(s): type=str, help='name of the task' ) -task_create_parser.add_argument( - '--labels', - default='[]', - type=parse_label_arg, - help='string or file containing JSON labels specification' -) -task_create_parser.add_argument( - '--project_id', - default=None, - type=int, - help='project ID if project exists' -) -task_create_parser.add_argument( - '--overlap', - default=0, - type=int, - help='the number of intersected frames between different segments' -) -task_create_parser.add_argument( - '--segment_size', - default=0, - type=int, - help='the number of frames in a segment' -) -task_create_parser.add_argument( - '--bug', - default='', - type=str, - help='bug tracker URL' -) task_create_parser.add_argument( 'resource_type', default='local', @@ -161,6 +133,18 @@ def argparse(s): type=str, help='format of the annotation file being uploaded, e.g. CVAT 1.1' ) +task_create_parser.add_argument( + '--bug_tracker', '--bug', + default=None, + type=str, + help='bug tracker URL' +) +task_create_parser.add_argument( + '--chunk_size', + default=None, + type=int, + help='the number of frames per chunk' +) task_create_parser.add_argument( '--completion_verification_period', default=20, @@ -168,6 +152,13 @@ def argparse(s): help='''number of seconds to wait until checking if data compression finished (necessary before uploading annotations)''' ) +task_create_parser.add_argument( + '--copy_data', + default=False, + action='store_true', + help='''set the option to copy the data, only used when resource type is + share (default: %(default)s)''' +) task_create_parser.add_argument( '--dataset_repository_url', default='', @@ -176,10 +167,11 @@ def argparse(s): ' https://github.com/user/repos [annotation/]') ) task_create_parser.add_argument( - '--lfs', - default=False, - action='store_true', - help='using lfs for dataset repository (default: %(default)s)' + '--frame_step', + default=None, + type=int, + help='''set the frame step option in the advanced configuration + when uploading image series or videos (default: %(default)s)''' ) task_create_parser.add_argument( '--image_quality', @@ -189,24 +181,34 @@ def argparse(s): when creating tasks.(default: %(default)s)''' ) task_create_parser.add_argument( - '--frame_step', - default=1, - type=int, - help='''set the frame step option in the advanced configuration - when uploading image series or videos (default: %(default)s)''' + '--labels', + default='[]', + type=parse_label_arg, + help='string or file containing JSON labels specification' ) task_create_parser.add_argument( - '--copy_data', + '--lfs', default=False, action='store_true', - help='''set the option to copy the data, only used when resource type is - share (default: %(default)s)''' + help='using lfs for dataset repository (default: %(default)s)' ) task_create_parser.add_argument( - '--use_cache', - default=True, - action='store_false', - help='''set the option to use the cache (default: %(default)s)''' + '--project_id', + default=None, + type=int, + help='project ID if project exists' +) +task_create_parser.add_argument( + '--overlap', + default=None, + type=int, + help='the number of intersected frames between different segments' +) +task_create_parser.add_argument( + '--segment_size', + default=None, + type=int, + help='the number of frames in a segment' ) task_create_parser.add_argument( '--sorting-method', @@ -214,6 +216,28 @@ def argparse(s): choices=['lexicographical', 'natural', 'predefined', 'random'], help='''data soring method (default: %(default)s)''' ) +task_create_parser.add_argument( + '--start_frame', + default=None, + type=int, + help='the start frame of the video' +) +task_create_parser.add_argument( + '--stop_frame', + default=None, + type=int, + help='the stop frame of the video' +) +task_create_parser.add_argument( + '--use_cache', + action='store_true', # automatically sets default=False + help='''use cache''' +) +task_create_parser.add_argument( + '--use_zip_chunks', + action='store_true', # automatically sets default=False + help='''zip chunks before sending them to the server''' +) ####################################################################### # Delete diff --git a/utils/cli/tests/test_cli.py b/utils/cli/tests/test_cli.py index 0e86bd65eba..1f1ca330971 100644 --- a/utils/cli/tests/test_cli.py +++ b/utils/cli/tests/test_cli.py @@ -27,7 +27,6 @@ def setUp(self, mock_stdout): self.taskname = 'test_task' self.cli.tasks_create(self.taskname, [{'name' : 'car'}, {'name': 'person'}], - 0, 0, '', ResourceType.LOCAL, [self.img_file]) # redirect logging to mocked stdout to test program output