From b5946c698783b4c1713aa99686daf1478f9c320b Mon Sep 17 00:00:00 2001 From: Felipe Almeida Date: Sat, 21 Oct 2017 00:40:59 -0200 Subject: [PATCH 01/16] add --zip flag and support for zipping a submission file prior to uploading it --- .gitignore | 1 + kaggle_cli/submit.py | 47 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index a317fe3..3efc1c3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ dist */__pycache__/ */*.pyc .kaggle-cli +.idea/ diff --git a/kaggle_cli/submit.py b/kaggle_cli/submit.py index 5272441..fe5b6d6 100644 --- a/kaggle_cli/submit.py +++ b/kaggle_cli/submit.py @@ -2,6 +2,9 @@ import time import re import json +import uuid +from argparse import ArgumentTypeError +import zipfile from cliff.command import Command @@ -21,6 +24,8 @@ def get_parser(self, prog_name): parser.add_argument('-c', '--competition', help='competition') parser.add_argument('-u', '--username', help='username') parser.add_argument('-p', '--password', help='password') + parser.add_argument('-z', '--zip', type=self._str2bool, nargs='?', const=True, default=False, + help='zip the submission file before uploading') return parser @@ -30,6 +35,7 @@ def take_action(self, parsed_args): username = config.get('username', '') password = config.get('password', '') competition = config.get('competition', '') + zip = config.get('zip', False) browser = common.login(username, password) base = 'https://www.kaggle.com' @@ -40,6 +46,12 @@ def take_action(self, parsed_args): entry = parsed_args.entry message = parsed_args.message + archive_name = Submit._rand_str(10)+'.zip' + + if zip: + with zipfile.ZipFile(archive_name, 'w', zipfile.ZIP_DEFLATED) as zf: + zf.write(entry) + competition_page = browser.get(competition_url) if competition_page.status_code == 404: @@ -51,18 +63,23 @@ def take_action(self, parsed_args): str(competition_page.soup) ).group(1) + if zip: + target_name = archive_name + else: + target_name = entry + form_submission = browser.post( file_form_submit_url, data={ - 'fileName': entry, - 'contentLength': os.path.getsize(entry), - 'lastModifiedDateUtc': int(os.path.getmtime(entry) * 1000) + 'fileName': target_name, + 'contentLength': os.path.getsize(target_name), + 'lastModifiedDateUtc': int(os.path.getmtime(target_name) * 1000) } ).json() file_submit_url = base + form_submission['createUrl'] - with open(entry, 'rb') as submission_file: + with open(target_name, 'rb') as submission_file: token = browser.post( file_submit_url, files={ @@ -98,3 +115,25 @@ def take_action(self, parsed_args): else: print('something went wrong') break + + if zip: + os.remove(target_name) + + @staticmethod + def _str2bool(v): + """ + parse boolean values + + https://stackoverflow.com/a/43357954/436721 + :return: + """ + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise ArgumentTypeError('Boolean value expected.') + + @staticmethod + def _rand_str(length): + return uuid.uuid4().hex[:length-1] From 7b33aaa5517ab3b3f6c112925bdc969b78cfae2f Mon Sep 17 00:00:00 2001 From: Felipe Almeida Date: Sat, 21 Oct 2017 15:57:54 -0200 Subject: [PATCH 02/16] add support for setting the --zip option in the config --- kaggle_cli/config.py | 11 ++++++++--- kaggle_cli/submit.py | 26 ++++++++++++++++++++------ 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/kaggle_cli/config.py b/kaggle_cli/config.py index d81d491..4ad5d9b 100644 --- a/kaggle_cli/config.py +++ b/kaggle_cli/config.py @@ -5,10 +5,9 @@ from cliff.command import Command - CONFIG_DIR_NAME = '.kaggle-cli' CONFIG_FILE_NAME = 'config' -DATA_OPTIONS = set(['username', 'password', 'competition']) +DATA_OPTIONS = set(['username', 'password', 'competition', 'zip']) def get_config(config_path): @@ -85,6 +84,7 @@ def get_parser(self, prog_name): parser.add_argument('-u', '--username', help='username') parser.add_argument('-p', '--password', help='password') parser.add_argument('-c', '--competition', help='competition') + parser.add_argument('-z', '--zip', help='zip the submission file before uploading?', action='store_true') parser.add_argument( '-g', '--global', @@ -98,7 +98,7 @@ def take_action(self, parsed_args): parsed_arg_dict = vars(parsed_args) if DATA_OPTIONS & set( - filter(lambda x: parsed_arg_dict[x], parsed_arg_dict) + filter(lambda x: parsed_arg_dict[x], parsed_arg_dict) ): if parsed_arg_dict['global']: config_dir = os.path.join( @@ -135,6 +135,11 @@ def take_action(self, parsed_args): parsed_arg_dict['competition'] ) + if parsed_arg_dict['zip']: + config.set('user', 'zip', 'yes') + else: + config.set('user', 'zip', 'no') + with open(config_path, 'w') as config_file: config.write(config_file) else: diff --git a/kaggle_cli/submit.py b/kaggle_cli/submit.py index fe5b6d6..ab2d234 100644 --- a/kaggle_cli/submit.py +++ b/kaggle_cli/submit.py @@ -25,7 +25,7 @@ def get_parser(self, prog_name): parser.add_argument('-u', '--username', help='username') parser.add_argument('-p', '--password', help='password') parser.add_argument('-z', '--zip', type=self._str2bool, nargs='?', const=True, default=False, - help='zip the submission file before uploading') + help='whether to zip the submission file before uploading') return parser @@ -35,7 +35,12 @@ def take_action(self, parsed_args): username = config.get('username', '') password = config.get('password', '') competition = config.get('competition', '') - zip = config.get('zip', False) + zip_flag = config.get('zip', 'no') + + if Submit._str2bool(zip_flag): + zip = True + else: + zip = False browser = common.login(username, password) base = 'https://www.kaggle.com' @@ -46,7 +51,7 @@ def take_action(self, parsed_args): entry = parsed_args.entry message = parsed_args.message - archive_name = Submit._rand_str(10)+'.zip' + archive_name = Submit._rand_str(10) + '.zip' if zip: with zipfile.ZipFile(archive_name, 'w', zipfile.ZIP_DEFLATED) as zf: @@ -122,10 +127,11 @@ def take_action(self, parsed_args): @staticmethod def _str2bool(v): """ - parse boolean values + parse truthy/falsy strings into booleans https://stackoverflow.com/a/43357954/436721 - :return: + :param v: the string to be parsed + :return: a boolean value """ if v.lower() in ('yes', 'true', 't', 'y', '1'): return True @@ -136,4 +142,12 @@ def _str2bool(v): @staticmethod def _rand_str(length): - return uuid.uuid4().hex[:length-1] + """ + this is used to prevent caching issues + + https://stackoverflow.com/a/34017605/436721 + + :param length: integer length + :return: a random string of the given length + """ + return uuid.uuid4().hex[:length - 1] From d3c68a49fa65386a7c0c200dd4abb291f7ee57a3 Mon Sep 17 00:00:00 2001 From: floydwch Date: Mon, 23 Oct 2017 08:11:01 +0800 Subject: [PATCH 03/16] add warning when the quota of submission ran out --- kaggle_cli/meta.py | 2 +- kaggle_cli/submit.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/kaggle_cli/meta.py b/kaggle_cli/meta.py index 58a6533..e64467c 100644 --- a/kaggle_cli/meta.py +++ b/kaggle_cli/meta.py @@ -1 +1 @@ -VERSION = '0.12.8' +VERSION = '0.12.9' diff --git a/kaggle_cli/submit.py b/kaggle_cli/submit.py index 5272441..bc55fd7 100644 --- a/kaggle_cli/submit.py +++ b/kaggle_cli/submit.py @@ -34,8 +34,8 @@ def take_action(self, parsed_args): browser = common.login(username, password) base = 'https://www.kaggle.com' competition_url = '/'.join([base, 'c', competition]) - file_form_submit_url = '/'.join([base, 'blobs/inbox/submissions']) - entry_form_submit_url = '/'.join([competition_url, 'submission.json']) + file_form_url = '/'.join([base, 'blobs/inbox/submissions']) + entry_form_url = '/'.join([competition_url, 'submission.json']) entry = parsed_args.entry message = parsed_args.message @@ -52,7 +52,7 @@ def take_action(self, parsed_args): ).group(1) form_submission = browser.post( - file_form_submit_url, + file_form_url, data={ 'fileName': entry, 'contentLength': os.path.getsize(entry), @@ -70,8 +70,8 @@ def take_action(self, parsed_args): } ).json()['token'] - browser.post( - entry_form_submit_url, + entry_form_resp = browser.post( + entry_form_url, data=json.dumps({ 'blobFileTokens': [token], 'submissionDescription': message if message else '' @@ -81,6 +81,10 @@ def take_action(self, parsed_args): } ) + if entry_form_resp.status_code == 400: + print('You have no more submissions remaining for today.') + return + status_url = ( 'https://www.kaggle.com/' 'c/{}/submissions/status.json' From f0869eef04f798e9b3dedc858c6e1147544b6199 Mon Sep 17 00:00:00 2001 From: floydwch Date: Mon, 23 Oct 2017 08:37:42 +0800 Subject: [PATCH 04/16] use response content to warn the user --- kaggle_cli/meta.py | 2 +- kaggle_cli/submit.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kaggle_cli/meta.py b/kaggle_cli/meta.py index e64467c..46dea4b 100644 --- a/kaggle_cli/meta.py +++ b/kaggle_cli/meta.py @@ -1 +1 @@ -VERSION = '0.12.9' +VERSION = '0.12.10' diff --git a/kaggle_cli/submit.py b/kaggle_cli/submit.py index bc55fd7..c9f5420 100644 --- a/kaggle_cli/submit.py +++ b/kaggle_cli/submit.py @@ -70,7 +70,7 @@ def take_action(self, parsed_args): } ).json()['token'] - entry_form_resp = browser.post( + entry_form_resp_message = browser.post( entry_form_url, data=json.dumps({ 'blobFileTokens': [token], @@ -79,10 +79,10 @@ def take_action(self, parsed_args): headers={ 'Content-Type': 'application/json' } - ) + ).json()['pageMessages'][0] - if entry_form_resp.status_code == 400: - print('You have no more submissions remaining for today.') + if entry_form_resp_message['type'] == 'error': + print(entry_form_resp_message['dangerousHtmlMessage']) return status_url = ( From f8a67f8da514c364279df07ebbe03daf77d61aec Mon Sep 17 00:00:00 2001 From: Felipe Almeida Date: Sat, 21 Oct 2017 00:40:59 -0200 Subject: [PATCH 05/16] add --zip flag and support for zipping a submission file prior to uploading it --- .gitignore | 1 + kaggle_cli/submit.py | 47 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index a317fe3..3efc1c3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ dist */__pycache__/ */*.pyc .kaggle-cli +.idea/ diff --git a/kaggle_cli/submit.py b/kaggle_cli/submit.py index c9f5420..58c2971 100644 --- a/kaggle_cli/submit.py +++ b/kaggle_cli/submit.py @@ -2,6 +2,9 @@ import time import re import json +import uuid +from argparse import ArgumentTypeError +import zipfile from cliff.command import Command @@ -21,6 +24,8 @@ def get_parser(self, prog_name): parser.add_argument('-c', '--competition', help='competition') parser.add_argument('-u', '--username', help='username') parser.add_argument('-p', '--password', help='password') + parser.add_argument('-z', '--zip', type=self._str2bool, nargs='?', const=True, default=False, + help='zip the submission file before uploading') return parser @@ -30,6 +35,7 @@ def take_action(self, parsed_args): username = config.get('username', '') password = config.get('password', '') competition = config.get('competition', '') + zip = config.get('zip', False) browser = common.login(username, password) base = 'https://www.kaggle.com' @@ -40,6 +46,12 @@ def take_action(self, parsed_args): entry = parsed_args.entry message = parsed_args.message + archive_name = Submit._rand_str(10)+'.zip' + + if zip: + with zipfile.ZipFile(archive_name, 'w', zipfile.ZIP_DEFLATED) as zf: + zf.write(entry) + competition_page = browser.get(competition_url) if competition_page.status_code == 404: @@ -51,18 +63,23 @@ def take_action(self, parsed_args): str(competition_page.soup) ).group(1) + if zip: + target_name = archive_name + else: + target_name = entry + form_submission = browser.post( file_form_url, data={ - 'fileName': entry, - 'contentLength': os.path.getsize(entry), - 'lastModifiedDateUtc': int(os.path.getmtime(entry) * 1000) + 'fileName': target_name, + 'contentLength': os.path.getsize(target_name), + 'lastModifiedDateUtc': int(os.path.getmtime(target_name) * 1000) } ).json() file_submit_url = base + form_submission['createUrl'] - with open(entry, 'rb') as submission_file: + with open(target_name, 'rb') as submission_file: token = browser.post( file_submit_url, files={ @@ -102,3 +119,25 @@ def take_action(self, parsed_args): else: print('something went wrong') break + + if zip: + os.remove(target_name) + + @staticmethod + def _str2bool(v): + """ + parse boolean values + + https://stackoverflow.com/a/43357954/436721 + :return: + """ + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise ArgumentTypeError('Boolean value expected.') + + @staticmethod + def _rand_str(length): + return uuid.uuid4().hex[:length-1] From 51160081c8951fb5ecb9402372770d05c4505368 Mon Sep 17 00:00:00 2001 From: Felipe Almeida Date: Sat, 21 Oct 2017 15:57:54 -0200 Subject: [PATCH 06/16] add support for setting the --zip option in the config --- kaggle_cli/config.py | 11 ++++++++--- kaggle_cli/submit.py | 26 ++++++++++++++++++++------ 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/kaggle_cli/config.py b/kaggle_cli/config.py index d81d491..4ad5d9b 100644 --- a/kaggle_cli/config.py +++ b/kaggle_cli/config.py @@ -5,10 +5,9 @@ from cliff.command import Command - CONFIG_DIR_NAME = '.kaggle-cli' CONFIG_FILE_NAME = 'config' -DATA_OPTIONS = set(['username', 'password', 'competition']) +DATA_OPTIONS = set(['username', 'password', 'competition', 'zip']) def get_config(config_path): @@ -85,6 +84,7 @@ def get_parser(self, prog_name): parser.add_argument('-u', '--username', help='username') parser.add_argument('-p', '--password', help='password') parser.add_argument('-c', '--competition', help='competition') + parser.add_argument('-z', '--zip', help='zip the submission file before uploading?', action='store_true') parser.add_argument( '-g', '--global', @@ -98,7 +98,7 @@ def take_action(self, parsed_args): parsed_arg_dict = vars(parsed_args) if DATA_OPTIONS & set( - filter(lambda x: parsed_arg_dict[x], parsed_arg_dict) + filter(lambda x: parsed_arg_dict[x], parsed_arg_dict) ): if parsed_arg_dict['global']: config_dir = os.path.join( @@ -135,6 +135,11 @@ def take_action(self, parsed_args): parsed_arg_dict['competition'] ) + if parsed_arg_dict['zip']: + config.set('user', 'zip', 'yes') + else: + config.set('user', 'zip', 'no') + with open(config_path, 'w') as config_file: config.write(config_file) else: diff --git a/kaggle_cli/submit.py b/kaggle_cli/submit.py index 58c2971..abdb2a2 100644 --- a/kaggle_cli/submit.py +++ b/kaggle_cli/submit.py @@ -25,7 +25,7 @@ def get_parser(self, prog_name): parser.add_argument('-u', '--username', help='username') parser.add_argument('-p', '--password', help='password') parser.add_argument('-z', '--zip', type=self._str2bool, nargs='?', const=True, default=False, - help='zip the submission file before uploading') + help='whether to zip the submission file before uploading') return parser @@ -35,7 +35,12 @@ def take_action(self, parsed_args): username = config.get('username', '') password = config.get('password', '') competition = config.get('competition', '') - zip = config.get('zip', False) + zip_flag = config.get('zip', 'no') + + if Submit._str2bool(zip_flag): + zip = True + else: + zip = False browser = common.login(username, password) base = 'https://www.kaggle.com' @@ -46,7 +51,7 @@ def take_action(self, parsed_args): entry = parsed_args.entry message = parsed_args.message - archive_name = Submit._rand_str(10)+'.zip' + archive_name = Submit._rand_str(10) + '.zip' if zip: with zipfile.ZipFile(archive_name, 'w', zipfile.ZIP_DEFLATED) as zf: @@ -126,10 +131,11 @@ def take_action(self, parsed_args): @staticmethod def _str2bool(v): """ - parse boolean values + parse truthy/falsy strings into booleans https://stackoverflow.com/a/43357954/436721 - :return: + :param v: the string to be parsed + :return: a boolean value """ if v.lower() in ('yes', 'true', 't', 'y', '1'): return True @@ -140,4 +146,12 @@ def _str2bool(v): @staticmethod def _rand_str(length): - return uuid.uuid4().hex[:length-1] + """ + this is used to prevent caching issues + + https://stackoverflow.com/a/34017605/436721 + + :param length: integer length + :return: a random string of the given length + """ + return uuid.uuid4().hex[:length - 1] From 23bb924f92790ce71348dce061636d1150522e05 Mon Sep 17 00:00:00 2001 From: Felipe Almeida Date: Mon, 23 Oct 2017 21:11:15 -0200 Subject: [PATCH 07/16] made suggested changes (where applicable) --- kaggle_cli/config.py | 2 +- kaggle_cli/submit.py | 54 ++++++++++++++++++++++++++------------------ 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/kaggle_cli/config.py b/kaggle_cli/config.py index 4ad5d9b..3b77b6f 100644 --- a/kaggle_cli/config.py +++ b/kaggle_cli/config.py @@ -98,7 +98,7 @@ def take_action(self, parsed_args): parsed_arg_dict = vars(parsed_args) if DATA_OPTIONS & set( - filter(lambda x: parsed_arg_dict[x], parsed_arg_dict) + filter(lambda x: parsed_arg_dict[x], parsed_arg_dict) ): if parsed_arg_dict['global']: config_dir = os.path.join( diff --git a/kaggle_cli/submit.py b/kaggle_cli/submit.py index abdb2a2..a00e65d 100644 --- a/kaggle_cli/submit.py +++ b/kaggle_cli/submit.py @@ -24,8 +24,7 @@ def get_parser(self, prog_name): parser.add_argument('-c', '--competition', help='competition') parser.add_argument('-u', '--username', help='username') parser.add_argument('-p', '--password', help='password') - parser.add_argument('-z', '--zip', type=self._str2bool, nargs='?', const=True, default=False, - help='whether to zip the submission file before uploading') + parser.add_argument('-z', '--zip', help='zip the submission file before uploading?', action='store_true') return parser @@ -37,10 +36,7 @@ def take_action(self, parsed_args): competition = config.get('competition', '') zip_flag = config.get('zip', 'no') - if Submit._str2bool(zip_flag): - zip = True - else: - zip = False + zip = Submit._str2bool(zip_flag) browser = common.login(username, password) base = 'https://www.kaggle.com' @@ -51,7 +47,12 @@ def take_action(self, parsed_args): entry = parsed_args.entry message = parsed_args.message - archive_name = Submit._rand_str(10) + '.zip' + archive_name = Submit._make_archive_name(entry) + + # print(archive_name) + # print(zip) + # + # return if zip: with zipfile.ZipFile(archive_name, 'w', zipfile.ZIP_DEFLATED) as zf: @@ -128,30 +129,39 @@ def take_action(self, parsed_args): if zip: os.remove(target_name) + @staticmethod + def _make_archive_name(original_file_path): + # if original name already has a suffix (csv,txt,etc), remove it + extension_pattern = r'(^.+)\.(.+)$' + + # file may be in another directory + original_basename = os.path.basename(original_file_path) + + if re.match(extension_pattern,original_basename): + archive_name = re.sub(extension_pattern,r'\1.zip',original_basename) + else: + archive_name = original_basename+".zip" + + # this is used to prevent caching issues + string_prefix = uuid.uuid4().hex[:4] + + prefixed_archive_name = string_prefix+"-"+archive_name + + original_directory_path = os.path.dirname(original_file_path) + + return os.path.join(original_directory_path,prefixed_archive_name) + @staticmethod def _str2bool(v): """ parse truthy/falsy strings into booleans - https://stackoverflow.com/a/43357954/436721 :param v: the string to be parsed :return: a boolean value """ - if v.lower() in ('yes', 'true', 't', 'y', '1'): + if v is True or v.lower() in ('yes', 'true', 't', 'y', '1'): return True - elif v.lower() in ('no', 'false', 'f', 'n', '0'): + elif v is False or v.lower() in ('no', 'false', 'f', 'n', '0'): return False else: raise ArgumentTypeError('Boolean value expected.') - - @staticmethod - def _rand_str(length): - """ - this is used to prevent caching issues - - https://stackoverflow.com/a/34017605/436721 - - :param length: integer length - :return: a random string of the given length - """ - return uuid.uuid4().hex[:length - 1] From 574019d9abc444639b3fb6a07520452b3860f533 Mon Sep 17 00:00:00 2001 From: floydwch Date: Tue, 24 Oct 2017 11:39:12 +0800 Subject: [PATCH 08/16] add type of config field --- kaggle_cli/config.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/kaggle_cli/config.py b/kaggle_cli/config.py index d81d491..ddcc44a 100644 --- a/kaggle_cli/config.py +++ b/kaggle_cli/config.py @@ -8,7 +8,17 @@ CONFIG_DIR_NAME = '.kaggle-cli' CONFIG_FILE_NAME = 'config' -DATA_OPTIONS = set(['username', 'password', 'competition']) +FIELD_OPTIONS = { + 'username': { + 'type': str + }, + 'password': { + 'type': str + }, + 'competition': { + 'type': str + } +} def get_config(config_path): @@ -97,7 +107,7 @@ def get_parser(self, prog_name): def take_action(self, parsed_args): parsed_arg_dict = vars(parsed_args) - if DATA_OPTIONS & set( + if set(FIELD_OPTIONS.keys()) & set( filter(lambda x: parsed_arg_dict[x], parsed_arg_dict) ): if parsed_arg_dict['global']: From 9158c36b2a294e685d610afd0dbb0b2009ee532e Mon Sep 17 00:00:00 2001 From: floydwch Date: Tue, 24 Oct 2017 12:20:32 +0800 Subject: [PATCH 09/16] add config field coercion --- kaggle_cli/config.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/kaggle_cli/config.py b/kaggle_cli/config.py index ddcc44a..2f1641a 100644 --- a/kaggle_cli/config.py +++ b/kaggle_cli/config.py @@ -59,11 +59,30 @@ def merge_dicts(x, y={}): return z +def config_section_to_dict(config, section, field_options): + result_dict = {} + for name, spec in field_options.items(): + if spec['type'] == bool: + value = config.getboolean(section, name, fallback=None) + elif spec['type'] == int: + value = config.getint(section, name, fallback=None) + elif spec['type'] == float: + value = config.getfloat(section, name, fallback=None) + else: + value = config.get(section, name, fallback=None) + if value: + result_dict[name] = value + return result_dict + + def get_working_config(configs): return reduce( lambda working_config, config: merge_dicts(config, working_config), - map(lambda config: dict(config['user']), configs), + map( + lambda config: + config_section_to_dict(config, 'user', FIELD_OPTIONS), + configs), {} ) From 551a1b2d259ef996517f245b1f34b97f55326ca3 Mon Sep 17 00:00:00 2001 From: floydwch Date: Tue, 24 Oct 2017 12:21:18 +0800 Subject: [PATCH 10/16] prepare zip field --- kaggle_cli/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kaggle_cli/config.py b/kaggle_cli/config.py index 2f1641a..26248a2 100644 --- a/kaggle_cli/config.py +++ b/kaggle_cli/config.py @@ -17,6 +17,9 @@ }, 'competition': { 'type': str + }, + 'zip': { + 'type': bool } } From 7f9b3c08465e2f1f0dd56c2eb47b7f3393e51e17 Mon Sep 17 00:00:00 2001 From: Felipe Almeida Date: Sat, 21 Oct 2017 00:40:59 -0200 Subject: [PATCH 11/16] add --zip flag and support for zipping a submission file prior to uploading it --- .gitignore | 1 + kaggle_cli/submit.py | 47 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index a317fe3..3efc1c3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ dist */__pycache__/ */*.pyc .kaggle-cli +.idea/ diff --git a/kaggle_cli/submit.py b/kaggle_cli/submit.py index c9f5420..58c2971 100644 --- a/kaggle_cli/submit.py +++ b/kaggle_cli/submit.py @@ -2,6 +2,9 @@ import time import re import json +import uuid +from argparse import ArgumentTypeError +import zipfile from cliff.command import Command @@ -21,6 +24,8 @@ def get_parser(self, prog_name): parser.add_argument('-c', '--competition', help='competition') parser.add_argument('-u', '--username', help='username') parser.add_argument('-p', '--password', help='password') + parser.add_argument('-z', '--zip', type=self._str2bool, nargs='?', const=True, default=False, + help='zip the submission file before uploading') return parser @@ -30,6 +35,7 @@ def take_action(self, parsed_args): username = config.get('username', '') password = config.get('password', '') competition = config.get('competition', '') + zip = config.get('zip', False) browser = common.login(username, password) base = 'https://www.kaggle.com' @@ -40,6 +46,12 @@ def take_action(self, parsed_args): entry = parsed_args.entry message = parsed_args.message + archive_name = Submit._rand_str(10)+'.zip' + + if zip: + with zipfile.ZipFile(archive_name, 'w', zipfile.ZIP_DEFLATED) as zf: + zf.write(entry) + competition_page = browser.get(competition_url) if competition_page.status_code == 404: @@ -51,18 +63,23 @@ def take_action(self, parsed_args): str(competition_page.soup) ).group(1) + if zip: + target_name = archive_name + else: + target_name = entry + form_submission = browser.post( file_form_url, data={ - 'fileName': entry, - 'contentLength': os.path.getsize(entry), - 'lastModifiedDateUtc': int(os.path.getmtime(entry) * 1000) + 'fileName': target_name, + 'contentLength': os.path.getsize(target_name), + 'lastModifiedDateUtc': int(os.path.getmtime(target_name) * 1000) } ).json() file_submit_url = base + form_submission['createUrl'] - with open(entry, 'rb') as submission_file: + with open(target_name, 'rb') as submission_file: token = browser.post( file_submit_url, files={ @@ -102,3 +119,25 @@ def take_action(self, parsed_args): else: print('something went wrong') break + + if zip: + os.remove(target_name) + + @staticmethod + def _str2bool(v): + """ + parse boolean values + + https://stackoverflow.com/a/43357954/436721 + :return: + """ + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise ArgumentTypeError('Boolean value expected.') + + @staticmethod + def _rand_str(length): + return uuid.uuid4().hex[:length-1] From bd0cfeda0757f4cd175fea1a737cc7f21dc2c66b Mon Sep 17 00:00:00 2001 From: Felipe Almeida Date: Sat, 21 Oct 2017 15:57:54 -0200 Subject: [PATCH 12/16] add support for setting the --zip option in the config --- kaggle_cli/submit.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/kaggle_cli/submit.py b/kaggle_cli/submit.py index 58c2971..abdb2a2 100644 --- a/kaggle_cli/submit.py +++ b/kaggle_cli/submit.py @@ -25,7 +25,7 @@ def get_parser(self, prog_name): parser.add_argument('-u', '--username', help='username') parser.add_argument('-p', '--password', help='password') parser.add_argument('-z', '--zip', type=self._str2bool, nargs='?', const=True, default=False, - help='zip the submission file before uploading') + help='whether to zip the submission file before uploading') return parser @@ -35,7 +35,12 @@ def take_action(self, parsed_args): username = config.get('username', '') password = config.get('password', '') competition = config.get('competition', '') - zip = config.get('zip', False) + zip_flag = config.get('zip', 'no') + + if Submit._str2bool(zip_flag): + zip = True + else: + zip = False browser = common.login(username, password) base = 'https://www.kaggle.com' @@ -46,7 +51,7 @@ def take_action(self, parsed_args): entry = parsed_args.entry message = parsed_args.message - archive_name = Submit._rand_str(10)+'.zip' + archive_name = Submit._rand_str(10) + '.zip' if zip: with zipfile.ZipFile(archive_name, 'w', zipfile.ZIP_DEFLATED) as zf: @@ -126,10 +131,11 @@ def take_action(self, parsed_args): @staticmethod def _str2bool(v): """ - parse boolean values + parse truthy/falsy strings into booleans https://stackoverflow.com/a/43357954/436721 - :return: + :param v: the string to be parsed + :return: a boolean value """ if v.lower() in ('yes', 'true', 't', 'y', '1'): return True @@ -140,4 +146,12 @@ def _str2bool(v): @staticmethod def _rand_str(length): - return uuid.uuid4().hex[:length-1] + """ + this is used to prevent caching issues + + https://stackoverflow.com/a/34017605/436721 + + :param length: integer length + :return: a random string of the given length + """ + return uuid.uuid4().hex[:length - 1] From 9e5d725fb10261420966b42a0efa3d6d8aed894f Mon Sep 17 00:00:00 2001 From: Felipe Almeida Date: Sat, 21 Oct 2017 00:40:59 -0200 Subject: [PATCH 13/16] add --zip flag and support for zipping a submission file prior to uploading it --- kaggle_cli/submit.py | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/kaggle_cli/submit.py b/kaggle_cli/submit.py index abdb2a2..58c2971 100644 --- a/kaggle_cli/submit.py +++ b/kaggle_cli/submit.py @@ -25,7 +25,7 @@ def get_parser(self, prog_name): parser.add_argument('-u', '--username', help='username') parser.add_argument('-p', '--password', help='password') parser.add_argument('-z', '--zip', type=self._str2bool, nargs='?', const=True, default=False, - help='whether to zip the submission file before uploading') + help='zip the submission file before uploading') return parser @@ -35,12 +35,7 @@ def take_action(self, parsed_args): username = config.get('username', '') password = config.get('password', '') competition = config.get('competition', '') - zip_flag = config.get('zip', 'no') - - if Submit._str2bool(zip_flag): - zip = True - else: - zip = False + zip = config.get('zip', False) browser = common.login(username, password) base = 'https://www.kaggle.com' @@ -51,7 +46,7 @@ def take_action(self, parsed_args): entry = parsed_args.entry message = parsed_args.message - archive_name = Submit._rand_str(10) + '.zip' + archive_name = Submit._rand_str(10)+'.zip' if zip: with zipfile.ZipFile(archive_name, 'w', zipfile.ZIP_DEFLATED) as zf: @@ -131,11 +126,10 @@ def take_action(self, parsed_args): @staticmethod def _str2bool(v): """ - parse truthy/falsy strings into booleans + parse boolean values https://stackoverflow.com/a/43357954/436721 - :param v: the string to be parsed - :return: a boolean value + :return: """ if v.lower() in ('yes', 'true', 't', 'y', '1'): return True @@ -146,12 +140,4 @@ def _str2bool(v): @staticmethod def _rand_str(length): - """ - this is used to prevent caching issues - - https://stackoverflow.com/a/34017605/436721 - - :param length: integer length - :return: a random string of the given length - """ - return uuid.uuid4().hex[:length - 1] + return uuid.uuid4().hex[:length-1] From d17fe8caf2becb51ea0e791945087b73f8c32103 Mon Sep 17 00:00:00 2001 From: Felipe Almeida Date: Sat, 21 Oct 2017 15:57:54 -0200 Subject: [PATCH 14/16] add support for setting the --zip option in the config --- kaggle_cli/submit.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/kaggle_cli/submit.py b/kaggle_cli/submit.py index 58c2971..abdb2a2 100644 --- a/kaggle_cli/submit.py +++ b/kaggle_cli/submit.py @@ -25,7 +25,7 @@ def get_parser(self, prog_name): parser.add_argument('-u', '--username', help='username') parser.add_argument('-p', '--password', help='password') parser.add_argument('-z', '--zip', type=self._str2bool, nargs='?', const=True, default=False, - help='zip the submission file before uploading') + help='whether to zip the submission file before uploading') return parser @@ -35,7 +35,12 @@ def take_action(self, parsed_args): username = config.get('username', '') password = config.get('password', '') competition = config.get('competition', '') - zip = config.get('zip', False) + zip_flag = config.get('zip', 'no') + + if Submit._str2bool(zip_flag): + zip = True + else: + zip = False browser = common.login(username, password) base = 'https://www.kaggle.com' @@ -46,7 +51,7 @@ def take_action(self, parsed_args): entry = parsed_args.entry message = parsed_args.message - archive_name = Submit._rand_str(10)+'.zip' + archive_name = Submit._rand_str(10) + '.zip' if zip: with zipfile.ZipFile(archive_name, 'w', zipfile.ZIP_DEFLATED) as zf: @@ -126,10 +131,11 @@ def take_action(self, parsed_args): @staticmethod def _str2bool(v): """ - parse boolean values + parse truthy/falsy strings into booleans https://stackoverflow.com/a/43357954/436721 - :return: + :param v: the string to be parsed + :return: a boolean value """ if v.lower() in ('yes', 'true', 't', 'y', '1'): return True @@ -140,4 +146,12 @@ def _str2bool(v): @staticmethod def _rand_str(length): - return uuid.uuid4().hex[:length-1] + """ + this is used to prevent caching issues + + https://stackoverflow.com/a/34017605/436721 + + :param length: integer length + :return: a random string of the given length + """ + return uuid.uuid4().hex[:length - 1] From 2bb95b41352092309edb9c9cee6c679e3c1d3588 Mon Sep 17 00:00:00 2001 From: Felipe Almeida Date: Mon, 23 Oct 2017 21:11:15 -0200 Subject: [PATCH 15/16] made suggested changes (where applicable) --- kaggle_cli/config.py | 10 ++++++++ kaggle_cli/submit.py | 57 ++++++++++++++++---------------------------- 2 files changed, 31 insertions(+), 36 deletions(-) diff --git a/kaggle_cli/config.py b/kaggle_cli/config.py index 26248a2..7fdcc59 100644 --- a/kaggle_cli/config.py +++ b/kaggle_cli/config.py @@ -117,6 +117,11 @@ def get_parser(self, prog_name): parser.add_argument('-u', '--username', help='username') parser.add_argument('-p', '--password', help='password') parser.add_argument('-c', '--competition', help='competition') + parser.add_argument( + '-z', + '--zip', + help='zip the submission file before uploading?', + action='store_true') parser.add_argument( '-g', '--global', @@ -167,6 +172,11 @@ def take_action(self, parsed_args): parsed_arg_dict['competition'] ) + if parsed_arg_dict['zip']: + config.set( + 'user','zip','yes' + ) + with open(config_path, 'w') as config_file: config.write(config_file) else: diff --git a/kaggle_cli/submit.py b/kaggle_cli/submit.py index abdb2a2..335231b 100644 --- a/kaggle_cli/submit.py +++ b/kaggle_cli/submit.py @@ -2,6 +2,7 @@ import time import re import json +import sys import uuid from argparse import ArgumentTypeError import zipfile @@ -24,8 +25,7 @@ def get_parser(self, prog_name): parser.add_argument('-c', '--competition', help='competition') parser.add_argument('-u', '--username', help='username') parser.add_argument('-p', '--password', help='password') - parser.add_argument('-z', '--zip', type=self._str2bool, nargs='?', const=True, default=False, - help='whether to zip the submission file before uploading') + parser.add_argument('-z', '--zip', help='zip the submission file before uploading?', action='store_true') return parser @@ -35,12 +35,7 @@ def take_action(self, parsed_args): username = config.get('username', '') password = config.get('password', '') competition = config.get('competition', '') - zip_flag = config.get('zip', 'no') - - if Submit._str2bool(zip_flag): - zip = True - else: - zip = False + zip = config.get('zip', False) browser = common.login(username, password) base = 'https://www.kaggle.com' @@ -51,7 +46,7 @@ def take_action(self, parsed_args): entry = parsed_args.entry message = parsed_args.message - archive_name = Submit._rand_str(10) + '.zip' + archive_name = make_archive_name(entry) if zip: with zipfile.ZipFile(archive_name, 'w', zipfile.ZIP_DEFLATED) as zf: @@ -101,10 +96,10 @@ def take_action(self, parsed_args): headers={ 'Content-Type': 'application/json' } - ).json()['pageMessages'][0] + ).json()['pageMessages'] - if entry_form_resp_message['type'] == 'error': - print(entry_form_resp_message['dangerousHtmlMessage']) + if entry_form_resp_message and entry_form_resp_message[0]['type'] == 'error': + print(entry_form_resp_message[0]['dangerousHtmlMessage']) return status_url = ( @@ -128,30 +123,20 @@ def take_action(self, parsed_args): if zip: os.remove(target_name) - @staticmethod - def _str2bool(v): - """ - parse truthy/falsy strings into booleans - - https://stackoverflow.com/a/43357954/436721 - :param v: the string to be parsed - :return: a boolean value - """ - if v.lower() in ('yes', 'true', 't', 'y', '1'): - return True - elif v.lower() in ('no', 'false', 'f', 'n', '0'): - return False - else: - raise ArgumentTypeError('Boolean value expected.') - @staticmethod - def _rand_str(length): - """ - this is used to prevent caching issues +def make_archive_name(original_file_path): + + # if original name already has a suffix (csv,txt,etc), remove it + extension_pattern = r'(^.+)\.(.+)$' + + # file may be in another directory + original_basename = os.path.basename(original_file_path) + + if re.match(extension_pattern,original_basename): + archive_name = re.sub(extension_pattern,r'\1.zip',original_basename) + else: + archive_name = original_basename+".zip" - https://stackoverflow.com/a/34017605/436721 + original_directory_path = os.path.dirname(original_file_path) - :param length: integer length - :return: a random string of the given length - """ - return uuid.uuid4().hex[:length - 1] + return os.path.join(original_directory_path,archive_name) From be32ad41d79aad07bdf94681ba867b8e6609bb3d Mon Sep 17 00:00:00 2001 From: Felipe Almeida Date: Thu, 26 Oct 2017 23:28:55 -0200 Subject: [PATCH 16/16] updated README.md --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index b0eb162..fa3b0c4 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,12 @@ To submit an entry. $ kg submit -u -p -c -m "" ``` +Optionally, add `-z` to zip the submission file before uploading: + +``` +$ kg submit -u -p -c -z -m "" +``` + ### Download To download the data files (resumable). @@ -53,6 +59,8 @@ $ kg dataset -u -p -o -d ### Config To set global config. +> Optional: add `-z` to zip submission files before uploading. + ``` $ kg config -g -u -p -c ```