diff --git a/gcexport.py b/gcexport.py old mode 100755 new mode 100644 index 57977a5..9671faa --- a/gcexport.py +++ b/gcexport.py @@ -9,6 +9,8 @@ Date: February 21, 2016 Fork author: Peter Steiner (https://github.com/pe-st/) Date: June 2017 +Porting to Python3 started from Thomas Th. (https://github.com/telemaxx/) +Date: March 2020 Description: Use this script to export your fitness data from Garmin Connect. See README.md for more information. @@ -29,8 +31,10 @@ from platform import python_version from subprocess import call from timeit import default_timer as timer +from os.path import sep import argparse + import csv import json import logging @@ -42,10 +46,9 @@ python3 = True try: # for python3 - from urllib.request import urlopen, Request + from urllib.request import urlopen, Request, HTTPError, URLError except ImportError: # or python2 - from urllib import urlencode - from urllib2 import urlopen, Request, HTTPError + from urllib2 import urlopen, Request, HTTPError, URLError python3 = False if python3: @@ -54,11 +57,13 @@ import urllib.parse import urllib.request import urllib + from urllib.parse import urlencode COOKIE_JAR = http.cookiejar.CookieJar() OPENER = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(COOKIE_JAR), urllib.request.HTTPSHandler(debuglevel=0)) else: import cookielib import urllib2 + from urllib import urlencode COOKIE_JAR = cookielib.CookieJar() OPENER = urllib2.build_opener(urllib2.HTTPCookieProcessor(COOKIE_JAR), urllib2.HTTPSHandler(debuglevel=0)) @@ -137,11 +142,8 @@ } # URLs for various services. -if python3: - URL_GC_LOGIN = "https://sso.garmin.com/sso/signin?" + urllib.parse.urlencode(DATA) -else: - URL_GC_LOGIN = 'https://sso.garmin.com/sso/signin?' + urlencode(DATA) +URL_GC_LOGIN = 'https://sso.garmin.com/sso/signin?' + urlencode(DATA) URL_GC_POST_AUTH = 'https://connect.garmin.com/modern/activities?' URL_GC_PROFILE = 'https://connect.garmin.com/modern/profile' URL_GC_USERSTATS = 'https://connect.garmin.com/modern/proxy/userstats-service/statistics/' @@ -199,8 +201,18 @@ def sanitize_filename(name, max_length=0): def write_to_file(filename, content, mode, file_time=None): """Helper function that persists content to file.""" - write_file = open(filename, mode) - write_file.write(content) + #print('write to file: ', filename, ' content: ', content) + if python3 and (filename.endswith('.json') or filename.endswith('.gpx') or filename.endswith('.tcx')): + #write_file = open(filename, mode) + write_file = open(filename, mode, encoding="utf-8") + if not isinstance(content,str): + content=content.decode("utf-8") + content=content.replace('\u0113','e') #for umlaut e in fEnix + write_file.write(content) + else: + write_file = open(filename, mode) + write_file.write(content) + #write_file.write(content) write_file.close() if file_time: utime(filename, (file_time, file_time)) @@ -209,14 +221,11 @@ def write_to_file(filename, content, mode, file_time=None): # url is a string, post is a dictionary of POST parameters, headers is a dictionary of headers. def http_req(url, post=None, headers=None): """Helper function that makes the HTTP requests.""" - if python3: - request = urllib.request.Request(url) - else: - request = urllib2.Request(url) - + request = Request(url) # Tell Garmin we're some supported browser. - request.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, \ - like Gecko) Chrome/54.0.2816.0 Safari/537.36') + request.add_header("User-Agent","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, \ + like Gecko) Chrome/54.0.2816.0 Safari/537.36", + ) if headers: if python3: for header_key, header_value in headers.items(): @@ -225,21 +234,19 @@ def http_req(url, post=None, headers=None): for header_key, header_value in headers.iteritems(): request.add_header(header_key, header_value) if post: + post = urlencode(post) # Convert dictionary to POST parameter string. if python3: - post = urllib.parse.urlencode(post) # Convert dictionary to POST parameter string. - else: - post = urlencode(post) # Convert dictionary to POST parameter string. - + post = post.encode("utf-8") start_time = timer() try: response = OPENER.open(request, data=post) - except urllib2.URLError as ex: + except URLError as ex: if hasattr(ex, 'reason'): logging.error('Failed to reach url %s, error: %s', url, ex) raise else: raise - logging.debug('Got %s in %s s from %s', response.getcode(), timer() - start_time, url) + logging.debug('Got %s in %s s from %s', response.getcode(), timer() - start_time, url) # N.B. urllib2 will follow any 302 redirects. # print(response.getcode()) @@ -255,26 +262,46 @@ def http_req(url, post=None, headers=None): # idea stolen from https://stackoverflow.com/a/31852401/3686 -def load_properties(multiline, sep='=', comment_char='#', keys=None): +def load_properties(multiline, separator='=', comment_char='#', keys=None): """ - Read a multiline string of properties (key/value pair separated by *sep*) into a dict + Read a multiline string of properties (key/value pair separated by *separator*) into a dict :param multiline: input string of properties - :param sep: separator between key and value + :param separator: separator between key and value :param comment_char: lines starting with this char are considered comments, not key/value pairs :param keys: list to append the keys to :return: """ + ##if python3: + ## multiline=multiline.decode("utf-8") + props = {} for line in multiline.splitlines(): stripped_line = line.strip() - if stripped_line and not stripped_line.startswith(comment_char): - key_value = stripped_line.split(sep) - key = key_value[0].strip() - value = sep.join(key_value[1:]).strip().strip('"') - props[key] = value - if keys != None: - keys.append(key) + if python3: + stripped_line = line.strip() + if stripped_line: + if (isinstance(stripped_line,bytes)): + stripped_line = stripped_line.decode('utf8') + if (isinstance(comment_char,bytes)): + comment_char=comment_char.decode('utf8') + if not stripped_line.startswith(comment_char): + #https://python-forum.io/Thread-Diff-between-Py-2-7-and-3--14749 + key_value = stripped_line.split(separator) + key = key_value[0].strip() + value = separator.join(key_value[1:]).strip().strip('"') + props[key] = value + if keys != None: + keys.append(key) + else: + stripped_line = line.strip() + if stripped_line and not stripped_line.startswith(comment_char): + key_value = stripped_line.split(separator) + key = key_value[0].strip() + value = separator.join(key_value[1:]).strip().strip('"') + props[key] = value + if keys != None: + keys.append(key) return props @@ -461,10 +488,7 @@ def login_to_garmin_connect(args): username = args.username if args.username else raw_input('Username: ') password = args.password if args.password else getpass() - if python3: - logging.debug("Login params: %s", urllib.parse.urlencode(DATA)) - else: - logging.debug("Login params: %s", urlencode(DATA)) + logging.debug("Login params: %s", urlencode(DATA)) # Initially, we need to get a valid session cookie, so we pull the login page. print('Connecting to Garmin Connect...', end='') @@ -489,7 +513,11 @@ def login_to_garmin_connect(args): } print('Requesting Login ticket...', end='') - login_response = http_req(URL_GC_LOGIN + '#', post_data, headers) + if python3: + login_response = http_req(URL_GC_LOGIN + "#", post_data, headers).decode() + else: + login_response = http_req(URL_GC_LOGIN + '#', post_data, headers) + for cookie in COOKIE_JAR: logging.debug("Cookie %s : %s", cookie.name, cookie.value) # write_to_file('login-response.html', login_response, 'w') @@ -634,7 +662,7 @@ def extract_device(device_dict, details, start_time_seconds, args, http_caller, device_id = device_meta['deviceId'] if present('deviceId', device_meta) else None if 'deviceId' not in device_meta or device_id and device_id != '0': device_json = http_caller(URL_GC_DEVICE + str(device_app_inst_id)) - file_writer(args.directory + '/device_' + str(device_app_inst_id) + '.json', + file_writer(args.directory + sep + 'device_' + str(device_app_inst_id) + '.json', device_json, 'w', start_time_seconds) if not device_json: @@ -658,14 +686,14 @@ def load_gear(activity_id, args): gear = json.loads(gear_json) if gear: del args # keep 'args' argument in case you need to uncomment write_to_file - # write_to_file(args.directory + '/activity_' + activity_id + '-gear.json', + # write_to_file(args.directory + sep + 'activity_' + activity_id + '-gear.json', # gear_json, 'w') gear_display_name = gear[0]['displayName'] if present('displayName', gear[0]) else None gear_model = gear[0]['customMakeModel'] if present('customMakeModel', gear[0]) else None logging.debug("Gear for %s = %s/%s", activity_id, gear_display_name, gear_model) return gear_display_name if gear_display_name else gear_model return None - except urllib2.HTTPError: + except HTTPError: pass # don't abort just for missing gear... # logging.info("Unable to get gear for %d", activity_id) # logging.exception(e) @@ -675,6 +703,10 @@ def export_data_file(activity_id, activity_details, args, file_time, append_desc """ Write the data of the activity to a file, depending on the chosen data format """ + #if python3: + # if (isinstance(start_time_locale,bytes)): + # start_time_locale = start_time_locale.decode('utf8') + # Time dependent subdirectory for activity files, e.g. '{YYYY} if not args.subdir is None: directory = resolve_path(args.directory, args.subdir, start_time_locale) @@ -687,27 +719,27 @@ def export_data_file(activity_id, activity_details, args, file_time, append_desc # timestamp as prefix for filename if args.fileprefix and args.fileprefix > 0: - prefix = "{}-".format(start_time_locale.replace("-", "").replace(":", b"").replace(" ", "-")) + prefix = "{}-".format(start_time_locale.replace("-", "").replace(":", "").replace(" ", "-")) else: prefix = "" fit_filename = None if args.format == 'gpx': - data_filename = directory + '/' + prefix + 'activity_' + activity_id + append_desc + '.gpx' + data_filename = directory + sep + prefix + 'activity_' + activity_id + append_desc + '.gpx' download_url = URL_GC_GPX_ACTIVITY + activity_id + '?full=true' file_mode = 'w' elif args.format == 'tcx': - data_filename = directory + '/' + prefix + 'activity_' + activity_id + append_desc + '.tcx' + data_filename = directory + sep + prefix + 'activity_' + activity_id + append_desc + '.tcx' download_url = URL_GC_TCX_ACTIVITY + activity_id + '?full=true' file_mode = 'w' elif args.format == 'original': - data_filename = directory + '/' + prefix + 'activity_' + activity_id + append_desc + '.zip' + data_filename = directory + sep + prefix + 'activity_' + activity_id + append_desc + '.zip' # TODO not all 'original' files are in FIT format, some are GPX or TCX... - fit_filename = directory + '/' + prefix + 'activity_' + activity_id + append_desc + '.fit' + fit_filename = directory + sep + prefix + 'activity_' + activity_id + append_desc + '.fit' download_url = URL_GC_ORIGINAL_ACTIVITY + activity_id file_mode = 'wb' elif args.format == 'json': - data_filename = directory + '/' + prefix + 'activity_' + activity_id + append_desc + '.json' + data_filename = directory + sep + prefix + 'activity_' + activity_id + append_desc + '.json' file_mode = 'w' else: raise Exception('Unrecognized format.') @@ -730,7 +762,7 @@ def export_data_file(activity_id, activity_details, args, file_time, append_desc try: data = http_req(download_url) - except urllib2.HTTPError as ex: + except HTTPError as ex: # Handle expected (though unfortunate) error codes; die on unexpected ones. if ex.code == 500 and args.format == 'tcx': # Garmin will give an internal server error (HTTP 500) when downloading TCX files @@ -799,11 +831,13 @@ def logging_verbosity(verbosity): for handler in logger.handlers: if isinstance(handler, logging.FileHandler): # this is the logfile handler + ###level = logging.DEBUG if verbosity > 0 else logging.INFO level = logging.DEBUG if verbosity and verbosity > 0 else logging.INFO handler.setLevel(level) logging.info('New logfile level: %s', logging.getLevelName(level)) elif isinstance(handler, logging.StreamHandler): # this is the console handler + ###level = logging.DEBUG if verbosity > 1 else (logging.INFO if verbosity > 0 else logging.WARN) level = logging.DEBUG if verbosity and verbosity > 1 else (logging.INFO if verbosity and verbosity > 0 else logging.WARN) handler.setLevel(level) logging.debug('New console log level: %s', logging.getLevelName(level)) @@ -834,6 +868,9 @@ def main(argv): logging.info("Starting %s version %s, using Python version %s", argv[0], SCRIPT_VERSION, python_version()) args = parse_arguments(argv) logging_verbosity(args.verbosity) + + print('verbosity: ' , args.verbosity) + logging_verbosity(0) print('Welcome to Garmin Connect Exporter!') @@ -916,14 +953,11 @@ def main(argv): print('Querying list of activities ' + str(total_downloaded + 1) \ + '..' + str(total_downloaded + num_to_download) \ + '...', end='') + + logging.info('Activity list URL %s', URL_GC_LIST + urlencode(search_params)) + result = http_req(URL_GC_LIST + urlencode(search_params)) - if python3: - logging.info('Activity list URL %s', URL_GC_LIST + urllib.parse.urlencode(search_params)) - result = http_req(URL_GC_LIST + urllib.parse.urlencode(search_params)) - else: - logging.info('Activity list URL %s', URL_GC_LIST + urlencode(search_params)) - result = http_req(URL_GC_LIST + urlencode(search_params)) - + print(' Done.') # Persist JSON activities list @@ -943,12 +977,12 @@ def main(argv): pass # Display which entry we're skipping. print('Skipping Garmin Connect activity ', end='') - print('(' + str(current_index) + '/' + str(total_to_download) + ') ', end='') + print('(' + str(current_index) + sep + str(total_to_download) + ') ', end='') print('[' + str(actvty['activityId']) + '] \n', end='') else: # Display which entry we're working on. print('Garmin Connect activity ', end='') - print('(' + str(current_index) + '/' + str(total_to_download) + ') ', end='') + print('(' + str(current_index) + sep + str(total_to_download) + ') ', end='') print('[' + str(actvty['activityId']) + '] ', end='') print(actvty['activityName']) @@ -1010,7 +1044,7 @@ def main(argv): start_time_seconds) samples = json.loads(activity_measurements) extract['samples'] = samples - except urllib2.HTTPError: + except HTTPError: pass # don't abort just for missing samples... # logging.info("Unable to get samples for %d", actvty['activityId']) # logging.exception(e)