Python3 and Python2 with latest changes from feature/p3

gcexport.py

@@ -9,6 +9,8 @@
 Date: February 21, 2016
 Fork author: Peter Steiner (https://github.com/pe-st/)
 Date: June 2017
+Porting to Python3 started from Thomas Th. (https://github.com/telemaxx/)
+Date: March 2020
 
 Description:    Use this script to export your fitness data from Garmin Connect.
                 See README.md for more information.
@@ -29,8 +31,10 @@
 from platform import python_version
 from subprocess import call
 from timeit import default_timer as timer
+from os.path import sep
 
 import argparse
+
 import csv
 import json
 import logging
@@ -42,10 +46,9 @@
 
 python3 = True
 try: # for python3
-    from urllib.request import urlopen, Request
+    from urllib.request import urlopen, Request, HTTPError, URLError
 except ImportError: # or python2
-    from urllib import urlencode
-    from urllib2 import urlopen, Request, HTTPError
+    from urllib2 import urlopen, Request, HTTPError, URLError
     python3 = False
 
 if python3:
@@ -54,11 +57,13 @@
     import urllib.parse
     import urllib.request
     import urllib
+    from urllib.parse import urlencode
     COOKIE_JAR = http.cookiejar.CookieJar()
     OPENER = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(COOKIE_JAR), urllib.request.HTTPSHandler(debuglevel=0))
 else:
     import cookielib
     import urllib2
+    from urllib import urlencode
     COOKIE_JAR = cookielib.CookieJar()
     OPENER = urllib2.build_opener(urllib2.HTTPCookieProcessor(COOKIE_JAR), urllib2.HTTPSHandler(debuglevel=0))
 
@@ -137,11 +142,8 @@
 }
 
 # URLs for various services.
-if python3:
-    URL_GC_LOGIN = "https://sso.garmin.com/sso/signin?" + urllib.parse.urlencode(DATA)
-else:
-    URL_GC_LOGIN = 'https://sso.garmin.com/sso/signin?' + urlencode(DATA)
 
+URL_GC_LOGIN = 'https://sso.garmin.com/sso/signin?' + urlencode(DATA)
 URL_GC_POST_AUTH = 'https://connect.garmin.com/modern/activities?'
 URL_GC_PROFILE = 'https://connect.garmin.com/modern/profile'
 URL_GC_USERSTATS = 'https://connect.garmin.com/modern/proxy/userstats-service/statistics/'
@@ -199,8 +201,18 @@ def sanitize_filename(name, max_length=0):
 
 def write_to_file(filename, content, mode, file_time=None):
     """Helper function that persists content to file."""
-    write_file = open(filename, mode)
-    write_file.write(content)
+    #print('write to file: ', filename, ' content: ', content)
+    if python3 and (filename.endswith('.json') or filename.endswith('.gpx') or filename.endswith('.tcx')):
+        #write_file = open(filename, mode)
+        write_file = open(filename, mode, encoding="utf-8")
+        if not isinstance(content,str):
+            content=content.decode("utf-8")
+        content=content.replace('\u0113','e') #for umlaut e in fEnix
+        write_file.write(content)
+    else:
+        write_file = open(filename, mode)
+        write_file.write(content)
+    #write_file.write(content)
     write_file.close()
     if file_time:
         utime(filename, (file_time, file_time))
@@ -209,14 +221,11 @@ def write_to_file(filename, content, mode, file_time=None):
 # url is a string, post is a dictionary of POST parameters, headers is a dictionary of headers.
 def http_req(url, post=None, headers=None):
     """Helper function that makes the HTTP requests."""
-    if python3:
-        request = urllib.request.Request(url)
-    else:
-        request = urllib2.Request(url)
-
+    request = Request(url)
     # Tell Garmin we're some supported browser.
-    request.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, \
-        like Gecko) Chrome/54.0.2816.0 Safari/537.36')
+    request.add_header("User-Agent","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, \
+    like Gecko) Chrome/54.0.2816.0 Safari/537.36",
+    )
     if headers:
         if python3:
             for header_key, header_value in headers.items():
@@ -225,21 +234,19 @@ def http_req(url, post=None, headers=None):
             for header_key, header_value in headers.iteritems():
                 request.add_header(header_key, header_value)
     if post:
+        post = urlencode(post)  # Convert dictionary to POST parameter string.
         if python3:
-            post = urllib.parse.urlencode(post)  # Convert dictionary to POST parameter string.
-        else:
-            post = urlencode(post)  # Convert dictionary to POST parameter string.
-
+            post = post.encode("utf-8")
     start_time = timer()
     try:
         response = OPENER.open(request, data=post)
-    except urllib2.URLError as ex:
+    except URLError as ex:
         if hasattr(ex, 'reason'):
             logging.error('Failed to reach url %s, error: %s', url, ex)
             raise
         else:
             raise
-    logging.debug('Got %s in %s s from %s', response.getcode(), timer() - start_time, url)
+        logging.debug('Got %s in %s s from %s', response.getcode(), timer() - start_time, url)
 
     # N.B. urllib2 will follow any 302 redirects.
     # print(response.getcode())
@@ -255,26 +262,46 @@ def http_req(url, post=None, headers=None):
 
 
 # idea stolen from https://stackoverflow.com/a/31852401/3686
-def load_properties(multiline, sep='=', comment_char='#', keys=None):
+def load_properties(multiline, separator='=', comment_char='#', keys=None):
     """
-    Read a multiline string of properties (key/value pair separated by *sep*) into a dict
+    Read a multiline string of properties (key/value pair separated by *separator*) into a dict
 
     :param multiline:    input string of properties
-    :param sep:          separator between key and value
+    :param separator:    separator between key and value
     :param comment_char: lines starting with this char are considered comments, not key/value pairs
     :param keys:         list to append the keys to
     :return:
     """
+    ##if python3:
+      ##  multiline=multiline.decode("utf-8")
+
     props = {}
     for line in multiline.splitlines():
         stripped_line = line.strip()
-        if stripped_line and not stripped_line.startswith(comment_char):
-            key_value = stripped_line.split(sep)
-            key = key_value[0].strip()
-            value = sep.join(key_value[1:]).strip().strip('"')
-            props[key] = value
-            if keys != None:
-                keys.append(key)
+        if python3:
+            stripped_line = line.strip()
+            if stripped_line:
+                if (isinstance(stripped_line,bytes)):
+                    stripped_line = stripped_line.decode('utf8')
+                if (isinstance(comment_char,bytes)):
+                    comment_char=comment_char.decode('utf8')             
+                if not stripped_line.startswith(comment_char):
+                    #https://python-forum.io/Thread-Diff-between-Py-2-7-and-3--14749
+                    key_value = stripped_line.split(separator)
+                    key = key_value[0].strip()
+                    value = separator.join(key_value[1:]).strip().strip('"')
+                    props[key] = value
+                    if keys != None:
+                        keys.append(key)
+        else:
+            stripped_line = line.strip()
+            if stripped_line and not stripped_line.startswith(comment_char):
+                key_value = stripped_line.split(separator)
+                key = key_value[0].strip()
+                value = separator.join(key_value[1:]).strip().strip('"')
+                props[key] = value
+                if keys != None:
+                    keys.append(key)
     return props
 
 
@@ -461,10 +488,7 @@ def login_to_garmin_connect(args):
     username = args.username if args.username else raw_input('Username: ')
     password = args.password if args.password else getpass()
 
-    if python3:
-        logging.debug("Login params: %s", urllib.parse.urlencode(DATA))
-    else:
-        logging.debug("Login params: %s", urlencode(DATA))
+    logging.debug("Login params: %s", urlencode(DATA))
 
     # Initially, we need to get a valid session cookie, so we pull the login page.
     print('Connecting to Garmin Connect...', end='')
@@ -489,7 +513,11 @@ def login_to_garmin_connect(args):
     }
 
     print('Requesting Login ticket...', end='')
-    login_response = http_req(URL_GC_LOGIN + '#', post_data, headers)
+    if python3:
+        login_response = http_req(URL_GC_LOGIN + "#", post_data, headers).decode()
+    else:
+        login_response = http_req(URL_GC_LOGIN + '#', post_data, headers)
+
     for cookie in COOKIE_JAR:
         logging.debug("Cookie %s : %s", cookie.name, cookie.value)
     # write_to_file('login-response.html', login_response, 'w')
@@ -634,7 +662,7 @@ def extract_device(device_dict, details, start_time_seconds, args, http_caller,
             device_id = device_meta['deviceId'] if present('deviceId', device_meta) else None
             if 'deviceId' not in device_meta or device_id and device_id != '0':
                 device_json = http_caller(URL_GC_DEVICE + str(device_app_inst_id))
-                file_writer(args.directory + '/device_' + str(device_app_inst_id) + '.json',
+                file_writer(args.directory + sep + 'device_' + str(device_app_inst_id) + '.json',
                             device_json, 'w',
                             start_time_seconds)
                 if not device_json:
@@ -658,14 +686,14 @@ def load_gear(activity_id, args):
         gear = json.loads(gear_json)
         if gear:
             del args # keep 'args' argument in case you need to uncomment write_to_file
-            # write_to_file(args.directory + '/activity_' + activity_id + '-gear.json',
+            # write_to_file(args.directory + sep + 'activity_' + activity_id + '-gear.json',
             #               gear_json, 'w')
             gear_display_name = gear[0]['displayName'] if present('displayName', gear[0]) else None
             gear_model = gear[0]['customMakeModel'] if present('customMakeModel', gear[0]) else None
             logging.debug("Gear for %s = %s/%s", activity_id, gear_display_name, gear_model)
             return gear_display_name if gear_display_name else gear_model
         return None
-    except urllib2.HTTPError:
+    except HTTPError:
         pass  # don't abort just for missing gear...
         # logging.info("Unable to get gear for %d", activity_id)
         # logging.exception(e)
@@ -675,6 +703,10 @@ def export_data_file(activity_id, activity_details, args, file_time, append_desc
     """
     Write the data of the activity to a file, depending on the chosen data format
     """
+    #if python3:
+    #    if (isinstance(start_time_locale,bytes)):
+    #        start_time_locale = start_time_locale.decode('utf8')
+
     # Time dependent subdirectory for activity files, e.g. '{YYYY}
     if not args.subdir is None:
         directory = resolve_path(args.directory, args.subdir, start_time_locale)
@@ -687,27 +719,27 @@ def export_data_file(activity_id, activity_details, args, file_time, append_desc
 
     # timestamp as prefix for filename
     if args.fileprefix and args.fileprefix > 0:
-        prefix = "{}-".format(start_time_locale.replace("-", "").replace(":", b"").replace(" ", "-"))
+        prefix = "{}-".format(start_time_locale.replace("-", "").replace(":", "").replace(" ", "-"))
     else:
         prefix = ""
 
     fit_filename = None
     if args.format == 'gpx':
-        data_filename = directory + '/' + prefix + 'activity_' + activity_id + append_desc + '.gpx'
+        data_filename = directory + sep + prefix + 'activity_' + activity_id + append_desc + '.gpx'
         download_url = URL_GC_GPX_ACTIVITY + activity_id + '?full=true'
         file_mode = 'w'
     elif args.format == 'tcx':
-        data_filename = directory + '/' + prefix + 'activity_' + activity_id + append_desc + '.tcx'
+        data_filename = directory + sep + prefix + 'activity_' + activity_id + append_desc + '.tcx'
         download_url = URL_GC_TCX_ACTIVITY + activity_id + '?full=true'
         file_mode = 'w'
     elif args.format == 'original':
-        data_filename = directory + '/' + prefix + 'activity_' + activity_id + append_desc + '.zip'
+        data_filename = directory + sep + prefix + 'activity_' + activity_id + append_desc + '.zip'
         # TODO not all 'original' files are in FIT format, some are GPX or TCX...
-        fit_filename = directory + '/' + prefix + 'activity_' + activity_id + append_desc + '.fit'
+        fit_filename = directory + sep + prefix + 'activity_' + activity_id + append_desc + '.fit'
         download_url = URL_GC_ORIGINAL_ACTIVITY + activity_id
         file_mode = 'wb'
     elif args.format == 'json':
-        data_filename = directory + '/' + prefix + 'activity_' + activity_id + append_desc + '.json'
+        data_filename = directory + sep + prefix + 'activity_' + activity_id + append_desc + '.json'
         file_mode = 'w'
     else:
         raise Exception('Unrecognized format.')
@@ -730,7 +762,7 @@ def export_data_file(activity_id, activity_details, args, file_time, append_desc
 
         try:
             data = http_req(download_url)
-        except urllib2.HTTPError as ex:
+        except HTTPError as ex:
             # Handle expected (though unfortunate) error codes; die on unexpected ones.
             if ex.code == 500 and args.format == 'tcx':
                 # Garmin will give an internal server error (HTTP 500) when downloading TCX files
@@ -799,11 +831,13 @@ def logging_verbosity(verbosity):
     for handler in logger.handlers:
         if isinstance(handler, logging.FileHandler):
             # this is the logfile handler
+            ###level = logging.DEBUG if verbosity > 0 else logging.INFO
             level = logging.DEBUG if verbosity and verbosity > 0 else logging.INFO
             handler.setLevel(level)
             logging.info('New logfile level: %s', logging.getLevelName(level))
         elif isinstance(handler, logging.StreamHandler):
             # this is the console handler
+            ###level = logging.DEBUG if verbosity > 1 else (logging.INFO if verbosity > 0 else logging.WARN)
             level = logging.DEBUG if verbosity and verbosity > 1 else (logging.INFO if verbosity and verbosity > 0 else logging.WARN)
             handler.setLevel(level)
             logging.debug('New console log level: %s', logging.getLevelName(level))
@@ -834,6 +868,9 @@ def main(argv):
     logging.info("Starting %s version %s, using Python version %s", argv[0], SCRIPT_VERSION, python_version())
     args = parse_arguments(argv)
     logging_verbosity(args.verbosity)
+
+    print('verbosity: ' , args.verbosity)
+    logging_verbosity(0)
 
     print('Welcome to Garmin Connect Exporter!')
 
@@ -916,14 +953,11 @@ def main(argv):
         print('Querying list of activities ' + str(total_downloaded + 1) \
               + '..' + str(total_downloaded + num_to_download) \
               + '...', end='')
+
+        logging.info('Activity list URL %s', URL_GC_LIST + urlencode(search_params))
+        result = http_req(URL_GC_LIST + urlencode(search_params))
 
-        if python3:
-            logging.info('Activity list URL %s', URL_GC_LIST + urllib.parse.urlencode(search_params))
-            result = http_req(URL_GC_LIST + urllib.parse.urlencode(search_params))
-        else:
-            logging.info('Activity list URL %s', URL_GC_LIST + urlencode(search_params))
-            result = http_req(URL_GC_LIST + urlencode(search_params))
-
+
         print(' Done.')
 
         # Persist JSON activities list
@@ -943,12 +977,12 @@ def main(argv):
                 pass
                 # Display which entry we're skipping.
                 print('Skipping Garmin Connect activity ', end='')
-                print('(' + str(current_index) + '/' + str(total_to_download) + ') ', end='')
+                print('(' + str(current_index) + sep + str(total_to_download) + ') ', end='')
                 print('[' + str(actvty['activityId']) + '] \n', end='')
             else:
                 # Display which entry we're working on.
                 print('Garmin Connect activity ', end='')
-                print('(' + str(current_index) + '/' + str(total_to_download) + ') ', end='')
+                print('(' + str(current_index) + sep + str(total_to_download) + ') ', end='')
                 print('[' + str(actvty['activityId']) + '] ', end='')
                 print(actvty['activityName'])
 
@@ -1010,7 +1044,7 @@ def main(argv):
                                       start_time_seconds)
                         samples = json.loads(activity_measurements)
                         extract['samples'] = samples
-                    except urllib2.HTTPError:
+                    except HTTPError:
                         pass # don't abort just for missing samples...
                         # logging.info("Unable to get samples for %d", actvty['activityId'])
                         # logging.exception(e)