From e14ac269d45b556e627a214cf2d996f072b926a0 Mon Sep 17 00:00:00 2001 From: rokdd Date: Mon, 26 Jul 2021 16:11:23 +0200 Subject: [PATCH 1/3] Minor fixes + support timezonefinder Some minor fixes (global do_merge not in split function,..) and adding support for timezonefinder --- gpx-split | 10 ++++- gpxtools.py | 124 +++++++++++++++++++++++++++++++++------------------- 2 files changed, 88 insertions(+), 46 deletions(-) diff --git a/gpx-split b/gpx-split index ca2987f..8f055da 100755 --- a/gpx-split +++ b/gpx-split @@ -16,19 +16,25 @@ import sys import gpxtools +from datetime import timedelta try: filename = sys.argv[1] except Exception: - print("Usage: gpx-split ") + print("Usage: gpx-split ") print("Split tracks from to individual files") sys.exit(1) try: tz = sys.argv[2] + # if timezone is automatic we use the timezonefinder to find by coordinates + if tz == "automatic": + import timezonefinder + + tz = timezonefinder.TimezoneFinder() except Exception: tz = None -gpxtools.split(filename, tz) +gpxtools.split(filename, tz, split_time=timedelta(hours=4)) # vim: ts=4 sw=4 et : diff --git a/gpxtools.py b/gpxtools.py index 913e08e..2b0eea4 100644 --- a/gpxtools.py +++ b/gpxtools.py @@ -15,42 +15,65 @@ import os import copy from iso8601 import parse_date -from pprint import pprint,pformat -from lxml import etree # apt-get install python3-lxml +from pprint import pprint, pformat +from lxml import etree # apt-get install python3-lxml from math import radians, sin, cos, atan2, sqrt -from pytz import timezone # apt-get install python3-tz +from pytz import timezone # apt-get install python3-tz +from datetime import timedelta -#ns = '{http://www.topografix.com/GPX/1/0}' +# ns = '{http://www.topografix.com/GPX/1/0}' ns = '{http://www.topografix.com/GPX/1/1}' do_merge = False base_file = '' + # Get unique dates from all tracks -def get_dates(tree, ns): +def get_dates(tree, ns, tz, split_time): dates = set() for trk in tree.iterchildren(ns + 'trk'): - dates.add(get_date(trk, ns)) + dates.add(get_date(trk, ns, tz, split_time=split_time)) return dates + def get_name(trk, ns): return trk.findtext(ns + 'name') or '' -def get_date(trk, ns, tz=None): + +def get_date(trk, ns, tz=None, split_time=timedelta()): gpxtime = trk.findtext(ns + 'trkseg/' + ns + 'trkpt/' + ns + 'time') + tzobj = None + + # use automatic timezone from coordinates + if type(tz).__name__ == "TimezoneFinder": + lat = trk.find(ns + 'trkseg/' + ns + 'trkpt[@lat]') + lng = lat.get('lon') + lat = lat.get('lat') + # From the lat/long, get the tz-database-style time zone name (e.g. 'America/Vancouver') or None + timezone_str = tz.certain_timezone_at(lat=float(lat), lng=float(lng)) + + if timezone_str is None: + # print("Could not determine the time zone") + pass + else: + # Display the current time in that time zone + tzobj = timezone(timezone_str) - # parse_date returns a datetime.datetime - if tz is not None: + # use timezone which is globally specified + elif tz is not None: tzobj = timezone(tz) - return parse_date(gpxtime).astimezone(tzobj).date() - return parse_date(gpxtime).date() + # parse_date returns a datetime.datetime by timezone if isset + if tzobj is not None: + return (parse_date(gpxtime) - split_time).astimezone(tzobj).date() + + return (parse_date(gpxtime) - split_time).parse_date(gpxtime).date() + def get_datetime(trk, ns, tz=None): gpxtime = trk.findtext(ns + 'trkseg/' + ns + 'trkpt/' + ns + 'time') if gpxtime is not None: - # parse_date returns a datetime.datetime if tz is not None: tzobj = timezone(tz) @@ -58,49 +81,57 @@ def get_datetime(trk, ns, tz=None): return parse_date(gpxtime) + def get_numpts(trk, ns): return len(trk.findall(ns + 'trkseg/' + ns + 'trkpt')) + def get_numtrk(root, ns): return len(root.findall(ns + 'trk')) + def get_numwpt(root, ns): return len(root.findall(ns + 'wpt')) + def get_numrte(root, ns): return len(root.findall(ns + 'rte')) + def get_numrtept(rte, ns): return len(rte.findall(ns + 'rtept')) + def distance(lat1, lon1, lat2, lon2): - radius = 6371000 # meter + radius = 6371000 # meter lat1, lon1, lat2, lon2 = map(radians, [lat1, lon1, lat2, lon2]) dlat = lat2 - lat1 dlon = lon2 - lon1 - a = sin(dlat/2) * sin(dlat/2) + cos(lat1) \ - * cos(lat2) * sin(dlon/2) * sin(dlon/2) - c = 2 * atan2(sqrt(a), sqrt(1-a)) + a = sin(dlat / 2) * sin(dlat / 2) + cos(lat1) \ + * cos(lat2) * sin(dlon / 2) * sin(dlon / 2) + c = 2 * atan2(sqrt(a), sqrt(1 - a)) d = radius * c return d + def make_filename(d, dir='.'): global do_merge, base_file do_merge = False n = 0 fn = "%s-%03d.gpx" % (d.isoformat(), n) - fname = os.path.join (dir, fn) + fname = os.path.join(dir, fn) base_file = fname while os.path.exists(fname): n += 1 fn = "%s-%03d.gpx" % (d.isoformat(), n) - fname = os.path.join (dir, fn) + fname = os.path.join(dir, fn) do_merge = True return fname -def split(filename, tz): - global ns + +def split(filename, tz, split_time=timedelta()): + global ns, do_merge try: tree0 = etree.parse(filename) @@ -113,7 +144,7 @@ def split(filename, tz): # Get the XML namespace from the tree ns = "{%s}" % root.nsmap[None] - dates = get_dates(root, ns) + dates = get_dates(root, ns, tz, split_time=split_time) # Iterate over the dates and remove non-matching tracks from the tree for d in dates: @@ -128,37 +159,41 @@ def split(filename, tz): root.remove(trk) continue - name = get_name(trk, ns) - trackdate = get_date(trk, ns, tz) + name = get_name(trk, ns) + + trackdate = get_date(trk, ns, tz, split_time=split_time) if trackdate != d: - #print("%-25s: date mismatch, removing %s" % (fname, name)) + # print("%-25s: date mismatch, removing %s" % (fname, name)) root.remove(trk) elif name in tracks: oldnum = tracks[name]['numpts'] newnum = get_numpts(trk, ns) if oldnum >= newnum: - print("%-25s: DUPLICATE %s (track points: old=%d, new=%d) -> removing" % (fname, name, oldnum, newnum)) + print("%-25s: DUPLICATE %s (track points: old=%d, new=%d) -> removing" % ( + fname, name, oldnum, newnum)) root.remove(trk) else: # newnum > oldnum. Old track should be removed and this one kept. - print("%-25s: duplicate %s (track points differ, old=%d new=%d) -> keeping and removing the old one" % \ + print( + "%-25s: duplicate %s (track points differ, old=%d new=%d) -> keeping and removing the old one" % \ (fname, name, oldnum, newnum)) root.remove(tracks[name]['track']) - tracks[name] = { 'numpts': newnum, 'track': trk } + tracks[name] = {'numpts': newnum, 'track': trk} else: numpts = get_numpts(trk, ns) - tracks[name] = { 'numpts': numpts, 'track': trk } - #print("%-25s: keeping %s" % (fname, name)) + tracks[name] = {'numpts': numpts, 'track': trk} + # print("%-25s: keeping %s" % (fname, name)) print("%-25s: writing file" % fname) - tree.write(fname, xml_declaration = True, encoding='utf-8') + tree.write(fname, xml_declaration=True, encoding='utf-8') # Merge if necessary if do_merge: print("%-25s: starting merge into %s" % (fname, base_file)) merge(base_file, fname, False) + # Merge tracks from file2 into file1. # On duplicate names, keep the track with the most track points. @@ -175,32 +210,32 @@ def merge(file1, file2, interactive=True): ns1 = "{%s}" % root1.nsmap[None] ns2 = "{%s}" % root2.nsmap[None] - #print("%-25s: Namespace: %s" % (file1, ns1)) - #print("%-25s: Namespace: %s" % (file2, ns2)) + # print("%-25s: Namespace: %s" % (file1, ns1)) + # print("%-25s: Namespace: %s" % (file2, ns2)) modified = False # Analyze the first file tracks1 = {} for trk in root1.iterchildren(ns1 + 'trk'): - name = get_name(trk, ns1) + name = get_name(trk, ns1) numpts = get_numpts(trk, ns1) if not name in tracks1: - tracks1[name] = { 'numpts': numpts, 'track': trk } + tracks1[name] = {'numpts': numpts, 'track': trk} else: print("Track '%s' already seen in '%s'. File contains dupes?" % (name, file1)) oldnum = tracks[name]['numpts'] if numpts > oldnum: print("Duplicate '%s' replacing old track (old=%d, new=%d points)" % (name, oldnum, numpts)) - tracks1[name] = { 'numpts': numpts, 'track': trk } + tracks1[name] = {'numpts': numpts, 'track': trk} for trk in root2.iterchildren(ns2 + 'trk'): - name = get_name(trk, ns2) + name = get_name(trk, ns2) numpts = get_numpts(trk, ns2) if not name in tracks1: print("%-25s: appending track '%s'" % (file1, name)) root1.append(copy.deepcopy(trk)) - tracks1[name] = { 'numpts': numpts, 'track': trk } + tracks1[name] = {'numpts': numpts, 'track': trk} modified = True else: oldpts = tracks1[name]['numpts'] @@ -208,33 +243,34 @@ def merge(file1, file2, interactive=True): print("%-25s: replacing track '%s'. oldpts=%d. newpts=%d" % (file2, name, oldpts, numpts)) root1.remove(tracks1[name]['track']) root1.append(copy.deepcopy(trk)) - tracks1[name] = { 'numpts': numpts, 'track': trk } + tracks1[name] = {'numpts': numpts, 'track': trk} modified = True - #else: + # else: # print("%-25s: skipping track '%s'. oldpts=%d. newpts=%d" % (file2, name, oldpts, numpts)) if modified: - yn=False + yn = False if interactive: - while yn not in ['y','n']: + while yn not in ['y', 'n']: yn = input("Overwrite '%s' and remove '%s' ? (y/n)" % (file1, file2)) if not interactive or yn == 'y': print("%-25s: Overwriting file" % file1) - tree1.write(file1, xml_declaration = True, encoding='utf-8') + tree1.write(file1, xml_declaration=True, encoding='utf-8') print("%-25s: Removing file" % file2) os.remove(file2) else: print("%-25s: No changes to write to file" % file1) - yn=False + yn = False if interactive: - while yn not in ['y','n']: + while yn not in ['y', 'n']: yn = raw_input("Remove '%s' ? (y/n)" % file2) if not interactive or yn == 'y': print("%-25s: Removing file" % file2) os.remove(file2) + def info(filename, tz): global ns From 1877f781514e8f260428cbb0c0cf3fc24f9c720c Mon Sep 17 00:00:00 2001 From: rokdd Date: Mon, 26 Jul 2021 16:17:35 +0200 Subject: [PATCH 2/3] Update gpx-split timedelta as argument --- gpx-split | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/gpx-split b/gpx-split index 8f055da..fc6900c 100755 --- a/gpx-split +++ b/gpx-split @@ -21,7 +21,7 @@ from datetime import timedelta try: filename = sys.argv[1] except Exception: - print("Usage: gpx-split ") + print("Usage: gpx-split ") print("Split tracks from to individual files") sys.exit(1) @@ -30,11 +30,15 @@ try: # if timezone is automatic we use the timezonefinder to find by coordinates if tz == "automatic": import timezonefinder - tz = timezonefinder.TimezoneFinder() except Exception: tz = None -gpxtools.split(filename, tz, split_time=timedelta(hours=4)) +try: + td = timedelta(hours=sys.argv[3]) +except Exception: + td = timedelta() + +gpxtools.split(filename, tz, split_time=td) # vim: ts=4 sw=4 et : From 5e64fb7f2c65fc37e068172fc001fa538052f2c0 Mon Sep 17 00:00:00 2001 From: rokdd Date: Tue, 27 Jul 2021 21:09:42 +0200 Subject: [PATCH 3/3] Option to not remove trks when duplicated --- gpx-split | 2 ++ gpxtools.py | 15 +++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/gpx-split b/gpx-split index fc6900c..292bc06 100755 --- a/gpx-split +++ b/gpx-split @@ -18,6 +18,8 @@ import sys import gpxtools from datetime import timedelta + + try: filename = sys.argv[1] except Exception: diff --git a/gpxtools.py b/gpxtools.py index 2b0eea4..03d2209 100644 --- a/gpxtools.py +++ b/gpxtools.py @@ -25,6 +25,7 @@ ns = '{http://www.topografix.com/GPX/1/1}' do_merge = False +do_duplicate_search=False base_file = '' @@ -131,7 +132,7 @@ def make_filename(d, dir='.'): def split(filename, tz, split_time=timedelta()): - global ns, do_merge + global ns, do_merge, do_duplicate_search try: tree0 = etree.parse(filename) @@ -169,7 +170,17 @@ def split(filename, tz, split_time=timedelta()): elif name in tracks: oldnum = tracks[name]['numpts'] newnum = get_numpts(trk, ns) - if oldnum >= newnum: + # sometimes there is no name and a huge file.. then just merge the segments.. + if do_duplicate_search == False: + print("%-25s: DUPLICATES %s (track points: old=%d + new=%d) -> concating" % ( + fname, name, oldnum, newnum)) + for trkseg in trk.iterchildren(): + tracks[name]['track'].append(trkseg) + tracks[name]['numpts']+=newnum + print("%-25s: DUPLICATES %s (track points: old=%d + new=%d) = concated %d" % ( + fname, name, oldnum, newnum, len(tracks[name]['track'].findall('.//'+ns+'trkpt')))) + root.remove(trk) + elif oldnum >= newnum: print("%-25s: DUPLICATE %s (track points: old=%d, new=%d) -> removing" % ( fname, name, oldnum, newnum)) root.remove(trk)