Save synced filenams and skip sync next times

package.json

@@ -37,7 +37,7 @@
   "license": "MIT",
   "private": true,
   "scripts": {
-    "data:clean": "rm scripts/data.db GPX_OUT/* activities/* src/static/activities.json",
+    "data:clean": "rm scripts/data.db {GPX,TCX,FIT}_OUT/* activities/* src/static/activities.json",
     "data:download:garmin": "python3 scripts/garmin_sync.py",
     "data:analysis": "python3 scripts/gen_svg.py --from-db --type github --output assets/github.svg",
     "build": "gatsby clean && gatsby build",

scripts/config.py

@@ -18,6 +18,7 @@
 }
 SQL_FILE = os.path.join(parent, "scripts", "data.db")
 JSON_FILE = os.path.join(parent, "src", "static", "activities.json")
+SYNCED_FILE_NAME = "imported.json"
 
 # TODO: Move into nike_sync
 BASE_URL = "https://api.nike.com/sport/v3/me"

scripts/generator/__init__.py

@@ -12,6 +12,10 @@
 from .db import Activity, init_db, update_or_create_activity
 
 
+sys.path.append("..")
+import utils
+
+
 IGNORE_BEFORE_SAVING = os.getenv("IGNORE_BEFORE_SAVING", False)
 
 
@@ -78,25 +82,33 @@ def sync_from_data_dir(self, data_dir, file_suffix="gpx"):
         if not tracks:
             print("No tracks found.")
             return
+
+        synced_files = []
+
         for t in tracks:
             created = update_or_create_activity(self.session, t.to_namedtuple())
             if created:
                 sys.stdout.write("+")
+                synced_files.extend(t.file_names)
             else:
                 sys.stdout.write(".")
             sys.stdout.flush()
 
+        utils.save_synced_data_file_list(data_dir, synced_files)
+
         self.session.commit()
 
     def sync_from_app(self, app_tracks):
         if not app_tracks:
             print("No tracks found.")
             return
         print("Syncing tracks '+' means new track '.' means update tracks")
+        synced_files = []
         for t in app_tracks:
             created = update_or_create_activity(self.session, t)
             if created:
                 sys.stdout.write("+")
+                synced_files.extend(t.file_names)
             else:
                 sys.stdout.write(".")
             sys.stdout.flush()

scripts/gpxtrackposter/track_loader.py

@@ -20,6 +20,8 @@
 from .track import Track
 from .year_range import YearRange
 
+import utils
+
 log = logging.getLogger(__name__)
 
 
@@ -167,12 +169,15 @@ def _load_data_tracks(file_names, load_func=load_gpx_file):
 
     @staticmethod
     def _list_data_files(data_dir, file_suffix):
+        synced_files = utils.load_synced_file_list(data_dir)
         data_dir = os.path.abspath(data_dir)
         if not os.path.isdir(data_dir):
             raise ParameterError(f"Not a directory: {data_dir}")
         for name in os.listdir(data_dir):
             if name.startswith("."):
                 continue
+            if name in synced_files:
+                continue
             path_name = os.path.join(data_dir, name)
             if name.endswith(f".{file_suffix}") and os.path.isfile(path_name):
                 yield path_name

scripts/utils.py

@@ -1,8 +1,10 @@
 import json
 import time
 from datetime import datetime
+from config import SYNCED_FILE_NAME
 
 import pytz
+import os
 
 try:
     from rich import print
@@ -104,3 +106,37 @@ def upload_file_to_strava(client, file_name, data_type):
         print(
             f"Uploading {data_type} file: {file_name} to strava, upload_id: {r.upload_id}."
         )
+
+
+def save_synced_data_file_list(data_dir: str, file_list: list):
+    data_dir = os.path.abspath(data_dir)
+    synced_json_file = os.path.join(data_dir, SYNCED_FILE_NAME)
+
+    old_list = []
+    if os.path.exists(synced_json_file):
+        with open(synced_json_file, "r") as f:
+            try:
+                old_list = json.load(f)
+            except Exception as e:
+                print(f"json load {synced_json_file} \nerror {e}")
+                pass
+
+    with open(synced_json_file, "w") as f:
+        file_list.extend(old_list)
+
+        json.dump(file_list, f)
+
+
+def load_synced_file_list(data_dir: str):
+    data_dir = os.path.abspath(data_dir)
+    synced_json_file = os.path.join(data_dir, SYNCED_FILE_NAME)
+
+    if os.path.exists(synced_json_file):
+        with open(synced_json_file, "r") as f:
+            try:
+                return json.load(f)
+            except Exception as e:
+                print(f"json load {synced_json_file} \nerror {e}")
+                pass
+
+    return []