Save synced filenams and skip sync next times (#454)

* feat: add synced file name in imported.json, then skip those files next time

* chore: add TCX_OUT\FIT_OUT dir in `yarn data:clean` cli

* reformated

* fix: add try except when load json

* save data files to cache

* Revert "save data files to cache"

This reverts commit cd3bcf9.

* fix: new synced_data_file_logger.py

* reformatted

* save GPX/FIT/TCX, activities, accets, data.db to github cache

* Revert "save GPX/FIT/TCX, activities, accets, data.db to github cache"

This reverts commit 55f48cc.

* resolve the code review issues.

* fix: put all types data files record in one json

* add update track data file names in the synced record file

package.json

@@ -37,7 +37,7 @@
   "license": "MIT",
   "private": true,
   "scripts": {
-    "data:clean": "rm scripts/data.db GPX_OUT/* activities/* src/static/activities.json",
+    "data:clean": "rm scripts/data.db {GPX,TCX,FIT}_OUT/* activities/* src/static/activities.json",
     "data:download:garmin": "python3 scripts/garmin_sync.py",
     "data:analysis": "python3 scripts/gen_svg.py --from-db --type github --output assets/github.svg",
     "build": "gatsby clean && gatsby build",

scripts/config.py

@@ -18,6 +18,7 @@
 }
 SQL_FILE = os.path.join(parent, "scripts", "data.db")
 JSON_FILE = os.path.join(parent, "src", "static", "activities.json")
+SYNCED_FILE = os.path.join(parent, "imported.json")
 
 # TODO: Move into nike_sync
 BASE_URL = "https://api.nike.com/sport/v3/me"

scripts/generator/__init__.py

@@ -11,6 +11,8 @@
 
 from .db import Activity, init_db, update_or_create_activity
 
+from synced_data_file_logger import save_synced_data_file_list
+
 
 IGNORE_BEFORE_SAVING = os.getenv("IGNORE_BEFORE_SAVING", False)
 
@@ -78,27 +80,35 @@ def sync_from_data_dir(self, data_dir, file_suffix="gpx"):
         if not tracks:
             print("No tracks found.")
             return
+
+        synced_files = []
+
         for t in tracks:
             created = update_or_create_activity(self.session, t.to_namedtuple())
             if created:
                 sys.stdout.write("+")
             else:
                 sys.stdout.write(".")
+            synced_files.extend(t.file_names)
             sys.stdout.flush()
 
+        save_synced_data_file_list(synced_files)
+
         self.session.commit()
 
     def sync_from_app(self, app_tracks):
         if not app_tracks:
             print("No tracks found.")
             return
         print("Syncing tracks '+' means new track '.' means update tracks")
+        synced_files = []
         for t in app_tracks:
             created = update_or_create_activity(self.session, t)
             if created:
                 sys.stdout.write("+")
             else:
                 sys.stdout.write(".")
+            synced_files.extend(t.file_names)
             sys.stdout.flush()
 
         self.session.commit()

scripts/gpxtrackposter/track_loader.py

@@ -20,6 +20,8 @@
 from .track import Track
 from .year_range import YearRange
 
+from synced_data_file_logger import load_synced_file_list
+
 log = logging.getLogger(__name__)
 
 
@@ -167,12 +169,15 @@ def _load_data_tracks(file_names, load_func=load_gpx_file):
 
     @staticmethod
     def _list_data_files(data_dir, file_suffix):
+        synced_files = load_synced_file_list()
         data_dir = os.path.abspath(data_dir)
         if not os.path.isdir(data_dir):
             raise ParameterError(f"Not a directory: {data_dir}")
         for name in os.listdir(data_dir):
             if name.startswith("."):
                 continue
+            if name in synced_files:
+                continue
             path_name = os.path.join(data_dir, name)
             if name.endswith(f".{file_suffix}") and os.path.isfile(path_name):
                 yield path_name

scripts/synced_data_file_logger.py

@@ -0,0 +1,24 @@
+import os
+from config import SYNCED_FILE
+import json
+
+
+def save_synced_data_file_list(file_list: list):
+    old_list = load_synced_file_list()
+
+    with open(SYNCED_FILE, "w") as f:
+        file_list.extend(old_list)
+
+        json.dump(file_list, f)
+
+
+def load_synced_file_list():
+    if os.path.exists(SYNCED_FILE):
+        with open(SYNCED_FILE, "r") as f:
+            try:
+                return json.load(f)
+            except Exception as e:
+                print(f"json load {SYNCED_FILE} \nerror {e}")
+                pass
+
+    return []