From 085a585503ce2761bce09ce9ef4d33b02b7eca7c Mon Sep 17 00:00:00 2001 From: Agassi Date: Sun, 23 Jul 2023 09:41:55 +0800 Subject: [PATCH] Save synced filenams and skip sync next times (#454) * feat: add synced file name in imported.json, then skip those files next time * chore: add TCX_OUT\FIT_OUT dir in `yarn data:clean` cli * reformated * fix: add try except when load json * save data files to cache * Revert "save data files to cache" This reverts commit cd3bcf93725d0f860fe901abeaf2cf974e0d5af7. * fix: new synced_data_file_logger.py * reformatted * save GPX/FIT/TCX, activities, accets, data.db to github cache * Revert "save GPX/FIT/TCX, activities, accets, data.db to github cache" This reverts commit 55f48cc7bc1d1598c63c5ab5ee80c28a8d598e59. * resolve the code review issues. * fix: put all types data files record in one json * add update track data file names in the synced record file --- package.json | 2 +- scripts/config.py | 1 + scripts/generator/__init__.py | 10 ++++++++++ scripts/gpxtrackposter/track_loader.py | 5 +++++ scripts/synced_data_file_logger.py | 24 ++++++++++++++++++++++++ 5 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 scripts/synced_data_file_logger.py diff --git a/package.json b/package.json index fe61e0e889b..a5a6ca3e630 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ "license": "MIT", "private": true, "scripts": { - "data:clean": "rm scripts/data.db GPX_OUT/* activities/* src/static/activities.json", + "data:clean": "rm scripts/data.db {GPX,TCX,FIT}_OUT/* activities/* src/static/activities.json", "data:download:garmin": "python3 scripts/garmin_sync.py", "data:analysis": "python3 scripts/gen_svg.py --from-db --type github --output assets/github.svg", "build": "gatsby clean && gatsby build", diff --git a/scripts/config.py b/scripts/config.py index ef4d3669029..6276c647be2 100644 --- a/scripts/config.py +++ b/scripts/config.py @@ -18,6 +18,7 @@ } SQL_FILE = os.path.join(parent, "scripts", "data.db") JSON_FILE = os.path.join(parent, "src", "static", "activities.json") +SYNCED_FILE = os.path.join(parent, "imported.json") # TODO: Move into nike_sync BASE_URL = "https://api.nike.com/sport/v3/me" diff --git a/scripts/generator/__init__.py b/scripts/generator/__init__.py index 3684ae3269d..6c10c603547 100644 --- a/scripts/generator/__init__.py +++ b/scripts/generator/__init__.py @@ -11,6 +11,8 @@ from .db import Activity, init_db, update_or_create_activity +from synced_data_file_logger import save_synced_data_file_list + IGNORE_BEFORE_SAVING = os.getenv("IGNORE_BEFORE_SAVING", False) @@ -78,14 +80,20 @@ def sync_from_data_dir(self, data_dir, file_suffix="gpx"): if not tracks: print("No tracks found.") return + + synced_files = [] + for t in tracks: created = update_or_create_activity(self.session, t.to_namedtuple()) if created: sys.stdout.write("+") else: sys.stdout.write(".") + synced_files.extend(t.file_names) sys.stdout.flush() + save_synced_data_file_list(synced_files) + self.session.commit() def sync_from_app(self, app_tracks): @@ -93,12 +101,14 @@ def sync_from_app(self, app_tracks): print("No tracks found.") return print("Syncing tracks '+' means new track '.' means update tracks") + synced_files = [] for t in app_tracks: created = update_or_create_activity(self.session, t) if created: sys.stdout.write("+") else: sys.stdout.write(".") + synced_files.extend(t.file_names) sys.stdout.flush() self.session.commit() diff --git a/scripts/gpxtrackposter/track_loader.py b/scripts/gpxtrackposter/track_loader.py index 007c3b2f24e..b63cb9ba969 100644 --- a/scripts/gpxtrackposter/track_loader.py +++ b/scripts/gpxtrackposter/track_loader.py @@ -20,6 +20,8 @@ from .track import Track from .year_range import YearRange +from synced_data_file_logger import load_synced_file_list + log = logging.getLogger(__name__) @@ -167,12 +169,15 @@ def _load_data_tracks(file_names, load_func=load_gpx_file): @staticmethod def _list_data_files(data_dir, file_suffix): + synced_files = load_synced_file_list() data_dir = os.path.abspath(data_dir) if not os.path.isdir(data_dir): raise ParameterError(f"Not a directory: {data_dir}") for name in os.listdir(data_dir): if name.startswith("."): continue + if name in synced_files: + continue path_name = os.path.join(data_dir, name) if name.endswith(f".{file_suffix}") and os.path.isfile(path_name): yield path_name diff --git a/scripts/synced_data_file_logger.py b/scripts/synced_data_file_logger.py new file mode 100644 index 00000000000..de9583b59d5 --- /dev/null +++ b/scripts/synced_data_file_logger.py @@ -0,0 +1,24 @@ +import os +from config import SYNCED_FILE +import json + + +def save_synced_data_file_list(file_list: list): + old_list = load_synced_file_list() + + with open(SYNCED_FILE, "w") as f: + file_list.extend(old_list) + + json.dump(file_list, f) + + +def load_synced_file_list(): + if os.path.exists(SYNCED_FILE): + with open(SYNCED_FILE, "r") as f: + try: + return json.load(f) + except Exception as e: + print(f"json load {SYNCED_FILE} \nerror {e}") + pass + + return []