From 0bc8cd0ff65c7074bde04963a21951911ce8429a Mon Sep 17 00:00:00 2001 From: Icemap Date: Sun, 23 Jun 2024 23:10:35 +0800 Subject: [PATCH 1/2] feat: incremental sync --- app/forum/get.py | 10 +++++++++- app/main.py | 7 ++++++- app/service/__init__.py | 17 ++++++++--------- app/service/translate.py | 9 ++++++++- 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/app/forum/get.py b/app/forum/get.py index 68d3169..290a8b1 100644 --- a/app/forum/get.py +++ b/app/forum/get.py @@ -49,6 +49,13 @@ def get_post(post_id: int) -> CnPosts: return dict_to_post(post) +def get_recent_updated_topics(page: int = 0) -> List[CnTopics]: + r = requests.get(f"https://asktug.com/latest.json?&page={page}") + topics = r.json()['topic_list']['topics'] + + return [dict_to_topic(topic) for topic in topics] + + def get_topics(page: int) -> List[CnTopics]: r = requests.get(f"https://asktug.com/latest.json?order=created&page={page}") topics = r.json()['topic_list']['topics'] @@ -85,7 +92,8 @@ def get_and_save_page_sync_progress(page: int, earliest: datetime) -> bool: # we use create time as the order to query the topics by page topics = get_topics(page) - topics = list(filter(lambda t: datetime.datetime.strptime(t.created_at, '%Y-%m-%dT%H:%M:%S.%fZ') > earliest, topics)) + topics = list( + filter(lambda t: datetime.datetime.strptime(t.created_at, '%Y-%m-%dT%H:%M:%S.%fZ') > earliest, topics)) save_page_topic_ids(topics) print(f"Got {len(topics)} topics in page {page}!") return len(topics) != 0 diff --git a/app/main.py b/app/main.py index e3184ba..8ecda64 100644 --- a/app/main.py +++ b/app/main.py @@ -1,5 +1,5 @@ from fastapi import FastAPI -from app.service.translate import translate_topic, translate_task +from app.service.translate import translate_topic, translate_task, translate_or_update_first_page from app.forum.get import save_all_topic_ids import datetime @@ -12,6 +12,11 @@ async def topic(topic_id: int): return translate_topic(topic_id) +@app.put("/page/{page_id}") +async def page(page_id: int): + return translate_or_update_first_page(page_id) + + @app.get("/sync/topic_ids") async def sync_topic_ids(): save_all_topic_ids(datetime.datetime.now() - datetime.timedelta(days=365*2)) diff --git a/app/service/__init__.py b/app/service/__init__.py index 96d0869..f7b3c29 100644 --- a/app/service/__init__.py +++ b/app/service/__init__.py @@ -13,28 +13,27 @@ # limitations under the License. import time -from app.service.translate import translate_task, task_error +from app.service.translate import translate_task, task_error, translate_or_update_first_page import threading -sleep_time = 0 +sleep_time = 60 def thread_loop(): while True: - progress = None try: - progress = translate_task() + incremental_loop() except Exception as e: - print(f"[Error] translate_task() {progress}: {e} ") - try: - task_error(progress) - except Exception as e: - print(f"[Error] task_error() {progress}: {e} ") + print(f"[Error] translate_or_update_first_page() {e} ") if sleep_time != 0: time.sleep(sleep_time) +def incremental_loop(): + translate_or_update_first_page() + + thread = threading.Thread(target=thread_loop) thread.setDaemon(True) thread.start() diff --git a/app/service/translate.py b/app/service/translate.py index e472abc..bbac5d8 100644 --- a/app/service/translate.py +++ b/app/service/translate.py @@ -14,7 +14,7 @@ import time from typing import List -from app.forum.get import get_topic_and_post_ids, get_post, get_topics +from app.forum.get import get_topic_and_post_ids, get_post, get_recent_updated_topics from app.forum.create import create_post, create_topic, update_post, update_topic, client from enum import Enum from app.db import db_query, db_exec, engine @@ -231,3 +231,10 @@ def translate_task(wait_when_none: int = 2): print(f"[{delta.seconds}s] merged {sync_progress}") return sync_progress + + +def translate_or_update_first_page(page: int = 0): + topics = get_recent_updated_topics(page) + for topic in topics: + topic_id = topic.id + translate_topic(topic_id=topic_id) From 44fd27d29062e3e3f128501e802365355f4826e4 Mon Sep 17 00:00:00 2001 From: Icemap Date: Sun, 23 Jun 2024 23:12:25 +0800 Subject: [PATCH 2/2] feat: stop loop first --- app/service/__init__.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/app/service/__init__.py b/app/service/__init__.py index f7b3c29..10edddb 100644 --- a/app/service/__init__.py +++ b/app/service/__init__.py @@ -19,22 +19,22 @@ sleep_time = 60 -def thread_loop(): - while True: - try: - incremental_loop() - except Exception as e: - print(f"[Error] translate_or_update_first_page() {e} ") - - if sleep_time != 0: - time.sleep(sleep_time) - - -def incremental_loop(): - translate_or_update_first_page() - - -thread = threading.Thread(target=thread_loop) -thread.setDaemon(True) -thread.start() -print(f"Sync task is running now.") +# def thread_loop(): +# while True: +# try: +# incremental_loop() +# except Exception as e: +# print(f"[Error] translate_or_update_first_page() {e} ") +# +# if sleep_time != 0: +# time.sleep(sleep_time) +# +# +# def incremental_loop(): +# translate_or_update_first_page() +# +# +# thread = threading.Thread(target=thread_loop) +# thread.setDaemon(True) +# thread.start() +# print(f"Sync task is running now.")