From 1f5f411a6a12bffeb4d7c58e33023bd5c047c42b Mon Sep 17 00:00:00 2001 From: "Alexie (Boyong) Madolid" Date: Thu, 4 Jul 2024 16:15:02 +0800 Subject: [PATCH] temporary --- jaseci_ai_kit/jac_misc/jac_misc/scraper/scraper.py | 4 +++- jaseci_ai_kit/jac_misc/jac_misc/scraper/utils.py | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/jaseci_ai_kit/jac_misc/jac_misc/scraper/scraper.py b/jaseci_ai_kit/jac_misc/jac_misc/scraper/scraper.py index 92c501c746..fd73d9644c 100644 --- a/jaseci_ai_kit/jac_misc/jac_misc/scraper/scraper.py +++ b/jaseci_ai_kit/jac_misc/jac_misc/scraper/scraper.py @@ -48,7 +48,9 @@ async def scrape(sr: ScraperRequest): ) ) return {"task": task.get_name()} - return await async_scrape(sr.pages, sr.pre_configs, sr.detailed, sr.target) + return await async_scrape( + sr.pages, sr.pre_configs, sr.detailed, sr.target, sr.trigger_id + ) @app.post("/scrape_preview/") async def scrape_preview(spr: ScraperPreviewRequest): diff --git a/jaseci_ai_kit/jac_misc/jac_misc/scraper/utils.py b/jaseci_ai_kit/jac_misc/jac_misc/scraper/utils.py index 44d23fe7f7..eb70529ce2 100644 --- a/jaseci_ai_kit/jac_misc/jac_misc/scraper/utils.py +++ b/jaseci_ai_kit/jac_misc/jac_misc/scraper/utils.py @@ -11,15 +11,24 @@ class Stopped(Exception): @staticmethod def add(trigger_id: str): Process.queue.add(trigger_id) + print(Process.queue) @staticmethod def can_continue(trigger_id: str): + print("########### can_continue") + print(Process.queue) + print(trigger_id) + print(trigger_id in Process.queue) if can := (trigger_id in Process.queue): Process.queue.remove(trigger_id) return not can @staticmethod def has_to_stop(trigger_id: str): + print("########### has_to_stop") + print(Process.queue) + print(trigger_id) + print(trigger_id in Process.queue) if trigger_id in Process.queue: Process.queue.remove(trigger_id) raise Process.Stopped()