Merge pull request #209 from martincollignon/fix/add-parsers-cleanup

feat: add cleanup of existing files at start of sync for cadastral an…
martincollignon · Dec 15, 2024 · 59e6c6f · 59e6c6f
2 parents d46b833 + d0a5af8
commit 59e6c6f
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 0 deletions.
diff --git a/backend/src/sources/parsers/agricultural_fields.py b/backend/src/sources/parsers/agricultural_fields.py
@@ -158,6 +158,16 @@ async def sync(self):
         self.features_processed = 0
         self.is_sync_complete = False
 
+        # Add cleanup
+        for blob_path in [
+            'raw/agricultural_fields/working.parquet',
+            'raw/agricultural_fields/current.parquet'
+        ]:
+            blob = self.bucket.blob(blob_path)
+            if blob.exists():
+                blob.delete()
+                logger.info(f"Deleted existing file: {blob_path}")
+
         try:
             conn = aiohttp.TCPConnector(limit=self.max_concurrent, ssl=self.ssl_context)
             async with aiohttp.ClientSession(timeout=self.timeout_config, connector=conn) as session:

diff --git a/backend/src/sources/parsers/cadastral.py b/backend/src/sources/parsers/cadastral.py
@@ -377,6 +377,16 @@ async def sync(self):
         logger.info("Starting cadastral sync...")
         self.is_sync_complete = False
 
+        # Add cleanup
+        for blob_path in [
+            'raw/cadastral/working.parquet',
+            'raw/cadastral/current.parquet'
+        ]:
+            blob = self.bucket.blob(blob_path)
+            if blob.exists():
+                blob.delete()
+                logger.info(f"Deleted existing file: {blob_path}")
+
         try:
             async with aiohttp.ClientSession(timeout=self.total_timeout_config) as session:
                 total_features = await self._get_total_count(session)