Merge pull request #212 from martincollignon/temp/file-comparison

Temp/file comparison

backend/scripts/sync_property_owners.py

@@ -0,0 +1,51 @@
+import asyncio
+import os
+from pathlib import Path
+import sys
+import logging
+from typing import Optional
+import signal
+from dotenv import load_dotenv
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[logging.StreamHandler()]
+)
+logger = logging.getLogger(__name__)
+
+backend_dir = Path(__file__).parent.parent
+sys.path.append(str(backend_dir))
+
+from src.sources.parsers.property_owners import PropertyOwnersParser
+from src.config import SOURCES
+
+shutdown = asyncio.Event()
+
+def handle_shutdown(signum, frame):
+    logger.info(f"Received signal {signum}. Starting graceful shutdown...")
+    shutdown.set()
+
+signal.signal(signal.SIGTERM, handle_shutdown)
+signal.signal(signal.SIGINT, handle_shutdown)
+
+async def main() -> Optional[int]:
+    """Sync property owners data to Cloud Storage"""
+    load_dotenv()
+    try:
+        property_owners = PropertyOwnersParser(SOURCES["property_owners"])
+        total_synced = await property_owners.sync()
+        logger.info(f"Total records synced: {total_synced:,}")
+        return total_synced
+    except Exception as e:
+        logger.error(f"Error during sync: {str(e)}")
+        raise
+
+if __name__ == "__main__":
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        logger.info("Received keyboard interrupt. Shutting down...")
+    except Exception as e:
+        logger.error(f"Unhandled exception: {str(e)}")
+        sys.exit(1) 

backend/src/config.py

@@ -49,5 +49,14 @@
         "frequency": "static",
         "enabled": True,
         "bucket": "landbrugsdata-raw-data"
+    },
+    "property_owners": {
+        "name": "Danish Property Owners",
+        "frequency": "weekly",
+        "enabled": True,
+        "type": "sftp",
+        "bucket": "landbrugsdata-raw-data",
+        "raw_folder": "raw",
+        "processed_folder": "processed"
     }
 }