diff --git a/backend/scripts/sync_property_owners.py b/backend/scripts/sync_property_owners.py new file mode 100644 index 00000000..1c4785b6 --- /dev/null +++ b/backend/scripts/sync_property_owners.py @@ -0,0 +1,51 @@ +import asyncio +import os +from pathlib import Path +import sys +import logging +from typing import Optional +import signal +from dotenv import load_dotenv + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + handlers=[logging.StreamHandler()] +) +logger = logging.getLogger(__name__) + +backend_dir = Path(__file__).parent.parent +sys.path.append(str(backend_dir)) + +from src.sources.parsers.property_owners import PropertyOwnersParser +from src.config import SOURCES + +shutdown = asyncio.Event() + +def handle_shutdown(signum, frame): + logger.info(f"Received signal {signum}. Starting graceful shutdown...") + shutdown.set() + +signal.signal(signal.SIGTERM, handle_shutdown) +signal.signal(signal.SIGINT, handle_shutdown) + +async def main() -> Optional[int]: + """Sync property owners data to Cloud Storage""" + load_dotenv() + try: + property_owners = PropertyOwnersParser(SOURCES["property_owners"]) + total_synced = await property_owners.sync() + logger.info(f"Total records synced: {total_synced:,}") + return total_synced + except Exception as e: + logger.error(f"Error during sync: {str(e)}") + raise + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + logger.info("Received keyboard interrupt. Shutting down...") + except Exception as e: + logger.error(f"Unhandled exception: {str(e)}") + sys.exit(1) \ No newline at end of file diff --git a/backend/src/config.py b/backend/src/config.py index 5701cf4f..db12b988 100644 --- a/backend/src/config.py +++ b/backend/src/config.py @@ -49,5 +49,14 @@ "frequency": "static", "enabled": True, "bucket": "landbrugsdata-raw-data" + }, + "property_owners": { + "name": "Danish Property Owners", + "frequency": "weekly", + "enabled": True, + "type": "sftp", + "bucket": "landbrugsdata-raw-data", + "raw_folder": "raw", + "processed_folder": "processed" } }