Merge pull request geosolutions-it#156 from randomorder/s2_product_zi…

…p_ingestion Move S2 DAG configuration to config file
randomorder · Aug 31, 2017 · 6b16b47 · 6b16b47
2 parents dd51159 + fa1ce82
commit 6b16b47
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 8 deletions.
diff --git a/airflow/dags/sentinel2/S2_Download.py b/airflow/dags/sentinel2/S2_Download.py
@@ -58,9 +58,9 @@
 
 # Archive Sentinel-2 RSYNC Task Operator
 archive_task = RSYNCOperator(task_id="sentinel2_upload_granules", 
-                             host = "localhost", 
-                             remote_usr = "moataz", 
-                             ssh_key_file = "/usr/local/airflow/id_rsa", 
+                             host = sentinel2_config['rsync_hostname'],
+                             remote_usr = sentinel2_config['rsync_username'],
+                             ssh_key_file = sentinel2_config['rsync_ssh_key'],
                              remote_dir = sentinel2_config['granules_upload_dir'], 
                              xk_pull_dag_id = 'S2_Download', 
                              xk_pull_task_id = 'dhus_download_task', 
@@ -69,9 +69,9 @@
 
 # Archive Sentinel-2 RSYNC with .prj and .tfw files Task Operator
 archive_tfwprj_task = RSYNCOperator(task_id="sentinel2_upload_granules_with_tfwprj", 
-                             host = "localhost", 
-                             remote_usr = "moataz", 
-                             ssh_key_file = "/usr/local/airflow/id_rsa", 
+                             host = sentinel2_config['rsync_hostname'],
+                             remote_usr = sentinel2_config['rsync_username'],
+                             ssh_key_file = sentinel2_config['rsync_ssh_key'],
                              remote_dir = sentinel2_config['granules_upload_dir'], 
                              xk_pull_dag_id = 'S2_Download', 
                              xk_pull_task_id = 'dhus_metadata_task', 
@@ -95,7 +95,7 @@
 generated_files_list = ['product/product.json','product/granules.json','product/thumbnail.jpeg']
 
 product_zip_task = Sentinel2ProductZipOperator(task_id = 'dhus_product_zip_task',
-                                               target_dir = "/var/data/download/Sentinel-2",
+                                               target_dir = sentinel2_config['product_zip_target_dir'],
                                                generated_files = generated_files_list,
                                                placeholders = placeholders_list,
                                                dag = dag)

diff --git a/airflow/dags/sentinel2/config.py b/airflow/dags/sentinel2/config.py
@@ -23,5 +23,8 @@
     'filename': 'S2A_MSIL1C*',
     'download_dir': os.path.join(download_base_dir, "Sentinel-2"),
     'granules_upload_dir': "/var/data/download/uploads",
-    'bands_res':{'10':("B02","B03","B04","B08"),'20':("B05","B06","B07","B8A","B11","B12"),'60':("B01","B09","B10")}
+    'bands_res':{'10':("B02","B03","B04","B08"),'20':("B05","B06","B07","B8A","B11","B12"),'60':("B01","B09","B10")},
+    'rsync_hostname': 'geoserver.cloudsdi.geo-solutions.it',
+    'rsync_username': 'ec2-user',
+    'rsync_ssh_key' : '/usr/local/airflow/id_rsa'
 }