Fix parallelization select

- Set a log until parallel develop is finished.
mjanez · Sep 13, 2023 · 6ca448d · 6ca448d
1 parent cb7f6e7
commit 6ca448d
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 12 deletions.
diff --git a/ogc2ckan/config/ckan_config.py b/ogc2ckan/config/ckan_config.py
@@ -26,9 +26,9 @@ def __init__(self):
         self.default_license = os.environ.get('DEFAULT_LICENSE', OGC2CKAN_CKANINFO_CONFIG['default_license'])
         self.default_license_id = os.environ.get('DEFAULT_LICENSE_ID', OGC2CKAN_CKANINFO_CONFIG['default_license_id'])
         self.ckan_harvester = OGC2CKAN_HARVESTER_CONFIG
-        self.ssl_unverified_mode = os.environ.get('SSL_UNVERIFIED_MODE', OGC2CKAN_CKANINFO_CONFIG['ssl_unverified_mode'])
-        self.metadata_distributions = os.environ.get('METADATA_DISTRIBUTIONS', OGC2CKAN_CKANINFO_CONFIG['metadata_distributions'])
-        self.parallelization = os.environ.get('PARALLELIZATION', OGC2CKAN_CKANINFO_CONFIG['parallelization'])
+        self.ssl_unverified_mode = True if os.environ.get('SSL_UNVERIFIED_MODE') == 'True' else OGC2CKAN_CKANINFO_CONFIG['ssl_unverified_mode']
+        self.metadata_distributions = True if os.environ.get('METADATA_DISTRIBUTIONS') == 'True' else OGC2CKAN_CKANINFO_CONFIG['metadata_distributions']
+        self.parallelization = True if os.environ.get('PARALLELIZATION') == 'True' else OGC2CKAN_CKANINFO_CONFIG['parallelization']
         self.dir3_soup = self.get_dir3_soup()
         self.ckan_dataset_schema = os.environ.get('CKAN_DATASET_SCHEMA', OGC2CKAN_CKANINFO_CONFIG['ckan_dataset_schema'])
 

diff --git a/ogc2ckan/ogc2ckan.py b/ogc2ckan/ogc2ckan.py
@@ -116,15 +116,17 @@ def start_harvesting(config_file):
         logging.info(f"{log_module}:CKAN_URL: {ckan_info.ckan_site_url}")
 
         try:
-            if harvest_servers is not None:
-                if ckan_info.parallelization is True:
-                    #TODO: Fix multicore parallel processing
-                    parallel_count = Parallel(n_jobs=processes)(delayed(launch_harvest)(harvest_server=endpoint, ckan_info=ckan_info) for endpoint in harvest_servers)
-                    new_records.append(sum(i[0] for i in parallel_count))
-                else:
-                    for endpoint in harvest_servers:
-                        harvester = launch_harvest(harvest_server=endpoint, ckan_info=ckan_info)
-                        new_records.append(harvester.ckan_dataset_count)
+            if harvest_servers is not None and ckan_info.parallelization is True:
+                #TODO: Fix multicore parallel processing
+                logging.warning(f'{log_module}:Parallel processing is not implemented yet.')
+                '''
+                parallel_count = Parallel(n_jobs=processes)(delayed(launch_harvest)(harvest_server=endpoint, ckan_info=ckan_info) for endpoint in harvest_servers)
+                new_records.append(sum(i[0] for i in parallel_count))
+                '''
+            elif harvest_servers and ckan_info.parallelization is False:
+                for endpoint in harvest_servers:
+                    harvester = launch_harvest(harvest_server=endpoint, ckan_info=ckan_info)
+                    new_records.append(harvester.ckan_dataset_count)
         except Exception as e:
             logging.error(f"{log_module}:Check invalid 'type' and 'active: True' in 'harvest_servers/{{my-harvest-server}}'at {config_file} Error: {e}")
             new_records = 0