diff --git a/tasks/models.py b/tasks/models.py index dcd3c2977..3d4900870 100644 --- a/tasks/models.py +++ b/tasks/models.py @@ -93,18 +93,22 @@ def fdownload(fname): valid=validators.url(fname) if valid==True: download_url = fname - try: - resp = requests.head(download_url) - filesize_bytes=int(resp.headers['Content-Length']) - except: - filesize_bytes=self.filesize_bytes absolute_download_url=download_url try : value = download_url.split('/') name=value[-1] - fname=name + split_name=name.split('_') + download_name=f"{split_name[0]}_{split_name[-1]}" # getting human redable name ignoring unique id + fname=download_name except: fname=f"""{self.run.job.name}_{self.name}.zip""" + try: + with open(os.path.join(settings.EXPORT_DOWNLOAD_ROOT, str(self.run.uid),f"{name}_size.txt")) as f : + size=f.readline() + filesize_bytes=int(size) + except: + filesize_bytes=0 + else: try: filesize_bytes = os.path.getsize(os.path.join(settings.EXPORT_DOWNLOAD_ROOT, str(self.run.uid), fname).encode('utf-8')) diff --git a/tasks/task_runners.py b/tasks/task_runners.py index 01a45fdaf..ab42caa77 100644 --- a/tasks/task_runners.py +++ b/tasks/task_runners.py @@ -81,13 +81,13 @@ def run_task(self, job_uid=None, user=None, ondemand=True): # noqa LOG.debug('Saved task: {0}'.format(format_name)) if ondemand: - # run_task_remote(run_uid) - # db.close_old_connections() - run_task_async_ondemand.send(run_uid) + run_task_remote(run_uid) + db.close_old_connections() + # run_task_async_ondemand.send(run_uid) else: - # run_task_remote(run_uid) - # db.close_old_connections() - run_task_async_scheduled.send(run_uid) + run_task_remote(run_uid) + db.close_old_connections() + # run_task_async_scheduled.send(run_uid) return run @dramatiq.actor(max_retries=0,queue_name='default',time_limit=1000*60*60*6) @@ -283,6 +283,13 @@ def add_metadata(z,theme): if geojson : try: response_back=geojson.fetch('GeoJSON',is_hdx_export=True) + try: + for r in response_back: + size_path=join(download_dir,f"{r['download_url'].split('/')[-1]}_size.txt") + with open(size_path, 'w') as f: + f.write(str(r['zip_file_size_bytes'][0])) + except: + LOG.error("Can not write filesize to text") finish_task('geojson',response_back=response_back) except Exception as ex : raise ex @@ -305,6 +312,13 @@ def add_metadata(z,theme): if shp: try: response_back=shp.fetch('shp',is_hdx_export=True) + try: + for r in response_back: + size_path=join(download_dir,f"{r['download_url'].split('/')[-1]}_size.txt") + with open(size_path, 'w') as f: + f.write(str(r['zip_file_size_bytes'][0])) + except: + LOG.error("Can not write filesize to text") finish_task('shp',response_back=response_back) except Exception as ex: raise ex @@ -383,7 +397,15 @@ def add_metadata(z,theme): if geojson : try: response_back=geojson.fetch('GeoJSON') + try: + for r in response_back: + size_path=join(download_dir,f"{r['download_url'].split('/')[-1]}_size.txt") + with open(size_path, 'w') as f: + f.write(str(r['zip_file_size_bytes'][0])) + except: + LOG.error("Can not write filesize to text") finish_task('geojson',response_back=response_back) + except Exception as ex : raise ex @@ -396,6 +418,13 @@ def add_metadata(z,theme): if shp: try : response_back=shp.fetch('shp') + try: + for r in response_back: + size_path=join(download_dir,f"{r['download_url'].split('/')[-1]}_size.txt") + with open(size_path, 'w') as f: + f.write(str(r['zip_file_size_bytes'][0])) + except: + LOG.error("Can not write filesize to text") finish_task('shp',response_back=response_back) except Exception as ex : raise ex