add logs

dataesr · Mar 28, 2024 · 8818b83 · 8818b83
1 parent a149b79
commit 8818b83
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 33 deletions.
diff --git a/ecorda/api.py b/ecorda/api.py
@@ -12,8 +12,12 @@
 
 @retry(delay=50, tries=25)
 def get_page(url):
-    r = requests.get(url, headers=get_headers())
-    return r.json()['data']
+    try:
+        r = requests.get(url, headers=get_headers())
+        return r.json()['data']
+    except Exception as e:
+        logger.error(f'Error fetching {url}: {e}')
+        raise e
 
 
 def base_api(base=None, framework=None, url_ue=None):
@@ -23,30 +27,29 @@ def base_api(base=None, framework=None, url_ue=None):
 
     tot_records = r.json().get("metadata").get("totalRecords")
     page_max = r.json().get("metadata").get("lastPage")
-    last_page_size = tot_records % PAGE_SIZE
+    # last_page_size = tot_records % PAGE_SIZE
+
+    started_at = time.strftime("%H:%M:%S")
 
     result = []
-    if r.json().get("metadata").get("totalRecords") == 0:
-        result = []
-        logger.debug(f'***{base} Empty***')
-    if tot_records <= PAGE_SIZE:
-        result = []
+    if tot_records != 0 and tot_records <= PAGE_SIZE:
         url1 = url_ue + base + "?framework=" + \
             framework + "&page=1&size=" + str(tot_records)
         time.sleep(0.2)
         result += get_page(url1)
-        logger.debug('1')
-    else:
-        logger.debug(
-            f'***{base} -> totalRecords:{r.json().get("metadata").get("totalRecords")}, totalPage:{page_max}, start request:{time.strftime("%H:%M:%S")}')
+        # logger.debug('1')
+    if tot_records > PAGE_SIZE:
         for page in range(1, page_max+1):
             url1 = url_ue + base + "?framework=" + framework + \
                 "&page=" + str(page) + "&size=" + SIZE
             time.sleep(0.2)
             result += get_page(url1)
-            logger.debug(f'{page}')
-        logger.debug(
-            f'Total records: {tot_records} // Nombre de résultats: {len(result)}')
+            # logger.debug(f'{page}')
+        # logger.debug(
+        #     f'Total records: {tot_records} // Nombre de résultats: {len(result)}')
+
+    logger.debug(
+        f'***{base} -> totalRecords:{tot_records}, totalPage:{page_max}, Nombre de résultats: {len(result)}, start:{started_at}, end:{time.strftime("%H:%M:%S")}***')
 
     if tot_records != len(result):
         raise Exception('matching records faild')

diff --git a/ecorda/load_all.py b/ecorda/load_all.py
@@ -1,4 +1,5 @@
-import os, json
+import os
+import json
 import pandas as pd
 from ecorda.api import base_api
 from utils.logger import get_logger
@@ -7,22 +8,27 @@
 
 os.system(f'mkdir -p /eCorda_data')
 
+
 def extraction_all(framework, liste_datas, url_ue):
-    counter=0
-    datas_load=[]
-    datas_empty=[]
-    for b in liste_datas:   
-        result = base_api(base=b, framework=framework, url_ue=url_ue)
-        b = b.replace("/", "_")
+    datas_load = []
+    datas_empty = []
+    datas_errors = []
+    for b in liste_datas:
+        try:
+            result = base_api(base=b, framework=framework, url_ue=url_ue)
+            b = b.replace("/", "_")
+
+            if result:
+                with open('/eCorda_data/'+b+'.json', 'w') as file:
+                    file.write(json.dumps(result, indent=4))
+                datas_load.append(b)
+            else:
+                datas_empty.append(b)
+        except Exception as e:
+            logger.error(f'Error in table {b}: {e}')
+            datas_errors.append([b, e.message])
 
-        if result:
-            with open('/eCorda_data/'+b+'.json','w') as file:
-                file.write(json.dumps(result, indent=4))
-            datas_load.append(b)
-            counter+=1
-        else:
-            datas_empty.append(b)
-
-    logger.debug(f'datas empty no load: {datas_empty}') 
-    No_load_datas = [i for i in liste_datas if (i.replace("/", "_") not in datas_empty) & (i.replace("/", "_") not in datas_load)]
-    logger.debug(f'datas no load because prob: {No_load_datas}')                          
+    logger.debug(f'Enpty tables: {datas_empty.join(",")}')
+    logger.debug(f'Loaded tables: {datas_load.join(",")}')
+    logger.debug(f'Error: {datas_errors}')
+    logger.debug(f'Job done! ')