From 71a247f1793864e59524564fd86da2e3b7cda7fa Mon Sep 17 00:00:00 2001 From: Toby James Date: Thu, 10 Aug 2023 13:20:50 +0100 Subject: [PATCH 01/10] Tidy rename to bp file using re --- src/wands/data_cache.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/wands/data_cache.py b/src/wands/data_cache.py index f5f20b8..5655e5b 100644 --- a/src/wands/data_cache.py +++ b/src/wands/data_cache.py @@ -1,4 +1,5 @@ import adios2 +import re import numpy as np from pathlib import Path from .adios import AdiosObject @@ -113,13 +114,9 @@ def check_availability(self, filename: str, data_list: list): -------- """ - # print( type(filename.replace('.h5','.bp'))) - # print(self._path) - # print(type(self._path)) - bpfilename_path = self._path / filename.replace(".h5", ".bp") - # print(type(bpfilename_path)) - # print(f"check av: data type data_list: {type(data_list)}") - # print(f"Check availability") + + bpfilename_path = self._path / re.sub("\.\w+$", ".bp", filename) + local_list = [] remote_list = [] # print(f"trying to open {bpfilename_path}") From f76962d17d3d906ee1f4a29c461ec95ec380c68a Mon Sep 17 00:00:00 2001 From: Toby James Date: Tue, 15 Aug 2023 16:00:00 +0100 Subject: [PATCH 02/10] Outline logger object --- src/wands/logger.py | 56 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 src/wands/logger.py diff --git a/src/wands/logger.py b/src/wands/logger.py new file mode 100644 index 0000000..8eded24 --- /dev/null +++ b/src/wands/logger.py @@ -0,0 +1,56 @@ +""" +import the object logger from this file and log to it. + +It writes all information to a log file, INFO to stdout, and warnings, +errors, to stderr. +""" +import sys +import os +import logging +from pathlib import Path +import re + +# Allow set the directory to write the log files in with the environment +# variable WANDSLOGPATH +# Default to CWD +if LOGPATH:=os.environ.get("WANDSLOGPATH"): + LOGPATH = Path(LOGPATH) +else: + LOGPATH = Path(".") + +LOGFILE = "WANDS" +n = 0 +EXT = ".log" + +pattern = LOGFILE + "(\d*)" + EXT + +# Increment the n value +for f in os.listdir(LOGPATH): + if res:=re.search(pattern, f): + if int(res.group(1)) > n: + n = int(res.group(1)) + +logging.basicConfig(filename=str(LOGPATH/(LOGFILE+n+EXT)), + level=logging.DEBUG) + +logger = logging.getLogger("WANDS") + +# Filter to isolate INFO for stdout +class _LoggingFilter(logging.Filter): + def __init__(self, level: int): + super().__init__() + self._level = level + def filter(self, record: logging.LogRecord) -> bool: + return record.levelno == self._level + +# INFO to stdout +outhandler = logging.StreamHandler(sys.stdout) +outhandler.setLevel(logging.INFO) +outhandler.addFilter(_LoggingFilter(logging.INFO)) + +# WARNING and above to stderr +errhandler = logging.StreamHandler(sys.stderr) +errhandler.setLevel(logging.WARNING) + +logger.addHandler(outhandler) +logger.addHandler(errhandler) From 3be59359461dd405726c4abe548c770ef2fd264e Mon Sep 17 00:00:00 2001 From: Toby James Date: Tue, 15 Aug 2023 16:01:11 +0100 Subject: [PATCH 03/10] Black logger.py --- src/wands/logger.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/wands/logger.py b/src/wands/logger.py index 8eded24..f3dae16 100644 --- a/src/wands/logger.py +++ b/src/wands/logger.py @@ -13,7 +13,7 @@ # Allow set the directory to write the log files in with the environment # variable WANDSLOGPATH # Default to CWD -if LOGPATH:=os.environ.get("WANDSLOGPATH"): +if LOGPATH := os.environ.get("WANDSLOGPATH"): LOGPATH = Path(LOGPATH) else: LOGPATH = Path(".") @@ -26,12 +26,11 @@ # Increment the n value for f in os.listdir(LOGPATH): - if res:=re.search(pattern, f): + if res := re.search(pattern, f): if int(res.group(1)) > n: n = int(res.group(1)) -logging.basicConfig(filename=str(LOGPATH/(LOGFILE+n+EXT)), - level=logging.DEBUG) +logging.basicConfig(filename=str(LOGPATH / (LOGFILE + n + EXT)), level=logging.DEBUG) logger = logging.getLogger("WANDS") @@ -40,9 +39,11 @@ class _LoggingFilter(logging.Filter): def __init__(self, level: int): super().__init__() self._level = level + def filter(self, record: logging.LogRecord) -> bool: return record.levelno == self._level + # INFO to stdout outhandler = logging.StreamHandler(sys.stdout) outhandler.setLevel(logging.INFO) From dd23ba653a9a64d7f92821565a7486a9e114a1d4 Mon Sep 17 00:00:00 2001 From: Toby James Date: Tue, 15 Aug 2023 16:05:57 +0100 Subject: [PATCH 04/10] Move import to top of file --- src/wands/wan.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/wands/wan.py b/src/wands/wan.py index e13ca1f..c471f43 100644 --- a/src/wands/wan.py +++ b/src/wands/wan.py @@ -4,6 +4,7 @@ import adios2 from .adios import AdiosObject from .datahub import RawData_fusion +from .data_cache import DataCache # maybe add a counter for number of declared io. To finalize adiosobject in the sense if all declared Io are finalized adios_io can be reset to None. ToBeDiscussed @@ -284,8 +285,6 @@ def receive_list_save_only( """ - from .data_cache import DataCache - print( f"STATUS: receiving data from remote not returned just saved in local data cache" ) From 9e21773ceb5836a8714c858820e5497a7b39c64b Mon Sep 17 00:00:00 2001 From: Toby James Date: Tue, 15 Aug 2023 16:19:46 +0100 Subject: [PATCH 05/10] Update to use logger --- src/wands/wan.py | 100 ++++++++++++++++++++++++----------------------- 1 file changed, 51 insertions(+), 49 deletions(-) diff --git a/src/wands/wan.py b/src/wands/wan.py index c471f43..be24203 100644 --- a/src/wands/wan.py +++ b/src/wands/wan.py @@ -6,6 +6,8 @@ from .datahub import RawData_fusion from .data_cache import DataCache +from logger import logger + # maybe add a counter for number of declared io. To finalize adiosobject in the sense if all declared Io are finalized adios_io can be reset to None. ToBeDiscussed class WandsWAN: @@ -31,7 +33,7 @@ def __init__(self, link="WAN", engine="Dataman", parameters=None): # global adios_io # if adios_io is None: # adios_io = adios2.ADIOS() - print(parameters) + logger.info(parameters) self._adob = AdiosObject(link, engine, parameters) # add requests.... @@ -94,11 +96,11 @@ def send_array(self, eng_name: str, var_name: str, data: np.ndarray): # count – local dimension # constantDims – true: shape, start, count won’t change, false: shape, start, count will change after definition shape = data.shape - # print(f"shape in send: {shape!s}") + logger.debug(f"shape in send: {shape!s}") count = shape - # print(f"count in send {count!s}") + logger.debug(f"count in send {count!s}") start = (0,) * len(shape) - # print(f"start in send {start!s}") + logger.debug(f"start in send {start!s}") sendbuffer = self._adob.get_IO().DefineVariable( var_name, data, shape, start, count, adios2.ConstantDims @@ -163,11 +165,11 @@ def send_dict_arrays(self, eng_name: str, data_dict: dict): # for var_name ,sendbuf in defined_vars.items(): for var_name, data in data_dict.items(): shape = data.shape - # (f"shape in send: {shape!s}") + logger.debug(f"shape in send: {shape!s}") count = shape - # print(f"count in send {count!s}") + logger.debug(f"count in send {count!s}") start = (0,) * len(shape) - # print(f"start in send {start!s}") + logger.debug(f"start in send {start!s}") sendbuffer = self._adob.get_IO().DefineVariable( var_name, data, shape, start, count, adios2.ConstantDims ) @@ -222,31 +224,31 @@ def receive_dict_arrays(self, eng_name: str, variable_list: list[str]) -> dict: ------- data """ - print(f"STATUS: receiving data from remote") + logger.info(f"STATUS: receiving data from remote") data_dict = {} reader = self._adob.get_IO().Open(eng_name, adios2.Mode.Read) - print(f"Status: Handshake successfull") + logger.info(f"Status: Handshake successfull") while True: stepStatus = reader.BeginStep() - # print(f"Stepstatus {stepStatus!s}") + logger.debug(f"Stepstatus {stepStatus!s}") if stepStatus == adios2.StepStatus.OK: # inquire for variable for name in variable_list: - # print(f"getting {name}") + logger.debug(f"getting {name}") recvar = self._adob.get_IO().InquireVariable(name) - # print(f"Variable inqurired {recvar!s}") + logger.debug(f"Variable inqurired {recvar!s}") if recvar: # determine the shape of the data that will be sent bufshape = recvar.Shape() # allocate buffer for now numpy # replace("_t","") necessary because - # print(f"Datatype {recvar.Type()}") + logger.debug(f"Received datatype {recvar.Type()}") data = np.ones(bufshape, dtype=recvar.Type().replace("_t", "")) - # print(f" in receive data {data!s}") - # print(f"data before Get: \n{data!s}") + logger.debug(f" in receive data {data!s}") + logger.debug(f"data before Get: \n{data!s}") reader.Get(recvar, data, adios2.Mode.Deferred) data_dict[name] = data - # print(f"data right after get This might be not right as data might not have been sent yet \n: {data!s}") + logger.debug(f"data right after get This might be not right as data might not have been sent yet \n: {data!s}") # currentStep = reader.CurrentStep() else: raise ValueError(f"InquireVariable failed {name!s}") @@ -255,9 +257,9 @@ def receive_dict_arrays(self, eng_name: str, variable_list: list[str]) -> dict: else: raise StopIteration(f"next step failed to initiate {stepStatus!s}") reader.EndStep() - # print(f"After end step \n{data!s}") + logger.debug(f"After end step \n{data!s}") reader.Close() - # print(f"after close \n {data!s}") + logger.debug(f"after close \n {data!s}") return data_dict def receive_list_save_only( @@ -285,10 +287,10 @@ def receive_list_save_only( """ - print( + logger.info( f"STATUS: receiving data from remote not returned just saved in local data cache" ) - bpfilename = filename.replace(".h5", ".bp") + bpfilename = filename.replace(".h5", ".bp") # TODO fix see issue #14 bpfilename_path = f"{path}/{bpfilename}" reader = self._adob.get_IO().Open(eng_name, adios2.Mode.Read) @@ -296,36 +298,36 @@ def receive_list_save_only( dc_obj = DataCache(path=path, link=link) writer = dc_obj._adob.get_IO().Open(bpfilename_path, adios2.Mode.Append) - print(f"Status: Handshake successfull") + logger.info(f"Status: Handshake successfull") while True: stepStatus = reader.BeginStep() - # print(f"Stepstatus {stepStatus!s}") + logger.debug(f"Stepstatus {stepStatus!s}") if stepStatus == adios2.StepStatus.OK: # inquire for variable writer.BeginStep() for name in variable_list: - # print(f"getting {name}") + logger.debug(f"getting {name}") recvar = self._adob.get_IO().InquireVariable(name) - # print(f"Variable inqurired {recvar!s}") + logger.debug(f"Variable inqurired {recvar!s}") if recvar: # determine the shape of the data that will be sent bufshape = recvar.Shape() # allocate buffer for now numpy # replace("_t","") necessary because - # print(f"Datatype {recvar.Type()}") + logger.debug(f"Datatype {recvar.Type()}") data = np.ones(bufshape, dtype=recvar.Type().replace("_t", "")) - # print(f" in receive data {data!s}") - # print(f"data before Get: \n{data!s}") + logger.debug(f" in receive data {data!s}") + logger.debug(f"data before Get: \n{data!s}") reader.Get(recvar, data, adios2.Mode.Sync) start = (0,) * len(bufshape) if dc_obj._adob.get_IO().InquireVariable(name): - print(f"Debug: variable {name} alrady in file") + logger.info(f"Debug: variable {name} alrady in file") continue writebuffer = dc_obj._adob.get_IO().DefineVariable( name, data, bufshape, start, bufshape, adios2.ConstantDims ) writer.Put(writebuffer, data, adios2.Mode.Deferred) - # print(f"data right after get This might be not right as data might not have been sent yet \n: {data!s}") + logger.debug(f"data right after get This might be not right as data might not have been sent yet \n: {data!s}") # currentStep = reader.CurrentStep() else: raise ValueError(f"InquireVariable failed {name!s}") @@ -335,10 +337,10 @@ def receive_list_save_only( raise StopIteration(f"next step failed to initiate {stepStatus!s}") writer.EndStep() reader.EndStep() - # print(f"After end step \n{data!s}") + logger.debug(f"After end step \n{data!s}") writer.Close() reader.Close() - # print(f"after close \n {data!s}") + logger.debug(f"after close \n {data!s}") # def receive_dict_arrays_multi_steps(self, eng_name:str, variable_list:list[str]): # """ @@ -420,7 +422,7 @@ def send_rawdata_fusion(self, eng_name: str, var_name: str, data: RawData_fusion if time is not None: sendmatrix.append(time) else: - warnings.warn( + logger.warn( f"could not retrieve time array for {var_name!s}, will not be sent" ) @@ -429,7 +431,7 @@ def send_rawdata_fusion(self, eng_name: str, var_name: str, data: RawData_fusion if data is not None: sendmatrix.append(data_array) else: - warnings.warn( + logger.warn( f"could not retrieve data array for {var_name!s}, will not be sent" ) @@ -438,7 +440,7 @@ def send_rawdata_fusion(self, eng_name: str, var_name: str, data: RawData_fusion if error is not None: sendmatrix.append(error) else: - warnings.warn( + logger.warn( f"could not retrieve error array for {var_name!s}, will not be sent" ) @@ -460,29 +462,29 @@ def send_rawdata_fusion_list(self, eng_name: str, rd_list: list[RawData_fusion]) None """ # defined_vars = self.define_variables(data_dict) - # print(f"sending raw list") - # print(rd_list) + logger.info(f"sending raw list") + logger.debug(rd_list) writer = self._adob.get_IO().Open(eng_name, adios2.Mode.Write) # name – unique variable identifier # shape – global dimension # start – local offset # count – local dimension # constantDims – true: shape, start, count won’t change, false: shape, start, count will change after definition - # print(f"opened ") + logger.debug(f"opened ") writer.BeginStep() # for var_name ,sendbuf in defined_vars.items(): for rd in rd_list: - # print(rd) + logger.debug(rd) name = rd.get_name() - # print(f"Signal: {name}") + logger.debug(f"Signal: {name}") sendmatrix = [] # send time if not None time = rd.get_time() if time is not None: sendmatrix.append(time) else: - warnings.warn( + logger.warn( f"could not retrieve time array for {name!s}, will not be sent" ) @@ -491,19 +493,19 @@ def send_rawdata_fusion_list(self, eng_name: str, rd_list: list[RawData_fusion]) if data is not None: sendmatrix.append(data) else: - warnings.warn( + logger.warn( f"could not retrieve data array for {name!s}, will not be sent" ) # self.send_array(eng_name,name,np.array(sendmatrix)) nd_matrix = np.array(sendmatrix) - # print(f"type matrix {type(nd_matrix)!s}") + logger.debug(f"type matrix {type(nd_matrix)!s}") shape = nd_matrix.shape - # print(f"shape in send: {shape!s}") + logger.debug(f"shape in send: {shape!s}") count = shape - # print(f"count in send {count!s}") + logger.debug(f"count in send {count!s}") start = (0,) * len(shape) - # print(f"start in send {start!s}") + logger.debug(f"start in send {start!s}") sendbuffer = self._adob.get_IO().DefineVariable( name, nd_matrix, shape, start, count, adios2.ConstantDims ) @@ -543,9 +545,9 @@ def receive_one_signal(self, eng_name: str, variable_name: str): # allocate buffer for now numpy Type() returns a cpp type e.g. int64_t # replace("_t","") necessary because data = np.ones(bufshape, dtype=recvar.Type().replace("_t", "")) - # print(f"data before Get: \n{data!s}") + logger.debug(f"data before Get: \n{data!s}") reader.Get(recvar, data, adios2.Mode.Deferred) - # print(f"data right after get This might be not right as data might not have been sent yet \n: {data!s}") + logger.debug(f"data right after get This might be not right as data might not have been sent yet \n: {data!s}") # currentStep = reader.CurrentStep() else: raise ValueError(f"InquireVariable failed {variable_name!s}") @@ -554,9 +556,9 @@ def receive_one_signal(self, eng_name: str, variable_name: str): else: raise StopIteration(f"next step failed to initiate {stepStatus!s}") reader.EndStep() - # print(f"After end step \n{data!s}") + logger.debug(f"After end step \n{data!s}") reader.Close() - # print(f"after close \n {data!s}") + logger.debug(f"after close \n {data!s}") return data # reader = self._io.Open(eng_name, adios2.Mode.Read) # recvar = self._io.InquireVariable(variable_name) From 7c765d11cebffb86aa562f654bb96c641248cea0 Mon Sep 17 00:00:00 2001 From: Toby James Date: Tue, 15 Aug 2023 16:22:08 +0100 Subject: [PATCH 06/10] Remove warnings import --- src/wands/wan.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/wands/wan.py b/src/wands/wan.py index be24203..8c475ba 100644 --- a/src/wands/wan.py +++ b/src/wands/wan.py @@ -1,6 +1,5 @@ # for now numpy import import numpy as np -import warnings import adios2 from .adios import AdiosObject from .datahub import RawData_fusion From 0935b080c1a2b8b0dfea03bd1e481b3387bda8ea Mon Sep 17 00:00:00 2001 From: Toby James Date: Tue, 15 Aug 2023 16:26:56 +0100 Subject: [PATCH 07/10] Use logger for prints --- src/wands/adios.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/wands/adios.py b/src/wands/adios.py index 2f1160d..25b52c8 100644 --- a/src/wands/adios.py +++ b/src/wands/adios.py @@ -5,6 +5,7 @@ # import numpy as np # import warnings import adios2 +from logger import logger # Adios object (only one per application) therefore a global variable within this module TODO rename to adios @@ -102,11 +103,11 @@ def get_avail_variables(self): return self._io.AvailableVariables() def print_info(self): - print(f"Name: {self.get_link()!s}") - print(f"Engine: {self.get_engine()!s}") - print(f"Parameters: {self.get_parameters()!s}") - print(f"Variables: {self.get_avail_variables()!s}") - print(f"Attributes: {self.get_avail_attributes()!s}") + logger.info(f"Name: {self.get_link()!s}") + logger.info(f"Engine: {self.get_engine()!s}") + logger.info(f"Parameters: {self.get_parameters()!s}") + logger.info(f"Variables: {self.get_avail_variables()!s}") + logger.info(f"Attributes: {self.get_avail_attributes()!s}") # def remove_all_variables(self): # self.RemoveAllVariables() From 34fb7d500dab00bebad8ff7152668a496f7cf641 Mon Sep 17 00:00:00 2001 From: Toby James Date: Tue, 15 Aug 2023 16:37:16 +0100 Subject: [PATCH 08/10] Add logger to wands.py --- src/wands/wands.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/wands/wands.py b/src/wands/wands.py index 0f0396e..656f9f1 100644 --- a/src/wands/wands.py +++ b/src/wands/wands.py @@ -3,6 +3,7 @@ # from pathlib import Path from .wan import WandsWAN from .data_cache import DataCache +from logger import logger class Wands: @@ -43,7 +44,8 @@ def request(self, filename: str, data_list: list) -> dict: # only for performance measurement from time import time # # remove locally available datasets from list - # only for performance measurement print(f"request: type datalist: {type(data_list)}") + # only for performance measurement + logger.debug(f"request: type datalist: {type(data_list)}") data_from_remote = {} # only for performance measurement timereq = time() @@ -60,8 +62,8 @@ def request(self, filename: str, data_list: list) -> dict: filename=filename, local_list=local_list ) # only for performance measurement timedatalocal = time() - # print(f"Signals found locally: {local_list}") - # print(f"Signals to be requested remotely: {remote_list}") + logger.info(f"Signals found locally: {local_list}") + logger.info(f"Signals to be requested remotely: {remote_list}") # fetch data remotely if remote_list: data = { @@ -69,15 +71,17 @@ def request(self, filename: str, data_list: list) -> dict: "signals": remote_list, } response = requests.post(self._webaddress, json=data) - print(response.status_code) - print(response.json()) + logger.info(response.status_code) + logger.info(response.json()) wandsWAN_obj = WandsWAN(parameters=self._Adiosparams) data_from_remote = wandsWAN_obj.receive(remote_list) # only for performance measurement timeremote = time() self.dataCache.write(filename=filename, data_dict=data_from_remote) # only for performance measurement timewrite = time() - # only for performance measurement print(f"Timings:\n check av = {timecheckav-timereq}\n t_lfc = {timedatalocal-timecheckav}\n ") + # only for performance measurement + logger.debug(f"Timings:\n check av = {timecheckav-timereq}\n t_lfc = {timedatalocal-timecheckav}\n ") # only for performance measurement if remote_list: - # only for performance measurement print(f"t_getremote = {timeremote-timedatalocal}\n t_toDB = {timewrite-timeremote}") + # only for performance measurement + logger.debug(f"t_getremote = {timeremote-timedatalocal}\n t_toDB = {timewrite-timeremote}") return data_from_remote | data_from_cache From dafa9eec0848610deb78aea1d37404d69427f8bd Mon Sep 17 00:00:00 2001 From: Toby James Date: Tue, 15 Aug 2023 16:37:29 +0100 Subject: [PATCH 09/10] Add logger to data_cache.py --- src/wands/data_cache.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/wands/data_cache.py b/src/wands/data_cache.py index 5655e5b..ec21ae1 100644 --- a/src/wands/data_cache.py +++ b/src/wands/data_cache.py @@ -4,6 +4,7 @@ from pathlib import Path from .adios import AdiosObject +from logger import logger class DataCache: """ @@ -61,22 +62,22 @@ def write(self, filename: str, data_dict: dict): # modify string to delete .hdf5 with string find bpfilename_path = self._path / filename.replace(".h5", ".bp") - # print(bpfilename_path) + logger.info(bpfilename_path) writer = self._adob.get_IO().Open(f"{bpfilename_path!s}", adios2.Mode.Append) writer.BeginStep() for var_name, data in data_dict.items(): # Test if the variable already exists if self._adob.get_IO().InquireVariable(var_name): - print(f" Debug: variable {var_name!s} exists already") + logger.info(f" Debug: variable {var_name!s} exists already") continue shape = data.shape - # print(f"shape in send: {shape!s}") + logger.debug(f"shape in send: {shape!s}") count = shape - # print(f"count in send {count!s}") + logger.debug(f"count in send {count!s}") start = (0,) * len(shape) - # print(f"start in send {start!s}") + logger.debug(f"start in send {start!s}") sendbuffer = self._adob.get_IO().DefineVariable( var_name, data, shape, start, count, adios2.ConstantDims ) @@ -119,7 +120,7 @@ def check_availability(self, filename: str, data_list: list): local_list = [] remote_list = [] - # print(f"trying to open {bpfilename_path}") + logger.info(f"trying to open {bpfilename_path}") if bpfilename_path.exists(): reader = self._adob.get_IO().Open(f"{bpfilename_path!s}", adios2.Mode.Read) if reader: @@ -132,7 +133,7 @@ def check_availability(self, filename: str, data_list: list): remote_list.append(signal) else: # This should never happen since we checked before that the file exists - print( + logger.warn( f"[WARNING] Even though the {bpfilename_path!s} exists WANDS was unable to open it. Requesting all datasets remotely" ) remote_list = data_list @@ -144,7 +145,7 @@ def check_availability(self, filename: str, data_list: list): def load_from_cache(self, filename: str, local_list: list): bpfilename_path = self._path / filename.replace(".h5", ".bp") - # print(f"beginning of cache: {local_list!s}") + logger.debug(f"beginning of cache: {local_list!s}") # self._adob.print_info() local_dict = {} if local_list: @@ -157,16 +158,16 @@ def load_from_cache(self, filename: str, local_list: list): while True: stepStatus = reader.BeginStep() - # print(stepStatus) + logger.debug(stepStatus) if stepStatus == adios2.StepStatus.OK: - # print(f"Current Step: {reader.CurrentStep()!s} reader Steps= {reader.Steps()!s} local dict = {local_dict!s} ") + logger.debug(f"Current Step: {reader.CurrentStep()!s} reader Steps= {reader.Steps()!s} local dict = {local_dict!s} ") for signal in local_list: if signal not in local_dict: variable = self._adob.get_IO().InquireVariable( signal ) if variable: - # print(variable.Type()) + logger.debug(variable.Type()) data = np.zeros( variable.Shape(), dtype=variable.Type() ) @@ -180,10 +181,10 @@ def load_from_cache(self, filename: str, local_list: list): ) reader.EndStep() - # print(f"call close next") + logger.debug(f"call close next") reader.Close() - # print(f"{bpfilename_path!s} closed") - # print(local_list) + logger.info(f"{bpfilename_path!s} closed") + logger.debug(local_list) else: raise ValueError(f" file {bpfilename_path} not found") From e9e1201f48fa40cac35bec793d9fb4cb648ac6e9 Mon Sep 17 00:00:00 2001 From: Toby James Date: Tue, 15 Aug 2023 16:37:45 +0100 Subject: [PATCH 10/10] Black all --- src/wands/data_cache.py | 5 ++++- src/wands/wan.py | 12 +++++++++--- src/wands/wands.py | 8 ++++++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/wands/data_cache.py b/src/wands/data_cache.py index ec21ae1..1310f70 100644 --- a/src/wands/data_cache.py +++ b/src/wands/data_cache.py @@ -6,6 +6,7 @@ from logger import logger + class DataCache: """ Use the received Data to build a local database @@ -160,7 +161,9 @@ def load_from_cache(self, filename: str, local_list: list): stepStatus = reader.BeginStep() logger.debug(stepStatus) if stepStatus == adios2.StepStatus.OK: - logger.debug(f"Current Step: {reader.CurrentStep()!s} reader Steps= {reader.Steps()!s} local dict = {local_dict!s} ") + logger.debug( + f"Current Step: {reader.CurrentStep()!s} reader Steps= {reader.Steps()!s} local dict = {local_dict!s} " + ) for signal in local_list: if signal not in local_dict: variable = self._adob.get_IO().InquireVariable( diff --git a/src/wands/wan.py b/src/wands/wan.py index 8c475ba..f11479a 100644 --- a/src/wands/wan.py +++ b/src/wands/wan.py @@ -247,7 +247,9 @@ def receive_dict_arrays(self, eng_name: str, variable_list: list[str]) -> dict: logger.debug(f"data before Get: \n{data!s}") reader.Get(recvar, data, adios2.Mode.Deferred) data_dict[name] = data - logger.debug(f"data right after get This might be not right as data might not have been sent yet \n: {data!s}") + logger.debug( + f"data right after get This might be not right as data might not have been sent yet \n: {data!s}" + ) # currentStep = reader.CurrentStep() else: raise ValueError(f"InquireVariable failed {name!s}") @@ -326,7 +328,9 @@ def receive_list_save_only( name, data, bufshape, start, bufshape, adios2.ConstantDims ) writer.Put(writebuffer, data, adios2.Mode.Deferred) - logger.debug(f"data right after get This might be not right as data might not have been sent yet \n: {data!s}") + logger.debug( + f"data right after get This might be not right as data might not have been sent yet \n: {data!s}" + ) # currentStep = reader.CurrentStep() else: raise ValueError(f"InquireVariable failed {name!s}") @@ -546,7 +550,9 @@ def receive_one_signal(self, eng_name: str, variable_name: str): data = np.ones(bufshape, dtype=recvar.Type().replace("_t", "")) logger.debug(f"data before Get: \n{data!s}") reader.Get(recvar, data, adios2.Mode.Deferred) - logger.debug(f"data right after get This might be not right as data might not have been sent yet \n: {data!s}") + logger.debug( + f"data right after get This might be not right as data might not have been sent yet \n: {data!s}" + ) # currentStep = reader.CurrentStep() else: raise ValueError(f"InquireVariable failed {variable_name!s}") diff --git a/src/wands/wands.py b/src/wands/wands.py index 656f9f1..46b1eb9 100644 --- a/src/wands/wands.py +++ b/src/wands/wands.py @@ -80,8 +80,12 @@ def request(self, filename: str, data_list: list) -> dict: self.dataCache.write(filename=filename, data_dict=data_from_remote) # only for performance measurement timewrite = time() # only for performance measurement - logger.debug(f"Timings:\n check av = {timecheckav-timereq}\n t_lfc = {timedatalocal-timecheckav}\n ") + logger.debug( + f"Timings:\n check av = {timecheckav-timereq}\n t_lfc = {timedatalocal-timecheckav}\n " + ) # only for performance measurement if remote_list: # only for performance measurement - logger.debug(f"t_getremote = {timeremote-timedatalocal}\n t_toDB = {timewrite-timeremote}") + logger.debug( + f"t_getremote = {timeremote-timedatalocal}\n t_toDB = {timewrite-timeremote}" + ) return data_from_remote | data_from_cache