Logger hierarchy (NVIDIA#3081)

* convert to logger hierarchy

* add functions to log_utils

docs/resources/log.config

@@ -5,7 +5,7 @@ keys=root
 keys=consoleHandler,errorFileHandler
 
 [formatters]
-keys=fullFormatter
+keys=baseFormatter
 
 [logger_root]
 level=INFO
@@ -14,14 +14,15 @@ handlers=consoleHandler,errorFileHandler
 [handler_consoleHandler]
 class=StreamHandler
 level=DEBUG
-formatter=fullFormatter
+formatter=baseFormatter
 args=(sys.stdout,)
 
 [handler_errorFileHandler]
 class=FileHandler
 level=ERROR
-formatter=fullFormatter
+formatter=baseFormatter
 args=('error_log.txt', 'a')
 
-[formatter_fullFormatter]
+[formatter_baseFormatter]
+class=nvflare.fuel.utils.log_utils.BaseFormatter
 format=%(asctime)s - %(name)s - %(levelname)s - %(message)s

nvflare/apis/fl_component.py

@@ -12,9 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import logging
-
 from nvflare.apis.utils.fl_context_utils import generate_log_message
+from nvflare.fuel.utils.log_utils import get_obj_logger
 from nvflare.security.logging import secure_format_traceback
 
 from .analytix import AnalyticsData, AnalyticsDataType
@@ -35,7 +34,7 @@ def __init__(self):
         FLComponents have the capability to handle and fire events and contain various methods for logging.
         """
         self._name = self.__class__.__name__
-        self.logger = logging.getLogger(self._name)
+        self.logger = get_obj_logger(self)
 
     @property
     def name(self):

nvflare/apis/fl_context.py

@@ -12,10 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import logging
 import threading
 from typing import Any, Dict, List
 
+from nvflare.fuel.utils.log_utils import get_obj_logger
+
 from .fl_constant import ReservedKey
 
 _update_lock = threading.Lock()
@@ -72,7 +73,7 @@ def __init__(self):
         """
         self.model = None
         self.props = {}
-        self.logger = logging.getLogger(self.__class__.__name__)
+        self.logger = get_obj_logger(self)
 
     def get_prop_keys(self) -> List[str]:
         return list(self.props.keys())

nvflare/apis/impl/task_manager.py

@@ -12,13 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import logging
 from enum import Enum
 from typing import Tuple
 
 from nvflare.apis.controller_spec import ClientTask, Task, TaskCompletionStatus
 from nvflare.apis.fl_context import FLContext
 from nvflare.apis.shareable import Shareable
+from nvflare.fuel.utils.log_utils import get_obj_logger
 
 
 class TaskCheckStatus(Enum):
@@ -39,7 +39,7 @@ def __init__(self):
         app-defined props.
         """
         self._name = self.__class__.__name__
-        self.logger = logging.getLogger(self._name)
+        self.logger = get_obj_logger(self)
 
     def check_task_send(self, client_task: ClientTask, fl_ctx: FLContext) -> TaskCheckStatus:
         """Determine whether the task should be sent to the client.

nvflare/apis/utils/fl_context_utils.py

@@ -13,16 +13,16 @@
 # limitations under the License.
 
 import copy
-import logging
 
 from nvflare.apis.fl_constant import FLContextKey, NonSerializableKeys
 from nvflare.apis.fl_context import FLContext
 from nvflare.apis.shareable import Shareable
 from nvflare.fuel.sec.audit import AuditService
 from nvflare.fuel.utils import fobs
+from nvflare.fuel.utils.log_utils import get_module_logger
 from nvflare.security.logging import secure_format_exception
 
-logger = logging.getLogger("fl_context_utils")
+logger = get_module_logger()
 
 
 def get_serializable_data(fl_ctx: FLContext):

nvflare/apis/utils/reliable_message.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import concurrent.futures
-import logging
 import threading
 import time
 import uuid
@@ -23,6 +22,7 @@
 from nvflare.apis.signal import Signal
 from nvflare.apis.utils.fl_context_utils import generate_log_message
 from nvflare.fuel.utils.config_service import ConfigService
+from nvflare.fuel.utils.log_utils import get_module_logger
 from nvflare.fuel.utils.validation_utils import check_positive_number
 from nvflare.security.logging import secure_format_exception, secure_format_traceback
 
@@ -236,7 +236,7 @@ class ReliableMessage:
     _reply_receivers = {}  # tx id => receiver
     _tx_lock = threading.Lock()
     _shutdown_asked = False
-    _logger = logging.getLogger("ReliableMessage")
+    _logger = get_module_logger(__module__, __qualname__)
 
     @classmethod
     def register_request_handler(cls, topic: str, handler_f, fl_ctx: FLContext):

nvflare/app_common/abstract/params_converter.py

@@ -12,19 +12,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import logging
 from abc import ABC, abstractmethod
 from typing import Any, List
 
 from nvflare.apis.dxo import from_shareable
 from nvflare.apis.fl_context import FLContext
 from nvflare.apis.shareable import Shareable
+from nvflare.fuel.utils.log_utils import get_obj_logger
 
 
 class ParamsConverter(ABC):
     def __init__(self, supported_tasks: List[str] = None):
         self.supported_tasks = supported_tasks
-        self.logger = logging.getLogger(self.__class__.__name__)
+        self.logger = get_obj_logger(self)
 
     def process(self, task_name: str, shareable: Shareable, fl_ctx: FLContext) -> Shareable:
         if not self.supported_tasks or task_name in self.supported_tasks:

nvflare/app_common/aggregators/dxo_aggregator.py

@@ -12,14 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import logging
 from typing import Any, Dict, Optional
 
 from nvflare.apis.dxo import DXO, DataKind, MetaKey
 from nvflare.apis.fl_component import FLComponent
 from nvflare.apis.fl_context import FLContext
 from nvflare.app_common.aggregators.weighted_aggregation_helper import WeightedAggregationHelper
 from nvflare.app_common.app_constant import AppConstants
+from nvflare.fuel.utils.log_utils import get_module_logger
 
 
 class DXOAggregator(FLComponent):
@@ -61,7 +61,7 @@ def __init__(
 
         if name_postfix:
             self._name += name_postfix
-            self.logger = logging.getLogger(self._name)
+            self.logger = get_module_logger(self.__module__, f"{self.__class__.__qualname__}{name_postfix}")
 
     def reset_aggregation_helper(self):
         if self.aggregation_helper:

nvflare/app_common/executors/multi_process_executor.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import logging
 import os
 import shlex
 import subprocess
@@ -41,6 +40,7 @@
 from nvflare.fuel.utils.class_utils import ModuleScanner
 from nvflare.fuel.utils.component_builder import ComponentBuilder
 from nvflare.fuel.utils.config_service import ConfigService
+from nvflare.fuel.utils.log_utils import get_obj_logger
 from nvflare.private.defs import CellChannel, CellChannelTopic, new_cell_message
 from nvflare.security.logging import secure_format_exception
 
@@ -85,7 +85,7 @@ def __init__(self, executor_id=None, num_of_processes=1, components=None):
         self.execute_complete = None
         self.engine = None
 
-        self.logger = logging.getLogger(self.__class__.__name__)
+        self.logger = get_obj_logger(self)
         self.conn_clients = []
         self.exe_process = None
 

nvflare/app_common/executors/task_script_runner.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import builtins
-import logging
 import os
 import runpy
 import sys
@@ -21,12 +20,13 @@
 from nvflare.client.in_process.api import TOPIC_ABORT
 from nvflare.fuel.data_event.data_bus import DataBus
 from nvflare.fuel.data_event.event_manager import EventManager
+from nvflare.fuel.utils.log_utils import get_module_logger
 
 print_fn = builtins.print
 
 
 class TaskScriptRunner:
-    logger = logging.getLogger(__name__)
+    logger = get_module_logger(__module__, __qualname__)
 
     def __init__(self, custom_dir: str, script_path: str, script_args: str = None, redirect_print_to_log=True):
         """Wrapper for function given function path and args
@@ -41,7 +41,6 @@ def __init__(self, custom_dir: str, script_path: str, script_args: str = None, r
         self.event_manager = EventManager(DataBus())
         self.script_args = script_args
         self.custom_dir = custom_dir
-        self.logger = logging.getLogger(self.__class__.__name__)
         self.script_path = script_path
         self.script_full_path = self.get_script_full_path(self.custom_dir, self.script_path)
 

nvflare/app_common/launchers/subprocess_launcher.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import logging
 import os
 import shlex
 import subprocess
@@ -24,6 +23,7 @@
 from nvflare.apis.shareable import Shareable
 from nvflare.apis.signal import Signal
 from nvflare.app_common.abstract.launcher import Launcher, LauncherRunStatus
+from nvflare.fuel.utils.log_utils import get_obj_logger
 from nvflare.utils.job_launcher_utils import add_custom_dir_to_path
 
 
@@ -47,7 +47,7 @@ def __init__(self, script: str, launch_once: bool = True, clean_up_script: Optio
         self._script = script
         self._launch_once = launch_once
         self._clean_up_script = clean_up_script
-        self.logger = logging.getLogger(self.__class__.__name__)
+        self.logger = get_obj_logger(self)
 
     def initialize(self, fl_ctx: FLContext):
         self._app_dir = self.get_app_dir(fl_ctx)

nvflare/app_common/np/np_validator.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import logging
 import time
 
 import numpy as np
@@ -24,6 +23,7 @@
 from nvflare.apis.shareable import Shareable, make_reply
 from nvflare.apis.signal import Signal
 from nvflare.app_common.app_constant import AppConstants
+from nvflare.fuel.utils.log_utils import get_obj_logger
 from nvflare.security.logging import secure_format_exception
 
 from .constants import NPConstants
@@ -41,7 +41,7 @@ def __init__(
         # for long time.
         super().__init__()
 
-        self.logger = logging.getLogger("NPValidator")
+        self.logger = get_obj_logger(self)
         self._random_epsilon = epsilon
         self._sleep_time = sleep_time
         self._validate_task_name = validate_task_name

nvflare/app_common/streamers/file_streamer.py

@@ -19,7 +19,7 @@
 from nvflare.apis.fl_context import FLContext
 from nvflare.apis.shareable import ReturnCode, Shareable, make_reply
 from nvflare.apis.streaming import ConsumerFactory, ObjectConsumer, ObjectProducer, StreamableEngine, StreamContext
-from nvflare.fuel.utils.obj_utils import get_logger
+from nvflare.fuel.utils.log_utils import get_obj_logger
 from nvflare.fuel.utils.validation_utils import check_positive_int, check_positive_number
 
 from .streamer_base import StreamerBase
@@ -36,7 +36,7 @@
 class _ChunkConsumer(ObjectConsumer):
     def __init__(self, stream_ctx: StreamContext, dest_dir):
         file_name = stream_ctx.get(_KEY_FILE_NAME)
-        self.logger = get_logger(self)
+        self.logger = get_obj_logger(self)
         self.file_name = file_name
         self.dest_dir = dest_dir
         self.file_size = stream_ctx.get(_KEY_FILE_SIZE)
@@ -101,7 +101,7 @@ def __init__(self, file, chunk_size, timeout):
         self.chunk_size = chunk_size
         self.timeout = timeout
         self.eof = False
-        self.logger = get_logger(self)
+        self.logger = get_obj_logger(self)
 
     def produce(
         self,

nvflare/app_common/tie/process_mgr.py

@@ -20,7 +20,7 @@
 from nvflare.apis.fl_constant import FLContextKey
 from nvflare.apis.fl_context import FLContext
 from nvflare.apis.workspace import Workspace
-from nvflare.fuel.utils.obj_utils import get_logger
+from nvflare.fuel.utils.log_utils import get_obj_logger
 from nvflare.fuel.utils.validation_utils import check_object_type, check_str
 
 
@@ -87,7 +87,7 @@ def __init__(self, cmd_desc: CommandDescriptor):
         self.log_file = None
         self.msg_prefix = None
         self.file_lock = threading.Lock()
-        self.logger = get_logger(self)
+        self.logger = get_obj_logger(self)
 
     def start(
         self,

nvflare/app_opt/confidential_computing/cc_helper.py

@@ -13,12 +13,13 @@
 # limitations under the License.
 # import os.path
 
-import logging
 import os
 from typing import Dict
 
 from nv_attestation_sdk.attestation import Attestation, Devices, Environment
 
+from nvflare.fuel.utils.log_utils import get_obj_logger
+
 
 class VerifierProp:
 
@@ -64,7 +65,7 @@ def __init__(self, site_name: str, verifiers: list):
         attestation.set_name(site_name)
         self.attestation = attestation
         self.token = None
-        self.logger = logging.getLogger(self.__class__.__name__)
+        self.logger = get_obj_logger(self)
         for v in verifiers:
             assert isinstance(v, dict)
             url = None

nvflare/app_opt/flower/grpc_client.py

@@ -16,7 +16,7 @@
 
 from nvflare.app_opt.flower.defs import GRPC_DEFAULT_OPTIONS
 from nvflare.fuel.utils.grpc_utils import create_channel
-from nvflare.fuel.utils.obj_utils import get_logger
+from nvflare.fuel.utils.log_utils import get_obj_logger
 
 from .utils import reply_should_exit
 
@@ -39,7 +39,7 @@ def __init__(self, server_addr, grpc_options=None):
         self.server_addr = server_addr
         self.grpc_options = grpc_options
         self.started = False
-        self.logger = get_logger(self)
+        self.logger = get_obj_logger(self)
 
     def start(self, ready_timeout=10):
         """Start the gRPC client and wait for the server to be ready.

nvflare/app_opt/flower/grpc_server.py

@@ -18,7 +18,7 @@
 from flwr.proto.grpcadapter_pb2_grpc import GrpcAdapterServicer, add_GrpcAdapterServicer_to_server
 
 from nvflare.app_opt.flower.defs import GRPC_DEFAULT_OPTIONS
-from nvflare.fuel.utils.obj_utils import get_logger
+from nvflare.fuel.utils.log_utils import get_obj_logger
 from nvflare.fuel.utils.validation_utils import check_object_type, check_positive_int
 from nvflare.security.logging import secure_format_exception
 
@@ -42,7 +42,7 @@ def __init__(self, addr, max_workers: int, grpc_options, servicer):
         check_positive_int("max_workers", max_workers)
         self.grpc_server = grpc.server(futures.ThreadPoolExecutor(max_workers=max_workers), options=grpc_options)
         add_GrpcAdapterServicer_to_server(servicer, self.grpc_server)
-        self.logger = get_logger(self)
+        self.logger = get_obj_logger(self)
 
         try:
             # TBD: will be enhanced to support secure port