Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added logging configuration to config file, and use it when opening logging #1635

Merged
merged 3 commits into from
Aug 28, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 46 additions & 1 deletion jwql/example_config.json
Original file line number Diff line number Diff line change
Expand Up @@ -47,5 +47,50 @@
"cores" : "",
"redis_host": "",
"redis_port": "",
"transfer_dir": ""
"transfer_dir": "",
"logging": {
"version": 1,
"disable_existing_loggers": true,
"formatters": {
"simple": {
"format": "%(asctime)s %(levelname)s: %(message)s",
"datefmt": "%m/%d/%Y %H:%M:%S %p"
}
},
"filters": {
"warnings_and_below": {
"()" : "jwql.utils.logging_functions.filter_maker",
"level": "WARNING"
}
},
"handlers": {
"stdout": {
"class": "logging.StreamHandler",
"level": "INFO",
"formatter": "simple",
"stream": "ext://sys.stdout",
"filters": ["warnings_and_below"]
},
"stderr": {
"class": "logging.StreamHandler",
"level": "ERROR",
"formatter": "simple",
"stream": "ext://sys.stderr"
},
"file": {
"class": "logging.FileHandler",
"formatter": "simple",
"filename": "app.log",
"mode": "a"
}
},
"root": {
"level": "DEBUG",
"handlers": [
"stderr",
"stdout",
"file"
]
}
}
}
52 changes: 42 additions & 10 deletions jwql/utils/logging_functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ def my_main_function():
import getpass
import importlib
import logging
import logging.config
import os
import pwd
import socket
Expand All @@ -76,8 +77,40 @@ def my_main_function():
from jwql.utils.utils import get_config, ensure_dir_exists


def filter_maker(level):
"""
This creates a logging filter that takes in an integer describing log level (with
DEBUG being the lowest value and CRITICAL the highest), and returns True if and only
if the logged message has a lower level than the filter level.

The filter is needed because the logging system is designed so that it outputs
messages of LogLevel *or higher*, because the assumption is you want to know if
something happens that's more serious than what you're looking at.

In this case, though, we're dividing printed-out log messages between the built-in
STDOUT and STDERR output streams, and we have assigned ERROR and above to go to
STDERR, while INFO and above go to STDOUT. So, by default, anything at ERROR or at
CRITICAL would go to *both* STDOUT and STDERR. This function lets you add a filter
that returns false for anything with a level above WARNING, so that STDOUT won't
duplicate those messages.
"""
level = getattr(logging, level)

def filter(record):
return record.levelno <= level

return filter


def configure_logging(module):
"""Configure the log file with a standard logging format.
"""
Configure the log file with a standard logging format. The format in question is
set up as follows:

- DEBUG messages are ignored
- INFO and WARNING messages go to both the log file and sys.stdout
- ERROR and CRITICAL messages go to both the log file and sys.stderr
- existing loggers are disabled before this configuration is applied

Parameters
----------
Expand All @@ -98,15 +131,14 @@ def configure_logging(module):
# Determine log file location
log_file = make_log_file(module)

# Make sure no other root lhandlers exist before configuring the logger
for handler in logging.root.handlers[:]:
logging.root.removeHandler(handler)
# Get the logging configuration dictionary
logging_config = get_config()['logging']

# Set the log file to the file that we got above
logging_config["handlers"]["file"]["filename"] = log_file

# Create the log file and set the permissions
logging.basicConfig(filename=log_file,
format='%(asctime)s %(levelname)s: %(message)s',
datefmt='%m/%d/%Y %H:%M:%S %p',
level=logging.INFO)
# Configure the logging system and set permissions for the file
logging.config.dictConfig(logging_config)
print('Log file initialized to {}'.format(log_file))
set_permissions(log_file)

Expand Down Expand Up @@ -243,7 +275,7 @@ def wrapped(*args, **kwargs):

# nosec comment added to ignore bandit security check
try:
environment = subprocess.check_output('conda env export', universal_newlines=True, shell=True) # nosec
environment = subprocess.check_output('conda env export', universal_newlines=True, shell=True) # nosec
logging.info('Environment:')
for line in environment.split('\n'):
logging.info(line)
Expand Down
Loading