Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add formatting to log and update get_logger function. #1204

Merged
merged 19 commits into from
Aug 28, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
c7e7362
use simplified default logger
jlewitt1 Aug 20, 2024
93b95a9
logger + log level updates
jlewitt1 Aug 20, 2024
2cb2773
use simplified default logger
jlewitt1 Aug 20, 2024
de13380
logger + log level updates
jlewitt1 Aug 20, 2024
70f376d
Merge branch 'update-default-logger' into log-level-updates
jlewitt1 Aug 20, 2024
7b883cc
use simplified default logger
jlewitt1 Aug 20, 2024
db1d247
Merge branch 'update-default-logger' into log-level-updates
jlewitt1 Aug 21, 2024
01dcbf3
Merge remote-tracking branch 'origin/main' into update-default-logger
jlewitt1 Aug 21, 2024
0b8428a
Merge branch 'update-default-logger' into log-level-updates
jlewitt1 Aug 21, 2024
8325af7
Merge remote-tracking branch 'origin/main' into update-default-logger
jlewitt1 Aug 21, 2024
9b372e0
Merge branch 'update-default-logger' into log-level-updates
jlewitt1 Aug 21, 2024
3ab8844
Merge branch 'main' into update-default-logger
jlewitt1 Aug 22, 2024
510f437
Merge branch 'update-default-logger' into log-level-updates
jlewitt1 Aug 22, 2024
5ee3139
Merge remote-tracking branch 'origin/main' into update-default-logger
jlewitt1 Aug 27, 2024
218312d
Merge branch 'update-default-logger' into log-level-updates
jlewitt1 Aug 27, 2024
f3e4aee
Merge remote-tracking branch 'origin/main' into update-default-logger
jlewitt1 Aug 27, 2024
dd2aab3
Merge branch 'update-default-logger' into log-level-updates
jlewitt1 Aug 27, 2024
911ad65
Add formatting to log and update `get_logger` function.
rohinb2 Aug 27, 2024
7e15ba4
Merge remote-tracking branch 'origin/main' into rohin/log-format-impr…
jlewitt1 Aug 28, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion examples/airflow-torch-training/callables.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from torch_example_for_airflow import DownloadData, SimpleTrainer


logger = get_logger(name=__name__)
logger = get_logger()


def bring_up_cluster_callable(**kwargs):
Expand Down
60 changes: 52 additions & 8 deletions runhouse/logger.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,59 @@
import logging
import os
import sys


def get_logger(name: str = __name__):
logger = logging.getLogger(name)
def get_logger(reinitialize: bool = False) -> logging.Logger:
"""
Creates and returns a logger with the specified name.
level = os.getenv("RH_LOG_LEVEL")
if level:
try:
logger.setLevel(getattr(logging, level.upper()))
except AttributeError as e:
raise e
Ensures a universal logger configuration across the codebase with the format:
"levelname - asctime - filename:lineno - message"
Args:
name (str): Name of the logger. Defaults to None, which gets the root logger.
Returns:
logging.Logger: Configured logger instance.
"""
# Create or retrieve the logger
logger = logging.getLogger("_rh_universal_logger")

level = os.getenv("RH_LOG_LEVEL") or "INFO"
try:
level = getattr(logging, level.upper())
except AttributeError as e:
raise e

if reinitialize:
for handler in logger.handlers:
logger.removeHandler(handler)

# Check if the logger has already been configured to prevent duplicate handlers
if not logger.handlers:

# Set the level for the new logger
logger.setLevel(level)

# Create a console handler that outputs to stdout
console_handler = logging.StreamHandler(stream=sys.stdout)

# The handler should capture all log levels, so we set the level to debug
console_handler.setLevel(logging.DEBUG)

# Define the log format
formatter = logging.Formatter(
"%(levelname)s | %(asctime)s | %(filename)s:%(lineno)d | %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)

# Apply the formatter to the handler
console_handler.setFormatter(formatter)

# Add the handler to the logger
logger.addHandler(console_handler)

# Prevent log messages from propagating to the root logger
logger.propagate = False

return logger
2 changes: 1 addition & 1 deletion runhouse/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
# For printing with typer
console = Console()

logger = get_logger(name=__name__)
logger = get_logger()


@app.command()
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/envs/conda_env.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from .env import Env

logger = get_logger(name=__name__)
logger = get_logger()


class CondaEnv(Env):
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/envs/env.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from runhouse.resources.resource import Resource
from runhouse.utils import run_with_logs

logger = get_logger(name=__name__)
logger = get_logger()


class Env(Resource):
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/folders/folder.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from runhouse.rns.utils.api import generate_uuid, relative_file_path
from runhouse.utils import locate_working_dir

logger = get_logger(name=__name__)
logger = get_logger()


class Folder(Resource):
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/folders/folder_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from runhouse.resources.folders.folder import Folder
from runhouse.resources.hardware.utils import _get_cluster_from

logger = get_logger(name=__name__)
logger = get_logger()


def folder(
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/folders/gcs_folder.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

from .folder import Folder

logger = get_logger(name=__name__)
logger = get_logger()


class GCSFolder(Folder):
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/folders/s3_folder.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
POLL_INTERVAL = 1
TIMEOUT_SECONDS = 3600

logger = get_logger(name=__name__)
logger = get_logger()


class S3Folder(Folder):
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/functions/aws_lambda.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
CRED_PATH = f"{Path.home()}/.aws/credentials"
LOG_GROUP_PREFIX = "/aws/lambda/"

logger = get_logger(name=__name__)
logger = get_logger()


class LambdaFunction(Function):
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/functions/function.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

from runhouse.resources.resource import Resource

logger = get_logger(name=__name__)
logger = get_logger()


class Function(Module):
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/functions/function_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from runhouse.resources.functions.function import Function
from runhouse.resources.packages import git_package

logger = get_logger(name=__name__)
logger = get_logger()


def function(
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/hardware/cluster.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@

from runhouse.servers.http import HTTPClient

logger = get_logger(name=__name__)
logger = get_logger()


class Cluster(Resource):
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/hardware/cluster_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from .on_demand_cluster import OnDemandCluster
from .sagemaker.sagemaker_cluster import SageMakerCluster

logger = get_logger(name=__name__)
logger = get_logger()

# Cluster factory method
def cluster(
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/hardware/on_demand_cluster.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@

from .cluster import Cluster

logger = get_logger(name=__name__)
logger = get_logger()


class OnDemandCluster(Cluster):
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/hardware/ray_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

from runhouse.logger import get_logger

logger = get_logger(name=__name__)
logger = get_logger()


def check_for_existing_ray_instance():
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/hardware/sagemaker/sagemaker_cluster.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
from runhouse.utils import generate_default_name


logger = get_logger(name=__name__)
logger = get_logger()
####################################################################################################
# Caching mechanisms for SSHTunnelForwarder
####################################################################################################
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/hardware/sky/command_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
##### RH modification #####
from runhouse.logger import get_logger

logger = get_logger(name=__name__)
logger = get_logger()
##### RH modification #####


Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/hardware/sky/subprocess_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

from runhouse.logger import get_logger

logger = get_logger(name=__name__)
logger = get_logger()

class CommandError(Exception):
"""Raised when a command fails.
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/hardware/sky_ssh_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
)


logger = get_logger(name=__name__)
logger = get_logger()

# Get rid of the constant "Found credentials in shared credentials file: ~/.aws/credentials" message
try:
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/module.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@
"_dumb_signature_cache",
]

logger = get_logger(name=__name__)
logger = get_logger()


class Module(Resource):
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/packages/package.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@

INSTALL_METHODS = {"local", "reqs", "pip", "conda", "rh"}

logger = get_logger(name=__name__)
logger = get_logger()


class CodeSyncError(Exception):
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/provenance.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from runhouse.rns.utils.api import log_timestamp, resolve_absolute_path
from runhouse.utils import StreamTee

logger = get_logger(name=__name__)
logger = get_logger()


class RunStatus(str, Enum):
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/resource.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
ResourceVisibility,
)

logger = get_logger(name=__name__)
logger = get_logger()


class Resource:
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/secrets/provider_secrets/ssh_secret.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from runhouse.resources.hardware.cluster import Cluster
from runhouse.resources.secrets.provider_secrets.provider_secret import ProviderSecret

logger = get_logger(name=__name__)
logger = get_logger()


class SSHSecret(ProviderSecret):
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/secrets/secret.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from runhouse.rns.utils.api import load_resp_content, read_resp_data
from runhouse.utils import generate_default_name

logger = get_logger(name=__name__)
logger = get_logger()


class Secret(Resource):
Expand Down
2 changes: 1 addition & 1 deletion runhouse/resources/secrets/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
USER_ENDPOINT = "user/secret"


logger = get_logger(name=__name__)
logger = get_logger()


def load_config(name: str, endpoint: str = USER_ENDPOINT):
Expand Down
2 changes: 1 addition & 1 deletion runhouse/rns/defaults.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

req_ctx = contextvars.ContextVar("rh_ctx", default={})

logger = get_logger(name=__name__)
logger = get_logger()


class Defaults:
Expand Down
2 changes: 1 addition & 1 deletion runhouse/rns/login.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from runhouse.globals import configs, rns_client
from runhouse.logger import get_logger

logger = get_logger(name=__name__)
logger = get_logger()


def is_interactive():
Expand Down
2 changes: 1 addition & 1 deletion runhouse/rns/rns_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

from runhouse.utils import locate_working_dir

logger = get_logger(name=__name__)
logger = get_logger()


# This is a copy of the Pydantic model that we use to validate in Den
Expand Down
2 changes: 1 addition & 1 deletion runhouse/rns/top_level_rns_fns.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from runhouse.logger import get_logger
from runhouse.servers.obj_store import ClusterServletSetupOption

logger = get_logger(name=__name__)
logger = get_logger()

logging.getLogger("numexpr").setLevel(logging.WARNING)

Expand Down
2 changes: 1 addition & 1 deletion runhouse/servers/autostop_helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

from runhouse.logger import get_logger

logger = get_logger(name=__name__)
logger = get_logger()


class AutostopHelper:
Expand Down
2 changes: 1 addition & 1 deletion runhouse/servers/caddy/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from runhouse.constants import DEFAULT_SERVER_PORT
from runhouse.logger import get_logger

logger = get_logger(name=__name__)
logger = get_logger()
SYSTEMCTL_ERROR = "systemctl: command not found"


Expand Down
2 changes: 1 addition & 1 deletion runhouse/servers/cluster_servlet.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
from runhouse.servers.http.auth import AuthCache
from runhouse.utils import ColoredFormatter, sync_function

logger = get_logger(name=__name__)
logger = get_logger()


class ClusterServletError(Exception):
Expand Down
2 changes: 1 addition & 1 deletion runhouse/servers/env_servlet.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from runhouse.servers.obj_store import ClusterServletSetupOption
from runhouse.utils import arun_in_thread, get_node_ip

logger = get_logger(name=__name__)
logger = get_logger()


def error_handling_decorator(func):
Expand Down
2 changes: 1 addition & 1 deletion runhouse/servers/http/auth.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from runhouse.rns.utils.api import load_resp_content, ResourceAccess
from runhouse.servers.http.http_utils import username_from_token

logger = get_logger(name=__name__)
logger = get_logger()


class AuthCache:
Expand Down
2 changes: 1 addition & 1 deletion runhouse/servers/http/certs.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@

from runhouse.rns.utils.api import resolve_absolute_path

logger = get_logger(name=__name__)
logger = get_logger()


class TLSCertConfig:
Expand Down
Loading
Loading