Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix petits logging regressions from feature/click-cli merge #6940

Merged
merged 7 commits into from
Feb 14, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changes/unreleased/Fixes-20230213-130522.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Fixes
body: Readd initialization events, --log-cache-events in new CLI
time: 2023-02-13T13:05:22.989477+01:00
custom:
Author: jtcohen6
Issue: "6933"
17 changes: 14 additions & 3 deletions core/dbt/cli/requires.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
from dbt.version import installed as installed_version
from dbt.adapters.factory import adapter_management, register_adapter
from dbt.flags import set_flags
from dbt.flags import set_flags, get_flag_dict
from dbt.cli.flags import Flags
from dbt.config import RuntimeConfig
from dbt.config.runtime import load_project, load_profile, UnsetProfile
from dbt.events.functions import setup_event_logger
from dbt.events.functions import setup_event_logger, fire_event, LOG_VERSION
from dbt.events.types import MainReportVersion, MainReportArgs, MainTrackingUserState
from dbt.exceptions import DbtProjectError
from dbt.parser.manifest import ManifestLoader, write_manifest
from dbt.profiler import profiler
from dbt.tracking import initialize_from_flags, track_run
from dbt.tracking import active_user, initialize_from_flags, track_run
from dbt.utils import cast_dict_to_dict_of_strings

from click import Context
from functools import update_wrapper
Expand Down Expand Up @@ -37,6 +40,14 @@ def wrapper(*args, **kwargs):
flags.DEBUG,
)

# Now that we have our logger, fire away!
fire_event(MainReportVersion(version=str(installed_version), log_version=LOG_VERSION))
flags_dict_str = cast_dict_to_dict_of_strings(get_flag_dict())
fire_event(MainReportArgs(args=flags_dict_str))
Comment on lines +45 to +46
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The values needs to be stringified first, because it's defined as Dict[str, str]:

// A002
message MainReportArgs {
map<string, string> args = 1;
}

(The good news is, this was caught by some of our functional tests!)

As discussed during BLG on Friday, we should investigate if we can support the Struct proto type (Dict[str, Any]): #6803 (comment)


if active_user is not None: # mypy appeasement, always true
fire_event(MainTrackingUserState(user_state=active_user.state()))

# Profiling
if flags.RECORD_TIMING_INFO:
ctx.with_resource(profiler(enable=True, outfile=flags.RECORD_TIMING_INFO))
Expand Down
Binary file modified core/dbt/docs/build/doctrees/environment.pickle
Binary file not shown.
22 changes: 14 additions & 8 deletions core/dbt/events/functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,7 @@
from dbt.events.eventmgr import EventManager, LoggerConfig, LineFormat, NoFilter
from dbt.events.helpers import env_secrets, scrub_secrets
from dbt.events.types import Formatting
from dbt.flags import get_flags
import dbt.flags as flags_module
from dbt.flags import get_flags, ENABLE_LEGACY_LOGGER
from dbt.logger import GLOBAL_LOGGER, make_log_dir_if_missing
from functools import partial
import json
Expand All @@ -23,7 +22,7 @@ def setup_event_logger(log_path: str, log_format: str, use_colors: bool, debug:
cleanup_event_logger()
make_log_dir_if_missing(log_path)

if get_flags().ENABLE_LEGACY_LOGGER:
if ENABLE_LEGACY_LOGGER:
EVENT_MANAGER.add_logger(_get_logbook_log_config(debug))
else:
EVENT_MANAGER.add_logger(_get_stdout_config(log_format, debug, use_colors))
Expand All @@ -43,12 +42,18 @@ def setup_event_logger(log_path: str, log_format: str, use_colors: bool, debug:


def _get_stdout_config(log_format: str, debug: bool, use_colors: bool) -> LoggerConfig:
flags = get_flags()
fmt = LineFormat.PlainText
if log_format == "json":
fmt = LineFormat.Json
elif debug:
fmt = LineFormat.DebugText
level = EventLevel.DEBUG if debug else EventLevel.INFO
# We don't have access to these values when we need to setup the default stdout logger!
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@peterallenwebb I don't think we'll be able to solve #6847 in the same way you did #6886, because we now need a click Context to access flags.QUIET (even if it's passed in on the CLI), and we don't have access to that as early as we used to.

That approach was always a bit opportunistic, so I don't feel like this is a huge deal. Open to ideas for sure.

log_cache_events = (
bool(flags.LOG_CACHE_EVENTS) if hasattr(flags, "LOG_CACHE_EVENTS") else False
)
quiet = bool(flags.QUIET) if hasattr(flags, "QUIET") else False
return LoggerConfig(
name="stdout_log",
level=level,
Expand All @@ -57,9 +62,9 @@ def _get_stdout_config(log_format: str, debug: bool, use_colors: bool) -> Logger
scrubber=env_scrubber,
filter=partial(
_stdout_filter,
bool(flags_module.LOG_CACHE_EVENTS),
log_cache_events,
debug,
bool(flags_module.QUIET),
quiet,
log_format,
),
output_stream=sys.stdout,
Expand Down Expand Up @@ -124,10 +129,11 @@ def cleanup_event_logger():
# currently fire before logs can be configured by setup_event_logger(), we
# create a default configuration with default settings and no file output.
EVENT_MANAGER: EventManager = EventManager()
# Problem: This needs to be set *BEFORE* we've resolved any flags (even CLI params)
EVENT_MANAGER.add_logger(
_get_logbook_log_config(flags_module.DEBUG) # type: ignore
if flags_module.ENABLE_LEGACY_LOGGER
else _get_stdout_config(flags_module.LOG_FORMAT, flags_module.DEBUG, flags_module.USE_COLORS) # type: ignore
_get_logbook_log_config(debug=False) # type: ignore
if ENABLE_LEGACY_LOGGER
else _get_stdout_config(log_format="text", debug=False, use_colors=True) # type: ignore
)

# This global, and the following two functions for capturing stdout logs are
Expand Down