Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

backport 7388 to 1.5.latest #7887

Closed
wants to merge 1 commit into from
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .changes/unreleased/Under the Hood-20230418-121236.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
kind: Under the Hood
body: 'Remove noisy parsing events: GenericTestFileParse, MacroFileParse, Note events
for static model parsing'
time: 2023-04-18T12:12:36.928665+02:00
custom:
Author: jtcohen6
Issue: "6671"
5 changes: 5 additions & 0 deletions core/dbt/events/functions.py
Original file line number Diff line number Diff line change
@@ -253,6 +253,11 @@ def fire_event_if(
fire_event(lazy_e(), level=level)


# a special case of fire_event_if, to only fire events in our unit/functional tests
def fire_event_if_test(lazy_e: Callable[[], BaseEvent], level: EventLevel = None) -> None:
fire_event_if(conditional=("pytest" in sys.modules), lazy_e=lazy_e, level=level)


# top-level method for accessing the new eventing system
# this is where all the side effects happen branched by event type
# (i.e. - mutating the event history, printing to stdout, logging
20 changes: 0 additions & 20 deletions core/dbt/events/types.proto
Original file line number Diff line number Diff line change
@@ -875,26 +875,6 @@ message ParsePerfInfoPathMsg {
ParsePerfInfoPath data = 2;
}

// I011
message GenericTestFileParse {
string path = 1;
}

message GenericTestFileParseMsg {
EventInfo info = 1;
GenericTestFileParse data = 2;
}

// I012
message MacroFileParse {
string path = 1;
}

message MacroFileParseMsg {
EventInfo info = 1;
MacroFileParse data = 2;
}

// Skipping I013

// I014
14 changes: 2 additions & 12 deletions core/dbt/events/types.py
Original file line number Diff line number Diff line change
@@ -828,20 +828,10 @@ def message(self) -> str:
return f"Performance info: {self.path}"


class GenericTestFileParse(DebugLevel):
def code(self):
return "I011"

def message(self) -> str:
return f"Parsing {self.path}"
# Removed I011: GenericTestFileParse


class MacroFileParse(DebugLevel):
def code(self):
return "I012"

def message(self) -> str:
return f"Parsing {self.path}"
# Removed I012: MacroFileParse


# Skipping I013
1,418 changes: 701 additions & 717 deletions core/dbt/events/types_pb2.py

Large diffs are not rendered by default.

5 changes: 0 additions & 5 deletions core/dbt/parser/generic_test.py
Original file line number Diff line number Diff line change
@@ -7,13 +7,10 @@
from dbt.contracts.graph.nodes import GenericTestNode, Macro
from dbt.contracts.graph.unparsed import UnparsedMacro
from dbt.contracts.files import SourceFile
from dbt.events.functions import fire_event
from dbt.events.types import GenericTestFileParse
from dbt.node_types import NodeType
from dbt.parser.base import BaseParser
from dbt.parser.search import FileBlock
from dbt.utils import MACRO_PREFIX
from dbt.flags import get_flags


class GenericTestParser(BaseParser[GenericTestNode]):
@@ -87,8 +84,6 @@ def parse_file(self, block: FileBlock):
source_file = block.file
assert isinstance(source_file.contents, str)
original_file_path = source_file.path.original_file_path
if get_flags().MACRO_DEBUGGING:
fire_event(GenericTestFileParse(path=original_file_path))

# this is really only used for error messages
base_node = UnparsedMacro(
5 changes: 0 additions & 5 deletions core/dbt/parser/macros.py
Original file line number Diff line number Diff line change
@@ -7,13 +7,10 @@
from dbt.contracts.graph.nodes import Macro
from dbt.contracts.files import FilePath, SourceFile
from dbt.exceptions import ParsingError
from dbt.events.functions import fire_event
from dbt.events.types import MacroFileParse
from dbt.node_types import NodeType
from dbt.parser.base import BaseParser
from dbt.parser.search import FileBlock, filesystem_search
from dbt.utils import MACRO_PREFIX
from dbt.flags import get_flags


class MacroParser(BaseParser[Macro]):
@@ -93,8 +90,6 @@ def parse_file(self, block: FileBlock):
source_file = block.file
assert isinstance(source_file.contents, str)
original_file_path = source_file.path.original_file_path
if get_flags().MACRO_DEBUGGING:
fire_event(MacroFileParse(path=original_file_path))

# this is really only used for error messages
base_node = UnparsedMacro(
48 changes: 28 additions & 20 deletions core/dbt/parser/models.py
Original file line number Diff line number Diff line change
@@ -3,7 +3,7 @@
from dbt.contracts.graph.nodes import ModelNode, RefArgs
from dbt.events.base_types import EventLevel
from dbt.events.types import Note
from dbt.events.functions import fire_event
from dbt.events.functions import fire_event_if_test
from dbt.flags import get_flags
from dbt.node_types import NodeType, ModelLanguage
from dbt.parser.base import SimpleSQLParser
@@ -246,8 +246,8 @@ def render_update(self, node: ModelNode, config: ContextConfig) -> None:
elif not flags.STATIC_PARSER:
# jinja rendering
super().render_update(node, config)
fire_event(
Note(
fire_event_if_test(
lambda: Note(
msg=f"1605: jinja rendering because of STATIC_PARSER flag. file: {node.path}"
),
EventLevel.DEBUG,
@@ -285,8 +285,8 @@ def render_update(self, node: ModelNode, config: ContextConfig) -> None:

# sample the experimental parser only during a normal run
if exp_sample and not flags.USE_EXPERIMENTAL_PARSER:
fire_event(
Note(msg=f"1610: conducting experimental parser sample on {node.path}"),
fire_event_if_test(
lambda: Note(msg=f"1610: conducting experimental parser sample on {node.path}"),
EventLevel.DEBUG,
)
experimental_sample = self.run_experimental_parser(node)
@@ -318,8 +318,10 @@ def render_update(self, node: ModelNode, config: ContextConfig) -> None:
# sampling rng here, but the effect would be the same since we would only roll
# it 40% of the time. So I've opted to keep all the rng code colocated above.
if stable_sample and not flags.USE_EXPERIMENTAL_PARSER:
fire_event(
Note(msg=f"1611: conducting full jinja rendering sample on {node.path}"),
fire_event_if_test(
lambda: Note(
msg=f"1611: conducting full jinja rendering sample on {node.path}"
),
EventLevel.DEBUG,
)
# if this will _never_ mutate anything `self` we could avoid these deep copies,
@@ -356,8 +358,9 @@ def render_update(self, node: ModelNode, config: ContextConfig) -> None:
else:
# jinja rendering
super().render_update(node, config)
fire_event(
Note(msg=f"1602: parser fallback to jinja rendering on {node.path}"),
# only for test purposes
fire_event_if_test(
lambda: Note(msg=f"1602: parser fallback to jinja rendering on {node.path}"),
EventLevel.DEBUG,
)

@@ -395,8 +398,8 @@ def run_static_parser(self, node: ModelNode) -> Optional[Union[str, Dict[str, Li
# this log line is used for integration testing. If you change
# the code at the beginning of the line change the tests in
# test/integration/072_experimental_parser_tests/test_all_experimental_parser.py
fire_event(
Note(
fire_event_if_test(
lambda: Note(
msg=f"1601: detected macro override of ref/source/config in the scope of {node.path}"
),
EventLevel.DEBUG,
@@ -406,15 +409,19 @@ def run_static_parser(self, node: ModelNode) -> Optional[Union[str, Dict[str, Li
# run the stable static parser and return the results
try:
statically_parsed = py_extract_from_source(node.raw_code)
fire_event(
Note(msg=f"1699: static parser successfully parsed {node.path}"), EventLevel.DEBUG
fire_event_if_test(
lambda: Note(msg=f"1699: static parser successfully parsed {node.path}"),
EventLevel.DEBUG,
)
return _shift_sources(statically_parsed)
# if we want information on what features are barring the static
# parser from reading model files, this is where we would add that
# since that information is stored in the `ExtractionError`.
except ExtractionError:
fire_event(Note(msg=f"1603: static parser failed on {node.path}"), EventLevel.DEBUG)
fire_event_if_test(
lambda: Note(msg=f"1603: static parser failed on {node.path}"),
EventLevel.DEBUG,
)
return "cannot_parse"

def run_experimental_parser(
@@ -425,8 +432,8 @@ def run_experimental_parser(
# this log line is used for integration testing. If you change
# the code at the beginning of the line change the tests in
# test/integration/072_experimental_parser_tests/test_all_experimental_parser.py
fire_event(
Note(
fire_event_if_test(
lambda: Note(
msg=f"1601: detected macro override of ref/source/config in the scope of {node.path}"
),
EventLevel.DEBUG,
@@ -439,17 +446,18 @@ def run_experimental_parser(
# experimental features. Change `py_extract_from_source` to the new
# experimental call when we add additional features.
experimentally_parsed = py_extract_from_source(node.raw_code)
fire_event(
Note(msg=f"1698: experimental parser successfully parsed {node.path}"),
fire_event_if_test(
lambda: Note(msg=f"1698: experimental parser successfully parsed {node.path}"),
EventLevel.DEBUG,
)
return _shift_sources(experimentally_parsed)
# if we want information on what features are barring the experimental
# parser from reading model files, this is where we would add that
# since that information is stored in the `ExtractionError`.
except ExtractionError:
fire_event(
Note(msg=f"1604: experimental parser failed on {node.path}"), EventLevel.DEBUG
fire_event_if_test(
lambda: Note(msg=f"1604: experimental parser failed on {node.path}"),
EventLevel.DEBUG,
)
return "cannot_parse"

2 changes: 0 additions & 2 deletions tests/unit/test_events.py
Original file line number Diff line number Diff line change
@@ -186,8 +186,6 @@ def test_event_codes(self):
types.InvalidValueForField(field_name="test", field_value="test"),
types.ValidationWarning(resource_type="model", field_name="access", node_name="my_macro"),
types.ParsePerfInfoPath(path=""),
types.GenericTestFileParse(path=""),
types.MacroFileParse(path=""),
types.PartialParsingErrorProcessingFile(file=""),
types.PartialParsingFile(file_id=""),
types.PartialParsingError(exc_info={}),