Skip to content

Commit

Permalink
add link to logs and commit
Browse files Browse the repository at this point in the history
fixed the dq cli test
  • Loading branch information
sushi30 committed Sep 25, 2024
1 parent 71bcc77 commit 214d5f3
Show file tree
Hide file tree
Showing 4 changed files with 39 additions and 25 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/py-cli-e2e-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -195,8 +195,8 @@ jobs:
with:
payload: |
{
"text": "🔥 Failed E2E Test for: ${{ matrix.e2e-test }} 🔥"
}
"text": "🔥 Failed E2E Test for: ${{ matrix.e2e-test }} 🔥\nLogs: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\nCommit: ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }}"
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.E2E_SLACK_WEBHOOK }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
Expand Down
7 changes: 6 additions & 1 deletion ingestion/src/metadata/utils/dict.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
class CustomDict(dict):
"""
A custom dictionary class that extends functionality.
"""


class ExtendedDict(dict):
def lower_case_keys(self):
return {k.lower(): v for k, v in self.items()}
49 changes: 29 additions & 20 deletions ingestion/tests/cli_e2e/base/test_cli_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -220,11 +220,11 @@ def test_data_quality(self) -> None:
return
self.delete_table_and_view()
self.create_table_and_view()
self.build_config_file()
self.run_command()
table: Table = self.openmetadata.get_by_name(
Table, self.get_data_quality_table(), nullable=False
)
self.build_config_file()
self.run_command()
test_case_definitions = self.get_test_case_definitions()
self.build_config_file(
E2EType.DATA_QUALITY,
Expand All @@ -236,26 +236,35 @@ def test_data_quality(self) -> None:
},
)
result = self.run_command("test")
sink_status, source_status = self.retrieve_statuses(result)
self.assert_status_for_data_quality(source_status, sink_status)
test_case_entities = [
self.openmetadata.get_by_name(
OMTestCase,
".".join([table.fullyQualifiedName.root, tcd.name]),
fields=["*"],
nullable=False,
)
for tcd in test_case_definitions
]
expected = self.get_expected_test_case_results()
try:
for test_case, expected in zip(test_case_entities, expected):
assert_equal_pydantic_objects(expected, test_case.testCaseResult)
finally:
for tc in test_case_entities:
self.openmetadata.delete(
OMTestCase, tc.id, recursive=True, hard_delete=True
sink_status, source_status = self.retrieve_statuses(result)
self.assert_status_for_data_quality(source_status, sink_status)
test_case_entities = [
self.openmetadata.get_by_name(
OMTestCase,
".".join([table.fullyQualifiedName.root, tcd.name]),
fields=["*"],
nullable=False,
)
for tcd in test_case_definitions
]
expected = self.get_expected_test_case_results()
try:
for test_case, expected in zip(test_case_entities, expected):
assert_equal_pydantic_objects(
expected.model_copy(
update={"timestamp": test_case.testCaseResult.timestamp}
),
test_case.testCaseResult,
)
finally:
for tc in test_case_entities:
self.openmetadata.delete(
OMTestCase, tc.id, recursive=True, hard_delete=True
)
except AssertionError:
print(result)
raise

def retrieve_table(self, table_name_fqn: str) -> Table:
return self.openmetadata.get_by_name(entity=Table, fqn=table_name_fqn)
Expand Down
4 changes: 2 additions & 2 deletions ingestion/tests/cli_e2e/test_cli_snowflake.py
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,7 @@ def wait_for_query_log(cls, timeout=600):
raise TimeoutError(f"Query log not updated for {timeout} seconds")

def get_data_quality_table(self):
return "e2e_snowflake.E2E_DB.E2E_TEST.PERSONS"
return self.fqn_created_table()

def get_test_case_definitions(self) -> List[TestCaseDefinition]:
return [
Expand All @@ -343,4 +343,4 @@ def get_test_case_definitions(self) -> List[TestCaseDefinition]:
]

def get_expected_test_case_results(self):
return [TestCaseResult(testCaseStatus=TestCaseStatus.Success)]
return [TestCaseResult(testCaseStatus=TestCaseStatus.Success, timestamp=0)]

0 comments on commit 214d5f3

Please sign in to comment.