Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sns reformat #69

Merged
merged 6 commits into from
May 3, 2023
Merged

Sns reformat #69

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
86cf6c3
reformat sns logic
May 2, 2023
bb45b60
reformat sns logic
May 2, 2023
ed4a066
Merge branch 'sns_reformat' of github.com:provectus/data-quality-gate…
May 3, 2023
de86cb5
fixed only_failed param init
May 3, 2023
0f98ae6
fixed bug_name param init
May 3, 2023
a255a57
fixed success message
May 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion configs/pipeline.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
"old_suite_name": "planner_raw_softfail",
"reuse_suite": false,
"use_old_suite_only": false,
"autobug": false
"autobug": false,
"only_failed": true
}

}
161 changes: 91 additions & 70 deletions functions/report_push/push_data_report.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ def handler(event, context):
run_name = report.get('run_name')
bucket = s3.Bucket(qa_bucket)
created_bug_count = 0
bug_name = []
items = []
df = wr.s3.read_json(
path=[
Expand Down Expand Up @@ -77,87 +78,30 @@ def handler(event, context):
for item in items:
batch.put_item(Item=item)

pipeline_config = json.loads(
wr.s3.read_json(path=f"s3://{qa_bucket}/test_configs/pipeline.json").to_json())
try:
pipeline_config = json.loads(
wr.s3.read_json(path=f"s3://{qa_bucket}/test_configs/pipeline.json").to_json())
autobug = pipeline_config[run_name]['autobug']
except KeyError:
autobug = False
print(f"Can't find config for {run_name}")
print(f"Can't find autobug param for {run_name}")
try:
only_failed = pipeline_config[run_name]["only_failed"]
except KeyError:
only_failed = True
print(f"Can't find only_failed param for {run_name}")


if autobug:
jira_project_key = os.environ['JIRA_PROJECT_KEY']
auth_in_jira()
created_bug_count, bug_name = create_jira_bugs_from_allure_result(bucket, key, replaced_allure_links, suite,
jira_project_key)
cloudwatch.put_metric_data(
Namespace='Data-QA',
MetricData=[
{
'MetricName': 'bug_created_count',
'Dimensions': [
{
'Name': 'table_name',
'Value': suite
},
{
'Name': 'Environment',
'Value': environment
}
],
'Value': created_bug_count,
'Unit': 'Count'
},
]
)
if not bug_name:
if sns_bugs_topic:
sns_message = {
"table": suite,
"pipeline_step": run_name,
"source_name": file,
"new_bugs": bug_name,
"new_bugs_count": created_bug_count,
"allure_report": f"https://{replaced_allure_links}"
}
sns.publish(TopicArn=sns_bugs_topic,
Message=json.dumps(sns_message),
MessageStructure='json',)

else:
cloudwatch.put_metric_data(
Namespace='Data-QA',
MetricData=[
{
'MetricName': 'suite_failed_count',
'Dimensions': [
{
'Name': 'table_name',
'Value': suite
},
{
'Name': 'Environment',
'Value': environment
}
],
'Value': failed,
'Unit': 'Count'
},
]
)
if sns_bugs_topic:
sns_message = {
"table": suite,
"pipeline_step": run_name,
"source_name": file,
"all": total,
"failed": failed,
"passed": passed,
"allure_report": f"https://{replaced_allure_links}"
}
sns.publish(TopicArn=sns_bugs_topic,
Message=json.dumps({"default": json.dumps(sns_message)}),
MessageStructure='json', )
push_cloudwatch_metrics(suite, environment, failed, created_bug_count)
push_sns_message(suite, run_name, file, bug_name, created_bug_count, replaced_allure_links, total, failed, passed,
sns_bugs_topic, only_failed)

return report


Expand Down Expand Up @@ -186,3 +130,80 @@ def create_jira_bugs_from_allure_result(bucket, key, replaced_allure_links, suit
jira_project_key,
))
return created_bug_count, bug_name


def push_sns_message(suite, run_name, file, bug_name, created_bug_count, replaced_allure_links, total, failed, passed,
sns_bugs_topic, only_failed):
message_structure = 'json'
if created_bug_count > 0:
sns_message = {
"table": suite,
"pipeline_step": run_name,
"source_name": file,
"new_bugs": bug_name,
"new_bugs_count": created_bug_count,
"allure_report": f"https://{replaced_allure_links}"
}
elif failed > 0:
sns_message = {
"table": suite,
"pipeline_step": run_name,
"source_name": file,
"all": total,
"failed": failed,
"passed": passed,
"allure_report": f"https://{replaced_allure_links}"
}
else:
sns_message = f"All {total} tests for source: {suite} were successful"
message_structure = 'string'

if message_structure == 'json':
sns_message = json.dumps({"default": json.dumps(sns_message)})

if only_failed and message_structure == 'json' or not only_failed:
sns.publish(TopicArn=sns_bugs_topic,
Message=sns_message,
MessageStructure=message_structure)


def push_cloudwatch_metrics(suite, environment, failed, created_bug_count):
if created_bug_count > 0:
metric_data = {
'MetricName': 'bug_created_count',
'Dimensions': [
{
'Name': 'table_name',
'Value': suite
},
{
'Name': 'Environment',
'Value': environment
}
],
'Value': created_bug_count,
'Unit': 'Count'
}
else:
metric_data = {
'MetricName': 'suite_failed_count',
'Dimensions': [
{
'Name': 'table_name',
'Value': suite
},
{
'Name': 'Environment',
'Value': environment
}
],
'Value': failed,
'Unit': 'Count'
}

cloudwatch.put_metric_data(
Namespace='Data-QA',
MetricData=[
metric_data,
]
)