Skip to content

Commit a14ed36

Browse files
leleamolleleamol
authored
Performance benchmarking for PyTorch (#78)
* Pytorch performance tests * Fixed the estimator * Fixed the training script for correct metrics generation * Added train duration metrics in the training script * Adjusted the alarm values * Adjusted the alarm values * Fixed the job name for no smdebug and no profiler * Optimized the training script and added comments in the driver script. * Updated the scripts for framework only training job * Removed the unenecessary code. * Updating the instance types.
1 parent b9a8634 commit a14ed36

File tree

4 files changed

+660
-0
lines changed

4 files changed

+660
-0
lines changed

‎benchmarks/pytorch_alarm_spec.json‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
[
2+
{
3+
"AlarmName": "SMProfiler PT1.5 p3.16x change in TrainingTime for profiler_fully_enabled vs smdebug_enabled",
4+
"ActionsEnabled": true,
5+
"MetricName": "Change in TrainingTimeInSeconds for profiler_fully_enabled compared to smdebug_enabled",
6+
"Namespace": "smprofiler-PT1.5-resnet50-ml.p3.16xlarge",
7+
"Statistic": "Maximum",
8+
"Period": 86400,
9+
"EvaluationPeriods": 1,
10+
"Threshold": 35,
11+
"ComparisonOperator": "GreaterThanThreshold",
12+
"TreatMissingData": "missing"
13+
},
14+
{
15+
"AlarmName": "SMProfiler PT1.5 p3.16x change in TrainingTime for system_profiler_enabled vs smdebug_enabled",
16+
"ActionsEnabled": true,
17+
"MetricName": "Change in TrainingTimeInSeconds for system_profiler_enabled compared to smdebug_enabled",
18+
"Namespace": "smprofiler-PT1.5-resnet50-ml.p3.16xlarge",
19+
"Statistic": "Maximum",
20+
"Period": 86400,
21+
"EvaluationPeriods": 1,
22+
"Threshold": 6,
23+
"ComparisonOperator": "GreaterThanThreshold",
24+
"TreatMissingData": "missing"
25+
},
26+
{
27+
"AlarmName": "SMProfiler PT1.5 p3.16x change in TrainingTime for framework_profiler_enabled vs smdebug_enabled",
28+
"ActionsEnabled": true,
29+
"MetricName": "Change in TrainingTimeInSeconds for framework_profiler_enabled compared to smdebug_enabled",
30+
"Namespace": "smprofiler-PT1.5-resnet50-ml.p3.16xlarge",
31+
"Statistic": "Maximum",
32+
"Period": 86400,
33+
"EvaluationPeriods": 1,
34+
"Threshold": 35,
35+
"ComparisonOperator": "GreaterThanThreshold",
36+
"TreatMissingData": "missing"
37+
},
38+
{
39+
"AlarmName": "SMProfiler PT1.5 p3.16x change in TrainingTime for profiler_fully_enabled vs profiler_disabled",
40+
"ActionsEnabled": true,
41+
"MetricName": "Change in TrainingTimeInSeconds for profiler_fully_enabled compared to profiler_disabled",
42+
"Namespace": "smprofiler-PT1.5-resnet50-ml.p3.16xlarge",
43+
"Statistic": "Maximum",
44+
"Period": 86400,
45+
"EvaluationPeriods": 1,
46+
"Threshold": 38,
47+
"ComparisonOperator": "GreaterThanThreshold",
48+
"TreatMissingData": "missing"
49+
},
50+
{
51+
"AlarmName": "SMProfiler PT1.5 p3.16x change in TrainingTime for system_profiler_enabled vs profiler_disabled",
52+
"ActionsEnabled": true,
53+
"MetricName": "Change in TrainingTimeInSeconds for system_profiler_enabled compared to profiler_disabled",
54+
"Namespace": "smprofiler-PT1.5-resnet50-ml.p3.16xlarge",
55+
"Statistic": "Maximum",
56+
"Period": 86400,
57+
"EvaluationPeriods": 1,
58+
"Threshold": 4,
59+
"ComparisonOperator": "GreaterThanThreshold",
60+
"TreatMissingData": "missing"
61+
},
62+
{
63+
"AlarmName": "SMProfiler PT1.5 p3.16x change in TrainingTime for framework_profiler_enabled vs profiler_disabled",
64+
"ActionsEnabled": true,
65+
"MetricName": "Change in TrainingTimeInSeconds for framework_profiler_enabled compared to profiler_disabled",
66+
"Namespace": "smprofiler-PT1.5-resnet50-ml.p3.16xlarge",
67+
"Statistic": "Maximum",
68+
"Period": 86400,
69+
"EvaluationPeriods": 1,
70+
"Threshold": 38,
71+
"ComparisonOperator": "GreaterThanThreshold",
72+
"TreatMissingData": "missing"
73+
},
74+
{
75+
"AlarmName": "SMProfiler PT1.5 p2.8x change in TrainingTime for profiler_fully_enabled vs smdebug_enabled",
76+
"ActionsEnabled": true,
77+
"MetricName": "Change in TrainingTimeInSeconds for profiler_fully_enabled compared to smdebug_enabled",
78+
"Namespace": "smprofiler-PT1.5-resnet50-ml.p2.8xlarge",
79+
"Statistic": "Maximum",
80+
"Period": 86400,
81+
"EvaluationPeriods": 1,
82+
"Threshold": 4,
83+
"ComparisonOperator": "GreaterThanThreshold",
84+
"TreatMissingData": "missing"
85+
},
86+
{
87+
"AlarmName": "SMProfiler PT1.5 p2.8x change in TrainingTime for system_profiler_enabled vs smdebug_enabled",
88+
"ActionsEnabled": true,
89+
"MetricName": "Change in TrainingTimeInSeconds for system_profiler_enabled compared to smdebug_enabled",
90+
"Namespace": "smprofiler-PT1.5-resnet50-ml.p2.8xlarge",
91+
"Statistic": "Maximum",
92+
"Period": 86400,
93+
"EvaluationPeriods": 1,
94+
"Threshold": 30,
95+
"ComparisonOperator": "GreaterThanThreshold",
96+
"TreatMissingData": "missing"
97+
},
98+
{
99+
"AlarmName": "SMProfiler PT1.5 p2.8x change in TrainingTime for framework_profiler_enabled vs smdebug_enabled",
100+
"ActionsEnabled": true,
101+
"MetricName": "Change in TrainingTimeInSeconds for framework_profiler_enabled compared to smdebug_enabled",
102+
"Namespace": "smprofiler-PT1.5-resnet50-ml.p2.8xlarge",
103+
"Statistic": "Maximum",
104+
"Period": 86400,
105+
"EvaluationPeriods": 1,
106+
"Threshold": 35,
107+
"ComparisonOperator": "GreaterThanThreshold",
108+
"TreatMissingData": "missing"
109+
},
110+
{
111+
"AlarmName": "SMProfiler PT1.5 p2.8x change in TrainingTime for profiler_fully_enabled vs profiler_disabled",
112+
"ActionsEnabled": true,
113+
"MetricName": "Change in TrainingTimeInSeconds for profiler_fully_enabled compared to profiler_disabled",
114+
"Namespace": "smprofiler-PT1.5-resnet50-ml.p2.8xlarge",
115+
"Statistic": "Maximum",
116+
"Period": 86400,
117+
"EvaluationPeriods": 1,
118+
"Threshold": 30,
119+
"ComparisonOperator": "GreaterThanThreshold",
120+
"TreatMissingData": "missing"
121+
},
122+
{
123+
"AlarmName": "SMProfiler PT1.5 p2.8x change in TrainingTime for system_profiler_enabled vs profiler_disabled",
124+
"ActionsEnabled": true,
125+
"MetricName": "Change in TrainingTimeInSeconds for system_profiler_enabled compared to profiler_disabled",
126+
"Namespace": "smprofiler-PT1.5-resnet50-ml.p2.8xlarge",
127+
"Statistic": "Maximum",
128+
"Period": 86400,
129+
"EvaluationPeriods": 1,
130+
"Threshold": 20,
131+
"ComparisonOperator": "GreaterThanThreshold",
132+
"TreatMissingData": "missing"
133+
},
134+
{
135+
"AlarmName": "SMProfiler PT1.5 p2.8x change in TrainingTime for framework_profiler_enabled vs profiler_disabled",
136+
"ActionsEnabled": true,
137+
"MetricName": "Change in TrainingTimeInSeconds for framework_profiler_enabled compared to profiler_disabled",
138+
"Namespace": "smprofiler-PT1.5-resnet50-ml.p2.8xlarge",
139+
"Statistic": "Maximum",
140+
"Period": 86400,
141+
"EvaluationPeriods": 1,
142+
"Threshold": 5,
143+
"ComparisonOperator": "GreaterThanThreshold",
144+
"TreatMissingData": "missing"
145+
}
146+
]

0 commit comments

Comments
 (0)