diff --git a/tests/perf/tooling/build_perf_tests.py b/products/nativescript/perf_helpers.py similarity index 70% rename from tests/perf/tooling/build_perf_tests.py rename to products/nativescript/perf_helpers.py index 7fae5281..b7479f01 100644 --- a/tests/perf/tooling/build_perf_tests.py +++ b/products/nativescript/perf_helpers.py @@ -1,14 +1,5 @@ -# pylint: disable=unused-argument -# pylint: disable=undefined-variable - import json import os -import unittest - -from parameterized import parameterized - -from core.base_test.tns_test import TnsTest -from core.enums.os_type import OSType from core.enums.platform_type import Platform from core.settings import Settings from core.utils.file_utils import Folder, File @@ -17,8 +8,7 @@ from core.utils.npm import Npm from core.utils.perf_utils import PerfUtils from core.utils.xcode import Xcode -from data.changes import Changes, Sync -from data.templates import Template +from data.changes import Sync from products.nativescript.tns import Tns RETRY_COUNT = 3 @@ -27,74 +17,6 @@ EXPECTED_RESULTS = JsonUtils.read(os.path.join(Settings.TEST_RUN_HOME, 'tests', 'perf', 'data.json')) -# noinspection PyMethodMayBeStatic,PyUnusedLocal -class PrepareAndBuildPerfTests(TnsTest): - TEST_DATA = [ - ('hello-world-js', Template.HELLO_WORLD_JS.local_package, Changes.JSHelloWord.JS), - ('hello-world-ng', Template.HELLO_WORLD_NG.local_package, Changes.NGHelloWorld.TS), - ('master-detail-ng', Template.MASTER_DETAIL_NG.local_package, Changes.MasterDetailNG.TS), - ] - - @classmethod - def setUpClass(cls): - TnsTest.setUpClass() - - def setUp(self): - TnsTest.setUp(self) - - @classmethod - def tearDownClass(cls): - TnsTest.tearDownClass() - - @parameterized.expand(TEST_DATA) - def test_001_prepare_data(self, template, template_package, change_set): - android_result_file = Helpers.get_result_file_name(template, Platform.ANDROID) - ios_result_file = Helpers.get_result_file_name(template, Platform.IOS) - Helpers.prepare_and_build(template=template_package, platform=Platform.ANDROID, - change_set=change_set, result_file=android_result_file) - Helpers.prepare_and_build(template=template_package, platform=Platform.IOS, - change_set=change_set, result_file=ios_result_file) - - @parameterized.expand(TEST_DATA) - def test_200_prepare_android_initial(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.ANDROID, 'prepare_initial') - expected = Helpers.get_expected_result(template, Platform.ANDROID, 'prepare_initial') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial android prepare time is not OK.' - - @parameterized.expand(TEST_DATA) - @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') - def test_201_prepare_ios_initial(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.IOS, 'prepare_initial') - expected = Helpers.get_expected_result(template, Platform.IOS, 'prepare_initial') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial ios prepare time is not OK.' - - @parameterized.expand(TEST_DATA) - def test_300_build_android_initial(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.ANDROID, 'build_initial') - expected = Helpers.get_expected_result(template, Platform.ANDROID, 'build_initial') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial android build time is not OK.' - - @parameterized.expand(TEST_DATA) - @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') - def test_301_build_ios_initial(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.IOS, 'build_initial') - expected = Helpers.get_expected_result(template, Platform.IOS, 'build_initial') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial ios build time is not OK.' - - @parameterized.expand(TEST_DATA) - def test_310_build_android_incremental(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.ANDROID, 'build_incremental') - expected = Helpers.get_expected_result(template, Platform.ANDROID, 'build_incremental') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental android build time is not OK.' - - @parameterized.expand(TEST_DATA) - @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') - def test_311_build_ios_incremental(self, template, template_package, change_set): - actual = Helpers.get_actual_result(template, Platform.IOS, 'build_incremental') - expected = Helpers.get_expected_result(template, Platform.IOS, 'build_incremental') - assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental ios build time is not OK.' - - class PrepareBuildInfo(object): prepare_initial = 0 prepare_skip = 0 @@ -159,3 +81,48 @@ def get_actual_result(template, platform, entry): def get_expected_result(template, platform, entry): platform = str(platform) return EXPECTED_RESULTS[template][platform][entry] + + @staticmethod + def prepare_data(template, change_set): + android_result_file = Helpers.get_result_file_name(template, Platform.ANDROID) + ios_result_file = Helpers.get_result_file_name(template, Platform.IOS) + Helpers.prepare_and_build(template=template, platform=Platform.ANDROID, + change_set=change_set, result_file=android_result_file) + Helpers.prepare_and_build(template=template, platform=Platform.IOS, + change_set=change_set, result_file=ios_result_file) + + @staticmethod + def prepare_android_initial(template): + actual = Helpers.get_actual_result(template, Platform.ANDROID, 'prepare_initial') + expected = Helpers.get_expected_result(template, Platform.ANDROID, 'prepare_initial') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial android prepare time is not OK.' + + @staticmethod + def prepare_ios_initial(template): + actual = Helpers.get_actual_result(template, Platform.IOS, 'prepare_initial') + expected = Helpers.get_expected_result(template, Platform.IOS, 'prepare_initial') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial ios prepare time is not OK.' + + @staticmethod + def build_android_initial(template): + actual = Helpers.get_actual_result(template, Platform.ANDROID, 'build_initial') + expected = Helpers.get_expected_result(template, Platform.ANDROID, 'build_initial') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial android build time is not OK.' + + @staticmethod + def build_ios_initial(template): + actual = Helpers.get_actual_result(template, Platform.IOS, 'build_initial') + expected = Helpers.get_expected_result(template, Platform.IOS, 'build_initial') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial ios build time is not OK.' + + @staticmethod + def build_android_incremental(template): + actual = Helpers.get_actual_result(template, Platform.ANDROID, 'build_incremental') + expected = Helpers.get_expected_result(template, Platform.ANDROID, 'build_incremental') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental android build time is not OK.' + + @staticmethod + def build_ios_incremental(template): + actual = Helpers.get_actual_result(template, Platform.IOS, 'build_incremental') + expected = Helpers.get_expected_result(template, Platform.IOS, 'build_incremental') + assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental ios build time is not OK.' diff --git a/tests/perf/tooling/build_perf_hello_world_js.py b/tests/perf/tooling/build_perf_hello_world_js.py new file mode 100644 index 00000000..028a45a1 --- /dev/null +++ b/tests/perf/tooling/build_perf_hello_world_js.py @@ -0,0 +1,52 @@ +# pylint: disable=unused-argument +# pylint: disable=undefined-variable + +import unittest + +from core.base_test.tns_test import TnsTest +from core.enums.os_type import OSType +from core.settings import Settings +from data.changes import Changes +from data.templates import Template +from products.nativescript.perf_helpers import Helpers + + +# noinspection PyMethodMayBeStatic,PyUnusedLocal +class PrepareAndBuildPerfTests(TnsTest): + template = Template.HELLO_WORLD_JS.local_package + change_set = Changes.JSHelloWord.JS + + @classmethod + def setUpClass(cls): + TnsTest.setUpClass() + + def setUp(self): + TnsTest.setUp(self) + + @classmethod + def tearDownClass(cls): + TnsTest.tearDownClass() + + def test_001_prepare_data(self): + Helpers.prepare_data(self.template, self.change_set) + + def test_200_prepare_android_initial(self): + Helpers.prepare_android_initial(self.template) + + @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') + def test_201_prepare_ios_initial(self): + Helpers.prepare_ios_initial(self.template) + + def test_300_build_android_initial(self): + Helpers.build_android_initial(self.template) + + @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') + def test_301_build_ios_initial(self): + Helpers.build_ios_initial(self.template) + + def test_310_build_android_incremental(self): + Helpers.build_android_incremental(self.template) + + @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') + def test_311_build_ios_incremental(self): + Helpers.build_ios_incremental(self.template) diff --git a/tests/perf/tooling/build_perf_hello_world_ng.py b/tests/perf/tooling/build_perf_hello_world_ng.py new file mode 100644 index 00000000..a5dd78d8 --- /dev/null +++ b/tests/perf/tooling/build_perf_hello_world_ng.py @@ -0,0 +1,62 @@ +# pylint: disable=unused-argument +# pylint: disable=undefined-variable + +import unittest + +from parameterized import parameterized + +from core.base_test.tns_test import TnsTest +from core.enums.os_type import OSType +from core.settings import Settings +from data.changes import Changes +from data.templates import Template +from products.nativescript.perf_helpers import Helpers + + +# noinspection PyMethodMayBeStatic,PyUnusedLocal +class PrepareAndBuildPerfTests(TnsTest): + TEST_DATA = [ + ('hello-world-ng', Template.HELLO_WORLD_NG.local_package, Changes.NGHelloWorld.TS) + ] + + @classmethod + def setUpClass(cls): + TnsTest.setUpClass() + + def setUp(self): + TnsTest.setUp(self) + + @classmethod + def tearDownClass(cls): + TnsTest.tearDownClass() + + @parameterized.expand(TEST_DATA) + def test_001_prepare_data(self, template, template_package, change_set): + Helpers.prepare_data(template, template_package, change_set) + + @parameterized.expand(TEST_DATA) + def test_200_prepare_android_initial(self, template, template_package, change_set): + Helpers.prepare_android_initial(template) + + @parameterized.expand(TEST_DATA) + @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') + def test_201_prepare_ios_initial(self, template, template_package, change_set): + Helpers.prepare_ios_initial(template) + + @parameterized.expand(TEST_DATA) + def test_300_build_android_initial(self, template, template_package, change_set): + Helpers.build_android_initial(template) + + @parameterized.expand(TEST_DATA) + @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') + def test_301_build_ios_initial(self, template, template_package, change_set): + Helpers.build_ios_initial(template) + + @parameterized.expand(TEST_DATA) + def test_310_build_android_incremental(self, template, template_package, change_set): + Helpers.build_android_incremental(template) + + @parameterized.expand(TEST_DATA) + @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') + def test_311_build_ios_incremental(self, template, template_package, change_set): + Helpers.build_ios_incremental(template) diff --git a/tests/perf/tooling/build_perf_master_detail_ng.py b/tests/perf/tooling/build_perf_master_detail_ng.py new file mode 100644 index 00000000..26b2bc53 --- /dev/null +++ b/tests/perf/tooling/build_perf_master_detail_ng.py @@ -0,0 +1,62 @@ +# pylint: disable=unused-argument +# pylint: disable=undefined-variable + +import unittest + +from parameterized import parameterized + +from core.base_test.tns_test import TnsTest +from core.enums.os_type import OSType +from core.settings import Settings +from data.changes import Changes +from data.templates import Template +from products.nativescript.perf_helpers import Helpers + + +# noinspection PyMethodMayBeStatic,PyUnusedLocal +class PrepareAndBuildPerfTests(TnsTest): + TEST_DATA = [ + ('master-detail-ng', Template.MASTER_DETAIL_NG.local_package, Changes.MasterDetailNG.TS) + ] + + @classmethod + def setUpClass(cls): + TnsTest.setUpClass() + + def setUp(self): + TnsTest.setUp(self) + + @classmethod + def tearDownClass(cls): + TnsTest.tearDownClass() + + @parameterized.expand(TEST_DATA) + def test_001_prepare_data(self, template, template_package, change_set): + Helpers.prepare_data(template, template_package, change_set) + + @parameterized.expand(TEST_DATA) + def test_200_prepare_android_initial(self, template, template_package, change_set): + Helpers.prepare_android_initial(template) + + @parameterized.expand(TEST_DATA) + @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') + def test_201_prepare_ios_initial(self, template, template_package, change_set): + Helpers.prepare_ios_initial(template) + + @parameterized.expand(TEST_DATA) + def test_300_build_android_initial(self, template, template_package, change_set): + Helpers.build_android_initial(template) + + @parameterized.expand(TEST_DATA) + @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') + def test_301_build_ios_initial(self, template, template_package, change_set): + Helpers.build_ios_initial(template) + + @parameterized.expand(TEST_DATA) + def test_310_build_android_incremental(self, template, template_package, change_set): + Helpers.build_android_incremental(template) + + @parameterized.expand(TEST_DATA) + @unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.') + def test_311_build_ios_incremental(self, template, template_package, change_set): + Helpers.build_ios_incremental(template)