Skip to content

fix: split perf tests #442

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
Open

fix: split perf tests #442

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a55d48a
split perf tests
miroslavaivanova Nov 18, 2019
12b9cb4
perf split
miroslavaivanova Nov 19, 2019
2791e3a
perf split
miroslavaivanova Nov 19, 2019
98d367b
pylint error
miroslavaivanova Nov 19, 2019
e8626e7
perf tests
miroslavaivanova Nov 20, 2019
4d3039a
pref
miroslavaivanova Nov 20, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 46 additions & 79 deletions tests/perf/tooling/build_perf_tests.py → products/nativescript/perf_helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,5 @@
# pylint: disable=unused-argument
# pylint: disable=undefined-variable

import json
import os
import unittest

from parameterized import parameterized

from core.base_test.tns_test import TnsTest
from core.enums.os_type import OSType
from core.enums.platform_type import Platform
from core.settings import Settings
from core.utils.file_utils import Folder, File
Expand All @@ -17,8 +8,7 @@
from core.utils.npm import Npm
from core.utils.perf_utils import PerfUtils
from core.utils.xcode import Xcode
from data.changes import Changes, Sync
from data.templates import Template
from data.changes import Sync
from products.nativescript.tns import Tns

RETRY_COUNT = 3
Expand All @@ -27,74 +17,6 @@
EXPECTED_RESULTS = JsonUtils.read(os.path.join(Settings.TEST_RUN_HOME, 'tests', 'perf', 'data.json'))


# noinspection PyMethodMayBeStatic,PyUnusedLocal
class PrepareAndBuildPerfTests(TnsTest):
TEST_DATA = [
('hello-world-js', Template.HELLO_WORLD_JS.local_package, Changes.JSHelloWord.JS),
('hello-world-ng', Template.HELLO_WORLD_NG.local_package, Changes.NGHelloWorld.TS),
('master-detail-ng', Template.MASTER_DETAIL_NG.local_package, Changes.MasterDetailNG.TS),
]

@classmethod
def setUpClass(cls):
TnsTest.setUpClass()

def setUp(self):
TnsTest.setUp(self)

@classmethod
def tearDownClass(cls):
TnsTest.tearDownClass()

@parameterized.expand(TEST_DATA)
def test_001_prepare_data(self, template, template_package, change_set):
android_result_file = Helpers.get_result_file_name(template, Platform.ANDROID)
ios_result_file = Helpers.get_result_file_name(template, Platform.IOS)
Helpers.prepare_and_build(template=template_package, platform=Platform.ANDROID,
change_set=change_set, result_file=android_result_file)
Helpers.prepare_and_build(template=template_package, platform=Platform.IOS,
change_set=change_set, result_file=ios_result_file)

@parameterized.expand(TEST_DATA)
def test_200_prepare_android_initial(self, template, template_package, change_set):
actual = Helpers.get_actual_result(template, Platform.ANDROID, 'prepare_initial')
expected = Helpers.get_expected_result(template, Platform.ANDROID, 'prepare_initial')
assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial android prepare time is not OK.'

@parameterized.expand(TEST_DATA)
@unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.')
def test_201_prepare_ios_initial(self, template, template_package, change_set):
actual = Helpers.get_actual_result(template, Platform.IOS, 'prepare_initial')
expected = Helpers.get_expected_result(template, Platform.IOS, 'prepare_initial')
assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial ios prepare time is not OK.'

@parameterized.expand(TEST_DATA)
def test_300_build_android_initial(self, template, template_package, change_set):
actual = Helpers.get_actual_result(template, Platform.ANDROID, 'build_initial')
expected = Helpers.get_expected_result(template, Platform.ANDROID, 'build_initial')
assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial android build time is not OK.'

@parameterized.expand(TEST_DATA)
@unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.')
def test_301_build_ios_initial(self, template, template_package, change_set):
actual = Helpers.get_actual_result(template, Platform.IOS, 'build_initial')
expected = Helpers.get_expected_result(template, Platform.IOS, 'build_initial')
assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial ios build time is not OK.'

@parameterized.expand(TEST_DATA)
def test_310_build_android_incremental(self, template, template_package, change_set):
actual = Helpers.get_actual_result(template, Platform.ANDROID, 'build_incremental')
expected = Helpers.get_expected_result(template, Platform.ANDROID, 'build_incremental')
assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental android build time is not OK.'

@parameterized.expand(TEST_DATA)
@unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.')
def test_311_build_ios_incremental(self, template, template_package, change_set):
actual = Helpers.get_actual_result(template, Platform.IOS, 'build_incremental')
expected = Helpers.get_expected_result(template, Platform.IOS, 'build_incremental')
assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental ios build time is not OK.'


class PrepareBuildInfo(object):
prepare_initial = 0
prepare_skip = 0
Expand Down Expand Up @@ -159,3 +81,48 @@ def get_actual_result(template, platform, entry):
def get_expected_result(template, platform, entry):
platform = str(platform)
return EXPECTED_RESULTS[template][platform][entry]

@staticmethod
def prepare_data(template, change_set):
android_result_file = Helpers.get_result_file_name(template, Platform.ANDROID)
ios_result_file = Helpers.get_result_file_name(template, Platform.IOS)
Helpers.prepare_and_build(template=template, platform=Platform.ANDROID,
change_set=change_set, result_file=android_result_file)
Helpers.prepare_and_build(template=template, platform=Platform.IOS,
change_set=change_set, result_file=ios_result_file)

@staticmethod
def prepare_android_initial(template):
actual = Helpers.get_actual_result(template, Platform.ANDROID, 'prepare_initial')
expected = Helpers.get_expected_result(template, Platform.ANDROID, 'prepare_initial')
assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial android prepare time is not OK.'

@staticmethod
def prepare_ios_initial(template):
actual = Helpers.get_actual_result(template, Platform.IOS, 'prepare_initial')
expected = Helpers.get_expected_result(template, Platform.IOS, 'prepare_initial')
assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial ios prepare time is not OK.'

@staticmethod
def build_android_initial(template):
actual = Helpers.get_actual_result(template, Platform.ANDROID, 'build_initial')
expected = Helpers.get_expected_result(template, Platform.ANDROID, 'build_initial')
assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial android build time is not OK.'

@staticmethod
def build_ios_initial(template):
actual = Helpers.get_actual_result(template, Platform.IOS, 'build_initial')
expected = Helpers.get_expected_result(template, Platform.IOS, 'build_initial')
assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Initial ios build time is not OK.'

@staticmethod
def build_android_incremental(template):
actual = Helpers.get_actual_result(template, Platform.ANDROID, 'build_incremental')
expected = Helpers.get_expected_result(template, Platform.ANDROID, 'build_incremental')
assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental android build time is not OK.'

@staticmethod
def build_ios_incremental(template):
actual = Helpers.get_actual_result(template, Platform.IOS, 'build_incremental')
expected = Helpers.get_expected_result(template, Platform.IOS, 'build_incremental')
assert PerfUtils.is_value_in_range(actual, expected, TOLERANCE), 'Incremental ios build time is not OK.'
52 changes: 52 additions & 0 deletions tests/perf/tooling/build_perf_hello_world_js.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
# pylint: disable=unused-argument
# pylint: disable=undefined-variable

import unittest

from core.base_test.tns_test import TnsTest
from core.enums.os_type import OSType
from core.settings import Settings
from data.changes import Changes
from data.templates import Template
from products.nativescript.perf_helpers import Helpers


# noinspection PyMethodMayBeStatic,PyUnusedLocal
class PrepareAndBuildPerfTests(TnsTest):
template = Template.HELLO_WORLD_JS.local_package
change_set = Changes.JSHelloWord.JS

@classmethod
def setUpClass(cls):
TnsTest.setUpClass()

def setUp(self):
TnsTest.setUp(self)

@classmethod
def tearDownClass(cls):
TnsTest.tearDownClass()

def test_001_prepare_data(self):
Helpers.prepare_data(self.template, self.change_set)

def test_200_prepare_android_initial(self):
Helpers.prepare_android_initial(self.template)

@unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.')
def test_201_prepare_ios_initial(self):
Helpers.prepare_ios_initial(self.template)

def test_300_build_android_initial(self):
Helpers.build_android_initial(self.template)

@unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.')
def test_301_build_ios_initial(self):
Helpers.build_ios_initial(self.template)

def test_310_build_android_incremental(self):
Helpers.build_android_incremental(self.template)

@unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.')
def test_311_build_ios_incremental(self):
Helpers.build_ios_incremental(self.template)
62 changes: 62 additions & 0 deletions tests/perf/tooling/build_perf_hello_world_ng.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
# pylint: disable=unused-argument
# pylint: disable=undefined-variable

import unittest

from parameterized import parameterized

from core.base_test.tns_test import TnsTest
from core.enums.os_type import OSType
from core.settings import Settings
from data.changes import Changes
from data.templates import Template
from products.nativescript.perf_helpers import Helpers


# noinspection PyMethodMayBeStatic,PyUnusedLocal
class PrepareAndBuildPerfTests(TnsTest):
TEST_DATA = [
('hello-world-ng', Template.HELLO_WORLD_NG.local_package, Changes.NGHelloWorld.TS)
]

@classmethod
def setUpClass(cls):
TnsTest.setUpClass()

def setUp(self):
TnsTest.setUp(self)

@classmethod
def tearDownClass(cls):
TnsTest.tearDownClass()

@parameterized.expand(TEST_DATA)
def test_001_prepare_data(self, template, template_package, change_set):
Helpers.prepare_data(template, template_package, change_set)

@parameterized.expand(TEST_DATA)
def test_200_prepare_android_initial(self, template, template_package, change_set):
Helpers.prepare_android_initial(template)

@parameterized.expand(TEST_DATA)
@unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.')
def test_201_prepare_ios_initial(self, template, template_package, change_set):
Helpers.prepare_ios_initial(template)

@parameterized.expand(TEST_DATA)
def test_300_build_android_initial(self, template, template_package, change_set):
Helpers.build_android_initial(template)

@parameterized.expand(TEST_DATA)
@unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.')
def test_301_build_ios_initial(self, template, template_package, change_set):
Helpers.build_ios_initial(template)

@parameterized.expand(TEST_DATA)
def test_310_build_android_incremental(self, template, template_package, change_set):
Helpers.build_android_incremental(template)

@parameterized.expand(TEST_DATA)
@unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.')
def test_311_build_ios_incremental(self, template, template_package, change_set):
Helpers.build_ios_incremental(template)
62 changes: 62 additions & 0 deletions tests/perf/tooling/build_perf_master_detail_ng.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
# pylint: disable=unused-argument
# pylint: disable=undefined-variable

import unittest

from parameterized import parameterized

from core.base_test.tns_test import TnsTest
from core.enums.os_type import OSType
from core.settings import Settings
from data.changes import Changes
from data.templates import Template
from products.nativescript.perf_helpers import Helpers


# noinspection PyMethodMayBeStatic,PyUnusedLocal
class PrepareAndBuildPerfTests(TnsTest):
TEST_DATA = [
('master-detail-ng', Template.MASTER_DETAIL_NG.local_package, Changes.MasterDetailNG.TS)
]

@classmethod
def setUpClass(cls):
TnsTest.setUpClass()

def setUp(self):
TnsTest.setUp(self)

@classmethod
def tearDownClass(cls):
TnsTest.tearDownClass()

@parameterized.expand(TEST_DATA)
def test_001_prepare_data(self, template, template_package, change_set):
Helpers.prepare_data(template, template_package, change_set)

@parameterized.expand(TEST_DATA)
def test_200_prepare_android_initial(self, template, template_package, change_set):
Helpers.prepare_android_initial(template)

@parameterized.expand(TEST_DATA)
@unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.')
def test_201_prepare_ios_initial(self, template, template_package, change_set):
Helpers.prepare_ios_initial(template)

@parameterized.expand(TEST_DATA)
def test_300_build_android_initial(self, template, template_package, change_set):
Helpers.build_android_initial(template)

@parameterized.expand(TEST_DATA)
@unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.')
def test_301_build_ios_initial(self, template, template_package, change_set):
Helpers.build_ios_initial(template)

@parameterized.expand(TEST_DATA)
def test_310_build_android_incremental(self, template, template_package, change_set):
Helpers.build_android_incremental(template)

@parameterized.expand(TEST_DATA)
@unittest.skipIf(Settings.HOST_OS != OSType.OSX, 'iOS tests can be executed only on macOS.')
def test_311_build_ios_incremental(self, template, template_package, change_set):
Helpers.build_ios_incremental(template)