From 98eef459355abf497e5b33df141de99d15e1f89c Mon Sep 17 00:00:00 2001 From: Jimmy Brisson Date: Mon, 9 Jul 2018 13:45:56 -0500 Subject: [PATCH 1/4] Enable -C= on command line --- tools/build.py | 18 +++++++++++++++++- tools/build_api.py | 7 +++++-- tools/config/__init__.py | 16 +++++++++++++++- tools/make.py | 16 +++++++++++++++- 4 files changed, 52 insertions(+), 5 deletions(-) diff --git a/tools/build.py b/tools/build.py index 52d8810eff5..ee45c49fd9e 100644 --- a/tools/build.py +++ b/tools/build.py @@ -38,6 +38,7 @@ from tools.build_api import build_library, build_mbed_libs, build_lib from tools.build_api import mcu_toolchain_matrix from tools.build_api import print_build_results +from tools.config import Config from tools.settings import CPPCHECK_CMD, CPPCHECK_MSG_FORMAT from tools.settings import CPPCHECK_CMD, CPPCHECK_MSG_FORMAT, CLI_COLOR_MAP from tools.notifier.term import TerminalNotifier @@ -95,6 +96,13 @@ dest="macros", help="Add a macro definition") + parser.add_argument( + "-C", + action="append", + dest="cli_config", + help="override a configuration value on the command line", + ) + parser.add_argument("-S", "--supported-toolchains", action="store_true", dest="supported_toolchains", @@ -188,6 +196,13 @@ mcu = TARGET_MAP[target] profile = extract_profile(parser, options, toolchain) if options.source_dir: + config = Config( + mcu, + app_config=Config.find_app_config( + options.source_dir + ), + symbols=options.cli_config, + ) lib_build_res = build_library( options.source_dir, options.build_dir, mcu, toolchain, jobs=options.jobs, @@ -197,7 +212,8 @@ name=options.artifact_name, build_profile=profile, ignore=options.ignore, - notify = notifier, + notify=notifier, + config=config, ) else: lib_build_res = build_mbed_libs( diff --git a/tools/build_api.py b/tools/build_api.py index 02672ca499c..5e968e41543 100644 --- a/tools/build_api.py +++ b/tools/build_api.py @@ -283,6 +283,9 @@ def target_supports_toolchain(target, toolchain_name): return toolchain_name in target.supported_toolchains +IS_CONFIG_PARAM = re.compile("\w+\.\w+=.*") + + def prepare_toolchain(src_paths, build_dir, target, toolchain_name, macros=None, clean=False, jobs=1, notify=None, config=None, app_config=None, @@ -616,7 +619,7 @@ def build_library(src_paths, build_path, target, toolchain_name, dependencies_paths=None, name=None, clean=False, archive=True, notify=None, macros=None, inc_dirs=None, jobs=1, report=None, properties=None, project_id=None, - remove_config_header_file=False, app_config=None, + remove_config_header_file=False, config=None, build_profile=None, ignore=None): """ Build a library @@ -666,7 +669,7 @@ def build_library(src_paths, build_path, target, toolchain_name, # Pass all params to the unified prepare_toolchain() toolchain = prepare_toolchain( src_paths, build_path, target, toolchain_name, macros=macros, - clean=clean, jobs=jobs, notify=notify, app_config=app_config, + clean=clean, jobs=jobs, notify=notify, config=config, build_profile=build_profile, ignore=ignore) # The first path will give the name to the library diff --git a/tools/config/__init__.py b/tools/config/__init__.py index 9dc7ddc3d95..b1e6fb5f238 100644 --- a/tools/config/__init__.py +++ b/tools/config/__init__.py @@ -427,7 +427,7 @@ def format_validation_error(self, error, path): path, ".".join(p for p in error.absolute_path), error.message.replace('u\'','\'')) - def __init__(self, tgt, top_level_dirs=None, app_config=None): + def __init__(self, tgt, top_level_dirs=None, app_config=None, symbols=None): """Construct a mbed configuration Positional arguments: @@ -500,6 +500,20 @@ def __init__(self, tgt, top_level_dirs=None, app_config=None): self.cumulative_overrides = {key: ConfigCumulativeOverride(key) for key in CUMULATIVE_ATTRIBUTES} + if symbols: + if not self.app_config_location: + self.app_config_location = "." + self.app_config_data.setdefault("target_overrides", {}) + self.app_config_data["target_overrides"].setdefault( + self.target.name, {}) + overrides = self.app_config_data["target_overrides"][self.target.name] + for symbol in symbols: + try: + name, value = symbol.split("=") + overrides[name] = value + except ValueError: + pass + self._process_config_and_overrides(self.app_config_data, {}, "app", "application") self.config_errors = config_errors diff --git a/tools/make.py b/tools/make.py index d59c5badf8f..73c4afbe639 100644 --- a/tools/make.py +++ b/tools/make.py @@ -31,6 +31,7 @@ ROOT = abspath(join(dirname(__file__), "..")) sys.path.insert(0, ROOT) +from tools.config import Config from tools.utils import args_error from tools.utils import NotSupportedException from tools.paths import BUILD_DIR @@ -99,6 +100,13 @@ dest="macros", help="Add a macro definition") + parser.add_argument( + "-C", + action="append", + dest="cli_config", + help="override a configuration value on the command line", + ) + group.add_argument( "-S", "--supported-toolchains", dest="supported_toolchains", @@ -273,6 +281,12 @@ build_dir = options.build_dir try: + config = Config( + mcu, + app_config=(options.app_config or + Config.find_app_config(test.source_dir)), + symbols=options.cli_config, + ) bin_file, update_file = build_project( test.source_dir, build_dir, @@ -286,7 +300,7 @@ macros=options.macros, jobs=options.jobs, name=options.artifact_name, - app_config=options.app_config, + config=config, inc_dirs=[dirname(MBED_LIBRARIES)], build_profile=extract_profile(parser, options, toolchain), stats_depth=options.stats_depth, From 8a34c4febe2b05ae5c8e877da6b77658911e612e Mon Sep 17 00:00:00 2001 From: Jimmy Brisson Date: Mon, 9 Jul 2018 14:00:14 -0500 Subject: [PATCH 2/4] Allow json as value to -C --- tools/config/__init__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/config/__init__.py b/tools/config/__init__.py index b1e6fb5f238..aaec23f49c5 100644 --- a/tools/config/__init__.py +++ b/tools/config/__init__.py @@ -510,9 +510,13 @@ def __init__(self, tgt, top_level_dirs=None, app_config=None, symbols=None): for symbol in symbols: try: name, value = symbol.split("=") - overrides[name] = value + except ValueError: + continue + try: + value = json.loads(value) except ValueError: pass + overrides[name] = value self._process_config_and_overrides(self.app_config_data, {}, "app", "application") From ea63e40c282240440f4924b42d15486459bbc323 Mon Sep 17 00:00:00 2001 From: Jimmy Brisson Date: Fri, 24 Aug 2018 13:48:44 -0500 Subject: [PATCH 3/4] Use Config instead of app_config in build_library tests --- tools/test/build_api/build_api_test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/test/build_api/build_api_test.py b/tools/test/build_api/build_api_test.py index 86ffde8edda..479140f9b5c 100644 --- a/tools/test/build_api/build_api_test.py +++ b/tools/test/build_api/build_api_test.py @@ -183,7 +183,7 @@ def test_build_project_no_app_config(self, mock_prepare_toolchain, mock_exists, @patch('tools.build_api.prepare_toolchain') def test_build_library_app_config(self, mock_prepare_toolchain, mock_exists, _, __): """ - Test that build_library uses app_config correctly + Test that build_library uses config correctly :param mock_prepare_toolchain: mock of function prepare_toolchain :param mock_exists: mock of function os.path.exists @@ -192,16 +192,16 @@ def test_build_library_app_config(self, mock_prepare_toolchain, mock_exists, _, :return: """ notify = MockNotifier() - app_config = "app_config" + config = MagicMock() mock_exists.return_value = False build_library(self.src_paths, self.build_path, self.target, - self.toolchain_name, app_config=app_config, notify=notify) + self.toolchain_name, config=config, notify=notify) args = mock_prepare_toolchain.call_args - self.assertTrue('app_config' in args[1], + self.assertTrue('config' in args[1], "prepare_toolchain was not called with app_config") - self.assertEqual(args[1]['app_config'], app_config, + self.assertEqual(args[1]['config'], config, "prepare_toolchain was called with an incorrect app_config") @patch('tools.build_api.Resources') @@ -210,7 +210,7 @@ def test_build_library_app_config(self, mock_prepare_toolchain, mock_exists, _, @patch('tools.build_api.prepare_toolchain') def test_build_library_no_app_config(self, mock_prepare_toolchain, mock_exists, _, __): """ - Test that build_library correctly deals with no app_config + Test that build_library correctly deals with no config :param mock_prepare_toolchain: mock of function prepare_toolchain :param mock_exists: mock of function os.path.exists @@ -225,9 +225,9 @@ def test_build_library_no_app_config(self, mock_prepare_toolchain, mock_exists, self.toolchain_name, notify=notify) args = mock_prepare_toolchain.call_args - self.assertTrue('app_config' in args[1], + self.assertTrue('config' in args[1], "prepare_toolchain was not called with app_config") - self.assertEqual(args[1]['app_config'], None, + self.assertEqual(args[1]['config'], None, "prepare_toolchain was called with an incorrect app_config") if __name__ == '__main__': From b120fc4a7e28116a9faf0f366d9221372782b6d8 Mon Sep 17 00:00:00 2001 From: Jimmy Brisson Date: Fri, 19 Oct 2018 10:51:36 -0500 Subject: [PATCH 4/4] Update test building to pass in Config Object And copy the base config to prevent conflicts --- tools/test.py | 7 ++++++- tools/test_api.py | 9 ++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tools/test.py b/tools/test.py index df9b8378445..24bc2079111 100644 --- a/tools/test.py +++ b/tools/test.py @@ -23,6 +23,7 @@ import os import json import fnmatch +from copy import deepcopy ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) sys.path.insert(0, ROOT) @@ -199,6 +200,10 @@ library_build_success = False profile = extract_profile(parser, options, toolchain) + config = Config( + mcu, + app_config=Config.find_app_config(base_source_paths), + ) try: # Build sources notify = TerminalNotifier(options.verbose) @@ -208,7 +213,7 @@ properties=build_properties, name="mbed-build", macros=options.macros, notify=notify, archive=False, - app_config=config, + config=deepcopy(config), build_profile=profile, ignore=options.ignore) diff --git a/tools/test_api.py b/tools/test_api.py index 8f19977f17d..952ca8c9f47 100644 --- a/tools/test_api.py +++ b/tools/test_api.py @@ -2196,7 +2196,7 @@ def build_test_worker(*args, **kwargs): def build_tests(tests, base_source_paths, build_path, target, toolchain_name, clean=False, notify=None, jobs=1, macros=None, silent=False, report=None, properties=None, - continue_on_build_fail=False, app_config=None, + continue_on_build_fail=False, config=None, build_profile=None, stats_depth=None, ignore=None): """Given the data structure from 'find_tests' and the typical build parameters, build all the tests @@ -2212,7 +2212,10 @@ def build_tests(tests, base_source_paths, build_path, target, toolchain_name, else: target_name = target target = TARGET_MAP[target_name] - cfg, _, _ = get_config(base_source_paths, target, app_config=app_config) + if config: + cfg = config.get_config_data() + else: + cfg, _, _ = get_config(base_source_paths, target) baud_rate = 9600 if 'platform.stdio-baud-rate' in cfg: @@ -2250,7 +2253,7 @@ def build_tests(tests, base_source_paths, build_path, target, toolchain_name, 'project_id': test_name, 'report': report, 'properties': properties, - 'app_config': app_config, + 'config': deepcopy(config), 'build_profile': build_profile, 'toolchain_paths': TOOLCHAIN_PATHS, 'stats_depth': stats_depth,