From ab0a5a53c24666115a07f58fd5f1fc93a55d986d Mon Sep 17 00:00:00 2001 From: Bogdan Marinescu Date: Thu, 16 Jun 2016 00:23:04 +0300 Subject: [PATCH 1/4] Configuration in header files - step 1 The current implementation of the configuration system "compiles" the configuration parameters to macros defined on the command line. This works, but has a few limitations: - it can bring back the maximum command line length issues in Windows. - depending on the type of the configuration parameter, it might require special quoting of its value in the command line. - various 3rd party IDE/tools seem to have some limitations regarding the total length of the macros that can be defined. This commit is the first step in replacing the current mechanism with one that generates configuration in header files that will be automatically included, instead of command line macro definitions. The commit only adds the method for generating the header file, it doesn't actually change the way config is used yet (that will happen in step 2), thus it is backwards compatible. The logic of the configuration system itself is unchanged (in fact, the whole change (not only this commit) is supposed to be completely transparent for the users of the configuration system). The commit also fixes an issue in tools/get_config.py that appeared as a result of a recent PR: the signature of "get_config" in tools/build_api.py changed, but tools/get_config.py was not updated. --- tools/config.py | 57 +++++++++++++++++++++++++-- tools/get_config.py | 4 +- tools/test/config_test/config_test.py | 3 +- 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/tools/config.py b/tools/config.py index c95b0206c75..b0e77bc23ab 100644 --- a/tools/config.py +++ b/tools/config.py @@ -132,8 +132,10 @@ def __init__(self, name, unit_name, unit_kind): if len(tmp) != 2: raise ValueError("Invalid macro definition '%s' in '%s'" % (name, self.defined_by)) self.macro_name = tmp[0] + self.macro_value = tmp[1] else: self.macro_name = name + self.macro_value = None # 'Config' implements the mbed configuration mechanism class Config: @@ -343,13 +345,13 @@ def get_app_config_data(self, params, macros): # Return the configuration data in two parts: # - params: a dictionary with (name, ConfigParam) entries - # - macros: the list of macros defined with "macros" in libraries and in the application + # - macros: the list of macros defined with "macros" in libraries and in the application (as ConfigMacro instances) def get_config_data(self): all_params = self.get_target_config_data() lib_params, macros = self.get_lib_config_data() all_params.update(lib_params) self.get_app_config_data(all_params, macros) - return all_params, [m.name for m in macros.values()] + return all_params, macros # Helper: verify if there are any required parameters without a value in 'params' def _check_required_parameters(self, params): @@ -363,11 +365,17 @@ def _check_required_parameters(self, params): def parameters_to_macros(params): return ['%s=%s' % (m.macro_name, m.value) for m in params.values() if m.value is not None] + # Return the macro definitions generated for a dictionary of ConfigMacros (as returned by get_config_data) + # params: a dictionary of (name, ConfigMacro instance) mappings + @staticmethod + def config_macros_to_macros(macros): + return [m.name for m in macros.values()] + # Return the configuration data converted to a list of C macros def get_config_data_macros(self): params, macros = self.get_config_data() self._check_required_parameters(params) - return macros + self.parameters_to_macros(params) + return self.config_macros_to_macros(macros) + self.parameters_to_macros(params) # Returns any features in the configuration data def get_features(self): @@ -387,4 +395,45 @@ def validate_config(self): if self.config_errors: raise self.config_errors[0] return True - + + # Return the configuration data converted to the content of a C header file, + # meant to be included to a C/C++ file. The content is returned as a string. + # If 'fname' is given, the content is also written to the file called "fname". + # WARNING: if 'fname' names an existing file, that file will be overwritten! + def get_config_data_header(self, fname = None): + params, macros = self.get_config_data() + self._check_required_parameters(params) + header_data = "// Automatically generated configuration file.\n" + header_data += "// DO NOT EDIT, content will be overwritten.\n\n" + header_data += "#ifndef __MBED_CONFIG_DATA__\n" + header_data += "#define __MBED_CONFIG_DATA__\n\n" + # Compute maximum length of macro names for proper alignment + max_param_macro_name_len = max([len(m.macro_name) for m in params.values() if m.value is not None]) if params else 0 + max_direct_macro_name_len = max([len(m.macro_name) for m in macros.values()]) if macros else 0 + max_macro_name_len = max(max_param_macro_name_len, max_direct_macro_name_len) + # Compute maximum length of macro values for proper alignment + max_param_macro_val_len = max([len(str(m.value)) for m in params.values() if m.value is not None]) if params else 0 + max_direct_macro_val_len = max([len(m.macro_value or "") for m in macros.values()]) if macros else 0 + max_macro_val_len = max(max_param_macro_val_len, max_direct_macro_val_len) + # Generate config parameters first + if params: + header_data += "// Configuration parameters\n" + for m in params.values(): + if m.value is not None: + header_data += "#define %s%s%s%s// set by %s\n" % (m.macro_name, " " * (max_macro_name_len - len(m.macro_name) + 1), str(m.value), + " " * (max_macro_val_len - len(str(m.value)) + 1), m.set_by) + # Then macros + if macros: + header_data += "// Macros\n" + for m in macros.values(): + if m.macro_value: + header_data += "#define %s%s%s%s// defined by %s\n" % (m.macro_name, " " * (max_macro_name_len - len(m.macro_name) + 1), str(m.macro_value), + " " * (max_macro_val_len - len(str(m.macro_value)) + 1), m.defined_by) + else: + header_data += "#define %s%s// defined by %s\n" % (m.macro_name, " " * (max_macro_name_len + max_macro_val_len - len(m.macro_name) + 2), m.defined_by) + header_data += "\n#endif\n" + # If fname is given, write "header_data" to it + if fname: + with open(fname, "wt") as f: + f.write(header_data) + return header_data diff --git a/tools/get_config.py b/tools/get_config.py index 6d8af838176..9d9e7dc5fa2 100644 --- a/tools/get_config.py +++ b/tools/get_config.py @@ -62,7 +62,7 @@ options.prefix = options.prefix or [""] try: - params, macros = get_config(options.source_dir, target, toolchain) + params, macros, features = get_config(options.source_dir, target, toolchain) if not params and not macros: print "No configuration data available." _exit(0) @@ -79,7 +79,7 @@ print "Macros" print "------" if macros: - print 'Defined with "macros":', macros + print 'Defined with "macros":', Config.config_macros_to_macros(macros) print "Generated from configuration parameters:", Config.parameters_to_macros(params) except KeyboardInterrupt, e: diff --git a/tools/test/config_test/config_test.py b/tools/test/config_test/config_test.py index a0e5eab8ff9..59c2b50c98d 100644 --- a/tools/test/config_test/config_test.py +++ b/tools/test/config_test/config_test.py @@ -16,7 +16,7 @@ """ from tools.build_api import get_config -from tools.config import ConfigException +from tools.config import ConfigException, Config import os, sys # Compare the output of config against a dictionary of known good results @@ -44,6 +44,7 @@ def test_tree(full_name, name): err_msg = None try: cfg, macros, features = get_config(full_name, target, "GCC_ARM") + macros = Config.config_macros_to_macros(macros) except ConfigException as e: err_msg = e.message if err_msg: From 80a70fcb514899dd98d89433e0c11cf03a09f622 Mon Sep 17 00:00:00 2001 From: Bogdan Marinescu Date: Thu, 16 Jun 2016 11:41:22 +0300 Subject: [PATCH 2/4] Simplified string formatting expressions in get_config_data_header --- tools/config.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tools/config.py b/tools/config.py index b0e77bc23ab..9e1364d5e07 100644 --- a/tools/config.py +++ b/tools/config.py @@ -420,17 +420,15 @@ def get_config_data_header(self, fname = None): header_data += "// Configuration parameters\n" for m in params.values(): if m.value is not None: - header_data += "#define %s%s%s%s// set by %s\n" % (m.macro_name, " " * (max_macro_name_len - len(m.macro_name) + 1), str(m.value), - " " * (max_macro_val_len - len(str(m.value)) + 1), m.set_by) + header_data += "#define {0:<{1}} {2!s:<{3}} // set by {4}\n".format(m.macro_name, max_macro_name_len, m.value, max_macro_val_len, m.set_by) # Then macros if macros: header_data += "// Macros\n" for m in macros.values(): if m.macro_value: - header_data += "#define %s%s%s%s// defined by %s\n" % (m.macro_name, " " * (max_macro_name_len - len(m.macro_name) + 1), str(m.macro_value), - " " * (max_macro_val_len - len(str(m.macro_value)) + 1), m.defined_by) + header_data += "#define {0:<{1}} {2!s:<{3}} // defined by {4}\n".format(m.macro_name, max_macro_name_len, m.macro_value, max_macro_val_len, m.defined_by) else: - header_data += "#define %s%s// defined by %s\n" % (m.macro_name, " " * (max_macro_name_len + max_macro_val_len - len(m.macro_name) + 2), m.defined_by) + header_data += "#define {0:<{1}} // defined by {2}\n".format(m.macro_name, max_macro_name_len + max_macro_val_len + 1, m.defined_by) header_data += "\n#endif\n" # If fname is given, write "header_data" to it if fname: From 85eca37d297ed6989dbfd086b043dc8e97ed830d Mon Sep 17 00:00:00 2001 From: Bogdan Marinescu Date: Thu, 16 Jun 2016 16:13:50 +0300 Subject: [PATCH 3/4] Added toolchain support for configuration in prefix headers This commit uses the previously introduced feature of generating configuration data as a C header file rather than as command line macro definitions. Each toolchain was modified to use prefix headers if requested, and build_api.py was modified to set up the toolchain's prefix header content using the data generated by the config system. Tested by compiling blinky for GCC and ARMCC. I'm having a few issues with my IAR license currently, but both ARMCC and IAR use the same `--preinclude` option for prefix headers, so this shouldn't be an issue. Note that at the moment all exporters still use the previous configuration data mechanism (individual macro definitions as opposed to a prefix header). Exporters will be updated in one or more PRs that will follow. --- tools/build_api.py | 8 ++++---- tools/toolchains/__init__.py | 23 +++++++++++++++++++++++ tools/toolchains/arm.py | 6 +++++- tools/toolchains/gcc.py | 6 +++++- tools/toolchains/iar.py | 6 +++++- 5 files changed, 42 insertions(+), 7 deletions(-) diff --git a/tools/build_api.py b/tools/build_api.py index f125edb53f3..cc2c2bb68c1 100644 --- a/tools/build_api.py +++ b/tools/build_api.py @@ -232,8 +232,8 @@ def build_project(src_path, build_path, target, toolchain_name, prev_features = features config.validate_config() - # And add the configuration macros to the toolchain - toolchain.add_macros(config.get_config_data_macros()) + # Set the toolchain's prefix header with the config data + toolchain.set_prefix_header_content(config.get_config_data_header()) # Compile Sources for path in src_paths: @@ -403,8 +403,8 @@ def build_library(src_paths, build_path, target, toolchain_name, prev_features = features config.validate_config() - # And add the configuration macros to the toolchain - toolchain.add_macros(config.get_config_data_macros()) + # Set the toolchain's prefix header with the config data + toolchain.set_prefix_header_content(config.get_config_data_header()) # Compile Sources for path in src_paths: diff --git a/tools/toolchains/__init__.py b/tools/toolchains/__init__.py index 043acaa2798..a4f0a7d36bf 100644 --- a/tools/toolchains/__init__.py +++ b/tools/toolchains/__init__.py @@ -265,6 +265,9 @@ def __init__(self, target, options=None, notify=None, macros=None, silent=False, self.flags = deepcopy(self.DEFAULT_FLAGS) + # prefix_header_content will hold the content of the prefix header (if used) + self.prefix_header_content = None + def get_output(self): return self.output @@ -867,6 +870,26 @@ def mem_stats(self, map): map_csv = splitext(map)[0] + "_map.csv" memap.generate_output('csv-ci', map_csv) + # "Prefix headers" are automatically included by the compiler at the beginning of + # each source file. + # header_content - the content of the prefix header file. + def set_prefix_header_content(self, header_content): + self.prefix_header_content = header_content + + # Return the location of the prefix header. This function will create the prefix + # header first if needed. The header will be written in a file called "mbed_prefix.h" + # located in the project's build directory. + # If prefix headers are not used (self.prefix_header_content is None), the function + # returns None + def get_prefix_header(self): + if self.prefix_header_content is None: + return None + prefix_file = join(self.build_dir, "mbed_prefix.h") + if not exists(prefix_file): + with open(prefix_file, "wt") as f: + f.write(self.prefix_header_content) + return prefix_file + from tools.settings import ARM_BIN from tools.settings import GCC_ARM_PATH, GCC_CR_PATH diff --git a/tools/toolchains/arm.py b/tools/toolchains/arm.py index f58a20b0179..6884b263bef 100644 --- a/tools/toolchains/arm.py +++ b/tools/toolchains/arm.py @@ -115,7 +115,11 @@ def get_dep_option(self, object): return ["--depend", dep_path] def get_compile_options(self, defines, includes): - return ['-D%s' % d for d in defines] + ['--via', self.get_inc_file(includes)] + opts = ['-D%s' % d for d in defines] + ['--via', self.get_inc_file(includes)] + prefix_header = self.get_prefix_header() + if prefix_header is not None: + opts = opts + ['--preinclude', prefix_header] + return opts @hook_tool def assemble(self, source, object, includes): diff --git a/tools/toolchains/gcc.py b/tools/toolchains/gcc.py index 3d0f9b5095a..f2dc9f78b24 100644 --- a/tools/toolchains/gcc.py +++ b/tools/toolchains/gcc.py @@ -166,7 +166,11 @@ def get_dep_option(self, object): return ["-MD", "-MF", dep_path] def get_compile_options(self, defines, includes): - return ['-D%s' % d for d in defines] + ['@%s' % self.get_inc_file(includes)] + opts = ['-D%s' % d for d in defines] + ['@%s' % self.get_inc_file(includes)] + prefix_header = self.get_prefix_header() + if prefix_header is not None: + opts = opts + ['-include', prefix_header] + return opts @hook_tool def assemble(self, source, object, includes): diff --git a/tools/toolchains/iar.py b/tools/toolchains/iar.py index aac31bc49df..256877451a3 100644 --- a/tools/toolchains/iar.py +++ b/tools/toolchains/iar.py @@ -119,7 +119,11 @@ def cc_extra(self, object): return ["-l", base + '.s.txt'] def get_compile_options(self, defines, includes): - return ['-D%s' % d for d in defines] + ['-f', self.get_inc_file(includes)] + opts = ['-D%s' % d for d in defines] + ['-f', self.get_inc_file(includes)] + prefix_header = self.get_prefix_header() + if prefix_header is not None: + opts = opts + ['--preinclude', prefix_header] + return opts @hook_tool def assemble(self, source, object, includes): From a164224acfa190115fcaa4f2f59e7fe409cff864 Mon Sep 17 00:00:00 2001 From: Bogdan Marinescu Date: Thu, 16 Jun 2016 16:57:33 +0300 Subject: [PATCH 4/4] Changed prefix file name to mbed_conf.h Also changed some function names to make it clear that the prefix headers feature is only used for config. --- tools/build_api.py | 8 ++++---- tools/toolchains/__init__.py | 32 ++++++++++++++++---------------- tools/toolchains/arm.py | 6 +++--- tools/toolchains/gcc.py | 6 +++--- tools/toolchains/iar.py | 6 +++--- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/tools/build_api.py b/tools/build_api.py index cc2c2bb68c1..86de024da9d 100644 --- a/tools/build_api.py +++ b/tools/build_api.py @@ -232,8 +232,8 @@ def build_project(src_path, build_path, target, toolchain_name, prev_features = features config.validate_config() - # Set the toolchain's prefix header with the config data - toolchain.set_prefix_header_content(config.get_config_data_header()) + # Set the toolchain's config header with the config data + toolchain.set_config_header_content(config.get_config_data_header()) # Compile Sources for path in src_paths: @@ -403,8 +403,8 @@ def build_library(src_paths, build_path, target, toolchain_name, prev_features = features config.validate_config() - # Set the toolchain's prefix header with the config data - toolchain.set_prefix_header_content(config.get_config_data_header()) + # Set the toolchain's config header with the config data + toolchain.set_config_header_content(config.get_config_data_header()) # Compile Sources for path in src_paths: diff --git a/tools/toolchains/__init__.py b/tools/toolchains/__init__.py index a4f0a7d36bf..28e2f170a95 100644 --- a/tools/toolchains/__init__.py +++ b/tools/toolchains/__init__.py @@ -265,8 +265,8 @@ def __init__(self, target, options=None, notify=None, macros=None, silent=False, self.flags = deepcopy(self.DEFAULT_FLAGS) - # prefix_header_content will hold the content of the prefix header (if used) - self.prefix_header_content = None + # config_header_content will hold the content of the config header (if used) + self.config_header_content = None def get_output(self): return self.output @@ -871,24 +871,24 @@ def mem_stats(self, map): memap.generate_output('csv-ci', map_csv) # "Prefix headers" are automatically included by the compiler at the beginning of - # each source file. - # header_content - the content of the prefix header file. - def set_prefix_header_content(self, header_content): - self.prefix_header_content = header_content + # each source file. They are used to provide configuration data. + # header_content - the content of the config header file. + def set_config_header_content(self, header_content): + self.config_header_content = header_content - # Return the location of the prefix header. This function will create the prefix - # header first if needed. The header will be written in a file called "mbed_prefix.h" + # Return the location of the config header. This function will create the config + # header first if needed. The header will be written in a file called "mbed_conf.h" # located in the project's build directory. - # If prefix headers are not used (self.prefix_header_content is None), the function + # If config headers are not used (self.config_header_content is None), the function # returns None - def get_prefix_header(self): - if self.prefix_header_content is None: + def get_config_header(self): + if self.config_header_content is None: return None - prefix_file = join(self.build_dir, "mbed_prefix.h") - if not exists(prefix_file): - with open(prefix_file, "wt") as f: - f.write(self.prefix_header_content) - return prefix_file + config_file = join(self.build_dir, "mbed_conf.h") + if not exists(config_file): + with open(config_file, "wt") as f: + f.write(self.config_header_content) + return config_file from tools.settings import ARM_BIN diff --git a/tools/toolchains/arm.py b/tools/toolchains/arm.py index 6884b263bef..deff6cf9fd3 100644 --- a/tools/toolchains/arm.py +++ b/tools/toolchains/arm.py @@ -116,9 +116,9 @@ def get_dep_option(self, object): def get_compile_options(self, defines, includes): opts = ['-D%s' % d for d in defines] + ['--via', self.get_inc_file(includes)] - prefix_header = self.get_prefix_header() - if prefix_header is not None: - opts = opts + ['--preinclude', prefix_header] + config_header = self.get_config_header() + if config_header is not None: + opts = opts + ['--preinclude', config_header] return opts @hook_tool diff --git a/tools/toolchains/gcc.py b/tools/toolchains/gcc.py index f2dc9f78b24..dc10c13d8c8 100644 --- a/tools/toolchains/gcc.py +++ b/tools/toolchains/gcc.py @@ -167,9 +167,9 @@ def get_dep_option(self, object): def get_compile_options(self, defines, includes): opts = ['-D%s' % d for d in defines] + ['@%s' % self.get_inc_file(includes)] - prefix_header = self.get_prefix_header() - if prefix_header is not None: - opts = opts + ['-include', prefix_header] + config_header = self.get_config_header() + if config_header is not None: + opts = opts + ['-include', config_header] return opts @hook_tool diff --git a/tools/toolchains/iar.py b/tools/toolchains/iar.py index 256877451a3..72f94a826cb 100644 --- a/tools/toolchains/iar.py +++ b/tools/toolchains/iar.py @@ -120,9 +120,9 @@ def cc_extra(self, object): def get_compile_options(self, defines, includes): opts = ['-D%s' % d for d in defines] + ['-f', self.get_inc_file(includes)] - prefix_header = self.get_prefix_header() - if prefix_header is not None: - opts = opts + ['--preinclude', prefix_header] + config_header = self.get_config_header() + if config_header is not None: + opts = opts + ['--preinclude', config_header] return opts @hook_tool