From 865c7f8f2f7c467ce3eea0a146326bf2eb2d23f8 Mon Sep 17 00:00:00 2001 From: Zach White Date: Sun, 3 Jan 2021 21:32:34 -0800 Subject: [PATCH 1/3] create an intermediary command to interface with make --- Makefile | 2 +- build_keyboard.mk | 6 --- lib/python/qmk/cli/__init__.py | 1 + lib/python/qmk/cli/make.py | 85 ++++++++++++++++++++++++++++++++++ tmk_core/rules.mk | 8 ++-- 5 files changed, 91 insertions(+), 11 deletions(-) create mode 100755 lib/python/qmk/cli/make.py diff --git a/Makefile b/Makefile index 91ab9e4e8ea6..dd0cb8e1c7cb 100644 --- a/Makefile +++ b/Makefile @@ -386,7 +386,7 @@ define PARSE_KEYMAP # Specify the variables that we are passing forward to submake MAKE_VARS := KEYBOARD=$$(CURRENT_KB) KEYMAP=$$(CURRENT_KM) REQUIRE_PLATFORM_KEY=$$(REQUIRE_PLATFORM_KEY) # And the first part of the make command - MAKE_CMD := $$(MAKE) -r -R -C $(ROOT_DIR) -f build_keyboard.mk $$(MAKE_TARGET) + MAKE_CMD := bin/qmk make -f build_keyboard.mk $$(MAKE_TARGET) # The message to display MAKE_MSG := $$(MSG_MAKE_KB) # We run the command differently, depending on if we want more output or not diff --git a/build_keyboard.mk b/build_keyboard.mk index d9bb2b92e23c..1b2aa0657bdd 100644 --- a/build_keyboard.mk +++ b/build_keyboard.mk @@ -313,12 +313,6 @@ endif CONFIG_H += $(KEYBOARD_OUTPUT)/src/info_config.h $(KEYBOARD_OUTPUT)/src/layouts.h -$(KEYBOARD_OUTPUT)/src/info_config.h: $(INFO_JSON_FILES) - bin/qmk generate-config-h --quiet --keyboard $(KEYBOARD) --output $(KEYBOARD_OUTPUT)/src/info_config.h - -$(KEYBOARD_OUTPUT)/src/layouts.h: $(INFO_JSON_FILES) - bin/qmk generate-layouts --quiet --keyboard $(KEYBOARD) --output $(KEYBOARD_OUTPUT)/src/layouts.h - # project specific files SRC += $(KEYBOARD_SRC) \ $(KEYMAP_C) \ diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index 372c40921a86..b0ca9b0061c9 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -24,6 +24,7 @@ from . import lint from . import list from . import kle2json +from . import make from . import new from . import pyformat from . import pytest diff --git a/lib/python/qmk/cli/make.py b/lib/python/qmk/cli/make.py new file mode 100755 index 000000000000..02aabbe555f8 --- /dev/null +++ b/lib/python/qmk/cli/make.py @@ -0,0 +1,85 @@ +"""Generate and run a make command for building a keyboard. +""" +import os +import subprocess +from argparse import FileType +from time import time + +from milc import cli + +from qmk.decorators import automagic_keyboard, automagic_keymap +from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json + + +file_stale_secs = 86400 # How many seconds old generated files are before being regenerated + + +@cli.argument('targets', nargs='*', arg_only=True, help='Make targets to run') +@cli.argument('-f', '--makefile', arg_only=True, default='build_keyboard.mk', help="The makefile to use") +@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run") +@cli.subcommand('Parse and run a QMK make command', hidden=True) +def make(cli): + """Run a qmk make command. + """ + # Determine the keyboard + silent = False + keyboard = None + build_dir = '.build' + + for target in cli.args.targets: + if target.startswith('KEYBOARD='): + keyboard = target.split('=', 1)[1] + + if target.startswith('BUILD_DIR='): + build_dir = target.split('=', 1)[1] + + if target == 'SILENT=true': + silent = True + + if target == 'SILENT=false': + silent = False + + if not keyboard: + cli.log.error('Could not identify a keyboard for this compile!') + return False + + keyboard_filesafe = keyboard.replace('/', '_') + keyboard_output = f'{build_dir}/obj_{keyboard_filesafe}' + + # Generate the include files + # FIXME(skullydazed/anyone): Instead of hard-coding a stale age we should compare the timestamp against all the info.json's involved + config_h_file = f'{keyboard_output}/src/info_config.h' + layouts_h_file = f'{keyboard_output}/src/layouts.h' + config_h_mtime = os.stat(config_h_file).st_mtime + layouts_h_mtime = os.stat(layouts_h_file).st_mtime + config_h_age = time() - config_h_mtime + layouts_h_age = time() - layouts_h_mtime + + if 'clean' not in cli.args.targets: + config_h_cmd = ['bin/qmk', 'generate-config-h', '--keyboard', keyboard, '--output', config_h_file] + layouts_h_cmd = ['bin/qmk', 'generate-layouts', '--keyboard', keyboard, '--output', layouts_h_file] + if silent: + config_h_cmd.append('-q') + layouts_h_cmd.append('-q') + + if config_h_age > file_stale_secs: + cli.run(config_h_cmd, capture_output=False) + if layouts_h_age > file_stale_secs: + cli.run(layouts_cmd, capture_output=False) + + # Call make + jobs = '1' + if '-j' in os.environ.get('MAKEFLAGS', ''): + for word in os.environ['MAKEFLAGS'].split(): + if word.startswith('-j'): + jobs = word[2:] + break + + shell_env = os.environ.copy() + for key in ('MAKEFLAGS', 'MAKELEVEL', 'MAKE_TERMERR', 'MFLAGS'): + if key in shell_env: + del shell_env[key] + + make_cmd = ['make', '-j', jobs, '-r', '-R', '-f', cli.args.makefile, *cli.args.targets] + cli.log.debug('Running: %s', ' '.join(make_cmd)) + cli.run(make_cmd, capture_output=False, env=shell_env) diff --git a/tmk_core/rules.mk b/tmk_core/rules.mk index b595ddb1db4b..a77e55dd13a7 100644 --- a/tmk_core/rules.mk +++ b/tmk_core/rules.mk @@ -324,27 +324,27 @@ $1_CXXFLAGS = $$(ALL_CXXFLAGS) $$($1_DEFS) $$($1_INCFLAGS) $$($1_CONFIG_FLAGS) $ $1_ASFLAGS = $$(ALL_ASFLAGS) $$($1_DEFS) $$($1_INCFLAGS) $$($1_CONFIG_FLAGS) # Compile: create object files from C source files. -$1/%.o : %.c $1/%.d $1/cflags.txt $1/compiler.txt $(KEYBOARD_OUTPUT)/src/info_config.h $(KEYBOARD_OUTPUT)/src/layouts.h | $(BEGIN) +$1/%.o : %.c $1/%.d $1/cflags.txt $1/compiler.txt | $(BEGIN) @mkdir -p $$(@D) @$$(SILENT) || printf "$$(MSG_COMPILING) $$<" | $$(AWK_CMD) $$(eval CMD := $$(CC) -c $$($1_CFLAGS) $$(INIT_HOOK_CFLAGS) $$(GENDEPFLAGS) $$< -o $$@ && $$(MOVE_DEP)) @$$(BUILD_CMD) # Compile: create object files from C++ source files. -$1/%.o : %.cpp $1/%.d $1/cxxflags.txt $1/compiler.txt $(KEYBOARD_OUTPUT)/src/info_config.h $(KEYBOARD_OUTPUT)/src/layouts.h | $(BEGIN) +$1/%.o : %.cpp $1/%.d $1/cxxflags.txt $1/compiler.txt | $(BEGIN) @mkdir -p $$(@D) @$$(SILENT) || printf "$$(MSG_COMPILING_CXX) $$<" | $$(AWK_CMD) $$(eval CMD=$$(CC) -c $$($1_CXXFLAGS) $$(INIT_HOOK_CFLAGS) $$(GENDEPFLAGS) $$< -o $$@ && $$(MOVE_DEP)) @$$(BUILD_CMD) -$1/%.o : %.cc $1/%.d $1/cxxflags.txt $1/compiler.txt $(KEYBOARD_OUTPUT)/src/info_config.h $(KEYBOARD_OUTPUT)/src/layouts.h | $(BEGIN) +$1/%.o : %.cc $1/%.d $1/cxxflags.txt $1/compiler.txt | $(BEGIN) @mkdir -p $$(@D) @$$(SILENT) || printf "$$(MSG_COMPILING_CXX) $$<" | $$(AWK_CMD) $$(eval CMD=$$(CC) -c $$($1_CXXFLAGS) $$(INIT_HOOK_CFLAGS) $$(GENDEPFLAGS) $$< -o $$@ && $$(MOVE_DEP)) @$$(BUILD_CMD) # Assemble: create object files from assembler source files. -$1/%.o : %.S $1/asflags.txt $1/compiler.txt $(KEYBOARD_OUTPUT)/src/info_config.h $(KEYBOARD_OUTPUT)/src/layouts.h | $(BEGIN) +$1/%.o : %.S $1/asflags.txt $1/compiler.txt | $(BEGIN) @mkdir -p $$(@D) @$(SILENT) || printf "$$(MSG_ASSEMBLING) $$<" | $$(AWK_CMD) $$(eval CMD=$$(CC) -c $$($1_ASFLAGS) $$< -o $$@) From 93036239d312f2bb4e8ddd9d1f4913e46761a679 Mon Sep 17 00:00:00 2001 From: Zach White Date: Sun, 3 Jan 2021 21:46:12 -0800 Subject: [PATCH 2/3] check file existance --- lib/python/qmk/cli/make.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/python/qmk/cli/make.py b/lib/python/qmk/cli/make.py index 02aabbe555f8..ca19c83edda5 100755 --- a/lib/python/qmk/cli/make.py +++ b/lib/python/qmk/cli/make.py @@ -49,11 +49,18 @@ def make(cli): # Generate the include files # FIXME(skullydazed/anyone): Instead of hard-coding a stale age we should compare the timestamp against all the info.json's involved config_h_file = f'{keyboard_output}/src/info_config.h' + if os.path.exists(config_h_file): + config_h_mtime = os.stat(config_h_file).st_mtime + config_h_age = time() - config_h_mtime + else: + config_h_age = file_stale_secs + 1 + layouts_h_file = f'{keyboard_output}/src/layouts.h' - config_h_mtime = os.stat(config_h_file).st_mtime - layouts_h_mtime = os.stat(layouts_h_file).st_mtime - config_h_age = time() - config_h_mtime - layouts_h_age = time() - layouts_h_mtime + if os.path.exists(layouts_h_file): + layouts_h_mtime = os.stat(layouts_h_file).st_mtime + layouts_h_age = time() - layouts_h_mtime + else: + layouts_h_age = file_stale_secs + 1 if 'clean' not in cli.args.targets: config_h_cmd = ['bin/qmk', 'generate-config-h', '--keyboard', keyboard, '--output', config_h_file] @@ -65,7 +72,7 @@ def make(cli): if config_h_age > file_stale_secs: cli.run(config_h_cmd, capture_output=False) if layouts_h_age > file_stale_secs: - cli.run(layouts_cmd, capture_output=False) + cli.run(layouts_h_cmd, capture_output=False) # Call make jobs = '1' From 06b56f054d01cf8b4da565e209cc8f6169f586d7 Mon Sep 17 00:00:00 2001 From: Zach White Date: Sun, 3 Jan 2021 22:06:24 -0800 Subject: [PATCH 3/3] be smarter about regenerating files --- lib/python/qmk/cli/make.py | 46 ++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/lib/python/qmk/cli/make.py b/lib/python/qmk/cli/make.py index ca19c83edda5..fbaed10f79fa 100755 --- a/lib/python/qmk/cli/make.py +++ b/lib/python/qmk/cli/make.py @@ -1,19 +1,36 @@ """Generate and run a make command for building a keyboard. """ import os -import subprocess -from argparse import FileType from time import time from milc import cli -from qmk.decorators import automagic_keyboard, automagic_keymap -from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json - +from qmk.info import find_info_json file_stale_secs = 86400 # How many seconds old generated files are before being regenerated +def is_stale(file_path, *dependencies): + """Checks to see if a file is stale and should be regenerated. + """ + if not os.path.exists(file_path): + return True + + file_mtime = os.stat(file_path).st_mtime + + # If the file is more than 24 hours old we consider it stale anyway + if time() - file_mtime > file_stale_secs: + return True + + # Iterate through dependencies, if we find one that's newer return True + for dep in dependencies: + dep_mtime = os.stat(dep).st_mtime + if dep_mtime > file_mtime: + return True + + return False + + @cli.argument('targets', nargs='*', arg_only=True, help='Make targets to run') @cli.argument('-f', '--makefile', arg_only=True, default='build_keyboard.mk', help="The makefile to use") @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run") @@ -47,31 +64,22 @@ def make(cli): keyboard_output = f'{build_dir}/obj_{keyboard_filesafe}' # Generate the include files - # FIXME(skullydazed/anyone): Instead of hard-coding a stale age we should compare the timestamp against all the info.json's involved + info_json_files = find_info_json(keyboard) config_h_file = f'{keyboard_output}/src/info_config.h' - if os.path.exists(config_h_file): - config_h_mtime = os.stat(config_h_file).st_mtime - config_h_age = time() - config_h_mtime - else: - config_h_age = file_stale_secs + 1 - layouts_h_file = f'{keyboard_output}/src/layouts.h' - if os.path.exists(layouts_h_file): - layouts_h_mtime = os.stat(layouts_h_file).st_mtime - layouts_h_age = time() - layouts_h_mtime - else: - layouts_h_age = file_stale_secs + 1 if 'clean' not in cli.args.targets: config_h_cmd = ['bin/qmk', 'generate-config-h', '--keyboard', keyboard, '--output', config_h_file] layouts_h_cmd = ['bin/qmk', 'generate-layouts', '--keyboard', keyboard, '--output', layouts_h_file] + if silent: config_h_cmd.append('-q') layouts_h_cmd.append('-q') - if config_h_age > file_stale_secs: + if is_stale(config_h_file, *info_json_files): cli.run(config_h_cmd, capture_output=False) - if layouts_h_age > file_stale_secs: + + if is_stale(layouts_h_file, *info_json_files): cli.run(layouts_h_cmd, capture_output=False) # Call make