From 50fdb2a52c67102cffe361f1cd867ab16dde828e Mon Sep 17 00:00:00 2001 From: Zach White Date: Thu, 24 Jun 2021 12:42:23 -0700 Subject: [PATCH 01/14] Rework `qmk compile` to bypass `Makefile`. Add new --filter option. --- Makefile | 2 +- lib/python/qmk/cli/clean.py | 6 +- lib/python/qmk/cli/compile.py | 140 ++++++++-- lib/python/qmk/cli/flash.py | 4 +- lib/python/qmk/commands.py | 24 +- lib/python/qmk/info.py | 61 +---- lib/python/qmk/keyboard.py | 11 + lib/python/qmk/keymap.py | 54 ++-- lib/python/qmk/metadata.py | 483 ++++++++++++++++++++++++++++++++++ 9 files changed, 670 insertions(+), 115 deletions(-) create mode 100644 lib/python/qmk/metadata.py diff --git a/Makefile b/Makefile index 269be720c2f9..7923a978cb16 100644 --- a/Makefile +++ b/Makefile @@ -398,7 +398,7 @@ define PARSE_KEYMAP # Specify the variables that we are passing forward to submake MAKE_VARS := KEYBOARD=$$(CURRENT_KB) KEYMAP=$$(CURRENT_KM) REQUIRE_PLATFORM_KEY=$$(REQUIRE_PLATFORM_KEY) QMK_BIN=$$(QMK_BIN) # And the first part of the make command - MAKE_CMD := $$(MAKE) -r -R -C $(ROOT_DIR) -f build_keyboard.mk $$(MAKE_TARGET) + MAKE_CMD := echo $$(MAKE) -r -R -C $(ROOT_DIR) -f build_keyboard.mk $$(MAKE_TARGET) # The message to display MAKE_MSG := $$(MSG_MAKE_KB) # We run the command differently, depending on if we want more output or not diff --git a/lib/python/qmk/cli/clean.py b/lib/python/qmk/cli/clean.py index 72b7ffe81071..f0b1a897af19 100644 --- a/lib/python/qmk/cli/clean.py +++ b/lib/python/qmk/cli/clean.py @@ -2,7 +2,7 @@ """ from subprocess import DEVNULL -from qmk.commands import create_make_target +from qmk.commands import _find_make from milc import cli @@ -11,4 +11,6 @@ def clean(cli): """Runs `make clean` (or `make distclean` if --all is passed) """ - cli.run(create_make_target('distclean' if cli.args.all else 'clean'), capture_output=False, stdin=DEVNULL) + make_cmd = [_find_make(), 'distclean' if cli.args.all else 'clean'] + + cli.run(make_cmd, capture_output=False, stdin=DEVNULL) diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index acbd77864986..66f55981ee35 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -5,20 +5,70 @@ from subprocess import DEVNULL from argcomplete.completers import FilesCompleter +from dotty_dict import dotty from milc import cli import qmk.path -from qmk.decorators import automagic_keyboard, automagic_keymap +from qmk.decorators import automagic_keyboard, automagic_keymap, lru_cache from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json -from qmk.keyboard import keyboard_completer, keyboard_folder -from qmk.keymap import keymap_completer +from qmk.info import info_json +from qmk.keyboard import keyboard_completer, is_keyboard_target, list_keyboards +from qmk.keymap import keymap_completer, list_keymaps +from qmk.metadata import true_values, false_values + + +@lru_cache() +def _keyboard_list(): + """Returns a list of keyboards matching cli.config.compile.keyboard. + """ + if cli.config.compile.keyboard == 'all': + return list_keyboards() + + elif cli.config.compile.keyboard.startswith('all-'): + return list_keyboards() + + return [cli.config.compile.keyboard] + + +def keyboard_keymap_iter(): + """Iterates over the keyboard/keymap for this command and yields a pairing of each. + """ + for keyboard in _keyboard_list(): + continue_flag = False + if cli.args.filter: + info_data = dotty(info_json(keyboard)) + for filter in cli.args.filter: + if '=' in filter: + key, value = filter.split('=', 1) + if value in true_values: + value = True + elif value in false_values: + value = False + elif value.isdigit(): + value = int(value) + elif '.' in value and value.replace('.').isdigit(): + value = float(value) + + if info_data.get(key) != value: + continue_flag = True + break + + if continue_flag: + continue + + if cli.config.compile.keymap == 'all': + for keymap in list_keymaps(keyboard): + yield keyboard, keymap + else: + yield keyboard, cli.config.compile.keymap @cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile') -@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') +@cli.argument('-kb', '--keyboard', type=is_keyboard_target, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") -@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") +@cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs to run.") +@cli.argument('-f', '--filter', arg_only=True, action='append', default=[], help="Filter your list against info.json.") @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.") @cli.subcommand('Compile a QMK Firmware.') @@ -31,47 +81,79 @@ def compile(cli): If a keyboard and keymap are provided this command will build a firmware based on that. """ - if cli.args.clean and not cli.args.filename and not cli.args.dry_run: - command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, 'clean') - cli.run(command, capture_output=False, stdin=DEVNULL) + envs = {'REQUIRE_PLATFORM_KEY': ''} + silent = cli.config.compile.keyboard == 'all' or cli.config.compile.keyboard.startswith('all-') or cli.config.compile.keymap == 'all' - # Build the environment vars - envs = {} + # Setup the environment for env in cli.args.env: if '=' in env: key, value = env.split('=', 1) + if key in envs: + cli.log.warning('Overwriting existing environment variable %s=%s with %s=%s!', key, envs[key], key, value) envs[key] = value else: cli.log.warning('Invalid environment variable: %s', env) - # Determine the compile command - command = None + if cli.config.compile.keyboard.startswith('all-'): + envs['REQUIRE_PLATFORM_KEY'] = cli.config.compile.keyboard[4:] + + # Run clean if necessary + if cli.args.clean and not cli.args.filename and not cli.args.dry_run: + for keyboard, keymap in keyboard_keymap_iter(): + cli.log.info('Cleaning previous build files for keyboard {fg_cyan}%s{fg_reset} keymap {fg_cyan}%s', keyboard, keymap) + _, _, make_cmd = create_make_command(keyboard, keymap, 'clean', cli.config.compile.parallel, silent, **envs) + cli.run(make_cmd, capture_output=False, stdin=DEVNULL) + + # Determine the compile command(s) + commands = None if cli.args.filename: # If a configurator JSON was provided generate a keymap and compile it user_keymap = parse_configurator_json(cli.args.filename) - command = compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, **envs) + commands = [compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, **envs)] - else: - if cli.config.compile.keyboard and cli.config.compile.keymap: - # Generate the make command for a specific keyboard/keymap. - command = create_make_command(cli.config.compile.keyboard, cli.config.compile.keymap, parallel=cli.config.compile.parallel, **envs) + elif cli.config.compile.keyboard and cli.config.compile.keymap: + if cli.args.filter: + cli.log.info('Generating the list of keyboards to compile, this may take some time.') + + commands = [create_make_command(keyboard, keymap, parallel=cli.config.compile.parallel, silent=silent, **envs) for keyboard, keymap in keyboard_keymap_iter()] - elif not cli.config.compile.keyboard: - cli.log.error('Could not determine keyboard!') - elif not cli.config.compile.keymap: - cli.log.error('Could not determine keymap!') + elif not cli.config.compile.keyboard: + cli.log.error('Could not determine keyboard!') + + elif not cli.config.compile.keymap: + cli.log.error('Could not determine keymap!') # Compile the firmware, if we're able to - if command: - cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) - if not cli.args.dry_run: - cli.echo('\n') - # FIXME(skullydazed/anyone): Remove text=False once milc 1.0.11 has had enough time to be installed everywhere. - compile = cli.run(command, capture_output=False, text=False) - return compile.returncode + if commands: + returncodes = [] + for keyboard, keymap, command in commands: + print() + cli.log.info('Building firmware for {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s', keyboard, keymap) + cli.log.debug('Running make command: {fg_blue}%s', ' '.join(command)) + + if not cli.args.dry_run: + compile = cli.run(command, capture_output=False) + returncodes.append(compile.returncode) + if compile.returncode == 0: + cli.log.info('Success!') + else: + cli.log.error('Failed!') + + if any(returncodes): + print() + cli.log.error('Could not compile all targets, look above this message for more details. Failing target(s):') + + for i, returncode in enumerate(returncodes): + if returncode != 0: + keyboard, keymap, command = commands[i] + cli.echo('\tkeyboard: {fg_cyan}%s{fg_reset} keymap: {fg_cyan}%s', keyboard, keymap) + + elif cli.args.filter: + cli.log.error('No keyboards found after applying filter(s)!') + return False else: cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') - cli.echo('usage: qmk compile [-h] [-b] [-kb KEYBOARD] [-km KEYMAP] [filename]') + cli.print_help() return False diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py index c2d9e09c693c..aa3d74e63a50 100644 --- a/lib/python/qmk/cli/flash.py +++ b/lib/python/qmk/cli/flash.py @@ -11,7 +11,7 @@ import qmk.path from qmk.decorators import automagic_keyboard, automagic_keymap from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json -from qmk.keyboard import keyboard_completer, keyboard_folder +from qmk.keyboard import keyboard_completer, is_keyboard_target def print_bootloader_help(): @@ -36,7 +36,7 @@ def print_bootloader_help(): @cli.argument('-b', '--bootloaders', action='store_true', help='List the available bootloaders.') @cli.argument('-bl', '--bootloader', default='flash', help='The flash command, corresponding to qmk\'s make options of bootloaders.') @cli.argument('-km', '--keymap', help='The keymap to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') -@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') +@cli.argument('-kb', '--keyboard', type=is_keyboard_target, completer=keyboard_completer, help='The keyboard to build a firmware for. Use this if you dont have a configurator file. Ignored when a configurator file is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.") @cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Set a variable to be passed to make. May be passed multiple times.") diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 01c23b261216..c791b745339e 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -55,7 +55,7 @@ def create_make_target(target, parallel=1, **env_vars): return [make_cmd, *get_make_parallel_args(parallel), *env, target] -def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars): +def create_make_command(keyboard, keymap, target=None, parallel=1, silent=False, **env_vars): """Create a make compile command Args: @@ -79,12 +79,26 @@ def create_make_command(keyboard, keymap, target=None, parallel=1, **env_vars): A command that can be run to make the specified keyboard and keymap """ - make_args = [keyboard, keymap] + make_cmd = [_find_make(), '--no-print-directory', '-r', '-R', '-C', './', '-f', 'build_keyboard.mk'] + + env_vars['KEYBOARD'] = keyboard + env_vars['KEYMAP'] = keymap + env_vars['QMK_BIN'] = 'bin/qmk' if 'DEPRECATED_BIN_QMK' in os.environ else 'qmk' + env_vars['VERBOSE'] = 'true' if cli.config.general.verbose else '' + env_vars['SILENT'] = 'true' if silent else 'false' + env_vars['COLOR'] = 'true' if cli.config.general.color else '' + + if parallel > 1: + make_cmd.append('-j') + make_cmd.append(parallel) if target: - make_args.append(target) + make_cmd.append(target) + + for key, value in env_vars.items(): + make_cmd.append(f'{key}={value}') - return create_make_target(':'.join(make_args), parallel, **env_vars) + return keyboard, keymap, make_cmd def get_git_version(current_time, repo_dir='.', check_dir='.'): @@ -236,7 +250,7 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va 'QMK_BIN="qmk"', ]) - return make_command + return user_keymap['keyboard'], user_keymap['keymap'], make_command def parse_configurator_json(configurator_file): diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index 7f3aabdc3b01..d868e9438bf1 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -1,22 +1,15 @@ """Functions that help us generate and use info.json files. """ -from glob import glob +from functools import lru_cache from pathlib import Path import jsonschema -from dotty_dict import dotty from milc import cli from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS -from qmk.c_parse import find_layouts -from qmk.json_schema import deep_update, json_load, validate -from qmk.keyboard import config_h, rules_mk +from qmk.json_schema import validate from qmk.keymap import list_keymaps -from qmk.makefile import parse_rules_mk_file -from qmk.math import compute - -true_values = ['1', 'on', 'yes'] -false_values = ['0', 'off', 'no'] +from qmk.metadata import basic_info_json, info_log_error def _valid_community_layout(layout): @@ -25,24 +18,11 @@ def _valid_community_layout(layout): return (Path('layouts/default') / layout).exists() +@lru_cache(maxsize=None) def info_json(keyboard): """Generate the info.json data for a specific keyboard. """ - cur_dir = Path('keyboards') - rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk') - if 'DEFAULT_FOLDER' in rules: - keyboard = rules['DEFAULT_FOLDER'] - rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk', rules) - - info_data = { - 'keyboard_name': str(keyboard), - 'keyboard_folder': str(keyboard), - 'keymaps': {}, - 'layouts': {}, - 'parse_errors': [], - 'parse_warnings': [], - 'maintainer': 'qmk', - } + info_data = basic_info_json(keyboard) # Populate the list of JSON keymaps for keymap in list_keymaps(keyboard, c=False, fullpath=True): @@ -81,20 +61,20 @@ def info_json(keyboard): _find_missing_layouts(info_data, keyboard) if not info_data.get('layouts'): - _log_error(info_data, 'No LAYOUTs defined! Need at least one layout defined in the keyboard.h or info.json.') + info_log_error(info_data, 'No LAYOUTs defined! Need at least one layout defined in the keyboard.h or info.json.') # Filter out any non-existing community layouts for layout in info_data.get('community_layouts', []): if not _valid_community_layout(layout): # Ignore layout from future checks info_data['community_layouts'].remove(layout) - _log_error(info_data, 'Claims to support a community layout that does not exist: %s' % (layout)) + info_log_error(info_data, 'Claims to support a community layout that does not exist: %s' % (layout)) # Make sure we supply layout macros for the community layouts we claim to support for layout in info_data.get('community_layouts', []): layout_name = 'LAYOUT_' + layout if layout_name not in info_data.get('layouts', {}) and layout_name not in info_data.get('layout_aliases', {}): - _log_error(info_data, 'Claims to support community layout %s but no %s() macro found' % (layout, layout_name)) + info_log_error(info_data, 'Claims to support community layout %s but no %s() macro found' % (layout, layout_name)) # Check that the reported matrix size is consistent with the actual matrix size _check_matrix(info_data) @@ -544,11 +524,11 @@ def _check_matrix(info_data): if col_count != actual_col_count and col_count != (actual_col_count / 2): # FIXME: once we can we should detect if split is enabled to do the actual_col_count/2 check. - _log_error(info_data, f'MATRIX_COLS is inconsistent with the size of MATRIX_COL_PINS: {col_count} != {actual_col_count}') + info_log_error(info_data, f'MATRIX_COLS is inconsistent with the size of MATRIX_COL_PINS: {col_count} != {actual_col_count}') if row_count != actual_row_count and row_count != (actual_row_count / 2): # FIXME: once we can we should detect if split is enabled to do the actual_row_count/2 check. - _log_error(info_data, f'MATRIX_ROWS is inconsistent with the size of MATRIX_ROW_PINS: {row_count} != {actual_row_count}') + info_log_error(info_data, f'MATRIX_ROWS is inconsistent with the size of MATRIX_ROW_PINS: {row_count} != {actual_row_count}') def _search_keyboard_h(keyboard): @@ -596,20 +576,6 @@ def _find_missing_layouts(info_data, keyboard): info_data['layout_aliases'][alias] = alias_text -def _log_error(info_data, message): - """Send an error message to both JSON and the log. - """ - info_data['parse_errors'].append(message) - cli.log.error('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message) - - -def _log_warning(info_data, message): - """Send a warning message to both JSON and the log. - """ - info_data['parse_warnings'].append(message) - cli.log.warning('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message) - - def arm_processor_rules(info_data, rules): """Setup the default info for an ARM board. """ @@ -668,7 +634,7 @@ def merge_info_jsons(keyboard, info_data): new_info_data = json_load(info_file) if not isinstance(new_info_data, dict): - _log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),)) + info_log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),)) continue try: @@ -692,7 +658,7 @@ def merge_info_jsons(keyboard, info_data): if layout_name in info_data['layouts']: if len(info_data['layouts'][layout_name]['layout']) != len(layout['layout']): msg = '%s: %s: Number of elements in info.json does not match! info.json:%s != %s:%s' - _log_error(info_data, msg % (info_data['keyboard_folder'], layout_name, len(layout['layout']), layout_name, len(info_data['layouts'][layout_name]['layout']))) + info_log_error(info_data, msg % (info_data['keyboard_folder'], layout_name, len(layout['layout']), layout_name, len(info_data['layouts'][layout_name]['layout']))) else: for new_key, existing_key in zip(layout['layout'], info_data['layouts'][layout_name]['layout']): existing_key.update(new_key) @@ -730,5 +696,4 @@ def find_info_json(keyboard): break keyboard_parent = keyboard_parent.parent - # Return a list of the info.json files that actually exist - return [info_json for info_json in info_jsons if info_json.exists()] + return info_data diff --git a/lib/python/qmk/keyboard.py b/lib/python/qmk/keyboard.py index c87ea9050b76..77fd9b92d599 100644 --- a/lib/python/qmk/keyboard.py +++ b/lib/python/qmk/keyboard.py @@ -64,6 +64,17 @@ def find_readme(keyboard): return cur_dir / 'readme.md' +def is_keyboard_target(keyboard_target): + """Checks to make sure the supplied keyboard_target is valid. + + This is mainly used by commands that accept --keyboard. + """ + if keyboard_target in ['all', 'all-avr', 'all-chibios', 'all-arm_atsam']: + return keyboard_target + + return keyboard_folder(keyboard_target) + + def keyboard_folder(keyboard): """Returns the actual keyboard folder. diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py index 2d5921e7a868..992b280296d0 100644 --- a/lib/python/qmk/keymap.py +++ b/lib/python/qmk/keymap.py @@ -14,6 +14,7 @@ import qmk.path from qmk.keyboard import find_keyboard_from_dir, rules_mk from qmk.errors import CppError +from qmk.metadata import basic_info_json # The `keymap.c` template to use when a keyboard doesn't have its own DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H @@ -327,36 +328,33 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa Returns: a sorted list of valid keymap names. """ - # parse all the rules.mk files for the keyboard - rules = rules_mk(keyboard) + info_data = basic_info_json(keyboard) names = set() - if rules: - keyboards_dir = Path('keyboards') - kb_path = keyboards_dir / keyboard - - # walk up the directory tree until keyboards_dir - # and collect all directories' name with keymap.c file in it - while kb_path != keyboards_dir: - keymaps_dir = kb_path / "keymaps" - - if keymaps_dir.is_dir(): - for keymap in keymaps_dir.iterdir(): - if is_keymap_dir(keymap, c, json, additional_files): - keymap = keymap if fullpath else keymap.name - names.add(keymap) - - kb_path = kb_path.parent - - # if community layouts are supported, get them - if "LAYOUTS" in rules: - for layout in rules["LAYOUTS"].split(): - cl_path = Path('layouts/community') / layout - if cl_path.is_dir(): - for keymap in cl_path.iterdir(): - if is_keymap_dir(keymap, c, json, additional_files): - keymap = keymap if fullpath else keymap.name - names.add(keymap) + keyboards_dir = Path('keyboards') + kb_path = keyboards_dir / info_data['keyboard_folder'] + + # walk up the directory tree until keyboards_dir + # and collect all directories' name with keymap.c file in it + while kb_path != keyboards_dir: + keymaps_dir = kb_path / "keymaps" + + if keymaps_dir.is_dir(): + for keymap in keymaps_dir.iterdir(): + if is_keymap_dir(keymap, c, json, additional_files): + keymap = keymap if fullpath else keymap.name + names.add(keymap) + + kb_path = kb_path.parent + + # if community layouts are supported, get them + for layout in info_data.get('community_layouts', []): + cl_path = Path('layouts/community') / layout + if cl_path.is_dir(): + for keymap in cl_path.iterdir(): + if is_keymap_dir(keymap, c, json, additional_files): + keymap = keymap if fullpath else keymap.name + names.add(keymap) return sorted(names) diff --git a/lib/python/qmk/metadata.py b/lib/python/qmk/metadata.py new file mode 100644 index 000000000000..4c0c8ce05fee --- /dev/null +++ b/lib/python/qmk/metadata.py @@ -0,0 +1,483 @@ +"""Functions that help us generate and use info.json files. +""" +from functools import lru_cache +from glob import glob +from pathlib import Path + +import jsonschema +from dotty_dict import dotty +from milc import cli + +from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS +from qmk.c_parse import find_layouts +from qmk.json_schema import deep_update, json_load, validate +from qmk.keyboard import config_h, rules_mk +from qmk.makefile import parse_rules_mk_file +from qmk.math import compute + +true_values = ['1', 'on', 'yes', 'true'] +false_values = ['0', 'off', 'no', 'false'] + + +@lru_cache(maxsize=None) +def basic_info_json(keyboard): + """Generate a subset of info.json for a specific keyboard. + + This does no validation, and should only be used as needed to avoid loops or when performance is critical. + """ + cur_dir = Path('keyboards') + rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk') + + if 'DEFAULT_FOLDER' in rules: + keyboard = rules['DEFAULT_FOLDER'] + rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk', rules) + + info_data = { + 'keyboard_name': str(keyboard), + 'keyboard_folder': str(keyboard), + 'keymaps': {}, + 'layouts': {}, + 'parse_errors': [], + 'parse_warnings': [], + 'maintainer': 'qmk', + } + + # Populate layout data + layouts, aliases = _find_all_layouts(info_data, keyboard) + + if aliases: + info_data['layout_aliases'] = aliases + + for layout_name, layout_json in layouts.items(): + if not layout_name.startswith('LAYOUT_kc'): + layout_json['c_macro'] = True + info_data['layouts'][layout_name] = layout_json + + # Merge in the data from info.json, config.h, and rules.mk + info_data = merge_info_jsons(keyboard, info_data) + info_data = _extract_config_h(info_data) + info_data = _extract_rules_mk(info_data) + + return info_data + + +def _extract_features(info_data, rules): + """Find all the features enabled in rules.mk. + """ + # Special handling for bootmagic which also supports a "lite" mode. + if rules.get('BOOTMAGIC_ENABLE') == 'lite': + rules['BOOTMAGIC_LITE_ENABLE'] = 'on' + del rules['BOOTMAGIC_ENABLE'] + if rules.get('BOOTMAGIC_ENABLE') == 'full': + rules['BOOTMAGIC_ENABLE'] = 'on' + + # Skip non-boolean features we haven't implemented special handling for + for feature in 'HAPTIC_ENABLE', 'QWIIC_ENABLE': + if rules.get(feature): + del rules[feature] + + # Process the rest of the rules as booleans + for key, value in rules.items(): + if key.endswith('_ENABLE'): + key = '_'.join(key.split('_')[:-1]).lower() + value = True if value.lower() in true_values else False if value.lower() in false_values else value + + if 'config_h_features' not in info_data: + info_data['config_h_features'] = {} + + if 'features' not in info_data: + info_data['features'] = {} + + if key in info_data['features']: + info_log_warning(info_data, 'Feature %s is specified in both info.json and rules.mk, the rules.mk value wins.' % (key,)) + + info_data['features'][key] = value + info_data['config_h_features'][key] = value + + return info_data + + +def _pin_name(pin): + """Returns the proper representation for a pin. + """ + pin = pin.strip() + + if not pin: + return None + + elif pin.isdigit(): + return int(pin) + + elif pin == 'NO_PIN': + return None + + return pin + + +def _extract_pins(pins): + """Returns a list of pins from a comma separated string of pins. + """ + return [_pin_name(pin) for pin in pins.split(',')] + + +def _extract_direct_matrix(info_data, direct_pins): + """ + """ + info_data['matrix_pins'] = {} + direct_pin_array = [] + + while direct_pins[-1] != '}': + direct_pins = direct_pins[:-1] + + for row in direct_pins.split('},{'): + if row.startswith('{'): + row = row[1:] + + if row.endswith('}'): + row = row[:-1] + + direct_pin_array.append([]) + + for pin in row.split(','): + if pin == 'NO_PIN': + pin = None + + direct_pin_array[-1].append(pin) + + return direct_pin_array + + +def _extract_matrix_info(info_data, config_c): + """Populate the matrix information. + """ + row_pins = config_c.get('MATRIX_ROW_PINS', '').replace('{', '').replace('}', '').strip() + col_pins = config_c.get('MATRIX_COL_PINS', '').replace('{', '').replace('}', '').strip() + direct_pins = config_c.get('DIRECT_PINS', '').replace(' ', '')[1:-1] + + if 'MATRIX_ROWS' in config_c and 'MATRIX_COLS' in config_c: + if 'matrix_size' in info_data: + info_log_warning(info_data, 'Matrix size is specified in both info.json and config.h, the config.h values win.') + + info_data['matrix_size'] = { + 'cols': compute(config_c.get('MATRIX_COLS', '0')), + 'rows': compute(config_c.get('MATRIX_ROWS', '0')), + } + + if row_pins and col_pins: + if 'matrix_pins' in info_data: + info_log_warning(info_data, 'Matrix pins are specified in both info.json and config.h, the config.h values win.') + + info_data['matrix_pins'] = { + 'cols': _extract_pins(col_pins), + 'rows': _extract_pins(row_pins), + } + + if direct_pins: + if 'matrix_pins' in info_data: + info_log_warning(info_data, 'Direct pins are specified in both info.json and config.h, the config.h values win.') + + info_data['matrix_pins']['direct'] = _extract_direct_matrix(info_data, direct_pins) + + return info_data + + +def _extract_config_h(info_data): + """Pull some keyboard information from existing config.h files + """ + config_c = config_h(info_data['keyboard_folder']) + + # Pull in data from the json map + dotty_info = dotty(info_data) + info_config_map = json_load(Path('data/mappings/info_config.json')) + + for config_key, info_dict in info_config_map.items(): + info_key = info_dict['info_key'] + key_type = info_dict.get('value_type', 'str') + + try: + if config_key in config_c and info_dict.get('to_json', True): + if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True): + info_log_warning(info_data, '%s in config.h is overwriting %s in info.json' % (config_key, info_key)) + + if key_type.startswith('array'): + if '.' in key_type: + key_type, array_type = key_type.split('.', 1) + else: + array_type = None + + config_value = config_c[config_key].replace('{', '').replace('}', '').strip() + + if array_type == 'int': + dotty_info[info_key] = list(map(int, config_value.split(','))) + else: + dotty_info[info_key] = config_value.split(',') + + elif key_type == 'bool': + dotty_info[info_key] = config_c[config_key] in true_values + + elif key_type == 'hex': + dotty_info[info_key] = '0x' + config_c[config_key][2:].upper() + + elif key_type == 'list': + dotty_info[info_key] = config_c[config_key].split() + + elif key_type == 'int': + dotty_info[info_key] = int(config_c[config_key]) + + else: + dotty_info[info_key] = config_c[config_key] + + except Exception as e: + info_log_warning(info_data, f'{config_key}->{info_key}: {e}') + + info_data.update(dotty_info) + + # Pull data that easily can't be mapped in json + _extract_matrix_info(info_data, config_c) + + return info_data + + +def _extract_rules_mk(info_data): + """Pull some keyboard information from existing rules.mk files + """ + rules = rules_mk(info_data['keyboard_folder']) + info_data['processor'] = rules.get('MCU', info_data.get('processor', 'atmega32u4')) + + if info_data['processor'] in CHIBIOS_PROCESSORS: + arm_processor_rules(info_data, rules) + + elif info_data['processor'] in LUFA_PROCESSORS + VUSB_PROCESSORS: + avr_processor_rules(info_data, rules) + + else: + cli.log.warning("%s: Unknown MCU: %s" % (info_data['keyboard_folder'], info_data['processor'])) + unknown_processor_rules(info_data, rules) + + # Pull in data from the json map + dotty_info = dotty(info_data) + info_rules_map = json_load(Path('data/mappings/info_rules.json')) + + for rules_key, info_dict in info_rules_map.items(): + info_key = info_dict['info_key'] + key_type = info_dict.get('value_type', 'str') + + try: + if rules_key in rules and info_dict.get('to_json', True): + if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True): + info_log_warning(info_data, '%s in rules.mk is overwriting %s in info.json' % (rules_key, info_key)) + + if key_type.startswith('array'): + if '.' in key_type: + key_type, array_type = key_type.split('.', 1) + else: + array_type = None + + rules_value = rules[rules_key].replace('{', '').replace('}', '').strip() + + if array_type == 'int': + dotty_info[info_key] = list(map(int, rules_value.split(','))) + else: + dotty_info[info_key] = rules_value.split(',') + + elif key_type == 'list': + dotty_info[info_key] = rules[rules_key].split() + + elif key_type == 'bool': + dotty_info[info_key] = rules[rules_key] in true_values + + elif key_type == 'hex': + dotty_info[info_key] = '0x' + rules[rules_key][2:].upper() + + elif key_type == 'int': + dotty_info[info_key] = int(rules[rules_key]) + + else: + dotty_info[info_key] = rules[rules_key] + + except Exception as e: + info_log_warning(info_data, f'{rules_key}->{info_key}: {e}') + + info_data.update(dotty_info) + + # Merge in config values that can't be easily mapped + _extract_features(info_data, rules) + + return info_data + + +def _search_keyboard_h(path): + current_path = Path('keyboards/') + aliases = {} + layouts = {} + + for directory in path.parts: + current_path = current_path / directory + keyboard_h = '%s.h' % (directory,) + keyboard_h_path = current_path / keyboard_h + if keyboard_h_path.exists(): + new_layouts, new_aliases = find_layouts(keyboard_h_path) + layouts.update(new_layouts) + + for alias, alias_text in new_aliases.items(): + if alias_text in layouts: + aliases[alias] = alias_text + + return layouts, aliases + + +def _find_all_layouts(info_data, keyboard): + """Looks for layout macros associated with this keyboard. + """ + layouts, aliases = _search_keyboard_h(Path(keyboard)) + + if not layouts: + # If we don't find any layouts from info.json or keyboard.h we widen our search. This is error prone which is why we want to encourage people to follow the standard above. + info_data['parse_warnings'].append('%s: Falling back to searching for KEYMAP/LAYOUT macros.' % (keyboard)) + + for file in glob('keyboards/%s/*.h' % keyboard): + if file.endswith('.h'): + these_layouts, these_aliases = find_layouts(file) + + if these_layouts: + layouts.update(these_layouts) + + for alias, alias_text in these_aliases.items(): + if alias_text in layouts: + aliases[alias] = alias_text + + return layouts, aliases + + +def info_log_error(info_data, message): + """Send an error message to both JSON and the log. + """ + info_data['parse_errors'].append(message) + cli.log.error('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message) + + +def info_log_warning(info_data, message): + """Send a warning message to both JSON and the log. + """ + info_data['parse_warnings'].append(message) + cli.log.warning('%s: %s', info_data.get('keyboard_folder', 'Unknown Keyboard!'), message) + + +def arm_processor_rules(info_data, rules): + """Setup the default info for an ARM board. + """ + info_data['processor_type'] = 'arm' + info_data['protocol'] = 'ChibiOS' + + if 'bootloader' not in info_data: + if 'STM32' in info_data['processor']: + info_data['bootloader'] = 'stm32-dfu' + else: + info_data['bootloader'] = 'unknown' + + if 'STM32' in info_data['processor']: + info_data['platform'] = 'STM32' + elif 'MCU_SERIES' in rules: + info_data['platform'] = rules['MCU_SERIES'] + elif 'ARM_ATSAM' in rules: + info_data['platform'] = 'ARM_ATSAM' + + return info_data + + +def avr_processor_rules(info_data, rules): + """Setup the default info for an AVR board. + """ + info_data['processor_type'] = 'avr' + info_data['platform'] = rules['ARCH'] if 'ARCH' in rules else 'unknown' + info_data['protocol'] = 'V-USB' if rules.get('MCU') in VUSB_PROCESSORS else 'LUFA' + + if 'bootloader' not in info_data: + info_data['bootloader'] = 'atmel-dfu' + + # FIXME(fauxpark/anyone): Eventually we should detect the protocol by looking at PROTOCOL inherited from mcu_selection.mk: + # info_data['protocol'] = 'V-USB' if rules.get('PROTOCOL') == 'VUSB' else 'LUFA' + + return info_data + + +def unknown_processor_rules(info_data, rules): + """Setup the default keyboard info for unknown boards. + """ + info_data['bootloader'] = 'unknown' + info_data['platform'] = 'unknown' + info_data['processor'] = 'unknown' + info_data['processor_type'] = 'unknown' + info_data['protocol'] = 'unknown' + + return info_data + + +def merge_info_jsons(keyboard, info_data): + """Return a merged copy of all the info.json files for a keyboard. + """ + for info_file in find_info_json(keyboard): + # Load and validate the JSON data + new_info_data = json_load(info_file) + + if not isinstance(new_info_data, dict): + info_log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),)) + continue + + try: + validate(new_info_data, 'qmk.keyboard.v1') + except jsonschema.ValidationError as e: + json_path = '.'.join([str(p) for p in e.absolute_path]) + cli.log.error('Not including data from file: %s', info_file) + cli.log.error('\t%s: %s', json_path, e.message) + continue + + # Merge layout data in + if 'layout_aliases' in new_info_data: + info_data['layout_aliases'] = {**info_data.get('layout_aliases', {}), **new_info_data['layout_aliases']} + del new_info_data['layout_aliases'] + + for layout_name, layout in new_info_data.get('layouts', {}).items(): + if layout_name in info_data.get('layout_aliases', {}): + info_log_warning(info_data, f"info.json uses alias name {layout_name} instead of {info_data['layout_aliases'][layout_name]}") + layout_name = info_data['layout_aliases'][layout_name] + + if layout_name in info_data['layouts']: + for new_key, existing_key in zip(layout['layout'], info_data['layouts'][layout_name]['layout']): + existing_key.update(new_key) + else: + layout['c_macro'] = False + info_data['layouts'][layout_name] = layout + + # Update info_data with the new data + if 'layouts' in new_info_data: + del new_info_data['layouts'] + + deep_update(info_data, new_info_data) + + return info_data + + +def find_info_json(keyboard): + """Finds all the info.json files associated with a keyboard. + """ + # Find the most specific first + base_path = Path('keyboards') + keyboard_path = base_path / keyboard + keyboard_parent = keyboard_path.parent + info_jsons = [keyboard_path / 'info.json'] + + # Add DEFAULT_FOLDER before parents, if present + rules = rules_mk(keyboard) + if 'DEFAULT_FOLDER' in rules: + info_jsons.append(Path(rules['DEFAULT_FOLDER']) / 'info.json') + + # Add in parent folders for least specific + for _ in range(5): + info_jsons.append(keyboard_parent / 'info.json') + if keyboard_parent.parent == base_path: + break + keyboard_parent = keyboard_parent.parent + + # Return a list of the info.json files that actually exist + return [info_json for info_json in info_jsons if info_json.exists()] From d3ed6fa8a4881157e4172ec55144f6719d34a5cd Mon Sep 17 00:00:00 2001 From: Zach White Date: Sat, 26 Jun 2021 16:06:36 -0700 Subject: [PATCH 02/14] eliminate the need for -kb all --- lib/python/qmk/cli/compile.py | 12 ++++++++++++ lib/python/qmk/keymap.py | 2 ++ 2 files changed, 14 insertions(+) diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 66f55981ee35..5ffae74ed1b8 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -82,6 +82,9 @@ def compile(cli): If a keyboard and keymap are provided this command will build a firmware based on that. """ envs = {'REQUIRE_PLATFORM_KEY': ''} + if cli.config.compile.keyboard is None: + cli.config.compile.keyboard = '' + silent = cli.config.compile.keyboard == 'all' or cli.config.compile.keyboard.startswith('all-') or cli.config.compile.keymap == 'all' # Setup the environment @@ -104,6 +107,11 @@ def compile(cli): _, _, make_cmd = create_make_command(keyboard, keymap, 'clean', cli.config.compile.parallel, silent, **envs) cli.run(make_cmd, capture_output=False, stdin=DEVNULL) + # If -f has been specified without a keyboard target, assume -kb all + if cli.args.filter and not cli.args.keyboard: + cli.log.debug('--filter supplied without --keyboard, assuming --keyboard all.') + cli.config.compile.keyboard = 'all' + # Determine the compile command(s) commands = None @@ -128,6 +136,10 @@ def compile(cli): if commands: returncodes = [] for keyboard, keymap, command in commands: + if keymap not in list_keymaps(keyboard): + cli.log.debug('Skipping keyboard %s, no %s keymap found.', keyboard, keymap) + continue + print() cli.log.info('Building firmware for {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s', keyboard, keymap) cli.log.debug('Running make command: {fg_blue}%s', ' '.join(command)) diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py index 992b280296d0..ea3eb94faeba 100644 --- a/lib/python/qmk/keymap.py +++ b/lib/python/qmk/keymap.py @@ -1,5 +1,6 @@ """Functions that help you work with QMK keymaps. """ +from functools import lru_cache import json import sys from pathlib import Path @@ -306,6 +307,7 @@ def locate_keymap(keyboard, keymap): return community_layout / 'keymap.c' +@lru_cache() def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=False): """List the available keymaps for a keyboard. From 7fe506006e8669a06adbb415d415b418440bcca4 Mon Sep 17 00:00:00 2001 From: Zach White Date: Sat, 26 Jun 2021 16:53:36 -0700 Subject: [PATCH 03/14] fix Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7923a978cb16..269be720c2f9 100644 --- a/Makefile +++ b/Makefile @@ -398,7 +398,7 @@ define PARSE_KEYMAP # Specify the variables that we are passing forward to submake MAKE_VARS := KEYBOARD=$$(CURRENT_KB) KEYMAP=$$(CURRENT_KM) REQUIRE_PLATFORM_KEY=$$(REQUIRE_PLATFORM_KEY) QMK_BIN=$$(QMK_BIN) # And the first part of the make command - MAKE_CMD := echo $$(MAKE) -r -R -C $(ROOT_DIR) -f build_keyboard.mk $$(MAKE_TARGET) + MAKE_CMD := $$(MAKE) -r -R -C $(ROOT_DIR) -f build_keyboard.mk $$(MAKE_TARGET) # The message to display MAKE_MSG := $$(MSG_MAKE_KB) # We run the command differently, depending on if we want more output or not From ea862e24f6fca3ab94e4fa0f450b19989cfd88c4 Mon Sep 17 00:00:00 2001 From: Zach White Date: Sat, 26 Jun 2021 18:55:51 -0700 Subject: [PATCH 04/14] refactor the compile code into commands.py --- lib/python/qmk/cli/compile.py | 146 ++------------------------------ lib/python/qmk/commands.py | 153 ++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 140 deletions(-) diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 5ffae74ed1b8..0f44835fbfb2 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -9,61 +9,14 @@ from milc import cli import qmk.path -from qmk.decorators import automagic_keyboard, automagic_keymap, lru_cache -from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json -from qmk.info import info_json -from qmk.keyboard import keyboard_completer, is_keyboard_target, list_keyboards -from qmk.keymap import keymap_completer, list_keymaps -from qmk.metadata import true_values, false_values - - -@lru_cache() -def _keyboard_list(): - """Returns a list of keyboards matching cli.config.compile.keyboard. - """ - if cli.config.compile.keyboard == 'all': - return list_keyboards() - - elif cli.config.compile.keyboard.startswith('all-'): - return list_keyboards() - - return [cli.config.compile.keyboard] - - -def keyboard_keymap_iter(): - """Iterates over the keyboard/keymap for this command and yields a pairing of each. - """ - for keyboard in _keyboard_list(): - continue_flag = False - if cli.args.filter: - info_data = dotty(info_json(keyboard)) - for filter in cli.args.filter: - if '=' in filter: - key, value = filter.split('=', 1) - if value in true_values: - value = True - elif value in false_values: - value = False - elif value.isdigit(): - value = int(value) - elif '.' in value and value.replace('.').isdigit(): - value = float(value) - - if info_data.get(key) != value: - continue_flag = True - break - - if continue_flag: - continue - - if cli.config.compile.keymap == 'all': - for keymap in list_keymaps(keyboard): - yield keyboard, keymap - else: - yield keyboard, cli.config.compile.keymap +from qmk.decorators import automagic_keyboard, automagic_keymap +from qmk.commands import do_compile +from qmk.keyboard import keyboard_completer, is_keyboard_target +from qmk.keymap import keymap_completer @cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile') +@cli.argument('-t', '--target', help="The make target to run. By default it compiles the keyboard only.") @cli.argument('-kb', '--keyboard', type=is_keyboard_target, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.') @cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually build, just show the make command to be run.") @@ -81,91 +34,4 @@ def compile(cli): If a keyboard and keymap are provided this command will build a firmware based on that. """ - envs = {'REQUIRE_PLATFORM_KEY': ''} - if cli.config.compile.keyboard is None: - cli.config.compile.keyboard = '' - - silent = cli.config.compile.keyboard == 'all' or cli.config.compile.keyboard.startswith('all-') or cli.config.compile.keymap == 'all' - - # Setup the environment - for env in cli.args.env: - if '=' in env: - key, value = env.split('=', 1) - if key in envs: - cli.log.warning('Overwriting existing environment variable %s=%s with %s=%s!', key, envs[key], key, value) - envs[key] = value - else: - cli.log.warning('Invalid environment variable: %s', env) - - if cli.config.compile.keyboard.startswith('all-'): - envs['REQUIRE_PLATFORM_KEY'] = cli.config.compile.keyboard[4:] - - # Run clean if necessary - if cli.args.clean and not cli.args.filename and not cli.args.dry_run: - for keyboard, keymap in keyboard_keymap_iter(): - cli.log.info('Cleaning previous build files for keyboard {fg_cyan}%s{fg_reset} keymap {fg_cyan}%s', keyboard, keymap) - _, _, make_cmd = create_make_command(keyboard, keymap, 'clean', cli.config.compile.parallel, silent, **envs) - cli.run(make_cmd, capture_output=False, stdin=DEVNULL) - - # If -f has been specified without a keyboard target, assume -kb all - if cli.args.filter and not cli.args.keyboard: - cli.log.debug('--filter supplied without --keyboard, assuming --keyboard all.') - cli.config.compile.keyboard = 'all' - - # Determine the compile command(s) - commands = None - - if cli.args.filename: - # If a configurator JSON was provided generate a keymap and compile it - user_keymap = parse_configurator_json(cli.args.filename) - commands = [compile_configurator_json(user_keymap, parallel=cli.config.compile.parallel, **envs)] - - elif cli.config.compile.keyboard and cli.config.compile.keymap: - if cli.args.filter: - cli.log.info('Generating the list of keyboards to compile, this may take some time.') - - commands = [create_make_command(keyboard, keymap, parallel=cli.config.compile.parallel, silent=silent, **envs) for keyboard, keymap in keyboard_keymap_iter()] - - elif not cli.config.compile.keyboard: - cli.log.error('Could not determine keyboard!') - - elif not cli.config.compile.keymap: - cli.log.error('Could not determine keymap!') - - # Compile the firmware, if we're able to - if commands: - returncodes = [] - for keyboard, keymap, command in commands: - if keymap not in list_keymaps(keyboard): - cli.log.debug('Skipping keyboard %s, no %s keymap found.', keyboard, keymap) - continue - - print() - cli.log.info('Building firmware for {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s', keyboard, keymap) - cli.log.debug('Running make command: {fg_blue}%s', ' '.join(command)) - - if not cli.args.dry_run: - compile = cli.run(command, capture_output=False) - returncodes.append(compile.returncode) - if compile.returncode == 0: - cli.log.info('Success!') - else: - cli.log.error('Failed!') - - if any(returncodes): - print() - cli.log.error('Could not compile all targets, look above this message for more details. Failing target(s):') - - for i, returncode in enumerate(returncodes): - if returncode != 0: - keyboard, keymap, command = commands[i] - cli.echo('\tkeyboard: {fg_cyan}%s{fg_reset} keymap: {fg_cyan}%s', keyboard, keymap) - - elif cli.args.filter: - cli.log.error('No keyboards found after applying filter(s)!') - return False - - else: - cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') - cli.print_help() - return False + do_compile(cli.config.compile.keyboard, cli.config.compile.keymap, cli.config.compile.parallel, cli.config.compile.target) diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index c791b745339e..360ad314223e 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -1,5 +1,6 @@ """Helper functions for commands. """ +from functools import lru_cache import json import os import sys @@ -8,11 +9,15 @@ from subprocess import DEVNULL from time import strftime +from dotty_dict import dotty from milc import cli import qmk.keymap from qmk.constants import QMK_FIRMWARE, KEYBOARD_OUTPUT_PREFIX +from qmk.info import info_json from qmk.json_schema import json_load +from qmk.keyboard import list_keyboards +from qmk.metadata import true_values, false_values time_fmt = '%Y-%m-%d-%H:%M:%S' @@ -346,3 +351,151 @@ def in_virtualenv(): """ active_prefix = getattr(sys, "base_prefix", None) or getattr(sys, "real_prefix", None) or sys.prefix return active_prefix != sys.prefix + + +def do_compile(keyboard, keymap, parallel, target=None): + """Shared code between `qmk compile` and `qmk flash`. + """ + if keyboard is None: + keyboard = '' + + envs = {'REQUIRE_PLATFORM_KEY': ''} + silent = keyboard == 'all' or keyboard.startswith('all-') or keymap == 'all' + + # Setup the environment + for env in cli.args.env: + if '=' in env: + key, value = env.split('=', 1) + if key in envs: + cli.log.warning('Overwriting existing environment variable %s=%s with %s=%s!', key, envs[key], key, value) + envs[key] = value + else: + cli.log.warning('Invalid environment variable: %s', env) + + if keyboard.startswith('all-'): + envs['REQUIRE_PLATFORM_KEY'] = keyboard[4:] + + # Run clean if necessary + if cli.args.clean and not cli.args.filename and not cli.args.dry_run: + for keyboard, keymap in keyboard_keymap_iter(keyboard, keymap): + cli.log.info('Cleaning previous build files for keyboard {fg_cyan}%s{fg_reset} keymap {fg_cyan}%s', keyboard, keymap) + _, _, make_cmd = create_make_command(keyboard, keymap, 'clean', parallel, silent, **envs) + cli.run(make_cmd, capture_output=False, stdin=DEVNULL) + + # If -f has been specified without a keyboard target, assume -kb all + if cli.args.filter and not cli.args.keyboard: + cli.log.debug('--filter supplied without --keyboard, assuming --keyboard all.') + keyboard = 'all' + + # Determine the compile command(s) + commands = None + + if cli.args.filename: + if cli.args.filter: + cli.log.warning('Ignoring --filter because a keymap.json was provided.') + + if cli.args.keyboard: + cli.log.warning('Ignoring --keyboard because a keymap.json was provided.') + + if cli.args.keymap: + cli.log.warning('Ignoring --keymap because a keymap.json was provided.') + + # If a configurator JSON was provided generate a keymap and compile it + user_keymap = parse_configurator_json(cli.args.filename) + commands = [compile_configurator_json(user_keymap, parallel=parallel, **envs)] + + elif keyboard and keymap: + if cli.args.filter: + cli.log.info('Generating the list of keyboards to compile, this may take some time.') + + commands = [create_make_command(keyboard, keymap, target=target, parallel=parallel, silent=silent, **envs) for keyboard, keymap in keyboard_keymap_iter(keyboard, keymap)] + + elif not keyboard: + cli.log.error('Could not determine keyboard!') + + elif not keymap: + cli.log.error('Could not determine keymap!') + + # Compile the firmware, if we're able to + if commands: + returncodes = [] + for keyboard, keymap, command in commands: + if keymap not in qmk.keymap.list_keymaps(keyboard): + cli.log.debug('Skipping keyboard %s, no %s keymap found.', keyboard, keymap) + continue + + print() + if target: + cli.log.info('Building firmware for {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s{fg_reset} and target {fg_cyan}%s', keyboard, keymap, target) + else: + cli.log.info('Building firmware for {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s', keyboard, keymap) + cli.log.debug('Running make command: {fg_blue}%s', ' '.join(command)) + + if not cli.args.dry_run: + compile = cli.run(command, capture_output=False) + returncodes.append(compile.returncode) + if compile.returncode == 0: + cli.log.info('Success!') + else: + cli.log.error('Failed!') + + if any(returncodes): + print() + cli.log.error('Could not compile all targets, look above this message for more details. Failing target(s):') + + for i, returncode in enumerate(returncodes): + if returncode != 0: + keyboard, keymap, command = commands[i] + cli.echo('\tkeyboard: {fg_cyan}%s{fg_reset} keymap: {fg_cyan}%s', keyboard, keymap) + + elif cli.args.filter: + cli.log.error('No keyboards found after applying filter(s)!') + return False + + else: + cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') + cli.print_help() + return False + + +@lru_cache() +def _keyboard_list(keyboard): + """Returns a list of keyboards matching keyboard. + """ + if keyboard == 'all' or keyboard.startswith('all-'): + return list_keyboards() + + return [keyboard] + + +def keyboard_keymap_iter(cli_keyboard, cli_keymap): + """Iterates over the keyboard/keymap for this command and yields a pairing of each. + """ + for keyboard in _keyboard_list(cli_keyboard): + continue_flag = False + if cli.args.filter: + info_data = dotty(info_json(keyboard)) + for filter in cli.args.filter: + if '=' in filter: + key, value = filter.split('=', 1) + if value in true_values: + value = True + elif value in false_values: + value = False + elif value.isdigit(): + value = int(value) + elif '.' in value and value.replace('.').isdigit(): + value = float(value) + + if info_data.get(key) != value: + continue_flag = True + break + + if continue_flag: + continue + + if cli_keymap == 'all': + for keymap in qmk.keymap.list_keymaps(keyboard): + yield keyboard, keymap + else: + yield keyboard, cli_keymap From 4f20c94b970445331cf83992bd8a2a9f6a7454f2 Mon Sep 17 00:00:00 2001 From: Zach White Date: Sat, 26 Jun 2021 19:46:00 -0700 Subject: [PATCH 05/14] unify the compile and flash commands --- lib/python/qmk/cli/compile.py | 30 ++++++++++++++++++++- lib/python/qmk/cli/flash.py | 51 +++-------------------------------- lib/python/qmk/commands.py | 47 ++++++++++++-------------------- 3 files changed, 49 insertions(+), 79 deletions(-) diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index 0f44835fbfb2..f83952143c42 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -13,6 +13,7 @@ from qmk.commands import do_compile from qmk.keyboard import keyboard_completer, is_keyboard_target from qmk.keymap import keymap_completer +from qmk.metadata import true_values, false_values @cli.argument('filename', nargs='?', arg_only=True, type=qmk.path.FileType('r'), completer=FilesCompleter('.json'), help='The configurator export to compile') @@ -34,4 +35,31 @@ def compile(cli): If a keyboard and keymap are provided this command will build a firmware based on that. """ - do_compile(cli.config.compile.keyboard, cli.config.compile.keymap, cli.config.compile.parallel, cli.config.compile.target) + # If -f has been specified without a keyboard target, assume -kb all + keyboard = cli.config.compile.keyboard or '' + + if cli.args.filter and not cli.args.keyboard: + cli.log.debug('--filter supplied without --keyboard, assuming --keyboard all.') + keyboard = 'all' + + if cli.args.filename and cli.args.filter: + cli.log.warning('Ignoring --filter because a keymap.json was provided.') + + filters = {} + + for filter in cli.args.filter: + if '=' in filter: + key, value = filter.split('=', 1) + + if value in true_values: + value = True + elif value in false_values: + value = False + elif value.isdigit(): + value = int(value) + elif '.' in value and value.replace('.').isdigit(): + value = float(value) + + filters[key] = value + + return do_compile(keyboard, cli.config.compile.keymap, cli.config.compile.parallel, cli.config.compile.target, filters) diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py index aa3d74e63a50..a77642cbaef1 100644 --- a/lib/python/qmk/cli/flash.py +++ b/lib/python/qmk/cli/flash.py @@ -10,7 +10,7 @@ import qmk.path from qmk.decorators import automagic_keyboard, automagic_keymap -from qmk.commands import compile_configurator_json, create_make_command, parse_configurator_json +from qmk.commands import do_compile from qmk.keyboard import keyboard_completer, is_keyboard_target @@ -54,55 +54,10 @@ def flash(cli): If bootloader is omitted the make system will use the configured bootloader for that keyboard. """ - if cli.args.clean and not cli.args.filename and not cli.args.dry_run: - command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, 'clean') - cli.run(command, capture_output=False, stdin=DEVNULL) - - # Build the environment vars - envs = {} - for env in cli.args.env: - if '=' in env: - key, value = env.split('=', 1) - envs[key] = value - else: - cli.log.warning('Invalid environment variable: %s', env) - - # Determine the compile command - command = '' - if cli.args.bootloaders: # Provide usage and list bootloaders - cli.echo('usage: qmk flash [-h] [-b] [-n] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]') + cli.print_usage() print_bootloader_help() return False - if cli.args.filename: - # Handle compiling a configurator JSON - user_keymap = parse_configurator_json(cli.args.filename) - keymap_path = qmk.path.keymap(user_keymap['keyboard']) - command = compile_configurator_json(user_keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs) - - cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap']) - - else: - if cli.config.flash.keyboard and cli.config.flash.keymap: - # Generate the make command for a specific keyboard/keymap. - command = create_make_command(cli.config.flash.keyboard, cli.config.flash.keymap, cli.args.bootloader, parallel=cli.config.flash.parallel, **envs) - - elif not cli.config.flash.keyboard: - cli.log.error('Could not determine keyboard!') - elif not cli.config.flash.keymap: - cli.log.error('Could not determine keymap!') - - # Compile the firmware, if we're able to - if command: - cli.log.info('Compiling keymap with {fg_cyan}%s', ' '.join(command)) - if not cli.args.dry_run: - cli.echo('\n') - compile = cli.run(command, capture_output=False, stdin=DEVNULL) - return compile.returncode - - else: - cli.log.error('You must supply a configurator export, both `--keyboard` and `--keymap`, or be in a directory for a keyboard or keymap.') - cli.echo('usage: qmk flash [-h] [-b] [-n] [-kb KEYBOARD] [-km KEYMAP] [-bl BOOTLOADER] [filename]') - return False + return do_compile(cli.config.flash.keyboard, cli.config.flash.keymap, cli.config.flash.parallel, cli.config.flash.bootloader) \ No newline at end of file diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 360ad314223e..5363a4d4e943 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -353,17 +353,20 @@ def in_virtualenv(): return active_prefix != sys.prefix -def do_compile(keyboard, keymap, parallel, target=None): +def do_compile(keyboard, keymap, parallel, target=None, filters=None, environment=None): """Shared code between `qmk compile` and `qmk flash`. """ if keyboard is None: keyboard = '' + if environment is None: + environment = {} + envs = {'REQUIRE_PLATFORM_KEY': ''} silent = keyboard == 'all' or keyboard.startswith('all-') or keymap == 'all' # Setup the environment - for env in cli.args.env: + for env in environment: if '=' in env: key, value = env.split('=', 1) if key in envs: @@ -382,18 +385,10 @@ def do_compile(keyboard, keymap, parallel, target=None): _, _, make_cmd = create_make_command(keyboard, keymap, 'clean', parallel, silent, **envs) cli.run(make_cmd, capture_output=False, stdin=DEVNULL) - # If -f has been specified without a keyboard target, assume -kb all - if cli.args.filter and not cli.args.keyboard: - cli.log.debug('--filter supplied without --keyboard, assuming --keyboard all.') - keyboard = 'all' - # Determine the compile command(s) commands = None if cli.args.filename: - if cli.args.filter: - cli.log.warning('Ignoring --filter because a keymap.json was provided.') - if cli.args.keyboard: cli.log.warning('Ignoring --keyboard because a keymap.json was provided.') @@ -405,10 +400,10 @@ def do_compile(keyboard, keymap, parallel, target=None): commands = [compile_configurator_json(user_keymap, parallel=parallel, **envs)] elif keyboard and keymap: - if cli.args.filter: + if filters: cli.log.info('Generating the list of keyboards to compile, this may take some time.') - commands = [create_make_command(keyboard, keymap, target=target, parallel=parallel, silent=silent, **envs) for keyboard, keymap in keyboard_keymap_iter(keyboard, keymap)] + commands = [create_make_command(keyboard, keymap, target=target, parallel=parallel, silent=silent, **envs) for keyboard, keymap in keyboard_keymap_iter(keyboard, keymap, filters)] elif not keyboard: cli.log.error('Could not determine keyboard!') @@ -448,7 +443,7 @@ def do_compile(keyboard, keymap, parallel, target=None): keyboard, keymap, command = commands[i] cli.echo('\tkeyboard: {fg_cyan}%s{fg_reset} keymap: {fg_cyan}%s', keyboard, keymap) - elif cli.args.filter: + elif filters: cli.log.error('No keyboards found after applying filter(s)!') return False @@ -468,28 +463,19 @@ def _keyboard_list(keyboard): return [keyboard] -def keyboard_keymap_iter(cli_keyboard, cli_keymap): +def keyboard_keymap_iter(cli_keyboard, cli_keymap, filters): """Iterates over the keyboard/keymap for this command and yields a pairing of each. """ for keyboard in _keyboard_list(cli_keyboard): continue_flag = False - if cli.args.filter: + + if filters: info_data = dotty(info_json(keyboard)) - for filter in cli.args.filter: - if '=' in filter: - key, value = filter.split('=', 1) - if value in true_values: - value = True - elif value in false_values: - value = False - elif value.isdigit(): - value = int(value) - elif '.' in value and value.replace('.').isdigit(): - value = float(value) - - if info_data.get(key) != value: - continue_flag = True - break + + for key, value in filters.items(): + if info_data.get(key) != value: + continue_flag = True + break if continue_flag: continue @@ -497,5 +483,6 @@ def keyboard_keymap_iter(cli_keyboard, cli_keymap): if cli_keymap == 'all': for keymap in qmk.keymap.list_keymaps(keyboard): yield keyboard, keymap + else: yield keyboard, cli_keymap From 07b8035ba903483cb7e76b3a8fa24f5a36cdeb6e Mon Sep 17 00:00:00 2001 From: Zach White Date: Sat, 26 Jun 2021 20:22:49 -0700 Subject: [PATCH 06/14] do some optimizing --- lib/python/qmk/info.py | 1 + lib/python/qmk/metadata.py | 32 ++++++++++++++++++++++++-------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index d868e9438bf1..18a34dd94eb9 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -12,6 +12,7 @@ from qmk.metadata import basic_info_json, info_log_error +@lru_cache(maxsize=None) def _valid_community_layout(layout): """Validate that a declared community list exists """ diff --git a/lib/python/qmk/metadata.py b/lib/python/qmk/metadata.py index 4c0c8ce05fee..b79625b434cd 100644 --- a/lib/python/qmk/metadata.py +++ b/lib/python/qmk/metadata.py @@ -97,6 +97,7 @@ def _extract_features(info_data, rules): return info_data +@lru_cache(maxsize=None) def _pin_name(pin): """Returns the proper representation for a pin. """ @@ -114,6 +115,7 @@ def _pin_name(pin): return pin +@lru_cache(maxsize=None) def _extract_pins(pins): """Returns a list of pins from a comma separated string of pins. """ @@ -306,6 +308,7 @@ def _extract_rules_mk(info_data): return info_data +@lru_cache(maxsize=None) def _search_keyboard_h(path): current_path = Path('keyboards/') aliases = {} @@ -334,17 +337,29 @@ def _find_all_layouts(info_data, keyboard): if not layouts: # If we don't find any layouts from info.json or keyboard.h we widen our search. This is error prone which is why we want to encourage people to follow the standard above. info_data['parse_warnings'].append('%s: Falling back to searching for KEYMAP/LAYOUT macros.' % (keyboard)) + layouts, new_aliases = _deep_search_layouts(keyboard) + aliases.update(new_aliases) + + return layouts, aliases + + +@lru_cache(maxsize=None) +def _deep_search_layouts(keyboard): + """Do a wider (error-prone) search for layout macros. + """ + layouts = {} + aliases = {} - for file in glob('keyboards/%s/*.h' % keyboard): - if file.endswith('.h'): - these_layouts, these_aliases = find_layouts(file) + for file in glob('keyboards/%s/*.h' % keyboard): + if file.endswith('.h'): + these_layouts, these_aliases = find_layouts(file) - if these_layouts: - layouts.update(these_layouts) + if these_layouts: + layouts.update(these_layouts) - for alias, alias_text in these_aliases.items(): - if alias_text in layouts: - aliases[alias] = alias_text + for alias, alias_text in these_aliases.items(): + if alias_text in layouts: + aliases[alias] = alias_text return layouts, aliases @@ -458,6 +473,7 @@ def merge_info_jsons(keyboard, info_data): return info_data +@lru_cache(maxsize=None) def find_info_json(keyboard): """Finds all the info.json files associated with a keyboard. """ From b4e18c9019e536d15018cd9b911ccda410698803 Mon Sep 17 00:00:00 2001 From: Zach White Date: Sat, 26 Jun 2021 23:33:13 -0700 Subject: [PATCH 07/14] Track mtimes for info.json files This allows us to skip validation when the file has not been changed since the last time it was validated. --- lib/python/qmk/metadata.py | 50 ++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/lib/python/qmk/metadata.py b/lib/python/qmk/metadata.py index b79625b434cd..cf269b48e58b 100644 --- a/lib/python/qmk/metadata.py +++ b/lib/python/qmk/metadata.py @@ -2,6 +2,7 @@ """ from functools import lru_cache from glob import glob +import os from pathlib import Path import jsonschema @@ -23,7 +24,7 @@ def basic_info_json(keyboard): """Generate a subset of info.json for a specific keyboard. - This does no validation, and should only be used as needed to avoid loops or when performance is critical. + This does minimal validation, and should only be used as needed to avoid loops or when performance is critical. """ cur_dir = Path('keyboards') rules = parse_rules_mk_file(cur_dir / keyboard / 'rules.mk') @@ -428,6 +429,36 @@ def unknown_processor_rules(info_data, rules): return info_data +def store_mtime(file): + """Stores the mtime for a json file. + """ + cli.log.debug('store_mtime(%s)', file) + mtime = str(os.stat(file).st_mtime) + cache_file = f'.build/json_times/{file}' + cache_dir = os.path.dirname(cache_file) + os.makedirs(cache_dir, exist_ok=True) + with open(cache_file, 'w') as fd: + fd.write(mtime) + + +def has_been_validated(file): + """Returns True if file is in the json cache. + """ + cli.log.debug('has_been_validated(%s)', file) + mtime_file = f'.build/json_times/{file}' + if os.path.exists(mtime_file): + with open(mtime_file) as fd: + cache_mtime = fd.read() + cache_mtime = cache_mtime.strip() + file_mtime = str(os.stat(file).st_mtime) + if cache_mtime == file_mtime: + return True + else: + os.remove(mtime_file) + + return False + + def merge_info_jsons(keyboard, info_data): """Return a merged copy of all the info.json files for a keyboard. """ @@ -439,13 +470,16 @@ def merge_info_jsons(keyboard, info_data): info_log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),)) continue - try: - validate(new_info_data, 'qmk.keyboard.v1') - except jsonschema.ValidationError as e: - json_path = '.'.join([str(p) for p in e.absolute_path]) - cli.log.error('Not including data from file: %s', info_file) - cli.log.error('\t%s: %s', json_path, e.message) - continue + if not has_been_validated(info_file): + try: + validate(new_info_data, 'qmk.keyboard.v1') + except jsonschema.ValidationError as e: + json_path = '.'.join([str(p) for p in e.absolute_path]) + cli.log.error('Not including data from file: %s', info_file) + cli.log.error('\t%s: %s', json_path, e.message) + continue + + store_mtime(info_file) # Merge layout data in if 'layout_aliases' in new_info_data: From 08b0ecb175a1246336645dff364a2ae713e9689e Mon Sep 17 00:00:00 2001 From: Zach White Date: Sun, 27 Jun 2021 00:11:51 -0700 Subject: [PATCH 08/14] compile matching boards as we find them, not after building the whole list --- lib/python/qmk/commands.py | 76 +++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 5363a4d4e943..7bad94bac090 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -103,7 +103,7 @@ def create_make_command(keyboard, keymap, target=None, parallel=1, silent=False, for key, value in env_vars.items(): make_cmd.append(f'{key}={value}') - return keyboard, keymap, make_cmd + return make_cmd def get_git_version(current_time, repo_dir='.', check_dir='.'): @@ -362,10 +362,13 @@ def do_compile(keyboard, keymap, parallel, target=None, filters=None, environmen if environment is None: environment = {} - envs = {'REQUIRE_PLATFORM_KEY': ''} - silent = keyboard == 'all' or keyboard.startswith('all-') or keymap == 'all' + all_keyboards = keyboard == 'all' or keyboard.startswith('all-') + all_keymaps = keymap == 'all' + multiple_compiles = all_keyboards or all_keymaps # Setup the environment + envs = {'REQUIRE_PLATFORM_KEY': ''} + for env in environment: if '=' in env: key, value = env.split('=', 1) @@ -382,11 +385,11 @@ def do_compile(keyboard, keymap, parallel, target=None, filters=None, environmen if cli.args.clean and not cli.args.filename and not cli.args.dry_run: for keyboard, keymap in keyboard_keymap_iter(keyboard, keymap): cli.log.info('Cleaning previous build files for keyboard {fg_cyan}%s{fg_reset} keymap {fg_cyan}%s', keyboard, keymap) - _, _, make_cmd = create_make_command(keyboard, keymap, 'clean', parallel, silent, **envs) + _, _, make_cmd = create_make_command(keyboard, keymap, 'clean', parallel, multiple_compiles, **envs) cli.run(make_cmd, capture_output=False, stdin=DEVNULL) # Determine the compile command(s) - commands = None + command = None if cli.args.filename: if cli.args.keyboard: @@ -397,13 +400,13 @@ def do_compile(keyboard, keymap, parallel, target=None, filters=None, environmen # If a configurator JSON was provided generate a keymap and compile it user_keymap = parse_configurator_json(cli.args.filename) - commands = [compile_configurator_json(user_keymap, parallel=parallel, **envs)] + command = compile_configurator_json(user_keymap, parallel=parallel, **envs) elif keyboard and keymap: - if filters: - cli.log.info('Generating the list of keyboards to compile, this may take some time.') - - commands = [create_make_command(keyboard, keymap, target=target, parallel=parallel, silent=silent, **envs) for keyboard, keymap in keyboard_keymap_iter(keyboard, keymap, filters)] + if multiple_compiles: + command = 'multiple' + else: + command = create_make_command(keyboard, keymap, target=target, parallel=parallel, silent=multiple_compiles, **envs) elif not keyboard: cli.log.error('Could not determine keyboard!') @@ -412,27 +415,11 @@ def do_compile(keyboard, keymap, parallel, target=None, filters=None, environmen cli.log.error('Could not determine keymap!') # Compile the firmware, if we're able to - if commands: + if command == 'multiple': returncodes = [] - for keyboard, keymap, command in commands: - if keymap not in qmk.keymap.list_keymaps(keyboard): - cli.log.debug('Skipping keyboard %s, no %s keymap found.', keyboard, keymap) - continue - - print() - if target: - cli.log.info('Building firmware for {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s{fg_reset} and target {fg_cyan}%s', keyboard, keymap, target) - else: - cli.log.info('Building firmware for {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s', keyboard, keymap) - cli.log.debug('Running make command: {fg_blue}%s', ' '.join(command)) - - if not cli.args.dry_run: - compile = cli.run(command, capture_output=False) - returncodes.append(compile.returncode) - if compile.returncode == 0: - cli.log.info('Success!') - else: - cli.log.error('Failed!') + for keyboard, keymap in keyboard_keymap_iter(keyboard, keymap, filters): + command = create_make_command(keyboard, keymap, target=target, parallel=parallel, silent=multiple_compiles, **envs) + _execute_compile(keyboard, keymap, command, target) if any(returncodes): print() @@ -443,6 +430,12 @@ def do_compile(keyboard, keymap, parallel, target=None, filters=None, environmen keyboard, keymap, command = commands[i] cli.echo('\tkeyboard: {fg_cyan}%s{fg_reset} keymap: {fg_cyan}%s', keyboard, keymap) + elif command: + if _execute_compile(keyboard, keymap, command, target) != 0: + print() + cli.log.error('Could not compile all targets, look above this message for more details. Failing target(s):') + cli.echo('\tkeyboard: {fg_cyan}%s{fg_reset} keymap: {fg_cyan}%s', keyboard, keymap) + elif filters: cli.log.error('No keyboards found after applying filter(s)!') return False @@ -453,6 +446,29 @@ def do_compile(keyboard, keymap, parallel, target=None, filters=None, environmen return False +def _execute_compile(keyboard, keymap, command, target): + if keymap not in qmk.keymap.list_keymaps(keyboard): + cli.log.debug('Skipping keyboard %s, no %s keymap found.', keyboard, keymap) + return 0 + + print() + if target: + cli.log.info('Building firmware for {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s{fg_reset} and target {fg_cyan}%s', keyboard, keymap, target) + else: + cli.log.info('Building firmware for {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s', keyboard, keymap) + cli.log.debug('Running make command: {fg_blue}%s', ' '.join(command)) + + if not cli.args.dry_run: + compile = cli.run(command, capture_output=False) + if compile.returncode == 0: + cli.log.info('Success!') + else: + cli.log.error('Failed!') + return compile.returncode + + return 0 + + @lru_cache() def _keyboard_list(keyboard): """Returns a list of keyboards matching keyboard. From 823a74ebae2858a21943950fe715ad289d8f42e7 Mon Sep 17 00:00:00 2001 From: Zach White Date: Sun, 27 Jun 2021 11:55:57 -0700 Subject: [PATCH 09/14] add support for building multiple keyboards in parallel --- lib/python/qmk/commands.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 7bad94bac090..bff4c2437e78 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -5,9 +5,10 @@ import os import sys import shutil +import threading from pathlib import Path from subprocess import DEVNULL -from time import strftime +from time import sleep, strftime from dotty_dict import dotty from milc import cli @@ -17,7 +18,6 @@ from qmk.info import info_json from qmk.json_schema import json_load from qmk.keyboard import list_keyboards -from qmk.metadata import true_values, false_values time_fmt = '%Y-%m-%d-%H:%M:%S' @@ -383,9 +383,9 @@ def do_compile(keyboard, keymap, parallel, target=None, filters=None, environmen # Run clean if necessary if cli.args.clean and not cli.args.filename and not cli.args.dry_run: - for keyboard, keymap in keyboard_keymap_iter(keyboard, keymap): - cli.log.info('Cleaning previous build files for keyboard {fg_cyan}%s{fg_reset} keymap {fg_cyan}%s', keyboard, keymap) - _, _, make_cmd = create_make_command(keyboard, keymap, 'clean', parallel, multiple_compiles, **envs) + for kb, km in keyboard_keymap_iter(keyboard, keymap, {}): + cli.log.info('Cleaning previous build files for keyboard {fg_cyan}%s{fg_reset} keymap {fg_cyan}%s', kb, km) + make_cmd = create_make_command(kb, km, 'clean', 1, multiple_compiles, **envs) cli.run(make_cmd, capture_output=False, stdin=DEVNULL) # Determine the compile command(s) @@ -418,8 +418,13 @@ def do_compile(keyboard, keymap, parallel, target=None, filters=None, environmen if command == 'multiple': returncodes = [] for keyboard, keymap in keyboard_keymap_iter(keyboard, keymap, filters): - command = create_make_command(keyboard, keymap, target=target, parallel=parallel, silent=multiple_compiles, **envs) - _execute_compile(keyboard, keymap, command, target) + command = create_make_command(keyboard, keymap, target=target, parallel=1, silent=multiple_compiles, **envs) + while threading.active_count() >= parallel + 1: + sleep(1) + threading.Thread(target=_execute_compile, args=(keyboard, keymap, command, target, returncodes)).start() + + while threading.active_count() > 1: + sleep(1) if any(returncodes): print() @@ -446,7 +451,10 @@ def do_compile(keyboard, keymap, parallel, target=None, filters=None, environmen return False -def _execute_compile(keyboard, keymap, command, target): +def _execute_compile(keyboard, keymap, command, target, returncodes=None): + if not returncodes: + returncodes = [] + if keymap not in qmk.keymap.list_keymaps(keyboard): cli.log.debug('Skipping keyboard %s, no %s keymap found.', keyboard, keymap) return 0 @@ -460,13 +468,15 @@ def _execute_compile(keyboard, keymap, command, target): if not cli.args.dry_run: compile = cli.run(command, capture_output=False) + + cli.acquire_lock() + returncodes.append(compile.returncode) + cli.release_lock() + if compile.returncode == 0: cli.log.info('Success!') else: cli.log.error('Failed!') - return compile.returncode - - return 0 @lru_cache() From dcbfdb5cfcd22653bb9aa146d6c64e92f2554bb6 Mon Sep 17 00:00:00 2001 From: Zach White Date: Sun, 27 Jun 2021 14:38:52 -0700 Subject: [PATCH 10/14] lru_cache everywhere --- lib/python/qmk/c_parse.py | 9 ++++++++- lib/python/qmk/commands.py | 3 +++ lib/python/qmk/comment_remover.py | 2 ++ lib/python/qmk/converter.py | 2 ++ lib/python/qmk/json_schema.py | 1 + lib/python/qmk/keyboard.py | 6 ++++-- lib/python/qmk/keymap.py | 13 +++++++++++-- lib/python/qmk/makefile.py | 2 ++ lib/python/qmk/math.py | 15 +++++++++++++-- lib/python/qmk/path.py | 6 ++++++ lib/python/qmk/submodules.py | 2 ++ 11 files changed, 54 insertions(+), 7 deletions(-) diff --git a/lib/python/qmk/c_parse.py b/lib/python/qmk/c_parse.py index 991373d5693a..d6582ac67030 100644 --- a/lib/python/qmk/c_parse.py +++ b/lib/python/qmk/c_parse.py @@ -1,7 +1,8 @@ """Functions for working with config.h files. """ -from pathlib import Path import re +from functools import lru_cache +from pathlib import Path from milc import cli @@ -12,18 +13,21 @@ multi_comment_regex = re.compile(r'/\*(.|\n)*?\*/', re.MULTILINE) +@lru_cache(maxsize=0) def strip_line_comment(string): """Removes comments from a single line string. """ return single_comment_regex.sub('', string) +@lru_cache(maxsize=0) def strip_multiline_comment(string): """Removes comments from a single line string. """ return multi_comment_regex.sub('', string) +@lru_cache(maxsize=0) def c_source_files(dir_names): """Returns a list of all *.c, *.h, and *.cpp files for a given list of directories @@ -38,6 +42,7 @@ def c_source_files(dir_names): return files +@lru_cache(maxsize=0) def find_layouts(file): """Returns list of parsed LAYOUT preprocessor macros found in the supplied include file. """ @@ -144,6 +149,7 @@ def _default_key(label=None): return new_key +@lru_cache(maxsize=0) def _parse_layout_macro(layout_macro): """Split the LAYOUT macro into its constituent parts """ @@ -154,6 +160,7 @@ def _parse_layout_macro(layout_macro): return macro_name, layout, matrix +@lru_cache(maxsize=0) def _parse_matrix_locations(matrix, file, macro_name): """Parse raw matrix data into a dictionary keyed by the LAYOUT identifier. """ diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index bff4c2437e78..7a067a7da61b 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -22,6 +22,7 @@ time_fmt = '%Y-%m-%d-%H:%M:%S' +@lru_cache(maxsize=0) def _find_make(): """Returns the correct make command for this environment. """ @@ -106,6 +107,7 @@ def create_make_command(keyboard, keymap, target=None, parallel=1, silent=False, return make_cmd +@lru_cache(maxsize=0) def get_git_version(current_time, repo_dir='.', check_dir='.'): """Returns the current git version for a repo, or the current time. """ @@ -258,6 +260,7 @@ def compile_configurator_json(user_keymap, bootloader=None, parallel=1, **env_va return user_keymap['keyboard'], user_keymap['keymap'], make_command +@lru_cache(maxsize=0) def parse_configurator_json(configurator_file): """Open and parse a configurator json export """ diff --git a/lib/python/qmk/comment_remover.py b/lib/python/qmk/comment_remover.py index 45a25257f8f5..e2edfcc667ef 100644 --- a/lib/python/qmk/comment_remover.py +++ b/lib/python/qmk/comment_remover.py @@ -3,6 +3,7 @@ Gratefully adapted from https://stackoverflow.com/a/241506 """ import re +from functools import lru_cache comment_pattern = re.compile(r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', re.DOTALL | re.MULTILINE) @@ -14,6 +15,7 @@ def _comment_stripper(match): return ' ' if s.startswith('/') else s +@lru_cache(maxsize=0) def comment_remover(text): """Remove C/C++ style comments from text. """ diff --git a/lib/python/qmk/converter.py b/lib/python/qmk/converter.py index bbd3531317d7..0dfbdf068b97 100644 --- a/lib/python/qmk/converter.py +++ b/lib/python/qmk/converter.py @@ -1,8 +1,10 @@ """Functions to convert to and from QMK formats """ from collections import OrderedDict +from functools import lru_cache +@lru_cache(maxsize=0) def kle2qmk(kle): """Convert a KLE layout to QMK's layout format. """ diff --git a/lib/python/qmk/json_schema.py b/lib/python/qmk/json_schema.py index ffc7c6bcd127..3db679cd2831 100644 --- a/lib/python/qmk/json_schema.py +++ b/lib/python/qmk/json_schema.py @@ -10,6 +10,7 @@ from milc import cli +@lru_cache(maxsize=0) def json_load(json_file): """Load a json file from disk. diff --git a/lib/python/qmk/keyboard.py b/lib/python/qmk/keyboard.py index 77fd9b92d599..d5598013c0af 100644 --- a/lib/python/qmk/keyboard.py +++ b/lib/python/qmk/keyboard.py @@ -1,10 +1,11 @@ """Functions that help us work with keyboards. """ +import os from array import array +from functools import lru_cache +from glob import glob from math import ceil from pathlib import Path -import os -from glob import glob import qmk.path from qmk.c_parse import parse_config_h_file @@ -217,6 +218,7 @@ def render_layout(layout_data, render_ascii, key_labels=None): return '\n'.join(lines) +@lru_cache(maxsize=0) def render_layouts(info_json, render_ascii): """Renders all the layouts from an `info_json` structure. """ diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py index ea3eb94faeba..b6a94536b60b 100644 --- a/lib/python/qmk/keymap.py +++ b/lib/python/qmk/keymap.py @@ -1,8 +1,8 @@ """Functions that help you work with QMK keymaps. """ -from functools import lru_cache import json import sys +from functools import lru_cache from pathlib import Path from subprocess import DEVNULL @@ -32,6 +32,7 @@ """ +@lru_cache(maxsize=0) def template_json(keyboard): """Returns a `keymap.json` template for a keyboard. @@ -49,6 +50,7 @@ def template_json(keyboard): return template +@lru_cache(maxsize=0) def template_c(keyboard): """Returns a `keymap.c` template for a keyboard. @@ -124,6 +126,7 @@ def keymap_completer(prefix, action, parser, parsed_args): return [] +@lru_cache(maxsize=0) def is_keymap_dir(keymap, c=True, json=True, additional_files=None): """Return True if Path object `keymap` has a keymap file inside. @@ -182,6 +185,7 @@ def generate_json(keymap, keyboard, layout, layers): return new_keymap +@lru_cache(maxsize=0) def generate_c(keyboard, layout, layers): """Returns a `keymap.c` or `keymap.json` for the specified keyboard, layout, and layers. @@ -268,6 +272,7 @@ def write(keyboard, keymap, layout, layers): return write_file(keymap_file, keymap_content) +@lru_cache(maxsize=0) def locate_keymap(keyboard, keymap): """Returns the path to a keymap for a specific keyboard. """ @@ -307,7 +312,7 @@ def locate_keymap(keyboard, keymap): return community_layout / 'keymap.c' -@lru_cache() +@lru_cache(maxsize=0) def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=False): """List the available keymaps for a keyboard. @@ -361,6 +366,7 @@ def list_keymaps(keyboard, c=True, json=True, additional_files=None, fullpath=Fa return sorted(names) +@lru_cache(maxsize=0) def _c_preprocess(path, stdin=DEVNULL): """ Run a file through the C pre-processor @@ -380,6 +386,7 @@ def _c_preprocess(path, stdin=DEVNULL): return pre_processed_keymap.stdout +@lru_cache(maxsize=0) def _get_layers(keymap): # noqa C901 : until someone has a good idea how to simplify/split up this code """ Find the layers in a keymap.c file. @@ -500,6 +507,7 @@ def _get_layers(keymap): # noqa C901 : until someone has a good idea how to sim return layers +@lru_cache(maxsize=0) def parse_keymap_c(keymap_file, use_cpp=True): """ Parse a keymap.c file. @@ -529,6 +537,7 @@ def parse_keymap_c(keymap_file, use_cpp=True): return keymap +@lru_cache(maxsize=0) def c2json(keyboard, keymap, keymap_file, use_cpp=True): """ Convert keymap.c to keymap.json diff --git a/lib/python/qmk/makefile.py b/lib/python/qmk/makefile.py index 02c2e70050c3..e2c051466790 100644 --- a/lib/python/qmk/makefile.py +++ b/lib/python/qmk/makefile.py @@ -1,8 +1,10 @@ """ Functions for working with Makefiles """ +from functools import lru_cache from pathlib import Path +@lru_cache(maxsize=0) def parse_rules_mk_file(file, rules_mk=None): """Turn a rules.mk file into a dictionary. diff --git a/lib/python/qmk/math.py b/lib/python/qmk/math.py index 88dc4a300c87..ec967f0807f3 100644 --- a/lib/python/qmk/math.py +++ b/lib/python/qmk/math.py @@ -3,12 +3,22 @@ Gratefully copied from https://stackoverflow.com/a/9558001 """ import ast -import operator as op +import operator +from functools import lru_cache # supported operators -operators = {ast.Add: op.add, ast.Sub: op.sub, ast.Mult: op.mul, ast.Div: op.truediv, ast.Pow: op.pow, ast.BitXor: op.xor, ast.USub: op.neg} +operators = { + ast.Add: operator.add, + ast.Sub: operator.sub, + ast.Mult: operator.mul, + ast.Div: operator.truediv, + ast.Pow: operator.pow, + ast.BitXor: operator.xor, + ast.USub: operator.neg, +} +@lru_cache(maxsize=0) def compute(expr): """Parse a mathematical expression and return the answer. @@ -22,6 +32,7 @@ def compute(expr): return _eval(ast.parse(expr, mode='eval').body) +@lru_cache(maxsize=0) def _eval(node): if isinstance(node, ast.Num): # return node.n diff --git a/lib/python/qmk/path.py b/lib/python/qmk/path.py index 72bae5927341..4425810867ce 100644 --- a/lib/python/qmk/path.py +++ b/lib/python/qmk/path.py @@ -3,12 +3,14 @@ import logging import os import argparse +from functools import lru_cache from pathlib import Path from qmk.constants import MAX_KEYBOARD_SUBFOLDERS, QMK_FIRMWARE from qmk.errors import NoSuchKeyboardError +@lru_cache(maxsize=0) def is_keyboard(keyboard_name): """Returns True if `keyboard_name` is a keyboard we can compile. """ @@ -19,6 +21,7 @@ def is_keyboard(keyboard_name): return rules_mk.exists() +@lru_cache(maxsize=0) def under_qmk_firmware(): """Returns a Path object representing the relative path under qmk_firmware, or None. """ @@ -30,12 +33,14 @@ def under_qmk_firmware(): return None +@lru_cache(maxsize=0) def keyboard(keyboard_name): """Returns the path to a keyboard's directory relative to the qmk root. """ return Path('keyboards') / keyboard_name +@lru_cache(maxsize=0) def keymap(keyboard_name): """Locate the correct directory for storing a keymap. @@ -56,6 +61,7 @@ def keymap(keyboard_name): raise NoSuchKeyboardError('Could not find keymaps directory for: %s' % keyboard_name) +@lru_cache(maxsize=0) def normpath(path): """Returns a `pathlib.Path()` object for a given path. diff --git a/lib/python/qmk/submodules.py b/lib/python/qmk/submodules.py index 6a272dae5025..09acd68161b2 100644 --- a/lib/python/qmk/submodules.py +++ b/lib/python/qmk/submodules.py @@ -1,8 +1,10 @@ """Functions for working with QMK's submodules. """ +from functools import lru_cache from milc import cli +@lru_cache(maxsize=0) def status(): """Returns a dictionary of submodules. From 335dd3c5c313009d5fdab04b69ef048dceaedf01 Mon Sep 17 00:00:00 2001 From: Zach White Date: Sun, 27 Jun 2021 15:59:54 -0700 Subject: [PATCH 11/14] ensure parallel is string --- lib/python/qmk/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 7a067a7da61b..495854c588d4 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -96,7 +96,7 @@ def create_make_command(keyboard, keymap, target=None, parallel=1, silent=False, if parallel > 1: make_cmd.append('-j') - make_cmd.append(parallel) + make_cmd.append(str(parallel)) if target: make_cmd.append(target) From 4fadb98a0270c9999abc1c9c029670bd61119d1b Mon Sep 17 00:00:00 2001 From: Zach White Date: Sun, 27 Jun 2021 17:04:24 -0700 Subject: [PATCH 12/14] cleanup output --- lib/python/qmk/commands.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 495854c588d4..a351f2842ec0 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -462,7 +462,6 @@ def _execute_compile(keyboard, keymap, command, target, returncodes=None): cli.log.debug('Skipping keyboard %s, no %s keymap found.', keyboard, keymap) return 0 - print() if target: cli.log.info('Building firmware for {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s{fg_reset} and target {fg_cyan}%s', keyboard, keymap, target) else: @@ -470,16 +469,15 @@ def _execute_compile(keyboard, keymap, command, target, returncodes=None): cli.log.debug('Running make command: {fg_blue}%s', ' '.join(command)) if not cli.args.dry_run: - compile = cli.run(command, capture_output=False) + compile = cli.run(command, combined_output=True) cli.acquire_lock() returncodes.append(compile.returncode) cli.release_lock() - if compile.returncode == 0: - cli.log.info('Success!') - else: - cli.log.error('Failed!') + if compile.returncode != 0: + cli.log.info('Could not build firmware for {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s', keyboard, keymap) + print(compile.stdout) @lru_cache() From 6f4742bde6582b7d978d6d0711c6bd9bddaefb7f Mon Sep 17 00:00:00 2001 From: Zach White Date: Sun, 27 Jun 2021 19:09:17 -0700 Subject: [PATCH 13/14] log output tweaks --- lib/python/qmk/commands.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index a351f2842ec0..08b95147180c 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -419,6 +419,8 @@ def do_compile(keyboard, keymap, parallel, target=None, filters=None, environmen # Compile the firmware, if we're able to if command == 'multiple': + cli.log.info('Building {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s', keyboard, keymap) + returncodes = [] for keyboard, keymap in keyboard_keymap_iter(keyboard, keymap, filters): command = create_make_command(keyboard, keymap, target=target, parallel=1, silent=multiple_compiles, **envs) @@ -439,6 +441,11 @@ def do_compile(keyboard, keymap, parallel, target=None, filters=None, environmen cli.echo('\tkeyboard: {fg_cyan}%s{fg_reset} keymap: {fg_cyan}%s', keyboard, keymap) elif command: + if target: + cli.log.info('Building {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s{fg_reset} and target {fg_cyan}%s', keyboard, keymap, target) + else: + cli.log.info('Building {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s', keyboard, keymap) + if _execute_compile(keyboard, keymap, command, target) != 0: print() cli.log.error('Could not compile all targets, look above this message for more details. Failing target(s):') @@ -462,10 +469,6 @@ def _execute_compile(keyboard, keymap, command, target, returncodes=None): cli.log.debug('Skipping keyboard %s, no %s keymap found.', keyboard, keymap) return 0 - if target: - cli.log.info('Building firmware for {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s{fg_reset} and target {fg_cyan}%s', keyboard, keymap, target) - else: - cli.log.info('Building firmware for {fg_cyan}%s{fg_reset} with keymap {fg_cyan}%s', keyboard, keymap) cli.log.debug('Running make command: {fg_blue}%s', ' '.join(command)) if not cli.args.dry_run: From 81b17125ebb6128df22fc55d008bd575f2e69950 Mon Sep 17 00:00:00 2001 From: Zach White Date: Thu, 9 Sep 2021 08:33:41 -0700 Subject: [PATCH 14/14] fix after rebase --- lib/python/qmk/cli/compile.py | 3 -- lib/python/qmk/cli/flash.py | 4 +-- lib/python/qmk/commands.py | 2 +- lib/python/qmk/info.py | 57 ++++++++++++++++++++--------------- lib/python/qmk/json_schema.py | 2 ++ 5 files changed, 37 insertions(+), 31 deletions(-) diff --git a/lib/python/qmk/cli/compile.py b/lib/python/qmk/cli/compile.py index f83952143c42..26bdf230d374 100755 --- a/lib/python/qmk/cli/compile.py +++ b/lib/python/qmk/cli/compile.py @@ -2,10 +2,7 @@ You can compile a keymap already in the repo or using a QMK Configurator export. """ -from subprocess import DEVNULL - from argcomplete.completers import FilesCompleter -from dotty_dict import dotty from milc import cli import qmk.path diff --git a/lib/python/qmk/cli/flash.py b/lib/python/qmk/cli/flash.py index a77642cbaef1..17eda7df8356 100644 --- a/lib/python/qmk/cli/flash.py +++ b/lib/python/qmk/cli/flash.py @@ -3,8 +3,6 @@ You can compile a keymap already in the repo or using a QMK Configurator export. A bootloader must be specified. """ -from subprocess import DEVNULL - from argcomplete.completers import FilesCompleter from milc import cli @@ -60,4 +58,4 @@ def flash(cli): print_bootloader_help() return False - return do_compile(cli.config.flash.keyboard, cli.config.flash.keymap, cli.config.flash.parallel, cli.config.flash.bootloader) \ No newline at end of file + return do_compile(cli.config.flash.keyboard, cli.config.flash.keymap, cli.config.flash.parallel, cli.config.flash.bootloader) diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py index 08b95147180c..93aeb45307e7 100644 --- a/lib/python/qmk/commands.py +++ b/lib/python/qmk/commands.py @@ -437,7 +437,7 @@ def do_compile(keyboard, keymap, parallel, target=None, filters=None, environmen for i, returncode in enumerate(returncodes): if returncode != 0: - keyboard, keymap, command = commands[i] + keyboard, keymap, command = returncodes[i] cli.echo('\tkeyboard: {fg_cyan}%s{fg_reset} keymap: {fg_cyan}%s', keyboard, keymap) elif command: diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py index 18a34dd94eb9..4824dfdf670b 100644 --- a/lib/python/qmk/info.py +++ b/lib/python/qmk/info.py @@ -1,15 +1,20 @@ """Functions that help us generate and use info.json files. """ from functools import lru_cache +from glob import glob from pathlib import Path import jsonschema +from dotty_dict import dotty from milc import cli from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS -from qmk.json_schema import validate +from qmk.c_parse import find_layouts +from qmk.json_schema import deep_update, json_load, validate +from qmk.keyboard import config_h, rules_mk from qmk.keymap import list_keymaps -from qmk.metadata import basic_info_json, info_log_error +from qmk.math import compute +from qmk.metadata import basic_info_json, info_log_error, info_log_warning, true_values, false_values @lru_cache(maxsize=None) @@ -111,7 +116,7 @@ def _extract_features(info_data, rules): info_data['features'] = {} if key in info_data['features']: - _log_warning(info_data, 'Feature %s is specified in both info.json and rules.mk, the rules.mk value wins.' % (key,)) + info_log_warning(info_data, 'Feature %s is specified in both info.json and rules.mk, the rules.mk value wins.' % (key,)) info_data['features'][key] = value info_data['config_h_features'][key] = value @@ -190,7 +195,7 @@ def _extract_split_main(info_data, config_c): info_data['split'] = {} if 'main' in info_data['split']: - _log_warning(info_data, 'Split main hand is specified in both config.h (SPLIT_HAND_PIN) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main']) + info_log_warning(info_data, 'Split main hand is specified in both config.h (SPLIT_HAND_PIN) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main']) info_data['split']['main'] = 'pin' @@ -199,7 +204,7 @@ def _extract_split_main(info_data, config_c): info_data['split'] = {} if 'main' in info_data['split']: - _log_warning(info_data, 'Split main hand is specified in both config.h (SPLIT_HAND_MATRIX_GRID) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main']) + info_log_warning(info_data, 'Split main hand is specified in both config.h (SPLIT_HAND_MATRIX_GRID) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main']) info_data['split']['main'] = 'matrix_grid' info_data['split']['matrix_grid'] = _extract_pins(config_c['SPLIT_HAND_MATRIX_GRID']) @@ -209,7 +214,7 @@ def _extract_split_main(info_data, config_c): info_data['split'] = {} if 'main' in info_data['split']: - _log_warning(info_data, 'Split main hand is specified in both config.h (EE_HANDS) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main']) + info_log_warning(info_data, 'Split main hand is specified in both config.h (EE_HANDS) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main']) info_data['split']['main'] = 'eeprom' @@ -218,7 +223,7 @@ def _extract_split_main(info_data, config_c): info_data['split'] = {} if 'main' in info_data['split']: - _log_warning(info_data, 'Split main hand is specified in both config.h (MASTER_RIGHT) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main']) + info_log_warning(info_data, 'Split main hand is specified in both config.h (MASTER_RIGHT) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main']) info_data['split']['main'] = 'right' @@ -227,7 +232,7 @@ def _extract_split_main(info_data, config_c): info_data['split'] = {} if 'main' in info_data['split']: - _log_warning(info_data, 'Split main hand is specified in both config.h (MASTER_LEFT) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main']) + info_log_warning(info_data, 'Split main hand is specified in both config.h (MASTER_LEFT) and info.json (split.main) (Value: %s), the config.h value wins.' % info_data['split']['main']) info_data['split']['main'] = 'left' @@ -242,7 +247,7 @@ def _extract_split_transport(info_data, config_c): info_data['split']['transport'] = {} if 'protocol' in info_data['split']['transport']: - _log_warning(info_data, 'Split transport is specified in both config.h (USE_I2C) and info.json (split.transport.protocol) (Value: %s), the config.h value wins.' % info_data['split']['transport']) + info_log_warning(info_data, 'Split transport is specified in both config.h (USE_I2C) and info.json (split.transport.protocol) (Value: %s), the config.h value wins.' % info_data['split']['transport']) info_data['split']['transport']['protocol'] = 'i2c' @@ -266,7 +271,7 @@ def _extract_split_right_pins(info_data, config_c): if row_pins and col_pins: if info_data.get('split', {}).get('matrix_pins', {}).get('right') in info_data: - _log_warning(info_data, 'Right hand matrix data is specified in both info.json and config.h, the config.h values win.') + info_log_warning(info_data, 'Right hand matrix data is specified in both info.json and config.h, the config.h values win.') if 'split' not in info_data: info_data['split'] = {} @@ -284,7 +289,7 @@ def _extract_split_right_pins(info_data, config_c): if direct_pins: if info_data.get('split', {}).get('matrix_pins', {}).get('right', {}): - _log_warning(info_data, 'Right hand matrix data is specified in both info.json and config.h, the config.h values win.') + info_log_warning(info_data, 'Right hand matrix data is specified in both info.json and config.h, the config.h values win.') if 'split' not in info_data: info_data['split'] = {} @@ -322,7 +327,7 @@ def _extract_matrix_info(info_data, config_c): if 'MATRIX_ROWS' in config_c and 'MATRIX_COLS' in config_c: if 'matrix_size' in info_data: - _log_warning(info_data, 'Matrix size is specified in both info.json and config.h, the config.h values win.') + info_log_warning(info_data, 'Matrix size is specified in both info.json and config.h, the config.h values win.') info_data['matrix_size'] = { 'cols': compute(config_c.get('MATRIX_COLS', '0')), @@ -331,14 +336,14 @@ def _extract_matrix_info(info_data, config_c): if row_pins and col_pins: if 'matrix_pins' in info_data and 'cols' in info_data['matrix_pins'] and 'rows' in info_data['matrix_pins']: - _log_warning(info_data, 'Matrix pins are specified in both info.json and config.h, the config.h values win.') + info_log_warning(info_data, 'Matrix pins are specified in both info.json and config.h, the config.h values win.') info_snippet['cols'] = _extract_pins(col_pins) info_snippet['rows'] = _extract_pins(row_pins) if direct_pins: if 'matrix_pins' in info_data and 'direct' in info_data['matrix_pins']: - _log_warning(info_data, 'Direct pins are specified in both info.json and config.h, the config.h values win.') + info_log_warning(info_data, 'Direct pins are specified in both info.json and config.h, the config.h values win.') info_snippet['direct'] = _extract_direct_matrix(direct_pins) @@ -350,7 +355,7 @@ def _extract_matrix_info(info_data, config_c): if config_c.get('CUSTOM_MATRIX', 'no') != 'no': if 'matrix_pins' in info_data and 'custom' in info_data['matrix_pins']: - _log_warning(info_data, 'Custom Matrix is specified in both info.json and config.h, the config.h values win.') + info_log_warning(info_data, 'Custom Matrix is specified in both info.json and config.h, the config.h values win.') info_snippet['custom'] = True @@ -379,7 +384,7 @@ def _extract_config_h(info_data): try: if config_key in config_c and info_dict.get('to_json', True): if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True): - _log_warning(info_data, '%s in config.h is overwriting %s in info.json' % (config_key, info_key)) + info_log_warning(info_data, '%s in config.h is overwriting %s in info.json' % (config_key, info_key)) if key_type.startswith('array'): if '.' in key_type: @@ -410,7 +415,7 @@ def _extract_config_h(info_data): dotty_info[info_key] = config_c[config_key] except Exception as e: - _log_warning(info_data, f'{config_key}->{info_key}: {e}') + info_log_warning(info_data, f'{config_key}->{info_key}: {e}') info_data.update(dotty_info) @@ -451,7 +456,7 @@ def _extract_rules_mk(info_data): try: if rules_key in rules and info_dict.get('to_json', True): if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True): - _log_warning(info_data, '%s in rules.mk is overwriting %s in info.json' % (rules_key, info_key)) + info_log_warning(info_data, '%s in rules.mk is overwriting %s in info.json' % (rules_key, info_key)) if key_type.startswith('array'): if '.' in key_type: @@ -482,7 +487,7 @@ def _extract_rules_mk(info_data): dotty_info[info_key] = rules[rules_key] except Exception as e: - _log_warning(info_data, f'{rules_key}->{info_key}: {e}') + info_log_warning(info_data, f'{rules_key}->{info_key}: {e}') info_data.update(dotty_info) @@ -558,7 +563,7 @@ def _find_missing_layouts(info_data, keyboard): If we don't find any layouts from info.json or keyboard.h we widen our search. This is error prone which is why we want to encourage people to follow the standard above. """ - _log_warning(info_data, '%s: Falling back to searching for KEYMAP/LAYOUT macros.' % (keyboard)) + info_log_warning(info_data, '%s: Falling back to searching for KEYMAP/LAYOUT macros.' % (keyboard)) for file in glob('keyboards/%s/*.h' % keyboard): these_layouts, these_aliases = find_layouts(file) @@ -653,7 +658,7 @@ def merge_info_jsons(keyboard, info_data): for layout_name, layout in new_info_data.get('layouts', {}).items(): if layout_name in info_data.get('layout_aliases', {}): - _log_warning(info_data, f"info.json uses alias name {layout_name} instead of {info_data['layout_aliases'][layout_name]}") + info_log_warning(info_data, f"info.json uses alias name {layout_name} instead of {info_data['layout_aliases'][layout_name]}") layout_name = info_data['layout_aliases'][layout_name] if layout_name in info_data['layouts']: @@ -687,14 +692,18 @@ def find_info_json(keyboard): # Add DEFAULT_FOLDER before parents, if present rules = rules_mk(keyboard) + if 'DEFAULT_FOLDER' in rules: info_jsons.append(Path(rules['DEFAULT_FOLDER']) / 'info.json') # Add in parent folders for least specific for _ in range(5): - info_jsons.append(keyboard_parent / 'info.json') + this_info_json = keyboard_parent / 'info.json' + + if this_info_json.exists(): + yield this_info_json + if keyboard_parent.parent == base_path: break - keyboard_parent = keyboard_parent.parent - return info_data + keyboard_parent = keyboard_parent.parent diff --git a/lib/python/qmk/json_schema.py b/lib/python/qmk/json_schema.py index 3db679cd2831..c2d8905d6da4 100644 --- a/lib/python/qmk/json_schema.py +++ b/lib/python/qmk/json_schema.py @@ -24,6 +24,8 @@ def json_load(json_file): exit(1) except Exception as e: cli.log.error('Unknown error attempting to load {fg_cyan}%s{fg_reset}:\n\t{fg_red}%s', json_file, e) + if cli.args.verbose: + cli.log.exception(e) exit(1)