CLI parallel search updates (#22525)

This commit is contained in:
Nick Brassel 2023-11-22 11:14:34 +11:00 committed by GitHub
parent 7e27d72cbc
commit 46b996a55e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 26 additions and 13 deletions

View file

@ -1,11 +1,8 @@
"""Validates the list of keyboard aliases. """Validates the list of keyboard aliases.
""" """
from pathlib import Path
from milc import cli from milc import cli
from qmk.json_schema import json_load from qmk.keyboard import resolve_keyboard, keyboard_folder, keyboard_alias_definitions
from qmk.keyboard import resolve_keyboard, keyboard_folder
def _safe_keyboard_folder(target): def _safe_keyboard_folder(target):
@ -34,7 +31,7 @@ def _target_keyboard_exists(target):
@cli.subcommand('Validates the list of keyboard aliases.', hidden=True) @cli.subcommand('Validates the list of keyboard aliases.', hidden=True)
def ci_validate_aliases(cli): def ci_validate_aliases(cli):
aliases = json_load(Path('data/mappings/keyboard_aliases.hjson')) aliases = keyboard_alias_definitions()
success = True success = True
for alias in aliases.keys(): for alias in aliases.keys():

View file

@ -10,7 +10,7 @@ from qmk.datetime import current_datetime
from qmk.info import info_json from qmk.info import info_json
from qmk.json_schema import json_load from qmk.json_schema import json_load
from qmk.keymap import list_keymaps from qmk.keymap import list_keymaps
from qmk.keyboard import find_readme, list_keyboards from qmk.keyboard import find_readme, list_keyboards, keyboard_alias_definitions
from qmk.keycodes import load_spec, list_versions, list_languages from qmk.keycodes import load_spec, list_versions, list_languages
DATA_PATH = Path('data') DATA_PATH = Path('data')
@ -166,7 +166,7 @@ def generate_api(cli):
# Generate data for the global files # Generate data for the global files
keyboard_list = sorted(kb_all) keyboard_list = sorted(kb_all)
keyboard_aliases = json_load(Path('data/mappings/keyboard_aliases.hjson')) keyboard_aliases = keyboard_alias_definitions()
keyboard_metadata = { keyboard_metadata = {
'last_updated': current_datetime(), 'last_updated': current_datetime(),
'keyboards': keyboard_list, 'keyboards': keyboard_list,

View file

@ -3,12 +3,12 @@
import os import os
import sys import sys
import shutil import shutil
from pathlib import Path
from milc import cli from milc import cli
import jsonschema import jsonschema
from qmk.json_schema import json_load, validate from qmk.json_schema import json_load, validate
from qmk.keyboard import keyboard_alias_definitions
def find_make(): def find_make():
@ -53,7 +53,7 @@ def parse_configurator_json(configurator_file):
exit(1) exit(1)
keyboard = user_keymap['keyboard'] keyboard = user_keymap['keyboard']
aliases = json_load(Path('data/mappings/keyboard_aliases.hjson')) aliases = keyboard_alias_definitions()
while keyboard in aliases: while keyboard in aliases:
last_keyboard = keyboard last_keyboard = keyboard

View file

@ -70,6 +70,11 @@ class AllKeyboards:
base_path = os.path.join(os.getcwd(), "keyboards") + os.path.sep base_path = os.path.join(os.getcwd(), "keyboards") + os.path.sep
@lru_cache(maxsize=1)
def keyboard_alias_definitions():
return json_load(Path('data/mappings/keyboard_aliases.hjson'))
def is_all_keyboards(keyboard): def is_all_keyboards(keyboard):
"""Returns True if the keyboard is an AllKeyboards object. """Returns True if the keyboard is an AllKeyboards object.
""" """
@ -112,7 +117,7 @@ def keyboard_folder(keyboard):
This checks aliases and DEFAULT_FOLDER to resolve the actual path for a keyboard. This checks aliases and DEFAULT_FOLDER to resolve the actual path for a keyboard.
""" """
aliases = json_load(Path('data/mappings/keyboard_aliases.hjson')) aliases = keyboard_alias_definitions()
while keyboard in aliases: while keyboard in aliases:
last_keyboard = keyboard last_keyboard = keyboard

View file

@ -6,7 +6,7 @@ import fnmatch
import logging import logging
import re import re
from typing import List, Tuple from typing import List, Tuple
from dotty_dict import dotty from dotty_dict import dotty, Dotty
from milc import cli from milc import cli
from qmk.util import parallel_map from qmk.util import parallel_map
@ -107,13 +107,22 @@ def expand_keymap_targets(targets: List[Tuple[str, str]]) -> List[Tuple[str, str
return list(sorted(set(overall_targets))) return list(sorted(set(overall_targets)))
def _construct_build_target_kb_km(e):
return KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1])
def _construct_build_target_kb_km_json(e):
return KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1], json=e[2])
def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = []) -> List[BuildTarget]: def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = []) -> List[BuildTarget]:
"""Filter a list of (keyboard, keymap) tuples based on the supplied filters. """Filter a list of (keyboard, keymap) tuples based on the supplied filters.
Optionally includes the values of the queried info.json keys. Optionally includes the values of the queried info.json keys.
""" """
if len(filters) == 0: if len(filters) == 0:
targets = [KeyboardKeymapBuildTarget(keyboard=kb, keymap=km) for kb, km in target_list] cli.log.info('Preparing target list...')
targets = list(parallel_map(_construct_build_target_kb_km, target_list))
else: else:
cli.log.info('Parsing data for all matching keyboard/keymap combinations...') cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)] valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)]
@ -172,7 +181,9 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str
cli.log.warning(f'Unrecognized filter expression: {filter_expr}') cli.log.warning(f'Unrecognized filter expression: {filter_expr}')
continue continue
targets = [KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1], json=e[2]) for e in valid_keymaps] cli.log.info('Preparing target list...')
valid_keymaps = [(e[0], e[1], e[2].to_dict() if isinstance(e[2], Dotty) else e[2]) for e in valid_keymaps] # need to convert dotty_dict back to dict because it doesn't survive parallelisation
targets = list(parallel_map(_construct_build_target_kb_km_json, list(valid_keymaps)))
return targets return targets