CLI parallel search updates (#22525)
This commit is contained in:
parent
7e27d72cbc
commit
46b996a55e
|
@ -1,11 +1,8 @@
|
||||||
"""Validates the list of keyboard aliases.
|
"""Validates the list of keyboard aliases.
|
||||||
"""
|
"""
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from milc import cli
|
from milc import cli
|
||||||
|
|
||||||
from qmk.json_schema import json_load
|
from qmk.keyboard import resolve_keyboard, keyboard_folder, keyboard_alias_definitions
|
||||||
from qmk.keyboard import resolve_keyboard, keyboard_folder
|
|
||||||
|
|
||||||
|
|
||||||
def _safe_keyboard_folder(target):
|
def _safe_keyboard_folder(target):
|
||||||
|
@ -34,7 +31,7 @@ def _target_keyboard_exists(target):
|
||||||
|
|
||||||
@cli.subcommand('Validates the list of keyboard aliases.', hidden=True)
|
@cli.subcommand('Validates the list of keyboard aliases.', hidden=True)
|
||||||
def ci_validate_aliases(cli):
|
def ci_validate_aliases(cli):
|
||||||
aliases = json_load(Path('data/mappings/keyboard_aliases.hjson'))
|
aliases = keyboard_alias_definitions()
|
||||||
|
|
||||||
success = True
|
success = True
|
||||||
for alias in aliases.keys():
|
for alias in aliases.keys():
|
||||||
|
|
|
@ -10,7 +10,7 @@ from qmk.datetime import current_datetime
|
||||||
from qmk.info import info_json
|
from qmk.info import info_json
|
||||||
from qmk.json_schema import json_load
|
from qmk.json_schema import json_load
|
||||||
from qmk.keymap import list_keymaps
|
from qmk.keymap import list_keymaps
|
||||||
from qmk.keyboard import find_readme, list_keyboards
|
from qmk.keyboard import find_readme, list_keyboards, keyboard_alias_definitions
|
||||||
from qmk.keycodes import load_spec, list_versions, list_languages
|
from qmk.keycodes import load_spec, list_versions, list_languages
|
||||||
|
|
||||||
DATA_PATH = Path('data')
|
DATA_PATH = Path('data')
|
||||||
|
@ -166,7 +166,7 @@ def generate_api(cli):
|
||||||
|
|
||||||
# Generate data for the global files
|
# Generate data for the global files
|
||||||
keyboard_list = sorted(kb_all)
|
keyboard_list = sorted(kb_all)
|
||||||
keyboard_aliases = json_load(Path('data/mappings/keyboard_aliases.hjson'))
|
keyboard_aliases = keyboard_alias_definitions()
|
||||||
keyboard_metadata = {
|
keyboard_metadata = {
|
||||||
'last_updated': current_datetime(),
|
'last_updated': current_datetime(),
|
||||||
'keyboards': keyboard_list,
|
'keyboards': keyboard_list,
|
||||||
|
|
|
@ -3,12 +3,12 @@
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import shutil
|
import shutil
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from milc import cli
|
from milc import cli
|
||||||
import jsonschema
|
import jsonschema
|
||||||
|
|
||||||
from qmk.json_schema import json_load, validate
|
from qmk.json_schema import json_load, validate
|
||||||
|
from qmk.keyboard import keyboard_alias_definitions
|
||||||
|
|
||||||
|
|
||||||
def find_make():
|
def find_make():
|
||||||
|
@ -53,7 +53,7 @@ def parse_configurator_json(configurator_file):
|
||||||
exit(1)
|
exit(1)
|
||||||
|
|
||||||
keyboard = user_keymap['keyboard']
|
keyboard = user_keymap['keyboard']
|
||||||
aliases = json_load(Path('data/mappings/keyboard_aliases.hjson'))
|
aliases = keyboard_alias_definitions()
|
||||||
|
|
||||||
while keyboard in aliases:
|
while keyboard in aliases:
|
||||||
last_keyboard = keyboard
|
last_keyboard = keyboard
|
||||||
|
|
|
@ -70,6 +70,11 @@ class AllKeyboards:
|
||||||
base_path = os.path.join(os.getcwd(), "keyboards") + os.path.sep
|
base_path = os.path.join(os.getcwd(), "keyboards") + os.path.sep
|
||||||
|
|
||||||
|
|
||||||
|
@lru_cache(maxsize=1)
|
||||||
|
def keyboard_alias_definitions():
|
||||||
|
return json_load(Path('data/mappings/keyboard_aliases.hjson'))
|
||||||
|
|
||||||
|
|
||||||
def is_all_keyboards(keyboard):
|
def is_all_keyboards(keyboard):
|
||||||
"""Returns True if the keyboard is an AllKeyboards object.
|
"""Returns True if the keyboard is an AllKeyboards object.
|
||||||
"""
|
"""
|
||||||
|
@ -112,7 +117,7 @@ def keyboard_folder(keyboard):
|
||||||
|
|
||||||
This checks aliases and DEFAULT_FOLDER to resolve the actual path for a keyboard.
|
This checks aliases and DEFAULT_FOLDER to resolve the actual path for a keyboard.
|
||||||
"""
|
"""
|
||||||
aliases = json_load(Path('data/mappings/keyboard_aliases.hjson'))
|
aliases = keyboard_alias_definitions()
|
||||||
|
|
||||||
while keyboard in aliases:
|
while keyboard in aliases:
|
||||||
last_keyboard = keyboard
|
last_keyboard = keyboard
|
||||||
|
|
|
@ -6,7 +6,7 @@ import fnmatch
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
from typing import List, Tuple
|
from typing import List, Tuple
|
||||||
from dotty_dict import dotty
|
from dotty_dict import dotty, Dotty
|
||||||
from milc import cli
|
from milc import cli
|
||||||
|
|
||||||
from qmk.util import parallel_map
|
from qmk.util import parallel_map
|
||||||
|
@ -107,13 +107,22 @@ def expand_keymap_targets(targets: List[Tuple[str, str]]) -> List[Tuple[str, str
|
||||||
return list(sorted(set(overall_targets)))
|
return list(sorted(set(overall_targets)))
|
||||||
|
|
||||||
|
|
||||||
|
def _construct_build_target_kb_km(e):
|
||||||
|
return KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1])
|
||||||
|
|
||||||
|
|
||||||
|
def _construct_build_target_kb_km_json(e):
|
||||||
|
return KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1], json=e[2])
|
||||||
|
|
||||||
|
|
||||||
def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = []) -> List[BuildTarget]:
|
def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = []) -> List[BuildTarget]:
|
||||||
"""Filter a list of (keyboard, keymap) tuples based on the supplied filters.
|
"""Filter a list of (keyboard, keymap) tuples based on the supplied filters.
|
||||||
|
|
||||||
Optionally includes the values of the queried info.json keys.
|
Optionally includes the values of the queried info.json keys.
|
||||||
"""
|
"""
|
||||||
if len(filters) == 0:
|
if len(filters) == 0:
|
||||||
targets = [KeyboardKeymapBuildTarget(keyboard=kb, keymap=km) for kb, km in target_list]
|
cli.log.info('Preparing target list...')
|
||||||
|
targets = list(parallel_map(_construct_build_target_kb_km, target_list))
|
||||||
else:
|
else:
|
||||||
cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
|
cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
|
||||||
valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)]
|
valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)]
|
||||||
|
@ -172,7 +181,9 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str
|
||||||
cli.log.warning(f'Unrecognized filter expression: {filter_expr}')
|
cli.log.warning(f'Unrecognized filter expression: {filter_expr}')
|
||||||
continue
|
continue
|
||||||
|
|
||||||
targets = [KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1], json=e[2]) for e in valid_keymaps]
|
cli.log.info('Preparing target list...')
|
||||||
|
valid_keymaps = [(e[0], e[1], e[2].to_dict() if isinstance(e[2], Dotty) else e[2]) for e in valid_keymaps] # need to convert dotty_dict back to dict because it doesn't survive parallelisation
|
||||||
|
targets = list(parallel_map(_construct_build_target_kb_km_json, list(valid_keymaps)))
|
||||||
|
|
||||||
return targets
|
return targets
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue