1 files changed, 373 insertions, 84 deletions

diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py
index f476dc666..2accaba9e 100644
--- a/lib/python/qmk/info.py
+++ b/lib/python/qmk/info.py
@@ -1,9 +1,13 @@
 """Functions that help us generate and use info.json files.
 """
 import json
+from collections.abc import Mapping
 from glob import glob
 from pathlib import Path
 
+import hjson
+import jsonschema
+from dotty_dict import dotty
 from milc import cli
 
 from qmk.constants import CHIBIOS_PROCESSORS, LUFA_PROCESSORS, VUSB_PROCESSORS
@@ -13,6 +17,9 @@ from qmk.keymap import list_keymaps
 from qmk.makefile import parse_rules_mk_file
 from qmk.math import compute
 
+true_values = ['1', 'on', 'yes']
+false_values = ['0', 'off', 'no']
+
 
 def info_json(keyboard):
     """Generate the info.json data for a specific keyboard.
@@ -38,8 +45,9 @@ def info_json(keyboard):
         info_data['keymaps'][keymap.name] = {'url': f'https://raw.githubusercontent.com/qmk/qmk_firmware/master/{keymap}/keymap.json'}
 
     # Populate layout data
-    for layout_name, layout_json in _find_all_layouts(info_data, keyboard, rules).items():
+    for layout_name, layout_json in _find_all_layouts(info_data, keyboard).items():
         if not layout_name.startswith('LAYOUT_kc'):
+            layout_json['c_macro'] = True
             info_data['layouts'][layout_name] = layout_json
 
     # Merge in the data from info.json, config.h, and rules.mk
@@ -47,54 +55,259 @@ def info_json(keyboard):
     info_data = _extract_config_h(info_data)
     info_data = _extract_rules_mk(info_data)
 
+    # Validate against the jsonschema
+    try:
+        keyboard_api_validate(info_data)
+
+    except jsonschema.ValidationError as e:
+        json_path = '.'.join([str(p) for p in e.absolute_path])
+        cli.log.error('Invalid API data: %s: %s: %s', keyboard, json_path, e.message)
+        exit()
+
+    # Make sure we have at least one layout
+    if not info_data.get('layouts'):
+        _log_error(info_data, 'No LAYOUTs defined! Need at least one layout defined in the keyboard.h or info.json.')
+
+    # Make sure we supply layout macros for the community layouts we claim to support
+    # FIXME(skullydazed): This should be populated into info.json and read from there instead
+    if 'LAYOUTS' in rules and info_data.get('layouts'):
+        # Match these up against the supplied layouts
+        supported_layouts = rules['LAYOUTS'].strip().split()
+        for layout_name in sorted(info_data['layouts']):
+            layout_name = layout_name[7:]
+
+            if layout_name in supported_layouts:
+                supported_layouts.remove(layout_name)
+
+        if supported_layouts:
+            for supported_layout in supported_layouts:
+                _log_error(info_data, 'Claims to support community layout %s but no LAYOUT_%s() macro found' % (supported_layout, supported_layout))
+
     return info_data
 
 
-def _extract_config_h(info_data):
-    """Pull some keyboard information from existing rules.mk files
+def _json_load(json_file):
+    """Load a json file from disk.
+
+    Note: file must be a Path object.
+    """
+    try:
+        return hjson.load(json_file.open())
+
+    except json.decoder.JSONDecodeError as e:
+        cli.log.error('Invalid JSON encountered attempting to load {fg_cyan}%s{fg_reset}:\n\t{fg_red}%s', json_file, e)
+        exit(1)
+
+
+def _jsonschema(schema_name):
+    """Read a jsonschema file from disk.
+
+    FIXME(skullydazed/anyone): Refactor to make this a public function.
+    """
+    schema_path = Path(f'data/schemas/{schema_name}.jsonschema')
+
+    if not schema_path.exists():
+        schema_path = Path('data/schemas/false.jsonschema')
+
+    return _json_load(schema_path)
+
+
+def keyboard_validate(data):
+    """Validates data against the keyboard jsonschema.
+    """
+    schema = _jsonschema('keyboard')
+    validator = jsonschema.Draft7Validator(schema).validate
+
+    return validator(data)
+
+
+def keyboard_api_validate(data):
+    """Validates data against the api_keyboard jsonschema.
+    """
+    base = _jsonschema('keyboard')
+    relative = _jsonschema('api_keyboard')
+    resolver = jsonschema.RefResolver.from_schema(base)
+    validator = jsonschema.Draft7Validator(relative, resolver=resolver).validate
+
+    return validator(data)
+
+
+def _extract_features(info_data, rules):
+    """Find all the features enabled in rules.mk.
+    """
+    # Special handling for bootmagic which also supports a "lite" mode.
+    if rules.get('BOOTMAGIC_ENABLE') == 'lite':
+        rules['BOOTMAGIC_LITE_ENABLE'] = 'on'
+        del rules['BOOTMAGIC_ENABLE']
+    if rules.get('BOOTMAGIC_ENABLE') == 'full':
+        rules['BOOTMAGIC_ENABLE'] = 'on'
+
+    # Skip non-boolean features we haven't implemented special handling for
+    for feature in 'HAPTIC_ENABLE', 'QWIIC_ENABLE':
+        if rules.get(feature):
+            del rules[feature]
+
+    # Process the rest of the rules as booleans
+    for key, value in rules.items():
+        if key.endswith('_ENABLE'):
+            key = '_'.join(key.split('_')[:-1]).lower()
+            value = True if value.lower() in true_values else False if value.lower() in false_values else value
+
+            if 'config_h_features' not in info_data:
+                info_data['config_h_features'] = {}
+
+            if 'features' not in info_data:
+                info_data['features'] = {}
+
+            if key in info_data['features']:
+                _log_warning(info_data, 'Feature %s is specified in both info.json and rules.mk, the rules.mk value wins.' % (key,))
+
+            info_data['features'][key] = value
+            info_data['config_h_features'][key] = value
+
+    return info_data
+
+
+def _pin_name(pin):
+    """Returns the proper representation for a pin.
+    """
+    pin = pin.strip()
+
+    if not pin:
+        return None
+
+    elif pin.isdigit():
+        return int(pin)
+
+    elif pin == 'NO_PIN':
+        return None
+
+    elif pin[0] in 'ABCDEFGHIJK' and pin[1].isdigit():
+        return pin
+
+    raise ValueError(f'Invalid pin: {pin}')
+
+
+def _extract_pins(pins):
+    """Returns a list of pins from a comma separated string of pins.
+    """
+    return [_pin_name(pin) for pin in pins.split(',')]
+
+
+def _extract_direct_matrix(info_data, direct_pins):
+    """
+    """
+    info_data['matrix_pins'] = {}
+    direct_pin_array = []
+
+    while direct_pins[-1] != '}':
+        direct_pins = direct_pins[:-1]
+
+    for row in direct_pins.split('},{'):
+        if row.startswith('{'):
+            row = row[1:]
+
+        if row.endswith('}'):
+            row = row[:-1]
+
+        direct_pin_array.append([])
+
+        for pin in row.split(','):
+            if pin == 'NO_PIN':
+                pin = None
+
+            direct_pin_array[-1].append(pin)
+
+    return direct_pin_array
+
+
+def _extract_matrix_info(info_data, config_c):
+    """Populate the matrix information.
     """
-    config_c = config_h(info_data['keyboard_folder'])
     row_pins = config_c.get('MATRIX_ROW_PINS', '').replace('{', '').replace('}', '').strip()
     col_pins = config_c.get('MATRIX_COL_PINS', '').replace('{', '').replace('}', '').strip()
     direct_pins = config_c.get('DIRECT_PINS', '').replace(' ', '')[1:-1]
 
-    info_data['diode_direction'] = config_c.get('DIODE_DIRECTION')
-    info_data['matrix_size'] = {
-        'rows': compute(config_c.get('MATRIX_ROWS', '0')),
-        'cols': compute(config_c.get('MATRIX_COLS', '0')),
-    }
-    info_data['matrix_pins'] = {}
+    if 'MATRIX_ROWS' in config_c and 'MATRIX_COLS' in config_c:
+        if 'matrix_size' in info_data:
+            _log_warning(info_data, 'Matrix size is specified in both info.json and config.h, the config.h values win.')
+
+        info_data['matrix_size'] = {
+            'cols': compute(config_c.get('MATRIX_COLS', '0')),
+            'rows': compute(config_c.get('MATRIX_ROWS', '0')),
+        }
 
-    if row_pins:
-        info_data['matrix_pins']['rows'] = row_pins.split(',')
-    if col_pins:
-        info_data['matrix_pins']['cols'] = col_pins.split(',')
+    if row_pins and col_pins:
+        if 'matrix_pins' in info_data:
+            _log_warning(info_data, 'Matrix pins are specified in both info.json and config.h, the config.h values win.')
+
+        info_data['matrix_pins'] = {
+            'cols': _extract_pins(col_pins),
+            'rows': _extract_pins(row_pins),
+        }
 
     if direct_pins:
-        direct_pin_array = []
-        for row in direct_pins.split('},{'):
-            if row.startswith('{'):
-                row = row[1:]
-            if row.endswith('}'):
-                row = row[:-1]
+        if 'matrix_pins' in info_data:
+            _log_warning(info_data, 'Direct pins are specified in both info.json and config.h, the config.h values win.')
 
-            direct_pin_array.append([])
+        info_data['matrix_pins']['direct'] = _extract_direct_matrix(info_data, direct_pins)
 
-            for pin in row.split(','):
-                if pin == 'NO_PIN':
-                    pin = None
+    return info_data
 
-                direct_pin_array[-1].append(pin)
 
-        info_data['matrix_pins']['direct'] = direct_pin_array
+def _extract_config_h(info_data):
+    """Pull some keyboard information from existing config.h files
+    """
+    config_c = config_h(info_data['keyboard_folder'])
 
-    info_data['usb'] = {
-        'vid': config_c.get('VENDOR_ID'),
-        'pid': config_c.get('PRODUCT_ID'),
-        'device_ver': config_c.get('DEVICE_VER'),
-        'manufacturer': config_c.get('MANUFACTURER'),
-        'product': config_c.get('PRODUCT'),
-    }
+    # Pull in data from the json map
+    dotty_info = dotty(info_data)
+    info_config_map = _json_load(Path('data/mappings/info_config.json'))
+
+    for config_key, info_dict in info_config_map.items():
+        info_key = info_dict['info_key']
+        key_type = info_dict.get('value_type', 'str')
+
+        try:
+            if config_key in config_c and info_dict.get('to_json', True):
+                if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True):
+                    _log_warning(info_data, '%s in config.h is overwriting %s in info.json' % (config_key, info_key))
+
+                if key_type.startswith('array'):
+                    if '.' in key_type:
+                        key_type, array_type = key_type.split('.', 1)
+                    else:
+                        array_type = None
+
+                    config_value = config_c[config_key].replace('{', '').replace('}', '').strip()
+
+                    if array_type == 'int':
+                        dotty_info[info_key] = list(map(int, config_value.split(',')))
+                    else:
+                        dotty_info[info_key] = config_value.split(',')
+
+                elif key_type == 'bool':
+                    dotty_info[info_key] = config_c[config_key] in true_values
+
+                elif key_type == 'hex':
+                    dotty_info[info_key] = '0x' + config_c[config_key][2:].upper()
+
+                elif key_type == 'list':
+                    dotty_info[info_key] = config_c[config_key].split()
+
+                elif key_type == 'int':
+                    dotty_info[info_key] = int(config_c[config_key])
+
+                else:
+                    dotty_info[info_key] = config_c[config_key]
+
+        except Exception as e:
+            _log_warning(info_data, f'{config_key}->{info_key}: {e}')
+
+    info_data.update(dotty_info)
+
+    # Pull data that easily can't be mapped in json
+    _extract_matrix_info(info_data, config_c)
 
     return info_data
 
@@ -103,19 +316,101 @@ def _extract_rules_mk(info_data):
     """Pull some keyboard information from existing rules.mk files
     """
     rules = rules_mk(info_data['keyboard_folder'])
-    mcu = rules.get('MCU')
+    info_data['processor'] = rules.get('MCU', info_data.get('processor', 'atmega32u4'))
+
+    if info_data['processor'] in CHIBIOS_PROCESSORS:
+        arm_processor_rules(info_data, rules)
+
+    elif info_data['processor'] in LUFA_PROCESSORS + VUSB_PROCESSORS:
+        avr_processor_rules(info_data, rules)
+
+    else:
+        cli.log.warning("%s: Unknown MCU: %s" % (info_data['keyboard_folder'], info_data['processor']))
+        unknown_processor_rules(info_data, rules)
+
+    # Pull in data from the json map
+    dotty_info = dotty(info_data)
+    info_rules_map = _json_load(Path('data/mappings/info_rules.json'))
 
-    if mcu in CHIBIOS_PROCESSORS:
-        return arm_processor_rules(info_data, rules)
+    for rules_key, info_dict in info_rules_map.items():
+        info_key = info_dict['info_key']
+        key_type = info_dict.get('value_type', 'str')
 
-    elif mcu in LUFA_PROCESSORS + VUSB_PROCESSORS:
-        return avr_processor_rules(info_data, rules)
+        try:
+            if rules_key in rules and info_dict.get('to_json', True):
+                if dotty_info.get(info_key) and info_dict.get('warn_duplicate', True):
+                    _log_warning(info_data, '%s in rules.mk is overwriting %s in info.json' % (rules_key, info_key))
+
+                if key_type.startswith('array'):
+                    if '.' in key_type:
+                        key_type, array_type = key_type.split('.', 1)
+                    else:
+                        array_type = None
 
-    msg = "Unknown MCU: " + str(mcu)
+                    rules_value = rules[rules_key].replace('{', '').replace('}', '').strip()
+
+                    if array_type == 'int':
+                        dotty_info[info_key] = list(map(int, rules_value.split(',')))
+                    else:
+                        dotty_info[info_key] = rules_value.split(',')
 
-    _log_warning(info_data, msg)
+                elif key_type == 'list':
+                    dotty_info[info_key] = rules[rules_key].split()
 
-    return unknown_processor_rules(info_data, rules)
+                elif key_type == 'bool':
+                    dotty_info[info_key] = rules[rules_key] in true_values
+
+                elif key_type == 'hex':
+                    dotty_info[info_key] = '0x' + rules[rules_key][2:].upper()
+
+                elif key_type == 'int':
+                    dotty_info[info_key] = int(rules[rules_key])
+
+                else:
+                    dotty_info[info_key] = rules[rules_key]
+
+        except Exception as e:
+            _log_warning(info_data, f'{rules_key}->{info_key}: {e}')
+
+    info_data.update(dotty_info)
+
+    # Merge in config values that can't be easily mapped
+    _extract_features(info_data, rules)
+
+    return info_data
+
+
+def _merge_layouts(info_data, new_info_data):
+    """Merge new_info_data into info_data in an intelligent way.
+    """
+    for layout_name, layout_json in new_info_data['layouts'].items():
+        if layout_name in info_data['layouts']:
+            # Pull in layouts we have a macro for
+            if len(info_data['layouts'][layout_name]['layout']) != len(layout_json['layout']):
+                msg = '%s: %s: Number of elements in info.json does not match! info.json:%s != %s:%s'
+                _log_error(info_data, msg % (info_data['keyboard_folder'], layout_name, len(layout_json['layout']), layout_name, len(info_data['layouts'][layout_name]['layout'])))
+            else:
+                for i, key in enumerate(info_data['layouts'][layout_name]['layout']):
+                    key.update(layout_json['layout'][i])
+        else:
+            # Pull in layouts that have matrix data
+            missing_matrix = False
+            for key in layout_json.get('layout', {}):
+                if 'matrix' not in key:
+                    missing_matrix = True
+
+            if not missing_matrix:
+                if layout_name in info_data['layouts']:
+                    # Update an existing layout with new data
+                    for i, key in enumerate(info_data['layouts'][layout_name]['layout']):
+                        key.update(layout_json['layout'][i])
+
+                else:
+                    # Copy in the new layout wholesale
+                    layout_json['c_macro'] = False
+                    info_data['layouts'][layout_name] = layout_json
+
+    return info_data
 
 
 def _search_keyboard_h(path):
@@ -131,34 +426,21 @@ def _search_keyboard_h(path):
     return layouts
 
 
-def _find_all_layouts(info_data, keyboard, rules):
+def _find_all_layouts(info_data, keyboard):
     """Looks for layout macros associated with this keyboard.
     """
     layouts = _search_keyboard_h(Path(keyboard))
 
     if not layouts:
-        # If we didn't find any layouts above we widen our search. This is error
-        # prone which is why we want to encourage people to follow the standard above.
-        _log_warning(info_data, 'Falling back to searching for KEYMAP/LAYOUT macros.')
+        # If we don't find any layouts from info.json or keyboard.h we widen our search. This is error prone which is why we want to encourage people to follow the standard above.
+        info_data['parse_warnings'].append('%s: Falling back to searching for KEYMAP/LAYOUT macros.' % (keyboard))
+
         for file in glob('keyboards/%s/*.h' % keyboard):
             if file.endswith('.h'):
                 these_layouts = find_layouts(file)
                 if these_layouts:
                     layouts.update(these_layouts)
 
-    if 'LAYOUTS' in rules:
-        # Match these up against the supplied layouts
-        supported_layouts = rules['LAYOUTS'].strip().split()
-        for layout_name in sorted(layouts):
-            if not layout_name.startswith('LAYOUT_'):
-                continue
-            layout_name = layout_name[7:]
-            if layout_name in supported_layouts:
-                supported_layouts.remove(layout_name)
-
-        if supported_layouts:
-            _log_error(info_data, 'Missing LAYOUT() macro for %s' % (', '.join(supported_layouts)))
-
     return layouts
 
 
@@ -180,13 +462,13 @@ def arm_processor_rules(info_data, rules):
     """Setup the default info for an ARM board.
     """
     info_data['processor_type'] = 'arm'
-    info_data['bootloader'] = rules['BOOTLOADER'] if 'BOOTLOADER' in rules else 'unknown'
-    info_data['processor'] = rules['MCU'] if 'MCU' in rules else 'unknown'
     info_data['protocol'] = 'ChibiOS'
 
-    if info_data['bootloader'] == 'unknown':
+    if 'bootloader' not in info_data:
         if 'STM32' in info_data['processor']:
             info_data['bootloader'] = 'stm32-dfu'
+        else:
+            info_data['bootloader'] = 'unknown'
 
     if 'STM32' in info_data['processor']:
         info_data['platform'] = 'STM32'
@@ -202,11 +484,12 @@ def avr_processor_rules(info_data, rules):
     """Setup the default info for an AVR board.
     """
     info_data['processor_type'] = 'avr'
-    info_data['bootloader'] = rules['BOOTLOADER'] if 'BOOTLOADER' in rules else 'atmel-dfu'
     info_data['platform'] = rules['ARCH'] if 'ARCH' in rules else 'unknown'
-    info_data['processor'] = rules['MCU'] if 'MCU' in rules else 'unknown'
     info_data['protocol'] = 'V-USB' if rules.get('MCU') in VUSB_PROCESSORS else 'LUFA'
 
+    if 'bootloader' not in info_data:
+        info_data['bootloader'] = 'atmel-dfu'
+
     # FIXME(fauxpark/anyone): Eventually we should detect the protocol by looking at PROTOCOL inherited from mcu_selection.mk:
     # info_data['protocol'] = 'V-USB' if rules.get('PROTOCOL') == 'VUSB' else 'LUFA'
 
@@ -225,38 +508,44 @@ def unknown_processor_rules(info_data, rules):
     return info_data
 
 
+def deep_update(origdict, newdict):
+    """Update a dictionary in place, recursing to do a deep copy.
+    """
+    for key, value in newdict.items():
+        if isinstance(value, Mapping):
+            origdict[key] = deep_update(origdict.get(key, {}), value)
+
+        else:
+            origdict[key] = value
+
+    return origdict
+
+
 def merge_info_jsons(keyboard, info_data):
     """Return a merged copy of all the info.json files for a keyboard.
     """
     for info_file in find_info_json(keyboard):
         # Load and validate the JSON data
-        try:
-            with info_file.open('r') as info_fd:
-                new_info_data = json.load(info_fd)
-        except Exception as e:
-            _log_error(info_data, "Invalid JSON in file %s: %s: %s" % (str(info_file), e.__class__.__name__, e))
-            continue
+        new_info_data = _json_load(info_file)
 
         if not isinstance(new_info_data, dict):
             _log_error(info_data, "Invalid file %s, root object should be a dictionary." % (str(info_file),))
             continue
 
-        # Copy whitelisted keys into `info_data`
-        for key in ('keyboard_name', 'manufacturer', 'identifier', 'url', 'maintainer', 'processor', 'bootloader', 'width', 'height'):
-            if key in new_info_data:
-                info_data[key] = new_info_data[key]
+        try:
+            keyboard_validate(new_info_data)
+        except jsonschema.ValidationError as e:
+            json_path = '.'.join([str(p) for p in e.absolute_path])
+            cli.log.error('Not including data from file: %s', info_file)
+            cli.log.error('\t%s: %s', json_path, e.message)
+            continue
+
+        # Mark the layouts as coming from json
+        for layout in new_info_data.get('layouts', {}).values():
+            layout['c_macro'] = False
 
-        # Merge the layouts in
-        if 'layouts' in new_info_data:
-            for layout_name, json_layout in new_info_data['layouts'].items():
-                # Only pull in layouts we have a macro for
-                if layout_name in info_data['layouts']:
-                    if info_data['layouts'][layout_name]['key_count'] != len(json_layout['layout']):
-                        msg = '%s: Number of elements in info.json does not match! info.json:%s != %s:%s'
-                        _log_error(info_data, msg % (layout_name, len(json_layout['layout']), layout_name, len(info_data['layouts'][layout_name]['layout'])))
-                    else:
-                        for i, key in enumerate(info_data['layouts'][layout_name]['layout']):
-                            key.update(json_layout['layout'][i])
+        # Update info_data with the new data
+        deep_update(info_data, new_info_data)
 
     return info_data