Add config.h and rules.mk support for data driven keymaps (#12859)

* Add config.h and rules.mk support for data driven keymaps

* tidy up after rebase

* Rename key as it can contain more than just keyboard overrides

* tidy up after rebase

* Add validation

10 files changed, 101 insertions, 69 deletions

diff --git a/build_json.mk b/build_json.mk
index 28659f260..0c034eb2a 100644
--- a/build_json.mk
+++ b/build_json.mk
@@ -1,31 +1,17 @@
 # Look for a json keymap file
 ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_5)/keymap.json)","")
-    KEYMAP_C := $(KEYMAP_OUTPUT)/keymap.c
     KEYMAP_JSON := $(MAIN_KEYMAP_PATH_5)/keymap.json
     KEYMAP_PATH := $(MAIN_KEYMAP_PATH_5)
 else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_4)/keymap.json)","")
-    KEYMAP_C := $(KEYMAP_OUTPUT)/keymap.c
     KEYMAP_JSON := $(MAIN_KEYMAP_PATH_4)/keymap.json
     KEYMAP_PATH := $(MAIN_KEYMAP_PATH_4)
 else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_3)/keymap.json)","")
-    KEYMAP_C := $(KEYMAP_OUTPUT)/keymap.c
     KEYMAP_JSON := $(MAIN_KEYMAP_PATH_3)/keymap.json
     KEYMAP_PATH := $(MAIN_KEYMAP_PATH_3)
 else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_2)/keymap.json)","")
-    KEYMAP_C := $(KEYMAP_OUTPUT)/keymap.c
     KEYMAP_JSON := $(MAIN_KEYMAP_PATH_2)/keymap.json
     KEYMAP_PATH := $(MAIN_KEYMAP_PATH_2)
 else ifneq ("$(wildcard $(MAIN_KEYMAP_PATH_1)/keymap.json)","")
-    KEYMAP_C := $(KEYMAP_OUTPUT)/keymap.c
     KEYMAP_JSON := $(MAIN_KEYMAP_PATH_1)/keymap.json
     KEYMAP_PATH := $(MAIN_KEYMAP_PATH_1)
 endif
-
-# Load the keymap-level rules.mk if exists
-ifneq ("$(wildcard $(KEYMAP_PATH))", "")
-    -include $(KEYMAP_PATH)/rules.mk
-endif
-
-# Generate the keymap.c
-$(KEYMAP_C): $(KEYMAP_JSON)
-        $(QMK_BIN) json2c --quiet --output $(KEYMAP_C) $(KEYMAP_JSON)
diff --git a/build_keyboard.mk b/build_keyboard.mk
index 46d1e4566..daef76080 100644
--- a/build_keyboard.mk
+++ b/build_keyboard.mk
@@ -23,6 +23,15 @@ KEYBOARD_OUTPUT := $(BUILD_DIR)/obj_$(KEYBOARD_FILESAFE)
 # Force expansion
 TARGET := $(TARGET)
 
+ifneq ($(FORCE_LAYOUT),)
+    TARGET := $(TARGET)_$(FORCE_LAYOUT)
+endif
+
+# Object files and generated keymap directory
+#     To put object files in current directory, use a dot (.), do NOT make
+#     this an empty or blank macro!
+KEYMAP_OUTPUT := $(BUILD_DIR)/obj_$(TARGET)
+
 # For split boards we need to set a master half.
 MASTER ?= left
 ifdef master
@@ -100,18 +109,9 @@ MAIN_KEYMAP_PATH_4 := $(KEYBOARD_PATH_4)/keymaps/$(KEYMAP)
 MAIN_KEYMAP_PATH_5 := $(KEYBOARD_PATH_5)/keymaps/$(KEYMAP)
 
 # Pull in rules from info.json
-INFO_RULES_MK = $(shell $(QMK_BIN) generate-rules-mk --quiet --escape --keyboard $(KEYBOARD) --output $(KEYBOARD_OUTPUT)/src/rules.mk)
+INFO_RULES_MK = $(shell $(QMK_BIN) generate-rules-mk --quiet --escape --keyboard $(KEYBOARD) --output $(KEYBOARD_OUTPUT)/src/info_rules.mk)
 include $(INFO_RULES_MK)
 
-ifneq ($(FORCE_LAYOUT),)
-    TARGET := $(TARGET)_$(FORCE_LAYOUT)
-endif
-
-# Object files and generated keymap directory
-#     To put object files in current directory, use a dot (.), do NOT make
-#     this an empty or blank macro!
-KEYMAP_OUTPUT := $(BUILD_DIR)/obj_$(TARGET)
-
 # Check for keymap.json first, so we can regenerate keymap.c
 include build_json.mk
 
@@ -146,6 +146,29 @@ ifeq ("$(wildcard $(KEYMAP_PATH))", "")
     endif
 endif
 
+# Have we found a keymap.json?
+ifneq ("$(wildcard $(KEYMAP_JSON))", "")
+    KEYMAP_C := $(KEYMAP_OUTPUT)/keymap.c
+    KEYMAP_H := $(KEYMAP_OUTPUT)/config.h
+
+    # Load the keymap-level rules.mk if exists
+    -include $(KEYMAP_PATH)/rules.mk
+
+    # Load any rules.mk content from keymap.json
+    INFO_RULES_MK = $(shell (QMK_BIN) generate-rules-mk --quiet --escape --keyboard $(KEYBOARD) --keymap $(KEYMAP) --output $(KEYMAP_OUTPUT)/rules.mk)
+    include $(INFO_RULES_MK)
+
+# Add rules to enerate the keymap files - indentation here is important
+$(KEYMAP_OUTPUT)/keymap.c: $(KEYMAP_JSON)
+        (QMK_BIN) json2c --quiet --output $(KEYMAP_C) $(KEYMAP_JSON)
+
+$(KEYMAP_OUTPUT)/config.h: $(KEYMAP_JSON)
+        (QMK_BIN) generate-config-h --quiet --keyboard $(KEYBOARD) --keymap $(KEYMAP) --output $(KEYMAP_OUTPUT)/config.h
+
+generated-files: $(KEYMAP_OUTPUT)/config.h $(KEYMAP_OUTPUT)/keymap.c
+
+endif
+
 ifeq ($(strip $(CTPC)), yes)
   CONVERT_TO_PROTON_C=yes
 endif
@@ -336,6 +359,9 @@ endif
 ifneq ("$(wildcard $(KEYMAP_PATH)/config.h)","")
     CONFIG_H += $(KEYMAP_PATH)/config.h
 endif
+ifneq ("$(KEYMAP_H)","")
+    CONFIG_H += $(KEYMAP_H)
+endif
 
 # project specific files
 SRC += \
diff --git a/build_layout.mk b/build_layout.mk
index 649dfb2c7..b4b00793e 100644
--- a/build_layout.mk
+++ b/build_layout.mk
@@ -7,7 +7,6 @@ define SEARCH_LAYOUTS_REPO
     LAYOUT_KEYMAP_C := $$(LAYOUT_KEYMAP_PATH)/keymap.c
     ifneq ("$$(wildcard $$(LAYOUT_KEYMAP_JSON))","")
         -include $$(LAYOUT_KEYMAP_PATH)/rules.mk
-        KEYMAP_C := $(KEYMAP_OUTPUT)/keymap.c
         KEYMAP_JSON := $$(LAYOUT_KEYMAP_JSON)
         KEYMAP_PATH := $$(LAYOUT_KEYMAP_PATH)
     else ifneq ("$$(wildcard $$(LAYOUT_KEYMAP_C))","")
@@ -31,7 +30,3 @@ ifneq ($(FORCE_LAYOUT),)
 endif
 
 $(foreach LAYOUT,$(LAYOUTS),$(eval $(call SEARCH_LAYOUTS)))
-
-# Use rule from build_json.mk, but update prerequisite in case KEYMAP_JSON was updated
-$(KEYMAP_C): $(KEYMAP_JSON)
-        $(QMK_BIN) json2c --quiet --output $(KEYMAP_C) $(KEYMAP_JSON)
diff --git a/data/mappings/info_config.json b/data/mappings/info_config.json
index 18477f912..72bb0f4a1 100644
--- a/data/mappings/info_config.json
+++ b/data/mappings/info_config.json
@@ -18,6 +18,8 @@
     "DESCRIPTION": {"info_key": "keyboard_folder", "to_json": false},
     "DIODE_DIRECTION": {"info_key": "diode_direction"},
     "FORCE_NKRO": {"info_key": "usb.force_nkro", "value_type": "bool"},
+    "DYNAMIC_KEYMAP_EEPROM_MAX_ADDR": {"info_key": "dynamic_keymap.eeprom_max_addr", "value_type": "int"},
+    "DYNAMIC_KEYMAP_LAYER_COUNT": {"info_key": "dynamic_keymap.layer_count", "value_type": "int"},
     "IGNORE_MOD_TAP_INTERRUPT": {"info_key": "tapping.ignore_mod_tap_interrupt", "value_type": "bool"},
     "IGNORE_MOD_TAP_INTERRUPT_PER_KEY": {"info_key": "tapping.ignore_mod_tap_interrupt_per_key", "value_type": "bool"},
     "LAYOUTS": {"info_key": "layout_aliases", "value_type": "mapping"},
diff --git a/data/schemas/keymap.jsonschema b/data/schemas/keymap.jsonschema
new file mode 100644
index 000000000..35c5b5c98
--- /dev/null
+++ b/data/schemas/keymap.jsonschema
@@ -0,0 +1,24 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$id": "qmk.keymap.v1",
+    "title": "Keymap Information",
+    "type": "object",
+    "properties": {
+        "author": {"$ref": "qmk.definitions.v1#/text_identifier"},
+        "keyboard": {"$ref": "qmk.definitions.v1#/text_identifier"},
+        "keymap": {"$ref": "qmk.definitions.v1#/text_identifier"},
+        "layout": {"$ref": "qmk.definitions.v1#/layout_macro"},
+        "layers": {
+            "type": "array",
+            "items": {
+                "type": "array",
+                "items": {"type": "string"}
+            }
+        },
+        "config": {"$ref": "qmk.keyboard.v1"},
+        "notes": {
+            "type": "string",
+            "description": "asdf"
+        }
+    }
+}
\ No newline at end of file
diff --git a/keyboards/thevankeyboards/minivan/keymaps/via/keymap.json b/keyboards/thevankeyboards/minivan/keymaps/via/keymap.json
index 1e9ef8810..54ceb5c34 100644
--- a/keyboards/thevankeyboards/minivan/keymaps/via/keymap.json
+++ b/keyboards/thevankeyboards/minivan/keymaps/via/keymap.json
@@ -7,5 +7,14 @@
     ["KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS"],
     ["KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS"],
     ["KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS", "KC_TRNS"]
-  ]
+  ],
+  "config": {
+    "features": {
+      "via": true,
+      "lto": true
+    },
+    "dynamic_keymap": {
+      "layer_count": 4
+    }
+  }
 }
diff --git a/keyboards/thevankeyboards/minivan/keymaps/via/rules.mk b/keyboards/thevankeyboards/minivan/keymaps/via/rules.mk
deleted file mode 100644
index 36b7ba9cb..000000000
--- a/keyboards/thevankeyboards/minivan/keymaps/via/rules.mk
+++ /dev/null
@@ -1,2 +0,0 @@
-VIA_ENABLE = yes
-LTO_ENABLE = yes
diff --git a/lib/python/qmk/cli/generate/config_h.py b/lib/python/qmk/cli/generate/config_h.py
index c0c148f1c..ca7e14fe6 100755
--- a/lib/python/qmk/cli/generate/config_h.py
+++ b/lib/python/qmk/cli/generate/config_h.py
@@ -5,11 +5,11 @@ from pathlib import Path
 from dotty_dict import dotty
 from milc import cli
 
-from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.info import info_json
-from qmk.json_schema import json_load
+from qmk.json_schema import json_load, validate
 from qmk.keyboard import keyboard_completer, keyboard_folder
-from qmk.path import is_keyboard, normpath
+from qmk.keymap import locate_keymap
+from qmk.path import normpath
 
 
 def direct_pins(direct_pins, postfix):
@@ -157,25 +157,22 @@ def generate_split_config(kb_info_json, config_h_lines):
 
 @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
 @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
-@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='Keyboard to generate config.h for.')
+@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='Keyboard to generate config.h for.')
+@cli.argument('-km', '--keymap', arg_only=True, help='Keymap to generate config.h for.')
 @cli.subcommand('Used by the make system to generate info_config.h from info.json', hidden=True)
-@automagic_keyboard
-@automagic_keymap
 def generate_config_h(cli):
     """Generates the info_config.h file.
     """
-    # Determine our keyboard(s)
-    if not cli.config.generate_config_h.keyboard:
-        cli.log.error('Missing parameter: --keyboard')
-        cli.subcommands['info'].print_help()
-        return False
-
-    if not is_keyboard(cli.config.generate_config_h.keyboard):
-        cli.log.error('Invalid keyboard: "%s"', cli.config.generate_config_h.keyboard)
-        return False
+    # Determine our keyboard/keymap
+    if cli.args.keymap:
+        km = locate_keymap(cli.args.keyboard, cli.args.keymap)
+        km_json = json_load(km)
+        validate(km_json, 'qmk.keymap.v1')
+        kb_info_json = dotty(km_json.get('config', {}))
+    else:
+        kb_info_json = dotty(info_json(cli.args.keyboard))
 
     # Build the info_config.h file.
-    kb_info_json = dotty(info_json(cli.config.generate_config_h.keyboard))
     config_h_lines = ['/* This file was generated by `qmk generate-config-h`. Do not edit or copy.' ' */', '', '#pragma once']
 
     generate_config_items(kb_info_json, config_h_lines)
diff --git a/lib/python/qmk/cli/generate/keyboard_h.py b/lib/python/qmk/cli/generate/keyboard_h.py
index 22500dbc9..c9d7f549b 100755
--- a/lib/python/qmk/cli/generate/keyboard_h.py
+++ b/lib/python/qmk/cli/generate/keyboard_h.py
@@ -2,7 +2,6 @@
 """
 from milc import cli
 
-from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.info import info_json
 from qmk.keyboard import keyboard_completer, keyboard_folder
 from qmk.path import normpath
@@ -29,14 +28,12 @@ def would_populate_layout_h(keyboard):
 
 @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
 @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
-@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, required=True, help='Keyboard to generate keyboard.h for.')
+@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='Keyboard to generate keyboard.h for.')
 @cli.subcommand('Used by the make system to generate keyboard.h from info.json', hidden=True)
-@automagic_keyboard
-@automagic_keymap
 def generate_keyboard_h(cli):
     """Generates the keyboard.h file.
     """
-    has_layout_h = would_populate_layout_h(cli.config.generate_keyboard_h.keyboard)
+    has_layout_h = would_populate_layout_h(cli.args.keyboard)
 
     # Build the layouts.h file.
     keyboard_h_lines = ['/* This file was generated by `qmk generate-keyboard-h`. Do not edit or copy.' ' */', '', '#pragma once', '#include "quantum.h"']
diff --git a/lib/python/qmk/cli/generate/rules_mk.py b/lib/python/qmk/cli/generate/rules_mk.py
index 2712b81cb..cdf17dfbc 100755
--- a/lib/python/qmk/cli/generate/rules_mk.py
+++ b/lib/python/qmk/cli/generate/rules_mk.py
@@ -5,11 +5,11 @@ from pathlib import Path
 from dotty_dict import dotty
 from milc import cli
 
-from qmk.decorators import automagic_keyboard, automagic_keymap
 from qmk.info import info_json
-from qmk.json_schema import json_load
+from qmk.json_schema import json_load, validate
 from qmk.keyboard import keyboard_completer, keyboard_folder
-from qmk.path import is_keyboard, normpath
+from qmk.keymap import locate_keymap
+from qmk.path import normpath
 
 
 def process_mapping_rule(kb_info_json, rules_key, info_dict):
@@ -39,23 +39,21 @@ def process_mapping_rule(kb_info_json, rules_key, info_dict):
 @cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
 @cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
 @cli.argument('-e', '--escape', arg_only=True, action='store_true', help="Escape spaces in quiet mode")
-@cli.argument('-kb', '--keyboard', type=keyboard_folder, completer=keyboard_completer, help='Keyboard to generate config.h for.')
-@cli.subcommand('Used by the make system to generate info_config.h from info.json', hidden=True)
-@automagic_keyboard
-@automagic_keymap
+@cli.argument('-kb', '--keyboard', arg_only=True, type=keyboard_folder, completer=keyboard_completer, required=True, help='Keyboard to generate rules.mk for.')
+@cli.argument('-km', '--keymap', arg_only=True, help='Keymap to generate rules.mk for.')
+@cli.subcommand('Used by the make system to generate rules.mk from info.json', hidden=True)
 def generate_rules_mk(cli):
     """Generates a rules.mk file from info.json.
     """
-    if not cli.config.generate_rules_mk.keyboard:
-        cli.log.error('Missing parameter: --keyboard')
-        cli.subcommands['info'].print_help()
-        return False
-
-    if not is_keyboard(cli.config.generate_rules_mk.keyboard):
-        cli.log.error('Invalid keyboard: "%s"', cli.config.generate_rules_mk.keyboard)
-        return False
+    # Determine our keyboard/keymap
+    if cli.args.keymap:
+        km = locate_keymap(cli.args.keyboard, cli.args.keymap)
+        km_json = json_load(km)
+        validate(km_json, 'qmk.keymap.v1')
+        kb_info_json = dotty(km_json.get('config', {}))
+    else:
+        kb_info_json = dotty(info_json(cli.args.keyboard))
 
-    kb_info_json = dotty(info_json(cli.config.generate_rules_mk.keyboard))
     info_rules_map = json_load(Path('data/mappings/info_rules.json'))
     rules_mk_lines = ['# This file was generated by `qmk generate-rules-mk`. Do not edit or copy.', '']