[CLI] Add c2json (#8817)

* Basic keymap parsing finally works

* Add 'keymap.json' creation to the qmk.keymap module

* Add tests and fix formatting

* Fix/exclude flake8 errors

* Convert keymap.c to valid keymap.json

* Fix some errors

* Add tests

* Finalize keymap.json creation, add json template

* Add docs

* Move pygments to the standard requirements

* Add support for nameless layers, fix tests

* Fix things after rebase

* Add missing 'keymap' value.

* Fix missing layer numbers from advanced keycodes

Buckwich noticed that if the advanced keycode / layer toggling key
contains a number, it goes missing.
Now we properly handle them.
Thx for noticing!

* Apply suggestions from code review

* fixup tests

Co-authored-by: Zach White <skullydazed@drpepper.org>
Co-authored-by: skullY <skullydazed@gmail.com>

7 files changed, 355 insertions, 35 deletions

diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
index 47f60c601..ba964ebbb 100644
--- a/lib/python/qmk/cli/__init__.py
+++ b/lib/python/qmk/cli/__init__.py
@@ -6,6 +6,7 @@ import sys
 
 from milc import cli
 
+from . import c2json
 from . import cformat
 from . import compile
 from . import config
diff --git a/lib/python/qmk/cli/c2json.py b/lib/python/qmk/cli/c2json.py
new file mode 100644
index 000000000..0267303fd
--- /dev/null
+++ b/lib/python/qmk/cli/c2json.py
@@ -0,0 +1,62 @@
+"""Generate a keymap.json from a keymap.c file.
+"""
+import json
+import sys
+
+from milc import cli
+
+import qmk.keymap
+import qmk.path
+
+
+@cli.argument('--no-cpp', arg_only=True, action='store_false', help='Do not use \'cpp\' on keymap.c')
+@cli.argument('-o', '--output', arg_only=True, type=qmk.path.normpath, help='File to write to')
+@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
+@cli.argument('-kb', '--keyboard', arg_only=True, required=True, help='The keyboard\'s name')
+@cli.argument('-km', '--keymap', arg_only=True, required=True, help='The keymap\'s name')
+@cli.argument('filename', arg_only=True, help='keymap.c file')
+@cli.subcommand('Creates a keymap.json from a keymap.c file.')
+def c2json(cli):
+    """Generate a keymap.json from a keymap.c file.
+
+    This command uses the `qmk.keymap` module to generate a keymap.json from a keymap.c file. The generated keymap is written to stdout, or to a file if -o is provided.
+    """
+    cli.args.filename = qmk.path.normpath(cli.args.filename)
+
+    # Error checking
+    if not cli.args.filename.exists():
+        cli.log.error('C file does not exist!')
+        cli.print_usage()
+        exit(1)
+
+    if str(cli.args.filename) == '-':
+        # TODO(skullydazed/anyone): Read file contents from STDIN
+        cli.log.error('Reading from STDIN is not (yet) supported.')
+        cli.print_usage()
+        exit(1)
+
+    # Environment processing
+    if cli.args.output == ('-'):
+        cli.args.output = None
+
+    # Parse the keymap.c
+    keymap_json = qmk.keymap.c2json(cli.args.keyboard, cli.args.keymap, cli.args.filename, use_cpp=cli.args.no_cpp)
+
+    # Generate the keymap.json
+    try:
+        keymap_json = qmk.keymap.generate(keymap_json['keyboard'], keymap_json['layout'], keymap_json['layers'], type='json', keymap=keymap_json['keymap'])
+    except KeyError:
+        cli.log.error('Something went wrong. Try to use --no-cpp.')
+        sys.exit(1)
+
+    if cli.args.output:
+        cli.args.output.parent.mkdir(parents=True, exist_ok=True)
+        if cli.args.output.exists():
+            cli.args.output.replace(cli.args.output.name + '.bak')
+        cli.args.output.write_text(json.dumps(keymap_json))
+
+        if not cli.args.quiet:
+            cli.log.info('Wrote keymap to %s.', cli.args.output)
+
+    else:
+        print(json.dumps(keymap_json))
diff --git a/lib/python/qmk/cli/doctor.py b/lib/python/qmk/cli/doctor.py
index 984c308d1..7fafd5757 100755
--- a/lib/python/qmk/cli/doctor.py
+++ b/lib/python/qmk/cli/doctor.py
@@ -58,7 +58,7 @@ def parse_gcc_version(version):
     return {
         'major': int(m.group(1)),
         'minor': int(m.group(2)) if m.group(2) else 0,
-        'patch': int(m.group(3)) if m.group(3) else 0
+        'patch': int(m.group(3)) if m.group(3) else 0,
     }
 
 
diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py
index 4db4667a8..5a6e60988 100644
--- a/lib/python/qmk/commands.py
+++ b/lib/python/qmk/commands.py
@@ -7,7 +7,6 @@ import subprocess
 import shlex
 import shutil
 
-from milc import cli
 import qmk.keymap
 
 
@@ -84,6 +83,4 @@ def run(command, *args, **kwargs):
         safecmd = ' '.join(safecmd)
         command = [os.environ['SHELL'], '-c', safecmd]
 
-    cli.log.debug('Running command: %s', command)
-
     return subprocess.run(command, *args, **kwargs)
diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py
index 78510a8a7..2b271fe80 100644
--- a/lib/python/qmk/keymap.py
+++ b/lib/python/qmk/keymap.py
@@ -1,11 +1,18 @@
 """Functions that help you work with QMK keymaps.
 """
 from pathlib import Path
+import json
+import subprocess
+
+from pygments.lexers.c_cpp import CLexer
+from pygments.token import Token
+from pygments import lex
 
 from milc import cli
 
 from qmk.keyboard import rules_mk
 import qmk.path
+import qmk.commands
 
 # The `keymap.c` template to use when a keyboard doesn't have its own
 DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H
@@ -22,22 +29,35 @@ __KEYMAP_GOES_HERE__
 """
 
 
-def template(keyboard):
-    """Returns the `keymap.c` template for a keyboard.
+def template(keyboard, type='c'):
+    """Returns the `keymap.c` or `keymap.json` template for a keyboard.
 
     If a template exists in `keyboards/<keyboard>/templates/keymap.c` that
     text will be used instead of `DEFAULT_KEYMAP_C`.
 
+    If a template exists in `keyboards/<keyboard>/templates/keymap.json` that
+    text will be used instead of an empty dictionary.
+
     Args:
         keyboard
             The keyboard to return a template for.
-    """
-    template_file = Path('keyboards/%s/templates/keymap.c' % keyboard)
 
-    if template_file.exists():
-        return template_file.read_text()
+        type
+            'json' for `keymap.json` and 'c' (or anything else) for `keymap.c`
+    """
+    if type == 'json':
+        template_file = Path('keyboards/%s/templates/keymap.json' % keyboard)
+        template = {'keyboard': keyboard}
+        if template_file.exists():
+            template.update(json.loads(template_file.read_text()))
+    else:
+        template_file = Path('keyboards/%s/templates/keymap.c' % keyboard)
+        if template_file.exists():
+            template = template_file.read_text()
+        else:
+            template = DEFAULT_KEYMAP_C
 
-    return DEFAULT_KEYMAP_C
+    return template
 
 
 def _strip_any(keycode):
@@ -57,8 +77,8 @@ def is_keymap_dir(keymap):
             return True
 
 
-def generate(keyboard, layout, layers):
-    """Returns a keymap.c for the specified keyboard, layout, and layers.
+def generate(keyboard, layout, layers, type='c', keymap=None):
+    """Returns a `keymap.c` or `keymap.json` for the specified keyboard, layout, and layers.
 
     Args:
         keyboard
@@ -69,24 +89,30 @@ def generate(keyboard, layout, layers):
 
         layers
             An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
-    """
-    layer_txt = []
-
-    for layer_num, layer in enumerate(layers):
-        if layer_num != 0:
-            layer_txt[-1] = layer_txt[-1] + ','
 
-        layer = map(_strip_any, layer)
-        layer_keys = ', '.join(layer)
-        layer_txt.append('\t[%s] = %s(%s)' % (layer_num, layout, layer_keys))
-
-    keymap = '\n'.join(layer_txt)
-    keymap_c = template(keyboard)
-
-    return keymap_c.replace('__KEYMAP_GOES_HERE__', keymap)
-
-
-def write(keyboard, keymap, layout, layers):
+        type
+            'json' for `keymap.json` and 'c' (or anything else) for `keymap.c`
+    """
+    new_keymap = template(keyboard, type)
+    if type == 'json':
+        new_keymap['keymap'] = keymap
+        new_keymap['layout'] = layout
+        new_keymap['layers'] = layers
+    else:
+        layer_txt = []
+        for layer_num, layer in enumerate(layers):
+            if layer_num != 0:
+                layer_txt[-1] = layer_txt[-1] + ','
+            layer_keys = ', '.join(layer)
+            layer_txt.append('\t[%s] = %s(%s)' % (layer_num, layout, layer_keys))
+
+        keymap = '\n'.join(layer_txt)
+        new_keymap = new_keymap.replace('__KEYMAP_GOES_HERE__', keymap)
+
+    return new_keymap
+
+
+def write(keyboard, keymap, layout, layers, type='c'):
     """Generate the `keymap.c` and write it to disk.
 
     Returns the filename written to.
@@ -103,12 +129,19 @@ def write(keyboard, keymap, layout, layers):
 
         layers
             An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
+
+        type
+            'json' for `keymap.json` and 'c' (or anything else) for `keymap.c`
     """
-    keymap_c = generate(keyboard, layout, layers)
-    keymap_file = qmk.path.keymap(keyboard) / keymap / 'keymap.c'
+    keymap_content = generate(keyboard, layout, layers, type)
+    if type == 'json':
+        keymap_file = qmk.path.keymap(keyboard) / keymap / 'keymap.json'
+        keymap_content = json.dumps(keymap_content)
+    else:
+        keymap_file = qmk.path.keymap(keyboard) / keymap / 'keymap.c'
 
     keymap_file.parent.mkdir(parents=True, exist_ok=True)
-    keymap_file.write_text(keymap_c)
+    keymap_file.write_text(keymap_content)
 
     cli.log.info('Wrote keymap to {fg_cyan}%s', keymap_file)
 
@@ -188,3 +221,192 @@ def list_keymaps(keyboard):
                     names = names.union([keymap.name for keymap in cl_path.iterdir() if is_keymap_dir(keymap)])
 
     return sorted(names)
+
+
+def _c_preprocess(path):
+    """ Run a file through the C pre-processor
+
+    Args:
+        path: path of the keymap.c file
+
+    Returns:
+        the stdout of the pre-processor
+    """
+    pre_processed_keymap = qmk.commands.run(['cpp', path], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+    return pre_processed_keymap.stdout
+
+
+def _get_layers(keymap):  # noqa C901 : until someone has a good idea how to simplify/split up this code
+    """ Find the layers in a keymap.c file.
+
+    Args:
+        keymap: the content of the keymap.c file
+
+    Returns:
+        a dictionary containing the parsed keymap
+    """
+    layers = list()
+    opening_braces = '({['
+    closing_braces = ')}]'
+    keymap_certainty = brace_depth = 0
+    is_keymap = is_layer = is_adv_kc = False
+    layer = dict(name=False, layout=False, keycodes=list())
+    for line in lex(keymap, CLexer()):
+        if line[0] is Token.Name:
+            if is_keymap:
+                # If we are inside the keymap array
+                # we know the keymap's name and the layout macro will come,
+                # followed by the keycodes
+                if not layer['name']:
+                    if line[1].startswith('LAYOUT') or line[1].startswith('KEYMAP'):
+                        # This can happen if the keymap array only has one layer,
+                        # for macropads and such
+                        layer['name'] = '0'
+                        layer['layout'] = line[1]
+                    else:
+                        layer['name'] = line[1]
+                elif not layer['layout']:
+                    layer['layout'] = line[1]
+                elif is_layer:
+                    # If we are inside a layout macro,
+                    # collect all keycodes
+                    if line[1] == '_______':
+                        kc = 'KC_TRNS'
+                    elif line[1] == 'XXXXXXX':
+                        kc = 'KC_NO'
+                    else:
+                        kc = line[1]
+                    if is_adv_kc:
+                        # If we are inside an advanced keycode
+                        # collect everything and hope the user
+                        # knew what he/she was doing
+                        layer['keycodes'][-1] += kc
+                    else:
+                        layer['keycodes'].append(kc)
+
+        # The keymaps array's signature:
+        # const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS]
+        #
+        # Only if we've found all 6 keywords in this specific order
+        # can we know for sure that we are inside the keymaps array
+            elif line[1] == 'PROGMEM' and keymap_certainty == 2:
+                keymap_certainty = 3
+            elif line[1] == 'keymaps' and keymap_certainty == 3:
+                keymap_certainty = 4
+            elif line[1] == 'MATRIX_ROWS' and keymap_certainty == 4:
+                keymap_certainty = 5
+            elif line[1] == 'MATRIX_COLS' and keymap_certainty == 5:
+                keymap_certainty = 6
+        elif line[0] is Token.Keyword:
+            if line[1] == 'const' and keymap_certainty == 0:
+                keymap_certainty = 1
+        elif line[0] is Token.Keyword.Type:
+            if line[1] == 'uint16_t' and keymap_certainty == 1:
+                keymap_certainty = 2
+        elif line[0] is Token.Punctuation:
+            if line[1] in opening_braces:
+                brace_depth += 1
+                if is_keymap:
+                    if is_layer:
+                        # We found the beginning of a non-basic keycode
+                        is_adv_kc = True
+                        layer['keycodes'][-1] += line[1]
+                    elif line[1] == '(' and brace_depth == 2:
+                        # We found the beginning of a layer
+                        is_layer = True
+                elif line[1] == '{' and keymap_certainty == 6:
+                    # We found the beginning of the keymaps array
+                    is_keymap = True
+            elif line[1] in closing_braces:
+                brace_depth -= 1
+                if is_keymap:
+                    if is_adv_kc:
+                        layer['keycodes'][-1] += line[1]
+                        if brace_depth == 2:
+                            # We found the end of a non-basic keycode
+                            is_adv_kc = False
+                    elif line[1] == ')' and brace_depth == 1:
+                        # We found the end of a layer
+                        is_layer = False
+                        layers.append(layer)
+                        layer = dict(name=False, layout=False, keycodes=list())
+                    elif line[1] == '}' and brace_depth == 0:
+                        # We found the end of the keymaps array
+                        is_keymap = False
+                        keymap_certainty = 0
+            elif is_adv_kc:
+                # Advanced keycodes can contain other punctuation
+                # e.g.: MT(MOD_LCTL | MOD_LSFT, KC_ESC)
+                layer['keycodes'][-1] += line[1]
+
+        elif line[0] is Token.Literal.Number.Integer and is_keymap and not is_adv_kc:
+            # If the pre-processor finds the 'meaning' of the layer names,
+            # they will be numbers
+            if not layer['name']:
+                layer['name'] = line[1]
+
+        else:
+            # We only care about
+            # operators and such if we
+            # are inside an advanced keycode
+            # e.g.: MT(MOD_LCTL | MOD_LSFT, KC_ESC)
+            if is_adv_kc:
+                layer['keycodes'][-1] += line[1]
+
+    return layers
+
+
+def parse_keymap_c(keymap_file, use_cpp=True):
+    """ Parse a keymap.c file.
+
+    Currently only cares about the keymaps array.
+
+    Args:
+        keymap_file: path of the keymap.c file
+
+        use_cpp: if True, pre-process the file with the C pre-processor
+
+    Returns:
+        a dictionary containing the parsed keymap
+    """
+    if use_cpp:
+        keymap_file = _c_preprocess(keymap_file)
+    else:
+        keymap_file = keymap_file.read_text()
+
+    keymap = dict()
+    keymap['layers'] = _get_layers(keymap_file)
+    return keymap
+
+
+def c2json(keyboard, keymap, keymap_file, use_cpp=True):
+    """ Convert keymap.c to keymap.json
+
+    Args:
+        keyboard: The name of the keyboard
+
+        keymap: The name of the keymap
+
+        layout: The LAYOUT macro this keymap uses.
+
+        keymap_file: path of the keymap.c file
+
+        use_cpp: if True, pre-process the file with the C pre-processor
+
+    Returns:
+        a dictionary in keymap.json format
+    """
+    keymap_json = parse_keymap_c(keymap_file, use_cpp)
+
+    dirty_layers = keymap_json.pop('layers', None)
+    keymap_json['layers'] = list()
+    for layer in dirty_layers:
+        layer.pop('name')
+        layout = layer.pop('layout')
+        if not keymap_json.get('layout', False):
+            keymap_json['layout'] = layout
+        keymap_json['layers'].append(layer.pop('keycodes'))
+
+    keymap_json['keyboard'] = keyboard
+    keymap_json['keymap'] = keymap
+    return keymap_json
diff --git a/lib/python/qmk/tests/test_cli_commands.py b/lib/python/qmk/tests/test_cli_commands.py
index 0b840b466..7ac0bcbde 100644
--- a/lib/python/qmk/tests/test_cli_commands.py
+++ b/lib/python/qmk/tests/test_cli_commands.py
@@ -1,10 +1,11 @@
-import subprocess
+from subprocess import STDOUT, PIPE
+
 from qmk.commands import run
 
 
 def check_subcommand(command, *args):
     cmd = ['bin/qmk', command] + list(args)
-    result = run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True)
+    result = run(cmd, stdout=PIPE, stderr=STDOUT, universal_newlines=True)
     return result
 
 
@@ -28,6 +29,11 @@ def test_compile():
     check_returncode(result)
 
 
+def test_compile_json():
+    result = check_subcommand('compile', '-kb', 'handwired/onekey/pytest', '-km', 'default_json')
+    check_returncode(result)
+
+
 def test_flash():
     result = check_subcommand('flash', '-kb', 'handwired/onekey/pytest', '-km', 'default', '-n')
     check_returncode(result)
@@ -153,3 +159,15 @@ def test_info_matrix_render():
     assert 'LAYOUT_ortho_1x1' in result.stdout
     assert '│0A│' in result.stdout
     assert 'Matrix for "LAYOUT_ortho_1x1"' in result.stdout
+
+
+def test_c2json():
+    result = check_subcommand("c2json", "-kb", "handwired/onekey/pytest", "-km", "default", "keyboards/handwired/onekey/keymaps/default/keymap.c")
+    check_returncode(result)
+    assert result.stdout.strip() == '{"keyboard": "handwired/onekey/pytest", "documentation": "This file is a keymap.json file for handwired/onekey/pytest", "keymap": "default", "layout": "LAYOUT_ortho_1x1", "layers": [["KC_A"]]}'
+
+
+def test_c2json_nocpp():
+    result = check_subcommand("c2json", "--no-cpp", "-kb", "handwired/onekey/pytest", "-km", "default", "keyboards/handwired/onekey/keymaps/pytest_nocpp/keymap.c")
+    check_returncode(result)
+    assert result.stdout.strip() == '{"keyboard": "handwired/onekey/pytest", "documentation": "This file is a keymap.json file for handwired/onekey/pytest", "keymap": "default", "layout": "LAYOUT", "layers": [["KC_ENTER"]]}'
diff --git a/lib/python/qmk/tests/test_qmk_keymap.py b/lib/python/qmk/tests/test_qmk_keymap.py
index d8669e549..7ef708e0d 100644
--- a/lib/python/qmk/tests/test_qmk_keymap.py
+++ b/lib/python/qmk/tests/test_qmk_keymap.py
@@ -6,14 +6,34 @@ def test_template_onekey_proton_c():
     assert templ == qmk.keymap.DEFAULT_KEYMAP_C
 
 
+def test_template_onekey_proton_c_json():
+    templ = qmk.keymap.template('handwired/onekey/proton_c', type='json')
+    assert templ == {'keyboard': 'handwired/onekey/proton_c'}
+
+
 def test_template_onekey_pytest():
     templ = qmk.keymap.template('handwired/onekey/pytest')
     assert templ == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {__KEYMAP_GOES_HERE__};\n'
 
 
+def test_template_onekey_pytest_json():
+    templ = qmk.keymap.template('handwired/onekey/pytest', type='json')
+    assert templ == {'keyboard': 'handwired/onekey/pytest', "documentation": "This file is a keymap.json file for handwired/onekey/pytest"}
+
+
 def test_generate_onekey_pytest():
     templ = qmk.keymap.generate('handwired/onekey/pytest', 'LAYOUT', [['KC_A']])
     assert templ == '#include QMK_KEYBOARD_H\nconst uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {\t[0] = LAYOUT(KC_A)};\n'
 
 
+def test_generate_onekey_pytest_json():
+    templ = qmk.keymap.generate('handwired/onekey/pytest', 'LAYOUT', [['KC_A']], type='json', keymap='default')
+    assert templ == {"keyboard": "handwired/onekey/pytest", "documentation": "This file is a keymap.json file for handwired/onekey/pytest", "keymap": "default", "layout": "LAYOUT", "layers": [["KC_A"]]}
+
+
+def test_parse_keymap_c():
+    parsed_keymap_c = qmk.keymap.parse_keymap_c('keyboards/handwired/onekey/keymaps/default/keymap.c')
+    assert parsed_keymap_c == {'layers': [{'name': '0', 'layout': 'LAYOUT_ortho_1x1', 'keycodes': ['KC_A']}]}
+
+
 # FIXME(skullydazed): Add a test for qmk.keymap.write that mocks up an FD.