QMK CLI and JSON keymap support (#6176)

* Script to generate keymap.c from JSON file.

* Support for keymap.json

* Add a warning about the keymap.c getting overwritten.

* Fix keymap generating

* Install the python deps

* Flesh out more of the python environment

* Remove defunct json2keymap

* Style everything with yapf

* Polish up python support

* Hide json keymap.c into the .build dir

* Polish up qmk-compile-json

* Make milc work with positional arguments

* Fix a couple small things

* Fix some errors and make the CLI more understandable

* Make the qmk wrapper more robust

* Add basic QMK Doctor

* Clean up docstrings and flesh them out as needed

* remove unused compile_firmware() function

10 files changed, 296 insertions, 0 deletions

diff --git a/lib/python/qmk/__init__.py b/lib/python/qmk/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/lib/python/qmk/__init__.py
diff --git a/lib/python/qmk/cli/compile/__init__.py b/lib/python/qmk/cli/compile/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/lib/python/qmk/cli/compile/__init__.py
diff --git a/lib/python/qmk/cli/compile/json.py b/lib/python/qmk/cli/compile/json.py
new file mode 100755
index 000000000..89c16b206
--- /dev/null
+++ b/lib/python/qmk/cli/compile/json.py
@@ -0,0 +1,44 @@
+"""Create a keymap directory from a configurator export.
+"""
+import json
+import os
+import sys
+import subprocess
+
+from milc import cli
+
+import qmk.keymap
+import qmk.path
+
+
+@cli.argument('filename', help='Configurator JSON export')
+@cli.entrypoint('Compile a QMK Configurator export.')
+def main(cli):
+    """Compile a QMK Configurator export.
+
+    This command creates a new keymap from a configurator export, overwriting an existing keymap if one exists.
+
+    FIXME(skullydazed): add code to check and warn if the keymap already exists
+    """
+    # Error checking
+    if cli.args.filename == ('-'):
+        cli.log.error('Reading from STDIN is not (yet) supported.')
+        exit(1)
+    if not os.path.exists(qmk.path.normpath(cli.args.filename)):
+        cli.log.error('JSON file does not exist!')
+        exit(1)
+
+    # Parse the configurator json
+    with open(qmk.path.normpath(cli.args.filename), 'r') as fd:
+        user_keymap = json.load(fd)
+
+    # Generate the keymap
+    keymap_path = qmk.path.keymap(user_keymap['keyboard'])
+    cli.log.info('Creating {fg_cyan}%s{style_reset_all} keymap in {fg_cyan}%s', user_keymap['keymap'], keymap_path)
+    qmk.keymap.write(user_keymap['keyboard'], user_keymap['keymap'], user_keymap['layout'], user_keymap['layers'])
+    cli.log.info('Wrote keymap to {fg_cyan}%s/%s/keymap.c', keymap_path, user_keymap['keymap'])
+
+    # Compile the keymap
+    command = ['make', ':'.join((user_keymap['keyboard'], user_keymap['keymap']))]
+    cli.log.info('Compiling keymap with {fg_cyan}%s\n\n', ' '.join(command))
+    subprocess.run(command)
diff --git a/lib/python/qmk/cli/doctor.py b/lib/python/qmk/cli/doctor.py
new file mode 100755
index 000000000..9ce765a4b
--- /dev/null
+++ b/lib/python/qmk/cli/doctor.py
@@ -0,0 +1,47 @@
+"""QMK Python Doctor
+
+Check up for QMK environment.
+"""
+import shutil
+import platform
+import os
+
+from milc import cli
+
+
+@cli.entrypoint('Basic QMK environment checks')
+def main(cli):
+    """Basic QMK environment checks.
+
+    This is currently very simple, it just checks that all the expected binaries are on your system.
+
+    TODO(unclaimed):
+        * [ ] Run the binaries to make sure they work
+        * [ ] Compile a trivial program with each compiler
+        * [ ] Check for udev entries on linux
+    """
+
+    binaries = ['dfu-programmer', 'avrdude', 'dfu-util', 'avr-gcc', 'arm-none-eabi-gcc']
+
+    cli.log.info('QMK Doctor is Checking your environment')
+
+    ok = True
+    for binary in binaries:
+        res = shutil.which(binary)
+        if res is None:
+            cli.log.error('{fg_red}QMK can\'t find ' + binary + ' in your path')
+            ok = False
+
+    OS = platform.system()
+    if OS == "Darwin":
+        cli.log.info("Detected {fg_cyan}macOS")
+    elif OS == "Linux":
+        cli.log.info("Detected {fg_cyan}linux")
+        test = 'systemctl list-unit-files | grep enabled | grep -i ModemManager'
+        if os.system(test) == 0:
+            cli.log.warn("{bg_yellow}Detected modem manager. Please disable it if you are using Pro Micros")
+    else:
+        cli.log.info("Assuming {fg_cyan}Windows")
+
+    if ok:
+        cli.log.info('{fg_green}QMK is ready to go')
diff --git a/lib/python/qmk/cli/hello.py b/lib/python/qmk/cli/hello.py
new file mode 100755
index 000000000..bc0cb6de1
--- /dev/null
+++ b/lib/python/qmk/cli/hello.py
@@ -0,0 +1,13 @@
+"""QMK Python Hello World
+
+This is an example QMK CLI script.
+"""
+from milc import cli
+
+
+@cli.argument('-n', '--name', default='World', help='Name to greet.')
+@cli.entrypoint('QMK Hello World.')
+def main(cli):
+    """Log a friendly greeting.
+    """
+    cli.log.info('Hello, %s!', cli.config.general.name)
diff --git a/lib/python/qmk/cli/json/__init__.py b/lib/python/qmk/cli/json/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/lib/python/qmk/cli/json/__init__.py
diff --git a/lib/python/qmk/cli/json/keymap.py b/lib/python/qmk/cli/json/keymap.py
new file mode 100755
index 000000000..35fc8f9c0
--- /dev/null
+++ b/lib/python/qmk/cli/json/keymap.py
@@ -0,0 +1,54 @@
+"""Generate a keymap.c from a configurator export.
+"""
+import json
+import os
+import sys
+
+from milc import cli
+
+import qmk.keymap
+
+
+@cli.argument('-o', '--output', help='File to write to')
+@cli.argument('filename', help='Configurator JSON file')
+@cli.entrypoint('Create a keymap.c from a QMK Configurator export.')
+def main(cli):
+    """Generate a keymap.c from a configurator export.
+
+    This command uses the `qmk.keymap` module to generate a keymap.c from a configurator export. The generated keymap is written to stdout, or to a file if -o is provided.
+    """
+    # Error checking
+    if cli.args.filename == ('-'):
+        cli.log.error('Reading from STDIN is not (yet) supported.')
+        cli.print_usage()
+        exit(1)
+    if not os.path.exists(qmk.path.normpath(cli.args.filename)):
+        cli.log.error('JSON file does not exist!')
+        cli.print_usage()
+        exit(1)
+
+    # Environment processing
+    if cli.args.output == ('-'):
+        cli.args.output = None
+
+    # Parse the configurator json
+    with open(qmk.path.normpath(cli.args.filename), 'r') as fd:
+        user_keymap = json.load(fd)
+
+    # Generate the keymap
+    keymap_c = qmk.keymap.generate(user_keymap['keyboard'], user_keymap['layout'], user_keymap['layers'])
+
+    if cli.args.output:
+        output_dir = os.path.dirname(cli.args.output)
+
+        if not os.path.exists(output_dir):
+            os.makedirs(output_dir)
+
+        output_file = qmk.path.normpath(cli.args.output)
+        with open(output_file, 'w') as keymap_fd:
+            keymap_fd.write(keymap_c)
+
+        cli.log.info('Wrote keymap to %s.', cli.args.output)
+
+    else:
+        print(keymap_c)
diff --git a/lib/python/qmk/errors.py b/lib/python/qmk/errors.py
new file mode 100644
index 000000000..f9bf5b9af
--- /dev/null
+++ b/lib/python/qmk/errors.py
@@ -0,0 +1,6 @@
+class NoSuchKeyboardError(Exception):
+    """Raised when we can't find a keyboard/keymap directory.
+    """
+
+    def __init__(self, message):
+        self.message = message
diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py
new file mode 100644
index 000000000..6eccab788
--- /dev/null
+++ b/lib/python/qmk/keymap.py
@@ -0,0 +1,100 @@
+"""Functions that help you work with QMK keymaps.
+"""
+import json
+import logging
+import os
+from traceback import format_exc
+
+import qmk.path
+from qmk.errors import NoSuchKeyboardError
+
+# The `keymap.c` template to use when a keyboard doesn't have its own
+DEFAULT_KEYMAP_C = """#include QMK_KEYBOARD_H
+
+/* THIS FILE WAS GENERATED!
+ *
+ * This file was generated by qmk-compile-json. You may or may not want to
+ * edit it directly.
+ */
+
+const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = {
+__KEYMAP_GOES_HERE__
+};
+"""
+
+
+def template(keyboard):
+    """Returns the `keymap.c` template for a keyboard.
+
+    If a template exists in `keyboards/<keyboard>/templates/keymap.c` that
+    text will be used instead of `DEFAULT_KEYMAP_C`.
+
+    Args:
+        keyboard
+            The keyboard to return a template for.
+    """
+    template_name = 'keyboards/%s/templates/keymap.c' % keyboard
+
+    if os.path.exists(template_name):
+        with open(template_name, 'r') as fd:
+            return fd.read()
+
+    return DEFAULT_KEYMAP_C
+
+
+def generate(keyboard, layout, layers):
+    """Returns a keymap.c for the specified keyboard, layout, and layers.
+
+    Args:
+        keyboard
+            The name of the keyboard
+
+        layout
+            The LAYOUT macro this keymap uses.
+
+        layers
+            An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
+    """
+    layer_txt = []
+    for layer_num, layer in enumerate(layers):
+        if layer_num != 0:
+            layer_txt[-1] = layer_txt[-1] + ','
+        layer_keys = ', '.join(layer)
+        layer_txt.append('\t[%s] = %s(%s)' % (layer_num, layout, layer_keys))
+
+    keymap = '\n'.join(layer_txt)
+    keymap_c = template(keyboard, keymap)
+
+    return keymap_c.replace('__KEYMAP_GOES_HERE__', keymap)
+
+
+def write(keyboard, keymap, layout, layers):
+    """Generate the `keymap.c` and write it to disk.
+
+    Returns the filename written to.
+
+    Args:
+        keyboard
+            The name of the keyboard
+
+        keymap
+            The name of the keymap
+
+        layout
+            The LAYOUT macro this keymap uses.
+
+        layers
+            An array of arrays describing the keymap. Each item in the inner array should be a string that is a valid QMK keycode.
+    """
+    keymap_c = generate(keyboard, layout, layers)
+    keymap_path = qmk.path.keymap(keyboard)
+    keymap_dir = os.path.join(keymap_path, keymap)
+    keymap_file = os.path.join(keymap_dir, 'keymap.c')
+
+    if not os.path.exists(keymap_dir):
+        os.makedirs(keymap_dir)
+
+    with open(keymap_file, 'w') as keymap_fd:
+        keymap_fd.write(keymap_c)
+
+    return keymap_file
diff --git a/lib/python/qmk/path.py b/lib/python/qmk/path.py
new file mode 100644
index 000000000..f2a8346a5
--- /dev/null
+++ b/lib/python/qmk/path.py
@@ -0,0 +1,32 @@
+"""Functions that help us work with files and folders.
+"""
+import os
+
+
+def keymap(keyboard):
+    """Locate the correct directory for storing a keymap.
+
+    Args:
+        keyboard
+            The name of the keyboard. Example: clueboard/66/rev3
+    """
+    for directory in ['.', '..', '../..', '../../..', '../../../..', '../../../../..']:
+        basepath = os.path.normpath(os.path.join('keyboards', keyboard, directory, 'keymaps'))
+
+        if os.path.exists(basepath):
+            return basepath
+
+    logging.error('Could not find keymaps directory!')
+    raise NoSuchKeyboardError('Could not find keymaps directory for: %s' % keyboard)
+
+
+def normpath(path):
+    """Returns the fully resolved absolute path to a file.
+
+    This function will return the absolute path to a file as seen from the
+    directory the script was called from.
+    """
+    if path and path[0] == '/':
+        return os.path.normpath(path)
+
+    return os.path.normpath(os.path.join(os.environ['ORIG_CWD'], path))