aboutsummaryrefslogtreecommitdiff
path: root/lib/python/qmk/cli
diff options
context:
space:
mode:
Diffstat (limited to 'lib/python/qmk/cli')
-rw-r--r--lib/python/qmk/cli/__init__.py4
-rwxr-xr-x[-rw-r--r--]lib/python/qmk/cli/cformat.py139
-rwxr-xr-xlib/python/qmk/cli/doctor/__init__.py5
-rw-r--r--lib/python/qmk/cli/doctor/check.py164
-rw-r--r--lib/python/qmk/cli/doctor/linux.py162
-rw-r--r--lib/python/qmk/cli/doctor/macos.py13
-rwxr-xr-xlib/python/qmk/cli/doctor/main.py (renamed from lib/python/qmk/cli/doctor.py)76
-rw-r--r--lib/python/qmk/cli/doctor/windows.py14
-rwxr-xr-x[-rw-r--r--]lib/python/qmk/cli/fileformat.py24
-rw-r--r--lib/python/qmk/cli/format/c.py137
-rwxr-xr-xlib/python/qmk/cli/format/json.py5
-rwxr-xr-xlib/python/qmk/cli/format/python.py26
-rw-r--r--lib/python/qmk/cli/format/text.py27
-rw-r--r--lib/python/qmk/cli/generate/version_h.py28
-rwxr-xr-xlib/python/qmk/cli/info.py2
-rwxr-xr-xlib/python/qmk/cli/kle2json.py2
-rwxr-xr-xlib/python/qmk/cli/pyformat.py32
17 files changed, 666 insertions, 194 deletions
diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py
index 1e1c26671..dea0eaeaf 100644
--- a/lib/python/qmk/cli/__init__.py
+++ b/lib/python/qmk/cli/__init__.py
@@ -40,7 +40,10 @@ subcommands = [
40 'qmk.cli.doctor', 40 'qmk.cli.doctor',
41 'qmk.cli.fileformat', 41 'qmk.cli.fileformat',
42 'qmk.cli.flash', 42 'qmk.cli.flash',
43 'qmk.cli.format.c',
43 'qmk.cli.format.json', 44 'qmk.cli.format.json',
45 'qmk.cli.format.python',
46 'qmk.cli.format.text',
44 'qmk.cli.generate.api', 47 'qmk.cli.generate.api',
45 'qmk.cli.generate.config_h', 48 'qmk.cli.generate.config_h',
46 'qmk.cli.generate.dfu_header', 49 'qmk.cli.generate.dfu_header',
@@ -50,6 +53,7 @@ subcommands = [
50 'qmk.cli.generate.layouts', 53 'qmk.cli.generate.layouts',
51 'qmk.cli.generate.rgb_breathe_table', 54 'qmk.cli.generate.rgb_breathe_table',
52 'qmk.cli.generate.rules_mk', 55 'qmk.cli.generate.rules_mk',
56 'qmk.cli.generate.version_h',
53 'qmk.cli.hello', 57 'qmk.cli.hello',
54 'qmk.cli.info', 58 'qmk.cli.info',
55 'qmk.cli.json2c', 59 'qmk.cli.json2c',
diff --git a/lib/python/qmk/cli/cformat.py b/lib/python/qmk/cli/cformat.py
index efeb45967..9d0ecaeba 100644..100755
--- a/lib/python/qmk/cli/cformat.py
+++ b/lib/python/qmk/cli/cformat.py
@@ -1,137 +1,28 @@
1"""Format C code according to QMK's style. 1"""Point people to the new command name.
2""" 2"""
3from os import path 3import sys
4from shutil import which 4from pathlib import Path
5from subprocess import CalledProcessError, DEVNULL, Popen, PIPE
6 5
7from argcomplete.completers import FilesCompleter
8from milc import cli 6from milc import cli
9 7
10from qmk.path import normpath
11from qmk.c_parse import c_source_files
12
13c_file_suffixes = ('c', 'h', 'cpp')
14core_dirs = ('drivers', 'quantum', 'tests', 'tmk_core', 'platforms')
15ignored = ('tmk_core/protocol/usb_hid', 'quantum/template', 'platforms/chibios')
16
17
18def find_clang_format():
19 """Returns the path to clang-format.
20 """
21 for clang_version in range(20, 6, -1):
22 binary = f'clang-format-{clang_version}'
23
24 if which(binary):
25 return binary
26
27 return 'clang-format'
28
29
30def find_diffs(files):
31 """Run clang-format and diff it against a file.
32 """
33 found_diffs = False
34
35 for file in files:
36 cli.log.debug('Checking for changes in %s', file)
37 clang_format = Popen([find_clang_format(), file], stdout=PIPE, stderr=PIPE, universal_newlines=True)
38 diff = cli.run(['diff', '-u', f'--label=a/{file}', f'--label=b/{file}', str(file), '-'], stdin=clang_format.stdout, capture_output=True)
39
40 if diff.returncode != 0:
41 print(diff.stdout)
42 found_diffs = True
43
44 return found_diffs
45
46
47def cformat_run(files):
48 """Spawn clang-format subprocess with proper arguments
49 """
50 # Determine which version of clang-format to use
51 clang_format = [find_clang_format(), '-i']
52
53 try:
54 cli.run([*clang_format, *map(str, files)], check=True, capture_output=False, stdin=DEVNULL)
55 cli.log.info('Successfully formatted the C code.')
56 return True
57
58 except CalledProcessError as e:
59 cli.log.error('Error formatting C code!')
60 cli.log.debug('%s exited with returncode %s', e.cmd, e.returncode)
61 cli.log.debug('STDOUT:')
62 cli.log.debug(e.stdout)
63 cli.log.debug('STDERR:')
64 cli.log.debug(e.stderr)
65 return False
66
67
68def filter_files(files, core_only=False):
69 """Yield only files to be formatted and skip the rest
70 """
71 if core_only:
72 # Filter non-core files
73 for index, file in enumerate(files):
74 # The following statement checks each file to see if the file path is
75 # - in the core directories
76 # - not in the ignored directories
77 if not any(i in str(file) for i in core_dirs) or any(i in str(file) for i in ignored):
78 files[index] = None
79 cli.log.debug("Skipping non-core file %s, as '--core-only' is used.", file)
80
81 for file in files:
82 if file and file.name.split('.')[-1] in c_file_suffixes:
83 yield file
84 else:
85 cli.log.debug('Skipping file %s', file)
86
87 8
88@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Flag only, don't automatically format.") 9@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Flag only, don't automatically format.")
89@cli.argument('-b', '--base-branch', default='origin/master', help='Branch to compare to diffs to.') 10@cli.argument('-b', '--base-branch', default='origin/master', help='Branch to compare to diffs to.')
90@cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all core files.') 11@cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all core files.')
91@cli.argument('--core-only', arg_only=True, action='store_true', help='Format core files only.') 12@cli.argument('--core-only', arg_only=True, action='store_true', help='Format core files only.')
92@cli.argument('files', nargs='*', arg_only=True, type=normpath, completer=FilesCompleter('.c'), help='Filename(s) to format.') 13@cli.argument('files', nargs='*', arg_only=True, help='Filename(s) to format.')
93@cli.subcommand("Format C code according to QMK's style.", hidden=False if cli.config.user.developer else True) 14@cli.subcommand('Pointer to the new command name: qmk format-c.', hidden=True)
94def cformat(cli): 15def cformat(cli):
95 """Format C code according to QMK's style. 16 """Pointer to the new command name: qmk format-c.
96 """ 17 """
97 # Find the list of files to format 18 cli.log.warning('"qmk cformat" has been renamed to "qmk format-c". Please use the new command in the future.')
98 if cli.args.files: 19 argv = [sys.executable, *sys.argv]
99 files = list(filter_files(cli.args.files, cli.args.core_only)) 20 argv[argv.index('cformat')] = 'format-c'
100 21 script_path = Path(argv[1])
101 if not files: 22 script_path_exe = Path(f'{argv[1]}.exe')
102 cli.log.error('No C files in filelist: %s', ', '.join(map(str, cli.args.files)))
103 exit(0)
104
105 if cli.args.all_files:
106 cli.log.warning('Filenames passed with -a, only formatting: %s', ','.join(map(str, files)))
107
108 elif cli.args.all_files:
109 all_files = c_source_files(core_dirs)
110 files = list(filter_files(all_files, True))
111
112 else:
113 git_diff_cmd = ['git', 'diff', '--name-only', cli.args.base_branch, *core_dirs]
114 git_diff = cli.run(git_diff_cmd, stdin=DEVNULL)
115
116 if git_diff.returncode != 0:
117 cli.log.error("Error running %s", git_diff_cmd)
118 print(git_diff.stderr)
119 return git_diff.returncode
120
121 files = []
122
123 for file in git_diff.stdout.strip().split('\n'):
124 if not any([file.startswith(ignore) for ignore in ignored]):
125 if path.exists(file) and file.split('.')[-1] in c_file_suffixes:
126 files.append(file)
127 23
128 # Sanity check 24 if not script_path.exists() and script_path_exe.exists():
129 if not files: 25 # For reasons I don't understand ".exe" is stripped from the script name on windows.
130 cli.log.error('No changed files detected. Use "qmk cformat -a" to format all core files') 26 argv[1] = str(script_path_exe)
131 return False
132 27
133 # Run clang-format on the files we've found 28 return cli.run(argv, capture_output=False).returncode
134 if cli.args.dry_run:
135 return not find_diffs(files)
136 else:
137 return cformat_run(files)
diff --git a/lib/python/qmk/cli/doctor/__init__.py b/lib/python/qmk/cli/doctor/__init__.py
new file mode 100755
index 000000000..272e04202
--- /dev/null
+++ b/lib/python/qmk/cli/doctor/__init__.py
@@ -0,0 +1,5 @@
1"""QMK Doctor
2
3Check out the user's QMK environment and make sure it's ready to compile.
4"""
5from .main import doctor
diff --git a/lib/python/qmk/cli/doctor/check.py b/lib/python/qmk/cli/doctor/check.py
new file mode 100644
index 000000000..a0bbb2816
--- /dev/null
+++ b/lib/python/qmk/cli/doctor/check.py
@@ -0,0 +1,164 @@
1"""Check for specific programs.
2"""
3from enum import Enum
4import re
5import shutil
6from subprocess import DEVNULL
7
8from milc import cli
9from qmk import submodules
10from qmk.constants import QMK_FIRMWARE
11
12
13class CheckStatus(Enum):
14 OK = 1
15 WARNING = 2
16 ERROR = 3
17
18
19ESSENTIAL_BINARIES = {
20 'dfu-programmer': {},
21 'avrdude': {},
22 'dfu-util': {},
23 'avr-gcc': {
24 'version_arg': '-dumpversion'
25 },
26 'arm-none-eabi-gcc': {
27 'version_arg': '-dumpversion'
28 },
29 'bin/qmk': {},
30}
31
32
33def _parse_gcc_version(version):
34 m = re.match(r"(\d+)(?:\.(\d+))?(?:\.(\d+))?", version)
35
36 return {
37 'major': int(m.group(1)),
38 'minor': int(m.group(2)) if m.group(2) else 0,
39 'patch': int(m.group(3)) if m.group(3) else 0,
40 }
41
42
43def _check_arm_gcc_version():
44 """Returns True if the arm-none-eabi-gcc version is not known to cause problems.
45 """
46 if 'output' in ESSENTIAL_BINARIES['arm-none-eabi-gcc']:
47 version_number = ESSENTIAL_BINARIES['arm-none-eabi-gcc']['output'].strip()
48 cli.log.info('Found arm-none-eabi-gcc version %s', version_number)
49
50 return CheckStatus.OK # Right now all known arm versions are ok
51
52
53def _check_avr_gcc_version():
54 """Returns True if the avr-gcc version is not known to cause problems.
55 """
56 rc = CheckStatus.ERROR
57 if 'output' in ESSENTIAL_BINARIES['avr-gcc']:
58 version_number = ESSENTIAL_BINARIES['avr-gcc']['output'].strip()
59
60 cli.log.info('Found avr-gcc version %s', version_number)
61 rc = CheckStatus.OK
62
63 parsed_version = _parse_gcc_version(version_number)
64 if parsed_version['major'] > 8:
65 cli.log.warning('{fg_yellow}We do not recommend avr-gcc newer than 8. Downgrading to 8.x is recommended.')
66 rc = CheckStatus.WARNING
67
68 return rc
69
70
71def _check_avrdude_version():
72 if 'output' in ESSENTIAL_BINARIES['avrdude']:
73 last_line = ESSENTIAL_BINARIES['avrdude']['output'].split('\n')[-2]
74 version_number = last_line.split()[2][:-1]
75 cli.log.info('Found avrdude version %s', version_number)
76
77 return CheckStatus.OK
78
79
80def _check_dfu_util_version():
81 if 'output' in ESSENTIAL_BINARIES['dfu-util']:
82 first_line = ESSENTIAL_BINARIES['dfu-util']['output'].split('\n')[0]
83 version_number = first_line.split()[1]
84 cli.log.info('Found dfu-util version %s', version_number)
85
86 return CheckStatus.OK
87
88
89def _check_dfu_programmer_version():
90 if 'output' in ESSENTIAL_BINARIES['dfu-programmer']:
91 first_line = ESSENTIAL_BINARIES['dfu-programmer']['output'].split('\n')[0]
92 version_number = first_line.split()[1]
93 cli.log.info('Found dfu-programmer version %s', version_number)
94
95 return CheckStatus.OK
96
97
98def check_binaries():
99 """Iterates through ESSENTIAL_BINARIES and tests them.
100 """
101 ok = True
102
103 for binary in sorted(ESSENTIAL_BINARIES):
104 if not is_executable(binary):
105 ok = False
106
107 return ok
108
109
110def check_binary_versions():
111 """Check the versions of ESSENTIAL_BINARIES
112 """
113 versions = []
114 for check in (_check_arm_gcc_version, _check_avr_gcc_version, _check_avrdude_version, _check_dfu_util_version, _check_dfu_programmer_version):
115 versions.append(check())
116 return versions
117
118
119def check_submodules():
120 """Iterates through all submodules to make sure they're cloned and up to date.
121 """
122 for submodule in submodules.status().values():
123 if submodule['status'] is None:
124 cli.log.error('Submodule %s has not yet been cloned!', submodule['name'])
125 return CheckStatus.ERROR
126 elif not submodule['status']:
127 cli.log.warning('Submodule %s is not up to date!', submodule['name'])
128 return CheckStatus.WARNING
129
130 return CheckStatus.OK
131
132
133def is_executable(command):
134 """Returns True if command exists and can be executed.
135 """
136 # Make sure the command is in the path.
137 res = shutil.which(command)
138 if res is None:
139 cli.log.error("{fg_red}Can't find %s in your path.", command)
140 return False
141
142 # Make sure the command can be executed
143 version_arg = ESSENTIAL_BINARIES[command].get('version_arg', '--version')
144 check = cli.run([command, version_arg], combined_output=True, stdin=DEVNULL, timeout=5)
145
146 ESSENTIAL_BINARIES[command]['output'] = check.stdout
147
148 if check.returncode in [0, 1]: # Older versions of dfu-programmer exit 1
149 cli.log.debug('Found {fg_cyan}%s', command)
150 return True
151
152 cli.log.error("{fg_red}Can't run `%s %s`", command, version_arg)
153 return False
154
155
156def check_git_repo():
157 """Checks that the .git directory exists inside QMK_HOME.
158
159 This is a decent enough indicator that the qmk_firmware directory is a
160 proper Git repository, rather than a .zip download from GitHub.
161 """
162 dot_git_dir = QMK_FIRMWARE / '.git'
163
164 return CheckStatus.OK if dot_git_dir.is_dir() else CheckStatus.WARNING
diff --git a/lib/python/qmk/cli/doctor/linux.py b/lib/python/qmk/cli/doctor/linux.py
new file mode 100644
index 000000000..c0b77216a
--- /dev/null
+++ b/lib/python/qmk/cli/doctor/linux.py
@@ -0,0 +1,162 @@
1"""OS-specific functions for: Linux
2"""
3import platform
4import shutil
5from pathlib import Path
6
7from milc import cli
8
9from qmk.constants import QMK_FIRMWARE
10from .check import CheckStatus
11
12
13def _udev_rule(vid, pid=None, *args):
14 """ Helper function that return udev rules
15 """
16 rule = ""
17 if pid:
18 rule = 'SUBSYSTEMS=="usb", ATTRS{idVendor}=="%s", ATTRS{idProduct}=="%s", TAG+="uaccess"' % (
19 vid,
20 pid,
21 )
22 else:
23 rule = 'SUBSYSTEMS=="usb", ATTRS{idVendor}=="%s", TAG+="uaccess"' % vid
24 if args:
25 rule = ', '.join([rule, *args])
26 return rule
27
28
29def _deprecated_udev_rule(vid, pid=None):
30 """ Helper function that return udev rules
31
32 Note: these are no longer the recommended rules, this is just used to check for them
33 """
34 if pid:
35 return 'SUBSYSTEMS=="usb", ATTRS{idVendor}=="%s", ATTRS{idProduct}=="%s", MODE:="0666"' % (vid, pid)
36 else:
37 return 'SUBSYSTEMS=="usb", ATTRS{idVendor}=="%s", MODE:="0666"' % vid
38
39
40def check_udev_rules():
41 """Make sure the udev rules look good.
42 """
43 rc = CheckStatus.OK
44 udev_dir = Path("/etc/udev/rules.d/")
45 desired_rules = {
46 'atmel-dfu': {
47 _udev_rule("03eb", "2fef"), # ATmega16U2
48 _udev_rule("03eb", "2ff0"), # ATmega32U2
49 _udev_rule("03eb", "2ff3"), # ATmega16U4
50 _udev_rule("03eb", "2ff4"), # ATmega32U4
51 _udev_rule("03eb", "2ff9"), # AT90USB64
52 _udev_rule("03eb", "2ffa"), # AT90USB162
53 _udev_rule("03eb", "2ffb") # AT90USB128
54 },
55 'kiibohd': {_udev_rule("1c11", "b007")},
56 'stm32': {
57 _udev_rule("1eaf", "0003"), # STM32duino
58 _udev_rule("0483", "df11") # STM32 DFU
59 },
60 'bootloadhid': {_udev_rule("16c0", "05df")},
61 'usbasploader': {_udev_rule("16c0", "05dc")},
62 'massdrop': {_udev_rule("03eb", "6124", 'ENV{ID_MM_DEVICE_IGNORE}="1"')},
63 'caterina': {
64 # Spark Fun Electronics
65 _udev_rule("1b4f", "9203", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # Pro Micro 3V3/8MHz
66 _udev_rule("1b4f", "9205", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # Pro Micro 5V/16MHz
67 _udev_rule("1b4f", "9207", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # LilyPad 3V3/8MHz (and some Pro Micro clones)
68 # Pololu Electronics
69 _udev_rule("1ffb", "0101", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # A-Star 32U4
70 # Arduino SA
71 _udev_rule("2341", "0036", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # Leonardo
72 _udev_rule("2341", "0037", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # Micro
73 # Adafruit Industries LLC
74 _udev_rule("239a", "000c", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # Feather 32U4
75 _udev_rule("239a", "000d", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # ItsyBitsy 32U4 3V3/8MHz
76 _udev_rule("239a", "000e", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # ItsyBitsy 32U4 5V/16MHz
77 # dog hunter AG
78 _udev_rule("2a03", "0036", 'ENV{ID_MM_DEVICE_IGNORE}="1"'), # Leonardo
79 _udev_rule("2a03", "0037", 'ENV{ID_MM_DEVICE_IGNORE}="1"') # Micro
80 }
81 }
82
83 # These rules are no longer recommended, only use them to check for their presence.
84 deprecated_rules = {
85 'atmel-dfu': {_deprecated_udev_rule("03eb", "2ff4"), _deprecated_udev_rule("03eb", "2ffb"), _deprecated_udev_rule("03eb", "2ff0")},
86 'kiibohd': {_deprecated_udev_rule("1c11")},
87 'stm32': {_deprecated_udev_rule("1eaf", "0003"), _deprecated_udev_rule("0483", "df11")},
88 'bootloadhid': {_deprecated_udev_rule("16c0", "05df")},
89 'caterina': {'ATTRS{idVendor}=="2a03", ENV{ID_MM_DEVICE_IGNORE}="1"', 'ATTRS{idVendor}=="2341", ENV{ID_MM_DEVICE_IGNORE}="1"'},
90 'tmk': {_deprecated_udev_rule("feed")}
91 }
92
93 if udev_dir.exists():
94 udev_rules = [rule_file for rule_file in udev_dir.glob('*.rules')]
95 current_rules = set()
96
97 # Collect all rules from the config files
98 for rule_file in udev_rules:
99 for line in rule_file.read_text(encoding='utf-8').split('\n'):
100 line = line.strip()
101 if not line.startswith("#") and len(line):
102 current_rules.add(line)
103
104 # Check if the desired rules are among the currently present rules
105 for bootloader, rules in desired_rules.items():
106 if not rules.issubset(current_rules):
107 deprecated_rule = deprecated_rules.get(bootloader)
108 if deprecated_rule and deprecated_rule.issubset(current_rules):
109 cli.log.warning("{fg_yellow}Found old, deprecated udev rules for '%s' boards. The new rules on https://docs.qmk.fm/#/faq_build?id=linux-udev-rules offer better security with the same functionality.", bootloader)
110 else:
111 # For caterina, check if ModemManager is running
112 if bootloader == "caterina":
113 if check_modem_manager():
114 rc = CheckStatus.WARNING
115 cli.log.warning("{fg_yellow}Detected ModemManager without the necessary udev rules. Please either disable it or set the appropriate udev rules if you are using a Pro Micro.")
116 rc = CheckStatus.WARNING
117 cli.log.warning("{fg_yellow}Missing or outdated udev rules for '%s' boards. Run 'sudo cp %s/util/udev/50-qmk.rules /etc/udev/rules.d/'.", bootloader, QMK_FIRMWARE)
118
119 else:
120 cli.log.warning("{fg_yellow}'%s' does not exist. Skipping udev rule checking...", udev_dir)
121
122 return rc
123
124
125def check_systemd():
126 """Check if it's a systemd system
127 """
128 return bool(shutil.which("systemctl"))
129
130
131def check_modem_manager():
132 """Returns True if ModemManager is running.
133
134 """
135 if check_systemd():
136 mm_check = cli.run(["systemctl", "--quiet", "is-active", "ModemManager.service"], timeout=10)
137 if mm_check.returncode == 0:
138 return True
139 else:
140 """(TODO): Add check for non-systemd systems
141 """
142 return False
143
144
145def os_test_linux():
146 """Run the Linux specific tests.
147 """
148 # Don't bother with udev on WSL, for now
149 if 'microsoft' in platform.uname().release.lower():
150 cli.log.info("Detected {fg_cyan}Linux (WSL){fg_reset}.")
151
152 # https://github.com/microsoft/WSL/issues/4197
153 if QMK_FIRMWARE.as_posix().startswith("/mnt"):
154 cli.log.warning("I/O performance on /mnt may be extremely slow.")
155 return CheckStatus.WARNING
156
157 return CheckStatus.OK
158 else:
159 cli.log.info("Detected {fg_cyan}Linux{fg_reset}.")
160 from .linux import check_udev_rules
161
162 return check_udev_rules()
diff --git a/lib/python/qmk/cli/doctor/macos.py b/lib/python/qmk/cli/doctor/macos.py
new file mode 100644
index 000000000..00fb27285
--- /dev/null
+++ b/lib/python/qmk/cli/doctor/macos.py
@@ -0,0 +1,13 @@
1import platform
2
3from milc import cli
4
5from .check import CheckStatus
6
7
8def os_test_macos():
9 """Run the Mac specific tests.
10 """
11 cli.log.info("Detected {fg_cyan}macOS %s{fg_reset}.", platform.mac_ver()[0])
12
13 return CheckStatus.OK
diff --git a/lib/python/qmk/cli/doctor.py b/lib/python/qmk/cli/doctor/main.py
index 327bc9cb3..6a31ccdfd 100755
--- a/lib/python/qmk/cli/doctor.py
+++ b/lib/python/qmk/cli/doctor/main.py
@@ -7,9 +7,11 @@ from subprocess import DEVNULL
7 7
8from milc import cli 8from milc import cli
9from milc.questions import yesno 9from milc.questions import yesno
10
10from qmk import submodules 11from qmk import submodules
11from qmk.constants import QMK_FIRMWARE 12from qmk.constants import QMK_FIRMWARE, QMK_FIRMWARE_UPSTREAM
12from qmk.os_helpers import CheckStatus, check_binaries, check_binary_versions, check_submodules, check_git_repo 13from .check import CheckStatus, check_binaries, check_binary_versions, check_submodules
14from qmk.commands import git_check_repo, git_get_branch, git_is_dirty, git_get_remotes, git_check_deviation, in_virtualenv
13 15
14 16
15def os_tests(): 17def os_tests():
@@ -18,51 +20,48 @@ def os_tests():
18 platform_id = platform.platform().lower() 20 platform_id = platform.platform().lower()
19 21
20 if 'darwin' in platform_id or 'macos' in platform_id: 22 if 'darwin' in platform_id or 'macos' in platform_id:
23 from .macos import os_test_macos
21 return os_test_macos() 24 return os_test_macos()
22 elif 'linux' in platform_id: 25 elif 'linux' in platform_id:
26 from .linux import os_test_linux
23 return os_test_linux() 27 return os_test_linux()
24 elif 'windows' in platform_id: 28 elif 'windows' in platform_id:
29 from .windows import os_test_windows
25 return os_test_windows() 30 return os_test_windows()
26 else: 31 else:
27 cli.log.warning('Unsupported OS detected: %s', platform_id) 32 cli.log.warning('Unsupported OS detected: %s', platform_id)
28 return CheckStatus.WARNING 33 return CheckStatus.WARNING
29 34
30 35
31def os_test_linux(): 36def git_tests():
32 """Run the Linux specific tests. 37 """Run Git-related checks
33 """ 38 """
34 # Don't bother with udev on WSL, for now 39 status = CheckStatus.OK
35 if 'microsoft' in platform.uname().release.lower():
36 cli.log.info("Detected {fg_cyan}Linux (WSL){fg_reset}.")
37
38 # https://github.com/microsoft/WSL/issues/4197
39 if QMK_FIRMWARE.as_posix().startswith("/mnt"):
40 cli.log.warning("I/O performance on /mnt may be extremely slow.")
41 return CheckStatus.WARNING
42 40
43 return CheckStatus.OK 41 # Make sure our QMK home is a Git repo
42 git_ok = git_check_repo()
43 if not git_ok:
44 cli.log.warning("{fg_yellow}QMK home does not appear to be a Git repository! (no .git folder)")
45 status = CheckStatus.WARNING
44 else: 46 else:
45 cli.log.info("Detected {fg_cyan}Linux{fg_reset}.") 47 git_branch = git_get_branch()
46 from qmk.os_helpers.linux import check_udev_rules 48 if git_branch:
47 49 cli.log.info('Git branch: %s', git_branch)
48 return check_udev_rules() 50 git_dirty = git_is_dirty()
49 51 if git_dirty:
50 52 cli.log.warning('{fg_yellow}Git has unstashed/uncommitted changes.')
51def os_test_macos(): 53 status = CheckStatus.WARNING
52 """Run the Mac specific tests. 54 git_remotes = git_get_remotes()
53 """ 55 if 'upstream' not in git_remotes.keys() or QMK_FIRMWARE_UPSTREAM not in git_remotes['upstream'].get('url', ''):
54 cli.log.info("Detected {fg_cyan}macOS %s{fg_reset}.", platform.mac_ver()[0]) 56 cli.log.warning('{fg_yellow}The official repository does not seem to be configured as git remote "upstream".')
55 57 status = CheckStatus.WARNING
56 return CheckStatus.OK 58 else:
57 59 git_deviation = git_check_deviation(git_branch)
58 60 if git_branch in ['master', 'develop'] and git_deviation:
59def os_test_windows(): 61 cli.log.warning('{fg_yellow}The local "%s" branch contains commits not found in the upstream branch.', git_branch)
60 """Run the Windows specific tests. 62 status = CheckStatus.WARNING
61 """ 63
62 win32_ver = platform.win32_ver() 64 return status
63 cli.log.info("Detected {fg_cyan}Windows %s (%s){fg_reset}.", win32_ver[0], win32_ver[1])
64
65 return CheckStatus.OK
66 65
67 66
68@cli.argument('-y', '--yes', action='store_true', arg_only=True, help='Answer yes to all questions.') 67@cli.argument('-y', '--yes', action='store_true', arg_only=True, help='Answer yes to all questions.')
@@ -82,12 +81,11 @@ def doctor(cli):
82 81
83 status = os_tests() 82 status = os_tests()
84 83
85 # Make sure our QMK home is a Git repo 84 status = git_tests()
86 git_ok = check_git_repo()
87 85
88 if git_ok == CheckStatus.WARNING: 86 venv = in_virtualenv()
89 cli.log.warning("QMK home does not appear to be a Git repository! (no .git folder)") 87 if venv:
90 status = CheckStatus.WARNING 88 cli.log.info('CLI installed in virtualenv.')
91 89
92 # Make sure the basic CLI tools we need are available and can be executed. 90 # Make sure the basic CLI tools we need are available and can be executed.
93 bin_ok = check_binaries() 91 bin_ok = check_binaries()
diff --git a/lib/python/qmk/cli/doctor/windows.py b/lib/python/qmk/cli/doctor/windows.py
new file mode 100644
index 000000000..381ab36fd
--- /dev/null
+++ b/lib/python/qmk/cli/doctor/windows.py
@@ -0,0 +1,14 @@
1import platform
2
3from milc import cli
4
5from .check import CheckStatus
6
7
8def os_test_windows():
9 """Run the Windows specific tests.
10 """
11 win32_ver = platform.win32_ver()
12 cli.log.info("Detected {fg_cyan}Windows %s (%s){fg_reset}.", win32_ver[0], win32_ver[1])
13
14 return CheckStatus.OK
diff --git a/lib/python/qmk/cli/fileformat.py b/lib/python/qmk/cli/fileformat.py
index 112d8d59d..cee4ba1ac 100644..100755
--- a/lib/python/qmk/cli/fileformat.py
+++ b/lib/python/qmk/cli/fileformat.py
@@ -1,13 +1,23 @@
1"""Format files according to QMK's style. 1"""Point people to the new command name.
2""" 2"""
3from milc import cli 3import sys
4from pathlib import Path
4 5
5import subprocess 6from milc import cli
6 7
7 8
8@cli.subcommand("Format files according to QMK's style.", hidden=True) 9@cli.subcommand('Pointer to the new command name: qmk format-text.', hidden=True)
9def fileformat(cli): 10def fileformat(cli):
10 """Run several general formatting commands. 11 """Pointer to the new command name: qmk format-text.
11 """ 12 """
12 dos2unix = subprocess.run(['bash', '-c', 'git ls-files -z | xargs -0 dos2unix'], stdout=subprocess.DEVNULL) 13 cli.log.warning('"qmk fileformat" has been renamed to "qmk format-text". Please use the new command in the future.')
13 return dos2unix.returncode 14 argv = [sys.executable, *sys.argv]
15 argv[argv.index('fileformat')] = 'format-text'
16 script_path = Path(argv[1])
17 script_path_exe = Path(f'{argv[1]}.exe')
18
19 if not script_path.exists() and script_path_exe.exists():
20 # For reasons I don't understand ".exe" is stripped from the script name on windows.
21 argv[1] = str(script_path_exe)
22
23 return cli.run(argv, capture_output=False).returncode
diff --git a/lib/python/qmk/cli/format/c.py b/lib/python/qmk/cli/format/c.py
new file mode 100644
index 000000000..b7263e19f
--- /dev/null
+++ b/lib/python/qmk/cli/format/c.py
@@ -0,0 +1,137 @@
1"""Format C code according to QMK's style.
2"""
3from os import path
4from shutil import which
5from subprocess import CalledProcessError, DEVNULL, Popen, PIPE
6
7from argcomplete.completers import FilesCompleter
8from milc import cli
9
10from qmk.path import normpath
11from qmk.c_parse import c_source_files
12
13c_file_suffixes = ('c', 'h', 'cpp')
14core_dirs = ('drivers', 'quantum', 'tests', 'tmk_core', 'platforms')
15ignored = ('tmk_core/protocol/usb_hid', 'quantum/template', 'platforms/chibios')
16
17
18def find_clang_format():
19 """Returns the path to clang-format.
20 """
21 for clang_version in range(20, 6, -1):
22 binary = f'clang-format-{clang_version}'
23
24 if which(binary):
25 return binary
26
27 return 'clang-format'
28
29
30def find_diffs(files):
31 """Run clang-format and diff it against a file.
32 """
33 found_diffs = False
34
35 for file in files:
36 cli.log.debug('Checking for changes in %s', file)
37 clang_format = Popen([find_clang_format(), file], stdout=PIPE, stderr=PIPE, universal_newlines=True)
38 diff = cli.run(['diff', '-u', f'--label=a/{file}', f'--label=b/{file}', str(file), '-'], stdin=clang_format.stdout, capture_output=True)
39
40 if diff.returncode != 0:
41 print(diff.stdout)
42 found_diffs = True
43
44 return found_diffs
45
46
47def cformat_run(files):
48 """Spawn clang-format subprocess with proper arguments
49 """
50 # Determine which version of clang-format to use
51 clang_format = [find_clang_format(), '-i']
52
53 try:
54 cli.run([*clang_format, *map(str, files)], check=True, capture_output=False, stdin=DEVNULL)
55 cli.log.info('Successfully formatted the C code.')
56 return True
57
58 except CalledProcessError as e:
59 cli.log.error('Error formatting C code!')
60 cli.log.debug('%s exited with returncode %s', e.cmd, e.returncode)
61 cli.log.debug('STDOUT:')
62 cli.log.debug(e.stdout)
63 cli.log.debug('STDERR:')
64 cli.log.debug(e.stderr)
65 return False
66
67
68def filter_files(files, core_only=False):
69 """Yield only files to be formatted and skip the rest
70 """
71 if core_only:
72 # Filter non-core files
73 for index, file in enumerate(files):
74 # The following statement checks each file to see if the file path is
75 # - in the core directories
76 # - not in the ignored directories
77 if not any(i in str(file) for i in core_dirs) or any(i in str(file) for i in ignored):
78 files[index] = None
79 cli.log.debug("Skipping non-core file %s, as '--core-only' is used.", file)
80
81 for file in files:
82 if file and file.name.split('.')[-1] in c_file_suffixes:
83 yield file
84 else:
85 cli.log.debug('Skipping file %s', file)
86
87
88@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Flag only, don't automatically format.")
89@cli.argument('-b', '--base-branch', default='origin/master', help='Branch to compare to diffs to.')
90@cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all core files.')
91@cli.argument('--core-only', arg_only=True, action='store_true', help='Format core files only.')
92@cli.argument('files', nargs='*', arg_only=True, type=normpath, completer=FilesCompleter('.c'), help='Filename(s) to format.')
93@cli.subcommand("Format C code according to QMK's style.", hidden=False if cli.config.user.developer else True)
94def format_c(cli):
95 """Format C code according to QMK's style.
96 """
97 # Find the list of files to format
98 if cli.args.files:
99 files = list(filter_files(cli.args.files, cli.args.core_only))
100
101 if not files:
102 cli.log.error('No C files in filelist: %s', ', '.join(map(str, cli.args.files)))
103 exit(0)
104
105 if cli.args.all_files:
106 cli.log.warning('Filenames passed with -a, only formatting: %s', ','.join(map(str, files)))
107
108 elif cli.args.all_files:
109 all_files = c_source_files(core_dirs)
110 files = list(filter_files(all_files, True))
111
112 else:
113 git_diff_cmd = ['git', 'diff', '--name-only', cli.args.base_branch, *core_dirs]
114 git_diff = cli.run(git_diff_cmd, stdin=DEVNULL)
115
116 if git_diff.returncode != 0:
117 cli.log.error("Error running %s", git_diff_cmd)
118 print(git_diff.stderr)
119 return git_diff.returncode
120
121 files = []
122
123 for file in git_diff.stdout.strip().split('\n'):
124 if not any([file.startswith(ignore) for ignore in ignored]):
125 if path.exists(file) and file.split('.')[-1] in c_file_suffixes:
126 files.append(file)
127
128 # Sanity check
129 if not files:
130 cli.log.error('No changed files detected. Use "qmk format-c -a" to format all core files')
131 return False
132
133 # Run clang-format on the files we've found
134 if cli.args.dry_run:
135 return not find_diffs(files)
136 else:
137 return cformat_run(files)
diff --git a/lib/python/qmk/cli/format/json.py b/lib/python/qmk/cli/format/json.py
index 1358c70e7..19d504491 100755
--- a/lib/python/qmk/cli/format/json.py
+++ b/lib/python/qmk/cli/format/json.py
@@ -8,7 +8,7 @@ from jsonschema import ValidationError
8from milc import cli 8from milc import cli
9 9
10from qmk.info import info_json 10from qmk.info import info_json
11from qmk.json_schema import json_load, keyboard_validate 11from qmk.json_schema import json_load, validate
12from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder 12from qmk.json_encoders import InfoJSONEncoder, KeymapJSONEncoder
13from qmk.path import normpath 13from qmk.path import normpath
14 14
@@ -23,14 +23,13 @@ def format_json(cli):
23 23
24 if cli.args.format == 'auto': 24 if cli.args.format == 'auto':
25 try: 25 try:
26 keyboard_validate(json_file) 26 validate(json_file, 'qmk.keyboard.v1')
27 json_encoder = InfoJSONEncoder 27 json_encoder = InfoJSONEncoder
28 28
29 except ValidationError as e: 29 except ValidationError as e:
30 cli.log.warning('File %s did not validate as a keyboard:\n\t%s', cli.args.json_file, e) 30 cli.log.warning('File %s did not validate as a keyboard:\n\t%s', cli.args.json_file, e)
31 cli.log.info('Treating %s as a keymap file.', cli.args.json_file) 31 cli.log.info('Treating %s as a keymap file.', cli.args.json_file)
32 json_encoder = KeymapJSONEncoder 32 json_encoder = KeymapJSONEncoder
33
34 elif cli.args.format == 'keyboard': 33 elif cli.args.format == 'keyboard':
35 json_encoder = InfoJSONEncoder 34 json_encoder = InfoJSONEncoder
36 elif cli.args.format == 'keymap': 35 elif cli.args.format == 'keymap':
diff --git a/lib/python/qmk/cli/format/python.py b/lib/python/qmk/cli/format/python.py
new file mode 100755
index 000000000..00612f97e
--- /dev/null
+++ b/lib/python/qmk/cli/format/python.py
@@ -0,0 +1,26 @@
1"""Format python code according to QMK's style.
2"""
3from subprocess import CalledProcessError, DEVNULL
4
5from milc import cli
6
7
8@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually format.")
9@cli.subcommand("Format python code according to QMK's style.", hidden=False if cli.config.user.developer else True)
10def format_python(cli):
11 """Format python code according to QMK's style.
12 """
13 edit = '--diff' if cli.args.dry_run else '--in-place'
14 yapf_cmd = ['yapf', '-vv', '--recursive', edit, 'bin/qmk', 'lib/python']
15 try:
16 cli.run(yapf_cmd, check=True, capture_output=False, stdin=DEVNULL)
17 cli.log.info('Python code in `bin/qmk` and `lib/python` is correctly formatted.')
18 return True
19
20 except CalledProcessError:
21 if cli.args.dry_run:
22 cli.log.error('Python code in `bin/qmk` and `lib/python` incorrectly formatted!')
23 else:
24 cli.log.error('Error formatting python code!')
25
26 return False
diff --git a/lib/python/qmk/cli/format/text.py b/lib/python/qmk/cli/format/text.py
new file mode 100644
index 000000000..e7e07b729
--- /dev/null
+++ b/lib/python/qmk/cli/format/text.py
@@ -0,0 +1,27 @@
1"""Ensure text files have the proper line endings.
2"""
3from subprocess import CalledProcessError
4
5from milc import cli
6
7
8@cli.subcommand("Ensure text files have the proper line endings.", hidden=True)
9def format_text(cli):
10 """Ensure text files have the proper line endings.
11 """
12 try:
13 file_list_cmd = cli.run(['git', 'ls-files', '-z'], check=True)
14 except CalledProcessError as e:
15 cli.log.error('Could not get file list: %s', e)
16 exit(1)
17 except Exception as e:
18 cli.log.error('Unhandled exception: %s: %s', e.__class__.__name__, e)
19 cli.log.exception(e)
20 exit(1)
21
22 dos2unix = cli.run(['xargs', '-0', 'dos2unix'], stdin=None, input=file_list_cmd.stdout)
23
24 if dos2unix.returncode != 0:
25 print(dos2unix.stderr)
26
27 return dos2unix.returncode
diff --git a/lib/python/qmk/cli/generate/version_h.py b/lib/python/qmk/cli/generate/version_h.py
new file mode 100644
index 000000000..b8e52588c
--- /dev/null
+++ b/lib/python/qmk/cli/generate/version_h.py
@@ -0,0 +1,28 @@
1"""Used by the make system to generate version.h for use in code.
2"""
3from milc import cli
4
5from qmk.commands import create_version_h
6from qmk.path import normpath
7
8
9@cli.argument('-o', '--output', arg_only=True, type=normpath, help='File to write to')
10@cli.argument('-q', '--quiet', arg_only=True, action='store_true', help="Quiet mode, only output error messages")
11@cli.argument('--skip-git', arg_only=True, action='store_true', help='Skip Git operations')
12@cli.argument('--skip-all', arg_only=True, action='store_true', help='Use placeholder values for all defines (implies --skip-git)')
13@cli.subcommand('Used by the make system to generate version.h for use in code', hidden=True)
14def generate_version_h(cli):
15 """Generates the version.h file.
16 """
17 if cli.args.skip_all:
18 cli.args.skip_git = True
19
20 version_h = create_version_h(cli.args.skip_git, cli.args.skip_all)
21
22 if cli.args.output:
23 cli.args.output.write_text(version_h)
24
25 if not cli.args.quiet:
26 cli.log.info('Wrote version.h to %s.', cli.args.output)
27 else:
28 print(version_h)
diff --git a/lib/python/qmk/cli/info.py b/lib/python/qmk/cli/info.py
index 0d08d242c..337b494a9 100755
--- a/lib/python/qmk/cli/info.py
+++ b/lib/python/qmk/cli/info.py
@@ -87,8 +87,6 @@ def print_friendly_output(kb_info_json):
87 cli.echo('{fg_blue}Maintainer{fg_reset}: %s', kb_info_json['maintainer']) 87 cli.echo('{fg_blue}Maintainer{fg_reset}: %s', kb_info_json['maintainer'])
88 cli.echo('{fg_blue}Keyboard Folder{fg_reset}: %s', kb_info_json.get('keyboard_folder', 'Unknown')) 88 cli.echo('{fg_blue}Keyboard Folder{fg_reset}: %s', kb_info_json.get('keyboard_folder', 'Unknown'))
89 cli.echo('{fg_blue}Layouts{fg_reset}: %s', ', '.join(sorted(kb_info_json['layouts'].keys()))) 89 cli.echo('{fg_blue}Layouts{fg_reset}: %s', ', '.join(sorted(kb_info_json['layouts'].keys())))
90 if 'width' in kb_info_json and 'height' in kb_info_json:
91 cli.echo('{fg_blue}Size{fg_reset}: %s x %s' % (kb_info_json['width'], kb_info_json['height']))
92 cli.echo('{fg_blue}Processor{fg_reset}: %s', kb_info_json.get('processor', 'Unknown')) 90 cli.echo('{fg_blue}Processor{fg_reset}: %s', kb_info_json.get('processor', 'Unknown'))
93 cli.echo('{fg_blue}Bootloader{fg_reset}: %s', kb_info_json.get('bootloader', 'Unknown')) 91 cli.echo('{fg_blue}Bootloader{fg_reset}: %s', kb_info_json.get('bootloader', 'Unknown'))
94 if 'layout_aliases' in kb_info_json: 92 if 'layout_aliases' in kb_info_json:
diff --git a/lib/python/qmk/cli/kle2json.py b/lib/python/qmk/cli/kle2json.py
index acb75ef4f..bbfddf426 100755
--- a/lib/python/qmk/cli/kle2json.py
+++ b/lib/python/qmk/cli/kle2json.py
@@ -44,8 +44,6 @@ def kle2json(cli):
44 'keyboard_name': kle.name, 44 'keyboard_name': kle.name,
45 'url': '', 45 'url': '',
46 'maintainer': 'qmk', 46 'maintainer': 'qmk',
47 'width': kle.columns,
48 'height': kle.rows,
49 'layouts': { 47 'layouts': {
50 'LAYOUT': { 48 'LAYOUT': {
51 'layout': kle2qmk(kle) 49 'layout': kle2qmk(kle)
diff --git a/lib/python/qmk/cli/pyformat.py b/lib/python/qmk/cli/pyformat.py
index abe5f6de1..c624f74ae 100755
--- a/lib/python/qmk/cli/pyformat.py
+++ b/lib/python/qmk/cli/pyformat.py
@@ -1,26 +1,24 @@
1"""Format python code according to QMK's style. 1"""Point people to the new command name.
2""" 2"""
3from subprocess import CalledProcessError, DEVNULL 3import sys
4from pathlib import Path
4 5
5from milc import cli 6from milc import cli
6 7
7 8
8@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Flag only, don't automatically format.") 9@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Don't actually format.")
9@cli.subcommand("Format python code according to QMK's style.", hidden=False if cli.config.user.developer else True) 10@cli.subcommand('Pointer to the new command name: qmk format-python.', hidden=False if cli.config.user.developer else True)
10def pyformat(cli): 11def pyformat(cli):
11 """Format python code according to QMK's style. 12 """Pointer to the new command name: qmk format-python.
12 """ 13 """
13 edit = '--diff' if cli.args.dry_run else '--in-place' 14 cli.log.warning('"qmk pyformat" has been renamed to "qmk format-python". Please use the new command in the future.')
14 yapf_cmd = ['yapf', '-vv', '--recursive', edit, 'bin/qmk', 'lib/python'] 15 argv = [sys.executable, *sys.argv]
15 try: 16 argv[argv.index('pyformat')] = 'format-python'
16 cli.run(yapf_cmd, check=True, capture_output=False, stdin=DEVNULL) 17 script_path = Path(argv[1])
17 cli.log.info('Python code in `bin/qmk` and `lib/python` is correctly formatted.') 18 script_path_exe = Path(f'{argv[1]}.exe')
18 return True
19 19
20 except CalledProcessError: 20 if not script_path.exists() and script_path_exe.exists():
21 if cli.args.dry_run: 21 # For reasons I don't understand ".exe" is stripped from the script name on windows.
22 cli.log.error('Python code in `bin/qmk` and `lib/python` incorrectly formatted!') 22 argv[1] = str(script_path_exe)
23 else:
24 cli.log.error('Error formatting python code!')
25 23
26 return False 24 return cli.run(argv, capture_output=False).returncode