Improve our CI tests (#11476)

* add a test and dry-run to qmk generate-api

* add a dry-run to qmk pyformat

* Add a --dry-run to qmk cformat

* reverse the order of nose2 and flake8 tests

* run CI test against cformat and pyformat

* fix programming errors

* tweak job name

* fix argument

* refine the files we select

* fix stack trace in --ci

* make cformat exit clean

* fix c file extensions

* decouple CI from pyformat

* remove --ci arg

* make ci happy

* use the environment var instead

* change output to text

* fix log message

* replace tabs

1 files changed, 90 insertions, 29 deletions

diff --git a/lib/python/qmk/cli/cformat.py b/lib/python/qmk/cli/cformat.py
index d0d3b3b0a..9333aaec4 100644
--- a/lib/python/qmk/cli/cformat.py
+++ b/lib/python/qmk/cli/cformat.py
@@ -1,6 +1,7 @@
 """Format C code according to QMK's style.
 """
 import subprocess
+from os import path
 from shutil import which
 
 from argcomplete.completers import FilesCompleter
@@ -9,58 +10,118 @@ from milc import cli
 from qmk.path import normpath
 from qmk.c_parse import c_source_files
 
+c_file_suffixes = ('c', 'h', 'cpp')
+core_dirs = ('drivers', 'quantum', 'tests', 'tmk_core', 'platforms')
+ignored = ('tmk_core/protocol/usb_hid', 'quantum/template', 'platforms/chibios')
 
-def cformat_run(files, all_files):
-    """Spawn clang-format subprocess with proper arguments
+
+def find_clang_format():
+    """Returns the path to clang-format.
     """
-    # Determine which version of clang-format to use
-    clang_format = ['clang-format', '-i']
     for clang_version in range(20, 6, -1):
-        binary = 'clang-format-%d' % clang_version
+        binary = f'clang-format-{clang_version}'
+
         if which(binary):
-            clang_format[0] = binary
-            break
+            return binary
+
+    return 'clang-format'
+
+
+def find_diffs(files):
+    """Run clang-format and diff it against a file.
+    """
+    found_diffs = False
+
+    for file in files:
+        cli.log.debug('Checking for changes in %s', file)
+        clang_format = subprocess.Popen([find_clang_format(), file], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
+        diff = cli.run(['diff', '-u', f'--label=a/{file}', f'--label=b/{file}', str(file), '-'], stdin=clang_format.stdout, capture_output=True)
+
+        if diff.returncode != 0:
+            print(diff.stdout)
+            found_diffs = True
+
+    return found_diffs
+
+
+def cformat_run(files):
+    """Spawn clang-format subprocess with proper arguments
+    """
+    # Determine which version of clang-format to use
+    clang_format = [find_clang_format(), '-i']
+
     try:
-        if not files:
-            cli.log.warn('No changes detected. Use "qmk cformat -a" to format all files')
-            return False
-        subprocess.run(clang_format + [file for file in files], check=True)
+        cli.run(clang_format + list(map(str, files)), check=True, capture_output=False)
         cli.log.info('Successfully formatted the C code.')
+        return True
 
-    except subprocess.CalledProcessError:
+    except subprocess.CalledProcessError as e:
         cli.log.error('Error formatting C code!')
+        cli.log.debug('%s exited with returncode %s', e.cmd, e.returncode)
+        cli.log.debug('STDOUT:')
+        cli.log.debug(e.stdout)
+        cli.log.debug('STDERR:')
+        cli.log.debug(e.stderr)
         return False
 
 
-@cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all core files.')
+def filter_files(files):
+    """Yield only files to be formatted and skip the rest
+    """
+    for file in files:
+        if file.name.split('.')[-1] in c_file_suffixes:
+            yield file
+        else:
+            cli.log.debug('Skipping file %s', file)
+
+
+@cli.argument('-n', '--dry-run', arg_only=True, action='store_true', help="Flag only, don't automatically format.")
 @cli.argument('-b', '--base-branch', default='origin/master', help='Branch to compare to diffs to.')
-@cli.argument('files', nargs='*', arg_only=True, completer=FilesCompleter('.c'), help='Filename(s) to format.')
+@cli.argument('-a', '--all-files', arg_only=True, action='store_true', help='Format all core files.')
+@cli.argument('files', nargs='*', arg_only=True, type=normpath, completer=FilesCompleter('.c'), help='Filename(s) to format.')
 @cli.subcommand("Format C code according to QMK's style.", hidden=False if cli.config.user.developer else True)
 def cformat(cli):
     """Format C code according to QMK's style.
     """
-    # Empty array for files
-    files = []
-    # Core directories for formatting
-    core_dirs = ['drivers', 'quantum', 'tests', 'tmk_core', 'platforms']
-    ignores = ['tmk_core/protocol/usb_hid', 'quantum/template', 'platforms/chibios']
     # Find the list of files to format
     if cli.args.files:
-        files.extend(normpath(file) for file in cli.args.files)
+        files = list(filter_files(cli.args.files))
+
+        if not files:
+            cli.log.error('No C files in filelist: %s', ', '.join(map(str, cli.args.files)))
+            exit(0)
+
         if cli.args.all_files:
             cli.log.warning('Filenames passed with -a, only formatting: %s', ','.join(map(str, files)))
-    # If -a is specified
+
     elif cli.args.all_files:
         all_files = c_source_files(core_dirs)
         # The following statement checks each file to see if the file path is in the ignored directories.
-        files.extend(file for file in all_files if not any(i in str(file) for i in ignores))
-    # No files specified & no -a flag
+        files = [file for file in all_files if not any(i in str(file) for i in ignored)]
+
     else:
-        base_args = ['git', 'diff', '--name-only', cli.args.base_branch]
-        out = subprocess.run(base_args + core_dirs, check=True, stdout=subprocess.PIPE)
-        changed_files = filter(None, out.stdout.decode('UTF-8').split('\n'))
-        filtered_files = [normpath(file) for file in changed_files if not any(i in file for i in ignores)]
-        files.extend(file for file in filtered_files if file.exists() and file.suffix in ['.c', '.h', '.cpp'])
+        git_diff_cmd = ['git', 'diff', '--name-only', cli.args.base_branch, *core_dirs]
+        git_diff = cli.run(git_diff_cmd)
+
+        if git_diff.returncode != 0:
+            cli.log.error("Error running %s", git_diff_cmd)
+            print(git_diff.stderr)
+            return git_diff.returncode
+
+        files = []
+
+        for file in git_diff.stdout.strip().split('\n'):
+            if not any([file.startswith(ignore) for ignore in ignored]):
+                if path.exists(file) and file.split('.')[-1] in c_file_suffixes:
+                    files.append(file)
+
+    # Sanity check
+    if not files:
+        cli.log.error('No changed files detected. Use "qmk cformat -a" to format all files')
+        return False
 
     # Run clang-format on the files we've found
-    cformat_run(files, cli.args.all_files)
+    if cli.args.dry_run:
+        return not find_diffs(files)
+    else:
+        return cformat_run(files)