From 0692756e656ffeb1864284c20e897fc38e8cbcfd Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Tue, 19 Sep 2023 21:20:06 +1000 Subject: [PATCH 01/14] Add `qmk license-check` command for RFC. --- lib/python/qmk/cli/__init__.py | 1 + lib/python/qmk/cli/license_check.py | 69 +++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 lib/python/qmk/cli/license_check.py diff --git a/lib/python/qmk/cli/__init__.py b/lib/python/qmk/cli/__init__.py index 9c3decf4f76a..749cf450c367 100644 --- a/lib/python/qmk/cli/__init__.py +++ b/lib/python/qmk/cli/__init__.py @@ -68,6 +68,7 @@ 'qmk.cli.import.keymap', 'qmk.cli.info', 'qmk.cli.json2c', + 'qmk.cli.license_check', 'qmk.cli.lint', 'qmk.cli.kle2json', 'qmk.cli.list.keyboards', diff --git a/lib/python/qmk/cli/license_check.py b/lib/python/qmk/cli/license_check.py new file mode 100644 index 000000000000..5cdcc45d6b6d --- /dev/null +++ b/lib/python/qmk/cli/license_check.py @@ -0,0 +1,69 @@ +import re +from pathlib import Path +from milc import cli + +licenses = [ + ('GPL-2.0-or-later', + [""" + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + """,""" + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or any later version. + """]), + ('GPL-2.0-only', + [""" + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; version 2. + """]) +] + +lparen = re.compile(r'\(\[\{\<') +rparen = re.compile(r'\)\]\}\>') +punctuation = re.compile(r'[\.,;:]+') +trash_prefix = re.compile(r'^(\s|/|\*|#)+') +trash_suffix = re.compile(r'(\s|/|\*|#|\\)+$') +spaces = re.compile(r'\s+') + +def _simplify_text(input): + lines = input.split('\n') + lines = [l.lower() for l in lines] + lines = [punctuation.sub('', l) for l in lines] + lines = [trash_prefix.sub('', l) for l in lines] + lines = [trash_suffix.sub('', l) for l in lines] + lines = [spaces.sub(' ', l) for l in lines] + lines = [lparen.sub('(', l) for l in lines] + lines = [rparen.sub(')', l) for l in lines] + lines = [l.strip() for l in lines] + lines = [l for l in lines if l is not None and l != ''] + return ' '.join(lines) + +@cli.argument('filename', arg_only=True, type=Path, help='Input file') +@cli.subcommand('File license check.', hidden=False if cli.config.user.developer else True) +def license_check(cli): + data = cli.args.filename.read_text() + if 'SPDX-License-Identifier:' in data: + + res = data.split('SPDX-License-Identifier:') + license = re.split(r'\s|//|\*', res[1].strip())[0].strip() + cli.log.info(f'{{fg_cyan}}{cli.args.filename}{{fg_reset}} -- license found: {license} (SPDX License Identifier)') + return True + + else: + + linear_text = _simplify_text(data) + + for short_license, long_licenses in licenses: + for long_license in long_licenses: + license_text = _simplify_text(long_license) + if license_text in linear_text: + cli.log.info(f'{{fg_cyan}}{cli.args.filename}{{fg_reset}} -- license found: {short_license} (Raw text)') + return True + + cli.log.error(f'{{fg_cyan}}{cli.args.filename}{{fg_reset}} -- no license found, or unknown license!') + return False From d1cc8811e7a14bc585baf4bec0dd3c64726d68e3 Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Tue, 19 Sep 2023 21:29:07 +1000 Subject: [PATCH 02/14] Multi-file support. --- lib/python/qmk/cli/license_check.py | 44 ++++++++++++++++++----------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/lib/python/qmk/cli/license_check.py b/lib/python/qmk/cli/license_check.py index 5cdcc45d6b6d..a5ada64f4c34 100644 --- a/lib/python/qmk/cli/license_check.py +++ b/lib/python/qmk/cli/license_check.py @@ -43,27 +43,37 @@ def _simplify_text(input): lines = [l for l in lines if l is not None and l != ''] return ' '.join(lines) -@cli.argument('filename', arg_only=True, type=Path, help='Input file') +@cli.argument('filenames', nargs='*', arg_only=True, type=Path, help='Input files') @cli.subcommand('File license check.', hidden=False if cli.config.user.developer else True) def license_check(cli): - data = cli.args.filename.read_text() - if 'SPDX-License-Identifier:' in data: + # Pre-format all the licenses + for short_license, long_licenses in licenses: + for i in range(len(long_licenses)): + long_licenses[i] = _simplify_text(long_licenses[i]) - res = data.split('SPDX-License-Identifier:') - license = re.split(r'\s|//|\*', res[1].strip())[0].strip() - cli.log.info(f'{{fg_cyan}}{cli.args.filename}{{fg_reset}} -- license found: {license} (SPDX License Identifier)') - return True + failed = False + for filename in sorted(cli.args.filenames): + data = filename.read_text() + if 'SPDX-License-Identifier:' in data: - else: + res = data.split('SPDX-License-Identifier:') + license = re.split(r'\s|//|\*', res[1].strip())[0].strip() + cli.log.info(f'{{fg_cyan}}{filename}{{fg_reset}} -- license found: {license} (SPDX License Identifier)') - linear_text = _simplify_text(data) + else: - for short_license, long_licenses in licenses: - for long_license in long_licenses: - license_text = _simplify_text(long_license) - if license_text in linear_text: - cli.log.info(f'{{fg_cyan}}{cli.args.filename}{{fg_reset}} -- license found: {short_license} (Raw text)') - return True + linear_text = _simplify_text(data) - cli.log.error(f'{{fg_cyan}}{cli.args.filename}{{fg_reset}} -- no license found, or unknown license!') - return False + found = False + for short_license, long_licenses in licenses: + for long_license in long_licenses: + if long_license in linear_text: + cli.log.info(f'{{fg_cyan}}{filename}{{fg_reset}} -- license found: {short_license} (Raw text)') + found = True + + if not found: + cli.log.error(f'{{fg_cyan}}{filename}{{fg_reset}} -- unknown license, or no license found!') + failed = True + + if failed: + return False \ No newline at end of file From 7c2813bb6afac661cfab937a7ba296cdeab3f795 Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Tue, 19 Sep 2023 21:35:29 +1000 Subject: [PATCH 03/14] `qmk pytest` --- lib/python/qmk/cli/license_check.py | 42 ++++++++++++++++------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/lib/python/qmk/cli/license_check.py b/lib/python/qmk/cli/license_check.py index a5ada64f4c34..e0143e318cbd 100644 --- a/lib/python/qmk/cli/license_check.py +++ b/lib/python/qmk/cli/license_check.py @@ -2,26 +2,25 @@ from pathlib import Path from milc import cli -licenses = [ - ('GPL-2.0-or-later', - [""" +licenses = [( + 'GPL-2.0-or-later', [ + """ This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - """,""" + """, """ This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or any later version. - """]), - ('GPL-2.0-only', - [""" + """ + ] +), ('GPL-2.0-only', [""" This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2. - """]) -] + """])] lparen = re.compile(r'\(\[\{\<') rparen = re.compile(r'\)\]\}\>') @@ -30,19 +29,21 @@ trash_suffix = re.compile(r'(\s|/|\*|#|\\)+$') spaces = re.compile(r'\s+') + def _simplify_text(input): lines = input.split('\n') - lines = [l.lower() for l in lines] - lines = [punctuation.sub('', l) for l in lines] - lines = [trash_prefix.sub('', l) for l in lines] - lines = [trash_suffix.sub('', l) for l in lines] - lines = [spaces.sub(' ', l) for l in lines] - lines = [lparen.sub('(', l) for l in lines] - lines = [rparen.sub(')', l) for l in lines] - lines = [l.strip() for l in lines] - lines = [l for l in lines if l is not None and l != ''] + lines = [line.lower() for line in lines] + lines = [punctuation.sub('', line) for line in lines] + lines = [trash_prefix.sub('', line) for line in lines] + lines = [trash_suffix.sub('', line) for line in lines] + lines = [spaces.sub(' ', line) for line in lines] + lines = [lparen.sub('(', line) for line in lines] + lines = [rparen.sub(')', line) for line in lines] + lines = [line.strip() for line in lines] + lines = [line for line in lines if line is not None and line != ''] return ' '.join(lines) + @cli.argument('filenames', nargs='*', arg_only=True, type=Path, help='Input files') @cli.subcommand('File license check.', hidden=False if cli.config.user.developer else True) def license_check(cli): @@ -66,14 +67,17 @@ def license_check(cli): found = False for short_license, long_licenses in licenses: + if found: + break for long_license in long_licenses: if long_license in linear_text: cli.log.info(f'{{fg_cyan}}{filename}{{fg_reset}} -- license found: {short_license} (Raw text)') found = True + break if not found: cli.log.error(f'{{fg_cyan}}{filename}{{fg_reset}} -- unknown license, or no license found!') failed = True if failed: - return False \ No newline at end of file + return False From 9db09c41270c681ddb9dec67cdac04fca53b4fa4 Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Tue, 19 Sep 2023 21:43:41 +1000 Subject: [PATCH 04/14] Short output --- lib/python/qmk/cli/license_check.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/python/qmk/cli/license_check.py b/lib/python/qmk/cli/license_check.py index e0143e318cbd..2f51b65c3bcc 100644 --- a/lib/python/qmk/cli/license_check.py +++ b/lib/python/qmk/cli/license_check.py @@ -45,6 +45,7 @@ def _simplify_text(input): @cli.argument('filenames', nargs='*', arg_only=True, type=Path, help='Input files') +@cli.argument('-s', '--short', action='store_true', help='Short output') @cli.subcommand('File license check.', hidden=False if cli.config.user.developer else True) def license_check(cli): # Pre-format all the licenses @@ -59,7 +60,10 @@ def license_check(cli): res = data.split('SPDX-License-Identifier:') license = re.split(r'\s|//|\*', res[1].strip())[0].strip() - cli.log.info(f'{{fg_cyan}}{filename}{{fg_reset}} -- license found: {license} (SPDX License Identifier)') + if cli.args.short: + print(f'{filename} {license}') + else: + cli.log.info(f'{{fg_cyan}}{filename}{{fg_reset}} -- license found: {license} (SPDX License Identifier)') else: @@ -71,12 +75,18 @@ def license_check(cli): break for long_license in long_licenses: if long_license in linear_text: - cli.log.info(f'{{fg_cyan}}{filename}{{fg_reset}} -- license found: {short_license} (Raw text)') + if cli.args.short: + print(f'{filename} {short_license}') + else: + cli.log.info(f'{{fg_cyan}}{filename}{{fg_reset}} -- license found: {short_license} (Raw text)') found = True break if not found: - cli.log.error(f'{{fg_cyan}}{filename}{{fg_reset}} -- unknown license, or no license found!') + if cli.args.short: + print(f'{filename} UNKNOWN') + else: + cli.log.error(f'{{fg_cyan}}{filename}{{fg_reset}} -- unknown license, or no license found!') failed = True if failed: From 288077b1961c829135e657ca2f13e2a08fb4b8c6 Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Tue, 19 Sep 2023 21:54:25 +1000 Subject: [PATCH 05/14] Apache-2.0 --- lib/python/qmk/cli/license_check.py | 41 ++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/lib/python/qmk/cli/license_check.py b/lib/python/qmk/cli/license_check.py index 2f51b65c3bcc..c78759aaa019 100644 --- a/lib/python/qmk/cli/license_check.py +++ b/lib/python/qmk/cli/license_check.py @@ -2,25 +2,52 @@ from pathlib import Path from milc import cli -licenses = [( - 'GPL-2.0-or-later', [ - """ +licenses = [ + ( + 'GPL-2.0-or-later', [ + """\ This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. - """, """ + """, """\ This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or any later version. """ - ] -), ('GPL-2.0-only', [""" + ] + ), + ('GPL-2.0-only', ["""\ This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2. - """])] + """]), + ( + 'GPL-3.0-or-later', [ + """\ + This program is free software: you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of + the License, or (at your option) any later version. + """, """\ + This program is free software: you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of + the License, or any later version. + """ + ] + ), + ('GPL-3.0-only', ["""\ + This program is free software: you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation, version 3. + """]), + ('Apache-2.0', ["""\ + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + """]), +] lparen = re.compile(r'\(\[\{\<') rparen = re.compile(r'\)\]\}\>') From 62e0f5a4e7c3a517d39b1015715e940e9f6e614e Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Tue, 19 Sep 2023 22:07:27 +1000 Subject: [PATCH 06/14] File extensions, directory search support. --- lib/python/qmk/cli/license_check.py | 78 +++++++++++++++++------------ 1 file changed, 46 insertions(+), 32 deletions(-) diff --git a/lib/python/qmk/cli/license_check.py b/lib/python/qmk/cli/license_check.py index c78759aaa019..325e528deb24 100644 --- a/lib/python/qmk/cli/license_check.py +++ b/lib/python/qmk/cli/license_check.py @@ -55,6 +55,7 @@ trash_prefix = re.compile(r'^(\s|/|\*|#)+') trash_suffix = re.compile(r'(\s|/|\*|#|\\)+$') spaces = re.compile(r'\s+') +suffixes = ['.c', '.h', '.cpp', '.cxx', '.hpp', '.hxx', '.def'] def _simplify_text(input): @@ -71,6 +72,43 @@ def _simplify_text(input): return ' '.join(lines) +def _check_file(filename): + data = filename.read_text(encoding='utf-8', errors='ignore') + if 'SPDX-License-Identifier:' in data: + + res = data.split('SPDX-License-Identifier:') + license = re.split(r'\s|//|\*', res[1].strip())[0].strip() + if cli.args.short: + print(f'{filename} {license}') + else: + cli.log.info(f'{{fg_cyan}}{filename}{{fg_reset}} -- license found: {license} (SPDX License Identifier)') + return True + + else: + + linear_text = _simplify_text(data) + + found = False + for short_license, long_licenses in licenses: + if found: + break + for long_license in long_licenses: + if long_license in linear_text: + if cli.args.short: + print(f'{filename} {short_license}') + else: + cli.log.info(f'{{fg_cyan}}{filename}{{fg_reset}} -- license found: {short_license} (Raw text)') + return True + + if not found: + if cli.args.short: + print(f'{filename} UNKNOWN') + else: + cli.log.error(f'{{fg_cyan}}{filename}{{fg_reset}} -- unknown license, or no license found!') + + return False + + @cli.argument('filenames', nargs='*', arg_only=True, type=Path, help='Input files') @cli.argument('-s', '--short', action='store_true', help='Short output') @cli.subcommand('File license check.', hidden=False if cli.config.user.developer else True) @@ -82,39 +120,15 @@ def license_check(cli): failed = False for filename in sorted(cli.args.filenames): - data = filename.read_text() - if 'SPDX-License-Identifier:' in data: - - res = data.split('SPDX-License-Identifier:') - license = re.split(r'\s|//|\*', res[1].strip())[0].strip() - if cli.args.short: - print(f'{filename} {license}') - else: - cli.log.info(f'{{fg_cyan}}{filename}{{fg_reset}} -- license found: {license} (SPDX License Identifier)') - + if filename.is_dir(): + for file in sorted(filename.rglob('*')): + if file.is_file() and file.suffix in suffixes: + if not _check_file(file): + failed = True else: - - linear_text = _simplify_text(data) - - found = False - for short_license, long_licenses in licenses: - if found: - break - for long_license in long_licenses: - if long_license in linear_text: - if cli.args.short: - print(f'{filename} {short_license}') - else: - cli.log.info(f'{{fg_cyan}}{filename}{{fg_reset}} -- license found: {short_license} (Raw text)') - found = True - break - - if not found: - if cli.args.short: - print(f'{filename} UNKNOWN') - else: - cli.log.error(f'{{fg_cyan}}{filename}{{fg_reset}} -- unknown license, or no license found!') - failed = True + if filename.suffix in suffixes: + if not _check_file(filename): + failed = True if failed: return False From 067eecab5ecfeee1c8e5ce1807c34c9c75a2aafc Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Tue, 19 Sep 2023 22:09:10 +1000 Subject: [PATCH 07/14] Yep. --- lib/python/qmk/cli/license_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/python/qmk/cli/license_check.py b/lib/python/qmk/cli/license_check.py index 325e528deb24..16129e284d17 100644 --- a/lib/python/qmk/cli/license_check.py +++ b/lib/python/qmk/cli/license_check.py @@ -125,7 +125,7 @@ def license_check(cli): if file.is_file() and file.suffix in suffixes: if not _check_file(file): failed = True - else: + elif filename.is_file(): if filename.suffix in suffixes: if not _check_file(filename): failed = True From c2570cdf8fdf9f0bf63203d4f01d3beb6b04b8dc Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Tue, 19 Sep 2023 22:56:04 +1000 Subject: [PATCH 08/14] Collect then process. --- lib/python/qmk/cli/license_check.py | 84 ++++++++++++++--------------- 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/lib/python/qmk/cli/license_check.py b/lib/python/qmk/cli/license_check.py index 16129e284d17..864b71a870ad 100644 --- a/lib/python/qmk/cli/license_check.py +++ b/lib/python/qmk/cli/license_check.py @@ -2,7 +2,7 @@ from pathlib import Path from milc import cli -licenses = [ +LICENSE_TEXTS = [ ( 'GPL-2.0-or-later', [ """\ @@ -49,86 +49,84 @@ """]), ] -lparen = re.compile(r'\(\[\{\<') -rparen = re.compile(r'\)\]\}\>') -punctuation = re.compile(r'[\.,;:]+') -trash_prefix = re.compile(r'^(\s|/|\*|#)+') -trash_suffix = re.compile(r'(\s|/|\*|#|\\)+$') -spaces = re.compile(r'\s+') -suffixes = ['.c', '.h', '.cpp', '.cxx', '.hpp', '.hxx', '.def'] +L_PAREN = re.compile(r'\(\[\{\<') +R_PAREN = re.compile(r'\)\]\}\>') +PUNCTUATION = re.compile(r'[\.,;:]+') +TRASH_PREFIX = re.compile(r'^(\s|/|\*|#)+') +TRASH_SUFFIX = re.compile(r'(\s|/|\*|#|\\)+$') +SPACE = re.compile(r'\s+') +SUFFIXES = ['.c', '.h', '.cpp', '.cxx', '.hpp', '.hxx', '.py'] # , '.def', '.mk'] # ? def _simplify_text(input): - lines = input.split('\n') - lines = [line.lower() for line in lines] - lines = [punctuation.sub('', line) for line in lines] - lines = [trash_prefix.sub('', line) for line in lines] - lines = [trash_suffix.sub('', line) for line in lines] - lines = [spaces.sub(' ', line) for line in lines] - lines = [lparen.sub('(', line) for line in lines] - lines = [rparen.sub(')', line) for line in lines] + lines = input.lower().split('\n') + lines = [PUNCTUATION.sub('', line) for line in lines] + lines = [TRASH_PREFIX.sub('', line) for line in lines] + lines = [TRASH_SUFFIX.sub('', line) for line in lines] + lines = [SPACE.sub(' ', line) for line in lines] + lines = [L_PAREN.sub('(', line) for line in lines] + lines = [R_PAREN.sub(')', line) for line in lines] lines = [line.strip() for line in lines] lines = [line for line in lines if line is not None and line != ''] return ' '.join(lines) -def _check_file(filename): +def _detect_license_from_file_contents(filename, absolute=False): data = filename.read_text(encoding='utf-8', errors='ignore') - if 'SPDX-License-Identifier:' in data: + filename_out = str(filename.absolute()) if absolute else str(filename) + if 'SPDX-License-Identifier:' in data: res = data.split('SPDX-License-Identifier:') license = re.split(r'\s|//|\*', res[1].strip())[0].strip() if cli.args.short: - print(f'{filename} {license}') + print(f'{filename_out} {license}') else: - cli.log.info(f'{{fg_cyan}}{filename}{{fg_reset}} -- license found: {license} (SPDX License Identifier)') + cli.log.info(f'{{fg_cyan}}{filename_out}{{fg_reset}} -- license detected: {license} (SPDX License Identifier)') return True else: - - linear_text = _simplify_text(data) - - found = False - for short_license, long_licenses in licenses: - if found: - break + simple_text = _simplify_text(data) + for short_license, long_licenses in LICENSE_TEXTS: for long_license in long_licenses: - if long_license in linear_text: + if long_license in simple_text: if cli.args.short: - print(f'{filename} {short_license}') + print(f'{filename_out} {short_license}') else: - cli.log.info(f'{{fg_cyan}}{filename}{{fg_reset}} -- license found: {short_license} (Raw text)') + cli.log.info(f'{{fg_cyan}}{filename_out}{{fg_reset}} -- license detected: {short_license} (Full text)') return True - if not found: - if cli.args.short: - print(f'{filename} UNKNOWN') - else: - cli.log.error(f'{{fg_cyan}}{filename}{{fg_reset}} -- unknown license, or no license found!') + if cli.args.short: + print(f'{filename_out} UNKNOWN') + else: + cli.log.error(f'{{fg_cyan}}{filename_out}{{fg_reset}} -- unknown license, or no license detected!') return False @cli.argument('filenames', nargs='*', arg_only=True, type=Path, help='Input files') @cli.argument('-s', '--short', action='store_true', help='Short output') +@cli.argument('-a', '--absolute', action='store_true', help='Print absolute paths') @cli.subcommand('File license check.', hidden=False if cli.config.user.developer else True) def license_check(cli): # Pre-format all the licenses - for short_license, long_licenses in licenses: + for _, long_licenses in LICENSE_TEXTS: for i in range(len(long_licenses)): long_licenses[i] = _simplify_text(long_licenses[i]) - failed = False + check_list = set() for filename in sorted(cli.args.filenames): if filename.is_dir(): for file in sorted(filename.rglob('*')): - if file.is_file() and file.suffix in suffixes: - if not _check_file(file): - failed = True + if file.is_file() and file.suffix in SUFFIXES: + check_list.add(file) elif filename.is_file(): - if filename.suffix in suffixes: - if not _check_file(filename): - failed = True + if filename.suffix in SUFFIXES: + check_list.add(filename) + + failed = False + for filename in sorted(check_list): + if not _detect_license_from_file_contents(filename, absolute=cli.args.absolute): + failed = True if failed: return False From 1664e005c7bff304349fcd48ea7d7cdf6007556d Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Tue, 19 Sep 2023 23:06:13 +1000 Subject: [PATCH 09/14] Ensure we know about the SPDX short-form license names. --- lib/python/qmk/cli/license_check.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/python/qmk/cli/license_check.py b/lib/python/qmk/cli/license_check.py index 864b71a870ad..15a504fe3efa 100644 --- a/lib/python/qmk/cli/license_check.py +++ b/lib/python/qmk/cli/license_check.py @@ -78,6 +78,20 @@ def _detect_license_from_file_contents(filename, absolute=False): if 'SPDX-License-Identifier:' in data: res = data.split('SPDX-License-Identifier:') license = re.split(r'\s|//|\*', res[1].strip())[0].strip() + found = False + for short_license, _ in LICENSE_TEXTS: + if license.lower() == short_license.lower(): + license = short_license + found = True + break + + if not found: + if cli.args.short: + print(f'{filename_out} UNKNOWN') + else: + cli.log.error(f'{{fg_cyan}}{filename_out}{{fg_reset}} -- unknown license, or no license detected!') + return False + if cli.args.short: print(f'{filename_out} {license}') else: From 1653b9c34b62d283d0e4b1a876a2096add209dfe Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Tue, 19 Sep 2023 23:38:35 +1000 Subject: [PATCH 10/14] LGPL, slight restructure. --- lib/python/qmk/cli/license_check.py | 48 +---------- lib/python/qmk/constants.py | 123 ++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 47 deletions(-) diff --git a/lib/python/qmk/cli/license_check.py b/lib/python/qmk/cli/license_check.py index 15a504fe3efa..c28dcfe0dffb 100644 --- a/lib/python/qmk/cli/license_check.py +++ b/lib/python/qmk/cli/license_check.py @@ -1,53 +1,7 @@ import re from pathlib import Path from milc import cli - -LICENSE_TEXTS = [ - ( - 'GPL-2.0-or-later', [ - """\ - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - """, """\ - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or any later version. - """ - ] - ), - ('GPL-2.0-only', ["""\ - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation; version 2. - """]), - ( - 'GPL-3.0-or-later', [ - """\ - This program is free software: you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of - the License, or (at your option) any later version. - """, """\ - This program is free software: you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation, either version 3 of - the License, or any later version. - """ - ] - ), - ('GPL-3.0-only', ["""\ - This program is free software: you can redistribute it and/or - modify it under the terms of the GNU General Public License as - published by the Free Software Foundation, version 3. - """]), - ('Apache-2.0', ["""\ - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - """]), -] +from qmk.constants import LICENSE_TEXTS L_PAREN = re.compile(r'\(\[\{\<') R_PAREN = re.compile(r'\)\]\}\>') diff --git a/lib/python/qmk/constants.py b/lib/python/qmk/constants.py index 97bd84aa2344..1967441fc89a 100644 --- a/lib/python/qmk/constants.py +++ b/lib/python/qmk/constants.py @@ -189,3 +189,126 @@ # ################################################################################ ''' + +LICENSE_TEXTS = [ + ( + 'GPL-2.0-or-later', [ + """\ + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + """, """\ + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or any later version. + """ + ] + ), + ('GPL-2.0-only', ["""\ + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation; version 2. + """]), + ( + 'GPL-3.0-or-later', [ + """\ + This program is free software: you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of + the License, or (at your option) any later version. + """, """\ + This program is free software: you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation, either version 3 of + the License, or any later version. + """ + ] + ), + ('GPL-3.0-only', ["""\ + This program is free software: you can redistribute it and/or + modify it under the terms of the GNU General Public License as + published by the Free Software Foundation, version 3. + """]), + ( + 'LGPL-2.1-or-later', [ + """\ + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License + as published by the Free Software Foundation; either version 2.1 + of the License, or (at your option) any later version. + """, """\ + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License + as published by the Free Software Foundation; either version 2.1 + of the License, or any later version. + """, """\ + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License + as published by the Free Software Foundation; either version 2.1 + of the License, or (at your option) any later version. + """, """\ + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License + as published by the Free Software Foundation; either version 2.1 + of the License, or any later version. + """ + ] + ), + ( + 'LGPL-2.1-only', [ + """\ + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation; version 2.1. + """, """\ + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation; version 2.1. + """ + ] + ), + ( + 'LGPL-3.0-or-later', [ + """\ + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License + as published by the Free Software Foundation; either version 3 + of the License, or (at your option) any later version. + """, """\ + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License + as published by the Free Software Foundation; either version 3 + of the License, or any later version. + """, """\ + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License + as published by the Free Software Foundation; either version 3 + of the License, or (at your option) any later version. + """, """\ + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License + as published by the Free Software Foundation; either version 3 + of the License, or any later version. + """ + ] + ), + ( + 'LGPL-3.0-only', [ + """\ + This program is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation; version 3. + """, """\ + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation; version 3. + """ + ] + ), + ('Apache-2.0', ["""\ + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + """]), +] From 0ae39538ff298cbf997a940d06014659ec8ac504 Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Fri, 22 Sep 2023 16:19:52 +1000 Subject: [PATCH 11/14] Extension specification. --- lib/python/qmk/cli/license_check.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/lib/python/qmk/cli/license_check.py b/lib/python/qmk/cli/license_check.py index c28dcfe0dffb..88a46e55acc1 100644 --- a/lib/python/qmk/cli/license_check.py +++ b/lib/python/qmk/cli/license_check.py @@ -9,7 +9,7 @@ TRASH_PREFIX = re.compile(r'^(\s|/|\*|#)+') TRASH_SUFFIX = re.compile(r'(\s|/|\*|#|\\)+$') SPACE = re.compile(r'\s+') -SUFFIXES = ['.c', '.h', '.cpp', '.cxx', '.hpp', '.hxx', '.py'] # , '.def', '.mk'] # ? +SUFFIXES = ['.c', '.h', '.cpp', '.cxx', '.hpp', '.hxx'] def _simplify_text(input): @@ -71,24 +71,31 @@ def _detect_license_from_file_contents(filename, absolute=False): return False -@cli.argument('filenames', nargs='*', arg_only=True, type=Path, help='Input files') -@cli.argument('-s', '--short', action='store_true', help='Short output') -@cli.argument('-a', '--absolute', action='store_true', help='Print absolute paths') +@cli.argument('inputs', nargs='*', arg_only=True, type=Path, help='List of input files or directories.') +@cli.argument('-s', '--short', action='store_true', help='Short output.') +@cli.argument('-a', '--absolute', action='store_true', help='Print absolute paths.') +@cli.argument('-e', '--extension', arg_only=True, action='append', default=[], help='Override list of extensions. Can be specified multiple times for multiple extensions.') @cli.subcommand('File license check.', hidden=False if cli.config.user.developer else True) def license_check(cli): + conditional = lambda s: s in SUFFIXES + + if len(cli.args.extension) > 0: + suffixes = [f'.{s}' if not s.startswith('.') else s for s in cli.args.extension] + conditional = lambda s: s in suffixes + # Pre-format all the licenses for _, long_licenses in LICENSE_TEXTS: for i in range(len(long_licenses)): long_licenses[i] = _simplify_text(long_licenses[i]) check_list = set() - for filename in sorted(cli.args.filenames): + for filename in sorted(cli.args.inputs): if filename.is_dir(): for file in sorted(filename.rglob('*')): - if file.is_file() and file.suffix in SUFFIXES: + if file.is_file() and conditional(file.suffix): check_list.add(file) elif filename.is_file(): - if filename.suffix in SUFFIXES: + if conditional(file.suffix): check_list.add(filename) failed = False From 76d76588d2a628fd30490634e198daa384f7623b Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Wed, 8 Nov 2023 21:04:29 +1100 Subject: [PATCH 12/14] Fixup CI/tests. --- lib/python/qmk/cli/license_check.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/python/qmk/cli/license_check.py b/lib/python/qmk/cli/license_check.py index 88a46e55acc1..3c5ed9796951 100644 --- a/lib/python/qmk/cli/license_check.py +++ b/lib/python/qmk/cli/license_check.py @@ -1,3 +1,5 @@ +# Copyright 2023 Nick Brassel (@tzarc) +# SPDX-License-Identifier: GPL-2.0-or-later import re from pathlib import Path from milc import cli @@ -77,11 +79,18 @@ def _detect_license_from_file_contents(filename, absolute=False): @cli.argument('-e', '--extension', arg_only=True, action='append', default=[], help='Override list of extensions. Can be specified multiple times for multiple extensions.') @cli.subcommand('File license check.', hidden=False if cli.config.user.developer else True) def license_check(cli): - conditional = lambda s: s in SUFFIXES + def defaultsuffix_condition(s): + return s in SUFFIXES + + conditional = defaultsuffix_condition if len(cli.args.extension) > 0: suffixes = [f'.{s}' if not s.startswith('.') else s for s in cli.args.extension] - conditional = lambda s: s in suffixes + + def specific_suffix_condition(s): + return s in suffixes + + conditional = specific_suffix_condition # Pre-format all the licenses for _, long_licenses in LICENSE_TEXTS: From 479903d0c7242154020e0695d56c02499d8cba2a Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Wed, 8 Nov 2023 21:09:17 +1100 Subject: [PATCH 13/14] Fixup CI/tests. --- lib/python/qmk/cli/license_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/python/qmk/cli/license_check.py b/lib/python/qmk/cli/license_check.py index 3c5ed9796951..1d30d760ef2c 100644 --- a/lib/python/qmk/cli/license_check.py +++ b/lib/python/qmk/cli/license_check.py @@ -104,7 +104,7 @@ def specific_suffix_condition(s): if file.is_file() and conditional(file.suffix): check_list.add(file) elif filename.is_file(): - if conditional(file.suffix): + if conditional(filename.suffix): check_list.add(filename) failed = False From a0d4e50fdb75c7f3c6fc1afce36a0325b22915ac Mon Sep 17 00:00:00 2001 From: Nick Brassel Date: Wed, 8 Nov 2023 22:20:45 +1100 Subject: [PATCH 14/14] Fixup CI/tests. --- lib/python/qmk/cli/license_check.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/python/qmk/cli/license_check.py b/lib/python/qmk/cli/license_check.py index 1d30d760ef2c..4bda272ec9bb 100644 --- a/lib/python/qmk/cli/license_check.py +++ b/lib/python/qmk/cli/license_check.py @@ -79,18 +79,18 @@ def _detect_license_from_file_contents(filename, absolute=False): @cli.argument('-e', '--extension', arg_only=True, action='append', default=[], help='Override list of extensions. Can be specified multiple times for multiple extensions.') @cli.subcommand('File license check.', hidden=False if cli.config.user.developer else True) def license_check(cli): - def defaultsuffix_condition(s): + def _default_suffix_condition(s): return s in SUFFIXES - conditional = defaultsuffix_condition + conditional = _default_suffix_condition if len(cli.args.extension) > 0: suffixes = [f'.{s}' if not s.startswith('.') else s for s in cli.args.extension] - def specific_suffix_condition(s): + def _specific_suffix_condition(s): return s in suffixes - conditional = specific_suffix_condition + conditional = _specific_suffix_condition # Pre-format all the licenses for _, long_licenses in LICENSE_TEXTS: