From 1db51e999610d3a14bed0172cc3e959cfab28e77 Mon Sep 17 00:00:00 2001 From: David Chalco <59750547+dachalco@users.noreply.github.com> Date: Wed, 2 Dec 2020 09:52:19 -0800 Subject: [PATCH] Header Checker: Misc Enhancements (#437) * +regex ignores, tentative complete lists for configs * Use header checks from user repo --- .github/scripts/check-header.py | 428 +++++++++++++++++++++++----- .github/workflows/header-checks.yml | 7 +- 2 files changed, 359 insertions(+), 76 deletions(-) diff --git a/.github/scripts/check-header.py b/.github/scripts/check-header.py index ce6de5534..e3f3607d6 100755 --- a/.github/scripts/check-header.py +++ b/.github/scripts/check-header.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import os, sys +import os, sys, re from argparse import ArgumentParser from difflib import unified_diff from json import load @@ -9,11 +9,11 @@ def dprint(msg): print('[DEBUG]: %s' % str(msg)) class HeaderChecker: - def __init__(self, header, padding=1000, ignored_files=[], ignored_ext=[], ignored_paths=[]): + def __init__(self, header, padding=1000, ignored_files=[], ignored_ext=[], ignored_patterns=[]): self.padding = padding self.header = header - self.ignorePathList = ignored_paths.copy() + self.ignorePatternList = ignored_patterns.copy() self.ignoreFileList = ignored_files.copy() self.ignoreExtList = ignored_ext.copy() @@ -67,9 +67,9 @@ class HeaderChecker: for f in args: self.ignoreFileList.append(f) - def ignorePath(self, *args): + def ignorePattern(self, *args): for p in args: - self.ignorePathList.append(p) + self.ignorePatternList.append(re.compile(p)) def isIgnoredFile(self, path): ''' @@ -77,11 +77,19 @@ class HeaderChecker: ''' assert os.path.exists(path), 'No such file: ' + path + + # Try simpler checks first filename = os.path.split(path)[-1] extension = os.path.splitext(filename)[-1] - return (path in self.ignorePathList - or extension in self.ignoreExtList - or filename in self.ignoreFileList) + if extension in self.ignoreExtList or filename in self.ignoreFileList: + return True + + # Then iterate against regex patterns. In future consider Trie + for pattern in self.ignorePatternList: + if pattern.match(path): + return True + + return False def configArgParser(): @@ -105,76 +113,352 @@ def configArgParser(): help = 'Treat arguments json files that store a list of files to check.') return parser +#-------------------------------------------------------------------------------------------------- +# CONFIG +#-------------------------------------------------------------------------------------------------- +FREERTOS_IGNORED_EXTENSIONS = [ + '.1', + '.ASM', + '.C', + '.DSW', + '.G_C', + '.H', + '.Hbp', + '.IDE', + '.LIB', + '.Opt', + '.PC', + '.PRM', + '.TXT', + '.URL', + '.UVL', + '.Uv2', + '.a', + '.ac', + '.am', + '.atsln', + '.atstart', + '.atsuo', + '.bash', + '.bat', + '.bbl', + '.bit', + '.board', + '.bsb', + '.bsdl', + '.bts', + '.ccxml', + '.cdkproj', + '.cdkws', + '.cfg', + '.cgp', + '.cmake', + '.cmd', + '.config', + '.cpp', + '.cproj', + '.crun', + '.css', + '.csv', + '.custom_argvars', + '.cxx', + '.cydwr', + '.cyprj', + '.cysch', + '.dat', + '.datas', + '.db', + '.dbgdt', + '.dep', + '.dni', + '.dnx', + '.doc', + '.dox', + '.doxygen', + '.ds', + '.dsk', + '.dtd', + '.dts', + '.elf', + '.env_conf', + '.ewd', + '.ewp', + '.ewt', + '.eww', + '.exe', + '.filters', + '.flash', + '.fmt', + '.ftl', + '.gdb', + '.gif', + '.gise', + '.gld', + '.gpdsc', + '.gui', + '.h_from_toolchain', + '.hdf', + '.hdp', + '.hex', + '.hist', + '.history', + '.hsf', + '.htm', + '.html', + '.hwc', + '.hwl', + '.hwp', + '.hws', + '.hzp', + '.hzs', + '.i', + '.icf', + '.ide', + '.idx', + '.in', + '.inc', + '.include', + '.index', + '.inf', + '.ini', + '.init', + '.ipcf', + '.ise', + '.jlink', + '.json', + '.la', + '.launch', + '.lcf', + '.lds', + '.lib', + '.lk1', + '.lkr', + '.lm', + '.lo', + '.lock', + '.lsl', + '.lst', + '.m4', + '.mac', + '.make', + '.map', + '.mbt', + '.mcp', + '.mcpar', + '.mcs', + '.mcw', + '.mdm', + '.mem', + '.mhs', + '.mk', + '.mk1', + '.mmi', + '.mrt', + '.mss', + '.mtpj', + '.nav', + '.ntrc_log', + '.opa', + '.opb', + '.opc', + '.opl', + '.opt', + '.opv', + '.out', + '.pack', + '.par', + '.patch', + '.pbd', + '.pdsc', + '.pe', + '.pem', + '.pgs', + '.pl', + '.plg', + '.png', + '.prc', + '.pref', + '.prefs', + '.prj', + '.properties', + '.ps1', + '.ptf', + '.r79', + '.rapp', + '.rc', + '.reggroups', + '.reglist', + '.resc', + '.resources', + '.rom', + '.rprj', + '.s79', + '.s82', + '.s90', + '.sc', + '.scf', + '.scfg', + '.script', + '.sct', + '.scvd', + '.session', + '.sfr', + '.sh', + '.shtml', + '.sig', + '.sln', + '.spec', + '.stf', + '.stg', + '.suo', + '.sup', + '.svg', + '.tags', + '.tcl', + '.tdt', + '.template', + '.tgt', + '.tps', + '.tra', + '.tree', + '.tws', + '.ucf', + '.url', + '.user', + '.ut', + '.uvmpw', + '.uvopt', + '.uvoptx', + '.uvproj', + '.uvprojx', + '.vcproj', + '.vcxproj', + '.version', + '.webserver', + '.wpj', + '.wsdt', + '.wsp', + '.wspos', + '.wsx', + '.x', + '.xbcd', + '.xcl', + '.xise', + '.xml', + '.xmp', + '.xmsgs', + '.xsl', + '.yml', + '.zip' +] + +FREERTOS_IGNORED_PATTERNS = [ + r'.*\.git.*', + r'.*mbedtls_config\.h.*', +] + +FREERTOS_HEADER = [ + '/*\n', + ' * FreeRTOS V202011.00\n', + ' * Copyright (C) 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n', + ' *\n', + ' * Permission is hereby granted, free of charge, to any person obtaining a copy of\n', + ' * this software and associated documentation files (the "Software"), to deal in\n', + ' * the Software without restriction, including without limitation the rights to\n', + ' * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\n', + ' * the Software, and to permit persons to whom the Software is furnished to do so,\n', + ' * subject to the following conditions:\n', + ' *\n', + ' * The above copyright notice and this permission notice shall be included in all\n', + ' * copies or substantial portions of the Software.\n', + ' *\n', + ' * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n', + ' * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\n', + ' * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\n', + ' * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\n', + ' * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\n', + ' * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n', + ' *\n', + ' * https://www.FreeRTOS.org\n', + ' * https://github.com/FreeRTOS\n', + ' *\n', + ' */\n', +] + +KERNEL_IGNORED_EXTENSIONS = [ + '.yml', + '.css', + '.idx', + '.md', + '.url', + '.sty', + '.0-rc2', + '.s82', + '.js', + '.out', + '.pack', + '.2', + '.1-kernel-only', + '.0-kernel-only', + '.0-rc1', + '.readme', + '.tex', + '.png', + '.bat', + '.sh' +] + +KERNEL_IGNORED_PATTERNS = [r'.*\.git.*'] + +KERNEL_HEADER = [ + '/*\n', + ' * FreeRTOS Kernel V10.4.2\n', + ' * Copyright (C) 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n', + ' *\n', + ' * Permission is hereby granted, free of charge, to any person obtaining a copy of\n', + ' * this software and associated documentation files (the "Software"), to deal in\n', + ' * the Software without restriction, including without limitation the rights to\n', + ' * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\n', + ' * the Software, and to permit persons to whom the Software is furnished to do so,\n', + ' * subject to the following conditions:\n', + ' *\n', + ' * The above copyright notice and this permission notice shall be included in all\n', + ' * copies or substantial portions of the Software.\n', + ' *\n', + ' * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n', + ' * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\n', + ' * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\n', + ' * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\n', + ' * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\n', + ' * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n', + ' *\n', + ' * https://www.FreeRTOS.org\n', + ' * https://github.com/FreeRTOS\n', + ' *\n', + ' */\n', +] + + +#-------------------------------------------------------------------------------------------------- +# MAIN +#-------------------------------------------------------------------------------------------------- def main(): parser = configArgParser() args = parser.parse_args() - freertos_header = [ - '/*\n', - ' * FreeRTOS V202011.00\n', - ' * Copyright (C) 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n', - ' *\n', - ' * Permission is hereby granted, free of charge, to any person obtaining a copy of\n', - ' * this software and associated documentation files (the "Software"), to deal in\n', - ' * the Software without restriction, including without limitation the rights to\n', - ' * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\n', - ' * the Software, and to permit persons to whom the Software is furnished to do so,\n', - ' * subject to the following conditions:\n', - ' *\n', - ' * The above copyright notice and this permission notice shall be included in all\n', - ' * copies or substantial portions of the Software.\n', - ' *\n', - ' * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n', - ' * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\n', - ' * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\n', - ' * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\n', - ' * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\n', - ' * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n', - ' *\n', - ' * https://www.FreeRTOS.org\n', - ' * https://github.com/FreeRTOS\n', - ' *\n', - ' */\n', - ] + # Configure checks + if args.kernel: + checker = HeaderChecker(KERNEL_HEADER) + checker.ignoreExtension(*KERNEL_IGNORED_EXTENSIONS) + checker.ignorePattern(*KERNEL_IGNORED_PATTERNS) + else: + checker = HeaderChecker(FREERTOS_HEADER) + checker.ignoreExtension(*FREERTOS_IGNORED_EXTENSIONS) + checker.ignorePattern(*FREERTOS_IGNORED_PATTERNS) - kernel_header = [ - '/*\n', - ' * FreeRTOS Kernel V10.4.2\n', - ' * Copyright (C) 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n', - ' *\n', - ' * Permission is hereby granted, free of charge, to any person obtaining a copy of\n', - ' * this software and associated documentation files (the "Software"), to deal in\n', - ' * the Software without restriction, including without limitation the rights to\n', - ' * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\n', - ' * the Software, and to permit persons to whom the Software is furnished to do so,\n', - ' * subject to the following conditions:\n', - ' *\n', - ' * The above copyright notice and this permission notice shall be included in all\n', - ' * copies or substantial portions of the Software.\n', - ' *\n', - ' * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n', - ' * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\n', - ' * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\n', - ' * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\n', - ' * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\n', - ' * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n', - ' *\n', - ' * https://www.FreeRTOS.org\n', - ' * https://github.com/FreeRTOS\n', - ' *\n', - ' */\n', - ] - - checker = HeaderChecker(kernel_header if args.kernel else freertos_header) - checker.ignoreExtension('.vcxproj', - '.vcxproj.filters', - '.sln' - '.md,' - '.gitmodules') - - checker.ignoreFile(os.path.split(__file__)[-1], # Add self - 'mbedtls_config.h') + checker.ignoreFile(os.path.split(__file__)[-1]) + # Check all input files print() n_failed = 0 for path in args.files_checked: diff --git a/.github/workflows/header-checks.yml b/.github/workflows/header-checks.yml index 30468ee0e..5b1f6371e 100644 --- a/.github/workflows/header-checks.yml +++ b/.github/workflows/header-checks.yml @@ -16,12 +16,11 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # Get latest checks from master + # Use the checks as defined by the user, so they can locally adjust as needed - name: Checkout FreeRTOS Tools uses: actions/checkout@v2 with: - repository: FreeRTOS/FreeRTOS - ref: master + ref: ${{ github.event.pull_request.head.sha }} path: tools # Checkout user pull request changes @@ -41,6 +40,6 @@ jobs: - name: Check File Headers run: | cd inspect - ../tools/.github/scripts/check-header.py --json ${HOME}/files.json + ../tools/.github/scripts/check-header.py --json ${HOME}/files_modified.json ${HOME}/files_added.json ${HOME}/files_renamed.json exit $?