6f10dc1101
for example this was not found: extensions/source/update/check/updatecheck.cxx:58:namespace frame = com::sun:⭐:frame ; due to the missing using and the space before the semicolon but IWYU correctly detected it as unused Change-Id: I404c93d41aec1a571c9ac49db7b7c7e8da824bf6 Reviewed-on: https://gerrit.libreoffice.org/c/core/+/165698 Tested-by: Jenkins Reviewed-by: Gabor Kelemen <gabor.kelemen.extern@allotropia.de>
567 lines
24 KiB
Python
Executable file
567 lines
24 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
#
|
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
#
|
|
# This parses the output of 'include-what-you-use', focusing on just removing
|
|
# not needed includes and providing a relatively conservative output by
|
|
# filtering out a number of LibreOffice-specific false positives.
|
|
#
|
|
# It assumes you have a 'compile_commands.json' around (similar to clang-tidy),
|
|
# you can generate one with 'make vim-ide-integration'.
|
|
#
|
|
# Design goals:
|
|
# - excludelist mechanism, so a warning is either fixed or excluded
|
|
# - works in a plugins-enabled clang build
|
|
# - no custom configure options required
|
|
# - no need to generate a dummy library to build a header
|
|
|
|
import json
|
|
import multiprocessing
|
|
import os
|
|
import queue
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import yaml
|
|
import argparse
|
|
import pathlib
|
|
|
|
|
|
def ignoreRemoval(include, toAdd, absFileName, moduleRules, noexclude):
|
|
# global rules
|
|
|
|
# Avoid replacing .hpp with .hdl in the com::sun::star and ooo::vba namespaces.
|
|
if ( include.startswith("com/sun/star") or include.startswith("ooo/vba") ) and include.endswith(".hpp"):
|
|
hdl = include.replace(".hpp", ".hdl")
|
|
if hdl in toAdd:
|
|
return True
|
|
|
|
# Avoid debug STL.
|
|
debugStl = {
|
|
"array": ("debug/array", ),
|
|
"bitset": ("debug/bitset", ),
|
|
"deque": ("debug/deque", ),
|
|
"forward_list": ("debug/forward_list", ),
|
|
"list": ("debug/list", ),
|
|
"map": ("debug/map.h", "debug/multimap.h"),
|
|
"set": ("debug/set.h", "debug/multiset.h"),
|
|
"unordered_map": ("debug/unordered_map", ),
|
|
"unordered_set": ("debug/unordered_set", ),
|
|
"vector": ("debug/vector", ),
|
|
}
|
|
for k, values in debugStl.items():
|
|
if include == k:
|
|
for value in values:
|
|
if value in toAdd:
|
|
return True
|
|
|
|
# Avoid proposing to use libstdc++ internal headers.
|
|
bits = {
|
|
"exception": "bits/exception.h",
|
|
"memory": "bits/shared_ptr.h",
|
|
"functional": "bits/std_function.h",
|
|
"cmath": "bits/std_abs.h",
|
|
"ctime": "bits/types/clock_t.h",
|
|
"cstdint": "bits/stdint-uintn.h",
|
|
}
|
|
for k, v in bits.items():
|
|
if include == k and v in toAdd:
|
|
return True
|
|
|
|
# Avoid proposing o3tl fw declaration
|
|
o3tl = {
|
|
"o3tl/typed_flags_set.hxx" : "namespace o3tl { template <typename T> struct typed_flags; }",
|
|
"o3tl/deleter.hxx" : "namespace o3tl { template <typename T> struct default_delete; }",
|
|
}
|
|
for k, v, in o3tl.items():
|
|
if include == k and v in toAdd:
|
|
return True
|
|
|
|
# Follow boost documentation.
|
|
if include == "boost/optional.hpp" and "boost/optional/optional.hpp" in toAdd:
|
|
return True
|
|
if include == "boost/intrusive_ptr.hpp" and "boost/smart_ptr/intrusive_ptr.hpp" in toAdd:
|
|
return True
|
|
if include == "boost/shared_ptr.hpp" and "boost/smart_ptr/shared_ptr.hpp" in toAdd:
|
|
return True
|
|
if include == "boost/variant.hpp" and "boost/variant/variant.hpp" in toAdd:
|
|
return True
|
|
if include == "boost/unordered_map.hpp" and "boost/unordered/unordered_map.hpp" in toAdd:
|
|
return True
|
|
if include == "boost/functional/hash.hpp" and "boost/container_hash/extensions.hpp" in toAdd:
|
|
return True
|
|
|
|
# Avoid .hxx to .h proposals in basic css/uno/* API
|
|
unoapi = {
|
|
"com/sun/star/uno/Any.hxx": "com/sun/star/uno/Any.h",
|
|
"com/sun/star/uno/Reference.hxx": "com/sun/star/uno/Reference.h",
|
|
"com/sun/star/uno/Sequence.hxx": "com/sun/star/uno/Sequence.h",
|
|
"com/sun/star/uno/Type.hxx": "com/sun/star/uno/Type.h"
|
|
}
|
|
for k, v in unoapi.items():
|
|
if include == k and v in toAdd:
|
|
return True
|
|
|
|
# 3rd-party, non-self-contained headers.
|
|
if include == "libepubgen/libepubgen.h" and "libepubgen/libepubgen-decls.h" in toAdd:
|
|
return True
|
|
if include == "librevenge/librevenge.h" and "librevenge/RVNGPropertyList.h" in toAdd:
|
|
return True
|
|
if include == "libetonyek/libetonyek.h" and "libetonyek/EtonyekDocument.h" in toAdd:
|
|
return True
|
|
|
|
noRemove = (
|
|
# <https://www.openoffice.org/tools/CodingGuidelines.sxw> insists on not
|
|
# removing this.
|
|
"sal/config.h",
|
|
# Works around a build breakage specific to the broken Android
|
|
# toolchain.
|
|
"android/compatibility.hxx",
|
|
# Removing this would change the meaning of '#if defined OSL_BIGENDIAN'.
|
|
"osl/endian.h",
|
|
)
|
|
if include in noRemove:
|
|
return True
|
|
|
|
# Ignore when <foo> is to be replaced with "foo".
|
|
if include in toAdd:
|
|
return True
|
|
|
|
fileName = os.path.relpath(absFileName, os.getcwd())
|
|
|
|
# Skip headers used only for compile test
|
|
if fileName == "cppu/qa/cppumaker/test_cppumaker.cxx":
|
|
if include.endswith(".hpp"):
|
|
return True
|
|
|
|
# yaml rules, except when --noexclude is given
|
|
|
|
if "excludelist" in moduleRules.keys() and not noexclude:
|
|
excludelistRules = moduleRules["excludelist"]
|
|
if fileName in excludelistRules.keys():
|
|
if include in excludelistRules[fileName]:
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
def unwrapInclude(include):
|
|
# Drop <> or "" around the include.
|
|
return include[1:-1]
|
|
|
|
|
|
def processIWYUOutput(iwyuOutput, moduleRules, fileName, noexclude, checknamespaces):
|
|
inAdd = False
|
|
toAdd = []
|
|
inRemove = False
|
|
toRemove = []
|
|
inFull = False
|
|
currentFileName = None
|
|
|
|
for line in iwyuOutput:
|
|
line = line.strip()
|
|
|
|
# Bail out if IWYU gave an error due to non self-containedness
|
|
if re.match ("(.*): error: (.*)", line):
|
|
return -1
|
|
|
|
if len(line) == 0:
|
|
if inRemove:
|
|
inRemove = False
|
|
continue
|
|
if inAdd:
|
|
inAdd = False
|
|
continue
|
|
if inFull:
|
|
inFull = False
|
|
continue
|
|
|
|
shouldAdd = fileName + " should add these lines:"
|
|
match = re.match(shouldAdd, line)
|
|
if match:
|
|
currentFileName = match.group(0).split(' ')[0]
|
|
inAdd = True
|
|
continue
|
|
|
|
shouldRemove = fileName + " should remove these lines:"
|
|
match = re.match(shouldRemove, line)
|
|
if match:
|
|
currentFileName = match.group(0).split(' ')[0]
|
|
inRemove = True
|
|
continue
|
|
|
|
if checknamespaces:
|
|
match = re.match("The full include-list for " + fileName, line)
|
|
if match:
|
|
inFull = True
|
|
continue
|
|
|
|
if inAdd:
|
|
match = re.match('#include ([^ ]+)', line)
|
|
if match:
|
|
include = unwrapInclude(match.group(1))
|
|
toAdd.append(include)
|
|
else:
|
|
# Forward declaration.
|
|
toAdd.append(line)
|
|
|
|
if inRemove and not checknamespaces:
|
|
match = re.match("- #include (.*) // lines (.*)-.*", line)
|
|
if match:
|
|
# Only suggest removals for now. Removing fwd decls is more complex: they may be
|
|
# indeed unused or they may removed to be replaced with an include. And we want to
|
|
# avoid the later.
|
|
include = unwrapInclude(match.group(1))
|
|
lineno = match.group(2)
|
|
if not ignoreRemoval(include, toAdd, currentFileName, moduleRules, noexclude):
|
|
toRemove.append("%s:%s: %s" % (currentFileName, lineno, include))
|
|
|
|
if inFull:
|
|
if checknamespaces:
|
|
# match for all possible URE/UNO namespaces, created with:
|
|
# find udkapi/com/sun/star/ -type d | sort| xargs basename -a | tr '\012' '|'
|
|
# find offapi/com/sun/star/ -type d | sort | xargs basename -a | tr '\012' '|'
|
|
# and ooo::vba namespaces
|
|
# plus a few popular ones about other modules
|
|
ns = re.compile(
|
|
'.*for\ ('
|
|
# URE namespaces
|
|
'beans|'
|
|
'bridge|oleautomation|'
|
|
'connection|'
|
|
'container|'
|
|
'io|'
|
|
'java|'
|
|
'lang|'
|
|
'loader|'
|
|
'reflection|'
|
|
'registry|'
|
|
'script|'
|
|
'security|'
|
|
'task|'
|
|
'uno|'
|
|
'uri|'
|
|
'util|'
|
|
# UNO namespaces
|
|
'accessibility|'
|
|
'animations|'
|
|
'auth|'
|
|
'awt|tab|tree|grid|'
|
|
'chart|'
|
|
'chart2|data|'
|
|
'configuration|bootstrap|backend|xml|'
|
|
'cui|'
|
|
'datatransfer|clipboard|dnd|'
|
|
'deployment|test|ui|'
|
|
'document|'
|
|
'drawing|framework|'
|
|
'embed|'
|
|
'form|binding|runtime|control|inspection|submission|component|validation|'
|
|
'formula|'
|
|
'frame|status|'
|
|
'gallery|'
|
|
'geometry|'
|
|
'graphic|'
|
|
'i18n|'
|
|
'image|'
|
|
'inspection|'
|
|
'ldap|'
|
|
'linguistic2|'
|
|
'logging|'
|
|
'mail|'
|
|
'media|'
|
|
'mozilla|'
|
|
'office|'
|
|
'packages|zip|manifest|'
|
|
'presentation|textfield|'
|
|
'qa|'
|
|
'rdf|'
|
|
'rendering|'
|
|
'report|inspection|meta|'
|
|
'resource|'
|
|
'scanner|'
|
|
'script|vba|browse|provider|'
|
|
'sdb|application|tools|'
|
|
'sdbc|'
|
|
'sdbcx|'
|
|
'security|'
|
|
'setup|'
|
|
'sheet|opencl|'
|
|
'smarttags|'
|
|
'style|'
|
|
'svg|'
|
|
'system|windows|'
|
|
'table|'
|
|
'task|'
|
|
'text|textfield|docinfo|fieldmaster|'
|
|
'tiledrendering|'
|
|
'ucb|'
|
|
'ui|dialogs|test|'
|
|
'util|'
|
|
'view|'
|
|
'xforms|'
|
|
'xml|xslt|wrapper|csax|sax|input|xpath|dom|views|events|crypto|sax|'
|
|
'xsd|'
|
|
# ooo::vba and its namespaces
|
|
'ooo|vba|excel|powerpoint|adodb|access|office|word|stdole|msforms|dao|'
|
|
# use of module namespaces, as spotted in the code
|
|
'analysis|pricing' # sca internals
|
|
'apphelper|CloneHelper|DataSeriesProperties|SceneProperties|wrapper|' # for chart internals
|
|
'basegfx|utils|'
|
|
'boost|posix_time|gregorian'
|
|
'cairo|'
|
|
'canvas|'
|
|
'chelp|'
|
|
'comphelper|'
|
|
'connectivity|'
|
|
'cpp|java|' # for codemaker::
|
|
'cppu|'
|
|
'dbaccess|dbahsql|dbaui|dbtools|'
|
|
'desktop|dp_misc|'
|
|
'drawinglayer|attribute|geometry|primitive2d|processor2d|'
|
|
'editeng|'
|
|
'emscripten|'
|
|
'formula|'
|
|
'framework|'
|
|
'frm|'
|
|
'http_dav_ucp|tdoc_ucp|package_ucp|hierarchy_ucp|gio|fileaccess|ucb_impl|hcp_impl|ucb_cmdenv|' # for ucb internal
|
|
'i18npool|'
|
|
'internal|ColorComponentTag|' # for slideshow internals
|
|
'jfw_plugin|'
|
|
'jni_uno|'
|
|
'librevenge|'
|
|
'linguistic|'
|
|
'lok|'
|
|
'mtv|' # for mdds::mtv
|
|
'nsSwDocInfoSubType|SWUnoHelper|nsHdFtFlags|' # sw internal
|
|
'o3tl|'
|
|
'odfflatxml|' # filter internal
|
|
'oox|core|drawingml|ole|vml|'
|
|
'OpenStormBento|'
|
|
'osl|'
|
|
'pdfi|pdfparse|'
|
|
'ppt|'
|
|
'pyuno|'
|
|
'reportdesign|'
|
|
'rptui|'
|
|
'rtl|math|textenc|'
|
|
'salhelper|'
|
|
'sax_fastparser|'
|
|
'sax|' # for xml::sax
|
|
'sc|'
|
|
'SchXMLTools|' # for xmloff
|
|
'sd|slidesorter|cache|controller|model|view|'
|
|
'sf_misc|'
|
|
'sfx2|DocTempl|'
|
|
'sidebar|' # for sfx2::sidebar
|
|
'skeletonmaker|'
|
|
'star|' # for com::sun::star
|
|
'std|chrono_literals|literals|'
|
|
'stoc_sec|'
|
|
'store|'
|
|
'svl|impl|'
|
|
'svt|'
|
|
'svtools|'
|
|
'svx|sdr|contact|table|'
|
|
'sw|access|annotation|mark|types|util|'
|
|
'toolkit|'
|
|
'treeview|'
|
|
'ucbhelper|'
|
|
'unodevtools'
|
|
'unopkg|'
|
|
'util|db|qe|' # for xmlsearch::
|
|
'utl|'
|
|
'vcl|psp|x11|'
|
|
'writerfilter|'
|
|
'xforms|'
|
|
'xmloff|token|EnhancedCustomShapeToken' # for xmloff::
|
|
'ZipUtils'
|
|
')$', re.VERBOSE
|
|
)
|
|
|
|
reason = re.match(ns, line)
|
|
if reason:
|
|
# Warn about namespaces: if a header is suggested only '// for $namespace', then the namespace is not used
|
|
# otherwise the used classes name would show up after the '// for'
|
|
# Cleaning out the respective header (if there is any
|
|
# - which is not always the case) is for the next run!
|
|
nameSpace = reason.group(1).split(' ')[0]
|
|
print("WARNING:", fileName, "This 'using namespace' is likely unnecessary:", nameSpace)
|
|
|
|
# Get the row number, normal IWYU output does not contain this info
|
|
subprocess.run(["git", "grep", "-n", "namespace.*[^a-zA-Z]"+nameSpace+" *;", fileName])
|
|
|
|
for remove in sorted(toRemove):
|
|
print("ERROR: %s: remove not needed include" % remove)
|
|
return len(toRemove)
|
|
|
|
|
|
def run_tool(task_queue, failed_files, dontstop, noexclude, checknamespaces):
|
|
while True:
|
|
invocation, moduleRules = task_queue.get()
|
|
if not len(failed_files):
|
|
print("[IWYU] " + invocation.split(' ')[-1])
|
|
p = subprocess.Popen(invocation, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
retcode = processIWYUOutput(p.communicate()[0].decode('utf-8').splitlines(), moduleRules, invocation.split(' ')[-1], noexclude, checknamespaces)
|
|
if retcode == -1 and not checknamespaces:
|
|
print("ERROR: A file is probably not self contained, check this commands output:\n" + invocation)
|
|
elif retcode > 0:
|
|
print("ERROR: The following command found unused includes:\n" + invocation)
|
|
if not dontstop:
|
|
failed_files.append(invocation)
|
|
task_queue.task_done()
|
|
if checknamespaces:
|
|
# Workaround: sometimes running git grep makes the letters typed into the terminal disappear after the script is finished
|
|
os.system('stty sane')
|
|
|
|
|
|
def isInUnoIncludeFile(path):
|
|
return path.startswith("include/com/") \
|
|
or path.startswith("include/cppu/") \
|
|
or path.startswith("include/cppuhelper/") \
|
|
or path.startswith("include/osl/") \
|
|
or path.startswith("include/rtl/") \
|
|
or path.startswith("include/sal/") \
|
|
or path.startswith("include/salhelper/") \
|
|
or path.startswith("include/systools/") \
|
|
or path.startswith("include/typelib/") \
|
|
or path.startswith("include/uno/")
|
|
|
|
|
|
def tidy(compileCommands, paths, dontstop, noexclude,checknamespaces):
|
|
return_code = 0
|
|
|
|
try:
|
|
max_task = multiprocessing.cpu_count()
|
|
task_queue = queue.Queue(max_task)
|
|
failed_files = []
|
|
for _ in range(max_task):
|
|
t = threading.Thread(target=run_tool, args=(task_queue, failed_files, dontstop, noexclude,checknamespaces))
|
|
t.daemon = True
|
|
t.start()
|
|
|
|
for path in sorted(paths):
|
|
if isInUnoIncludeFile(path):
|
|
continue
|
|
|
|
# IWYU fails on these with #error: don't use this in new code
|
|
if path.startswith("include/vcl/toolkit"):
|
|
continue
|
|
|
|
moduleName = path.split("/")[0]
|
|
|
|
rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml")
|
|
moduleRules = {}
|
|
if os.path.exists(rulePath):
|
|
moduleRules = yaml.full_load(open(rulePath))
|
|
assume = None
|
|
pathAbs = os.path.abspath(path)
|
|
compileFile = pathAbs
|
|
matches = [i for i in compileCommands if i["file"] == compileFile]
|
|
if not len(matches):
|
|
# Only use assume-filename for headers, so we don't try to analyze e.g. Windows-only
|
|
# code on Linux.
|
|
if "assumeFilename" in moduleRules.keys() and not path.endswith("cxx"):
|
|
assume = moduleRules["assumeFilename"]
|
|
if assume:
|
|
assumeAbs = os.path.abspath(assume)
|
|
compileFile = assumeAbs
|
|
matches = [i for i in compileCommands if i["file"] == compileFile]
|
|
if not len(matches):
|
|
print("WARNING: no compile commands for '" + path + "' (assumed filename: '" + assume + "'")
|
|
continue
|
|
else:
|
|
print("WARNING: no compile commands for '" + path + "'")
|
|
continue
|
|
|
|
_, _, args = matches[0]["command"].partition(" ")
|
|
if assume:
|
|
args = args.replace(assumeAbs, "-x c++ " + pathAbs)
|
|
|
|
invocation = "include-what-you-use -Xiwyu --no_fwd_decls -Xiwyu --max_line_length=200 " + args
|
|
task_queue.put((invocation, moduleRules))
|
|
|
|
task_queue.join()
|
|
if len(failed_files):
|
|
return_code = 1
|
|
|
|
except KeyboardInterrupt:
|
|
print('\nCtrl-C detected, goodbye.')
|
|
os.kill(0, 9)
|
|
|
|
sys.exit(return_code)
|
|
|
|
|
|
def main(argv):
|
|
parser = argparse.ArgumentParser(description='Check source files for unneeded includes.')
|
|
parser.add_argument('--continue', action='store_true',
|
|
help='Don\'t stop on errors. Useful for periodic re-check of large amount of files')
|
|
parser.add_argument('Files' , nargs='*',
|
|
help='The files to be checked')
|
|
parser.add_argument('--recursive', metavar='DIR', nargs=1, type=str,
|
|
help='Recursively search a directory for source files to check')
|
|
parser.add_argument('--headers', action='store_true',
|
|
help='Check header files. If omitted, check source files. Use with --recursive.')
|
|
parser.add_argument('--noexclude', action='store_true',
|
|
help='Ignore excludelist. Useful to check whether its exclusions are still all valid.')
|
|
parser.add_argument('--ns', action='store_true',
|
|
help='Warn about unused "using namespace" statements. '
|
|
'Removing these may uncover more removable headers '
|
|
'in a subsequent normal run')
|
|
|
|
args = parser.parse_args()
|
|
|
|
if not len(argv):
|
|
parser.print_help()
|
|
return
|
|
|
|
list_of_files = []
|
|
if args.recursive:
|
|
for root, dirs, files in os.walk(args.recursive[0]):
|
|
for file in files:
|
|
if args.headers:
|
|
if (file.endswith(".hxx") or file.endswith(".hrc") or file.endswith(".h")):
|
|
list_of_files.append(os.path.join(root,file))
|
|
else:
|
|
if (file.endswith(".cxx") or file.endswith(".c")):
|
|
list_of_files.append(os.path.join(root,file))
|
|
else:
|
|
list_of_files = args.Files
|
|
|
|
try:
|
|
with open("compile_commands.json", 'r') as compileCommandsSock:
|
|
compileCommands = json.load(compileCommandsSock)
|
|
except FileNotFoundError:
|
|
print ("File 'compile_commands.json' does not exist, please run:\nmake vim-ide-integration")
|
|
sys.exit(-1)
|
|
|
|
# quickly sanity check whether files with exceptions in yaml still exists
|
|
# only check for the module of the very first filename passed
|
|
|
|
# Verify there are files selected for checking, with --recursive it
|
|
# may happen that there are in fact no C/C++ files in a module directory
|
|
if not list_of_files:
|
|
print("No files found to check!")
|
|
sys.exit(-2)
|
|
|
|
moduleName = sorted(list_of_files)[0].split("/")[0]
|
|
rulePath = os.path.join(moduleName, "IwyuFilter_" + moduleName + ".yaml")
|
|
moduleRules = {}
|
|
if os.path.exists(rulePath):
|
|
moduleRules = yaml.full_load(open(rulePath))
|
|
if "excludelist" in moduleRules.keys():
|
|
excludelistRules = moduleRules["excludelist"]
|
|
for pathname in excludelistRules.keys():
|
|
file = pathlib.Path(pathname)
|
|
if not file.exists():
|
|
print("WARNING: File listed in " + rulePath + " no longer exists: " + pathname)
|
|
|
|
tidy(compileCommands, paths=list_of_files, dontstop=vars(args)["continue"], noexclude=args.noexclude, checknamespaces=args.ns)
|
|
|
|
if __name__ == '__main__':
|
|
main(sys.argv[1:])
|
|
|
|
# vim:set shiftwidth=4 softtabstop=4 expandtab:
|