Skip to content

bpo-36876: Small adjustments to the C-analyzer tool. #23045

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
8 changes: 5 additions & 3 deletions Tools/c-analyzer/c_analyzer/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,12 @@
from c_parser.info import (
KIND,
TypeDeclaration,
filter_by_kind,
collate_by_kind_group,
resolve_parsed,
)
from c_parser.match import (
filter_by_kind,
group_by_kinds,
)
from . import (
analyze as _analyze,
datafiles as _datafiles,
Expand Down Expand Up @@ -55,7 +57,7 @@ def analyze_decls(decls, known, *,
)

decls = list(decls)
collated = collate_by_kind_group(decls)
collated = group_by_kinds(decls)

types = {decl: None for decl in collated['type']}
typespecs = _analyze.get_typespecs(types)
Expand Down
104 changes: 68 additions & 36 deletions Tools/c-analyzer/c_analyzer/__main__.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import io
import logging
import os
import os.path
import re
import sys
Expand All @@ -9,6 +10,7 @@
add_verbosity_cli,
add_traceback_cli,
add_sepval_cli,
add_progress_cli,
add_files_cli,
add_commands_cli,
process_args_by_key,
Expand All @@ -17,11 +19,13 @@
filter_filenames,
iter_marks,
)
from c_parser.info import KIND, is_type_decl
from c_parser.info import KIND
from c_parser.match import is_type_decl
from .match import filter_forward
from . import (
analyze as _analyze,
check_all as _check_all,
datafiles as _datafiles,
check_all as _check_all,
)


Expand All @@ -44,7 +48,7 @@
TABLE_SECTIONS = {
'types': (
['kind', 'name', 'data', 'file'],
is_type_decl,
KIND.is_type_decl,
(lambda v: (v.kind.value, v.filename or '', v.name)),
),
'typedefs': 'types',
Expand Down Expand Up @@ -167,9 +171,7 @@ def handle_failure(failure, data):
print(f'{data.filename}:{name} - {failure}')
elif fmt == 'summary':
def handle_failure(failure, data):
parent = data.parent or ''
funcname = parent if isinstance(parent, str) else parent.name
print(f'{data.filename:35}\t{funcname or "-":35}\t{data.name:40}\t{failure}')
print(_fmt_one_summary(data, failure))
elif fmt == 'full':
div = ''
def handle_failure(failure, data):
Expand Down Expand Up @@ -230,6 +232,15 @@ def section(name):
yield f'grand total: {total}'


def _fmt_one_summary(item, extra=None):
parent = item.parent or ''
funcname = parent if isinstance(parent, str) else parent.name
if extra:
return f'{item.filename:35}\t{funcname or "-":35}\t{item.name:40}\t{extra}'
else:
return f'{item.filename:35}\t{funcname or "-":35}\t{item.name}'


def fmt_full(analysis):
# XXX Support sorting.
items = sorted(analysis, key=lambda v: v.key)
Expand Down Expand Up @@ -272,10 +283,12 @@ def process_checks(args):
args.checks = [check]
else:
process_checks = add_checks_cli(parser, checks=checks)
process_progress = add_progress_cli(parser)
process_output = add_output_cli(parser, default=None)
process_files = add_files_cli(parser, **kwargs)
return [
process_checks,
process_progress,
process_output,
process_files,
]
Expand All @@ -288,6 +301,7 @@ def cmd_check(filenames, *,
relroot=None,
failfast=False,
iter_filenames=None,
track_progress=None,
verbosity=VERBOSITY,
_analyze=_analyze,
_CHECKS=CHECKS,
Expand All @@ -304,36 +318,53 @@ def cmd_check(filenames, *,
) = _get_check_handlers(fmt, printer, verbosity)

filenames = filter_filenames(filenames, iter_filenames)
if track_progress:
filenames = track_progress(filenames)

logger.info('analyzing...')
logger.info('analyzing files...')
analyzed = _analyze(filenames, **kwargs)
if relroot:
analyzed.fix_filenames(relroot)
decls = filter_forward(analyzed, markpublic=True)

logger.info('checking...')
numfailed = 0
for data, failure in _check_all(analyzed, checks, failfast=failfast):
logger.info('checking analysis results...')
failed = []
for data, failure in _check_all(decls, checks, failfast=failfast):
if data is None:
printer.info('stopping after one failure')
break
if div is not None and numfailed > 0:
if div is not None and len(failed) > 0:
printer.info(div)
numfailed += 1
failed.append(data)
handle_failure(failure, data)
handle_after()

printer.info('-------------------------')
logger.info(f'total failures: {numfailed}')
logger.info(f'total failures: {len(failed)}')
logger.info('done checking')

if numfailed > 0:
sys.exit(numfailed)
if fmt == 'summary':
print('Categorized by storage:')
print()
from .match import group_by_storage
grouped = group_by_storage(failed, ignore_non_match=False)
for group, decls in grouped.items():
print()
print(group)
for decl in decls:
print(' ', _fmt_one_summary(decl))
print(f'subtotal: {len(decls)}')

if len(failed) > 0:
sys.exit(len(failed))


def _cli_analyze(parser, **kwargs):
process_progress = add_progress_cli(parser)
process_output = add_output_cli(parser)
process_files = add_files_cli(parser, **kwargs)
return [
process_progress,
process_output,
process_files,
]
Expand All @@ -343,6 +374,7 @@ def _cli_analyze(parser, **kwargs):
def cmd_analyze(filenames, *,
fmt=None,
iter_filenames=None,
track_progress=None,
verbosity=None,
_analyze=_analyze,
formats=FORMATS,
Expand All @@ -356,56 +388,54 @@ def cmd_analyze(filenames, *,
raise ValueError(f'unsupported fmt {fmt!r}')

filenames = filter_filenames(filenames, iter_filenames)
if verbosity == 2:
def iter_filenames(filenames=filenames):
marks = iter_marks()
for filename in filenames:
print(next(marks), end='')
yield filename
filenames = iter_filenames()
elif verbosity > 2:
def iter_filenames(filenames=filenames):
for filename in filenames:
print(f'<{filename}>')
yield filename
filenames = iter_filenames()

logger.info('analyzing...')
if track_progress:
filenames = track_progress(filenames)

logger.info('analyzing files...')
analyzed = _analyze(filenames, **kwargs)
decls = filter_forward(analyzed, markpublic=True)

for line in do_fmt(analyzed):
for line in do_fmt(decls):
print(line)


def _cli_data(parser, filenames=None, known=None):
ArgumentParser = type(parser)
common = ArgumentParser(add_help=False)
if filenames is None:
common.add_argument('filenames', metavar='FILE', nargs='+')
# These flags will get processed by the top-level parse_args().
add_verbosity_cli(common)
add_traceback_cli(common)

subs = parser.add_subparsers(dest='datacmd')

sub = subs.add_parser('show', parents=[common])
if known is None:
sub.add_argument('--known', required=True)
if filenames is None:
sub.add_argument('filenames', metavar='FILE', nargs='+')

sub = subs.add_parser('dump')
sub = subs.add_parser('dump', parents=[common])
if known is None:
sub.add_argument('--known')
sub.add_argument('--show', action='store_true')
process_progress = add_progress_cli(sub)

sub = subs.add_parser('check')
sub = subs.add_parser('check', parents=[common])
if known is None:
sub.add_argument('--known', required=True)

return None
def process_args(args):
if args.datacmd == 'dump':
process_progress(args)
return process_args


def cmd_data(datacmd, filenames, known=None, *,
_analyze=_analyze,
formats=FORMATS,
extracolumns=None,
relroot=None,
track_progress=None,
**kwargs
):
kwargs.pop('verbosity', None)
Expand All @@ -417,6 +447,8 @@ def cmd_data(datacmd, filenames, known=None, *,
for line in do_fmt(known):
print(line)
elif datacmd == 'dump':
if track_progress:
filenames = track_progress(filenames)
analyzed = _analyze(filenames, **kwargs)
if known is None or usestdout:
outfile = io.StringIO()
Expand Down
6 changes: 5 additions & 1 deletion Tools/c-analyzer/c_analyzer/analyze.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,19 @@
TypeDeclaration,
POTSType,
FuncPtr,
)
from c_parser.match import (
is_pots,
is_funcptr,
)
from .info import (
IGNORED,
UNKNOWN,
is_system_type,
SystemType,
)
from .match import (
is_system_type,
)


def get_typespecs(typedecls):
Expand Down
3 changes: 2 additions & 1 deletion Tools/c-analyzer/c_analyzer/datafiles.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import c_common.tables as _tables
import c_parser.info as _info
import c_parser.match as _match
import c_parser.datafiles as _parser
from . import analyze as _analyze

Expand All @@ -17,7 +18,7 @@ def analyze_known(known, *,
handle_unresolved=True,
):
knowntypes = knowntypespecs = {}
collated = _info.collate_by_kind_group(known)
collated = _match.group_by_kinds(known)
types = {decl: None for decl in collated['type']}
typespecs = _analyze.get_typespecs(types)
def analyze_decl(decl):
Expand Down
42 changes: 4 additions & 38 deletions Tools/c-analyzer/c_analyzer/info.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@
HighlevelParsedItem,
Declaration,
TypeDeclaration,
)
from c_parser.match import (
is_type_decl,
)
from .match import (
is_process_global,
)

Expand All @@ -16,44 +20,6 @@
UNKNOWN = _misc.Labeled('UNKNOWN')


# XXX Use known.tsv for these?
SYSTEM_TYPES = {
'int8_t',
'uint8_t',
'int16_t',
'uint16_t',
'int32_t',
'uint32_t',
'int64_t',
'uint64_t',
'size_t',
'ssize_t',
'intptr_t',
'uintptr_t',
'wchar_t',
'',
# OS-specific
'pthread_cond_t',
'pthread_mutex_t',
'pthread_key_t',
'atomic_int',
'atomic_uintptr_t',
'',
# lib-specific
'WINDOW', # curses
'XML_LChar',
'XML_Size',
'XML_Parser',
'enum XML_Error',
'enum XML_Status',
'',
}


def is_system_type(typespec):
return typespec in SYSTEM_TYPES


class SystemType(TypeDeclaration):

def __init__(self, name):
Expand Down
Loading