Commit d9542db4 authored by Alex Richardson's avatar Alex Richardson
Browse files

[UpdateTestChecks] Share the code to parse RUN: lines between all scripts 

Summary:
This commit also introduces a common.debug() function to avoid many
`if args.verbose:` statements. Depends on D70428.

Reviewers: xbolva00, MaskRay, jdoerfert

Reviewed By: MaskRay

Subscribers: llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D70432
parent 160a5045

llvm/utils/UpdateTestChecks/common.py

+29 −2
Original line number Diff line number Diff line
@@ -14,12 +14,17 @@ else: 
##### Common utilities for update_*test_checks.py





_verbose = False



def parse_commandline_args(parser):

  parser.add_argument('-v', '--verbose', action='store_true',

                      help='Show verbose output')

  parser.add_argument('-u', '--update-only', action='store_true',

                      help='Only update test if it was already autogened')

  return parser.parse_args()

  args = parser.parse_args()

  global _verbose

  _verbose = args.verbose

  return args



def should_add_line_to_output(input_line, prefix_set):

  # Skip any blank comment lines in the IR.
@@ -53,7 +58,7 @@ def invoke_tool(exe, cmd_args, ir): 


##### LLVM IR parser



RUN_LINE_RE = re.compile(r'^\s*[;#]\s*RUN:\s*(.*)$')

RUN_LINE_RE = re.compile(r'^\s*(?://|[;#])\s*RUN:\s*(.*)$')

CHECK_PREFIX_RE = re.compile(r'--?check-prefix(?:es)?[= ](\S+)')

PREFIX_RE = re.compile('^[a-zA-Z0-9_-]+$')

CHECK_RE = re.compile(r'^\s*[;#]\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL|-SAME)?:')
@@ -91,6 +96,28 @@ def warn(msg, test_file=None): 
    msg = '{}: {}'.format(msg, test_file)

  print('WARNING: {}'.format(msg), file=sys.stderr)



def debug(*args, **kwargs):

  # Python2 does not allow def debug(*args, file=sys.stderr, **kwargs):

  if 'file' not in kwargs:

    kwargs['file'] = sys.stderr

  if _verbose:

    print(*args, **kwargs)



def find_run_lines(test, lines):

  debug('Scanning for RUN lines in test file:', test)

  raw_lines = [m.group(1)

               for m in [RUN_LINE_RE.match(l) for l in lines] if m]

  run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []

  for l in raw_lines[1:]:

    if run_lines[-1].endswith('\\'):

      run_lines[-1] = run_lines[-1].rstrip('\\' + ' ' + l)

    else:

      run_lines.append(l)

  debug('Found {} RUN lines in {}:'.format(len(run_lines), test))

  for l in run_lines:

    debug('  RUN: {}'.format(l))

  return run_lines



def scrub_body(body):

  # Scrub runs of whitespace out of the assembly, but leave the leading

  # whitespace in place.

llvm/utils/update_analyze_test_checks.py

+5 −23
Original line number Diff line number Diff line
@@ -69,8 +69,6 @@ def main(): 


  test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]

  for test in test_paths:

    if args.verbose:

      print('Scanning for RUN lines in test file: %s' % (test,), file=sys.stderr)

    with open(test) as f:

      input_lines = [l.rstrip() for l in f]
@@ -84,20 +82,7 @@ def main(): 
        common.warn("Skipping test which isn't autogenerated: " + test)

        continue



    raw_lines = [m.group(1)

                 for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m]

    run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []

    for l in raw_lines[1:]:

      if run_lines[-1].endswith("\\"):

        run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l

      else:

        run_lines.append(l)



    if args.verbose:

      print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)

      for l in run_lines:

        print('  RUN: ' + l, file=sys.stderr)



    run_lines = common.find_run_lines(test, input_lines)

    prefix_list = []

    for l in run_lines:

      if '|' not in l:
@@ -132,9 +117,8 @@ def main(): 
      for prefix in prefixes:

        func_dict.update({prefix: dict()})

    for prefixes, opt_args in prefix_list:

      if args.verbose:

        print('Extracted opt cmd: ' + opt_basename + ' ' + opt_args, file=sys.stderr)

        print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)

      common.debug('Extracted opt cmd:', opt_basename, opt_args, file=sys.stderr)

      common.debug('Extracted FileCheck prefixes:', str(prefixes), file=sys.stderr)



      raw_tool_outputs = common.invoke_tool(args.opt_binary, opt_args, test)
@@ -147,8 +131,7 @@ def main(): 
    is_in_function = False

    is_in_function_start = False

    prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])

    if args.verbose:

      print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)

    common.debug('Rewriting FileCheck prefixes:', str(prefix_set), file=sys.stderr)

    output_lines = []

    output_lines.append(autogenerated_note)
@@ -194,8 +177,7 @@ def main(): 
        continue

      is_in_function = is_in_function_start = True



    if args.verbose:

      print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)

    common.debug('Writing %d lines to %s...' % (len(output_lines), test))



    with open(test, 'wb') as f:

      f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])

llvm/utils/update_cc_test_checks.py

+9 −27
Original line number Diff line number Diff line
@@ -29,7 +29,6 @@ from UpdateTestChecks import asm, common 
ADVERT = '// NOTE: Assertions have been autogenerated by '



CHECK_RE = re.compile(r'^\s*//\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL)?:')

RUN_LINE_RE = re.compile(r'^//\s*RUN:\s*(.*)$')



SUBST = {

    '%clang': [],
@@ -38,9 +37,6 @@ SUBST = { 
}



def get_line2spell_and_mangled(args, clang_args):

  def debug_mangled(*print_args, **kwargs):

    if args.verbose:

      print(*print_args, file=sys.stderr, **kwargs)

  ret = {}

  # Use clang's JSON AST dump to get the mangled name

  json_dump_args = [args.clang, *clang_args, '-fsyntax-only', '-o', '-']
@@ -49,7 +45,7 @@ def get_line2spell_and_mangled(args, clang_args): 
    # -Xclang -ast-dump=json instead:

    json_dump_args.append('-Xclang')

  json_dump_args.append('-ast-dump=json')

  debug_mangled('Running', ' '.join(json_dump_args))

  common.debug('Running', ' '.join(json_dump_args))

  status = subprocess.run(json_dump_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

  if status.returncode != 0:

    sys.stderr.write('Failed to run ' + ' '.join(json_dump_args) + '\n')
@@ -67,20 +63,19 @@ def get_line2spell_and_mangled(args, clang_args): 
    if node['kind'] != 'FunctionDecl':

      continue

    if node.get('isImplicit') is True and node.get('storageClass') == 'extern':

      debug_mangled('Skipping builtin function:', node['name'], '@', node['loc'])

      common.debug('Skipping builtin function:', node['name'], '@', node['loc'])

      continue

    debug_mangled('Found function:', node['kind'], node['name'], '@', node['loc'])

    common.debug('Found function:', node['kind'], node['name'], '@', node['loc'])

    line = node['loc'].get('line')

    # If there is no line it is probably a builtin function -> skip

    if line is None:

      debug_mangled('Skipping function without line number:', node['name'], '@', node['loc'])

      common.debug('Skipping function without line number:', node['name'], '@', node['loc'])

      continue

    spell = node['name']

    mangled = node.get('mangledName', spell)

    ret[int(line)-1] = (spell, mangled)

  if args.verbose:

  for line, func_name in sorted(ret.items()):

      print('line {}: found function {}'.format(line+1, func_name), file=sys.stderr)

    common.debug('line {}: found function {}'.format(line+1, func_name), file=sys.stderr)

  if not ret:

    common.warn('Did not find any functions using', ' '.join(json_dump_args))

  return ret
@@ -191,19 +186,7 @@ def main(): 
        continue



    # Extract RUN lines.

    raw_lines = [m.group(1)

                 for m in [RUN_LINE_RE.match(l) for l in input_lines] if m]

    run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []

    for l in raw_lines[1:]:

      if run_lines[-1].endswith("\\"):

        run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l

      else:

        run_lines.append(l)



    if args.verbose:

      print('Found {} RUN lines:'.format(len(run_lines)), file=sys.stderr)

      for l in run_lines:

        print('  RUN: ' + l, file=sys.stderr)

    run_lines = common.find_run_lines(filename, input_lines)



    # Build a list of clang command lines and check prefixes from RUN lines.

    run_list = []
@@ -260,9 +243,8 @@ def main(): 
      for prefix in prefixes:

        func_dict.update({prefix: dict()})

    for prefixes, clang_args, extra_commands, triple_in_cmd in run_list:

      if args.verbose:

        print('Extracted clang cmd: clang {}'.format(clang_args), file=sys.stderr)

        print('Extracted FileCheck prefixes: {}'.format(prefixes), file=sys.stderr)

      common.debug('Extracted clang cmd: clang {}'.format(clang_args))

      common.debug('Extracted FileCheck prefixes: {}'.format(prefixes))



      get_function_body(args, filename, clang_args, extra_commands, prefixes, triple_in_cmd, func_dict)

llvm/utils/update_llc_test_checks.py

+11 −29
Original line number Diff line number Diff line
@@ -43,8 +43,6 @@ def main(): 


  test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]

  for test in test_paths:

    if args.verbose:

      print('Scanning for RUN lines in test file: %s' % (test,), file=sys.stderr)

    with open(test) as f:

      input_lines = [l.rstrip() for l in f]
@@ -65,20 +63,7 @@ def main(): 
        triple_in_ir = m.groups()[0]

        break



    raw_lines = [m.group(1)

                 for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m]

    run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []

    for l in raw_lines[1:]:

      if run_lines[-1].endswith("\\"):

        run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l

      else:

        run_lines.append(l)



    if args.verbose:

      print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)

      for l in run_lines:

        print('  RUN: ' + l, file=sys.stderr)



    run_lines = common.find_run_lines(test, input_lines)

    run_list = []

    for l in run_lines:

      if '|' not in l:
@@ -115,12 +100,6 @@ def main(): 
      llc_cmd_args = llc_cmd_args.replace('< %s', '').replace('%s', '').strip()

      if test.endswith('.mir'):

        llc_cmd_args += ' -x mir'

        comment_sym = '#'

        check_indent = '  '

      else:

        comment_sym = ';'

        check_indent = ''



      check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)

                               for item in m.group(1).split(',')]

      if not check_prefixes:
@@ -130,6 +109,12 @@ def main(): 
      # now, we just ignore all but the last.

      run_list.append((check_prefixes, llc_cmd_args, triple_in_cmd, march_in_cmd))



    if test.endswith('.mir'):

      comment_sym = '#'

      check_indent = '  '

    else:

      comment_sym = ';'

      check_indent = ''

    autogenerated_note = (comment_sym + ADVERT + 'utils/' + script_name)



    func_dict = {}
@@ -138,9 +123,8 @@ def main(): 
      for prefix in prefixes:

        func_dict.update({prefix: dict()})

    for prefixes, llc_args, triple_in_cmd, march_in_cmd in run_list:

      if args.verbose:

        print('Extracted LLC cmd: ' + llc_tool + ' ' + llc_args, file=sys.stderr)

        print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)

      common.debug('Extracted LLC cmd:', llc_tool, llc_args)

      common.debug('Extracted FileCheck prefixes:', str(prefixes))



      raw_tool_output = common.invoke_tool(args.llc_binary, llc_args, test)

      triple = triple_in_cmd or triple_in_ir
@@ -154,8 +138,7 @@ def main(): 
    is_in_function_start = False

    func_name = None

    prefix_set = set([prefix for p in run_list for prefix in p[0]])

    if args.verbose:

      print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)

    common.debug('Rewriting FileCheck prefixes:', str(prefix_set))

    output_lines = []

    output_lines.append(autogenerated_note)
@@ -199,8 +182,7 @@ def main(): 
        continue

      is_in_function = is_in_function_start = True



    if args.verbose:

      print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)

    common.debug('Writing %d lines to %s...' % (len(output_lines), test))



    with open(test, 'wb') as f:

      f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])

llvm/utils/update_mca_test_checks.py

+2 −28
Original line number Diff line number Diff line
@@ -83,26 +83,6 @@ def _parse_args(): 
  return args





def _find_run_lines(input_lines, args):

  raw_lines = [m.group(1)

               for m in [common.RUN_LINE_RE.match(l) for l in input_lines]

               if m]

  run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []

  for l in raw_lines[1:]:

    if run_lines[-1].endswith(r'\\'):

      run_lines[-1] = run_lines[-1].rstrip('\\') + ' ' + l

    else:

      run_lines.append(l)



  if args.verbose:

    sys.stderr.write('Found {} RUN line{}:\n'.format(

        len(run_lines), '' if len(run_lines) == 1 else 's'))

    for line in run_lines:

      sys.stderr.write('  RUN: {}\n'.format(line))



  return run_lines





def _get_run_infos(run_lines, args):

  run_infos = []

  for run_line in run_lines:
@@ -544,9 +524,7 @@ def _write_output(test_path, input_lines, prefix_list, block_infos, # noqa 
    return

  sys.stderr.write('      [{} lines total]\n'.format(len(output_lines)))



  if args.verbose:

    sys.stderr.write(

        'Writing {} lines to {}...\n\n'.format(len(output_lines), test_path))

  common.debug('Writing', len(output_lines), 'lines to', test_path, '..\n\n')



  with open(test_path, 'wb') as f:

    f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
@@ -562,17 +540,13 @@ def main(): 
    # will be written once per source location per test.

    _configure_warnings(args)



    if args.verbose:

      sys.stderr.write(

          'Scanning for RUN lines in test file: {}\n'.format(test_path))



    if not os.path.isfile(test_path):

      raise Error('could not find test file: {}'.format(test_path))



    with open(test_path) as f:

      input_lines = [l.rstrip() for l in f]



    run_lines = _find_run_lines(input_lines, args)

    run_lines = common.find_run_lines(test_path, input_lines)

    run_infos = _get_run_infos(run_lines, args)

    common_prefix, prefix_pad = _get_useful_prefix_info(run_infos)

    block_infos = _get_block_infos(run_infos, test_path, args, common_prefix)