PRESUBMIT.py 58.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Top-level presubmit script for Chromium.

See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
for more details about the presubmit API built into gcl.
"""


import re
import sys


_EXCLUDED_PATHS = (
    r"^breakpad[\\\/].*",
    r"^native_client_sdk[\\\/]src[\\\/]build_tools[\\\/]make_rules.py",
    r"^native_client_sdk[\\\/]src[\\\/]build_tools[\\\/]make_simple.py",
20
    r"^native_client_sdk[\\\/]src[\\\/]tools[\\\/].*.mk",
21 22 23 24 25 26
    r"^net[\\\/]tools[\\\/]spdyshark[\\\/].*",
    r"^skia[\\\/].*",
    r"^v8[\\\/].*",
    r".*MakeFile$",
    r".+_autogen\.h$",
    r".+[\\\/]pnacl_shim\.c$",
27
    r"^gpu[\\\/]config[\\\/].*_list_json\.cc$",
28 29 30
    r"^chrome[\\\/]browser[\\\/]resources[\\\/]pdf[\\\/]index.js"
)

31 32 33
# The NetscapePlugIn library is excluded from pan-project as it will soon
# be deleted together with the rest of the NPAPI and it's not worthwhile to
# update the coding style until then.
34 35
_TESTRUNNER_PATHS = (
    r"^content[\\\/]shell[\\\/]tools[\\\/]plugin[\\\/].*",
36 37
)

38 39 40 41 42 43 44
# Fragment of a regular expression that matches C++ and Objective-C++
# implementation files.
_IMPLEMENTATION_EXTENSIONS = r'\.(cc|cpp|cxx|mm)$'

# Regular expression that matches code only used for test binaries
# (best effort).
_TEST_CODE_EXCLUDED_PATHS = (
45
    r'.*[\\\/](fake_|test_|mock_).+%s' % _IMPLEMENTATION_EXTENSIONS,
46
    r'.+_test_(base|support|util)%s' % _IMPLEMENTATION_EXTENSIONS,
47
    r'.+_(api|browser|kif|perf|pixel|unit|ui)?test(_[a-z]+)?%s' %
48
        _IMPLEMENTATION_EXTENSIONS,
49
    r'.+profile_sync_service_harness%s' % _IMPLEMENTATION_EXTENSIONS,
50
    r'.*[\\\/](test|tool(s)?)[\\\/].*',
51
    # content_shell is used for running layout tests.
52
    r'content[\\\/]shell[\\\/].*',
53
    # At request of folks maintaining this folder.
54
    r'chrome[\\\/]browser[\\\/]automation[\\\/].*',
55
    # Non-production example code.
56
    r'mojo[\\\/]examples[\\\/].*',
57
    # Launcher for running iOS tests on the simulator.
58
    r'testing[\\\/]iossim[\\\/]iossim\.mm$',
59
)
60 61 62 63 64

_TEST_ONLY_WARNING = (
    'You might be calling functions intended only for testing from\n'
    'production code.  It is OK to ignore this warning if you know what\n'
    'you are doing, as the heuristics used to detect the situation are\n'
65
    'not perfect.  The commit queue will not block on this warning.')
66 67


68 69 70 71 72
_INCLUDE_ORDER_WARNING = (
    'Your #include order seems to be broken. Send mail to\n'
    'marja@chromium.org if this is not the case.')


73 74 75 76 77 78 79 80 81 82 83
_BANNED_OBJC_FUNCTIONS = (
    (
      'addTrackingRect:',
      (
       'The use of -[NSView addTrackingRect:owner:userData:assumeInside:] is'
       'prohibited. Please use CrTrackingArea instead.',
       'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
      ),
      False,
    ),
    (
84
      r'/NSTrackingArea\W',
85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159
      (
       'The use of NSTrackingAreas is prohibited. Please use CrTrackingArea',
       'instead.',
       'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
      ),
      False,
    ),
    (
      'convertPointFromBase:',
      (
       'The use of -[NSView convertPointFromBase:] is almost certainly wrong.',
       'Please use |convertPoint:(point) fromView:nil| instead.',
       'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
      ),
      True,
    ),
    (
      'convertPointToBase:',
      (
       'The use of -[NSView convertPointToBase:] is almost certainly wrong.',
       'Please use |convertPoint:(point) toView:nil| instead.',
       'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
      ),
      True,
    ),
    (
      'convertRectFromBase:',
      (
       'The use of -[NSView convertRectFromBase:] is almost certainly wrong.',
       'Please use |convertRect:(point) fromView:nil| instead.',
       'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
      ),
      True,
    ),
    (
      'convertRectToBase:',
      (
       'The use of -[NSView convertRectToBase:] is almost certainly wrong.',
       'Please use |convertRect:(point) toView:nil| instead.',
       'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
      ),
      True,
    ),
    (
      'convertSizeFromBase:',
      (
       'The use of -[NSView convertSizeFromBase:] is almost certainly wrong.',
       'Please use |convertSize:(point) fromView:nil| instead.',
       'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
      ),
      True,
    ),
    (
      'convertSizeToBase:',
      (
       'The use of -[NSView convertSizeToBase:] is almost certainly wrong.',
       'Please use |convertSize:(point) toView:nil| instead.',
       'http://dev.chromium.org/developers/coding-style/cocoa-dos-and-donts',
      ),
      True,
    ),
)


_BANNED_CPP_FUNCTIONS = (
    # Make sure that gtest's FRIEND_TEST() macro is not used; the
    # FRIEND_TEST_ALL_PREFIXES() macro from base/gtest_prod_util.h should be
    # used instead since that allows for FLAKY_ and DISABLED_ prefixes.
    (
      'FRIEND_TEST(',
      (
       'Chromium code should not use gtest\'s FRIEND_TEST() macro. Include',
       'base/gtest_prod_util.h and use FRIEND_TEST_ALL_PREFIXES() instead.',
      ),
      False,
160
      (),
161 162 163 164 165 166 167 168 169
    ),
    (
      'ScopedAllowIO',
      (
       'New code should not use ScopedAllowIO. Post a task to the blocking',
       'pool or the FILE thread instead.',
      ),
      True,
      (
170
        r"^base[\\\/]process[\\\/]process_metrics_linux\.cc$",
171
        r"^chrome[\\\/]browser[\\\/]chromeos[\\\/]boot_times_loader\.cc$",
172
        r"^components[\\\/]crash[\\\/]app[\\\/]breakpad_mac\.mm$",
173 174
        r"^content[\\\/]shell[\\\/]browser[\\\/]shell_browser_main\.cc$",
        r"^content[\\\/]shell[\\\/]browser[\\\/]shell_message_filter\.cc$",
175
        r"^mojo[\\\/]system[\\\/]raw_shared_buffer_posix\.cc$",
176
        r"^net[\\\/]disk_cache[\\\/]cache_util\.cc$",
177
        r"^net[\\\/]url_request[\\\/]test_url_fetcher_factory\.cc$",
178 179
      ),
    ),
180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216
    (
      'SkRefPtr',
      (
        'The use of SkRefPtr is prohibited. ',
        'Please use skia::RefPtr instead.'
      ),
      True,
      (),
    ),
    (
      'SkAutoRef',
      (
        'The indirect use of SkRefPtr via SkAutoRef is prohibited. ',
        'Please use skia::RefPtr instead.'
      ),
      True,
      (),
    ),
    (
      'SkAutoTUnref',
      (
        'The use of SkAutoTUnref is dangerous because it implicitly ',
        'converts to a raw pointer. Please use skia::RefPtr instead.'
      ),
      True,
      (),
    ),
    (
      'SkAutoUnref',
      (
        'The indirect use of SkAutoTUnref through SkAutoUnref is dangerous ',
        'because it implicitly converts to a raw pointer. ',
        'Please use skia::RefPtr instead.'
      ),
      True,
      (),
    ),
217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240
    (
      r'/HANDLE_EINTR\(.*close',
      (
       'HANDLE_EINTR(close) is invalid. If close fails with EINTR, the file',
       'descriptor will be closed, and it is incorrect to retry the close.',
       'Either call close directly and ignore its return value, or wrap close',
       'in IGNORE_EINTR to use its return value. See http://crbug.com/269623'
      ),
      True,
      (),
    ),
    (
      r'/IGNORE_EINTR\((?!.*close)',
      (
       'IGNORE_EINTR is only valid when wrapping close. To wrap other system',
       'calls, use HANDLE_EINTR. See http://crbug.com/269623',
      ),
      True,
      (
        # Files that #define IGNORE_EINTR.
        r'^base[\\\/]posix[\\\/]eintr_wrapper\.h$',
        r'^ppapi[\\\/]tests[\\\/]test_broker\.cc$',
      ),
    ),
241 242 243 244 245 246 247
    (
      r'/v8::Extension\(',
      (
        'Do not introduce new v8::Extensions into the code base, use',
        'gin::Wrappable instead. See http://crbug.com/334679',
      ),
      True,
248
      (
249
        r'extensions[\\\/]renderer[\\\/]safe_builtins\.*',
250
      ),
251
    ),
252 253
)

254 255 256 257
_IPC_ENUM_TRAITS_DEPRECATED = (
    'You are using IPC_ENUM_TRAITS() in your code. It has been deprecated.\n'
    'See http://www.chromium.org/Home/chromium-security/education/security-tips-for-ipc')

258

259 260 261
_VALID_OS_MACROS = (
    # Please keep sorted.
    'OS_ANDROID',
262
    'OS_ANDROID_HOST',
263 264 265 266 267 268 269 270 271 272
    'OS_BSD',
    'OS_CAT',       # For testing.
    'OS_CHROMEOS',
    'OS_FREEBSD',
    'OS_IOS',
    'OS_LINUX',
    'OS_MACOSX',
    'OS_NACL',
    'OS_OPENBSD',
    'OS_POSIX',
273
    'OS_QNX',
274 275 276 277 278
    'OS_SOLARIS',
    'OS_WIN',
)


279 280 281 282 283 284 285 286 287
def _CheckNoProductionCodeUsingTestOnlyFunctions(input_api, output_api):
  """Attempts to prevent use of functions intended only for testing in
  non-testing code. For now this is just a best-effort implementation
  that ignores header files and may have some false positives. A
  better implementation would probably need a proper C++ parser.
  """
  # We only scan .cc files and the like, as the declaration of
  # for-testing functions in header files are hard to distinguish from
  # calls to such functions without a proper C++ parser.
288
  file_inclusion_pattern = r'.+%s' % _IMPLEMENTATION_EXTENSIONS
289

290
  base_function_pattern = r'[ :]test::[^\s]+|ForTest(ing)?|for_test(ing)?'
291
  inclusion_pattern = input_api.re.compile(r'(%s)\s*\(' % base_function_pattern)
292
  comment_pattern = input_api.re.compile(r'//.*(%s)' % base_function_pattern)
293 294 295 296 297
  exclusion_pattern = input_api.re.compile(
    r'::[A-Za-z0-9_]+(%s)|(%s)[^;]+\{' % (
      base_function_pattern, base_function_pattern))

  def FilterFile(affected_file):
298 299 300
    black_list = (_EXCLUDED_PATHS +
                  _TEST_CODE_EXCLUDED_PATHS +
                  input_api.DEFAULT_BLACK_LIST)
301 302 303 304 305 306 307 308
    return input_api.FilterSourceFile(
      affected_file,
      white_list=(file_inclusion_pattern, ),
      black_list=black_list)

  problems = []
  for f in input_api.AffectedSourceFiles(FilterFile):
    local_path = f.LocalPath()
309
    for line_number, line in f.ChangedContents():
310
      if (inclusion_pattern.search(line) and
311
          not comment_pattern.search(line) and
312 313 314 315 316
          not exclusion_pattern.search(line)):
        problems.append(
          '%s:%d\n    %s' % (local_path, line_number, line.strip()))

  if problems:
317
    return [output_api.PresubmitPromptOrNotify(_TEST_ONLY_WARNING, problems)]
318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350
  else:
    return []


def _CheckNoIOStreamInHeaders(input_api, output_api):
  """Checks to make sure no .h files include <iostream>."""
  files = []
  pattern = input_api.re.compile(r'^#include\s*<iostream>',
                                 input_api.re.MULTILINE)
  for f in input_api.AffectedSourceFiles(input_api.FilterSourceFile):
    if not f.LocalPath().endswith('.h'):
      continue
    contents = input_api.ReadFile(f)
    if pattern.search(contents):
      files.append(f)

  if len(files):
    return [ output_api.PresubmitError(
        'Do not #include <iostream> in header files, since it inserts static '
        'initialization into every file including the header. Instead, '
        '#include <ostream>. See http://crbug.com/94794',
        files) ]
  return []


def _CheckNoUNIT_TESTInSourceFiles(input_api, output_api):
  """Checks to make sure no source files use UNIT_TEST"""
  problems = []
  for f in input_api.AffectedFiles():
    if (not f.LocalPath().endswith(('.cc', '.mm'))):
      continue

    for line_num, line in f.ChangedContents():
351
      if 'UNIT_TEST ' in line or line.endswith('UNIT_TEST'):
352 353 354 355 356 357 358 359 360 361 362 363 364
        problems.append('    %s:%d' % (f.LocalPath(), line_num))

  if not problems:
    return []
  return [output_api.PresubmitPromptWarning('UNIT_TEST is only for headers.\n' +
      '\n'.join(problems))]


def _CheckNoNewWStrings(input_api, output_api):
  """Checks to make sure we don't introduce use of wstrings."""
  problems = []
  for f in input_api.AffectedFiles():
    if (not f.LocalPath().endswith(('.cc', '.h')) or
365
        f.LocalPath().endswith(('test.cc', '_win.cc', '_win.h'))):
366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393
      continue

    allowWString = False
    for line_num, line in f.ChangedContents():
      if 'presubmit: allow wstring' in line:
        allowWString = True
      elif not allowWString and 'wstring' in line:
        problems.append('    %s:%d' % (f.LocalPath(), line_num))
        allowWString = False
      else:
        allowWString = False

  if not problems:
    return []
  return [output_api.PresubmitPromptWarning('New code should not use wstrings.'
      '  If you are calling a cross-platform API that accepts a wstring, '
      'fix the API.\n' +
      '\n'.join(problems))]


def _CheckNoDEPSGIT(input_api, output_api):
  """Make sure .DEPS.git is never modified manually."""
  if any(f.LocalPath().endswith('.DEPS.git') for f in
      input_api.AffectedFiles()):
    return [output_api.PresubmitError(
      'Never commit changes to .DEPS.git. This file is maintained by an\n'
      'automated system based on what\'s in DEPS and your changes will be\n'
      'overwritten.\n'
394
      'See https://sites.google.com/a/chromium.org/dev/developers/how-tos/get-the-code#Rolling_DEPS\n'
395 396 397 398
      'for more information')]
  return []


399 400 401 402 403 404 405 406 407 408 409 410 411 412 413
def _CheckValidHostsInDEPS(input_api, output_api):
  """Checks that DEPS file deps are from allowed_hosts."""
  # Run only if DEPS file has been modified to annoy fewer bystanders.
  if all(f.LocalPath() != 'DEPS' for f in input_api.AffectedFiles()):
    return []
  # Outsource work to gclient verify
  try:
    input_api.subprocess.check_output(['gclient', 'verify'])
    return []
  except input_api.subprocess.CalledProcessError, error:
    return [output_api.PresubmitError(
        'DEPS file must have only git dependencies.',
        long_text=error.output)]


414 415 416 417 418 419 420 421 422
def _CheckNoBannedFunctions(input_api, output_api):
  """Make sure that banned functions are not used."""
  warnings = []
  errors = []

  file_filter = lambda f: f.LocalPath().endswith(('.mm', '.m', '.h'))
  for f in input_api.AffectedFiles(file_filter=file_filter):
    for line_num, line in f.ChangedContents():
      for func_name, message, error in _BANNED_OBJC_FUNCTIONS:
423 424 425 426 427 428 429 430
        matched = False
        if func_name[0:1] == '/':
          regex = func_name[1:]
          if input_api.re.search(regex, line):
            matched = True
        elif func_name in line:
            matched = True
        if matched:
431 432 433 434 435 436 437 438 439 440
          problems = warnings;
          if error:
            problems = errors;
          problems.append('    %s:%d:' % (f.LocalPath(), line_num))
          for message_line in message:
            problems.append('      %s' % message_line)

  file_filter = lambda f: f.LocalPath().endswith(('.cc', '.mm', '.h'))
  for f in input_api.AffectedFiles(file_filter=file_filter):
    for line_num, line in f.ChangedContents():
441 442 443 444 445 446 447 448 449
      for func_name, message, error, excluded_paths in _BANNED_CPP_FUNCTIONS:
        def IsBlacklisted(affected_file, blacklist):
          local_path = affected_file.LocalPath()
          for item in blacklist:
            if input_api.re.match(item, local_path):
              return True
          return False
        if IsBlacklisted(f, excluded_paths):
          continue
450 451 452 453 454 455 456 457
        matched = False
        if func_name[0:1] == '/':
          regex = func_name[1:]
          if input_api.re.search(regex, line):
            matched = True
        elif func_name in line:
            matched = True
        if matched:
458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524
          problems = warnings;
          if error:
            problems = errors;
          problems.append('    %s:%d:' % (f.LocalPath(), line_num))
          for message_line in message:
            problems.append('      %s' % message_line)

  result = []
  if (warnings):
    result.append(output_api.PresubmitPromptWarning(
        'Banned functions were used.\n' + '\n'.join(warnings)))
  if (errors):
    result.append(output_api.PresubmitError(
        'Banned functions were used.\n' + '\n'.join(errors)))
  return result


def _CheckNoPragmaOnce(input_api, output_api):
  """Make sure that banned functions are not used."""
  files = []
  pattern = input_api.re.compile(r'^#pragma\s+once',
                                 input_api.re.MULTILINE)
  for f in input_api.AffectedSourceFiles(input_api.FilterSourceFile):
    if not f.LocalPath().endswith('.h'):
      continue
    contents = input_api.ReadFile(f)
    if pattern.search(contents):
      files.append(f)

  if files:
    return [output_api.PresubmitError(
        'Do not use #pragma once in header files.\n'
        'See http://www.chromium.org/developers/coding-style#TOC-File-headers',
        files)]
  return []


def _CheckNoTrinaryTrueFalse(input_api, output_api):
  """Checks to make sure we don't introduce use of foo ? true : false."""
  problems = []
  pattern = input_api.re.compile(r'\?\s*(true|false)\s*:\s*(true|false)')
  for f in input_api.AffectedFiles():
    if not f.LocalPath().endswith(('.cc', '.h', '.inl', '.m', '.mm')):
      continue

    for line_num, line in f.ChangedContents():
      if pattern.match(line):
        problems.append('    %s:%d' % (f.LocalPath(), line_num))

  if not problems:
    return []
  return [output_api.PresubmitPromptWarning(
      'Please consider avoiding the "? true : false" pattern if possible.\n' +
      '\n'.join(problems))]


def _CheckUnwantedDependencies(input_api, output_api):
  """Runs checkdeps on #include statements added in this
  change. Breaking - rules is an error, breaking ! rules is a
  warning.
  """
  # We need to wait until we have an input_api object and use this
  # roundabout construct to import checkdeps because this file is
  # eval-ed and thus doesn't have __file__.
  original_sys_path = sys.path
  try:
    sys.path = sys.path + [input_api.os_path.join(
525
        input_api.PresubmitLocalPath(), 'buildtools', 'checkdeps')]
526 527 528 529 530 531 532 533 534 535 536 537 538 539 540
    import checkdeps
    from cpp_checker import CppChecker
    from rules import Rule
  finally:
    # Restore sys.path to what it was before.
    sys.path = original_sys_path

  added_includes = []
  for f in input_api.AffectedFiles():
    if not CppChecker.IsCppFile(f.LocalPath()):
      continue

    changed_lines = [line for line_num, line in f.ChangedContents()]
    added_includes.append([f.LocalPath(), changed_lines])

541
  deps_checker = checkdeps.DepsChecker(input_api.PresubmitLocalPath())
542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558

  error_descriptions = []
  warning_descriptions = []
  for path, rule_type, rule_description in deps_checker.CheckAddedCppIncludes(
      added_includes):
    description_with_path = '%s\n    %s' % (path, rule_description)
    if rule_type == Rule.DISALLOW:
      error_descriptions.append(description_with_path)
    else:
      warning_descriptions.append(description_with_path)

  results = []
  if error_descriptions:
    results.append(output_api.PresubmitError(
        'You added one or more #includes that violate checkdeps rules.',
        error_descriptions))
  if warning_descriptions:
559
    results.append(output_api.PresubmitPromptOrNotify(
560 561 562 563 564 565 566 567 568
        'You added one or more #includes of files that are temporarily\n'
        'allowed but being removed. Can you avoid introducing the\n'
        '#include? See relevant DEPS file(s) for details and contacts.',
        warning_descriptions))
  return results


def _CheckFilePermissions(input_api, output_api):
  """Check that all files have their permissions properly set."""
569 570
  if input_api.platform == 'win32':
    return []
571 572 573 574
  args = [sys.executable, 'tools/checkperms/checkperms.py', '--root',
          input_api.change.RepositoryRoot()]
  for f in input_api.AffectedFiles():
    args += ['--file', f.LocalPath()]
575 576 577
  checkperms = input_api.subprocess.Popen(args,
                                          stdout=input_api.subprocess.PIPE)
  errors = checkperms.communicate()[0].strip()
578
  if errors:
579 580 581
    return [output_api.PresubmitError('checkperms.py failed.',
                                      errors.splitlines())]
  return []
582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603


def _CheckNoAuraWindowPropertyHInHeaders(input_api, output_api):
  """Makes sure we don't include ui/aura/window_property.h
  in header files.
  """
  pattern = input_api.re.compile(r'^#include\s*"ui/aura/window_property.h"')
  errors = []
  for f in input_api.AffectedFiles():
    if not f.LocalPath().endswith('.h'):
      continue
    for line_num, line in f.ChangedContents():
      if pattern.match(line):
        errors.append('    %s:%d' % (f.LocalPath(), line_num))

  results = []
  if errors:
    results.append(output_api.PresubmitError(
      'Header files should not include ui/aura/window_property.h', errors))
  return results


604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656
def _CheckIncludeOrderForScope(scope, input_api, file_path, changed_linenums):
  """Checks that the lines in scope occur in the right order.

  1. C system files in alphabetical order
  2. C++ system files in alphabetical order
  3. Project's .h files
  """

  c_system_include_pattern = input_api.re.compile(r'\s*#include <.*\.h>')
  cpp_system_include_pattern = input_api.re.compile(r'\s*#include <.*>')
  custom_include_pattern = input_api.re.compile(r'\s*#include ".*')

  C_SYSTEM_INCLUDES, CPP_SYSTEM_INCLUDES, CUSTOM_INCLUDES = range(3)

  state = C_SYSTEM_INCLUDES

  previous_line = ''
  previous_line_num = 0
  problem_linenums = []
  for line_num, line in scope:
    if c_system_include_pattern.match(line):
      if state != C_SYSTEM_INCLUDES:
        problem_linenums.append((line_num, previous_line_num))
      elif previous_line and previous_line > line:
        problem_linenums.append((line_num, previous_line_num))
    elif cpp_system_include_pattern.match(line):
      if state == C_SYSTEM_INCLUDES:
        state = CPP_SYSTEM_INCLUDES
      elif state == CUSTOM_INCLUDES:
        problem_linenums.append((line_num, previous_line_num))
      elif previous_line and previous_line > line:
        problem_linenums.append((line_num, previous_line_num))
    elif custom_include_pattern.match(line):
      if state != CUSTOM_INCLUDES:
        state = CUSTOM_INCLUDES
      elif previous_line and previous_line > line:
        problem_linenums.append((line_num, previous_line_num))
    else:
      problem_linenums.append(line_num)
    previous_line = line
    previous_line_num = line_num

  warnings = []
  for (line_num, previous_line_num) in problem_linenums:
    if line_num in changed_linenums or previous_line_num in changed_linenums:
      warnings.append('    %s:%d' % (file_path, line_num))
  return warnings


def _CheckIncludeOrderInFile(input_api, f, changed_linenums):
  """Checks the #include order for the given file f."""

  system_include_pattern = input_api.re.compile(r'\s*#include \<.*')
657 658 659 660 661 662
  # Exclude the following includes from the check:
  # 1) #include <.../...>, e.g., <sys/...> includes often need to appear in a
  # specific order.
  # 2) <atlbase.h>, "build/build_config.h"
  excluded_include_pattern = input_api.re.compile(
      r'\s*#include (\<.*/.*|\<atlbase\.h\>|"build/build_config.h")')
663
  custom_include_pattern = input_api.re.compile(r'\s*#include "(?P<FILE>.*)"')
664 665 666 667
  # Match the final or penultimate token if it is xxxtest so we can ignore it
  # when considering the special first include.
  test_file_tag_pattern = input_api.re.compile(
    r'_[a-z]+test(?=(_[a-zA-Z0-9]+)?\.)')
668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683
  if_pattern = input_api.re.compile(
      r'\s*#\s*(if|elif|else|endif|define|undef).*')
  # Some files need specialized order of includes; exclude such files from this
  # check.
  uncheckable_includes_pattern = input_api.re.compile(
      r'\s*#include '
      '("ipc/.*macros\.h"|<windows\.h>|".*gl.*autogen.h")\s*')

  contents = f.NewContents()
  warnings = []
  line_num = 0

  # Handle the special first include. If the first include file is
  # some/path/file.h, the corresponding including file can be some/path/file.cc,
  # some/other/path/file.cc, some/path/file_platform.cc, some/path/file-suffix.h
  # etc. It's also possible that no special first include exists.
684 685 686 687 688
  # If the included file is some/path/file_platform.h the including file could
  # also be some/path/file_xxxtest_platform.h.
  including_file_base_name = test_file_tag_pattern.sub(
    '', input_api.os_path.basename(f.LocalPath()))

689 690 691 692 693 694 695 696 697 698
  for line in contents:
    line_num += 1
    if system_include_pattern.match(line):
      # No special first include -> process the line again along with normal
      # includes.
      line_num -= 1
      break
    match = custom_include_pattern.match(line)
    if match:
      match_dict = match.groupdict()
699 700 701 702
      header_basename = test_file_tag_pattern.sub(
        '', input_api.os_path.basename(match_dict['FILE'])).replace('.h', '')

      if header_basename not in including_file_base_name:
703 704 705 706 707 708 709 710 711 712 713
        # No special first include -> process the line again along with normal
        # includes.
        line_num -= 1
      break

  # Split into scopes: Each region between #if and #endif is its own scope.
  scopes = []
  current_scope = []
  for line in contents[line_num:]:
    line_num += 1
    if uncheckable_includes_pattern.match(line):
714
      continue
715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740
    if if_pattern.match(line):
      scopes.append(current_scope)
      current_scope = []
    elif ((system_include_pattern.match(line) or
           custom_include_pattern.match(line)) and
          not excluded_include_pattern.match(line)):
      current_scope.append((line_num, line))
  scopes.append(current_scope)

  for scope in scopes:
    warnings.extend(_CheckIncludeOrderForScope(scope, input_api, f.LocalPath(),
                                               changed_linenums))
  return warnings


def _CheckIncludeOrder(input_api, output_api):
  """Checks that the #include order is correct.

  1. The corresponding header for source files.
  2. C system files in alphabetical order
  3. C++ system files in alphabetical order
  4. Project's .h files in alphabetical order

  Each region separated by #if, #elif, #else, #endif, #define and #undef follows
  these rules separately.
  """
741 742 743
  def FileFilterIncludeOrder(affected_file):
    black_list = (_EXCLUDED_PATHS + input_api.DEFAULT_BLACK_LIST)
    return input_api.FilterSourceFile(affected_file, black_list=black_list)
744 745

  warnings = []
746
  for f in input_api.AffectedFiles(file_filter=FileFilterIncludeOrder):
747 748 749 750 751 752
    if f.LocalPath().endswith(('.cc', '.h')):
      changed_linenums = set(line_num for line_num, _ in f.ChangedContents())
      warnings.extend(_CheckIncludeOrderInFile(input_api, f, changed_linenums))

  results = []
  if warnings:
753
    results.append(output_api.PresubmitPromptOrNotify(_INCLUDE_ORDER_WARNING,
754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793
                                                      warnings))
  return results


def _CheckForVersionControlConflictsInFile(input_api, f):
  pattern = input_api.re.compile('^(?:<<<<<<<|>>>>>>>) |^=======$')
  errors = []
  for line_num, line in f.ChangedContents():
    if pattern.match(line):
      errors.append('    %s:%d %s' % (f.LocalPath(), line_num, line))
  return errors


def _CheckForVersionControlConflicts(input_api, output_api):
  """Usually this is not intentional and will cause a compile failure."""
  errors = []
  for f in input_api.AffectedFiles():
    errors.extend(_CheckForVersionControlConflictsInFile(input_api, f))

  results = []
  if errors:
    results.append(output_api.PresubmitError(
      'Version control conflict markers found, please resolve.', errors))
  return results


def _CheckHardcodedGoogleHostsInLowerLayers(input_api, output_api):
  def FilterFile(affected_file):
    """Filter function for use with input_api.AffectedSourceFiles,
    below.  This filters out everything except non-test files from
    top-level directories that generally speaking should not hard-code
    service URLs (e.g. src/android_webview/, src/content/ and others).
    """
    return input_api.FilterSourceFile(
      affected_file,
      white_list=(r'^(android_webview|base|content|net)[\\\/].*', ),
      black_list=(_EXCLUDED_PATHS +
                  _TEST_CODE_EXCLUDED_PATHS +
                  input_api.DEFAULT_BLACK_LIST))

794 795 796
  base_pattern = '"[^"]*google\.com[^"]*"'
  comment_pattern = input_api.re.compile('//.*%s' % base_pattern)
  pattern = input_api.re.compile(base_pattern)
797 798 799
  problems = []  # items are (filename, line_number, line)
  for f in input_api.AffectedSourceFiles(FilterFile):
    for line_num, line in f.ChangedContents():
800
      if not comment_pattern.search(line) and pattern.search(line):
801 802 803
        problems.append((f.LocalPath(), line_num, line))

  if problems:
804
    return [output_api.PresubmitPromptOrNotify(
805
        'Most layers below src/chrome/ should not hardcode service URLs.\n'
806
        'Are you sure this is correct?',
807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830
        ['  %s:%d:  %s' % (
            problem[0], problem[1], problem[2]) for problem in problems])]
  else:
    return []


def _CheckNoAbbreviationInPngFileName(input_api, output_api):
  """Makes sure there are no abbreviations in the name of PNG files.
  """
  pattern = input_api.re.compile(r'.*_[a-z]_.*\.png$|.*_[a-z]\.png$')
  errors = []
  for f in input_api.AffectedFiles(include_deletes=False):
    if pattern.match(f.LocalPath()):
      errors.append('    %s' % f.LocalPath())

  results = []
  if errors:
    results.append(output_api.PresubmitError(
        'The name of PNG files should not have abbreviations. \n'
        'Use _hover.png, _center.png, instead of _h.png, _c.png.\n'
        'Contact oshima@chromium.org if you have questions.', errors))
  return results


831
def _FilesToCheckForIncomingDeps(re, changed_lines):
832
  """Helper method for _CheckAddedDepsHaveTargetApprovals. Returns
833 834 835 836 837 838
  a set of DEPS entries that we should look up.

  For a directory (rather than a specific filename) we fake a path to
  a specific filename by adding /DEPS. This is chosen as a file that
  will seldom or never be subject to per-file include_rules.
  """
839 840
  # We ignore deps entries on auto-generated directories.
  AUTO_GENERATED_DIRS = ['grit', 'jni']
841 842 843 844 845 846 847

  # This pattern grabs the path without basename in the first
  # parentheses, and the basename (if present) in the second. It
  # relies on the simple heuristic that if there is a basename it will
  # be a header file ending in ".h".
  pattern = re.compile(
      r"""['"]\+([^'"]+?)(/[a-zA-Z0-9_]+\.h)?['"].*""")
848
  results = set()
849 850 851 852
  for changed_line in changed_lines:
    m = pattern.match(changed_line)
    if m:
      path = m.group(1)
853
      if path.split('/')[0] not in AUTO_GENERATED_DIRS:
854 855 856 857
        if m.group(2):
          results.add('%s%s' % (path, m.group(2)))
        else:
          results.add('%s/DEPS' % path)
858 859 860
  return results


861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876
def _CheckAddedDepsHaveTargetApprovals(input_api, output_api):
  """When a dependency prefixed with + is added to a DEPS file, we
  want to make sure that the change is reviewed by an OWNER of the
  target file or directory, to avoid layering violations from being
  introduced. This check verifies that this happens.
  """
  changed_lines = set()
  for f in input_api.AffectedFiles():
    filename = input_api.os_path.basename(f.LocalPath())
    if filename == 'DEPS':
      changed_lines |= set(line.strip()
                           for line_num, line
                           in f.ChangedContents())
  if not changed_lines:
    return []

877 878
  virtual_depended_on_files = _FilesToCheckForIncomingDeps(input_api.re,
                                                           changed_lines)
879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901
  if not virtual_depended_on_files:
    return []

  if input_api.is_committing:
    if input_api.tbr:
      return [output_api.PresubmitNotifyResult(
          '--tbr was specified, skipping OWNERS check for DEPS additions')]
    if not input_api.change.issue:
      return [output_api.PresubmitError(
          "DEPS approval by OWNERS check failed: this change has "
          "no Rietveld issue number, so we can't check it for approvals.")]
    output = output_api.PresubmitError
  else:
    output = output_api.PresubmitNotifyResult

  owners_db = input_api.owners_db
  owner_email, reviewers = input_api.canned_checks._RietveldOwnerAndReviewers(
      input_api,
      owners_db.email_regexp,
      approval_needed=input_api.is_committing)

  owner_email = owner_email or input_api.change.author_email

902 903 904
  reviewers_plus_owner = set(reviewers)
  if owner_email:
    reviewers_plus_owner.add(owner_email)
905 906
  missing_files = owners_db.files_not_covered_by(virtual_depended_on_files,
                                                 reviewers_plus_owner)
907 908 909 910 911 912 913 914 915 916 917

  # We strip the /DEPS part that was added by
  # _FilesToCheckForIncomingDeps to fake a path to a file in a
  # directory.
  def StripDeps(path):
    start_deps = path.rfind('/DEPS')
    if start_deps != -1:
      return path[:start_deps]
    else:
      return path
  unapproved_dependencies = ["'+%s'," % StripDeps(path)
918 919 920 921
                             for path in missing_files]

  if unapproved_dependencies:
    output_list = [
922
      output('Missing LGTM from OWNERS of dependencies added to DEPS:\n    %s' %
923 924 925 926 927 928 929 930 931 932 933
             '\n    '.join(sorted(unapproved_dependencies)))]
    if not input_api.is_committing:
      suggested_owners = owners_db.reviewers_for(missing_files, owner_email)
      output_list.append(output(
          'Suggested missing target path OWNERS:\n    %s' %
          '\n    '.join(suggested_owners or [])))
    return output_list

  return []


934 935 936 937 938 939
def _CheckSpamLogging(input_api, output_api):
  file_inclusion_pattern = r'.+%s' % _IMPLEMENTATION_EXTENSIONS
  black_list = (_EXCLUDED_PATHS +
                _TEST_CODE_EXCLUDED_PATHS +
                input_api.DEFAULT_BLACK_LIST +
                (r"^base[\\\/]logging\.h$",
940
                 r"^base[\\\/]logging\.cc$",
941 942
                 r"^chrome[\\\/]app[\\\/]chrome_main_delegate\.cc$",
                 r"^chrome[\\\/]browser[\\\/]chrome_browser_main\.cc$",
943 944
                 r"^chrome[\\\/]browser[\\\/]ui[\\\/]startup[\\\/]"
                     r"startup_browser_creator\.cc$",
945
                 r"^chrome[\\\/]installer[\\\/]setup[\\\/].*",
946 947
                 r"chrome[\\\/]browser[\\\/]diagnostics[\\\/]" +
                     r"diagnostics_writer\.cc$",
948 949 950
                 r"^chrome_elf[\\\/]dll_hash[\\\/]dll_hash_main\.cc$",
                 r"^chromecast[\\\/]",
                 r"^cloud_print[\\\/]",
951 952
                 r"^content[\\\/]common[\\\/]gpu[\\\/]client[\\\/]"
                     r"gl_helper_benchmark\.cc$",
953
                 r"^courgette[\\\/]courgette_tool\.cc$",
954
                 r"^extensions[\\\/]renderer[\\\/]logging_native_handler\.cc$",
955
                 r"^native_client_sdk[\\\/]",
956
                 r"^remoting[\\\/]base[\\\/]logging\.h$",
957 958 959
                 r"^remoting[\\\/]host[\\\/].*",
                 r"^sandbox[\\\/]linux[\\\/].*",
                 r"^tools[\\\/]",
960 961 962
                 r"^ui[\\\/]aura[\\\/]bench[\\\/]bench_main\.cc$",
                 r"^webkit[\\\/]browser[\\\/]fileapi[\\\/]" +
                     r"dump_file_system.cc$",))
963 964 965 966 967 968 969 970 971 972
  source_file_filter = lambda x: input_api.FilterSourceFile(
      x, white_list=(file_inclusion_pattern,), black_list=black_list)

  log_info = []
  printf = []

  for f in input_api.AffectedSourceFiles(source_file_filter):
    contents = input_api.ReadFile(f, 'rb')
    if re.search(r"\bD?LOG\s*\(\s*INFO\s*\)", contents):
      log_info.append(f.LocalPath())
973
    elif re.search(r"\bD?LOG_IF\s*\(\s*INFO\s*,", contents):
974
      log_info.append(f.LocalPath())
975 976 977 978

    if re.search(r"\bprintf\(", contents):
      printf.append(f.LocalPath())
    elif re.search(r"\bfprintf\((stdout|stderr)", contents):
979 980 981 982 983 984 985 986 987 988 989 990 991
      printf.append(f.LocalPath())

  if log_info:
    return [output_api.PresubmitError(
      'These files spam the console log with LOG(INFO):',
      items=log_info)]
  if printf:
    return [output_api.PresubmitError(
      'These files spam the console log with printf/fprintf:',
      items=printf)]
  return []


992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065
def _CheckForAnonymousVariables(input_api, output_api):
  """These types are all expected to hold locks while in scope and
     so should never be anonymous (which causes them to be immediately
     destroyed)."""
  they_who_must_be_named = [
    'base::AutoLock',
    'base::AutoReset',
    'base::AutoUnlock',
    'SkAutoAlphaRestore',
    'SkAutoBitmapShaderInstall',
    'SkAutoBlitterChoose',
    'SkAutoBounderCommit',
    'SkAutoCallProc',
    'SkAutoCanvasRestore',
    'SkAutoCommentBlock',
    'SkAutoDescriptor',
    'SkAutoDisableDirectionCheck',
    'SkAutoDisableOvalCheck',
    'SkAutoFree',
    'SkAutoGlyphCache',
    'SkAutoHDC',
    'SkAutoLockColors',
    'SkAutoLockPixels',
    'SkAutoMalloc',
    'SkAutoMaskFreeImage',
    'SkAutoMutexAcquire',
    'SkAutoPathBoundsUpdate',
    'SkAutoPDFRelease',
    'SkAutoRasterClipValidate',
    'SkAutoRef',
    'SkAutoTime',
    'SkAutoTrace',
    'SkAutoUnref',
  ]
  anonymous = r'(%s)\s*[({]' % '|'.join(they_who_must_be_named)
  # bad: base::AutoLock(lock.get());
  # not bad: base::AutoLock lock(lock.get());
  bad_pattern = input_api.re.compile(anonymous)
  # good: new base::AutoLock(lock.get())
  good_pattern = input_api.re.compile(r'\bnew\s*' + anonymous)
  errors = []

  for f in input_api.AffectedFiles():
    if not f.LocalPath().endswith(('.cc', '.h', '.inl', '.m', '.mm')):
      continue
    for linenum, line in f.ChangedContents():
      if bad_pattern.search(line) and not good_pattern.search(line):
        errors.append('%s:%d' % (f.LocalPath(), linenum))

  if errors:
    return [output_api.PresubmitError(
      'These lines create anonymous variables that need to be named:',
      items=errors)]
  return []


def _CheckCygwinShell(input_api, output_api):
  source_file_filter = lambda x: input_api.FilterSourceFile(
      x, white_list=(r'.+\.(gyp|gypi)$',))
  cygwin_shell = []

  for f in input_api.AffectedSourceFiles(source_file_filter):
    for linenum, line in f.ChangedContents():
      if 'msvs_cygwin_shell' in line:
        cygwin_shell.append(f.LocalPath())
        break

  if cygwin_shell:
    return [output_api.PresubmitError(
      'These files should not use msvs_cygwin_shell (the default is 0):',
      items=cygwin_shell)]
  return []


1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097
def _CheckUserActionUpdate(input_api, output_api):
  """Checks if any new user action has been added."""
  if any('actions.xml' == input_api.os_path.basename(f) for f in
         input_api.LocalPaths()):
    # If actions.xml is already included in the changelist, the PRESUBMIT
    # for actions.xml will do a more complete presubmit check.
    return []

  file_filter = lambda f: f.LocalPath().endswith(('.cc', '.mm'))
  action_re = r'[^a-zA-Z]UserMetricsAction\("([^"]*)'
  current_actions = None
  for f in input_api.AffectedFiles(file_filter=file_filter):
    for line_num, line in f.ChangedContents():
      match = input_api.re.search(action_re, line)
      if match:
        # Loads contents in tools/metrics/actions/actions.xml to memory. It's
        # loaded only once.
        if not current_actions:
          with open('tools/metrics/actions/actions.xml') as actions_f:
            current_actions = actions_f.read()
        # Search for the matched user action name in |current_actions|.
        for action_name in match.groups():
          action = 'name="{0}"'.format(action_name)
          if action not in current_actions:
            return [output_api.PresubmitPromptWarning(
              'File %s line %d: %s is missing in '
              'tools/metrics/actions/actions.xml. Please run '
              'tools/metrics/actions/extract_actions.py to update.'
              % (f.LocalPath(), line_num, action_name))]
  return []


1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143
def _GetJSONParseError(input_api, filename, eat_comments=True):
  try:
    contents = input_api.ReadFile(filename)
    if eat_comments:
      json_comment_eater = input_api.os_path.join(
          input_api.PresubmitLocalPath(),
          'tools', 'json_comment_eater', 'json_comment_eater.py')
      process = input_api.subprocess.Popen(
          [input_api.python_executable, json_comment_eater],
          stdin=input_api.subprocess.PIPE,
          stdout=input_api.subprocess.PIPE,
          universal_newlines=True)
      (contents, _) = process.communicate(input=contents)

    input_api.json.loads(contents)
  except ValueError as e:
    return e
  return None


def _GetIDLParseError(input_api, filename):
  try:
    contents = input_api.ReadFile(filename)
    idl_schema = input_api.os_path.join(
        input_api.PresubmitLocalPath(),
        'tools', 'json_schema_compiler', 'idl_schema.py')
    process = input_api.subprocess.Popen(
        [input_api.python_executable, idl_schema],
        stdin=input_api.subprocess.PIPE,
        stdout=input_api.subprocess.PIPE,
        stderr=input_api.subprocess.PIPE,
        universal_newlines=True)
    (_, error) = process.communicate(input=contents)
    return error or None
  except ValueError as e:
    return e


def _CheckParseErrors(input_api, output_api):
  """Check that IDL and JSON files do not contain syntax errors."""
  actions = {
    '.idl': _GetIDLParseError,
    '.json': _GetJSONParseError,
  }
  # These paths contain test data and other known invalid JSON files.
  excluded_patterns = [
1144 1145
    r'test[\\\/]data[\\\/]',
    r'^components[\\\/]policy[\\\/]resources[\\\/]policy_templates\.json$',
1146 1147 1148
  ]
  # Most JSON files are preprocessed and support comments, but these do not.
  json_no_comments_patterns = [
1149
    r'^testing[\\\/]',
1150 1151 1152
  ]
  # Only run IDL checker on files in these directories.
  idl_included_patterns = [
1153 1154
    r'^chrome[\\\/]common[\\\/]extensions[\\\/]api[\\\/]',
    r'^extensions[\\\/]common[\\\/]api[\\\/]',
1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197
  ]

  def get_action(affected_file):
    filename = affected_file.LocalPath()
    return actions.get(input_api.os_path.splitext(filename)[1])

  def MatchesFile(patterns, path):
    for pattern in patterns:
      if input_api.re.search(pattern, path):
        return True
    return False

  def FilterFile(affected_file):
    action = get_action(affected_file)
    if not action:
      return False
    path = affected_file.LocalPath()

    if MatchesFile(excluded_patterns, path):
      return False

    if (action == _GetIDLParseError and
        not MatchesFile(idl_included_patterns, path)):
      return False
    return True

  results = []
  for affected_file in input_api.AffectedFiles(
      file_filter=FilterFile, include_deletes=False):
    action = get_action(affected_file)
    kwargs = {}
    if (action == _GetJSONParseError and
        MatchesFile(json_no_comments_patterns, affected_file.LocalPath())):
      kwargs['eat_comments'] = False
    parse_error = action(input_api,
                         affected_file.AbsoluteLocalPath(),
                         **kwargs)
    if parse_error:
      results.append(output_api.PresubmitError('%s could not be parsed: %s' %
          (affected_file.LocalPath(), parse_error)))
  return results


1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211
def _CheckJavaStyle(input_api, output_api):
  """Runs checkstyle on changed java files and returns errors if any exist."""
  original_sys_path = sys.path
  try:
    sys.path = sys.path + [input_api.os_path.join(
        input_api.PresubmitLocalPath(), 'tools', 'android', 'checkstyle')]
    import checkstyle
  finally:
    # Restore sys.path to what it was before.
    sys.path = original_sys_path

  return checkstyle.RunCheckstyle(
      input_api, output_api, 'tools/android/checkstyle/chromium-style-5.0.xml')

1212

1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238
_DEPRECATED_CSS = [
  # Values
  ( "-webkit-box", "flex" ),
  ( "-webkit-inline-box", "inline-flex" ),
  ( "-webkit-flex", "flex" ),
  ( "-webkit-inline-flex", "inline-flex" ),
  ( "-webkit-min-content", "min-content" ),
  ( "-webkit-max-content", "max-content" ),

  # Properties
  ( "-webkit-background-clip", "background-clip" ),
  ( "-webkit-background-origin", "background-origin" ),
  ( "-webkit-background-size", "background-size" ),
  ( "-webkit-box-shadow", "box-shadow" ),

  # Functions
  ( "-webkit-gradient", "gradient" ),
  ( "-webkit-repeating-gradient", "repeating-gradient" ),
  ( "-webkit-linear-gradient", "linear-gradient" ),
  ( "-webkit-repeating-linear-gradient", "repeating-linear-gradient" ),
  ( "-webkit-radial-gradient", "radial-gradient" ),
  ( "-webkit-repeating-radial-gradient", "repeating-radial-gradient" ),
]

def _CheckNoDeprecatedCSS(input_api, output_api):
  """ Make sure that we don't use deprecated CSS
1239 1240 1241
      properties, functions or values. Our external
      documentation is ignored by the hooks as it
      needs to be consumed by WebKit. """
1242
  results = []
1243 1244 1245 1246 1247 1248 1249 1250 1251
  file_inclusion_pattern = (r".+\.css$")
  black_list = (_EXCLUDED_PATHS +
                _TEST_CODE_EXCLUDED_PATHS +
                input_api.DEFAULT_BLACK_LIST +
                (r"^chrome/common/extensions/docs",
                 r"^chrome/docs",
                 r"^native_client_sdk"))
  file_filter = lambda f: input_api.FilterSourceFile(
      f, white_list=file_inclusion_pattern, black_list=black_list)
1252 1253 1254 1255 1256 1257 1258 1259 1260
  for fpath in input_api.AffectedFiles(file_filter=file_filter):
    for line_num, line in fpath.ChangedContents():
      for (deprecated_value, value) in _DEPRECATED_CSS:
        if input_api.re.search(deprecated_value, line):
          results.append(output_api.PresubmitError(
              "%s:%d: Use of deprecated CSS %s, use %s instead" %
              (fpath.LocalPath(), line_num, deprecated_value, value)))
  return results

1261 1262 1263 1264
def _CommonChecks(input_api, output_api):
  """Checks common to both upload and commit."""
  results = []
  results.extend(input_api.canned_checks.PanProjectChecks(
1265 1266
      input_api, output_api,
      excluded_paths=_EXCLUDED_PATHS + _TESTRUNNER_PATHS))
1267 1268
  results.extend(_CheckAuthorizedAuthor(input_api, output_api))
  results.extend(
1269
      _CheckNoProductionCodeUsingTestOnlyFunctions(input_api, output_api))
1270 1271 1272 1273 1274 1275 1276 1277 1278 1279
  results.extend(_CheckNoIOStreamInHeaders(input_api, output_api))
  results.extend(_CheckNoUNIT_TESTInSourceFiles(input_api, output_api))
  results.extend(_CheckNoNewWStrings(input_api, output_api))
  results.extend(_CheckNoDEPSGIT(input_api, output_api))
  results.extend(_CheckNoBannedFunctions(input_api, output_api))
  results.extend(_CheckNoPragmaOnce(input_api, output_api))
  results.extend(_CheckNoTrinaryTrueFalse(input_api, output_api))
  results.extend(_CheckUnwantedDependencies(input_api, output_api))
  results.extend(_CheckFilePermissions(input_api, output_api))
  results.extend(_CheckNoAuraWindowPropertyHInHeaders(input_api, output_api))
1280 1281 1282 1283 1284
  results.extend(_CheckIncludeOrder(input_api, output_api))
  results.extend(_CheckForVersionControlConflicts(input_api, output_api))
  results.extend(_CheckPatchFiles(input_api, output_api))
  results.extend(_CheckHardcodedGoogleHostsInLowerLayers(input_api, output_api))
  results.extend(_CheckNoAbbreviationInPngFileName(input_api, output_api))
1285
  results.extend(_CheckForInvalidOSMacros(input_api, output_api))
1286
  results.extend(_CheckAddedDepsHaveTargetApprovals(input_api, output_api))
1287 1288 1289 1290 1291
  results.extend(
      input_api.canned_checks.CheckChangeHasNoTabs(
          input_api,
          output_api,
          source_file_filter=lambda x: x.LocalPath().endswith('.grd')))
1292
  results.extend(_CheckSpamLogging(input_api, output_api))
1293 1294
  results.extend(_CheckForAnonymousVariables(input_api, output_api))
  results.extend(_CheckCygwinShell(input_api, output_api))
1295
  results.extend(_CheckUserActionUpdate(input_api, output_api))
1296
  results.extend(_CheckNoDeprecatedCSS(input_api, output_api))
1297
  results.extend(_CheckParseErrors(input_api, output_api))
1298
  results.extend(_CheckForIPCRules(input_api, output_api))
1299 1300 1301 1302 1303 1304

  if any('PRESUBMIT.py' == f.LocalPath() for f in input_api.AffectedFiles()):
    results.extend(input_api.canned_checks.RunUnitTestsInDirectory(
        input_api, output_api,
        input_api.PresubmitLocalPath(),
        whitelist=[r'^PRESUBMIT_test\.py$']))
1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325
  return results


def _CheckAuthorizedAuthor(input_api, output_api):
  """For non-googler/chromites committers, verify the author's email address is
  in AUTHORS.
  """
  # TODO(maruel): Add it to input_api?
  import fnmatch

  author = input_api.change.author_email
  if not author:
    input_api.logging.info('No author, skipping AUTHOR check')
    return []
  authors_path = input_api.os_path.join(
      input_api.PresubmitLocalPath(), 'AUTHORS')
  valid_authors = (
      input_api.re.match(r'[^#]+\s+\<(.+?)\>\s*$', line)
      for line in open(authors_path))
  valid_authors = [item.group(1).lower() for item in valid_authors if item]
  if not any(fnmatch.fnmatch(author.lower(), valid) for valid in valid_authors):
1326
    input_api.logging.info('Valid authors are %s', ', '.join(valid_authors))
1327 1328 1329 1330 1331 1332 1333 1334 1335 1336
    return [output_api.PresubmitPromptWarning(
        ('%s is not in AUTHORS file. If you are a new contributor, please visit'
        '\n'
        'http://www.chromium.org/developers/contributing-code and read the '
        '"Legal" section\n'
        'If you are a chromite, verify the contributor signed the CLA.') %
        author)]
  return []


1337 1338 1339 1340 1341 1342 1343 1344 1345 1346
def _CheckPatchFiles(input_api, output_api):
  problems = [f.LocalPath() for f in input_api.AffectedFiles()
      if f.LocalPath().endswith(('.orig', '.rej'))]
  if problems:
    return [output_api.PresubmitError(
        "Don't commit .rej and .orig files.", problems)]
  else:
    return []


1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395
def _DidYouMeanOSMacro(bad_macro):
  try:
    return {'A': 'OS_ANDROID',
            'B': 'OS_BSD',
            'C': 'OS_CHROMEOS',
            'F': 'OS_FREEBSD',
            'L': 'OS_LINUX',
            'M': 'OS_MACOSX',
            'N': 'OS_NACL',
            'O': 'OS_OPENBSD',
            'P': 'OS_POSIX',
            'S': 'OS_SOLARIS',
            'W': 'OS_WIN'}[bad_macro[3].upper()]
  except KeyError:
    return ''


def _CheckForInvalidOSMacrosInFile(input_api, f):
  """Check for sensible looking, totally invalid OS macros."""
  preprocessor_statement = input_api.re.compile(r'^\s*#')
  os_macro = input_api.re.compile(r'defined\((OS_[^)]+)\)')
  results = []
  for lnum, line in f.ChangedContents():
    if preprocessor_statement.search(line):
      for match in os_macro.finditer(line):
        if not match.group(1) in _VALID_OS_MACROS:
          good = _DidYouMeanOSMacro(match.group(1))
          did_you_mean = ' (did you mean %s?)' % good if good else ''
          results.append('    %s:%d %s%s' % (f.LocalPath(),
                                             lnum,
                                             match.group(1),
                                             did_you_mean))
  return results


def _CheckForInvalidOSMacros(input_api, output_api):
  """Check all affected files for invalid OS macros."""
  bad_macros = []
  for f in input_api.AffectedFiles():
    if not f.LocalPath().endswith(('.py', '.js', '.html', '.css')):
      bad_macros.extend(_CheckForInvalidOSMacrosInFile(input_api, f))

  if not bad_macros:
    return []

  return [output_api.PresubmitError(
      'Possibly invalid OS macro[s] found. Please fix your code\n'
      'or add your macro to src/PRESUBMIT.py.', bad_macros)]

1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419
def _CheckForIPCRules(input_api, output_api):
  """Check for same IPC rules described in
  http://www.chromium.org/Home/chromium-security/education/security-tips-for-ipc
  """
  base_pattern = r'IPC_ENUM_TRAITS\('
  inclusion_pattern = input_api.re.compile(r'(%s)' % base_pattern)
  comment_pattern = input_api.re.compile(r'//.*(%s)' % base_pattern)

  problems = []
  for f in input_api.AffectedSourceFiles(None):
    local_path = f.LocalPath()
    if not local_path.endswith('.h'):
      continue
    for line_number, line in f.ChangedContents():
      if inclusion_pattern.search(line) and not comment_pattern.search(line):
        problems.append(
          '%s:%d\n    %s' % (local_path, line_number, line.strip()))

  if problems:
    return [output_api.PresubmitPromptWarning(
        _IPC_ENUM_TRAITS_DEPRECATED, problems)]
  else:
    return []

1420

1421 1422 1423
def CheckChangeOnUpload(input_api, output_api):
  results = []
  results.extend(_CommonChecks(input_api, output_api))
1424
  results.extend(_CheckValidHostsInDEPS(input_api, output_api))
1425
  results.extend(_CheckJavaStyle(input_api, output_api))
1426 1427 1428
  return results


1429 1430 1431
def GetTryServerMasterForBot(bot):
  """Returns the Try Server master for the given bot.

1432 1433 1434 1435 1436
  It tries to guess the master from the bot name, but may still fail
  and return None.  There is no longer a default master.
  """
  # Potentially ambiguous bot names are listed explicitly.
  master_map = {
1437
      'linux_gpu': 'tryserver.chromium.gpu',
1438
      'mac_gpu': 'tryserver.chromium.gpu',
1439
      'win_gpu': 'tryserver.chromium.gpu',
1440 1441 1442
      'chromium_presubmit': 'tryserver.chromium.linux',
      'blink_presubmit': 'tryserver.chromium.linux',
      'tools_build_presubmit': 'tryserver.chromium.linux',
1443
  }
1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454
  master = master_map.get(bot)
  if not master:
    if 'gpu' in bot:
      master = 'tryserver.chromium.gpu'
    elif 'linux' in bot or 'android' in bot or 'presubmit' in bot:
      master = 'tryserver.chromium.linux'
    elif 'win' in bot:
      master = 'tryserver.chromium.win'
    elif 'mac' in bot or 'ios' in bot:
      master = 'tryserver.chromium.mac'
  return master
1455 1456


1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500
def GetDefaultTryConfigs(bots=None):
  """Returns a list of ('bot', set(['tests']), optionally filtered by [bots].

  To add tests to this list, they MUST be in the the corresponding master's
  gatekeeper config. For example, anything on master.chromium would be closed by
  tools/build/masters/master.chromium/master_gatekeeper_cfg.py.

  If 'bots' is specified, will only return configurations for bots in that list.
  """

  standard_tests = [
      'base_unittests',
      'browser_tests',
      'cacheinvalidation_unittests',
      'check_deps',
      'check_deps2git',
      'content_browsertests',
      'content_unittests',
      'crypto_unittests',
      'gpu_unittests',
      'interactive_ui_tests',
      'ipc_tests',
      'jingle_unittests',
      'media_unittests',
      'net_unittests',
      'ppapi_unittests',
      'printing_unittests',
      'sql_unittests',
      'sync_unit_tests',
      'unit_tests',
      # Broken in release.
      #'url_unittests',
      #'webkit_unit_tests',
  ]

  builders_and_tests = {
      # TODO(maruel): Figure out a way to run 'sizes' where people can
      # effectively update the perf expectation correctly.  This requires a
      # clobber=True build running 'sizes'. 'sizes' is not accurate with
      # incremental build. Reference:
      # http://chromium.org/developers/tree-sheriffs/perf-sheriffs.
      # TODO(maruel): An option would be to run 'sizes' but not count a failure
      # of this step as a try job failure.
      'android_aosp': ['compile'],
1501 1502
      'android_arm64_dbg_recipe': ['slave_steps'],
      'android_chromium_gn_compile_dbg': ['compile'],
1503
      'android_chromium_gn_compile_rel': ['compile'],
1504
      'android_clang_dbg': ['slave_steps'],
1505
      'android_clang_dbg_recipe': ['slave_steps'],
1506
      'android_dbg_tests_recipe': ['slave_steps'],
1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518
      'cros_x86': ['defaulttests'],
      'ios_dbg_simulator': [
          'compile',
          'base_unittests',
          'content_unittests',
          'crypto_unittests',
          'url_unittests',
          'net_unittests',
          'sql_unittests',
          'ui_unittests',
      ],
      'ios_rel_device': ['compile'],
1519
      'ios_rel_device_ninja': ['compile'],
1520 1521
      'linux_asan': ['compile'],
      'mac_asan': ['compile'],
1522 1523
      #TODO(stip): Change the name of this builder to reflect that it's release.
      'linux_gtk': standard_tests,
1524
      'linux_chromeos_asan': ['compile'],
1525
      'linux_chromium_chromeos_clang_dbg': ['defaulttests'],
1526
      'linux_chromium_chromeos_rel_swarming': ['defaulttests'],
1527
      'linux_chromium_compile_dbg': ['defaulttests'],
1528
      'linux_chromium_gn_dbg': ['compile'],
1529
      'linux_chromium_gn_rel': ['defaulttests'],
1530
      'linux_chromium_rel_swarming': ['defaulttests'],
1531
      'linux_chromium_clang_dbg': ['defaulttests'],
1532
      'linux_gpu': ['defaulttests'],
1533
      'linux_nacl_sdk_build': ['compile'],
1534
      'mac_chromium_compile_dbg': ['defaulttests'],
1535
      'mac_chromium_rel_swarming': ['defaulttests'],
1536
      'mac_gpu': ['defaulttests'],
1537
      'mac_nacl_sdk_build': ['compile'],
1538
      'win_chromium_compile_dbg': ['defaulttests'],
1539
      'win_chromium_dbg': ['defaulttests'],
1540
      'win_chromium_rel_swarming': ['defaulttests'],
1541
      'win_chromium_x64_rel_swarming': ['defaulttests'],
1542
      'win_gpu': ['defaulttests'],
1543
      'win_nacl_sdk_build': ['compile'],
1544
      'win8_chromium_rel': ['defaulttests'],
1545 1546 1547
  }

  if bots:
1548 1549
    filtered_builders_and_tests = dict((bot, set(builders_and_tests[bot]))
                                       for bot in bots)
1550
  else:
1551 1552 1553 1554 1555 1556 1557 1558 1559
    filtered_builders_and_tests = dict(
        (bot, set(tests))
        for bot, tests in builders_and_tests.iteritems())

  # Build up the mapping from tryserver master to bot/test.
  out = dict()
  for bot, tests in filtered_builders_and_tests.iteritems():
    out.setdefault(GetTryServerMasterForBot(bot), {})[bot] = tests
  return out
1560 1561


1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580
def CheckChangeOnCommit(input_api, output_api):
  results = []
  results.extend(_CommonChecks(input_api, output_api))
  # TODO(thestig) temporarily disabled, doesn't work in third_party/
  #results.extend(input_api.canned_checks.CheckSvnModifiedDirectories(
  #    input_api, output_api, sources))
  # Make sure the tree is 'open'.
  results.extend(input_api.canned_checks.CheckTreeIsOpen(
      input_api,
      output_api,
      json_url='http://chromium-status.appspot.com/current?format=json'))

  results.extend(input_api.canned_checks.CheckChangeHasBugField(
      input_api, output_api))
  results.extend(input_api.canned_checks.CheckChangeHasDescription(
      input_api, output_api))
  return results


1581
def GetPreferredTryMasters(project, change):
1582 1583
  files = change.LocalPaths()

1584
  if not files or all(re.search(r'[\\\/]OWNERS$', f) for f in files):
1585
    return {}
1586

1587
  if all(re.search(r'\.(m|mm)$|(^|[\\\/_])mac[\\\/_.]', f) for f in files):
1588 1589
    return GetDefaultTryConfigs([
        'mac_chromium_compile_dbg',
1590
        'mac_chromium_rel_swarming',
1591
    ])
1592
  if all(re.search('(^|[/_])win[/_.]', f) for f in files):
1593 1594 1595
    return GetDefaultTryConfigs([
        'win_chromium_dbg',
        'win_chromium_rel_swarming',
1596
        'win8_chromium_rel',
1597
    ])
1598
  if all(re.search(r'(^|[\\\/_])android[\\\/_.]', f) for f in files):
1599 1600 1601
    return GetDefaultTryConfigs([
        'android_aosp',
        'android_clang_dbg',
1602
        'android_dbg_tests_recipe',
1603
    ])
1604
  if all(re.search(r'[\\\/_]ios[\\\/_.]', f) for f in files):
1605
    return GetDefaultTryConfigs(['ios_rel_device', 'ios_dbg_simulator'])
1606

1607
  builders = [
1608
      'android_arm64_dbg_recipe',
1609
      'android_chromium_gn_compile_rel',
1610
      'android_chromium_gn_compile_dbg',
1611
      'android_clang_dbg',
1612
      'android_clang_dbg_recipe',
1613
      'android_dbg_tests_recipe',
1614 1615
      'ios_dbg_simulator',
      'ios_rel_device',
1616
      'ios_rel_device_ninja',
1617
      'linux_chromium_chromeos_rel_swarming',
1618
      'linux_chromium_clang_dbg',
1619
      'linux_chromium_gn_dbg',
1620
      'linux_chromium_gn_rel',
1621
      'linux_chromium_rel_swarming',
1622
      'linux_gpu',
1623
      'mac_chromium_compile_dbg',
1624
      'mac_chromium_rel_swarming',
1625
      'mac_gpu',
1626
      'win_chromium_compile_dbg',
1627
      'win_chromium_rel_swarming',
1628
      'win_chromium_x64_rel_swarming',
1629
      'win_gpu',
1630
      'win8_chromium_rel',
1631
  ]
1632 1633

  # Match things like path/aura/file.cc and path/file_aura.cc.
1634
  # Same for chromeos.
1635
  if any(re.search(r'[\\\/_](aura|chromeos)', f) for f in files):
1636 1637 1638 1639
    builders.extend([
        'linux_chromeos_asan',
        'linux_chromium_chromeos_clang_dbg'
    ])
1640

1641 1642 1643 1644 1645 1646
  # If there are gyp changes to base, build, or chromeos, run a full cros build
  # in addition to the shorter linux_chromeos build. Changes to high level gyp
  # files have a much higher chance of breaking the cros build, which is
  # differnt from the linux_chromeos build that most chrome developers test
  # with.
  if any(re.search('^(base|build|chromeos).*\.gypi?$', f) for f in files):
1647
    builders.extend(['cros_x86'])
1648

1649 1650 1651 1652 1653
  # The AOSP bot doesn't build the chrome/ layer, so ignore any changes to it
  # unless they're .gyp(i) files as changes to those files can break the gyp
  # step on that bot.
  if (not all(re.search('^chrome', f) for f in files) or
      any(re.search('\.gypi?$', f) for f in files)):
1654
    builders.extend(['android_aosp'])
1655

1656
  return GetDefaultTryConfigs(builders)