"""This script automatically disables tests, given an ID and a set of
configurations on which it should be disabled. See the README for more details.
"""
import argparse
import os
import sys
import subprocess
import traceback
from typing import List, Optional, Tuple
import urllib.parse
import conditions
import errors
import expectations
import gtest
import resultdb
SRC_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
def main(argv: List[str]) -> int:
valid_conds = ' '.join(
sorted(f'\t{term.name}' for term in conditions.TERMINALS))
parser = argparse.ArgumentParser(
description='Disables tests.',
epilog=f"Valid conditions are:\n{valid_conds}")
parser.add_argument(
'build',
type=str,
help='the Buildbucket build ID to search for tests to disable ')
parser.add_argument('test_regex',
type=str,
help='the regex for the test to disable. For example: ' +
'".*CompressionUtilsTest.GzipCompression.*". Currently' +
'we assume that there is at most one test matching' +
'the regex. Disabling multiple tests at the same time' +
'is not currently supported (crbug.com/1364416)')
parser.add_argument('conditions',
type=str,
nargs='*',
help="the conditions under which to disable the test. " +
"Each entry consists of any number of conditions joined" +
" with '&', specifying the conjunction of these values." +
" All entries will be 'OR'ed together, along with any " +
"existing conditions from the file.")
parser.add_argument('-c',
'--cache',
action='store_true',
help='cache ResultDB rpc results, useful for testing.')
parser.add_argument(
'-b',
'--bug',
help="write a TODO referencing this bug in a comment " +
"next to the disabled test. Bug can be given as just the" +
" ID or a URL (e.g. 123456, crbug.com/v8/654321).")
parser.add_argument('-m',
'--message',
help="write a comment containing this message next to " +
"the disabled test.")
args = parser.parse_args(argv[1:])
if args.cache:
resultdb.CANNED_RESPONSE_FILE = os.path.join(os.path.dirname(__file__),
'.canned_responses.json')
message = args.message
if args.bug is not None:
try:
message = make_bug_message(args.bug, message)
except Exception:
print(
'Invalid value for --bug. Should have one of the following forms:\n' +
'\t1234\n' + '\tcrbug/1234\n' + '\tcrbug/project/1234\n' +
'\tcrbug.com/1234\n' + '\tcrbug.com/project/1234\n' +
'\tbugs.chromium.org/p/project/issues/detail?id=1234\n',
file=sys.stderr)
return 1
try:
disable_test(args.build, args.test_regex, args.conditions, message)
return 0
except errors.UserError as e:
print(e, file=sys.stderr)
return 1
except errors.InternalError as e:
trace = traceback.format_exc()
print(f"Internal error: {e}", file=sys.stderr)
print('Please file a bug using the following link:', file=sys.stderr)
print(generate_bug_link(args, trace), file=sys.stderr)
return 1
except Exception:
trace = traceback.format_exc()
print(f'Error: unhandled exception at top-level\n{trace}', file=sys.stderr)
print('Please file a bug using the following link:', file=sys.stderr)
print(generate_bug_link(args, trace), file=sys.stderr)
return 1
def make_bug_message(bug: str, message: str) -> str:
bug_id, project = parse_bug(bug)
project_component = '' if project == 'chromium' else f'{project}/'
bug_url = f"crbug.com/{project_component}{bug_id}"
if not message:
message = "Re-enable this test"
return f"TODO({bug_url}): {message}"
def parse_bug(bug: str) -> Tuple[int, str]:
try:
bug_id = int(bug)
return bug_id, 'chromium'
except ValueError:
pass
if '//' not in bug:
bug = f"https://{bug}"
url = urllib.parse.urlparse(bug)
if url.netloc in {'crbug', 'crbug.com'}:
parts = url.path.split('/')[1:]
if len(parts) == 1:
return int(parts[0]), 'chromium'
return int(parts[1]), parts[0]
if url.netloc == 'bugs.chromium.org':
parts = url.path.split('/')[1:]
project = parts[1]
bug_id = int(urllib.parse.parse_qs(url.query)['id'][0])
return bug_id, project
raise ValueError()
def disable_test(build: str, test_regex: str, cond_strs: List[str],
message: Optional[str]):
conds = conditions.parse(cond_strs)
invocation = "invocations/build-" + build
test_name, filename = resultdb.get_test_metadata(invocation, test_regex)
test_name = extract_name_and_suite(test_name)
full_path = os.path.join(SRC_ROOT, filename.lstrip('/'))
_, extension = os.path.splitext(full_path)
extension = extension.lstrip('.')
if extension == 'html':
full_path = expectations.search_for_expectations(full_path, test_name)
try:
with open(full_path, 'r') as f:
source_file = f.read()
except FileNotFoundError as e:
raise errors.UserError(
f"Couldn't open file {filename}. Either this test has moved file very" +
"recently, or your checkout isn't up-to-date.") from e
if extension == 'cc':
disabler = gtest.disabler
elif extension == 'html':
disabler = expectations.disabler
else:
raise errors.UserError(
f"Don't know how to disable tests for this file format ({extension})")
new_content = disabler(test_name, source_file, conds, message)
with open(full_path, 'w') as f:
f.write(new_content)
def extract_name_and_suite(test_name: str) -> str:
if test_name.endswith('.html'):
return test_name
for part in test_name.split('/'):
if '.' in part:
return part
raise errors.UserError(f"Couldn't parse test name: {test_name}")
def get_current_commit_hash() -> Optional[str]:
proc = subprocess.run(['git', 'rev-parse', 'HEAD'],
check=False,
capture_output=True,
text=True)
if proc.returncode != 0:
return None
return proc.stdout.strip()
def generate_bug_link(args: argparse.Namespace, trace: str) -> str:
trace = trace.replace(SRC_ROOT, '/')
args_list = '\n'.join(f'{k} = {v}' for k, v in args.__dict__.items())
summary = f'Test disabler failed for {args.test_id}'
description = f'''
<Please describe the problem here>
========== Debug info ==========
Exception:
{trace}
Args:
{args_list}'''
if (git_hash := get_current_commit_hash()) is not None:
description += f'''
Checked out chromium/src revision:
{git_hash}
'''
params = urllib.parse.urlencode(
dict(
labels='Type-Bug,Pri-2',
components='Infra>Sheriffing>SheriffOMatic',
summary=summary,
description=description,
))
return urllib.parse.urlunsplit(
('https', 'bugs.chromium.org', '/p/chromium/issues/entry', params, ''))
if __name__ == '__main__':
sys.exit(main(sys.argv))