chen3feng / blade-build / 862

pending completion

Build # 862

Build Type

Pull #706

travis-ci

Committed by

web-flow

Commit Message

<a href="https://github.com/chen3feng/blade-build/commit/<a class=hub.com/chen3feng/blade-build/commit/7a1cccd19391d9d159442b15dca23e81d82f5fd8">7a1cccd19<a href="https://github.com/chen3feng/blade-build/commit/7a1cccd19391d9d159442b15dca23e81d82f5fd8">">Merge </a><a class="double-link" href="https://github.com/chen3feng/blade-build/commit/<a class="double-link" href="https://github.com/chen3feng/blade-build/commit/94f50e55a7824b21930024fe8358ad926d7286c6">94f50e55a</a>">94f50e55a</a><a href="https://github.com/chen3feng/blade-build/commit/7a1cccd19391d9d159442b15dca23e81d82f5fd8"> into 475ac03ee">475ac03ee</a>

Pull Request Pull Request #706: Change target.key to string

Run Details

90 of 90 new or added lines in 11 files covered. (100.0%)

2784 of 7257 relevant lines covered (38.36%)

0.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

53.76

/src/blade/load_build_files.py

# Copyright (c) 2011 Tencent Inc.
# All rights reserved.
#
# Author: Huan Yu <huanyu@tencent.com>
#         Feng Chen <phongchen@tencent.com>
#         Yi Wang <yiwang@tencent.com>
#         Chong Peng <michaelpeng@tencent.com>
#         Wenting Li <wentingli@tencent.com>
# Date:   October 20, 2011


"""
 This is the CmdOptions module which parses the users'
 input and provides hint for users.

"""

from __future__ import absolute_import

import os
import traceback

from blade import build_attributes
from blade import build_rules
from blade import config
from blade import console
from blade.blade_util import var_to_list, exec_file, source_location
from blade.pathlib import Path


# import these modules make build functions registered into build_rules
# TODO(chen3feng): Load build modules dynamically to enable extension.


def _load_build_rules():
    # pylint: disable=import-outside-toplevel,unused-import
    import blade.cc_targets
    import blade.cu_targets
    import blade.gen_rule_target
    import blade.go_targets
    import blade.java_targets
    import blade.scala_targets
    import blade.lex_yacc_target
    import blade.package_target
    import blade.proto_library_target
    import blade.py_targets
    import blade.resource_library_target
    import blade.sh_test_target
    import blade.swig_library_target
    import blade.thrift_library
    import blade.fbthrift_library


def _find_dir_dependent(dir, blade):
    """Find which target depends on the dir. """
    target_database = blade.get_target_database()
    for key in target_database:
        target = target_database[key]
        for dkey in target.expanded_deps:
            if dkey.split(':')[0] == dir:
                return target
    return None


def _report_not_exist(kind, path, source_dir, blade):
    """Report dir or BUILD file does not exist. """
    msg = '%s "//%s" does not exist' % (kind, path)
    dependent = _find_dir_dependent(source_dir, blade)
    (dependent or console).fatal(msg)


def enable_if(cond, true_value, false_value=None):
    """A global function can be called in BUILD to filter srcs/deps by target"""
    if cond:
        ret = true_value
    else:
        ret = false_value
    if ret is None:
        ret = []
    return ret


def glob(include, exclude=None, excludes=None, allow_empty=False):
    """This function can be called in BUILD to specify a set of files using patterns.
    Args:
        include:List[str], file patterns to be matched.
        exclude:Optional[List[str]], file patterns to be removed from the result.
        allow_empty:bool: Whether a empty result is a error.

    Patterns may contain shell-like wildcards, such as * , ? , or [charset].
    Additionally, the path element '**' matches any subpath.
    """
    from blade import build_manager  # pylint: disable=import-outside-toplevel
    source_dir = Path(build_manager.instance.get_current_source_path())
    source_loc = source_location(os.path.join(str(source_dir), 'BUILD'))
    include = var_to_list(include)
    severity = config.get_item('global_config', 'glob_error_severity')
    output = getattr(console, severity)
    if excludes:
        output('%s %s: "excludes" is deprecated, use "exclude" instead' % (source_loc, severity),
               prefix=False)
    exclude = var_to_list(exclude) + var_to_list(excludes)

    def includes_iterator():
        results = []
        for pattern in include:
            for path in source_dir.glob(pattern):
                if path.is_file() and not path.name.startswith('.'):
                    results.append(path.relative_to(source_dir))

        return results

    def is_special(pattern):
        return '*' in pattern or '?' in pattern or '[' in pattern

    non_special_excludes = set()
    match_excludes = set()
    for pattern in exclude:
        if is_special(pattern):
            match_excludes.add(pattern)
        else:
            non_special_excludes.add(pattern)

    def exclusion(path):
        if str(path) in non_special_excludes:
            return True
        for pattern in match_excludes:
            ret = path.match(pattern)
            if ret:
                return True
        return False

    result = sorted({str(p) for p in includes_iterator() if not exclusion(p)})
    if not result and not allow_empty:
        args = repr(include)
        if exclude:
            args += ', exclude=%s' % repr(exclude)
        output('%s %s: "glob(%s)" got an empty result. If it is the expected behavior, '
               'specify "allow_empty=True" to eliminate this message' % (source_loc, severity, args),
               prefix=False)

    return result


# Each include in a BUILD file can only affect itself
__current_globles = None


# Include a defination file in a BUILD file
def include(name):
    from blade import build_manager  # pylint: disable=import-outside-toplevel
    if name.startswith('//'):
        dir = build_manager.instance.get_root_dir()
        name = name[2:]
    else:
        dir = build_manager.instance.get_current_source_path()
    exec_file(os.path.join(dir, name), __current_globles, None)


build_rules.register_function(enable_if)
build_rules.register_function(glob)
build_rules.register_function(include)


def _load_build_file(source_dir, processed_source_dirs, blade):
    """Load the BUILD and place the targets into database.

    Invoked by _load_targets.  Load and execute the BUILD
    file, which is a Python script, in source_dir.  Statements in BUILD
    depends on global variable current_source_dir, and will register build
    target/rules into global variables target_database.  Report error
    and exit if path/BUILD does NOT exist.
    The parameters processed_source_dirs refers to a set defined in the
    caller and used to avoid duplicated execution of BUILD files.

    """
    source_dir = os.path.normpath(source_dir)
    # TODO(yiwang): the character '#' is a magic value.
    if source_dir in processed_source_dirs or source_dir == '#':
        return
    processed_source_dirs.add(source_dir)

    if not os.path.exists(source_dir):
        _report_not_exist('Directory', source_dir, source_dir, blade)
        return

    old_current_source_path = blade.get_current_source_path()
    try:
        blade.set_current_source_path(source_dir)
        build_file = os.path.join(source_dir, 'BUILD')
        if os.path.exists(build_file) and not os.path.isdir(build_file):
            try:
                # The magic here is that a BUILD file is a Python script,
                # which can be loaded and executed by execfile().
                global __current_globles
                __current_globles = build_rules.get_all()
                exec_file(build_file, __current_globles, None)
            except SystemExit:
                console.fatal('%s: Fatal error' % build_file)
            except:  # pylint: disable=bare-except
                console.fatal('Parse error in %s\n%s' % (
                    build_file, traceback.format_exc()))
        else:
            _report_not_exist('File', build_file, source_dir, blade)
    finally:
        blade.set_current_source_path(old_current_source_path)


def _find_dependent(dkey, blade):
    """Find which target depends on the target with dkey. """
    target_database = blade.get_target_database()
    for key in target_database:
        target = target_database[key]
        if dkey in target.expanded_deps:
            return target
    return None


# File names should be skipped
_SKIP_FILES = ['BLADE_ROOT', '.bladeskip']


def _is_load_excluded(root, d):
    """Whether exclude the directory when loading BUILD.
    """
    # TODO(wentingli): Exclude directories matching patterns configured globally

    # Exclude directories starting with '.', e.g. '.', '..', '.svn', '.git'.
    if d.startswith('.'):
        return True

    # Exclude build dirs
    for build_dir in ('build32_debug', 'build32_release',
                       'build64_debug', 'build64_release'):
        if d.startswith(build_dir):
            return True

    # Exclude directories containing special files
    for skip_file in _SKIP_FILES:
        if os.path.exists(os.path.join(root, d, skip_file)):
            console.info('Skip "%s" due to "%s" file' % (os.path.join(root, d), skip_file))
            return True

    return False


def load_targets(target_ids, blade_root_dir, blade):
    """load_targets.

    Parse and load targets, including those specified in command line
    and their direct and indirect dependencies, by loading related BUILD
    files.  Returns a map which contains all these targets.

    """
    _load_build_rules()

    # pylint: disable=too-many-locals
    build_rules.register_variable('build_target', build_attributes.attributes)
    target_database = blade.get_target_database()
    # targets specified in command line
    cited_targets = set()
    # cited_targets and all its dependencies
    related_targets = {}
    # source dirs mentioned in command line
    source_dirs = []
    # to prevent duplicated loading of BUILD files
    processed_source_dirs = set()

    direct_targets = []
    all_command_targets = []
    # Parse command line target_ids.  For those in the form of <path>:<target>,
    # record (<path>,<target>) in cited_targets; for the rest (with <path>
    # but without <target>), record <path> into paths.
    for target_id in target_ids:
        source_dir, target_name = target_id.rsplit(':', 1)
        if not os.path.exists(source_dir):
            _report_not_exist('Directory', source_dir, source_dir, blade)

        if target_name not in ('*', '...'):
            cited_targets.add(source_dir + ':' + target_name)
        elif target_name == '...':
            for root, dirs, files in os.walk(source_dir):
                # Note the dirs[:] = slice assignment; we are replacing the
                # elements in dirs (and not the list referred to by dirs) so
                # that os.walk() will not process deleted directories.
                dirs[:] = [d for d in dirs if not _is_load_excluded(root, d)]
                if 'BUILD' in files:
                    source_dirs.append(root)
        else:
            source_dirs.append(source_dir)

    direct_targets = list(cited_targets)

    # Load BUILD files in paths, and add all loaded targets into
    # cited_targets.  Together with above step, we can ensure that all
    # targets mentioned in the command line are now in cited_targets.
    for source_dir in source_dirs:
        _load_build_file(source_dir,
                         processed_source_dirs,
                         blade)

    for key in target_database:
        cited_targets.add(key)
    all_command_targets = list(cited_targets)

    # Starting from targets specified in command line, breath-first
    # propagate to load BUILD files containing directly and indirectly
    # dependent targets.  All these targets form related_targets,
    # which is a subset of target_databased created by loading  BUILD files.
    while cited_targets:
        target_id = cited_targets.pop()
        source_dir, target_name = target_id.split(':')
        if target_id in related_targets:
            continue

        _load_build_file(source_dir,
                         processed_source_dirs,
                         blade)

        if target_id not in target_database:
            msg = 'Target "//%s" does not exist' % target_id
            dependent = _find_dependent(target_id, blade)
            (dependent or console).error(msg)
            continue

        related_targets[target_id] = target_database[target_id]
        for key in related_targets[target_id].expanded_deps:
            if key not in related_targets:
                cited_targets.add(key)

    # Iterating to get svn root dirs
    for target_id in related_targets:  # pylint: disable=dict-iter-missing-items
        path, name = target_id.rsplit(':')
        root_dir = path.split('/')[0].strip()
        if root_dir not in blade.svn_root_dirs and '#' not in root_dir:
            blade.svn_root_dirs.append(root_dir)

    return direct_targets, all_command_targets, related_targets

1	# Copyright (c) 2011 Tencent Inc.
2	# All rights reserved.
3	#
4	# Author: Huan Yu <huanyu@tencent.com>
5	# Feng Chen <phongchen@tencent.com>
6	# Yi Wang <yiwang@tencent.com>
7	# Chong Peng <michaelpeng@tencent.com>
8	# Wenting Li <wentingli@tencent.com>
9	# Date: October 20, 2011
10
11
12	"""	2✔
13	This is the CmdOptions module which parses the users'
14	input and provides hint for users.
15
16	"""
17
18	from __future__ import absolute_import	2✔
19
20	import os	2✔
21	import traceback	2✔
22
23	from blade import build_attributes	2✔
24	from blade import build_rules	2✔
25	from blade import config	2✔
26	from blade import console	2✔
27	from blade.blade_util import var_to_list, exec_file, source_location	2✔
28	from blade.pathlib import Path	2✔
29
30
31	# import these modules make build functions registered into build_rules
32	# TODO(chen3feng): Load build modules dynamically to enable extension.
33
34
35	def _load_build_rules():	2✔
36	# pylint: disable=import-outside-toplevel,unused-import
37	import blade.cc_targets	2✔
38	import blade.cu_targets	2✔
39	import blade.gen_rule_target	2✔
40	import blade.go_targets	2✔
41	import blade.java_targets	2✔
42	import blade.scala_targets	2✔
43	import blade.lex_yacc_target	2✔
44	import blade.package_target	2✔
45	import blade.proto_library_target	2✔
46	import blade.py_targets	2✔
47	import blade.resource_library_target	2✔
48	import blade.sh_test_target	2✔
49	import blade.swig_library_target	2✔
50	import blade.thrift_library	2✔
51	import blade.fbthrift_library	2✔
52
53
54	def _find_dir_dependent(dir, blade):	2✔
55	"""Find which target depends on the dir. """
56	target_database = blade.get_target_database()	×
57	for key in target_database:	×
58	target = target_database[key]	×
59	for dkey in target.expanded_deps:	×
60	if dkey.split(':')[0] == dir:	×
61	return target	×
62	return None	×
63
64
65	def _report_not_exist(kind, path, source_dir, blade):	2✔
66	"""Report dir or BUILD file does not exist. """
67	msg = '%s "//%s" does not exist' % (kind, path)	×
68	dependent = _find_dir_dependent(source_dir, blade)	×
69	(dependent or console).fatal(msg)	×
70
71
72	def enable_if(cond, true_value, false_value=None):	2✔
73	"""A global function can be called in BUILD to filter srcs/deps by target"""
74	if cond:	×
75	ret = true_value	×
76	else:
77	ret = false_value	×
78	if ret is None:	×
79	ret = []	×
80	return ret	×
81
82
83	def glob(include, exclude=None, excludes=None, allow_empty=False):	2✔
84	"""This function can be called in BUILD to specify a set of files using patterns.
85	Args:
86	include:List[str], file patterns to be matched.
87	exclude:Optional[List[str]], file patterns to be removed from the result.
88	allow_empty:bool: Whether a empty result is a error.
89
90	Patterns may contain shell-like wildcards, such as * , ? , or [charset].
91	Additionally, the path element '**' matches any subpath.
92	"""
93	from blade import build_manager # pylint: disable=import-outside-toplevel	×
94	source_dir = Path(build_manager.instance.get_current_source_path())	×
95	source_loc = source_location(os.path.join(str(source_dir), 'BUILD'))	×
96	include = var_to_list(include)	×
97	severity = config.get_item('global_config', 'glob_error_severity')	×
98	output = getattr(console, severity)	×
99	if excludes:	×
100	output('%s %s: "excludes" is deprecated, use "exclude" instead' % (source_loc, severity),	×
101	prefix=False)
102	exclude = var_to_list(exclude) + var_to_list(excludes)	×
103
104	def includes_iterator():	×
105	results = []	×
106	for pattern in include:	×
107	for path in source_dir.glob(pattern):	×
108	if path.is_file() and not path.name.startswith('.'):	×
109	results.append(path.relative_to(source_dir))	×
110
111	return results	×
112
113	def is_special(pattern):	×
114	return '*' in pattern or '?' in pattern or '[' in pattern	×
115
116	non_special_excludes = set()	×
117	match_excludes = set()	×
118	for pattern in exclude:	×
119	if is_special(pattern):	×
120	match_excludes.add(pattern)	×
121	else:
122	non_special_excludes.add(pattern)	×
123
124	def exclusion(path):	×
125	if str(path) in non_special_excludes:	×
126	return True	×
127	for pattern in match_excludes:	×
128	ret = path.match(pattern)	×
129	if ret:	×
130	return True	×
131	return False	×
132
133	result = sorted({str(p) for p in includes_iterator() if not exclusion(p)})	×
134	if not result and not allow_empty:	×
135	args = repr(include)	×
136	if exclude:	×
137	args += ', exclude=%s' % repr(exclude)	×
138	output('%s %s: "glob(%s)" got an empty result. If it is the expected behavior, '	×
139	'specify "allow_empty=True" to eliminate this message' % (source_loc, severity, args),
140	prefix=False)
141
142	return result	×
143
144
145	# Each include in a BUILD file can only affect itself
146	__current_globles = None	2✔
147
148
149	# Include a defination file in a BUILD file
150	def include(name):	2✔
151	from blade import build_manager # pylint: disable=import-outside-toplevel	×
152	if name.startswith('//'):	×
153	dir = build_manager.instance.get_root_dir()	×
154	name = name[2:]	×
155	else:
156	dir = build_manager.instance.get_current_source_path()	×
157	exec_file(os.path.join(dir, name), __current_globles, None)	×
158
159
160	build_rules.register_function(enable_if)	2✔
161	build_rules.register_function(glob)	2✔
162	build_rules.register_function(include)	2✔
163
164
165	def _load_build_file(source_dir, processed_source_dirs, blade):	2✔
166	"""Load the BUILD and place the targets into database.
167
168	Invoked by _load_targets. Load and execute the BUILD
169	file, which is a Python script, in source_dir. Statements in BUILD
170	depends on global variable current_source_dir, and will register build
171	target/rules into global variables target_database. Report error
172	and exit if path/BUILD does NOT exist.
173	The parameters processed_source_dirs refers to a set defined in the
174	caller and used to avoid duplicated execution of BUILD files.
175
176	"""
177	source_dir = os.path.normpath(source_dir)	2✔
178	# TODO(yiwang): the character '#' is a magic value.
179	if source_dir in processed_source_dirs or source_dir == '#':	2✔
180	return	2✔
181	processed_source_dirs.add(source_dir)	2✔
182
183	if not os.path.exists(source_dir):	2✔
184	_report_not_exist('Directory', source_dir, source_dir, blade)	×
185	return	×
186
187	old_current_source_path = blade.get_current_source_path()	2✔
188	try:	2✔
189	blade.set_current_source_path(source_dir)	2✔
190	build_file = os.path.join(source_dir, 'BUILD')	2✔
191	if os.path.exists(build_file) and not os.path.isdir(build_file):	2✔
192	try:	2✔
193	# The magic here is that a BUILD file is a Python script,
194	# which can be loaded and executed by execfile().
195	global __current_globles
196	__current_globles = build_rules.get_all()	2✔
197	exec_file(build_file, __current_globles, None)	2✔
198	except SystemExit:	×
199	console.fatal('%s: Fatal error' % build_file)	×
200	except: # pylint: disable=bare-except	×
201	console.fatal('Parse error in %s\n%s' % (	×
202	build_file, traceback.format_exc()))
203	else:
204	_report_not_exist('File', build_file, source_dir, blade)	×
205	finally:
206	blade.set_current_source_path(old_current_source_path)	2✔
207
208
209	def _find_dependent(dkey, blade):	2✔
210	"""Find which target depends on the target with dkey. """
211	target_database = blade.get_target_database()	×
212	for key in target_database:	×
213	target = target_database[key]	×
214	if dkey in target.expanded_deps:	×
215	return target	×
216	return None	×
217
218
219	# File names should be skipped
220	_SKIP_FILES = ['BLADE_ROOT', '.bladeskip']	2✔
221
222
223	def _is_load_excluded(root, d):	2✔
224	"""Whether exclude the directory when loading BUILD.
225	"""
226	# TODO(wentingli): Exclude directories matching patterns configured globally
227
228	# Exclude directories starting with '.', e.g. '.', '..', '.svn', '.git'.
229	if d.startswith('.'):	2✔
230	return True	×
231
232	# Exclude build dirs
233	for build_dir in ('build32_debug', 'build32_release',	2✔
234	'build64_debug', 'build64_release'):
235	if d.startswith(build_dir):	2✔
236	return True	×
237
238	# Exclude directories containing special files
239	for skip_file in _SKIP_FILES:	2✔
240	if os.path.exists(os.path.join(root, d, skip_file)):	2✔
241	console.info('Skip "%s" due to "%s" file' % (os.path.join(root, d), skip_file))	×
242	return True	×
243
244	return False	2✔
245
246
247	def load_targets(target_ids, blade_root_dir, blade):	2✔
248	"""load_targets.
249
250	Parse and load targets, including those specified in command line
251	and their direct and indirect dependencies, by loading related BUILD
252	files. Returns a map which contains all these targets.
253
254	"""
255	_load_build_rules()	2✔
256
257	# pylint: disable=too-many-locals
258	build_rules.register_variable('build_target', build_attributes.attributes)	2✔
259	target_database = blade.get_target_database()	2✔
260	# targets specified in command line
261	cited_targets = set()	2✔
262	# cited_targets and all its dependencies
263	related_targets = {}	2✔
264	# source dirs mentioned in command line
265	source_dirs = []	2✔
266	# to prevent duplicated loading of BUILD files
267	processed_source_dirs = set()	2✔
268
269	direct_targets = []	2✔
270	all_command_targets = []	2✔
271	# Parse command line target_ids. For those in the form of <path>:<target>,
272	# record (<path>,<target>) in cited_targets; for the rest (with <path>
273	# but without <target>), record <path> into paths.
274	for target_id in target_ids:	2✔
275	source_dir, target_name = target_id.rsplit(':', 1)	2✔
276	if not os.path.exists(source_dir):	2✔
277	_report_not_exist('Directory', source_dir, source_dir, blade)	×
278
279	if target_name not in ('*', '...'):	2✔
280	cited_targets.add(source_dir + ':' + target_name)	×
281	elif target_name == '...':	2✔
282	for root, dirs, files in os.walk(source_dir):	2✔
283	# Note the dirs[:] = slice assignment; we are replacing the
284	# elements in dirs (and not the list referred to by dirs) so
285	# that os.walk() will not process deleted directories.
286	dirs[:] = [d for d in dirs if not _is_load_excluded(root, d)]	2✔
287	if 'BUILD' in files:	2✔
288	source_dirs.append(root)	2✔
289	else:
290	source_dirs.append(source_dir)	×
291
292	direct_targets = list(cited_targets)	2✔
293
294	# Load BUILD files in paths, and add all loaded targets into
295	# cited_targets. Together with above step, we can ensure that all
296	# targets mentioned in the command line are now in cited_targets.
297	for source_dir in source_dirs:	2✔
298	_load_build_file(source_dir,	2✔
299	processed_source_dirs,
300	blade)
301
302	for key in target_database:	2✔
303	cited_targets.add(key)	2✔
304	all_command_targets = list(cited_targets)	2✔
305
306	# Starting from targets specified in command line, breath-first
307	# propagate to load BUILD files containing directly and indirectly
308	# dependent targets. All these targets form related_targets,
309	# which is a subset of target_databased created by loading BUILD files.
310	while cited_targets:	2✔
311	target_id = cited_targets.pop()	2✔
312	source_dir, target_name = target_id.split(':')	2✔
313	if target_id in related_targets:	2✔
314	continue	×
315
316	_load_build_file(source_dir,	2✔
317	processed_source_dirs,
318	blade)
319
320	if target_id not in target_database:	2✔
321	msg = 'Target "//%s" does not exist' % target_id	×
322	dependent = _find_dependent(target_id, blade)	×
323	(dependent or console).error(msg)	×
324	continue	×
325
326	related_targets[target_id] = target_database[target_id]	2✔
327	for key in related_targets[target_id].expanded_deps:	2✔
328	if key not in related_targets:	2✔
329	cited_targets.add(key)	1✔
330
331	# Iterating to get svn root dirs
332	for target_id in related_targets: # pylint: disable=dict-iter-missing-items	2✔
333	path, name = target_id.rsplit(':')	2✔
334	root_dir = path.split('/')[0].strip()	2✔
335	if root_dir not in blade.svn_root_dirs and '#' not in root_dir:	2✔
336	blade.svn_root_dirs.append(root_dir)	2✔
337
338	return direct_targets, all_command_targets, related_targets	2✔

chen3feng / blade-build / 862

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous