5994

pending completion

Build # 5994

Build Type

push

travis-ci

Committed by

jsirois

Commit Message

Revert "fix misc ws issues"

This reverts commit 3f3a41c25.

Testing Done:
I ran a git bisect session using this as the test case:
```console
./pants test \
  tests/python/pants_test/backend/jvm/tasks/jvm_compile/java:java_compile_jvm_platform_integration \
  tests/python/pants_test/backend/jvm/tasks/jvm_compile/java:java_zinc_compile_jvm_platform_integration \
    -- -vk test_compile_target_coercion
```

CI went green here:
  https://travis-ci.org/pantsbuild/pants/builds/74161665

Bugs closed: 1924, 1925

Reviewed at https://rbcommons.com/s/twitter/r/2573/

Run Details

66 of 66 new or added lines in 37 files covered. (100.0%)

15252 of 20825 relevant lines covered (73.24%)

1.38 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

20.0

/src/python/pants/backend/jvm/tasks/jvm_compile/jvm_compile_isolated_strategy.py

# coding=utf-8
# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
                        unicode_literals, with_statement)

import os
import shutil
from collections import OrderedDict, defaultdict

from pants.backend.jvm.tasks.classpath_util import ClasspathUtil
from pants.backend.jvm.tasks.jvm_compile.execution_graph import (ExecutionFailure, ExecutionGraph,
                                                                 Job)
from pants.backend.jvm.tasks.jvm_compile.jvm_compile_strategy import JvmCompileStrategy
from pants.backend.jvm.tasks.jvm_compile.resource_mapping import ResourceMapping
from pants.base.build_environment import get_buildroot
from pants.base.exceptions import TaskError
from pants.base.worker_pool import Work, WorkerPool
from pants.util.dirutil import safe_mkdir, safe_walk
from pants.util.fileutil import atomic_copy


class JvmCompileIsolatedStrategy(JvmCompileStrategy):
  """A strategy for JVM compilation that uses per-target classpaths and analysis."""

  @classmethod
  def register_options(cls, register, language, supports_concurrent_execution):
    if supports_concurrent_execution:
      register('--worker-count', type=int, default=1, advanced=True,
               help='The number of concurrent workers to use compiling {lang} sources with the isolated'
                    ' strategy.'.format(lang=language))
    register('--capture-log', action='store_true', default=False, advanced=True,
            help='Capture compilation output to per-target logs.')

  def __init__(self, context, options, workdir, analysis_tools, language, sources_predicate):
    super(JvmCompileIsolatedStrategy, self).__init__(context, options, workdir, analysis_tools,
                                                     language, sources_predicate)

    # Various working directories.
    self._analysis_dir = os.path.join(workdir, 'isolated-analysis')
    self._classes_dir = os.path.join(workdir, 'isolated-classes')
    self._logs_dir = os.path.join(workdir, 'isolated-logs')

    self._capture_log = options.capture_log

    try:
      worker_count = options.worker_count
    except AttributeError:
      # tasks that don't support concurrent execution have no worker_count registered
      worker_count = 1
    self._worker_count = worker_count

    self._worker_pool = None

  def name(self):
    return 'isolated'

  def compile_context(self, target):
    analysis_file = JvmCompileStrategy._analysis_for_target(self._analysis_dir, target)
    classes_dir = os.path.join(self._classes_dir, target.id)
    return self.CompileContext(target,
                               analysis_file,
                               classes_dir,
                               self._sources_for_target(target))

  def _create_compile_contexts_for_targets(self, targets):
    compile_contexts = OrderedDict()
    for target in targets:
      compile_context = self.compile_context(target)
      compile_contexts[target] = compile_context
    return compile_contexts

  def pre_compile(self):
    super(JvmCompileIsolatedStrategy, self).pre_compile()
    safe_mkdir(self._analysis_dir)
    safe_mkdir(self._classes_dir)
    safe_mkdir(self._logs_dir)

  def prepare_compile(self, cache_manager, all_targets, relevant_targets):
    super(JvmCompileIsolatedStrategy, self).prepare_compile(cache_manager, all_targets,
                                                            relevant_targets)

    # Update the classpath by adding relevant target's classes directories to its classpath.
    compile_classpaths = self.context.products.get_data('compile_classpath')

    with self.context.new_workunit('validate-{}-analysis'.format(self._language)):
      for target in relevant_targets:
        cc = self.compile_context(target)
        safe_mkdir(cc.classes_dir)
        compile_classpaths.add_for_target(target, [(conf, cc.classes_dir) for conf in self._confs])
        self.validate_analysis(cc.analysis_file)

    # This ensures the workunit for the worker pool is set
    with self.context.new_workunit('isolation-{}-pool-bootstrap'.format(self._language)) \
            as workunit:
      # This uses workunit.parent as the WorkerPool's parent so that child workunits
      # of different pools will show up in order in the html output. This way the current running
      # workunit is on the bottom of the page rather than possibly in the middle.
      self._worker_pool = WorkerPool(workunit.parent,
                                     self.context.run_tracker,
                                     self._worker_count)

  def invalidation_hints(self, relevant_targets):
    # No partitioning.
    return (0, None)

  def compute_classes_by_source(self, compile_contexts):
    buildroot = get_buildroot()
    # Build a mapping of srcs to classes for each context.
    classes_by_src_by_context = defaultdict(dict)
    for compile_context in compile_contexts:
      # Walk the class directory to build a set of unclaimed classfiles.
      unclaimed_classes = set()
      for dirpath, _, filenames in safe_walk(compile_context.classes_dir):
        unclaimed_classes.update(os.path.join(dirpath, f) for f in filenames)

      # Grab the analysis' view of which classfiles were generated.
      classes_by_src = classes_by_src_by_context[compile_context]
      if os.path.exists(compile_context.analysis_file):
        products = self._analysis_parser.parse_products_from_path(compile_context.analysis_file,
                                                                  compile_context.classes_dir)
        for src, classes in products.items():
          relsrc = os.path.relpath(src, buildroot)
          classes_by_src[relsrc] = classes
          unclaimed_classes.difference_update(classes)

      # Any remaining classfiles were unclaimed by sources/analysis.
      classes_by_src[None] = list(unclaimed_classes)
    return classes_by_src_by_context

  def _compute_classpath_entries(self, compile_classpaths,
                                 target_closure,
                                 compile_context,
                                 extra_compile_time_classpath):
    # Generate a classpath specific to this compile and target.
    return ClasspathUtil.compute_classpath_for_target(compile_context.target, compile_classpaths,
                                                      extra_compile_time_classpath, self._confs,
                                                      target_closure)

  def _upstream_analysis(self, compile_contexts, classpath_entries):
    """Returns tuples of classes_dir->analysis_file for the closure of the target."""
    # Reorganize the compile_contexts by class directory.
    compile_contexts_by_directory = {}
    for compile_context in compile_contexts.values():
      compile_contexts_by_directory[compile_context.classes_dir] = compile_context
    # If we have a compile context for the target, include it.
    for entry in classpath_entries:
      if not entry.endswith('.jar'):
        compile_context = compile_contexts_by_directory.get(entry)
        if not compile_context:
          self.context.log.debug('Missing upstream analysis for {}'.format(entry))
        else:
          yield compile_context.classes_dir, compile_context.analysis_file

  def _capture_log_file(self, target):
    if self._capture_log:
      return os.path.join(self._logs_dir, "{}.log".format(target.id))
    return None

  def exec_graph_key_for_target(self, compile_target):
    return "compile({})".format(compile_target.address.spec)

  def _create_compile_jobs(self, compile_classpaths, compile_contexts, extra_compile_time_classpath,
                           invalid_targets, invalid_vts_partitioned, compile_vts, register_vts,
                           update_artifact_cache_vts_work):
    def create_work_for_vts(vts, compile_context, target_closure):
      def work():
        progress_message = compile_context.target.address.spec
        cp_entries = self._compute_classpath_entries(compile_classpaths,
                                                     target_closure,
                                                     compile_context,
                                                     extra_compile_time_classpath)

        upstream_analysis = dict(self._upstream_analysis(compile_contexts, cp_entries))

        # Capture a compilation log if requested.
        log_file = self._capture_log_file(compile_context.target)

        # Mutate analysis within a temporary directory, and move it to the final location
        # on success.
        tmpdir = os.path.join(self.analysis_tmpdir, compile_context.target.id)
        safe_mkdir(tmpdir)
        tmp_analysis_file = JvmCompileStrategy._analysis_for_target(
            tmpdir, compile_context.target)
        if os.path.exists(compile_context.analysis_file):
           shutil.copy(compile_context.analysis_file, tmp_analysis_file)
        target, = vts.targets
        compile_vts(vts,
                    compile_context.sources,
                    tmp_analysis_file,
                    upstream_analysis,
                    cp_entries,
                    compile_context.classes_dir,
                    log_file,
                    progress_message,
                    target.platform)
        atomic_copy(tmp_analysis_file, compile_context.analysis_file)

        # Update the products with the latest classes.
        register_vts([compile_context])

        # Kick off the background artifact cache write.
        if update_artifact_cache_vts_work:
          self._write_to_artifact_cache(vts, compile_context, update_artifact_cache_vts_work)

      return work

    jobs = []
    invalid_target_set = set(invalid_targets)
    for vts in invalid_vts_partitioned:
      assert len(vts.targets) == 1, ("Requested one target per partition, got {}".format(vts))

      # Invalidated targets are a subset of relevant targets: get the context for this one.
      compile_target = vts.targets[0]
      compile_context = compile_contexts[compile_target]
      compile_target_closure = compile_target.closure()

      # dependencies of the current target which are invalid for this chunk
      invalid_dependencies = (compile_target_closure & invalid_target_set) - [compile_target]

      jobs.append(Job(self.exec_graph_key_for_target(compile_target),
                      create_work_for_vts(vts, compile_context, compile_target_closure),
                      [self.exec_graph_key_for_target(target) for target in invalid_dependencies],
                      # If compilation and analysis work succeeds, validate the vts.
                      # Otherwise, fail it.
                      on_success=vts.update,
                      on_failure=vts.force_invalidate))
    return jobs

  def compile_chunk(self,
                    invalidation_check,
                    all_targets,
                    relevant_targets,
                    invalid_targets,
                    extra_compile_time_classpath_elements,
                    compile_vts,
                    register_vts,
                    update_artifact_cache_vts_work):
    """Executes compilations for the invalid targets contained in a single chunk."""
    assert invalid_targets, "compile_chunk should only be invoked if there are invalid targets."
    # Get the classpath generated by upstream JVM tasks and our own prepare_compile().
    compile_classpaths = self.context.products.get_data('compile_classpath')

    extra_compile_time_classpath = self._compute_extra_classpath(
        extra_compile_time_classpath_elements)

    compile_contexts = self._create_compile_contexts_for_targets(all_targets)

    # Now create compile jobs for each invalid target one by one.
    jobs = self._create_compile_jobs(compile_classpaths,
                                     compile_contexts,
                                     extra_compile_time_classpath,
                                     invalid_targets,
                                     invalidation_check.invalid_vts_partitioned,
                                     compile_vts,
                                     register_vts,
                                     update_artifact_cache_vts_work)

    exec_graph = ExecutionGraph(jobs)
    try:
      exec_graph.execute(self._worker_pool, self.context.log)
    except ExecutionFailure as e:
      raise TaskError("Compilation failure: {}".format(e))

  def compute_resource_mapping(self, compile_contexts):
    return ResourceMapping(self._classes_dir)

  def post_process_cached_vts(self, cached_vts):
    """Localizes the fetched analysis for targets we found in the cache.

    This is the complement of `_write_to_artifact_cache`.
    """
    compile_contexts = []
    for vt in cached_vts:
      for target in vt.targets:
        compile_contexts.append(self.compile_context(target))

    for compile_context in compile_contexts:
      portable_analysis_file = JvmCompileStrategy._portable_analysis_for_target(
          self._analysis_dir, compile_context.target)
      if os.path.exists(portable_analysis_file):
        self._analysis_tools.localize(portable_analysis_file, compile_context.analysis_file)

  def _write_to_artifact_cache(self, vts, compile_context, get_update_artifact_cache_work):
    assert len(vts.targets) == 1
    assert vts.targets[0] == compile_context.target

    # Noop if the target is uncacheable.
    if (compile_context.target.has_label('no_cache')):
      return
    vt = vts.versioned_targets[0]

    # Set up args to relativize analysis in the background.
    portable_analysis_file = JvmCompileStrategy._portable_analysis_for_target(
        self._analysis_dir, compile_context.target)
    relativize_args_tuple = (compile_context.analysis_file, portable_analysis_file)

    # Collect the artifacts for this target.
    artifacts = []
    def add_abs_products(p):
      if p:
        for _, paths in p.abs_paths():
          artifacts.extend(paths)
    # Resources.
    resources_by_target = self.context.products.get_data('resources_by_target')
    add_abs_products(resources_by_target.get(compile_context.target))
    # Classes.
    classes_by_target = self.context.products.get_data('classes_by_target')
    add_abs_products(classes_by_target.get(compile_context.target))
    # Log file.
    log_file = self._capture_log_file(compile_context.target)
    if log_file and os.path.exists(log_file):
      artifacts.append(log_file)

    # Get the 'work' that will publish these artifacts to the cache.
    # NB: the portable analysis_file won't exist until we finish.
    vts_artifactfiles_pair = (vt, artifacts + [portable_analysis_file])
    update_artifact_cache_work = get_update_artifact_cache_work([vts_artifactfiles_pair])

    # And execute it.
    if update_artifact_cache_work:
      work_chain = [
          Work(self._analysis_tools.relativize, [relativize_args_tuple], 'relativize'),
          update_artifact_cache_work
      ]
      self.context.submit_background_work_chain(work_chain, parent_workunit_name='cache')

1	# coding=utf-8
2	# Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
3	# Licensed under the Apache License, Version 2.0 (see LICENSE).
4
5	from __future__ import (absolute_import, division, generators, nested_scopes, print_function,	2✔
6	unicode_literals, with_statement)
7
8	import os	2✔
9	import shutil	2✔
10	from collections import OrderedDict, defaultdict	2✔
11
12	from pants.backend.jvm.tasks.classpath_util import ClasspathUtil	2✔
13	from pants.backend.jvm.tasks.jvm_compile.execution_graph import (ExecutionFailure, ExecutionGraph,	2✔
14	Job)
15	from pants.backend.jvm.tasks.jvm_compile.jvm_compile_strategy import JvmCompileStrategy	2✔
16	from pants.backend.jvm.tasks.jvm_compile.resource_mapping import ResourceMapping	2✔
17	from pants.base.build_environment import get_buildroot	2✔
18	from pants.base.exceptions import TaskError	2✔
19	from pants.base.worker_pool import Work, WorkerPool	2✔
20	from pants.util.dirutil import safe_mkdir, safe_walk	2✔
21	from pants.util.fileutil import atomic_copy	2✔
22
23
24	class JvmCompileIsolatedStrategy(JvmCompileStrategy):	2✔
25	"""A strategy for JVM compilation that uses per-target classpaths and analysis."""
26
27	@classmethod	2✔
28	def register_options(cls, register, language, supports_concurrent_execution):
29	if supports_concurrent_execution:	2✔
30	register('--worker-count', type=int, default=1, advanced=True,	×
31	help='The number of concurrent workers to use compiling {lang} sources with the isolated'
32	' strategy.'.format(lang=language))
33	register('--capture-log', action='store_true', default=False, advanced=True,	2✔
34	help='Capture compilation output to per-target logs.')
35
36	def __init__(self, context, options, workdir, analysis_tools, language, sources_predicate):	2✔
37	super(JvmCompileIsolatedStrategy, self).__init__(context, options, workdir, analysis_tools,	×
38	language, sources_predicate)
39
40	# Various working directories.
41	self._analysis_dir = os.path.join(workdir, 'isolated-analysis')	×
42	self._classes_dir = os.path.join(workdir, 'isolated-classes')	×
43	self._logs_dir = os.path.join(workdir, 'isolated-logs')	×
44
45	self._capture_log = options.capture_log	×
46
47	try:	×
48	worker_count = options.worker_count	×
49	except AttributeError:	×
50	# tasks that don't support concurrent execution have no worker_count registered
51	worker_count = 1	×
52	self._worker_count = worker_count	×
53
54	self._worker_pool = None	×
55
56	def name(self):	2✔
57	return 'isolated'	×
58
59	def compile_context(self, target):	2✔
60	analysis_file = JvmCompileStrategy._analysis_for_target(self._analysis_dir, target)	×
61	classes_dir = os.path.join(self._classes_dir, target.id)	×
62	return self.CompileContext(target,	×
63	analysis_file,
64	classes_dir,
65	self._sources_for_target(target))
66
67	def _create_compile_contexts_for_targets(self, targets):	2✔
68	compile_contexts = OrderedDict()	×
69	for target in targets:	×
70	compile_context = self.compile_context(target)	×
71	compile_contexts[target] = compile_context	×
72	return compile_contexts	×
73
74	def pre_compile(self):	2✔
75	super(JvmCompileIsolatedStrategy, self).pre_compile()	×
76	safe_mkdir(self._analysis_dir)	×
77	safe_mkdir(self._classes_dir)	×
78	safe_mkdir(self._logs_dir)	×
79
80	def prepare_compile(self, cache_manager, all_targets, relevant_targets):	2✔
81	super(JvmCompileIsolatedStrategy, self).prepare_compile(cache_manager, all_targets,	×
82	relevant_targets)
83
84	# Update the classpath by adding relevant target's classes directories to its classpath.
85	compile_classpaths = self.context.products.get_data('compile_classpath')	×
86
87	with self.context.new_workunit('validate-{}-analysis'.format(self._language)):	×
88	for target in relevant_targets:	×
89	cc = self.compile_context(target)	×
90	safe_mkdir(cc.classes_dir)	×
91	compile_classpaths.add_for_target(target, [(conf, cc.classes_dir) for conf in self._confs])	×
92	self.validate_analysis(cc.analysis_file)	×
93
94	# This ensures the workunit for the worker pool is set
95	with self.context.new_workunit('isolation-{}-pool-bootstrap'.format(self._language)) \	×
96	as workunit:
97	# This uses workunit.parent as the WorkerPool's parent so that child workunits
98	# of different pools will show up in order in the html output. This way the current running
99	# workunit is on the bottom of the page rather than possibly in the middle.
100	self._worker_pool = WorkerPool(workunit.parent,	×
101	self.context.run_tracker,
102	self._worker_count)
103
104	def invalidation_hints(self, relevant_targets):	2✔
105	# No partitioning.
106	return (0, None)	×
107
108	def compute_classes_by_source(self, compile_contexts):	2✔
109	buildroot = get_buildroot()	×
110	# Build a mapping of srcs to classes for each context.
111	classes_by_src_by_context = defaultdict(dict)	×
112	for compile_context in compile_contexts:	×
113	# Walk the class directory to build a set of unclaimed classfiles.
114	unclaimed_classes = set()	×
115	for dirpath, _, filenames in safe_walk(compile_context.classes_dir):	×
116	unclaimed_classes.update(os.path.join(dirpath, f) for f in filenames)	×
117
118	# Grab the analysis' view of which classfiles were generated.
119	classes_by_src = classes_by_src_by_context[compile_context]	×
120	if os.path.exists(compile_context.analysis_file):	×
121	products = self._analysis_parser.parse_products_from_path(compile_context.analysis_file,	×
122	compile_context.classes_dir)
123	for src, classes in products.items():	×
124	relsrc = os.path.relpath(src, buildroot)	×
125	classes_by_src[relsrc] = classes	×
126	unclaimed_classes.difference_update(classes)	×
127
128	# Any remaining classfiles were unclaimed by sources/analysis.
129	classes_by_src[None] = list(unclaimed_classes)	×
130	return classes_by_src_by_context	×
131
132	def _compute_classpath_entries(self, compile_classpaths,	2✔
133	target_closure,
134	compile_context,
135	extra_compile_time_classpath):
136	# Generate a classpath specific to this compile and target.
137	return ClasspathUtil.compute_classpath_for_target(compile_context.target, compile_classpaths,	×
138	extra_compile_time_classpath, self._confs,
139	target_closure)
140
141	def _upstream_analysis(self, compile_contexts, classpath_entries):	2✔
142	"""Returns tuples of classes_dir->analysis_file for the closure of the target."""
143	# Reorganize the compile_contexts by class directory.
144	compile_contexts_by_directory = {}	×
145	for compile_context in compile_contexts.values():	×
146	compile_contexts_by_directory[compile_context.classes_dir] = compile_context	×
147	# If we have a compile context for the target, include it.
148	for entry in classpath_entries:	×
149	if not entry.endswith('.jar'):	×
150	compile_context = compile_contexts_by_directory.get(entry)	×
151	if not compile_context:	×
152	self.context.log.debug('Missing upstream analysis for {}'.format(entry))	×
153	else:
154	yield compile_context.classes_dir, compile_context.analysis_file	×
155
156	def _capture_log_file(self, target):	2✔
157	if self._capture_log:	×
158	return os.path.join(self._logs_dir, "{}.log".format(target.id))	×
159	return None	×
160
161	def exec_graph_key_for_target(self, compile_target):	2✔
162	return "compile({})".format(compile_target.address.spec)	×
163
164	def _create_compile_jobs(self, compile_classpaths, compile_contexts, extra_compile_time_classpath,	2✔
165	invalid_targets, invalid_vts_partitioned, compile_vts, register_vts,
166	update_artifact_cache_vts_work):
167	def create_work_for_vts(vts, compile_context, target_closure):	×
168	def work():	×
169	progress_message = compile_context.target.address.spec	×
170	cp_entries = self._compute_classpath_entries(compile_classpaths,	×
171	target_closure,
172	compile_context,
173	extra_compile_time_classpath)
174
175	upstream_analysis = dict(self._upstream_analysis(compile_contexts, cp_entries))	×
176
177	# Capture a compilation log if requested.
178	log_file = self._capture_log_file(compile_context.target)	×
179
180	# Mutate analysis within a temporary directory, and move it to the final location
181	# on success.
182	tmpdir = os.path.join(self.analysis_tmpdir, compile_context.target.id)	×
183	safe_mkdir(tmpdir)	×
184	tmp_analysis_file = JvmCompileStrategy._analysis_for_target(	×
185	tmpdir, compile_context.target)
186	if os.path.exists(compile_context.analysis_file):	×
187	shutil.copy(compile_context.analysis_file, tmp_analysis_file)	×
188	target, = vts.targets	×
189	compile_vts(vts,	×
190	compile_context.sources,
191	tmp_analysis_file,
192	upstream_analysis,
193	cp_entries,
194	compile_context.classes_dir,
195	log_file,
196	progress_message,
197	target.platform)
198	atomic_copy(tmp_analysis_file, compile_context.analysis_file)	×
199
200	# Update the products with the latest classes.
201	register_vts([compile_context])	×
202
203	# Kick off the background artifact cache write.
204	if update_artifact_cache_vts_work:	×
205	self._write_to_artifact_cache(vts, compile_context, update_artifact_cache_vts_work)	×
206
207	return work	×
208
209	jobs = []	×
210	invalid_target_set = set(invalid_targets)	×
211	for vts in invalid_vts_partitioned:	×
212	assert len(vts.targets) == 1, ("Requested one target per partition, got {}".format(vts))	×
213
214	# Invalidated targets are a subset of relevant targets: get the context for this one.
215	compile_target = vts.targets[0]	×
216	compile_context = compile_contexts[compile_target]	×
217	compile_target_closure = compile_target.closure()	×
218
219	# dependencies of the current target which are invalid for this chunk
220	invalid_dependencies = (compile_target_closure & invalid_target_set) - [compile_target]	×
221
222	jobs.append(Job(self.exec_graph_key_for_target(compile_target),	×
223	create_work_for_vts(vts, compile_context, compile_target_closure),
224	[self.exec_graph_key_for_target(target) for target in invalid_dependencies],
225	# If compilation and analysis work succeeds, validate the vts.
226	# Otherwise, fail it.
227	on_success=vts.update,
228	on_failure=vts.force_invalidate))
229	return jobs	×
230
231	def compile_chunk(self,	2✔
232	invalidation_check,
233	all_targets,
234	relevant_targets,
235	invalid_targets,
236	extra_compile_time_classpath_elements,
237	compile_vts,
238	register_vts,
239	update_artifact_cache_vts_work):
240	"""Executes compilations for the invalid targets contained in a single chunk."""
241	assert invalid_targets, "compile_chunk should only be invoked if there are invalid targets."	×
242	# Get the classpath generated by upstream JVM tasks and our own prepare_compile().
243	compile_classpaths = self.context.products.get_data('compile_classpath')	×
244
245	extra_compile_time_classpath = self._compute_extra_classpath(	×
246	extra_compile_time_classpath_elements)
247
248	compile_contexts = self._create_compile_contexts_for_targets(all_targets)	×
249
250	# Now create compile jobs for each invalid target one by one.
251	jobs = self._create_compile_jobs(compile_classpaths,	×
252	compile_contexts,
253	extra_compile_time_classpath,
254	invalid_targets,
255	invalidation_check.invalid_vts_partitioned,
256	compile_vts,
257	register_vts,
258	update_artifact_cache_vts_work)
259
260	exec_graph = ExecutionGraph(jobs)	×
261	try:	×
262	exec_graph.execute(self._worker_pool, self.context.log)	×
263	except ExecutionFailure as e:	×
264	raise TaskError("Compilation failure: {}".format(e))	×
265
266	def compute_resource_mapping(self, compile_contexts):	2✔
267	return ResourceMapping(self._classes_dir)	×
268
269	def post_process_cached_vts(self, cached_vts):	2✔
270	"""Localizes the fetched analysis for targets we found in the cache.
271
272	This is the complement of `_write_to_artifact_cache`.
273	"""
274	compile_contexts = []	×
275	for vt in cached_vts:	×
276	for target in vt.targets:	×
277	compile_contexts.append(self.compile_context(target))	×
278
279	for compile_context in compile_contexts:	×
280	portable_analysis_file = JvmCompileStrategy._portable_analysis_for_target(	×
281	self._analysis_dir, compile_context.target)
282	if os.path.exists(portable_analysis_file):	×
283	self._analysis_tools.localize(portable_analysis_file, compile_context.analysis_file)	×
284
285	def _write_to_artifact_cache(self, vts, compile_context, get_update_artifact_cache_work):	2✔
286	assert len(vts.targets) == 1	×
287	assert vts.targets[0] == compile_context.target	×
288
289	# Noop if the target is uncacheable.
290	if (compile_context.target.has_label('no_cache')):	×
291	return	×
292	vt = vts.versioned_targets[0]	×
293
294	# Set up args to relativize analysis in the background.
295	portable_analysis_file = JvmCompileStrategy._portable_analysis_for_target(	×
296	self._analysis_dir, compile_context.target)
297	relativize_args_tuple = (compile_context.analysis_file, portable_analysis_file)	×
298
299	# Collect the artifacts for this target.
300	artifacts = []	×
301	def add_abs_products(p):	×
302	if p:	×
303	for _, paths in p.abs_paths():	×
304	artifacts.extend(paths)	×
305	# Resources.
306	resources_by_target = self.context.products.get_data('resources_by_target')	×
307	add_abs_products(resources_by_target.get(compile_context.target))	×
308	# Classes.
309	classes_by_target = self.context.products.get_data('classes_by_target')	×
310	add_abs_products(classes_by_target.get(compile_context.target))	×
311	# Log file.
312	log_file = self._capture_log_file(compile_context.target)	×
313	if log_file and os.path.exists(log_file):	×
314	artifacts.append(log_file)	×
315
316	# Get the 'work' that will publish these artifacts to the cache.
317	# NB: the portable analysis_file won't exist until we finish.
318	vts_artifactfiles_pair = (vt, artifacts + [portable_analysis_file])	×
319	update_artifact_cache_work = get_update_artifact_cache_work([vts_artifactfiles_pair])	×
320
321	# And execute it.
322	if update_artifact_cache_work:	×
323	work_chain = [	×
324	Work(self._analysis_tools.relativize, [relativize_args_tuple], 'relativize'),
325	update_artifact_cache_work
326	]
327	self.context.submit_background_work_chain(work_chain, parent_workunit_name='cache')	×

pantsbuild / pants / 5994

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous