22740642519

Committed 05 Mar 2026 11:00PM UTC coverage: 52.677% (-40.3%) from 92.931%

Build # 22740642519

Build Type

Pull #23157

github

Committed by

web-flow

Commit Message

Merge 2aa18e6d4 into f0030f5e7

Pull Request Pull Request #23157: [pants ng] Partition source files by config.

Run Details

31678 of 60136 relevant lines covered (52.68%)

0.53 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

75.42

/src/python/pants/backend/go/util_rules/coverage.py

# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).
from __future__ import annotations

import enum
import hashlib
import os
from dataclasses import dataclass
from pathlib import PurePath

import chevron

from pants.backend.go.util_rules.sdk import GoSdkProcess, GoSdkToolIDRequest, compute_go_tool_id
from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior
from pants.build_graph.address import Address
from pants.core.goals.test import CoverageData
from pants.engine.fs import CreateDigest, DigestSubset, FileContent, PathGlobs
from pants.engine.internals.native_engine import Digest, MergeDigests
from pants.engine.internals.selectors import concurrently
from pants.engine.intrinsics import create_digest, digest_subset_to_digest, merge_digests
from pants.engine.process import fallible_to_exec_result_or_raise
from pants.engine.rules import collect_rules, implicitly, rule
from pants.util.ordered_set import FrozenOrderedSet


@dataclass(frozen=True)
class GoCoverageData(CoverageData):
    coverage_digest: Digest
    import_path: str
    sources_digest: Digest
    sources_dir_path: str
    pkg_target_address: Address


class GoCoverMode(enum.Enum):
    SET = "set"
    COUNT = "count"
    ATOMIC = "atomic"


@dataclass(frozen=True)
class GoCoverageConfig:
    # How to count the code usage.
    cover_mode: GoCoverMode

    # Import path patterns for packages which should be instrumented for code coverage.
    import_path_include_patterns: tuple[str, ...] = ()


@dataclass(frozen=True)
class ApplyCodeCoverageRequest:
    """Apply code coverage to a package using `go tool cover`."""

    digest: Digest
    dir_path: str
    go_files: tuple[str, ...]
    cgo_files: tuple[str, ...]
    cover_mode: GoCoverMode
    import_path: str


@dataclass(frozen=True)
class FileCodeCoverageMetadata:
    """Metadata for code coverage applied to a single Go file."""

    file_id: str
    go_file: str
    cover_go_file: str
    cover_var: str


@dataclass(frozen=True)
class BuiltGoPackageCodeCoverageMetadata:
    import_path: str
    cover_file_metadatas: tuple[FileCodeCoverageMetadata, ...]
    sources_digest: Digest
    sources_dir_path: str


@dataclass(frozen=True)
class ApplyCodeCoverageResult:
    digest: Digest
    cover_file_metadatas: tuple[FileCodeCoverageMetadata, ...]
    go_files: tuple[str, ...]
    cgo_files: tuple[str, ...]


@dataclass(frozen=True)
class ApplyCodeCoverageToFileRequest:
    digest: Digest
    go_file: str
    cover_go_file: str
    mode: GoCoverMode
    cover_var: str


@dataclass(frozen=True)
class ApplyCodeCoverageToFileResult:
    digest: Digest
    cover_go_file: str


@rule
async def go_apply_code_coverage_to_file(
    request: ApplyCodeCoverageToFileRequest,
) -> ApplyCodeCoverageToFileResult:
    cover_tool_id = await compute_go_tool_id(GoSdkToolIDRequest("cover"))

    result = await fallible_to_exec_result_or_raise(
        **implicitly(
            GoSdkProcess(
                input_digest=request.digest,
                command=[
                    "tool",
                    "cover",
                    "-mode",
                    request.mode.value,
                    "-var",
                    request.cover_var,
                    "-o",
                    request.cover_go_file,
                    request.go_file,
                ],
                description=f"Apply Go coverage to: {request.go_file}",
                output_files=(str(request.cover_go_file),),
                env={"__PANTS_GO_COVER_TOOL_ID": cover_tool_id.tool_id},
            )
        )
    )

    return ApplyCodeCoverageToFileResult(
        digest=result.output_digest,
        cover_go_file=request.cover_go_file,
    )


def _hash_string(s: str) -> str:
    h = hashlib.sha256(s.encode())
    return h.hexdigest()[:12]


def _is_test_file(s: str) -> bool:
    return s.endswith("_test.go")


@rule
async def go_apply_code_coverage(request: ApplyCodeCoverageRequest) -> ApplyCodeCoverageResult:
    # Setup metadata for each file to which code coverage will be applied by assigning the name of the exported
    # variable which holds coverage counters for each file.
    file_metadatas: list[FileCodeCoverageMetadata] = []
    output_go_files = []
    output_cgo_files = []
    import_path_hash = _hash_string(request.import_path)
    for i, go_file in enumerate(request.go_files + request.cgo_files):
        if _is_test_file(go_file):
            if i < len(request.go_files):
                output_go_files.append(go_file)
            else:
                output_cgo_files.append(go_file)
            continue

        p = PurePath(go_file)
        cover_go_file = str(p.with_name(f"{p.stem}.cover.go"))
        file_metadatas.append(
            FileCodeCoverageMetadata(
                file_id=f"{request.import_path}/{go_file}",
                go_file=go_file,
                cover_go_file=cover_go_file,
                cover_var=f"GoCover_{import_path_hash}_{i}",
            )
        )
        if i < len(request.go_files):
            output_go_files.append(cover_go_file)
        else:
            output_cgo_files.append(cover_go_file)

    subsetted_digests = await concurrently(
        digest_subset_to_digest(
            DigestSubset(
                request.digest,
                PathGlobs(
                    [os.path.join(request.dir_path, file_metadata.go_file)],
                    glob_match_error_behavior=GlobMatchErrorBehavior.error,
                    description_of_origin="coverage",
                ),
            )
        )
        for file_metadata in file_metadatas
    )

    # Apply code coverage codegen to each file that will be analyzed.
    cover_results = await concurrently(
        go_apply_code_coverage_to_file(
            ApplyCodeCoverageToFileRequest(
                digest=go_file_digest,
                go_file=os.path.join(request.dir_path, file_metadata.go_file),
                cover_go_file=os.path.join(request.dir_path, file_metadata.cover_go_file),
                mode=request.cover_mode,
                cover_var=file_metadata.cover_var,
            )
        )
        for file_metadata, go_file_digest in zip(file_metadatas, subsetted_digests)
    )

    # Merge the coverage codegen back into the original digest so that non-covered and covered sources are in
    # the same digest.
    digest = await merge_digests(MergeDigests([request.digest, *(r.digest for r in cover_results)]))

    return ApplyCodeCoverageResult(
        digest=digest,
        cover_file_metadatas=tuple(file_metadatas),
        go_files=tuple(output_go_files),
        cgo_files=tuple(output_cgo_files),
    )


@dataclass(frozen=True)
class GenerateCoverageSetupCodeRequest:
    packages: FrozenOrderedSet[BuiltGoPackageCodeCoverageMetadata]
    cover_mode: GoCoverMode


@dataclass(frozen=True)
class GenerateCoverageSetupCodeResult:
    PATH = "pants_cover_setup.go"
    digest: Digest


COVERAGE_SETUP_CODE = """\
package main

import (
    "testing"

{{#imports}}
    _cover{{i}} "{{import_path}}"
{{/imports}}
)

var (
    coverCounters = make(map[string][]uint32)
    coverBlocks = make(map[string][]testing.CoverBlock)
)

func coverRegisterFile(fileName string, counter []uint32, pos []uint32, numStmts []uint16) {
    if 3*len(counter) != len(pos) || len(counter) != len(numStmts) {
        panic("coverage: mismatched sizes")
    }
    if coverCounters[fileName] != nil {
        // Already registered.
        return
    }
    coverCounters[fileName] = counter
    block := make([]testing.CoverBlock, len(counter))
    for i := range counter {
        block[i] = testing.CoverBlock{
            Line0: pos[3*i+0],
            Col0: uint16(pos[3*i+2]),
            Line1: pos[3*i+1],
            Col1: uint16(pos[3*i+2]>>16),
            Stmts: numStmts[i],
        }
    }
    coverBlocks[fileName] = block
}

func init() {
{{#registrations}}
    coverRegisterFile("{{file_id}}", _cover{{i}}.{{cover_var}}.Count[:], _cover{{i}}.{{cover_var}}.Pos[:], _cover{{i}}.{{cover_var}}.NumStmt[:])
{{/registrations}}
}

func registerCover() {
    testing.RegisterCover(testing.Cover{
        Mode: "{{cover_mode}}",
        Counters: coverCounters,
        Blocks: coverBlocks,
        CoveredPackages: "",
    })
}
"""


@rule
async def generate_go_coverage_setup_code(
    request: GenerateCoverageSetupCodeRequest,
) -> GenerateCoverageSetupCodeResult:
    content = chevron.render(
        template=COVERAGE_SETUP_CODE,
        data={
            "imports": [
                {"i": i, "import_path": pkg.import_path} for i, pkg in enumerate(request.packages)
            ],
            "registrations": [
                {
                    "i": i,
                    "file_id": m.file_id,
                    "cover_var": m.cover_var,
                }
                for i, pkg in enumerate(request.packages)
                for m in pkg.cover_file_metadatas
            ],
            "cover_mode": request.cover_mode.value,
        },
    )

    digest = await create_digest(
        CreateDigest([FileContent(GenerateCoverageSetupCodeResult.PATH, content.encode())])
    )
    return GenerateCoverageSetupCodeResult(digest=digest)


def rules():
    return collect_rules()

1	# Copyright 2022 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3	from __future__ import annotations	1✔
4
5	import enum	1✔
6	import hashlib	1✔
7	import os	1✔
8	from dataclasses import dataclass	1✔
9	from pathlib import PurePath	1✔
10
11	import chevron	1✔
12
13	from pants.backend.go.util_rules.sdk import GoSdkProcess, GoSdkToolIDRequest, compute_go_tool_id	1✔
14	from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior	1✔
15	from pants.build_graph.address import Address	1✔
16	from pants.core.goals.test import CoverageData	1✔
17	from pants.engine.fs import CreateDigest, DigestSubset, FileContent, PathGlobs	1✔
18	from pants.engine.internals.native_engine import Digest, MergeDigests	1✔
19	from pants.engine.internals.selectors import concurrently	1✔
20	from pants.engine.intrinsics import create_digest, digest_subset_to_digest, merge_digests	1✔
21	from pants.engine.process import fallible_to_exec_result_or_raise	1✔
22	from pants.engine.rules import collect_rules, implicitly, rule	1✔
23	from pants.util.ordered_set import FrozenOrderedSet	1✔
24
25
26	@dataclass(frozen=True)	1✔
27	class GoCoverageData(CoverageData):	1✔
28	coverage_digest: Digest	1✔
29	import_path: str	1✔
30	sources_digest: Digest	1✔
31	sources_dir_path: str	1✔
32	pkg_target_address: Address	1✔
33
34
35	class GoCoverMode(enum.Enum):	1✔
36	SET = "set"	1✔
37	COUNT = "count"	1✔
38	ATOMIC = "atomic"	1✔
39
40
41	@dataclass(frozen=True)	1✔
42	class GoCoverageConfig:	1✔
43	# How to count the code usage.
44	cover_mode: GoCoverMode	1✔
45
46	# Import path patterns for packages which should be instrumented for code coverage.
47	import_path_include_patterns: tuple[str, ...] = ()	1✔
48
49
50	@dataclass(frozen=True)	1✔
51	class ApplyCodeCoverageRequest:	1✔
52	"""Apply code coverage to a package using `go tool cover`."""
53
54	digest: Digest	1✔
55	dir_path: str	1✔
56	go_files: tuple[str, ...]	1✔
57	cgo_files: tuple[str, ...]	1✔
58	cover_mode: GoCoverMode	1✔
59	import_path: str	1✔
60
61
62	@dataclass(frozen=True)	1✔
63	class FileCodeCoverageMetadata:	1✔
64	"""Metadata for code coverage applied to a single Go file."""
65
66	file_id: str	1✔
67	go_file: str	1✔
68	cover_go_file: str	1✔
69	cover_var: str	1✔
70
71
72	@dataclass(frozen=True)	1✔
73	class BuiltGoPackageCodeCoverageMetadata:	1✔
74	import_path: str	1✔
75	cover_file_metadatas: tuple[FileCodeCoverageMetadata, ...]	1✔
76	sources_digest: Digest	1✔
77	sources_dir_path: str	1✔
78
79
80	@dataclass(frozen=True)	1✔
81	class ApplyCodeCoverageResult:	1✔
82	digest: Digest	1✔
83	cover_file_metadatas: tuple[FileCodeCoverageMetadata, ...]	1✔
84	go_files: tuple[str, ...]	1✔
85	cgo_files: tuple[str, ...]	1✔
86
87
88	@dataclass(frozen=True)	1✔
89	class ApplyCodeCoverageToFileRequest:	1✔
90	digest: Digest	1✔
91	go_file: str	1✔
92	cover_go_file: str	1✔
93	mode: GoCoverMode	1✔
94	cover_var: str	1✔
95
96
97	@dataclass(frozen=True)	1✔
98	class ApplyCodeCoverageToFileResult:	1✔
99	digest: Digest	1✔
100	cover_go_file: str	1✔
101
102
103	@rule	1✔
104	async def go_apply_code_coverage_to_file(	1✔
105	request: ApplyCodeCoverageToFileRequest,
106	) -> ApplyCodeCoverageToFileResult:
107	cover_tool_id = await compute_go_tool_id(GoSdkToolIDRequest("cover"))	×
108
109	result = await fallible_to_exec_result_or_raise(	×
110	**implicitly(
111	GoSdkProcess(
112	input_digest=request.digest,
113	command=[
114	"tool",
115	"cover",
116	"-mode",
117	request.mode.value,
118	"-var",
119	request.cover_var,
120	"-o",
121	request.cover_go_file,
122	request.go_file,
123	],
124	description=f"Apply Go coverage to: {request.go_file}",
125	output_files=(str(request.cover_go_file),),
126	env={"__PANTS_GO_COVER_TOOL_ID": cover_tool_id.tool_id},
127	)
128	)
129	)
130
131	return ApplyCodeCoverageToFileResult(	×
132	digest=result.output_digest,
133	cover_go_file=request.cover_go_file,
134	)
135
136
137	def _hash_string(s: str) -> str:	1✔
138	h = hashlib.sha256(s.encode())	×
139	return h.hexdigest()[:12]	×
140
141
142	def _is_test_file(s: str) -> bool:	1✔
143	return s.endswith("_test.go")	×
144
145
146	@rule	1✔
147	async def go_apply_code_coverage(request: ApplyCodeCoverageRequest) -> ApplyCodeCoverageResult:	1✔
148	# Setup metadata for each file to which code coverage will be applied by assigning the name of the exported
149	# variable which holds coverage counters for each file.
150	file_metadatas: list[FileCodeCoverageMetadata] = []	×
151	output_go_files = []	×
152	output_cgo_files = []	×
153	import_path_hash = _hash_string(request.import_path)	×
154	for i, go_file in enumerate(request.go_files + request.cgo_files):	×
155	if _is_test_file(go_file):	×
156	if i < len(request.go_files):	×
157	output_go_files.append(go_file)	×
158	else:
159	output_cgo_files.append(go_file)	×
160	continue	×
161
162	p = PurePath(go_file)	×
163	cover_go_file = str(p.with_name(f"{p.stem}.cover.go"))	×
164	file_metadatas.append(	×
165	FileCodeCoverageMetadata(
166	file_id=f"{request.import_path}/{go_file}",
167	go_file=go_file,
168	cover_go_file=cover_go_file,
169	cover_var=f"GoCover_{import_path_hash}_{i}",
170	)
171	)
172	if i < len(request.go_files):	×
173	output_go_files.append(cover_go_file)	×
174	else:
175	output_cgo_files.append(cover_go_file)	×
176
177	subsetted_digests = await concurrently(	×
178	digest_subset_to_digest(
179	DigestSubset(
180	request.digest,
181	PathGlobs(
182	[os.path.join(request.dir_path, file_metadata.go_file)],
183	glob_match_error_behavior=GlobMatchErrorBehavior.error,
184	description_of_origin="coverage",
185	),
186	)
187	)
188	for file_metadata in file_metadatas
189	)
190
191	# Apply code coverage codegen to each file that will be analyzed.
192	cover_results = await concurrently(	×
193	go_apply_code_coverage_to_file(
194	ApplyCodeCoverageToFileRequest(
195	digest=go_file_digest,
196	go_file=os.path.join(request.dir_path, file_metadata.go_file),
197	cover_go_file=os.path.join(request.dir_path, file_metadata.cover_go_file),
198	mode=request.cover_mode,
199	cover_var=file_metadata.cover_var,
200	)
201	)
202	for file_metadata, go_file_digest in zip(file_metadatas, subsetted_digests)
203	)
204
205	# Merge the coverage codegen back into the original digest so that non-covered and covered sources are in
206	# the same digest.
207	digest = await merge_digests(MergeDigests([request.digest, *(r.digest for r in cover_results)]))	×
208
209	return ApplyCodeCoverageResult(	×
210	digest=digest,
211	cover_file_metadatas=tuple(file_metadatas),
212	go_files=tuple(output_go_files),
213	cgo_files=tuple(output_cgo_files),
214	)
215
216
217	@dataclass(frozen=True)	1✔
218	class GenerateCoverageSetupCodeRequest:	1✔
219	packages: FrozenOrderedSet[BuiltGoPackageCodeCoverageMetadata]	1✔
220	cover_mode: GoCoverMode	1✔
221
222
223	@dataclass(frozen=True)	1✔
224	class GenerateCoverageSetupCodeResult:	1✔
225	PATH = "pants_cover_setup.go"	1✔
226	digest: Digest	1✔
227
228
229	COVERAGE_SETUP_CODE = """\	1✔
230	package main
231
232	import (
233	"testing"
234
235	{{#imports}}
236	_cover{{i}} "{{import_path}}"
237	{{/imports}}
238	)
239
240	var (
241	coverCounters = make(map[string][]uint32)
242	coverBlocks = make(map[string][]testing.CoverBlock)
243	)
244
245	func coverRegisterFile(fileName string, counter []uint32, pos []uint32, numStmts []uint16) {
246	if 3*len(counter) != len(pos) \|\| len(counter) != len(numStmts) {
247	panic("coverage: mismatched sizes")
248	}
249	if coverCounters[fileName] != nil {
250	// Already registered.
251	return
252	}
253	coverCounters[fileName] = counter
254	block := make([]testing.CoverBlock, len(counter))
255	for i := range counter {
256	block[i] = testing.CoverBlock{
257	Line0: pos[3*i+0],
258	Col0: uint16(pos[3*i+2]),
259	Line1: pos[3*i+1],
260	Col1: uint16(pos[3*i+2]>>16),
261	Stmts: numStmts[i],
262	}
263	}
264	coverBlocks[fileName] = block
265	}
266
267	func init() {
268	{{#registrations}}
269	coverRegisterFile("{{file_id}}", _cover{{i}}.{{cover_var}}.Count[:], _cover{{i}}.{{cover_var}}.Pos[:], _cover{{i}}.{{cover_var}}.NumStmt[:])
270	{{/registrations}}
271	}
272
273	func registerCover() {
274	testing.RegisterCover(testing.Cover{
275	Mode: "{{cover_mode}}",
276	Counters: coverCounters,
277	Blocks: coverBlocks,
278	CoveredPackages: "",
279	})
280	}
281	"""
282
283
284	@rule	1✔
285	async def generate_go_coverage_setup_code(	1✔
286	request: GenerateCoverageSetupCodeRequest,
287	) -> GenerateCoverageSetupCodeResult:
288	content = chevron.render(	×
289	template=COVERAGE_SETUP_CODE,
290	data={
291	"imports": [
292	{"i": i, "import_path": pkg.import_path} for i, pkg in enumerate(request.packages)
293	],
294	"registrations": [
295	{
296	"i": i,
297	"file_id": m.file_id,
298	"cover_var": m.cover_var,
299	}
300	for i, pkg in enumerate(request.packages)
301	for m in pkg.cover_file_metadatas
302	],
303	"cover_mode": request.cover_mode.value,
304	},
305	)
306
307	digest = await create_digest(	×
308	CreateDigest([FileContent(GenerateCoverageSetupCodeResult.PATH, content.encode())])
309	)
310	return GenerateCoverageSetupCodeResult(digest=digest)	×
311
312
313	def rules():	1✔
314	return collect_rules()	1✔

pantsbuild / pants / 22740642519

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous