18252174847

Committed 05 Oct 2025 01:36AM UTC coverage: 43.382% (-36.9%) from 80.261%

Build # 18252174847

Build Type

push

github

Committed by

web-flow

Commit Message

run tests on mac arm (#22717)

Just doing the minimal to pull forward the x86_64 pattern.

ref #20993

Run Details

25776 of 59416 relevant lines covered (43.38%)

1.3 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

26.97

/src/python/pants/option/options_fingerprinter.py

# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).
import hashlib
import json
import os
from collections import defaultdict
from enum import Enum
from hashlib import sha1
from typing import Any, cast

from pants.base.build_environment import get_buildroot
from pants.option.custom_types import UnsetBool, dict_with_files_option, dir_option, file_option
from pants.option.options import Options
from pants.util.strutil import softwrap


class OptionEncoder(json.JSONEncoder):
    def default(self, o):
        if o is UnsetBool:
            return "_UNSET_BOOL_ENCODING"
        if isinstance(o, Enum):
            return o.value
        if isinstance(o, dict):
            # Sort by key to ensure that we don't invalidate if the insertion order changes.
            return {k: self.default(v) for k, v in sorted(o.items())}
        return super().default(o)


def stable_option_fingerprint(obj):
    json_str = json.dumps(
        obj, ensure_ascii=True, allow_nan=False, sort_keys=True, cls=OptionEncoder
    )
    digest = hashlib.sha1()
    digest.update(json_str.encode("utf8"))
    return digest.hexdigest()


class OptionsFingerprinter:
    """Handles fingerprinting options under a given build_graph.

    :API: public
    """

    @classmethod
    def options_map_for_scope(cls, scope: str, options: Options) -> dict[str, Any]:
        """Given options and a scope, create a JSON-serializable map of all fingerprintable options
        in scope to their values."""
        options_map = {}
        option_items = options.get_fingerprintable_for_scope(scope, daemon_only=False)
        fingerprinter = cls()

        for option_name, option_type, option_value in option_items:
            value = option_value
            if option_type in (dir_option, file_option, dict_with_files_option):
                fingerprint = fingerprinter.fingerprint(option_type, option_value)
                if fingerprint is None:
                    # This isn't necessarily a good value to be using here, but it preserves behavior from
                    # before the commit which added it. I suspect that using the empty string would be
                    # reasonable too, but haven't done any archaeology to check.
                    fingerprint = "None"
                value = f"{option_value} (hash: {fingerprint})"
            options_map[option_name] = value

        serializable_map = cast(
            dict[str, Any], json.loads(json.dumps(options_map, cls=OptionEncoder))
        )

        return serializable_map

    def fingerprint(self, option_type, option_val):
        """Returns a hash of the given option_val based on the option_type.

        :API: public

        Returns None if option_val is None.
        """
        if option_val is None:
            return None

        # Wrapping all other values in a list here allows us to easily handle single-valued and
        # list-valued options uniformly. For non-list-valued options, this will be a singleton list
        # (with the exception of dict, which is not modified). This dict exception works because we do
        # not currently have any "list of dict" type, so there is no ambiguity.
        if not isinstance(option_val, (list, tuple, dict)):
            option_val = [option_val]

        if option_type == dir_option:
            return self._fingerprint_dirs(option_val)
        elif option_type == file_option:
            return self._fingerprint_files(option_val)
        elif option_type == dict_with_files_option:
            return self._fingerprint_dict_with_files(option_val)
        else:
            return self._fingerprint_primitives(option_val)

    def _assert_in_buildroot(self, filepath):
        """Raises an error if the given filepath isn't in the buildroot.

        Returns the normalized, absolute form of the path.
        """
        filepath = os.path.normpath(filepath)
        root = get_buildroot()
        if not os.path.abspath(filepath) == filepath:
            # If not absolute, assume relative to the build root.
            return os.path.join(root, filepath)
        else:
            if ".." in os.path.relpath(filepath, root).split(os.path.sep):
                # The path wasn't in the buildroot. This is an error because it violates pants being
                # hermetic.
                raise ValueError(
                    softwrap(
                        f"""
                        Received a file_option that was not inside the build root:

                            file_option: {filepath}
                            build_root:  {root}
                        """
                    )
                )
            return filepath

    def _fingerprint_dirs(self, dirpaths, topdown=True, onerror=None, followlinks=False):
        """Returns a fingerprint of the given file directories and all their sub contents.

        This assumes that the file directories are of reasonable size to cause memory or performance
        issues.
        """
        # Note that we don't sort the dirpaths, as their order may have meaning.
        filepaths = []
        for dirpath in dirpaths:
            dirs = os.walk(dirpath, topdown=topdown, onerror=onerror, followlinks=followlinks)
            sorted_dirs = sorted(dirs, key=lambda d: d[0])
            filepaths.extend(
                [
                    os.path.join(dirpath, filename)
                    for dirpath, dirnames, filenames in sorted_dirs
                    for filename in sorted(filenames)
                ]
            )
        return self._fingerprint_files(filepaths)

    def _fingerprint_files(self, filepaths):
        """Returns a fingerprint of the given filepaths and their contents.

        This assumes the files are small enough to be read into memory.
        """
        hasher = sha1()
        # Note that we don't sort the filepaths, as their order may have meaning.
        for filepath in filepaths:
            filepath = self._assert_in_buildroot(filepath)
            hasher.update(os.path.relpath(filepath, get_buildroot()).encode())
            with open(filepath, "rb") as f:
                hasher.update(f.read())
        return hasher.hexdigest()

    def _fingerprint_primitives(self, val):
        return stable_option_fingerprint(val)

    @staticmethod
    def _fingerprint_dict_with_files(option_val):
        """Returns a fingerprint of the given dictionary containing file paths.

        Any value which is a file path which exists on disk will be fingerprinted by that file's
        contents rather than by its path.

        This assumes the files are small enough to be read into memory.

        NB: The keys of the dict are assumed to be strings -- if they are not, the dict should be
        converted to encode its keys with `stable_option_fingerprint()`, as is done in the `fingerprint()`
        method.
        """
        final = defaultdict(list)
        for k, v in option_val.items():
            for sub_value in sorted(v.split(",")):
                if os.path.isfile(sub_value):
                    with open(sub_value) as f:
                        final[k].append(f.read())
                else:
                    final[k].append(sub_value)
        fingerprint = stable_option_fingerprint(final)
        return fingerprint

1	# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3	import hashlib	3✔
4	import json	3✔
5	import os	3✔
6	from collections import defaultdict	3✔
7	from enum import Enum	3✔
8	from hashlib import sha1	3✔
9	from typing import Any, cast	3✔
10
11	from pants.base.build_environment import get_buildroot	3✔
12	from pants.option.custom_types import UnsetBool, dict_with_files_option, dir_option, file_option	3✔
13	from pants.option.options import Options	3✔
14	from pants.util.strutil import softwrap	3✔
15
16
17	class OptionEncoder(json.JSONEncoder):	3✔
18	def default(self, o):	3✔
19	if o is UnsetBool:	×
20	return "_UNSET_BOOL_ENCODING"	×
21	if isinstance(o, Enum):	×
22	return o.value	×
23	if isinstance(o, dict):	×
24	# Sort by key to ensure that we don't invalidate if the insertion order changes.
25	return {k: self.default(v) for k, v in sorted(o.items())}	×
26	return super().default(o)	×
27
28
29	def stable_option_fingerprint(obj):	3✔
30	json_str = json.dumps(	×
31	obj, ensure_ascii=True, allow_nan=False, sort_keys=True, cls=OptionEncoder
32	)
33	digest = hashlib.sha1()	×
34	digest.update(json_str.encode("utf8"))	×
35	return digest.hexdigest()	×
36
37
38	class OptionsFingerprinter:	3✔
39	"""Handles fingerprinting options under a given build_graph.
40
41	:API: public
42	"""
43
44	@classmethod	3✔
45	def options_map_for_scope(cls, scope: str, options: Options) -> dict[str, Any]:	3✔
46	"""Given options and a scope, create a JSON-serializable map of all fingerprintable options
47	in scope to their values."""
48	options_map = {}	×
49	option_items = options.get_fingerprintable_for_scope(scope, daemon_only=False)	×
50	fingerprinter = cls()	×
51
52	for option_name, option_type, option_value in option_items:	×
53	value = option_value	×
54	if option_type in (dir_option, file_option, dict_with_files_option):	×
55	fingerprint = fingerprinter.fingerprint(option_type, option_value)	×
56	if fingerprint is None:	×
57	# This isn't necessarily a good value to be using here, but it preserves behavior from
58	# before the commit which added it. I suspect that using the empty string would be
59	# reasonable too, but haven't done any archaeology to check.
60	fingerprint = "None"	×
61	value = f"{option_value} (hash: {fingerprint})"	×
62	options_map[option_name] = value	×
63
64	serializable_map = cast(	×
65	dict[str, Any], json.loads(json.dumps(options_map, cls=OptionEncoder))
66	)
67
68	return serializable_map	×
69
70	def fingerprint(self, option_type, option_val):	3✔
71	"""Returns a hash of the given option_val based on the option_type.
72
73	:API: public
74
75	Returns None if option_val is None.
76	"""
77	if option_val is None:	×
78	return None	×
79
80	# Wrapping all other values in a list here allows us to easily handle single-valued and
81	# list-valued options uniformly. For non-list-valued options, this will be a singleton list
82	# (with the exception of dict, which is not modified). This dict exception works because we do
83	# not currently have any "list of dict" type, so there is no ambiguity.
84	if not isinstance(option_val, (list, tuple, dict)):	×
85	option_val = [option_val]	×
86
87	if option_type == dir_option:	×
88	return self._fingerprint_dirs(option_val)	×
89	elif option_type == file_option:	×
90	return self._fingerprint_files(option_val)	×
91	elif option_type == dict_with_files_option:	×
92	return self._fingerprint_dict_with_files(option_val)	×
93	else:
94	return self._fingerprint_primitives(option_val)	×
95
96	def _assert_in_buildroot(self, filepath):	3✔
97	"""Raises an error if the given filepath isn't in the buildroot.
98
99	Returns the normalized, absolute form of the path.
100	"""
101	filepath = os.path.normpath(filepath)	×
102	root = get_buildroot()	×
103	if not os.path.abspath(filepath) == filepath:	×
104	# If not absolute, assume relative to the build root.
105	return os.path.join(root, filepath)	×
106	else:
107	if ".." in os.path.relpath(filepath, root).split(os.path.sep):	×
108	# The path wasn't in the buildroot. This is an error because it violates pants being
109	# hermetic.
110	raise ValueError(	×
111	softwrap(
112	f"""
113	Received a file_option that was not inside the build root:
114
115	file_option: {filepath}
116	build_root: {root}
117	"""
118	)
119	)
120	return filepath	×
121
122	def _fingerprint_dirs(self, dirpaths, topdown=True, onerror=None, followlinks=False):	3✔
123	"""Returns a fingerprint of the given file directories and all their sub contents.
124
125	This assumes that the file directories are of reasonable size to cause memory or performance
126	issues.
127	"""
128	# Note that we don't sort the dirpaths, as their order may have meaning.
129	filepaths = []	×
130	for dirpath in dirpaths:	×
131	dirs = os.walk(dirpath, topdown=topdown, onerror=onerror, followlinks=followlinks)	×
132	sorted_dirs = sorted(dirs, key=lambda d: d[0])	×
133	filepaths.extend(	×
134	[
135	os.path.join(dirpath, filename)
136	for dirpath, dirnames, filenames in sorted_dirs
137	for filename in sorted(filenames)
138	]
139	)
140	return self._fingerprint_files(filepaths)	×
141
142	def _fingerprint_files(self, filepaths):	3✔
143	"""Returns a fingerprint of the given filepaths and their contents.
144
145	This assumes the files are small enough to be read into memory.
146	"""
147	hasher = sha1()	×
148	# Note that we don't sort the filepaths, as their order may have meaning.
149	for filepath in filepaths:	×
150	filepath = self._assert_in_buildroot(filepath)	×
151	hasher.update(os.path.relpath(filepath, get_buildroot()).encode())	×
152	with open(filepath, "rb") as f:	×
153	hasher.update(f.read())	×
154	return hasher.hexdigest()	×
155
156	def _fingerprint_primitives(self, val):	3✔
157	return stable_option_fingerprint(val)	×
158
159	@staticmethod	3✔
160	def _fingerprint_dict_with_files(option_val):	3✔
161	"""Returns a fingerprint of the given dictionary containing file paths.
162
163	Any value which is a file path which exists on disk will be fingerprinted by that file's
164	contents rather than by its path.
165
166	This assumes the files are small enough to be read into memory.
167
168	NB: The keys of the dict are assumed to be strings -- if they are not, the dict should be
169	converted to encode its keys with `stable_option_fingerprint()`, as is done in the `fingerprint()`
170	method.
171	"""
172	final = defaultdict(list)	×
173	for k, v in option_val.items():	×
174	for sub_value in sorted(v.split(",")):	×
175	if os.path.isfile(sub_value):	×
176	with open(sub_value) as f:	×
177	final[k].append(f.read())	×
178	else:
179	final[k].append(sub_value)	×
180	fingerprint = stable_option_fingerprint(final)	×
181	return fingerprint	×

pantsbuild / pants / 18252174847

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous