19015773527

Committed 02 Nov 2025 05:33PM UTC coverage: 17.872% (-62.4%) from 80.3%

Build # 19015773527

Build Type

Pull #22816

github

Committed by

web-flow

Commit Message

Merge a12d75757 into 6c024e162

Pull Request Pull Request #22816: Update Pants internal Python to 3.14

Run Details

4 of 5 new or added lines in 3 files covered. (80.0%)

28452 existing lines in 683 files now uncovered.

9831 of 55007 relevant lines covered (17.87%)

0.18 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

78.08

/src/python/pants/util/memo.py

# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

import functools
import inspect
from collections.abc import Callable
from contextlib import contextmanager
from typing import Any, TypeVar

from pants.util.meta import T, classproperty

FuncType = Callable[..., Any]
F = TypeVar("F", bound=FuncType)


# Used as a sentinel that disambiguates tuples passed in *args from coincidentally matching tuples
# formed from kwargs item pairs.
_kwargs_separator = (object(),)


def equal_args(*args, **kwargs):
    """A memoized key factory that compares the equality (`==`) of a stable sort of the
    parameters."""
    key = args
    if kwargs:
        key += _kwargs_separator + tuple(sorted(kwargs.items()))
    return key


class InstanceKey:
    """An equality wrapper for an arbitrary object instance.

    This wrapper leverages `id` and `is` for fast `__hash__` and `__eq__` but both of these rely on
    the object in question not being gc'd since both `id` and `is` rely on the instance address
    which can be recycled; so we retain a strong reference to the instance to ensure no recycling
    can occur.
    """

    def __init__(self, instance):
        self._instance = instance
        self._hash = id(instance)

    def __hash__(self):
        return self._hash

    def __eq__(self, other):
        if self._instance is other:
            return True
        if isinstance(other, InstanceKey):
            return self._instance is other._instance
        return False


def per_instance(*args, **kwargs):
    """A memoized key factory that works like `equal_args` except that the first parameter's
    identity is used when forming the key.

    This is a useful key factory when you want to enforce memoization happens per-instance for an
    instance method in a class hierarchy that defines a custom `__hash__`/`__eq__`.
    """
    instance_and_rest = (InstanceKey(args[0]),) + args[1:]
    return equal_args(*instance_and_rest, **kwargs)


def memoized(func: F | None = None, key_factory=equal_args, cache_factory=dict) -> F:
    """Memoizes the results of a function call.

    By default, exactly one result is memoized for each unique combination of function arguments.

    Note that memoization is not thread-safe and the default result cache will grow without bound;
    so care must be taken to only apply this decorator to functions with single threaded access and
    an expected reasonably small set of unique call parameters.

    Note that the wrapped function comes equipped with 3 helper function attributes:

    + `put(*args, **kwargs)`: A context manager that takes the same arguments as the memoized
                              function and yields a setter function to set the value in the
                              memoization cache.
    + `forget(*args, **kwargs)`: Takes the same arguments as the memoized function and causes the
                                 memoization cache to forget the computed value, if any, for those
                                 arguments.
    + `clear()`: Causes the memoization cache to be fully cleared.

    :API: public

    :param func: The function to wrap.  Only generally passed by the python runtime and should be
                 omitted when passing a custom `key_factory` or `cache_factory`.
    :param key_factory: A function that can form a cache key from the arguments passed to the
                        wrapped, memoized function; by default uses simple parameter-set equality;
                        ie `equal_args`.
    :param cache_factory: A no-arg callable that produces a mapping object to use for the memoized
                          method's value cache.  By default the `dict` constructor, but could be a
                          a factory for an LRU cache for example.
    :raises: `ValueError` if the wrapper is applied to anything other than a function.
    :returns: A wrapped function that memoizes its results or else a function wrapper that does this.
    """
    if func is None:
        # We're being applied as a decorator factory; ie: the user has supplied args, like so:
        # >>> @memoized(cache_factory=lru_cache)
        # ... def expensive_operation(user):
        # ...   pass
        # So we return a decorator with the user-supplied args curried in for the python decorator
        # machinery to use to wrap the upcoming func.
        #
        # NB: This is just a tricky way to allow for both `@memoized` and `@memoized(...params...)`
        # application forms.  Without this trick, ie: using a decorator class or nested decorator
        # function, the no-params application would have to be `@memoized()`.  It still can, but need
        # not be and a bare `@memoized` will work as well as a `@memoized()`.
        return functools.partial(  # type: ignore[return-value]
            memoized, key_factory=key_factory, cache_factory=cache_factory
        )

    if not inspect.isfunction(func):
        raise ValueError("The @memoized decorator must be applied innermost of all decorators.")

    key_func = key_factory or equal_args
    memoized_results = cache_factory() if cache_factory else {}

    @functools.wraps(func)
    def memoize(*args, **kwargs):
        key = key_func(*args, **kwargs)
        if key in memoized_results:
            return memoized_results[key]
        result = func(*args, **kwargs)
        memoized_results[key] = result
        return result

    @contextmanager
    def put(*args, **kwargs):
        key = key_func(*args, **kwargs)
        yield functools.partial(memoized_results.__setitem__, key)

    memoize.put = put  # type: ignore[attr-defined]

    def forget(*args, **kwargs):
        key = key_func(*args, **kwargs)
        if key in memoized_results:
            del memoized_results[key]

    memoize.forget = forget  # type: ignore[attr-defined]

    def clear():
        memoized_results.clear()

    memoize.clear = clear  # type: ignore[attr-defined]

    return memoize  # type: ignore[return-value]


def memoized_method(func: F | None = None, key_factory=per_instance, cache_factory=dict) -> F:
    """A convenience wrapper for memoizing instance methods.

    Typically you'd expect a memoized instance method to hold a cached value per class instance;
    however, for classes that implement a custom `__hash__`/`__eq__` that can hash separate instances
    the same, `@memoized` will share cached values across `==` class instances.  Using
    `@memoized_method` defaults to a `per_instance` key for the cache to provide the expected cached
    value per-instance behavior.

    Applied like so:

    >>> class Foo:
    ...   @memoized_method
    ...   def name(self):
    ...     pass

    Is equivalent to:

    >>> class Foo:
    ...   @memoized(key_factory=per_instance)
    ...   def name(self):
    ...     pass

    :API: public

    :param func: The function to wrap.  Only generally passed by the python runtime and should be
                 omitted when passing a custom `key_factory` or `cache_factory`.
    :param key_factory: A function that can form a cache key from the arguments passed to the
                        wrapped, memoized function; by default `per_instance`.
    :param kwargs: Any extra keyword args accepted by `memoized`.
    :raises: `ValueError` if the wrapper is applied to anything other than a function.
    :returns: A wrapped function that memoizes its results or else a function wrapper that does this.
    """
    return memoized(func=func, key_factory=key_factory, cache_factory=cache_factory)


def memoized_property(
    func: Callable[..., T] | None = None, key_factory=per_instance, cache_factory=dict
) -> T:
    """A convenience wrapper for memoizing properties.

    Applied like so:

    >>> class Foo:
    ...   @memoized_property
    ...   def name(self):
    ...     pass

    Is equivalent to:

    >>> class Foo:
    ...   @property
    ...   @memoized_method
    ...   def name(self):
    ...     pass

    Which is equivalent to:

    >>> class Foo:
    ...   @property
    ...   @memoized(key_factory=per_instance)
    ...   def name(self):
    ...     pass

    By default a deleter for the property is setup that un-caches the property such that a subsequent
    property access re-computes the value.  In other words, for this `now` @memoized_property:

    >>> import time
    >>> class Bar:
    ...   @memoized_property
    ...   def now(self):
    ...     return time.time()

    You could write code like so:

    >>> bar = Bar()
    >>> bar.now
    1433267312.622095
    >>> time.sleep(5)
    >>> bar.now
    1433267312.622095
    >>> del bar.now
    >>> bar.now
    1433267424.056189
    >>> time.sleep(5)
    >>> bar.now
    1433267424.056189
    >>>

    :API: public

    :param func: The property getter method to wrap.  Only generally passed by the python runtime and
                 should be omitted when passing a custom `key_factory` or `cache_factory`.
    :param key_factory: A function that can form a cache key from the arguments passed to the
                        wrapped, memoized function; by default `per_instance`.
    :param kwargs: Any extra keyword args accepted by `memoized`.
    :raises: `ValueError` if the wrapper is applied to anything other than a function.
    :returns: A read-only property that memoizes its calculated value and un-caches its value when
              `del`ed.
    """
    getter = memoized_method(func=func, key_factory=key_factory, cache_factory=cache_factory)
    return property(  # type: ignore[return-value]
        fget=getter,
        fdel=lambda self: getter.forget(self),  # type: ignore[attr-defined]
    )


# TODO[13244]: fix type hint issue when using @memoized_classmethod and friends
def memoized_classmethod(func: F | None = None, key_factory=per_instance, cache_factory=dict) -> F:
    return classmethod(  # type: ignore[return-value]
        memoized_method(func, key_factory=key_factory, cache_factory=cache_factory)
    )


def memoized_classproperty(
    func: Callable[..., T] | None = None, key_factory=per_instance, cache_factory=dict
) -> T:
    return classproperty(
        memoized_classmethod(func, key_factory=key_factory, cache_factory=cache_factory)
    )


def testable_memoized_property(
    func: Callable[..., T] | None = None, key_factory=per_instance, cache_factory=dict
) -> T:
    """A variant of `memoized_property` that allows for setting of properties (for tests, etc)."""
    getter = memoized_method(func=func, key_factory=key_factory, cache_factory=cache_factory)

    def setter(self, val):
        with getter.put(self) as putter:
            putter(val)

    return property(  # type: ignore[return-value]
        fget=getter,
        fset=setter,
        fdel=lambda self: getter.forget(self),  # type: ignore[attr-defined]
    )

1	# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
2	# Licensed under the Apache License, Version 2.0 (see LICENSE).
3
4	import functools	1✔
5	import inspect	1✔
6	from collections.abc import Callable	1✔
7	from contextlib import contextmanager	1✔
8	from typing import Any, TypeVar	1✔
9
10	from pants.util.meta import T, classproperty	1✔
11
12	FuncType = Callable[..., Any]	1✔
13	F = TypeVar("F", bound=FuncType)	1✔
14
15
16	# Used as a sentinel that disambiguates tuples passed in *args from coincidentally matching tuples
17	# formed from kwargs item pairs.
18	_kwargs_separator = (object(),)	1✔
19
20
21	def equal_args(args, *kwargs):	1✔
22	"""A memoized key factory that compares the equality (`==`) of a stable sort of the
23	parameters."""
24	key = args	1✔
25	if kwargs:	1✔
UNCOV 26	key += _kwargs_separator + tuple(sorted(kwargs.items()))	×
27	return key	1✔
28
29
30	class InstanceKey:	1✔
31	"""An equality wrapper for an arbitrary object instance.
32
33	This wrapper leverages `id` and `is` for fast `__hash__` and `__eq__` but both of these rely on
34	the object in question not being gc'd since both `id` and `is` rely on the instance address
35	which can be recycled; so we retain a strong reference to the instance to ensure no recycling
36	can occur.
37	"""
38
39	def __init__(self, instance):	1✔
40	self._instance = instance	1✔
41	self._hash = id(instance)	1✔
42
43	def __hash__(self):	1✔
44	return self._hash	1✔
45
46	def __eq__(self, other):	1✔
47	if self._instance is other:	1✔
48	return True	×
49	if isinstance(other, InstanceKey):	1✔
50	return self._instance is other._instance	1✔
51	return False	×
52
53
54	def per_instance(args, *kwargs):	1✔
55	"""A memoized key factory that works like `equal_args` except that the first parameter's
56	identity is used when forming the key.
57
58	This is a useful key factory when you want to enforce memoization happens per-instance for an
59	instance method in a class hierarchy that defines a custom `__hash__`/`__eq__`.
60	"""
61	instance_and_rest = (InstanceKey(args[0]),) + args[1:]	1✔
62	return equal_args(instance_and_rest, *kwargs)	1✔
63
64
65	def memoized(func: F \| None = None, key_factory=equal_args, cache_factory=dict) -> F:	1✔
66	"""Memoizes the results of a function call.
67
68	By default, exactly one result is memoized for each unique combination of function arguments.
69
70	Note that memoization is not thread-safe and the default result cache will grow without bound;
71	so care must be taken to only apply this decorator to functions with single threaded access and
72	an expected reasonably small set of unique call parameters.
73
74	Note that the wrapped function comes equipped with 3 helper function attributes:
75
76	+ `put(args, *kwargs)`: A context manager that takes the same arguments as the memoized
77	function and yields a setter function to set the value in the
78	memoization cache.
79	+ `forget(args, *kwargs)`: Takes the same arguments as the memoized function and causes the
80	memoization cache to forget the computed value, if any, for those
81	arguments.
82	+ `clear()`: Causes the memoization cache to be fully cleared.
83
84	:API: public
85
86	:param func: The function to wrap. Only generally passed by the python runtime and should be
87	omitted when passing a custom `key_factory` or `cache_factory`.
88	:param key_factory: A function that can form a cache key from the arguments passed to the
89	wrapped, memoized function; by default uses simple parameter-set equality;
90	ie `equal_args`.
91	:param cache_factory: A no-arg callable that produces a mapping object to use for the memoized
92	method's value cache. By default the `dict` constructor, but could be a
93	a factory for an LRU cache for example.
94	:raises: `ValueError` if the wrapper is applied to anything other than a function.
95	:returns: A wrapped function that memoizes its results or else a function wrapper that does this.
96	"""
97	if func is None:	1✔
98	# We're being applied as a decorator factory; ie: the user has supplied args, like so:
99	# >>> @memoized(cache_factory=lru_cache)
100	# ... def expensive_operation(user):
101	# ... pass
102	# So we return a decorator with the user-supplied args curried in for the python decorator
103	# machinery to use to wrap the upcoming func.
104	#
105	# NB: This is just a tricky way to allow for both `@memoized` and `@memoized(...params...)`
106	# application forms. Without this trick, ie: using a decorator class or nested decorator
107	# function, the no-params application would have to be `@memoized()`. It still can, but need
108	# not be and a bare `@memoized` will work as well as a `@memoized()`.
UNCOV 109	return functools.partial( # type: ignore[return-value]	×
110	memoized, key_factory=key_factory, cache_factory=cache_factory
111	)
112
113	if not inspect.isfunction(func):	1✔
UNCOV 114	raise ValueError("The @memoized decorator must be applied innermost of all decorators.")	×
115
116	key_func = key_factory or equal_args	1✔
117	memoized_results = cache_factory() if cache_factory else {}	1✔
118
119	@functools.wraps(func)	1✔
120	def memoize(args, *kwargs):	1✔
121	key = key_func(args, *kwargs)	1✔
122	if key in memoized_results:	1✔
123	return memoized_results[key]	1✔
124	result = func(args, *kwargs)	1✔
125	memoized_results[key] = result	1✔
126	return result	1✔
127
128	@contextmanager	1✔
129	def put(args, *kwargs):	1✔
UNCOV 130	key = key_func(args, *kwargs)	×
UNCOV 131	yield functools.partial(memoized_results.__setitem__, key)	×
132
133	memoize.put = put # type: ignore[attr-defined]	1✔
134
135	def forget(args, *kwargs):	1✔
UNCOV 136	key = key_func(args, *kwargs)	×
UNCOV 137	if key in memoized_results:	×
UNCOV 138	del memoized_results[key]	×
139
140	memoize.forget = forget # type: ignore[attr-defined]	1✔
141
142	def clear():	1✔
UNCOV 143	memoized_results.clear()	×
144
145	memoize.clear = clear # type: ignore[attr-defined]	1✔
146
147	return memoize # type: ignore[return-value]	1✔
148
149
150	def memoized_method(func: F \| None = None, key_factory=per_instance, cache_factory=dict) -> F:	1✔
151	"""A convenience wrapper for memoizing instance methods.
152
153	Typically you'd expect a memoized instance method to hold a cached value per class instance;
154	however, for classes that implement a custom `__hash__`/`__eq__` that can hash separate instances
155	the same, `@memoized` will share cached values across `==` class instances. Using
156	`@memoized_method` defaults to a `per_instance` key for the cache to provide the expected cached
157	value per-instance behavior.
158
159	Applied like so:
160
161	>>> class Foo:
162	... @memoized_method
163	... def name(self):
164	... pass
165
166	Is equivalent to:
167
168	>>> class Foo:
169	... @memoized(key_factory=per_instance)
170	... def name(self):
171	... pass
172
173	:API: public
174
175	:param func: The function to wrap. Only generally passed by the python runtime and should be
176	omitted when passing a custom `key_factory` or `cache_factory`.
177	:param key_factory: A function that can form a cache key from the arguments passed to the
178	wrapped, memoized function; by default `per_instance`.
179	:param kwargs: Any extra keyword args accepted by `memoized`.
180	:raises: `ValueError` if the wrapper is applied to anything other than a function.
181	:returns: A wrapped function that memoizes its results or else a function wrapper that does this.
182	"""
183	return memoized(func=func, key_factory=key_factory, cache_factory=cache_factory)	1✔
184
185
186	def memoized_property(	1✔
187	func: Callable[..., T] \| None = None, key_factory=per_instance, cache_factory=dict
188	) -> T:
189	"""A convenience wrapper for memoizing properties.
190
191	Applied like so:
192
193	>>> class Foo:
194	... @memoized_property
195	... def name(self):
196	... pass
197
198	Is equivalent to:
199
200	>>> class Foo:
201	... @property
202	... @memoized_method
203	... def name(self):
204	... pass
205
206	Which is equivalent to:
207
208	>>> class Foo:
209	... @property
210	... @memoized(key_factory=per_instance)
211	... def name(self):
212	... pass
213
214	By default a deleter for the property is setup that un-caches the property such that a subsequent
215	property access re-computes the value. In other words, for this `now` @memoized_property:
216
217	>>> import time
218	>>> class Bar:
219	... @memoized_property
220	... def now(self):
221	... return time.time()
222
223	You could write code like so:
224
225	>>> bar = Bar()
226	>>> bar.now
227	1433267312.622095
228	>>> time.sleep(5)
229	>>> bar.now
230	1433267312.622095
231	>>> del bar.now
232	>>> bar.now
233	1433267424.056189
234	>>> time.sleep(5)
235	>>> bar.now
236	1433267424.056189
237	>>>
238
239	:API: public
240
241	:param func: The property getter method to wrap. Only generally passed by the python runtime and
242	should be omitted when passing a custom `key_factory` or `cache_factory`.
243	:param key_factory: A function that can form a cache key from the arguments passed to the
244	wrapped, memoized function; by default `per_instance`.
245	:param kwargs: Any extra keyword args accepted by `memoized`.
246	:raises: `ValueError` if the wrapper is applied to anything other than a function.
247	:returns: A read-only property that memoizes its calculated value and un-caches its value when
248	`del`ed.
249	"""
250	getter = memoized_method(func=func, key_factory=key_factory, cache_factory=cache_factory)	1✔
251	return property( # type: ignore[return-value]	1✔
252	fget=getter,
253	fdel=lambda self: getter.forget(self), # type: ignore[attr-defined]
254	)
255
256
257	# TODO[13244]: fix type hint issue when using @memoized_classmethod and friends
258	def memoized_classmethod(func: F \| None = None, key_factory=per_instance, cache_factory=dict) -> F:	1✔
259	return classmethod( # type: ignore[return-value]	1✔
260	memoized_method(func, key_factory=key_factory, cache_factory=cache_factory)
261	)
262
263
264	def memoized_classproperty(	1✔
265	func: Callable[..., T] \| None = None, key_factory=per_instance, cache_factory=dict
266	) -> T:
267	return classproperty(	1✔
268	memoized_classmethod(func, key_factory=key_factory, cache_factory=cache_factory)
269	)
270
271
272	def testable_memoized_property(	1✔
273	func: Callable[..., T] \| None = None, key_factory=per_instance, cache_factory=dict
274	) -> T:
275	"""A variant of `memoized_property` that allows for setting of properties (for tests, etc)."""
UNCOV 276	getter = memoized_method(func=func, key_factory=key_factory, cache_factory=cache_factory)	×
277
UNCOV 278	def setter(self, val):	×
UNCOV 279	with getter.put(self) as putter:	×
UNCOV 280	putter(val)	×
281
UNCOV 282	return property( # type: ignore[return-value]	×
283	fget=getter,
284	fset=setter,
285	fdel=lambda self: getter.forget(self), # type: ignore[attr-defined]
286	)

pantsbuild / pants / 19015773527

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous