10519623169

Committed 23 Aug 2024 03:59AM UTC coverage: 15.581% (+0.009%) from 15.572%

Build # 10519623169

Build Type

push

github

Committed by

yeliudev

Commit Message

Update logger API

Run Details

8 of 13 new or added lines in 4 files covered. (61.54%)

679 of 4358 relevant lines covered (15.58%)

3.09 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

3.45

/nncore/engine/buffer.py

# Copyright (c) Ye Liu. Licensed under the MIT License.

from collections import OrderedDict

import torch

import nncore


@nncore.bind_getter('max_size')
@nncore.bind_method('_data', ['get', 'pop', 'keys', 'values', 'items'])
class Buffer(object):
    """
    A buffer that tracks a series of values and provide access to smoothed
    scalar values over a window.

    Args:
        max_size (int, optional): Maximal number of internal values that can
            be stored in the buffer. When the capacity of the buffer is
            exhausted, old values will be removed. Default: ``100000``.
        logger (:obj:`logging.Logger` | str | None, optional): The logger or
            name of the logger to use. Default: ``None``.
    """

    def __init__(self, max_size=100000, logger=None):
        self._max_size = max_size
        self._logger = logger
        self._data = OrderedDict()

    def update(self, key, value, warning=True):
        """
        Add a new value. If the length of the buffer exceeds
        :obj:`self._max_size`, the oldest element will be removed from the
        buffer.

        Args:
            key (str): The key of the values.
            value (any): The new value.
            warning (bool, optional): Whether to display warning when removing
                values. Default: ``True``.
        """
        if key not in self._data:
            self._data[key] = []
        elif not key.startswith('_') and len(
                self._data[key]) == self._max_size:
            if warning:
                nncore.log(
                    "Number of '{}' values in the buffer exceeds max size "
                    "({}), removing the oldest element".format(
                        key, self._max_size),
                    self._logger,
                    log_level='WARNING')
            self._data[key].pop(0)

        self._data[key].append(value)

    def count(self, key):
        """
        Return the number of values according to the key.

        Args:
            key (str): The key of the values.
        """
        return len(self._data[key])

    def clear(self):
        """
        Remove all values from the buffer.
        """
        self._data = OrderedDict()

    def latest(self, key):
        """
        Return the latest value in the buffer.

        Args:
            key (str): The key of the values.
        """
        return self._data[key][-1]

    def median(self, key, window_size=None):
        """
        Return the median of the latest ``window_size`` values in the buffer.

        Args:
            key (str): The key of the values.
            window_size (int | None, optional): The window size of the values
                to be computed. If not specified, all the values will be taken
                into account. Default: ``None``.

        Returns:
            float: The median of the latest ``window_size`` values.
        """
        if window_size is None or window_size > len(self._data[key]):
            window_size = len(self._data[key])

        if isinstance(self._data[key][0], dict):
            data = nncore.to_dict_of_list(self._data[key][-window_size:])
            median = {
                k: torch.Tensor(v).median().item()
                for k, v in data.items()
            }
        else:
            median = torch.Tensor(
                self._data[key][-window_size:]).median().item()

        return median

    def mean(self, key, window_size=None):
        """
        Return the mean of the latest ``window_size`` values in the buffer.

        Args:
            key (str): The key of the values.
            window_size (int | None, optional): The window size of the values
                to be computed. If not specified, all the values will be taken
                into account. Default: ``None``.

        Returns:
            float: The mean of the latest ``window_size`` values.
        """
        if window_size is None or window_size > len(self._data[key]):
            window_size = len(self._data[key])

        if isinstance(self._data[key][0], dict):
            data = nncore.to_dict_of_list(self._data[key][-window_size:])
            mean = {k: torch.Tensor(v).mean().item() for k, v in data.items()}
        else:
            mean = torch.Tensor(self._data[key][-window_size:]).mean().item()

        return mean

    def sum(self, key, window_size=None):
        """
        Return the sum of the latest ``window_size`` values in the buffer.

        Args:
            key (str): The key of the values.
            window_size (int | None, optional): The window size of the values
                to be computed. If not specified, all the values will be taken
                into account. Default: ``None``.

        Returns:
            float: The sum of the latest ``window_size`` values.
        """
        if window_size is None or window_size > len(self._data[key]):
            window_size = len(self._data[key])

        if isinstance(self._data[key][0], dict):
            data = nncore.to_dict_of_list(self._data[key][-window_size:])
            sum = {k: torch.Tensor(v).sum().item() for k, v in data.items()}
        else:
            sum = torch.Tensor(self._data[key][-window_size:]).sum().item()

        return sum

    def avg(self, key, factor='_avg_factor', window_size=None):
        """
        Return the average of the latest ``window_size`` values in the buffer.
        Note that since not all the values in the buffer are count from the
        same number of samples, the exact average of these values should be
        computed with the number of samples.

        Args:
            key (str): The key of the values.
            factor (str, optional): The key of average factor. Default:
                ``'_avg_factor'``.
            window_size (int | None, optional): The window size of the values
                to be computed. If not specified, all the values will be taken
                into account. Default: ``None``.

        Returns:
            float: The average of the latest ``window_size`` values.
        """
        if window_size is None or window_size > len(self._data[key]):
            window_size = len(self._data[key])

        num_samples = torch.Tensor(self._data[factor][-window_size:])

        if isinstance(self._data[key][0], dict):
            data = nncore.to_dict_of_list(self._data[key][-window_size:])
            avg = {
                k: ((torch.Tensor(v) * num_samples).sum() /
                    num_samples.sum()).item()
                for k, v in data.items()
            }
        else:
            scalar = torch.Tensor(self._data[key][-window_size:])
            avg = ((scalar * num_samples).sum() / num_samples.sum()).item()

        return avg

1	# Copyright (c) Ye Liu. Licensed under the MIT License.
2
3	from collections import OrderedDict	20✔
4
5	import torch	20✔
6
7	import nncore	×
8
9
10	@nncore.bind_getter('max_size')	×
11	@nncore.bind_method('_data', ['get', 'pop', 'keys', 'values', 'items'])	×
12	class Buffer(object):	×
13	"""
14	A buffer that tracks a series of values and provide access to smoothed
15	scalar values over a window.
16
17	Args:
18	max_size (int, optional): Maximal number of internal values that can
19	be stored in the buffer. When the capacity of the buffer is
20	exhausted, old values will be removed. Default: ``100000``.
21	logger (:obj:`logging.Logger` \| str \| None, optional): The logger or
22	name of the logger to use. Default: ``None``.
23	"""
24
25	def __init__(self, max_size=100000, logger=None):	×
26	self._max_size = max_size	×
27	self._logger = logger	×
28	self._data = OrderedDict()	×
29
30	def update(self, key, value, warning=True):	×
31	"""
32	Add a new value. If the length of the buffer exceeds
33	:obj:`self._max_size`, the oldest element will be removed from the
34	buffer.
35
36	Args:
37	key (str): The key of the values.
38	value (any): The new value.
39	warning (bool, optional): Whether to display warning when removing
40	values. Default: ``True``.
41	"""
42	if key not in self._data:	×
43	self._data[key] = []	×
44	elif not key.startswith('_') and len(	×
45	self._data[key]) == self._max_size:
46	if warning:	×
NEW 47	nncore.log(	×
48	"Number of '{}' values in the buffer exceeds max size "
49	"({}), removing the oldest element".format(
50	key, self._max_size),
51	self._logger,
52	log_level='WARNING')
53	self._data[key].pop(0)	×
54
55	self._data[key].append(value)	×
56
57	def count(self, key):	×
58	"""
59	Return the number of values according to the key.
60
61	Args:
62	key (str): The key of the values.
63	"""
64	return len(self._data[key])	×
65
66	def clear(self):	×
67	"""
68	Remove all values from the buffer.
69	"""
70	self._data = OrderedDict()	×
71
72	def latest(self, key):	×
73	"""
74	Return the latest value in the buffer.
75
76	Args:
77	key (str): The key of the values.
78	"""
79	return self._data[key][-1]	×
80
81	def median(self, key, window_size=None):	×
82	"""
83	Return the median of the latest ``window_size`` values in the buffer.
84
85	Args:
86	key (str): The key of the values.
87	window_size (int \| None, optional): The window size of the values
88	to be computed. If not specified, all the values will be taken
89	into account. Default: ``None``.
90
91	Returns:
92	float: The median of the latest ``window_size`` values.
93	"""
94	if window_size is None or window_size > len(self._data[key]):	×
95	window_size = len(self._data[key])	×
96
97	if isinstance(self._data[key][0], dict):	×
98	data = nncore.to_dict_of_list(self._data[key][-window_size:])	×
99	median = {	×
100	k: torch.Tensor(v).median().item()
101	for k, v in data.items()
102	}
103	else:
104	median = torch.Tensor(	×
105	self._data[key][-window_size:]).median().item()
106
107	return median	×
108
109	def mean(self, key, window_size=None):	×
110	"""
111	Return the mean of the latest ``window_size`` values in the buffer.
112
113	Args:
114	key (str): The key of the values.
115	window_size (int \| None, optional): The window size of the values
116	to be computed. If not specified, all the values will be taken
117	into account. Default: ``None``.
118
119	Returns:
120	float: The mean of the latest ``window_size`` values.
121	"""
122	if window_size is None or window_size > len(self._data[key]):	×
123	window_size = len(self._data[key])	×
124
125	if isinstance(self._data[key][0], dict):	×
126	data = nncore.to_dict_of_list(self._data[key][-window_size:])	×
127	mean = {k: torch.Tensor(v).mean().item() for k, v in data.items()}	×
128	else:
129	mean = torch.Tensor(self._data[key][-window_size:]).mean().item()	×
130
131	return mean	×
132
133	def sum(self, key, window_size=None):	×
134	"""
135	Return the sum of the latest ``window_size`` values in the buffer.
136
137	Args:
138	key (str): The key of the values.
139	window_size (int \| None, optional): The window size of the values
140	to be computed. If not specified, all the values will be taken
141	into account. Default: ``None``.
142
143	Returns:
144	float: The sum of the latest ``window_size`` values.
145	"""
146	if window_size is None or window_size > len(self._data[key]):	×
147	window_size = len(self._data[key])	×
148
149	if isinstance(self._data[key][0], dict):	×
150	data = nncore.to_dict_of_list(self._data[key][-window_size:])	×
151	sum = {k: torch.Tensor(v).sum().item() for k, v in data.items()}	×
152	else:
153	sum = torch.Tensor(self._data[key][-window_size:]).sum().item()	×
154
155	return sum	×
156
157	def avg(self, key, factor='_avg_factor', window_size=None):	×
158	"""
159	Return the average of the latest ``window_size`` values in the buffer.
160	Note that since not all the values in the buffer are count from the
161	same number of samples, the exact average of these values should be
162	computed with the number of samples.
163
164	Args:
165	key (str): The key of the values.
166	factor (str, optional): The key of average factor. Default:
167	``'_avg_factor'``.
168	window_size (int \| None, optional): The window size of the values
169	to be computed. If not specified, all the values will be taken
170	into account. Default: ``None``.
171
172	Returns:
173	float: The average of the latest ``window_size`` values.
174	"""
175	if window_size is None or window_size > len(self._data[key]):	×
176	window_size = len(self._data[key])	×
177
178	num_samples = torch.Tensor(self._data[factor][-window_size:])	×
179
180	if isinstance(self._data[key][0], dict):	×
181	data = nncore.to_dict_of_list(self._data[key][-window_size:])	×
182	avg = {	×
183	k: ((torch.Tensor(v) * num_samples).sum() /
184	num_samples.sum()).item()
185	for k, v in data.items()
186	}
187	else:
188	scalar = torch.Tensor(self._data[key][-window_size:])	×
189	avg = ((scalar * num_samples).sum() / num_samples.sum()).item()	×
190
191	return avg	×

yeliudev / nncore / 10519623169

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous