15655387894

Committed 14 Jun 2025 07:27PM UTC coverage: 65.21% (+0.6%) from 64.638%

Build # 15655387894

Build Type

push

github

Committed by

paulmthompson

Commit Message

fixed data aggregation error with DataArrayIndex vs TimeFrameIndex problem

Run Details

18 of 20 new or added lines in 1 file covered. (90.0%)

183 existing lines in 13 files now uncovered.

7996 of 12262 relevant lines covered (65.21%)

602.13 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.09

/src/WhiskerToolbox/DataManager/utils/DataAggregation/DataAggregation.cpp

#include "DataAggregation.hpp"

#include "AnalogTimeSeries/utils/statistics.hpp"

#include <algorithm>
#include <cmath>
#include <optional>
#include <stdexcept>

namespace DataAggregation {

int64_t calculateOverlapDuration(Interval const & a, Interval const & b) {
    const int64_t overlap_start = std::max(a.start, b.start);
    const int64_t overlap_end = std::min(a.end, b.end);

    if (overlap_start <= overlap_end) {
        return overlap_end - overlap_start + 1;
    }
    return 0;
}

int findOverlappingIntervalIndex(Interval const & target_interval,
                                 std::vector<Interval> const & reference_intervals,
                                 OverlapStrategy strategy) {
    std::vector<int> overlapping_indices;

    // Find all overlapping intervals using existing is_overlapping function
    for (size_t i = 0; i < reference_intervals.size(); ++i) {
        if (is_overlapping(target_interval, reference_intervals[i])) {
            overlapping_indices.push_back(static_cast<int>(i));
        }
    }

    if (overlapping_indices.empty()) {
        return -1;// No overlap found
    }

    switch (strategy) {
        case OverlapStrategy::First:
            return overlapping_indices.front();

        case OverlapStrategy::Last:
            return overlapping_indices.back();

        case OverlapStrategy::MaxOverlap: {
            int best_index = overlapping_indices[0];
            int64_t max_overlap = calculateOverlapDuration(target_interval, reference_intervals[best_index]);

            for (size_t i = 1; i < overlapping_indices.size(); ++i) {
                const int current_index = overlapping_indices[i];
                const int64_t current_overlap = calculateOverlapDuration(target_interval, reference_intervals[current_index]);

                if (current_overlap > max_overlap) {
                    max_overlap = current_overlap;
                    best_index = current_index;
                }
            }
            return best_index;
        }

        default:
            throw std::invalid_argument("Unknown overlap strategy");
    }
}

double applyTransformation(Interval const & interval,
                           TransformationConfig const & config,
                           std::map<std::string, std::vector<Interval>> const & reference_intervals,
                           std::map<std::string, std::shared_ptr<AnalogTimeSeries>> const & reference_analog,
                           std::map<std::string, std::shared_ptr<PointData>> const & reference_points) {
    switch (config.type) {
        case TransformationType::IntervalStart:
            return static_cast<double>(interval.start);

        case TransformationType::IntervalEnd:
            return static_cast<double>(interval.end);

        case TransformationType::IntervalDuration:
            return static_cast<double>(interval.end - interval.start + 1);

        case TransformationType::IntervalID: {
            auto it = reference_intervals.find(config.reference_data_key);
            if (it == reference_intervals.end()) {
                return std::nan("");// Reference data not found
            }

            const int overlap_index = findOverlappingIntervalIndex(interval, it->second, config.overlap_strategy);
            if (overlap_index == -1) {
                return std::nan("");// No overlap found
            }

            return static_cast<double>(overlap_index);// 0-based indexing
        }

        case TransformationType::IntervalCount: {
            auto it = reference_intervals.find(config.reference_data_key);
            if (it == reference_intervals.end()) {
                return std::nan("");// Reference data not found
            }

            // Count all overlapping intervals
            int count = 0;
            for (auto const & ref_interval: it->second) {
                if (is_overlapping(interval, ref_interval)) {
                    count++;
                }
            }

            return static_cast<double>(count);
        }

        case TransformationType::AnalogMean: {
            auto it = reference_analog.find(config.reference_data_key);
            if (it == reference_analog.end() || !it->second) {
                return std::nan("");// Reference data not found
            }

            // Get data within the time interval
            auto start_index = it->second->findDataArrayIndexGreaterOrEqual(TimeFrameIndex(interval.start));
            auto end_index = it->second->findDataArrayIndexLessOrEqual(TimeFrameIndex(interval.end));

            if (!start_index.has_value() || !end_index.has_value()) {
                return std::nan("");// No data found in interval
            }

            return calculate_mean(*it->second, start_index.value().getValue(), end_index.value().getValue() + 1); // exclusive end
        }

        case TransformationType::AnalogMin: {
            auto it = reference_analog.find(config.reference_data_key);
            if (it == reference_analog.end() || !it->second) {
                return std::nan("");// Reference data not found
            }

            auto start_index = it->second->findDataArrayIndexGreaterOrEqual(TimeFrameIndex(interval.start));
            auto end_index = it->second->findDataArrayIndexLessOrEqual(TimeFrameIndex(interval.end));

            if (!start_index.has_value() || !end_index.has_value()) {
                return std::nan("");// No data found in interval
            }

            // Get data within the time interval
            return calculate_min(*it->second, start_index.value().getValue(), end_index.value().getValue() + 1); // exclusive end
        }

        case TransformationType::AnalogMax: {
            auto it = reference_analog.find(config.reference_data_key);
            if (it == reference_analog.end() || !it->second) {
                return std::nan("");// Reference data not found
            }

            auto start_index = it->second->findDataArrayIndexGreaterOrEqual(TimeFrameIndex(interval.start));
            auto end_index = it->second->findDataArrayIndexLessOrEqual(TimeFrameIndex(interval.end));

            if (!start_index.has_value() || !end_index.has_value()) {
                return std::nan("");// No data found in interval
            }

            // Get data within the time interval
            return calculate_max(*it->second, start_index.value().getValue(), end_index.value().getValue() + 1); // exclusive end
        }

        case TransformationType::AnalogStdDev: {
            auto it = reference_analog.find(config.reference_data_key);
            if (it == reference_analog.end() || !it->second) {
                return std::nan("");// Reference data not found
            }

            // Get data within the time interval
            auto start_index = it->second->findDataArrayIndexGreaterOrEqual(TimeFrameIndex(interval.start));
            auto end_index = it->second->findDataArrayIndexLessOrEqual(TimeFrameIndex(interval.end));

            if (!start_index.has_value() || !end_index.has_value()) {
                return std::nan("");// No data found in interval
            }

            return calculate_std_dev(*it->second, start_index.value().getValue(), end_index.value().getValue() + 1); // exclusive end
        }

        case TransformationType::PointMeanX: {
            auto it = reference_points.find(config.reference_data_key);
            if (it == reference_points.end() || !it->second) {
                return std::nan("");// Reference data not found
            }

            // Collect all X coordinates within the interval
            std::vector<float> x_values;
            for (int64_t t = interval.start; t <= interval.end; ++t) {
                const auto& points = it->second->getPointsAtTime(static_cast<int>(t));
                for (const auto& point : points) {
                    x_values.push_back(point.x);
                }
            }

            if (x_values.empty()) {
                return std::nan("");// No points found in interval
            }

            // Calculate mean
            double sum = 0.0;
            for (float x : x_values) {
                sum += static_cast<double>(x);
            }
            return sum / static_cast<double>(x_values.size());
        }

        case TransformationType::PointMeanY: {
            auto it = reference_points.find(config.reference_data_key);
            if (it == reference_points.end() || !it->second) {
                return std::nan("");// Reference data not found
            }

            // Collect all Y coordinates within the interval
            std::vector<float> y_values;
            for (int64_t t = interval.start; t <= interval.end; ++t) {
                const auto& points = it->second->getPointsAtTime(static_cast<int>(t));
                for (const auto& point : points) {
                    y_values.push_back(point.y);
                }
            }

            if (y_values.empty()) {
                return std::nan("");// No points found in interval
            }

            // Calculate mean
            double sum = 0.0;
            for (float y : y_values) {
                sum += static_cast<double>(y);
            }
            return sum / static_cast<double>(y_values.size());
        }

        default:
            throw std::invalid_argument("Unknown transformation type");
    }
}

std::vector<std::vector<double>> aggregateData(
        std::vector<Interval> const & row_intervals,
        std::vector<TransformationConfig> const & transformations,
        std::map<std::string, std::vector<Interval>> const & reference_intervals,
        std::map<std::string, std::shared_ptr<AnalogTimeSeries>> const & reference_analog,
        std::map<std::string, std::shared_ptr<PointData>> const & reference_points) {

    std::vector<std::vector<double>> result;
    result.reserve(row_intervals.size());

    for (auto const & interval: row_intervals) {
        std::vector<double> row;
        row.reserve(transformations.size());

        for (auto const & transformation: transformations) {
            const double value = applyTransformation(interval, transformation, reference_intervals, reference_analog, reference_points);
            row.push_back(value);
        }

        result.push_back(std::move(row));
    }

    return result;
}

}// namespace DataAggregation

1	#include "DataAggregation.hpp"
2
3	#include "AnalogTimeSeries/utils/statistics.hpp"
4
5	#include <algorithm>
6	#include <cmath>
7	#include <optional>
8	#include <stdexcept>
9
10	namespace DataAggregation {
11
12	int64_t calculateOverlapDuration(Interval const & a, Interval const & b) {	11✔
13	const int64_t overlap_start = std::max(a.start, b.start);	11✔
14	const int64_t overlap_end = std::min(a.end, b.end);	11✔
15
16	if (overlap_start <= overlap_end) {	11✔
17	return overlap_end - overlap_start + 1;	9✔
18	}
19	return 0;	2✔
20	}
21
22	int findOverlappingIntervalIndex(Interval const & target_interval,	18✔
23	std::vector<Interval> const & reference_intervals,
24	OverlapStrategy strategy) {
25	std::vector<int> overlapping_indices;	18✔
26
27	// Find all overlapping intervals using existing is_overlapping function
28	for (size_t i = 0; i < reference_intervals.size(); ++i) {	63✔
29	if (is_overlapping(target_interval, reference_intervals[i])) {	45✔
30	overlapping_indices.push_back(static_cast<int>(i));	24✔
31	}
32	}
33
34	if (overlapping_indices.empty()) {	18✔
35	return -1;// No overlap found	1✔
36	}
37
38	switch (strategy) {	17✔
39	case OverlapStrategy::First:	12✔
40	return overlapping_indices.front();	12✔
41
42	case OverlapStrategy::Last:	1✔
43	return overlapping_indices.back();	1✔
44
45	case OverlapStrategy::MaxOverlap: {	4✔
46	int best_index = overlapping_indices[0];	4✔
47	int64_t max_overlap = calculateOverlapDuration(target_interval, reference_intervals[best_index]);	4✔
48
49	for (size_t i = 1; i < overlapping_indices.size(); ++i) {	6✔
50	const int current_index = overlapping_indices[i];	2✔
51	const int64_t current_overlap = calculateOverlapDuration(target_interval, reference_intervals[current_index]);	2✔
52
53	if (current_overlap > max_overlap) {	2✔
54	max_overlap = current_overlap;	1✔
55	best_index = current_index;	1✔
56	}
57	}
58	return best_index;	4✔
59	}
60
UNCOV 61	default:	×
UNCOV 62	throw std::invalid_argument("Unknown overlap strategy");	×
63	}
64	}	18✔
65
66	double applyTransformation(Interval const & interval,	115✔
67	TransformationConfig const & config,
68	std::map<std::string, std::vector<Interval>> const & reference_intervals,
69	std::map<std::string, std::shared_ptr<AnalogTimeSeries>> const & reference_analog,
70	std::map<std::string, std::shared_ptr<PointData>> const & reference_points) {
71	switch (config.type) {	115✔
72	case TransformationType::IntervalStart:	22✔
73	return static_cast<double>(interval.start);	22✔
74
75	case TransformationType::IntervalEnd:	13✔
76	return static_cast<double>(interval.end);	13✔
77
78	case TransformationType::IntervalDuration:	10✔
79	return static_cast<double>(interval.end - interval.start + 1);	10✔
80
81	case TransformationType::IntervalID: {	19✔
82	auto it = reference_intervals.find(config.reference_data_key);	19✔
83	if (it == reference_intervals.end()) {	19✔
84	return std::nan("");// Reference data not found	1✔
85	}
86
87	const int overlap_index = findOverlappingIntervalIndex(interval, it->second, config.overlap_strategy);	18✔
88	if (overlap_index == -1) {	18✔
89	return std::nan("");// No overlap found	1✔
90	}
91
92	return static_cast<double>(overlap_index);// 0-based indexing	17✔
93	}
94
95	case TransformationType::IntervalCount: {	9✔
96	auto it = reference_intervals.find(config.reference_data_key);	9✔
97	if (it == reference_intervals.end()) {	9✔
98	return std::nan("");// Reference data not found	1✔
99	}
100
101	// Count all overlapping intervals
102	int count = 0;	8✔
103	for (auto const & ref_interval: it->second) {	29✔
104	if (is_overlapping(interval, ref_interval)) {	21✔
105	count++;	10✔
106	}
107	}
108
109	return static_cast<double>(count);	8✔
110	}
111
112	case TransformationType::AnalogMean: {	14✔
113	auto it = reference_analog.find(config.reference_data_key);	14✔
114	if (it == reference_analog.end() \|\| !it->second) {	14✔
115	return std::nan("");// Reference data not found	2✔
116	}
117
118	// Get data within the time interval
119	auto start_index = it->second->findDataArrayIndexGreaterOrEqual(TimeFrameIndex(interval.start));	12✔
120	auto end_index = it->second->findDataArrayIndexLessOrEqual(TimeFrameIndex(interval.end));	12✔
121
122	if (!start_index.has_value() \|\| !end_index.has_value()) {	12✔
123	return std::nan("");// No data found in interval	1✔
124	}
125
126	return calculate_mean(*it->second, start_index.value().getValue(), end_index.value().getValue() + 1); // exclusive end	11✔
127	}
128
129	case TransformationType::AnalogMin: {	8✔
130	auto it = reference_analog.find(config.reference_data_key);	8✔
131	if (it == reference_analog.end() \|\| !it->second) {	8✔
UNCOV 132	return std::nan("");// Reference data not found	×
133	}
134
135	auto start_index = it->second->findDataArrayIndexGreaterOrEqual(TimeFrameIndex(interval.start));	8✔
136	auto end_index = it->second->findDataArrayIndexLessOrEqual(TimeFrameIndex(interval.end));	8✔
137
138	if (!start_index.has_value() \|\| !end_index.has_value()) {	8✔
139	return std::nan("");// No data found in interval	1✔
140	}
141
142	// Get data within the time interval
143	return calculate_min(*it->second, start_index.value().getValue(), end_index.value().getValue() + 1); // exclusive end	7✔
144	}
145
146	case TransformationType::AnalogMax: {	7✔
147	auto it = reference_analog.find(config.reference_data_key);	7✔
148	if (it == reference_analog.end() \|\| !it->second) {	7✔
UNCOV 149	return std::nan("");// Reference data not found	×
150	}
151
152	auto start_index = it->second->findDataArrayIndexGreaterOrEqual(TimeFrameIndex(interval.start));	7✔
153	auto end_index = it->second->findDataArrayIndexLessOrEqual(TimeFrameIndex(interval.end));	7✔
154
155	if (!start_index.has_value() \|\| !end_index.has_value()) {	7✔
NEW 156	return std::nan("");// No data found in interval	×
157	}
158
159	// Get data within the time interval
160	return calculate_max(*it->second, start_index.value().getValue(), end_index.value().getValue() + 1); // exclusive end	7✔
161	}
162
163	case TransformationType::AnalogStdDev: {	3✔
164	auto it = reference_analog.find(config.reference_data_key);	3✔
165	if (it == reference_analog.end() \|\| !it->second) {	3✔
UNCOV 166	return std::nan("");// Reference data not found	×
167	}
168
169	// Get data within the time interval
170	auto start_index = it->second->findDataArrayIndexGreaterOrEqual(TimeFrameIndex(interval.start));	3✔
171	auto end_index = it->second->findDataArrayIndexLessOrEqual(TimeFrameIndex(interval.end));	3✔
172
173	if (!start_index.has_value() \|\| !end_index.has_value()) {	3✔
NEW 174	return std::nan("");// No data found in interval	×
175	}
176
177	return calculate_std_dev(*it->second, start_index.value().getValue(), end_index.value().getValue() + 1); // exclusive end	3✔
178	}
179
180	case TransformationType::PointMeanX: {	6✔
181	auto it = reference_points.find(config.reference_data_key);	6✔
182	if (it == reference_points.end() \|\| !it->second) {	6✔
183	return std::nan("");// Reference data not found	2✔
184	}
185
186	// Collect all X coordinates within the interval
187	std::vector<float> x_values;	4✔
188	for (int64_t t = interval.start; t <= interval.end; ++t) {	15✔
189	const auto& points = it->second->getPointsAtTime(static_cast<int>(t));	11✔
190	for (const auto& point : points) {	21✔
191	x_values.push_back(point.x);	10✔
192	}
193	}
194
195	if (x_values.empty()) {	4✔
196	return std::nan("");// No points found in interval	1✔
197	}
198
199	// Calculate mean
200	double sum = 0.0;	3✔
201	for (float x : x_values) {	13✔
202	sum += static_cast<double>(x);	10✔
203	}
204	return sum / static_cast<double>(x_values.size());	3✔
205	}	4✔
206
207	case TransformationType::PointMeanY: {	4✔
208	auto it = reference_points.find(config.reference_data_key);	4✔
209	if (it == reference_points.end() \|\| !it->second) {	4✔
UNCOV 210	return std::nan("");// Reference data not found	×
211	}
212
213	// Collect all Y coordinates within the interval
214	std::vector<float> y_values;	4✔
215	for (int64_t t = interval.start; t <= interval.end; ++t) {	15✔
216	const auto& points = it->second->getPointsAtTime(static_cast<int>(t));	11✔
217	for (const auto& point : points) {	21✔
218	y_values.push_back(point.y);	10✔
219	}
220	}
221
222	if (y_values.empty()) {	4✔
223	return std::nan("");// No points found in interval	1✔
224	}
225
226	// Calculate mean
227	double sum = 0.0;	3✔
228	for (float y : y_values) {	13✔
229	sum += static_cast<double>(y);	10✔
230	}
231	return sum / static_cast<double>(y_values.size());	3✔
232	}	4✔
233
UNCOV 234	default:	×
UNCOV 235	throw std::invalid_argument("Unknown transformation type");	×
236	}
237	}
238
239	std::vector<std::vector<double>> aggregateData(	31✔
240	std::vector<Interval> const & row_intervals,
241	std::vector<TransformationConfig> const & transformations,
242	std::map<std::string, std::vector<Interval>> const & reference_intervals,
243	std::map<std::string, std::shared_ptr<AnalogTimeSeries>> const & reference_analog,
244	std::map<std::string, std::shared_ptr<PointData>> const & reference_points) {
245
246	std::vector<std::vector<double>> result;	31✔
247	result.reserve(row_intervals.size());	31✔
248
249	for (auto const & interval: row_intervals) {	76✔
250	std::vector<double> row;	45✔
251	row.reserve(transformations.size());	45✔
252
253	for (auto const & transformation: transformations) {	160✔
254	const double value = applyTransformation(interval, transformation, reference_intervals, reference_analog, reference_points);	115✔
255	row.push_back(value);	115✔
256	}
257
258	result.push_back(std::move(row));	45✔
259	}	45✔
260
261	return result;	31✔
UNCOV 262	}	×
263
264	}// namespace DataAggregation

paulmthompson / WhiskerToolbox / 15655387894

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous