15430655463

Committed 04 Jun 2025 12:26AM UTC coverage: 40.974% (+17.1%) from 23.832%

Build # 15430655463

Build Type

push

github

Committed by

paulmthompson

Commit Message

fix self copy with mask data

Run Details

5 of 5 new or added lines in 2 files covered. (100.0%)

945 existing lines in 24 files now uncovered.

3003 of 7329 relevant lines covered (40.97%)

731.35 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

24.45

/src/WhiskerToolbox/DataManager/DataManager.cpp


#include "DataManager.hpp"
#include "AnalogTimeSeries/Analog_Time_Series.hpp"
#include "DigitalTimeSeries/Digital_Event_Series.hpp"
#include "DigitalTimeSeries/Digital_Interval_Series.hpp"
#include "Lines/Line_Data.hpp"
#include "Masks/Mask_Data.hpp"
#include "Media/Media_Data.hpp"
#include "Media/Video_Data.hpp"
#include "Points/Point_Data.hpp"
#include "Tensors/Tensor_Data.hpp"

#include "AnalogTimeSeries/IO/JSON/Analog_Time_Series_JSON.hpp"
#include "DigitalTimeSeries/IO/JSON/Digital_Event_Series_JSON.hpp"
#include "DigitalTimeSeries/IO/CSV/Digital_Interval_Series_CSV.hpp"
#include "DigitalTimeSeries/IO/JSON/Digital_Interval_Series_JSON.hpp"
#include "Lines/IO/JSON/Line_Data_JSON.hpp"
#include "Masks/IO/JSON/Mask_Data_JSON.hpp"
#include "Media/Video_Data_Loader.hpp"
#include "Points/IO/JSON/Point_Data_JSON.hpp"

#include "loaders/binary_loaders.hpp"
#include "transforms/Masks/mask_area.hpp"

#include "TimeFrame.hpp"

#include "nlohmann/json.hpp"
#include "utils/string_manip.hpp"

#include <filesystem>
#include <fstream>
#include <iostream>
#include <optional>
#include <regex>

using namespace nlohmann;

DataManager::DataManager() {
    _times["time"] = std::make_shared<TimeFrame>();
    _data["media"] = std::make_shared<MediaData>();

    setTimeFrame("media", "time");
    _output_path = std::filesystem::current_path();
}

bool DataManager::setTime(std::string const & key, std::shared_ptr<TimeFrame> timeframe, bool overwrite) {

    if (!timeframe) {
        std::cerr << "Error: Cannot register a nullptr TimeFrame for key: " << key << std::endl;
        return false;
    }

    if (_times.find(key) != _times.end()) {
        if (overwrite) {
            _times[key] = std::move(timeframe);
            return true;
        } else {
            std::cerr << "Error: Time key already exists in DataManager: " << key << std::endl;
            return false;
        }
    }

    _times[key] = std::move(timeframe);
    return true;
}

std::shared_ptr<TimeFrame>  DataManager::getTime() {
    return _times["time"];
};

std::shared_ptr<TimeFrame> DataManager::getTime(std::string const & key) {
    if (_times.find(key) != _times.end()) {
        return _times[key];
    }
    return nullptr;
};

bool DataManager::removeTime(std::string const & key)
{
    if (_times.find(key) == _times.end()) {
        std::cerr << "Error: could not find time key in DataManager: " << key << std::endl;
        return false;
    }

    auto it = _times.find(key);
    _times.erase(it);
    return true;
}

bool DataManager::setTimeFrame(std::string const & data_key, std::string const & time_key) {
    if (_data.find(data_key) == _data.end()) {
        std::cerr << "Error: Data key not found in DataManager: " << data_key << std::endl;
        return false;
    }

    if (_times.find(time_key) == _times.end()) {
        std::cerr << "Error: Time key not found in DataManager: " << time_key << std::endl;
        return false;
    }

    _time_frames[data_key] = time_key;
    return true;
}

std::string DataManager::getTimeFrame(std::string const & data_key) {
    // check if data_key exists
    if (_data.find(data_key) == _data.end()) {
        std::cerr << "Error: Data key not found in DataManager: " << data_key << std::endl;
        return "";
    }

    // check if data key has time frame
    if (_time_frames.find(data_key) == _time_frames.end()) {
        std::cerr << "Error: Data key "
                  << data_key
                  << " exists, but not assigned to a TimeFrame" <<  std::endl;
        return "";
    }

    return _time_frames[data_key];
}

std::vector<std::string> DataManager::getTimeFrameKeys() {
    std::vector<std::string> keys;
    keys.reserve(_times.size());
    for (auto const & [key, value]: _times) {

        keys.push_back(key);
    }
    return keys;
}

int DataManager::addCallbackToData(std::string const & key, ObserverCallback callback) {

    int id = -1;

    if (_data.find(key) != _data.end()) {
        auto data = _data[key];

        id = std::visit([callback](auto & x) {
            return x.get()->addObserver(callback);
        }, data);
    }

    return id;
}

bool DataManager::removeCallbackFromData(std::string const & key, int callback_id) {
    if (_data.find(key) != _data.end()) {
        auto data = _data[key];

        std::visit([callback_id](auto & x) {
            x.get()->removeObserver(callback_id);
        }, data);

        return true;
    }

    return false;
}

void DataManager::addObserver(ObserverCallback callback) {
    _observers.push_back(std::move(callback));
}

void DataManager::_notifyObservers() {
    for (auto & observer: _observers) {
        observer();
    }
}

std::vector<std::string> DataManager::getAllKeys() {
    std::vector<std::string> keys;
    keys.reserve(_data.size());
    for (auto const & [key, value]: _data) {

        keys.push_back(key);
    }
    return keys;
}

std::optional<DataTypeVariant> DataManager::getDataVariant(std::string const & key) {
    if (_data.find(key) != _data.end()) {
        return _data[key];
    }
    return std::nullopt;
}

void DataManager::setData(std::string const & key, DataTypeVariant data) {
    _data[key] = data;
    setTimeFrame(key, "time");
    _notifyObservers();
}

std::optional<std::string> processFilePath(
        std::string const & file_path,
        std::filesystem::path const & base_path) {
    std::filesystem::path full_path = file_path;

    // Check for wildcard character
    if (file_path.find('*') != std::string::npos) {
        // Convert wildcard pattern to regex
        std::string const pattern = std::regex_replace(full_path.string(), std::regex("\\*"), ".*");
        std::regex const regex_pattern(pattern);

        // Iterate through the directory to find matching files
        for (auto const & entry: std::filesystem::directory_iterator(base_path)) {
            std::cout << "Checking " << entry.path().string() << " with full path " << full_path << std::endl;
            if (std::regex_match(entry.path().string(), regex_pattern)) {
                std::cout << "Loading file " << entry.path().string() << std::endl;
                return entry.path().string();
            }
        }
        return std::nullopt;
    } else {
        // Check if the file path is relative
        if (!std::filesystem::path(file_path).is_absolute()) {
            full_path = base_path / file_path;
        }
        // Check for the presence of the file
        if (std::filesystem::exists(full_path)) {
            std::cout << "Loading file " << full_path.string() << std::endl;
            return full_path.string();
        } else {
            return std::nullopt;
        }
    }
}

bool checkRequiredFields(json const & item, std::vector<std::string> const & requiredFields) {
    for (auto const & field: requiredFields) {
        if (!item.contains(field)) {
            std::cerr << "Error: Missing required field \"" << field << "\" in JSON item." << std::endl;
            return false;
        }
    }
    return true;
}

void checkOptionalFields(json const & item, std::vector<std::string> const & optionalFields) {
    for (auto const & field: optionalFields) {
        if (!item.contains(field)) {
            std::cout << "Warning: Optional field \"" << field << "\" is missing in JSON item." << std::endl;
        }
    }
}

DM_DataType stringToDataType(std::string const & data_type_str) {
    if (data_type_str == "video") return DM_DataType::Video;
    if (data_type_str == "points") return DM_DataType::Points;
    if (data_type_str == "mask") return DM_DataType::Mask;
    if (data_type_str == "line") return DM_DataType::Line;
    if (data_type_str == "analog") return DM_DataType::Analog;
    if (data_type_str == "digital_event") return DM_DataType::DigitalEvent;
    if (data_type_str == "digital_interval") return DM_DataType::DigitalInterval;
    if (data_type_str == "tensor") return DM_DataType::Tensor;
    if (data_type_str == "time") return DM_DataType::Time;
    return DM_DataType::Unknown;
}

std::vector<DataInfo> load_data_from_json_config(DataManager * dm, std::string const & json_filepath) {
    std::vector<DataInfo> data_info_list;
    // Open JSON file
    std::ifstream ifs(json_filepath);
    if (!ifs.is_open()) {
        std::cerr << "Failed to open JSON file: " << json_filepath << std::endl;
        return data_info_list;
    }

    // Parse JSON
    json j;
    ifs >> j;

    // get base path of filepath
    std::filesystem::path const base_path = std::filesystem::path(json_filepath).parent_path();

    // Iterate through JSON array
    for (auto const & item: j) {

        if (!checkRequiredFields(item, {"data_type", "name", "filepath"})) {
            continue;// Exit if any required field is missing
        }

        std::string const data_type_str = item["data_type"];
        auto const data_type = stringToDataType(data_type_str);
        if (data_type == DM_DataType::Unknown) {
            std::cout << "Unknown data type: " << data_type_str << std::endl;
            continue;
        }

        std::string const name = item["name"];

        auto file_exists = processFilePath(item["filepath"], base_path);
        if (!file_exists) {
            std::cerr << "File does not exist: " << item["filepath"] << std::endl;
            continue;
        }

        std::string const file_path = file_exists.value();

        switch (data_type) {
            case DM_DataType::Video: {

                auto video_data = load_video_into_VideoData(file_path);
                dm->setData<VideoData>("media", video_data);

                data_info_list.push_back({name, "VideoData", ""});
                break;
            }
            case DM_DataType::Points: {

                auto point_data = load_into_PointData(file_path, item);

                dm->setData<PointData>(name, point_data);

                std::string const color = item.value("color", "#0000FF");
                data_info_list.push_back({name, "PointData", color});
                break;
            }
            case DM_DataType::Mask: {

                auto mask_data = load_into_MaskData(file_path, item);

                std::string const color = item.value("color", "0000FF");
                dm->setData<MaskData>(name, mask_data);

                data_info_list.push_back({name, "MaskData", color});

                if (item.contains("operations")) {

                    for (auto const & operation: item["operations"]) {

                        std::string const operation_type = operation["type"];

                        if (operation_type == "area") {
                            std::cout << "Calculating area for mask: " << name << std::endl;
                            auto area_data = area(dm->getData<MaskData>(name).get());
                            std::string const output_name = name + "_area";
                            dm->setData<AnalogTimeSeries>(output_name, area_data);
                        }
                    }
                }
                break;
            }
            case DM_DataType::Line: {

                auto line_data = load_into_LineData(file_path, item);

                dm->setData<LineData>(name, line_data);

                std::string const color = item.value("color", "0000FF");

                data_info_list.push_back({name, "LineData", color});

                break;
            }
            case DM_DataType::Analog: {

                auto analog_time_series = load_into_AnalogTimeSeries(file_path, item);

                for (int channel = 0; channel < analog_time_series.size(); channel++) {
                    std::string const channel_name = name + "_" + std::to_string(channel);

                    dm->setData<AnalogTimeSeries>(channel_name, analog_time_series[channel]);

                    if (item.contains("clock")) {
                        std::string const clock = item["clock"];
                        dm->setTimeFrame(channel_name, clock);
                    }
                }
                break;
            }
            case DM_DataType::DigitalEvent: {

                auto digital_event_series = load_into_DigitalEventSeries(file_path, item);

                for (int channel = 0; channel < digital_event_series.size(); channel++) {
                    std::string const channel_name = name + "_" + std::to_string(channel);

                    dm->setData<DigitalEventSeries>(channel_name, digital_event_series[channel]);

                    if (item.contains("clock")) {
                        std::string const clock = item["clock"];
                        dm->setTimeFrame(channel_name, clock);
                    }
                }
                break;
            }
            case DM_DataType::DigitalInterval: {

                auto digital_interval_series = load_into_DigitalIntervalSeries(file_path, item);
                dm->setData<DigitalIntervalSeries>(name, digital_interval_series);

                break;
            }
            case DM_DataType::Tensor: {

                if (item["format"] == "numpy") {

                    TensorData tensor_data;
                    loadNpyToTensorData(file_path, tensor_data);

                    dm->setData<TensorData>(name, std::make_shared<TensorData>(tensor_data));

                } else {
                    std::cout << "Format " << item["format"] << " not found for " << name << std::endl;
                }
                break;
            }
            case DM_DataType::Time: {

                if (item["format"] == "uint16") {

                    int const channel = item["channel"];
                    std::string const transition = item["transition"];

                    int const header_size = item.value("header_size", 0);

                    auto opts = Loader::BinaryAnalogOptions{.file_path = file_path,
                                                            .header_size_bytes = static_cast<size_t>(header_size)};
                    auto data = readBinaryFile<uint16_t>(opts);

                    auto digital_data = Loader::extractDigitalData(data, channel);
                    auto events = Loader::extractEvents(digital_data, transition);

                    // convert to int with std::transform
                    std::vector<int> events_int;
                    events_int.reserve(events.size());
                    for (auto e: events) {
                        events_int.push_back(static_cast<int>(e));
                    }
                    std::cout << "Loaded " << events_int.size() << " events for " << name << std::endl;

                    auto timeframe = std::make_shared<TimeFrame>(events_int);
                    dm->setTime(name, timeframe, true);
                }

                if (item["format"] == "uint16_length") {

                    int const header_size = item.value("header_size", 0);

                    auto opts = Loader::BinaryAnalogOptions{.file_path = file_path,
                                                            .header_size_bytes = static_cast<size_t>(header_size)};
                    auto data = readBinaryFile<uint16_t>(opts);

                    std::vector<int> t(data.size());
                    std::iota(std::begin(t), std::end(t), 0);

                    std::cout << "Total of " << t.size() << " timestamps for " << name << std::endl;

                    auto timeframe = std::make_shared<TimeFrame>(t);
                    dm->setTime(name, timeframe, true);
                }
                break;
            }
            default:
                std::cout << "Unsupported data type: " << data_type_str << std::endl;
                continue;
        }
        if (item.contains("clock")) {
            std::string const clock = item["clock"];
            std::cout << "Setting time for " << name << " to " << clock << std::endl;
            dm->setTimeFrame(name, clock);
        }
    }

    return data_info_list;
}

DM_DataType DataManager::getType(std::string const & key) const {
    auto it = _data.find(key);
    if (it != _data.end()) {
        if (std::holds_alternative<std::shared_ptr<MediaData>>(it->second)) {
            return DM_DataType::Video;
        } else if (std::holds_alternative<std::shared_ptr<PointData>>(it->second)) {
            return DM_DataType::Points;
        } else if (std::holds_alternative<std::shared_ptr<LineData>>(it->second)) {
            return DM_DataType::Line;
        } else if (std::holds_alternative<std::shared_ptr<MaskData>>(it->second)) {
            return DM_DataType::Mask;
        } else if (std::holds_alternative<std::shared_ptr<AnalogTimeSeries>>(it->second)) {
            return DM_DataType::Analog;
        } else if (std::holds_alternative<std::shared_ptr<DigitalEventSeries>>(it->second)) {
            return DM_DataType::DigitalEvent;
        } else if (std::holds_alternative<std::shared_ptr<DigitalIntervalSeries>>(it->second)) {
            return DM_DataType::DigitalInterval;
        } else if (std::holds_alternative<std::shared_ptr<TensorData>>(it->second)) {
            return DM_DataType::Tensor;
        }
        return DM_DataType::Unknown;
    }
    return DM_DataType::Unknown;
}

std::string convert_data_type_to_string(DM_DataType type) {
    switch (type) {
        case DM_DataType::Video:
            return "video";
        case DM_DataType::Points:
            return "points";
        case DM_DataType::Mask:
            return "mask";
        case DM_DataType::Line:
            return "line";
        case DM_DataType::Analog:
            return "analog";
        case DM_DataType::DigitalEvent:
            return "digital_event";
        case DM_DataType::DigitalInterval:
            return "digital_interval";
        case DM_DataType::Tensor:
            return "tensor";
        case DM_DataType::Time:
            return "time";
        default:
            return "unknown";
    }
}

1
2	#include "DataManager.hpp"
3	#include "AnalogTimeSeries/Analog_Time_Series.hpp"
4	#include "DigitalTimeSeries/Digital_Event_Series.hpp"
5	#include "DigitalTimeSeries/Digital_Interval_Series.hpp"
6	#include "Lines/Line_Data.hpp"
7	#include "Masks/Mask_Data.hpp"
8	#include "Media/Media_Data.hpp"
9	#include "Media/Video_Data.hpp"
10	#include "Points/Point_Data.hpp"
11	#include "Tensors/Tensor_Data.hpp"
12
13	#include "AnalogTimeSeries/IO/JSON/Analog_Time_Series_JSON.hpp"
14	#include "DigitalTimeSeries/IO/JSON/Digital_Event_Series_JSON.hpp"
15	#include "DigitalTimeSeries/IO/CSV/Digital_Interval_Series_CSV.hpp"
16	#include "DigitalTimeSeries/IO/JSON/Digital_Interval_Series_JSON.hpp"
17	#include "Lines/IO/JSON/Line_Data_JSON.hpp"
18	#include "Masks/IO/JSON/Mask_Data_JSON.hpp"
19	#include "Media/Video_Data_Loader.hpp"
20	#include "Points/IO/JSON/Point_Data_JSON.hpp"
21
22	#include "loaders/binary_loaders.hpp"
23	#include "transforms/Masks/mask_area.hpp"
24
25	#include "TimeFrame.hpp"
26
27	#include "nlohmann/json.hpp"
28	#include "utils/string_manip.hpp"
29
30	#include <filesystem>
31	#include <fstream>
32	#include <iostream>
33	#include <optional>
34	#include <regex>
35
36	using namespace nlohmann;
37
38	DataManager::DataManager() {	46✔
39	_times["time"] = std::make_shared<TimeFrame>();	138✔
40	_data["media"] = std::make_shared<MediaData>();	138✔
41
42	setTimeFrame("media", "time");	230✔
43	_output_path = std::filesystem::current_path();	46✔
44	}	46✔
45
46	bool DataManager::setTime(std::string const & key, std::shared_ptr<TimeFrame> timeframe, bool overwrite) {	26✔
47
48	if (!timeframe) {	26✔
49	std::cerr << "Error: Cannot register a nullptr TimeFrame for key: " << key << std::endl;	1✔
50	return false;	1✔
51	}
52
53	if (_times.find(key) != _times.end()) {	25✔
54	if (overwrite) {	2✔
UNCOV 55	_times[key] = std::move(timeframe);	×
UNCOV 56	return true;	×
57	} else {
58	std::cerr << "Error: Time key already exists in DataManager: " << key << std::endl;	2✔
59	return false;	2✔
60	}
61	}
62
63	_times[key] = std::move(timeframe);	23✔
64	return true;	23✔
65	}
66
67	std::shared_ptr<TimeFrame> DataManager::getTime() {	1✔
68	return _times["time"];	3✔
69	};
70
71	std::shared_ptr<TimeFrame> DataManager::getTime(std::string const & key) {	10✔
72	if (_times.find(key) != _times.end()) {	10✔
73	return _times[key];	8✔
74	}
75	return nullptr;	2✔
76	};
77
UNCOV 78	bool DataManager::removeTime(std::string const & key)	×
79	{
80	if (_times.find(key) == _times.end()) {	×
81	std::cerr << "Error: could not find time key in DataManager: " << key << std::endl;	×
82	return false;	×
83	}
84
UNCOV 85	auto it = _times.find(key);	×
UNCOV 86	_times.erase(it);	×
UNCOV 87	return true;	×
88	}
89
90	bool DataManager::setTimeFrame(std::string const & data_key, std::string const & time_key) {	92✔
91	if (_data.find(data_key) == _data.end()) {	92✔
92	std::cerr << "Error: Data key not found in DataManager: " << data_key << std::endl;	1✔
93	return false;	1✔
94	}
95
96	if (_times.find(time_key) == _times.end()) {	91✔
97	std::cerr << "Error: Time key not found in DataManager: " << time_key << std::endl;	1✔
98	return false;	1✔
99	}
100
101	_time_frames[data_key] = time_key;	90✔
102	return true;	90✔
103	}
104
105	std::string DataManager::getTimeFrame(std::string const & data_key) {	9✔
106	// check if data_key exists
107	if (_data.find(data_key) == _data.end()) {	9✔
108	std::cerr << "Error: Data key not found in DataManager: " << data_key << std::endl;	1✔
109	return "";	3✔
110	}
111
112	// check if data key has time frame
113	if (_time_frames.find(data_key) == _time_frames.end()) {	8✔
114	std::cerr << "Error: Data key "
115	<< data_key
UNCOV 116	<< " exists, but not assigned to a TimeFrame" << std::endl;	×
UNCOV 117	return "";	×
118	}
119
120	return _time_frames[data_key];	8✔
121	}
122
123	std::vector<std::string> DataManager::getTimeFrameKeys() {	8✔
124	std::vector<std::string> keys;	8✔
125	keys.reserve(_times.size());	8✔
126	for (auto const & [key, value]: _times) {	24✔
127
128	keys.push_back(key);	16✔
129	}
130	return keys;	8✔
UNCOV 131	}	×
132
133	int DataManager::addCallbackToData(std::string const & key, ObserverCallback callback) {	7✔
134
135	int id = -1;	7✔
136
137	if (_data.find(key) != _data.end()) {	7✔
138	auto data = _data[key];	6✔
139
140	id = std::visit([callback](auto & x) {	12✔
141	return x.get()->addObserver(callback);	6✔
142	}, data);
143	}	6✔
144
145	return id;	7✔
146	}
147
148	bool DataManager::removeCallbackFromData(std::string const & key, int callback_id) {	4✔
149	if (_data.find(key) != _data.end()) {	4✔
150	auto data = _data[key];	3✔
151
152	std::visit([callback_id](auto & x) {	6✔
153	x.get()->removeObserver(callback_id);	3✔
154	}, data);	3✔
155
156	return true;	3✔
157	}	3✔
158
159	return false;	1✔
160	}
161
162	void DataManager::addObserver(ObserverCallback callback) {	5✔
163	_observers.push_back(std::move(callback));	5✔
164	}	5✔
165
166	void DataManager::_notifyObservers() {	40✔
167	for (auto & observer: _observers) {	48✔
168	observer();	8✔
169	}
170	}	40✔
171
172	std::vector<std::string> DataManager::getAllKeys() {	5✔
173	std::vector<std::string> keys;	5✔
174	keys.reserve(_data.size());	5✔
175	for (auto const & [key, value]: _data) {	17✔
176
177	keys.push_back(key);	12✔
178	}
179	return keys;	5✔
UNCOV 180	}	×
181
UNCOV 182	std::optional<DataTypeVariant> DataManager::getDataVariant(std::string const & key) {	×
UNCOV 183	if (_data.find(key) != _data.end()) {	×
UNCOV 184	return _data[key];	×
185	}
UNCOV 186	return std::nullopt;	×
187	}
188
189	void DataManager::setData(std::string const & key, DataTypeVariant data) {	1✔
190	_data[key] = data;	1✔
191	setTimeFrame(key, "time");	3✔
192	_notifyObservers();	1✔
193	}	1✔
194
UNCOV 195	std::optional<std::string> processFilePath(	×
196	std::string const & file_path,
197	std::filesystem::path const & base_path) {
198	std::filesystem::path full_path = file_path;	×
199
200	// Check for wildcard character
UNCOV 201	if (file_path.find('*') != std::string::npos) {	×
202	// Convert wildcard pattern to regex
203	std::string const pattern = std::regex_replace(full_path.string(), std::regex("\\"), ".");	×
204	std::regex const regex_pattern(pattern);	×
205
206	// Iterate through the directory to find matching files
UNCOV 207	for (auto const & entry: std::filesystem::directory_iterator(base_path)) {	×
208	std::cout << "Checking " << entry.path().string() << " with full path " << full_path << std::endl;	×
209	if (std::regex_match(entry.path().string(), regex_pattern)) {	×
210	std::cout << "Loading file " << entry.path().string() << std::endl;	×
UNCOV 211	return entry.path().string();	×
212	}
213	}	×
UNCOV 214	return std::nullopt;	×
UNCOV 215	} else {	×
216	// Check if the file path is relative
217	if (!std::filesystem::path(file_path).is_absolute()) {	×
218	full_path = base_path / file_path;	×
219	}
220	// Check for the presence of the file
UNCOV 221	if (std::filesystem::exists(full_path)) {	×
UNCOV 222	std::cout << "Loading file " << full_path.string() << std::endl;	×
223	return full_path.string();	×
224	} else {
225	return std::nullopt;	×
226	}
227	}
228	}	×
229
UNCOV 230	bool checkRequiredFields(json const & item, std::vector<std::string> const & requiredFields) {	×
UNCOV 231	for (auto const & field: requiredFields) {	×
232	if (!item.contains(field)) {	×
UNCOV 233	std::cerr << "Error: Missing required field \"" << field << "\" in JSON item." << std::endl;	×
UNCOV 234	return false;	×
235	}
236	}
237	return true;	×
238	}
239
UNCOV 240	void checkOptionalFields(json const & item, std::vector<std::string> const & optionalFields) {	×
241	for (auto const & field: optionalFields) {	×
UNCOV 242	if (!item.contains(field)) {	×
243	std::cout << "Warning: Optional field \"" << field << "\" is missing in JSON item." << std::endl;	×
244	}
245	}
246	}	×
247
248	DM_DataType stringToDataType(std::string const & data_type_str) {	×
249	if (data_type_str == "video") return DM_DataType::Video;	×
250	if (data_type_str == "points") return DM_DataType::Points;	×
251	if (data_type_str == "mask") return DM_DataType::Mask;	×
252	if (data_type_str == "line") return DM_DataType::Line;	×
253	if (data_type_str == "analog") return DM_DataType::Analog;	×
UNCOV 254	if (data_type_str == "digital_event") return DM_DataType::DigitalEvent;	×
UNCOV 255	if (data_type_str == "digital_interval") return DM_DataType::DigitalInterval;	×
256	if (data_type_str == "tensor") return DM_DataType::Tensor;	×
257	if (data_type_str == "time") return DM_DataType::Time;	×
UNCOV 258	return DM_DataType::Unknown;	×
259	}
260
261	std::vector<DataInfo> load_data_from_json_config(DataManager * dm, std::string const & json_filepath) {	×
262	std::vector<DataInfo> data_info_list;	×
263	// Open JSON file
UNCOV 264	std::ifstream ifs(json_filepath);	×
UNCOV 265	if (!ifs.is_open()) {	×
266	std::cerr << "Failed to open JSON file: " << json_filepath << std::endl;	×
267	return data_info_list;	×
268	}
269
270	// Parse JSON
UNCOV 271	json j;	×
UNCOV 272	ifs >> j;	×
273
274	// get base path of filepath
275	std::filesystem::path const base_path = std::filesystem::path(json_filepath).parent_path();	×
276
277	// Iterate through JSON array
UNCOV 278	for (auto const & item: j) {	×
279
280	if (!checkRequiredFields(item, {"data_type", "name", "filepath"})) {	×
281	continue;// Exit if any required field is missing	×
282	}
283
UNCOV 284	std::string const data_type_str = item["data_type"];	×
UNCOV 285	auto const data_type = stringToDataType(data_type_str);	×
286	if (data_type == DM_DataType::Unknown) {	×
UNCOV 287	std::cout << "Unknown data type: " << data_type_str << std::endl;	×
288	continue;	×
289	}
290
291	std::string const name = item["name"];	×
292
UNCOV 293	auto file_exists = processFilePath(item["filepath"], base_path);	×
294	if (!file_exists) {	×
UNCOV 295	std::cerr << "File does not exist: " << item["filepath"] << std::endl;	×
296	continue;	×
297	}
298
299	std::string const file_path = file_exists.value();	×
300
UNCOV 301	switch (data_type) {	×
302	case DM_DataType::Video: {	×
303
304	auto video_data = load_video_into_VideoData(file_path);	×
305	dm->setData<VideoData>("media", video_data);	×
306
307	data_info_list.push_back({name, "VideoData", ""});	×
UNCOV 308	break;	×
309	}	×
UNCOV 310	case DM_DataType::Points: {	×
311
312	auto point_data = load_into_PointData(file_path, item);	×
313
314	dm->setData<PointData>(name, point_data);	×
315
UNCOV 316	std::string const color = item.value("color", "#0000FF");	×
317	data_info_list.push_back({name, "PointData", color});	×
UNCOV 318	break;	×
319	}	×
320	case DM_DataType::Mask: {	×
321
322	auto mask_data = load_into_MaskData(file_path, item);	×
323
324	std::string const color = item.value("color", "0000FF");	×
UNCOV 325	dm->setData<MaskData>(name, mask_data);	×
326
UNCOV 327	data_info_list.push_back({name, "MaskData", color});	×
328
UNCOV 329	if (item.contains("operations")) {	×
330
331	for (auto const & operation: item["operations"]) {	×
332
333	std::string const operation_type = operation["type"];	×
334
335	if (operation_type == "area") {	×
336	std::cout << "Calculating area for mask: " << name << std::endl;	×
UNCOV 337	auto area_data = area(dm->getData<MaskData>(name).get());	×
338	std::string const output_name = name + "_area";	×
339	dm->setData<AnalogTimeSeries>(output_name, area_data);	×
340	}	×
UNCOV 341	}	×
342	}
UNCOV 343	break;	×
UNCOV 344	}	×
345	case DM_DataType::Line: {	×
346
UNCOV 347	auto line_data = load_into_LineData(file_path, item);	×
348
UNCOV 349	dm->setData<LineData>(name, line_data);	×
350
UNCOV 351	std::string const color = item.value("color", "0000FF");	×
352
UNCOV 353	data_info_list.push_back({name, "LineData", color});	×
354
UNCOV 355	break;	×
356	}	×
357	case DM_DataType::Analog: {	×
358
UNCOV 359	auto analog_time_series = load_into_AnalogTimeSeries(file_path, item);	×
360
UNCOV 361	for (int channel = 0; channel < analog_time_series.size(); channel++) {	×
362	std::string const channel_name = name + "_" + std::to_string(channel);	×
363
UNCOV 364	dm->setData<AnalogTimeSeries>(channel_name, analog_time_series[channel]);	×
365
UNCOV 366	if (item.contains("clock")) {	×
367	std::string const clock = item["clock"];	×
368	dm->setTimeFrame(channel_name, clock);	×
369	}	×
370	}	×
371	break;	×
372	}	×
373	case DM_DataType::DigitalEvent: {	×
374
UNCOV 375	auto digital_event_series = load_into_DigitalEventSeries(file_path, item);	×
376
UNCOV 377	for (int channel = 0; channel < digital_event_series.size(); channel++) {	×
378	std::string const channel_name = name + "_" + std::to_string(channel);	×
379
UNCOV 380	dm->setData<DigitalEventSeries>(channel_name, digital_event_series[channel]);	×
381
UNCOV 382	if (item.contains("clock")) {	×
383	std::string const clock = item["clock"];	×
384	dm->setTimeFrame(channel_name, clock);	×
385	}	×
386	}	×
387	break;	×
388	}	×
389	case DM_DataType::DigitalInterval: {	×
390
UNCOV 391	auto digital_interval_series = load_into_DigitalIntervalSeries(file_path, item);	×
392	dm->setData<DigitalIntervalSeries>(name, digital_interval_series);	×
393
UNCOV 394	break;	×
395	}	×
396	case DM_DataType::Tensor: {	×
397
UNCOV 398	if (item["format"] == "numpy") {	×
399
UNCOV 400	TensorData tensor_data;	×
401	loadNpyToTensorData(file_path, tensor_data);	×
402
UNCOV 403	dm->setData<TensorData>(name, std::make_shared<TensorData>(tensor_data));	×
404
UNCOV 405	} else {	×
406	std::cout << "Format " << item["format"] << " not found for " << name << std::endl;	×
407	}
UNCOV 408	break;	×
409	}
UNCOV 410	case DM_DataType::Time: {	×
411
UNCOV 412	if (item["format"] == "uint16") {	×
413
UNCOV 414	int const channel = item["channel"];	×
415	std::string const transition = item["transition"];	×
416
UNCOV 417	int const header_size = item.value("header_size", 0);	×
418
UNCOV 419	auto opts = Loader::BinaryAnalogOptions{.file_path = file_path,	×
420	.header_size_bytes = static_cast<size_t>(header_size)};	×
421	auto data = readBinaryFile<uint16_t>(opts);	×
422
UNCOV 423	auto digital_data = Loader::extractDigitalData(data, channel);	×
424	auto events = Loader::extractEvents(digital_data, transition);	×
425
426	// convert to int with std::transform
UNCOV 427	std::vector<int> events_int;	×
428	events_int.reserve(events.size());	×
429	for (auto e: events) {	×
430	events_int.push_back(static_cast<int>(e));	×
431	}
UNCOV 432	std::cout << "Loaded " << events_int.size() << " events for " << name << std::endl;	×
433
UNCOV 434	auto timeframe = std::make_shared<TimeFrame>(events_int);	×
435	dm->setTime(name, timeframe, true);	×
436	}	×
437
UNCOV 438	if (item["format"] == "uint16_length") {	×
439
UNCOV 440	int const header_size = item.value("header_size", 0);	×
441
UNCOV 442	auto opts = Loader::BinaryAnalogOptions{.file_path = file_path,	×
443	.header_size_bytes = static_cast<size_t>(header_size)};	×
444	auto data = readBinaryFile<uint16_t>(opts);	×
445
UNCOV 446	std::vector<int> t(data.size());	×
447	std::iota(std::begin(t), std::end(t), 0);	×
448
UNCOV 449	std::cout << "Total of " << t.size() << " timestamps for " << name << std::endl;	×
450
UNCOV 451	auto timeframe = std::make_shared<TimeFrame>(t);	×
452	dm->setTime(name, timeframe, true);	×
453	}	×
454	break;	×
455	}
UNCOV 456	default:	×
457	std::cout << "Unsupported data type: " << data_type_str << std::endl;	×
458	continue;	×
459	}	×
460	if (item.contains("clock")) {	×
461	std::string const clock = item["clock"];	×
462	std::cout << "Setting time for " << name << " to " << clock << std::endl;	×
463	dm->setTimeFrame(name, clock);	×
464	}	×
465	}	×
466
467	return data_info_list;
UNCOV 468	}	×
469
UNCOV 470	DM_DataType DataManager::getType(std::string const & key) const {	×
471	auto it = _data.find(key);	×
472	if (it != _data.end()) {	×
473	if (std::holds_alternative<std::shared_ptr<MediaData>>(it->second)) {	×
474	return DM_DataType::Video;	×
475	} else if (std::holds_alternative<std::shared_ptr<PointData>>(it->second)) {	×
476	return DM_DataType::Points;	×
477	} else if (std::holds_alternative<std::shared_ptr<LineData>>(it->second)) {	×
478	return DM_DataType::Line;	×
479	} else if (std::holds_alternative<std::shared_ptr<MaskData>>(it->second)) {	×
480	return DM_DataType::Mask;	×
481	} else if (std::holds_alternative<std::shared_ptr<AnalogTimeSeries>>(it->second)) {	×
482	return DM_DataType::Analog;	×
483	} else if (std::holds_alternative<std::shared_ptr<DigitalEventSeries>>(it->second)) {	×
484	return DM_DataType::DigitalEvent;	×
485	} else if (std::holds_alternative<std::shared_ptr<DigitalIntervalSeries>>(it->second)) {	×
486	return DM_DataType::DigitalInterval;	×
487	} else if (std::holds_alternative<std::shared_ptr<TensorData>>(it->second)) {	×
488	return DM_DataType::Tensor;	×
489	}
UNCOV 490	return DM_DataType::Unknown;	×
491	}
UNCOV 492	return DM_DataType::Unknown;	×
493	}
494
UNCOV 495	std::string convert_data_type_to_string(DM_DataType type) {	×
496	switch (type) {	×
497	case DM_DataType::Video:	×
498	return "video";	×
499	case DM_DataType::Points:	×
500	return "points";	×
501	case DM_DataType::Mask:	×
502	return "mask";	×
503	case DM_DataType::Line:	×
504	return "line";	×
505	case DM_DataType::Analog:	×
506	return "analog";	×
507	case DM_DataType::DigitalEvent:	×
508	return "digital_event";	×
509	case DM_DataType::DigitalInterval:	×
510	return "digital_interval";	×
511	case DM_DataType::Tensor:	×
512	return "tensor";	×
513	case DM_DataType::Time:	×
514	return "time";	×
515	default:	×
516	return "unknown";	×
517	}
518	}

paulmthompson / WhiskerToolbox / 15430655463

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous