14313763633

Committed 07 Apr 2025 03:53PM UTC coverage: 14.375% (-0.09%) from 14.464%

Build # 14313763633

Build Type

push

github

Committed by

paulmthompson

Commit Message

tests should use new options format

Coverage Stats

208 of 1447 relevant lines covered (14.37%)

1.4 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

3.37

/src/WhiskerToolbox/DataManager/DataManager.cpp


#include "DataManager.hpp"
#include "AnalogTimeSeries/Analog_Time_Series.hpp"
#include "DigitalTimeSeries/Digital_Event_Series.hpp"
#include "DigitalTimeSeries/Digital_Interval_Series.hpp"
#include "Lines/Line_Data.hpp"
#include "Masks/Mask_Data.hpp"
#include "Media/Video_Data.hpp"
#include "Points/Point_Data.hpp"
#include "Tensors/Tensor_Data.hpp"

#include "loaders/CSV_Loaders.hpp"
#include "loaders/binary_loaders.hpp"
#include "transforms/data_transforms.hpp"

#include "TimeFrame.hpp"

#include "utils/container.hpp"
#include "utils/glob.hpp"
#include "utils/hdf5_mask_load.hpp"

#include "nlohmann/json.hpp"
#include "utils/string_manip.hpp"

#include <filesystem>
#include <fstream>
#include <iostream>
#include <optional>
#include <regex>

using namespace nlohmann;

DataManager::DataManager() {
    _times["time"] = std::make_shared<TimeFrame>();
    _data["media"] = std::make_shared<MediaData>();

    setTimeFrame("media", "time");
    _output_path = std::filesystem::current_path();
}

void DataManager::setTimeFrame(std::string const & data_key, std::string const & time_key) {
    //Check that data_key is in _data
    if (_data.find(data_key) == _data.end()) {
        std::cerr << "Data key not found in DataManager: " << data_key << std::endl;
        return;
    }

    //Check that time_key is in _times
    if (_times.find(time_key) == _times.end()) {
        std::cerr << "Time key not found in DataManager: " << time_key << std::endl;
        return;
    }

    _time_frames[data_key] = time_key;
}

std::vector<std::vector<float>> read_ragged_hdf5(std::string const & filepath, std::string const & key) {
    auto myvector = load_ragged_array<float>(filepath, key);
    return myvector;
}

std::vector<int> read_array_hdf5(std::string const & filepath, std::string const & key) {
    auto myvector = load_array<int>(filepath, key);
    return myvector;
}

std::optional<std::string> processFilePath(
        std::string const & file_path,
        std::filesystem::path const & base_path) {
    std::filesystem::path full_path = file_path;

    // Check for wildcard character
    if (file_path.find('*') != std::string::npos) {
        // Convert wildcard pattern to regex
        std::string const pattern = std::regex_replace(full_path.string(), std::regex("\\*"), ".*");
        std::regex const regex_pattern(pattern);

        // Iterate through the directory to find matching files
        for (auto const & entry: std::filesystem::directory_iterator(base_path)) {
            std::cout << "Checking " << entry.path().string() << " with full path " << full_path << std::endl;
            if (std::regex_match(entry.path().string(), regex_pattern)) {
                std::cout << "Loading file " << entry.path().string() << std::endl;
                return entry.path().string();
            }
        }
        return std::nullopt;
    } else {
        // Check if the file path is relative
        if (!std::filesystem::path(file_path).is_absolute()) {
            full_path = base_path / file_path;
        }
        // Check for the presence of the file
        if (std::filesystem::exists(full_path)) {
            std::cout << "Loading file " << full_path.string() << std::endl;
            return full_path.string();
        } else {
            return std::nullopt;
        }
    }
}

bool checkRequiredFields(json const & item, std::vector<std::string> const & requiredFields) {
    for (auto const & field: requiredFields) {
        if (!item.contains(field)) {
            std::cerr << "Error: Missing required field \"" << field << "\" in JSON item." << std::endl;
            return false;
        }
    }
    return true;
}

int DataManager::addCallbackToData(std::string const & key, ObserverCallback callback) {

    int id = -1;

    if (_data.find(key) != _data.end()) {
        auto data = _data[key];

        id = std::visit([callback](auto & x) {
            return x.get()->addObserver(callback);
        },
                        data);
    }

    return id;
}

void DataManager::removeCallbackFromData(std::string const & key, int callback_id) {
    if (_data.find(key) != _data.end()) {
        auto data = _data[key];

        std::visit([callback_id](auto & x) {
            x.get()->removeObserver(callback_id);
        },
                   data);
    }
}

void checkOptionalFields(json const & item, std::vector<std::string> const & optionalFields) {
    for (auto const & field: optionalFields) {
        if (!item.contains(field)) {
            std::cout << "Warning: Optional field \"" << field << "\" is missing in JSON item." << std::endl;
        }
    }
}


Loader::BinaryAnalogOptions createBinaryAnalogOptions(std::string const & file_path, nlohmann::basic_json<> const & item) {

    int const header_size = item.value("header_size", 0);
    int const num_channels = item.value("channel_count", 1);

    auto opts = Loader::BinaryAnalogOptions{
            .file_path = file_path,
            .header_size_bytes = static_cast<size_t>(header_size),
            .num_channels = static_cast<size_t>(num_channels)};

    return opts;
}

std::vector<DataInfo> load_data_from_json_config(DataManager * dm, std::string const & json_filepath) {
    std::vector<DataInfo> data_info_list;
    // Open JSON file
    std::ifstream ifs(json_filepath);
    if (!ifs.is_open()) {
        std::cerr << "Failed to open JSON file: " << json_filepath << std::endl;
        return data_info_list;
    }

    // Parse JSON
    json j;
    ifs >> j;

    // get base path of filepath
    std::filesystem::path const base_path = std::filesystem::path(json_filepath).parent_path();

    // Iterate through JSON array
    for (auto const & item: j) {

        if (!checkRequiredFields(item, {"data_type", "name", "filepath"})) {
            continue;// Exit if any required field is missing
        }

        std::string const data_type = item["data_type"];
        std::string const name = item["name"];

        auto file_exists = processFilePath(item["filepath"], base_path);
        if (!file_exists) {
            std::cerr << "File does not exist: " << item["filepath"] << std::endl;
            continue;
        }

        std::string const file_path = file_exists.value();

        if (data_type == "video") {
            // Create VideoData object
            auto video_data = std::make_shared<VideoData>();

            video_data->LoadMedia(file_path);

            std::cout << "Video has " << video_data->getTotalFrameCount() << " frames" << std::endl;
            // Add VideoData to DataManager
            dm->setMedia(video_data);

            data_info_list.push_back({name, "VideoData", ""});

        } else if (data_type == "points") {

            int const frame_column = item["frame_column"];
            int const x_column = item["x_column"];
            int const y_column = item["y_column"];

            std::string const color = item.value("color", "#0000FF");
            std::string const delim = item.value("delim", " ");

            int const height = item.value("height", -1);
            int const width = item.value("width", -1);

            int const scaled_height = item.value("scale_to_height", -1);
            int const scaled_width = item.value("scale_to_width", -1);

            auto keypoints = load_points_from_csv(
                    file_path,
                    frame_column,
                    x_column,
                    y_column,
                    delim.c_str()[0]);

            std::cout << "There are " << keypoints.size() << " keypoints " << std::endl;

            auto point_data = std::make_shared<PointData>(keypoints);
            point_data->setImageSize(ImageSize{width, height});

            if (scaled_height > 0 && scaled_width > 0) {
                scale(point_data, ImageSize{scaled_width, scaled_height});
            }

            dm->setData<PointData>(name, point_data);

            data_info_list.push_back({name, "PointData", color});

        } else if (data_type == "mask") {

            std::string const frame_key = item["frame_key"];
            std::string const prob_key = item["probability_key"];
            std::string const x_key = item["x_key"];
            std::string const y_key = item["y_key"];

            int const height = item.value("height", -1);
            int const width = item.value("width", -1);

            std::string const color = item.value("color", "0000FF");

            auto frames = read_array_hdf5(file_path, frame_key);
            auto probs = read_ragged_hdf5(file_path, prob_key);
            auto y_coords = read_ragged_hdf5(file_path, y_key);
            auto x_coords = read_ragged_hdf5(file_path, x_key);

            auto mask_data = std::make_shared<MaskData>();
            mask_data->setImageSize(ImageSize{width, height});

            for (std::size_t i = 0; i < frames.size(); i++) {
                auto frame = frames[i];
                auto prob = probs[i];
                auto x = x_coords[i];
                auto y = y_coords[i];
                mask_data->addMaskAtTime(frame, x, y);
            }

            dm->setData<MaskData>(name, mask_data);

            data_info_list.push_back({name, "MaskData", color});

            if (item.contains("operations")) {

                for (auto const & operation: item["operations"]) {

                    std::string const operation_type = operation["type"];

                    if (operation_type == "area") {
                        std::cout << "Calculating area for mask: " << name << std::endl;
                        auto area_data = area(dm->getData<MaskData>(name));
                        std::string const output_name = name + "_area";
                        dm->setData<AnalogTimeSeries>(output_name, area_data);
                    }
                }
            }
        } else if (data_type == "line") {

            auto line_map = load_line_csv(file_path);

            //Get the whisker name from the filename using filesystem
            auto whisker_filename = std::filesystem::path(file_path).filename().string();

            //Remove .csv suffix from filename
            auto whisker_name = remove_extension(whisker_filename);

            dm->setData<LineData>(whisker_name, std::make_shared<LineData>(line_map));

            std::string const color = item.value("color", "0000FF");

            data_info_list.push_back({name, "LineData", color});

        } else if (data_type == "analog") {

            if (item["format"] == "int16") {

                auto opts = createBinaryAnalogOptions(file_path, item);

                if (opts.num_channels > 1) {

                    auto data = readBinaryFileMultiChannel<int16_t>(opts);

                    std::cout << "Read " << data.size() << " channels" << std::endl;

                    for (int channel = 0; channel < data.size(); channel++) {
                        // convert to float with std::transform
                        std::vector<float> data_float;
                        std::transform(
                                data[channel].begin(),
                                data[channel].end(),
                                std::back_inserter(data_float), [](int16_t i) { return i; });

                        auto analog_time_series = std::make_shared<AnalogTimeSeries>();
                        analog_time_series->setData(data_float);

                        std::string const channel_name = name + "_" + std::to_string(channel);

                        dm->setData<AnalogTimeSeries>(channel_name, analog_time_series);

                        if (item.contains("clock")) {
                            std::string const clock = item["clock"];
                            dm->setTimeFrame(channel_name, clock);
                        }
                    }

                } else {

                    auto data = readBinaryFile<int16_t>(opts);

                    // convert to float with std::transform
                    std::vector<float> data_float;
                    std::transform(
                            data.begin(),
                            data.end(),
                            std::back_inserter(data_float), [](int16_t i) { return i; });

                    auto analog_time_series = std::make_shared<AnalogTimeSeries>();
                    analog_time_series->setData(data_float);
                    dm->setData<AnalogTimeSeries>(name, analog_time_series);
                }

            } else {
                std::cout << "Format " << item["format"] << " not found for " << name << std::endl;
            }

        } else if (data_type == "digital_event") {

            if (item["format"] == "uint16") {

                int const channel = item["channel"];
                std::string const transition = item["transition"];

                auto opts = createBinaryAnalogOptions(file_path, item);
                auto data = readBinaryFile<uint16_t>(opts);

                auto digital_data = Loader::extractDigitalData(data, channel);
                auto events = Loader::extractEvents(digital_data, transition);
                std::cout << "Loaded " << events.size() << " events for " << name << std::endl;

                auto digital_event_series = std::make_shared<DigitalEventSeries>();
                digital_event_series->setData(events);
                dm->setData<DigitalEventSeries>(name, digital_event_series);
            } else if (item["format"] == "csv") {

                auto opts = Loader::CSVSingleColumnOptions{.filename = file_path};

                auto events = Loader::loadSingleColumnCSV(opts);
                std::cout << "Loaded " << events.size() << " events for " << name << std::endl;

                float const scale = item.value("scale", 1.0f);
                bool const scale_divide = item.value("scale_divide", false);

                if (scale_divide) {
                    for (auto & e: events) {
                        e /= scale;
                    }
                } else {
                    for (auto & e: events) {
                        e *= scale;
                    }
                }

                auto digital_event_series = std::make_shared<DigitalEventSeries>();
                digital_event_series->setData(events);
                dm->setData<DigitalEventSeries>(name, digital_event_series);

            } else {
                std::cout << "Format " << item["format"] << " not found for " << name << std::endl;
            }
        } else if (data_type == "digital_interval") {

            if (item["format"] == "uint16") {

                int const channel = item["channel"];
                std::string const transition = item["transition"];

                auto opts = createBinaryAnalogOptions(file_path, item);
                auto data = readBinaryFile<uint16_t>(opts);

                auto digital_data = Loader::extractDigitalData(data, channel);

                auto intervals = Loader::extractIntervals(digital_data, transition);
                std::cout << "Loaded " << intervals.size() << " intervals for " << name << std::endl;

                auto digital_interval_series = std::make_shared<DigitalIntervalSeries>();
                digital_interval_series->setData(intervals);
                dm->setData<DigitalIntervalSeries>(name, digital_interval_series);

            } else if (item["format"] == "csv") {

                auto opts = Loader::CSVMultiColumnOptions{.filename = file_path};

                auto intervals = Loader::loadPairColumnCSV(opts);
                std::cout << "Loaded " << intervals.size() << " intervals for " << name << std::endl;
                auto digital_interval_series = std::make_shared<DigitalIntervalSeries>();
                digital_interval_series->setData(intervals);
                dm->setData<DigitalIntervalSeries>(name, digital_interval_series);
            } else {
                std::cout << "Format " << item["format"] << " not found for " << name << std::endl;
            }
        } else if (data_type == "tensor") {

            if (item["format"] == "numpy") {

                TensorData tensor_data;
                loadNpyToTensorData(file_path, tensor_data);

                dm->setData<TensorData>(name, std::make_shared<TensorData>(tensor_data));

            } else {
                std::cout << "Format " << item["format"] << " not found for " << name << std::endl;
            }

        } else if (data_type == "time") {

            if (item["format"] == "uint16") {

                int const channel = item["channel"];
                std::string const transition = item["transition"];

                auto opts = createBinaryAnalogOptions(file_path, item);
                auto data = readBinaryFile<uint16_t>(opts);

                auto digital_data = Loader::extractDigitalData(data, channel);
                auto events = Loader::extractEvents(digital_data, transition);

                // convert to int with std::transform
                std::vector<int> events_int;
                events_int.reserve(events.size());
                for (auto e: events) {
                    events_int.push_back(static_cast<int>(e));
                }
                std::cout << "Loaded " << events_int.size() << " events for " << name << std::endl;

                auto timeframe = std::make_shared<TimeFrame>(events_int);
                dm->setTime(name, timeframe);
            }

            if (item["format"] == "uint16_length") {

                auto opts = createBinaryAnalogOptions(file_path, item);
                auto data = readBinaryFile<uint16_t>(opts);

                std::vector<int> t(data.size());
                std::iota(std::begin(t), std::end(t), 0);

                std::cout << "Total of " << t.size() << " timestamps for " << name << std::endl;

                auto timeframe = std::make_shared<TimeFrame>(t);
                dm->setTime(name, timeframe);
            }
        }
        if (item.contains("clock")) {
            std::string const clock = item["clock"];
            std::cout << "Setting time for " << name << " to " << clock << std::endl;
            dm->setTimeFrame(name, clock);
        }
    }

    return data_info_list;
}

std::string DataManager::getType(std::string const & key) const {
    auto it = _data.find(key);
    if (it != _data.end()) {
        if (std::holds_alternative<std::shared_ptr<MediaData>>(it->second)) {
            return "MediaData";
        } else if (std::holds_alternative<std::shared_ptr<PointData>>(it->second)) {
            return "PointData";
        } else if (std::holds_alternative<std::shared_ptr<LineData>>(it->second)) {
            return "LineData";
        } else if (std::holds_alternative<std::shared_ptr<MaskData>>(it->second)) {
            return "MaskData";
        } else if (std::holds_alternative<std::shared_ptr<AnalogTimeSeries>>(it->second)) {
            return "AnalogTimeSeries";
        } else if (std::holds_alternative<std::shared_ptr<DigitalEventSeries>>(it->second)) {
            return "DigitalEventSeries";
        } else if (std::holds_alternative<std::shared_ptr<DigitalIntervalSeries>>(it->second)) {
            return "DigitalIntervalSeries";
        } else if (std::holds_alternative<std::shared_ptr<TensorData>>(it->second)) {
            return "TensorData";
        }
        return "Unknown";
    }
}

1
2	#include "DataManager.hpp"
3	#include "AnalogTimeSeries/Analog_Time_Series.hpp"
4	#include "DigitalTimeSeries/Digital_Event_Series.hpp"
5	#include "DigitalTimeSeries/Digital_Interval_Series.hpp"
6	#include "Lines/Line_Data.hpp"
7	#include "Masks/Mask_Data.hpp"
8	#include "Media/Video_Data.hpp"
9	#include "Points/Point_Data.hpp"
10	#include "Tensors/Tensor_Data.hpp"
11
12	#include "loaders/CSV_Loaders.hpp"
13	#include "loaders/binary_loaders.hpp"
14	#include "transforms/data_transforms.hpp"
15
16	#include "TimeFrame.hpp"
17
18	#include "utils/container.hpp"
19	#include "utils/glob.hpp"
20	#include "utils/hdf5_mask_load.hpp"
21
22	#include "nlohmann/json.hpp"
23	#include "utils/string_manip.hpp"
24
25	#include <filesystem>
26	#include <fstream>
27	#include <iostream>
28	#include <optional>
29	#include <regex>
30
31	using namespace nlohmann;
32
33	DataManager::DataManager() {	2✔
34	_times["time"] = std::make_shared<TimeFrame>();	6✔
35	_data["media"] = std::make_shared<MediaData>();	6✔
36
37	setTimeFrame("media", "time");	10✔
38	_output_path = std::filesystem::current_path();	2✔
39	}	2✔
40
41	void DataManager::setTimeFrame(std::string const & data_key, std::string const & time_key) {	2✔
42	//Check that data_key is in _data
43	if (_data.find(data_key) == _data.end()) {	2✔
44	std::cerr << "Data key not found in DataManager: " << data_key << std::endl;	×
45	return;	×
46	}
47
48	//Check that time_key is in _times
49	if (_times.find(time_key) == _times.end()) {	2✔
50	std::cerr << "Time key not found in DataManager: " << time_key << std::endl;	×
51	return;	×
52	}
53
54	_time_frames[data_key] = time_key;	2✔
55	}
56
57	std::vector<std::vector<float>> read_ragged_hdf5(std::string const & filepath, std::string const & key) {	×
58	auto myvector = load_ragged_array<float>(filepath, key);	×
59	return myvector;	×
60	}
61
62	std::vector<int> read_array_hdf5(std::string const & filepath, std::string const & key) {	×
63	auto myvector = load_array<int>(filepath, key);	×
64	return myvector;	×
65	}
66
67	std::optional<std::string> processFilePath(	×
68	std::string const & file_path,
69	std::filesystem::path const & base_path) {
70	std::filesystem::path full_path = file_path;	×
71
72	// Check for wildcard character
73	if (file_path.find('*') != std::string::npos) {	×
74	// Convert wildcard pattern to regex
75	std::string const pattern = std::regex_replace(full_path.string(), std::regex("\\"), ".");	×
76	std::regex const regex_pattern(pattern);	×
77
78	// Iterate through the directory to find matching files
79	for (auto const & entry: std::filesystem::directory_iterator(base_path)) {	×
80	std::cout << "Checking " << entry.path().string() << " with full path " << full_path << std::endl;	×
81	if (std::regex_match(entry.path().string(), regex_pattern)) {	×
82	std::cout << "Loading file " << entry.path().string() << std::endl;	×
83	return entry.path().string();	×
84	}
85	}	×
86	return std::nullopt;	×
87	} else {	×
88	// Check if the file path is relative
89	if (!std::filesystem::path(file_path).is_absolute()) {	×
90	full_path = base_path / file_path;	×
91	}
92	// Check for the presence of the file
93	if (std::filesystem::exists(full_path)) {	×
94	std::cout << "Loading file " << full_path.string() << std::endl;	×
95	return full_path.string();	×
96	} else {
97	return std::nullopt;	×
98	}
99	}
100	}	×
101
102	bool checkRequiredFields(json const & item, std::vector<std::string> const & requiredFields) {	×
103	for (auto const & field: requiredFields) {	×
104	if (!item.contains(field)) {	×
105	std::cerr << "Error: Missing required field \"" << field << "\" in JSON item." << std::endl;	×
106	return false;	×
107	}
108	}
109	return true;	×
110	}
111
112	int DataManager::addCallbackToData(std::string const & key, ObserverCallback callback) {	×
113
114	int id = -1;	×
115
116	if (_data.find(key) != _data.end()) {	×
117	auto data = _data[key];	×
118
119	id = std::visit([callback](auto & x) {	×
120	return x.get()->addObserver(callback);	×
121	},
122	data);
123	}	×
124
125	return id;	×
126	}
127
128	void DataManager::removeCallbackFromData(std::string const & key, int callback_id) {	×
129	if (_data.find(key) != _data.end()) {	×
130	auto data = _data[key];	×
131
132	std::visit([callback_id](auto & x) {	×
133	x.get()->removeObserver(callback_id);	×
134	},	×
135	data);
136	}	×
137	}	×
138
139	void checkOptionalFields(json const & item, std::vector<std::string> const & optionalFields) {	×
140	for (auto const & field: optionalFields) {	×
141	if (!item.contains(field)) {	×
142	std::cout << "Warning: Optional field \"" << field << "\" is missing in JSON item." << std::endl;	×
143	}
144	}
145	}	×
146
147
148	Loader::BinaryAnalogOptions createBinaryAnalogOptions(std::string const & file_path, nlohmann::basic_json<> const & item) {	×
149
150	int const header_size = item.value("header_size", 0);	×
151	int const num_channels = item.value("channel_count", 1);	×
152
153	auto opts = Loader::BinaryAnalogOptions{
154	.file_path = file_path,
155	.header_size_bytes = static_cast<size_t>(header_size),	×
156	.num_channels = static_cast<size_t>(num_channels)};	×
157
158	return opts;	×
159	}
160
161	std::vector<DataInfo> load_data_from_json_config(DataManager * dm, std::string const & json_filepath) {	×
162	std::vector<DataInfo> data_info_list;	×
163	// Open JSON file
164	std::ifstream ifs(json_filepath);	×
165	if (!ifs.is_open()) {	×
166	std::cerr << "Failed to open JSON file: " << json_filepath << std::endl;	×
167	return data_info_list;	×
168	}
169
170	// Parse JSON
171	json j;	×
172	ifs >> j;	×
173
174	// get base path of filepath
175	std::filesystem::path const base_path = std::filesystem::path(json_filepath).parent_path();	×
176
177	// Iterate through JSON array
178	for (auto const & item: j) {	×
179
180	if (!checkRequiredFields(item, {"data_type", "name", "filepath"})) {	×
181	continue;// Exit if any required field is missing	×
182	}
183
184	std::string const data_type = item["data_type"];	×
185	std::string const name = item["name"];	×
186
187	auto file_exists = processFilePath(item["filepath"], base_path);	×
188	if (!file_exists) {	×
189	std::cerr << "File does not exist: " << item["filepath"] << std::endl;	×
190	continue;	×
191	}
192
193	std::string const file_path = file_exists.value();	×
194
195	if (data_type == "video") {	×
196	// Create VideoData object
197	auto video_data = std::make_shared<VideoData>();	×
198
199	video_data->LoadMedia(file_path);	×
200
201	std::cout << "Video has " << video_data->getTotalFrameCount() << " frames" << std::endl;	×
202	// Add VideoData to DataManager
203	dm->setMedia(video_data);	×
204
205	data_info_list.push_back({name, "VideoData", ""});	×
206
207	} else if (data_type == "points") {	×
208
209	int const frame_column = item["frame_column"];	×
210	int const x_column = item["x_column"];	×
211	int const y_column = item["y_column"];	×
212
213	std::string const color = item.value("color", "#0000FF");	×
214	std::string const delim = item.value("delim", " ");	×
215
216	int const height = item.value("height", -1);	×
217	int const width = item.value("width", -1);	×
218
219	int const scaled_height = item.value("scale_to_height", -1);	×
220	int const scaled_width = item.value("scale_to_width", -1);	×
221
222	auto keypoints = load_points_from_csv(	×
223	file_path,
224	frame_column,
225	x_column,
226	y_column,
227	delim.c_str()[0]);	×
228
229	std::cout << "There are " << keypoints.size() << " keypoints " << std::endl;	×
230
231	auto point_data = std::make_shared<PointData>(keypoints);	×
232	point_data->setImageSize(ImageSize{width, height});	×
233
234	if (scaled_height > 0 && scaled_width > 0) {	×
235	scale(point_data, ImageSize{scaled_width, scaled_height});	×
236	}
237
238	dm->setData<PointData>(name, point_data);	×
239
240	data_info_list.push_back({name, "PointData", color});	×
241
242	} else if (data_type == "mask") {	×
243
244	std::string const frame_key = item["frame_key"];	×
245	std::string const prob_key = item["probability_key"];	×
246	std::string const x_key = item["x_key"];	×
247	std::string const y_key = item["y_key"];	×
248
249	int const height = item.value("height", -1);	×
250	int const width = item.value("width", -1);	×
251
252	std::string const color = item.value("color", "0000FF");	×
253
254	auto frames = read_array_hdf5(file_path, frame_key);	×
255	auto probs = read_ragged_hdf5(file_path, prob_key);	×
256	auto y_coords = read_ragged_hdf5(file_path, y_key);	×
257	auto x_coords = read_ragged_hdf5(file_path, x_key);	×
258
259	auto mask_data = std::make_shared<MaskData>();	×
260	mask_data->setImageSize(ImageSize{width, height});	×
261
262	for (std::size_t i = 0; i < frames.size(); i++) {	×
263	auto frame = frames[i];	×
264	auto prob = probs[i];	×
265	auto x = x_coords[i];	×
266	auto y = y_coords[i];	×
267	mask_data->addMaskAtTime(frame, x, y);	×
268	}	×
269
270	dm->setData<MaskData>(name, mask_data);	×
271
272	data_info_list.push_back({name, "MaskData", color});	×
273
274	if (item.contains("operations")) {	×
275
276	for (auto const & operation: item["operations"]) {	×
277
278	std::string const operation_type = operation["type"];	×
279
280	if (operation_type == "area") {	×
281	std::cout << "Calculating area for mask: " << name << std::endl;	×
282	auto area_data = area(dm->getData<MaskData>(name));	×
283	std::string const output_name = name + "_area";	×
284	dm->setData<AnalogTimeSeries>(output_name, area_data);	×
285	}	×
286	}	×
287	}
288	} else if (data_type == "line") {	×
289
290	auto line_map = load_line_csv(file_path);	×
291
292	//Get the whisker name from the filename using filesystem
293	auto whisker_filename = std::filesystem::path(file_path).filename().string();	×
294
295	//Remove .csv suffix from filename
296	auto whisker_name = remove_extension(whisker_filename);	×
297
298	dm->setData<LineData>(whisker_name, std::make_shared<LineData>(line_map));	×
299
300	std::string const color = item.value("color", "0000FF");	×
301
302	data_info_list.push_back({name, "LineData", color});	×
303
304	} else if (data_type == "analog") {	×
305
306	if (item["format"] == "int16") {	×
307
308	auto opts = createBinaryAnalogOptions(file_path, item);	×
309
310	if (opts.num_channels > 1) {	×
311
312	auto data = readBinaryFileMultiChannel<int16_t>(opts);	×
313
314	std::cout << "Read " << data.size() << " channels" << std::endl;	×
315
316	for (int channel = 0; channel < data.size(); channel++) {	×
317	// convert to float with std::transform
318	std::vector<float> data_float;	×
319	std::transform(	×
320	data[channel].begin(),	×
321	data[channel].end(),	×
322	std::back_inserter(data_float), [](int16_t i) { return i; });	×
323
324	auto analog_time_series = std::make_shared<AnalogTimeSeries>();	×
325	analog_time_series->setData(data_float);	×
326
327	std::string const channel_name = name + "_" + std::to_string(channel);	×
328
329	dm->setData<AnalogTimeSeries>(channel_name, analog_time_series);	×
330
331	if (item.contains("clock")) {	×
332	std::string const clock = item["clock"];	×
333	dm->setTimeFrame(channel_name, clock);	×
334	}	×
335	}	×
336
337	} else {	×
338
339	auto data = readBinaryFile<int16_t>(opts);	×
340
341	// convert to float with std::transform
342	std::vector<float> data_float;	×
343	std::transform(	×
344	data.begin(),
345	data.end(),
346	std::back_inserter(data_float), [](int16_t i) { return i; });	×
347
348	auto analog_time_series = std::make_shared<AnalogTimeSeries>();	×
349	analog_time_series->setData(data_float);	×
350	dm->setData<AnalogTimeSeries>(name, analog_time_series);	×
351	}	×
352
353	} else {	×
354	std::cout << "Format " << item["format"] << " not found for " << name << std::endl;	×
355	}
356
357	} else if (data_type == "digital_event") {	×
358
359	if (item["format"] == "uint16") {	×
360
361	int const channel = item["channel"];	×
362	std::string const transition = item["transition"];	×
363
364	auto opts = createBinaryAnalogOptions(file_path, item);	×
365	auto data = readBinaryFile<uint16_t>(opts);	×
366
367	auto digital_data = Loader::extractDigitalData(data, channel);	×
368	auto events = Loader::extractEvents(digital_data, transition);	×
369	std::cout << "Loaded " << events.size() << " events for " << name << std::endl;	×
370
371	auto digital_event_series = std::make_shared<DigitalEventSeries>();	×
372	digital_event_series->setData(events);	×
373	dm->setData<DigitalEventSeries>(name, digital_event_series);	×
374	} else if (item["format"] == "csv") {	×
375
376	auto opts = Loader::CSVSingleColumnOptions{.filename = file_path};	×
377
378	auto events = Loader::loadSingleColumnCSV(opts);	×
379	std::cout << "Loaded " << events.size() << " events for " << name << std::endl;	×
380
381	float const scale = item.value("scale", 1.0f);	×
382	bool const scale_divide = item.value("scale_divide", false);	×
383
384	if (scale_divide) {	×
385	for (auto & e: events) {	×
386	e /= scale;	×
387	}
388	} else {
389	for (auto & e: events) {	×
390	e *= scale;	×
391	}
392	}
393
394	auto digital_event_series = std::make_shared<DigitalEventSeries>();	×
395	digital_event_series->setData(events);	×
396	dm->setData<DigitalEventSeries>(name, digital_event_series);	×
397
398	} else {	×
399	std::cout << "Format " << item["format"] << " not found for " << name << std::endl;	×
400	}
401	} else if (data_type == "digital_interval") {	×
402
403	if (item["format"] == "uint16") {	×
404
405	int const channel = item["channel"];	×
406	std::string const transition = item["transition"];	×
407
408	auto opts = createBinaryAnalogOptions(file_path, item);	×
409	auto data = readBinaryFile<uint16_t>(opts);	×
410
411	auto digital_data = Loader::extractDigitalData(data, channel);	×
412
413	auto intervals = Loader::extractIntervals(digital_data, transition);	×
414	std::cout << "Loaded " << intervals.size() << " intervals for " << name << std::endl;	×
415
416	auto digital_interval_series = std::make_shared<DigitalIntervalSeries>();	×
417	digital_interval_series->setData(intervals);	×
418	dm->setData<DigitalIntervalSeries>(name, digital_interval_series);	×
419
420	} else if (item["format"] == "csv") {	×
421
422	auto opts = Loader::CSVMultiColumnOptions{.filename = file_path};	×
423
424	auto intervals = Loader::loadPairColumnCSV(opts);	×
425	std::cout << "Loaded " << intervals.size() << " intervals for " << name << std::endl;	×
426	auto digital_interval_series = std::make_shared<DigitalIntervalSeries>();	×
427	digital_interval_series->setData(intervals);	×
428	dm->setData<DigitalIntervalSeries>(name, digital_interval_series);	×
429	} else {	×
430	std::cout << "Format " << item["format"] << " not found for " << name << std::endl;	×
431	}
432	} else if (data_type == "tensor") {	×
433
434	if (item["format"] == "numpy") {	×
435
436	TensorData tensor_data;	×
437	loadNpyToTensorData(file_path, tensor_data);	×
438
439	dm->setData<TensorData>(name, std::make_shared<TensorData>(tensor_data));	×
440
441	} else {	×
442	std::cout << "Format " << item["format"] << " not found for " << name << std::endl;	×
443	}
444
445	} else if (data_type == "time") {	×
446
447	if (item["format"] == "uint16") {	×
448
449	int const channel = item["channel"];	×
450	std::string const transition = item["transition"];	×
451
452	auto opts = createBinaryAnalogOptions(file_path, item);	×
453	auto data = readBinaryFile<uint16_t>(opts);	×
454
455	auto digital_data = Loader::extractDigitalData(data, channel);	×
456	auto events = Loader::extractEvents(digital_data, transition);	×
457
458	// convert to int with std::transform
459	std::vector<int> events_int;	×
460	events_int.reserve(events.size());	×
461	for (auto e: events) {	×
462	events_int.push_back(static_cast<int>(e));	×
463	}
464	std::cout << "Loaded " << events_int.size() << " events for " << name << std::endl;	×
465
466	auto timeframe = std::make_shared<TimeFrame>(events_int);	×
467	dm->setTime(name, timeframe);	×
468	}	×
469
470	if (item["format"] == "uint16_length") {	×
471
472	auto opts = createBinaryAnalogOptions(file_path, item);	×
473	auto data = readBinaryFile<uint16_t>(opts);	×
474
475	std::vector<int> t(data.size());	×
476	std::iota(std::begin(t), std::end(t), 0);	×
477
478	std::cout << "Total of " << t.size() << " timestamps for " << name << std::endl;	×
479
480	auto timeframe = std::make_shared<TimeFrame>(t);	×
481	dm->setTime(name, timeframe);	×
482	}	×
483	}
484	if (item.contains("clock")) {	×
485	std::string const clock = item["clock"];	×
486	std::cout << "Setting time for " << name << " to " << clock << std::endl;	×
487	dm->setTimeFrame(name, clock);	×
488	}	×
489	}	×
490
491	return data_info_list;
492	}	×
493
494	std::string DataManager::getType(std::string const & key) const {	×
495	auto it = _data.find(key);	×
496	if (it != _data.end()) {	×
497	if (std::holds_alternative<std::shared_ptr<MediaData>>(it->second)) {	×
498	return "MediaData";	×
499	} else if (std::holds_alternative<std::shared_ptr<PointData>>(it->second)) {	×
500	return "PointData";	×
501	} else if (std::holds_alternative<std::shared_ptr<LineData>>(it->second)) {	×
502	return "LineData";	×
503	} else if (std::holds_alternative<std::shared_ptr<MaskData>>(it->second)) {	×
504	return "MaskData";	×
505	} else if (std::holds_alternative<std::shared_ptr<AnalogTimeSeries>>(it->second)) {	×
506	return "AnalogTimeSeries";	×
507	} else if (std::holds_alternative<std::shared_ptr<DigitalEventSeries>>(it->second)) {	×
508	return "DigitalEventSeries";	×
509	} else if (std::holds_alternative<std::shared_ptr<DigitalIntervalSeries>>(it->second)) {	×
510	return "DigitalIntervalSeries";	×
511	} else if (std::holds_alternative<std::shared_ptr<TensorData>>(it->second)) {	×
512	return "TensorData";	×
513	}
514	return "Unknown";	×
515	}
516	}

paulmthompson / WhiskerToolbox / 14313763633

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous