14316300640

Committed 07 Apr 2025 06:06PM UTC coverage: 13.96% (-0.4%) from 14.375%

Build # 14316300640

Build Type

push

github

Committed by

paulmthompson

Commit Message

start moving some loading methods into their own header files. use enum for loading data types

Coverage Stats

0 of 214 new or added lines in 4 files covered. (0.0%)

6 existing lines in 2 files now uncovered.

208 of 1490 relevant lines covered (13.96%)

1.36 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

3.18

/src/WhiskerToolbox/DataManager/DataManager.cpp


#include "DataManager.hpp"
#include "AnalogTimeSeries/Analog_Time_Series.hpp"
#include "DigitalTimeSeries/Digital_Event_Series.hpp"
#include "DigitalTimeSeries/Digital_Interval_Series.hpp"
#include "Lines/Line_Data.hpp"
#include "Masks/Mask_Data.hpp"
#include "Media/Video_Data.hpp"
#include "Points/Point_Data.hpp"
#include "Tensors/Tensor_Data.hpp"

#include "Media/Video_Data_Loader.hpp"
#include "Points/Point_Data_Loader.hpp"

#include "loaders/CSV_Loaders.hpp"
#include "loaders/binary_loaders.hpp"
#include "transforms/data_transforms.hpp"

#include "TimeFrame.hpp"

#include "utils/container.hpp"
#include "utils/glob.hpp"
#include "utils/hdf5_mask_load.hpp"

#include "nlohmann/json.hpp"
#include "utils/string_manip.hpp"

#include <filesystem>
#include <fstream>
#include <iostream>
#include <optional>
#include <regex>

using namespace nlohmann;

DataManager::DataManager() {
    _times["time"] = std::make_shared<TimeFrame>();
    _data["media"] = std::make_shared<MediaData>();

    setTimeFrame("media", "time");
    _output_path = std::filesystem::current_path();
}

void DataManager::setTimeFrame(std::string const & data_key, std::string const & time_key) {
    //Check that data_key is in _data
    if (_data.find(data_key) == _data.end()) {
        std::cerr << "Data key not found in DataManager: " << data_key << std::endl;
        return;
    }

    //Check that time_key is in _times
    if (_times.find(time_key) == _times.end()) {
        std::cerr << "Time key not found in DataManager: " << time_key << std::endl;
        return;
    }

    _time_frames[data_key] = time_key;
}

std::vector<std::vector<float>> read_ragged_hdf5(std::string const & filepath, std::string const & key) {
    auto myvector = load_ragged_array<float>(filepath, key);
    return myvector;
}

std::vector<int> read_array_hdf5(std::string const & filepath, std::string const & key) {
    auto myvector = load_array<int>(filepath, key);
    return myvector;
}

enum class DataType {
    Video,
    Points,
    Mask,
    Line,
    Analog,
    DigitalEvent,
    DigitalInterval,
    Tensor,
    Time,
    Unknown
};

DataType stringToDataType(std::string const & data_type_str) {
    if (data_type_str == "video") return DataType::Video;
    if (data_type_str == "points") return DataType::Points;
    if (data_type_str == "mask") return DataType::Mask;
    if (data_type_str == "line") return DataType::Line;
    if (data_type_str == "analog") return DataType::Analog;
    if (data_type_str == "digital_event") return DataType::DigitalEvent;
    if (data_type_str == "digital_interval") return DataType::DigitalInterval;
    if (data_type_str == "tensor") return DataType::Tensor;
    if (data_type_str == "time") return DataType::Time;
    return DataType::Unknown;
}

std::optional<std::string> processFilePath(
        std::string const & file_path,
        std::filesystem::path const & base_path) {
    std::filesystem::path full_path = file_path;

    // Check for wildcard character
    if (file_path.find('*') != std::string::npos) {
        // Convert wildcard pattern to regex
        std::string const pattern = std::regex_replace(full_path.string(), std::regex("\\*"), ".*");
        std::regex const regex_pattern(pattern);

        // Iterate through the directory to find matching files
        for (auto const & entry: std::filesystem::directory_iterator(base_path)) {
            std::cout << "Checking " << entry.path().string() << " with full path " << full_path << std::endl;
            if (std::regex_match(entry.path().string(), regex_pattern)) {
                std::cout << "Loading file " << entry.path().string() << std::endl;
                return entry.path().string();
            }
        }
        return std::nullopt;
    } else {
        // Check if the file path is relative
        if (!std::filesystem::path(file_path).is_absolute()) {
            full_path = base_path / file_path;
        }
        // Check for the presence of the file
        if (std::filesystem::exists(full_path)) {
            std::cout << "Loading file " << full_path.string() << std::endl;
            return full_path.string();
        } else {
            return std::nullopt;
        }
    }
}

bool checkRequiredFields(json const & item, std::vector<std::string> const & requiredFields) {
    for (auto const & field: requiredFields) {
        if (!item.contains(field)) {
            std::cerr << "Error: Missing required field \"" << field << "\" in JSON item." << std::endl;
            return false;
        }
    }
    return true;
}

int DataManager::addCallbackToData(std::string const & key, ObserverCallback callback) {

    int id = -1;

    if (_data.find(key) != _data.end()) {
        auto data = _data[key];

        id = std::visit([callback](auto & x) {
            return x.get()->addObserver(callback);
        },
                        data);
    }

    return id;
}

void DataManager::removeCallbackFromData(std::string const & key, int callback_id) {
    if (_data.find(key) != _data.end()) {
        auto data = _data[key];

        std::visit([callback_id](auto & x) {
            x.get()->removeObserver(callback_id);
        },
                   data);
    }
}

void checkOptionalFields(json const & item, std::vector<std::string> const & optionalFields) {
    for (auto const & field: optionalFields) {
        if (!item.contains(field)) {
            std::cout << "Warning: Optional field \"" << field << "\" is missing in JSON item." << std::endl;
        }
    }
}


Loader::BinaryAnalogOptions createBinaryAnalogOptions(std::string const & file_path, nlohmann::basic_json<> const & item) {

    int const header_size = item.value("header_size", 0);
    int const num_channels = item.value("channel_count", 1);

    auto opts = Loader::BinaryAnalogOptions{
            .file_path = file_path,
            .header_size_bytes = static_cast<size_t>(header_size),
            .num_channels = static_cast<size_t>(num_channels)};

    return opts;
}

std::vector<DataInfo> load_data_from_json_config(DataManager * dm, std::string const & json_filepath) {
    std::vector<DataInfo> data_info_list;
    // Open JSON file
    std::ifstream ifs(json_filepath);
    if (!ifs.is_open()) {
        std::cerr << "Failed to open JSON file: " << json_filepath << std::endl;
        return data_info_list;
    }

    // Parse JSON
    json j;
    ifs >> j;

    // get base path of filepath
    std::filesystem::path const base_path = std::filesystem::path(json_filepath).parent_path();

    // Iterate through JSON array
    for (auto const & item: j) {

        if (!checkRequiredFields(item, {"data_type", "name", "filepath"})) {
            continue;// Exit if any required field is missing
        }

        std::string const data_type_str = item["data_type"];
        DataType data_type = stringToDataType(data_type_str);
        if (data_type == DataType::Unknown) {
            std::cout << "Unknown data type: " << data_type_str << std::endl;
            continue;
        }

        std::string const name = item["name"];

        auto file_exists = processFilePath(item["filepath"], base_path);
        if (!file_exists) {
            std::cerr << "File does not exist: " << item["filepath"] << std::endl;
            continue;
        }

        std::string const file_path = file_exists.value();

        switch (data_type) {
            case DataType::Video: {
                // Create VideoData object
                auto video_data = load_video_into_VideoData(file_path);

                // Add VideoData to DataManager
                dm->setMedia(video_data);

                data_info_list.push_back({name, "VideoData", ""});
                break;
            }
            case DataType::Points: {

                auto point_data = load_into_PointData(file_path, item);

                dm->setData<PointData>(name, point_data);

                std::string const color = item.value("color", "#0000FF");
                data_info_list.push_back({name, "PointData", color});
                break;
            }
            case DataType::Mask: {

                std::string const frame_key = item["frame_key"];
                std::string const prob_key = item["probability_key"];
                std::string const x_key = item["x_key"];
                std::string const y_key = item["y_key"];

                int const height = item.value("height", -1);
                int const width = item.value("width", -1);

                std::string const color = item.value("color", "0000FF");

                auto frames = read_array_hdf5(file_path, frame_key);
                auto probs = read_ragged_hdf5(file_path, prob_key);
                auto y_coords = read_ragged_hdf5(file_path, y_key);
                auto x_coords = read_ragged_hdf5(file_path, x_key);

                auto mask_data = std::make_shared<MaskData>();
                mask_data->setImageSize(ImageSize{width, height});

                for (std::size_t i = 0; i < frames.size(); i++) {
                    auto frame = frames[i];
                    auto prob = probs[i];
                    auto x = x_coords[i];
                    auto y = y_coords[i];
                    mask_data->addMaskAtTime(frame, x, y);
                }

                dm->setData<MaskData>(name, mask_data);

                data_info_list.push_back({name, "MaskData", color});

                if (item.contains("operations")) {

                    for (auto const & operation: item["operations"]) {

                        std::string const operation_type = operation["type"];

                        if (operation_type == "area") {
                            std::cout << "Calculating area for mask: " << name << std::endl;
                            auto area_data = area(dm->getData<MaskData>(name));
                            std::string const output_name = name + "_area";
                            dm->setData<AnalogTimeSeries>(output_name, area_data);
                        }
                    }
                }
                break;
            }
            case DataType::Line: {

                auto line_map = load_line_csv(file_path);

                //Get the whisker name from the filename using filesystem
                auto whisker_filename = std::filesystem::path(file_path).filename().string();

                //Remove .csv suffix from filename
                auto whisker_name = remove_extension(whisker_filename);

                dm->setData<LineData>(whisker_name, std::make_shared<LineData>(line_map));

                std::string const color = item.value("color", "0000FF");

                data_info_list.push_back({name, "LineData", color});

                break;
            }
            case DataType::Analog: {

                if (item["format"] == "int16") {

                    auto opts = createBinaryAnalogOptions(file_path, item);

                    if (opts.num_channels > 1) {

                        auto data = readBinaryFileMultiChannel<int16_t>(opts);

                        std::cout << "Read " << data.size() << " channels" << std::endl;

                        for (int channel = 0; channel < data.size(); channel++) {
                            // convert to float with std::transform
                            std::vector<float> data_float;
                            std::transform(
                                    data[channel].begin(),
                                    data[channel].end(),
                                    std::back_inserter(data_float), [](int16_t i) { return i; });

                            auto analog_time_series = std::make_shared<AnalogTimeSeries>();
                            analog_time_series->setData(data_float);

                            std::string const channel_name = name + "_" + std::to_string(channel);

                            dm->setData<AnalogTimeSeries>(channel_name, analog_time_series);

                            if (item.contains("clock")) {
                                std::string const clock = item["clock"];
                                dm->setTimeFrame(channel_name, clock);
                            }
                        }

                    } else {

                        auto data = readBinaryFile<int16_t>(opts);

                        // convert to float with std::transform
                        std::vector<float> data_float;
                        std::transform(
                                data.begin(),
                                data.end(),
                                std::back_inserter(data_float), [](int16_t i) { return i; });

                        auto analog_time_series = std::make_shared<AnalogTimeSeries>();
                        analog_time_series->setData(data_float);
                        dm->setData<AnalogTimeSeries>(name, analog_time_series);
                    }

                } else {
                    std::cout << "Format " << item["format"] << " not found for " << name << std::endl;
                }
                break;
            }
            case DataType::DigitalEvent: {

                if (item["format"] == "uint16") {

                    int const channel = item["channel"];
                    std::string const transition = item["transition"];

                    auto opts = createBinaryAnalogOptions(file_path, item);
                    auto data = readBinaryFile<uint16_t>(opts);

                    auto digital_data = Loader::extractDigitalData(data, channel);
                    auto events = Loader::extractEvents(digital_data, transition);
                    std::cout << "Loaded " << events.size() << " events for " << name << std::endl;

                    auto digital_event_series = std::make_shared<DigitalEventSeries>();
                    digital_event_series->setData(events);
                    dm->setData<DigitalEventSeries>(name, digital_event_series);
                } else if (item["format"] == "csv") {

                    auto opts = Loader::CSVSingleColumnOptions{.filename = file_path};

                    auto events = Loader::loadSingleColumnCSV(opts);
                    std::cout << "Loaded " << events.size() << " events for " << name << std::endl;

                    float const scale = item.value("scale", 1.0f);
                    bool const scale_divide = item.value("scale_divide", false);

                    if (scale_divide) {
                        for (auto & e: events) {
                            e /= scale;
                        }
                    } else {
                        for (auto & e: events) {
                            e *= scale;
                        }
                    }

                    auto digital_event_series = std::make_shared<DigitalEventSeries>();
                    digital_event_series->setData(events);
                    dm->setData<DigitalEventSeries>(name, digital_event_series);

                } else {
                    std::cout << "Format " << item["format"] << " not found for " << name << std::endl;
                }
                break;
            }
            case DataType::DigitalInterval: {

                if (item["format"] == "uint16") {

                    int const channel = item["channel"];
                    std::string const transition = item["transition"];

                    auto opts = createBinaryAnalogOptions(file_path, item);
                    auto data = readBinaryFile<uint16_t>(opts);

                    auto digital_data = Loader::extractDigitalData(data, channel);

                    auto intervals = Loader::extractIntervals(digital_data, transition);
                    std::cout << "Loaded " << intervals.size() << " intervals for " << name << std::endl;

                    auto digital_interval_series = std::make_shared<DigitalIntervalSeries>();
                    digital_interval_series->setData(intervals);
                    dm->setData<DigitalIntervalSeries>(name, digital_interval_series);

                } else if (item["format"] == "csv") {

                    auto opts = Loader::CSVMultiColumnOptions{.filename = file_path};

                    auto intervals = Loader::loadPairColumnCSV(opts);
                    std::cout << "Loaded " << intervals.size() << " intervals for " << name << std::endl;
                    auto digital_interval_series = std::make_shared<DigitalIntervalSeries>();
                    digital_interval_series->setData(intervals);
                    dm->setData<DigitalIntervalSeries>(name, digital_interval_series);
                } else {
                    std::cout << "Format " << item["format"] << " not found for " << name << std::endl;
                }
                break;
            }
            case DataType::Tensor: {

                if (item["format"] == "numpy") {

                    TensorData tensor_data;
                    loadNpyToTensorData(file_path, tensor_data);

                    dm->setData<TensorData>(name, std::make_shared<TensorData>(tensor_data));

                } else {
                    std::cout << "Format " << item["format"] << " not found for " << name << std::endl;
                }
                break;
            }
            case DataType::Time: {

                if (item["format"] == "uint16") {

                    int const channel = item["channel"];
                    std::string const transition = item["transition"];

                    auto opts = createBinaryAnalogOptions(file_path, item);
                    auto data = readBinaryFile<uint16_t>(opts);

                    auto digital_data = Loader::extractDigitalData(data, channel);
                    auto events = Loader::extractEvents(digital_data, transition);

                    // convert to int with std::transform
                    std::vector<int> events_int;
                    events_int.reserve(events.size());
                    for (auto e: events) {
                        events_int.push_back(static_cast<int>(e));
                    }
                    std::cout << "Loaded " << events_int.size() << " events for " << name << std::endl;

                    auto timeframe = std::make_shared<TimeFrame>(events_int);
                    dm->setTime(name, timeframe);
                }

                if (item["format"] == "uint16_length") {

                    auto opts = createBinaryAnalogOptions(file_path, item);
                    auto data = readBinaryFile<uint16_t>(opts);

                    std::vector<int> t(data.size());
                    std::iota(std::begin(t), std::end(t), 0);

                    std::cout << "Total of " << t.size() << " timestamps for " << name << std::endl;

                    auto timeframe = std::make_shared<TimeFrame>(t);
                    dm->setTime(name, timeframe);
                }
                break;
            }
            default:
                std::cout << "Unsupported data type: " << data_type_str << std::endl;
                continue;
        }
        if (item.contains("clock")) {
            std::string const clock = item["clock"];
            std::cout << "Setting time for " << name << " to " << clock << std::endl;
            dm->setTimeFrame(name, clock);
        }
    }

    return data_info_list;
}

std::string DataManager::getType(std::string const & key) const {
    auto it = _data.find(key);
    if (it != _data.end()) {
        if (std::holds_alternative<std::shared_ptr<MediaData>>(it->second)) {
            return "MediaData";
        } else if (std::holds_alternative<std::shared_ptr<PointData>>(it->second)) {
            return "PointData";
        } else if (std::holds_alternative<std::shared_ptr<LineData>>(it->second)) {
            return "LineData";
        } else if (std::holds_alternative<std::shared_ptr<MaskData>>(it->second)) {
            return "MaskData";
        } else if (std::holds_alternative<std::shared_ptr<AnalogTimeSeries>>(it->second)) {
            return "AnalogTimeSeries";
        } else if (std::holds_alternative<std::shared_ptr<DigitalEventSeries>>(it->second)) {
            return "DigitalEventSeries";
        } else if (std::holds_alternative<std::shared_ptr<DigitalIntervalSeries>>(it->second)) {
            return "DigitalIntervalSeries";
        } else if (std::holds_alternative<std::shared_ptr<TensorData>>(it->second)) {
            return "TensorData";
        }
        return "Unknown";
    }
}

1
2	#include "DataManager.hpp"
3	#include "AnalogTimeSeries/Analog_Time_Series.hpp"
4	#include "DigitalTimeSeries/Digital_Event_Series.hpp"
5	#include "DigitalTimeSeries/Digital_Interval_Series.hpp"
6	#include "Lines/Line_Data.hpp"
7	#include "Masks/Mask_Data.hpp"
8	#include "Media/Video_Data.hpp"
9	#include "Points/Point_Data.hpp"
10	#include "Tensors/Tensor_Data.hpp"
11
12	#include "Media/Video_Data_Loader.hpp"
13	#include "Points/Point_Data_Loader.hpp"
14
15	#include "loaders/CSV_Loaders.hpp"
16	#include "loaders/binary_loaders.hpp"
17	#include "transforms/data_transforms.hpp"
18
19	#include "TimeFrame.hpp"
20
21	#include "utils/container.hpp"
22	#include "utils/glob.hpp"
23	#include "utils/hdf5_mask_load.hpp"
24
25	#include "nlohmann/json.hpp"
26	#include "utils/string_manip.hpp"
27
28	#include <filesystem>
29	#include <fstream>
30	#include <iostream>
31	#include <optional>
32	#include <regex>
33
34	using namespace nlohmann;
35
36	DataManager::DataManager() {	2✔
37	_times["time"] = std::make_shared<TimeFrame>();	6✔
38	_data["media"] = std::make_shared<MediaData>();	6✔
39
40	setTimeFrame("media", "time");	10✔
41	_output_path = std::filesystem::current_path();	2✔
42	}	2✔
43
44	void DataManager::setTimeFrame(std::string const & data_key, std::string const & time_key) {	2✔
45	//Check that data_key is in _data
46	if (_data.find(data_key) == _data.end()) {	2✔
47	std::cerr << "Data key not found in DataManager: " << data_key << std::endl;	×
48	return;	×
49	}
50
51	//Check that time_key is in _times
52	if (_times.find(time_key) == _times.end()) {	2✔
53	std::cerr << "Time key not found in DataManager: " << time_key << std::endl;	×
54	return;	×
55	}
56
57	_time_frames[data_key] = time_key;	2✔
58	}
59
60	std::vector<std::vector<float>> read_ragged_hdf5(std::string const & filepath, std::string const & key) {	×
61	auto myvector = load_ragged_array<float>(filepath, key);	×
62	return myvector;	×
63	}
64
65	std::vector<int> read_array_hdf5(std::string const & filepath, std::string const & key) {	×
66	auto myvector = load_array<int>(filepath, key);	×
67	return myvector;	×
68	}
69
70	enum class DataType {
71	Video,
72	Points,
73	Mask,
74	Line,
75	Analog,
76	DigitalEvent,
77	DigitalInterval,
78	Tensor,
79	Time,
80	Unknown
81	};
82
NEW 83	DataType stringToDataType(std::string const & data_type_str) {	×
NEW 84	if (data_type_str == "video") return DataType::Video;	×
NEW 85	if (data_type_str == "points") return DataType::Points;	×
NEW 86	if (data_type_str == "mask") return DataType::Mask;	×
NEW 87	if (data_type_str == "line") return DataType::Line;	×
NEW 88	if (data_type_str == "analog") return DataType::Analog;	×
NEW 89	if (data_type_str == "digital_event") return DataType::DigitalEvent;	×
NEW 90	if (data_type_str == "digital_interval") return DataType::DigitalInterval;	×
NEW 91	if (data_type_str == "tensor") return DataType::Tensor;	×
NEW 92	if (data_type_str == "time") return DataType::Time;	×
NEW 93	return DataType::Unknown;	×
94	}
95
UNCOV 96	std::optional<std::string> processFilePath(	×
97	std::string const & file_path,
98	std::filesystem::path const & base_path) {
99	std::filesystem::path full_path = file_path;	×
100
101	// Check for wildcard character
102	if (file_path.find('*') != std::string::npos) {	×
103	// Convert wildcard pattern to regex
104	std::string const pattern = std::regex_replace(full_path.string(), std::regex("\\"), ".");	×
105	std::regex const regex_pattern(pattern);	×
106
107	// Iterate through the directory to find matching files
108	for (auto const & entry: std::filesystem::directory_iterator(base_path)) {	×
109	std::cout << "Checking " << entry.path().string() << " with full path " << full_path << std::endl;	×
110	if (std::regex_match(entry.path().string(), regex_pattern)) {	×
111	std::cout << "Loading file " << entry.path().string() << std::endl;	×
112	return entry.path().string();	×
113	}
114	}	×
115	return std::nullopt;	×
116	} else {	×
117	// Check if the file path is relative
118	if (!std::filesystem::path(file_path).is_absolute()) {	×
119	full_path = base_path / file_path;	×
120	}
121	// Check for the presence of the file
122	if (std::filesystem::exists(full_path)) {	×
123	std::cout << "Loading file " << full_path.string() << std::endl;	×
124	return full_path.string();	×
125	} else {
126	return std::nullopt;	×
127	}
128	}
129	}	×
130
131	bool checkRequiredFields(json const & item, std::vector<std::string> const & requiredFields) {	×
132	for (auto const & field: requiredFields) {	×
133	if (!item.contains(field)) {	×
134	std::cerr << "Error: Missing required field \"" << field << "\" in JSON item." << std::endl;	×
135	return false;	×
136	}
137	}
138	return true;	×
139	}
140
141	int DataManager::addCallbackToData(std::string const & key, ObserverCallback callback) {	×
142
143	int id = -1;	×
144
145	if (_data.find(key) != _data.end()) {	×
146	auto data = _data[key];	×
147
148	id = std::visit([callback](auto & x) {	×
149	return x.get()->addObserver(callback);	×
150	},
151	data);
152	}	×
153
154	return id;	×
155	}
156
157	void DataManager::removeCallbackFromData(std::string const & key, int callback_id) {	×
158	if (_data.find(key) != _data.end()) {	×
159	auto data = _data[key];	×
160
161	std::visit([callback_id](auto & x) {	×
162	x.get()->removeObserver(callback_id);	×
163	},	×
164	data);
165	}	×
166	}	×
167
168	void checkOptionalFields(json const & item, std::vector<std::string> const & optionalFields) {	×
169	for (auto const & field: optionalFields) {	×
170	if (!item.contains(field)) {	×
171	std::cout << "Warning: Optional field \"" << field << "\" is missing in JSON item." << std::endl;	×
172	}
173	}
174	}	×
175
176
177	Loader::BinaryAnalogOptions createBinaryAnalogOptions(std::string const & file_path, nlohmann::basic_json<> const & item) {	×
178
179	int const header_size = item.value("header_size", 0);	×
180	int const num_channels = item.value("channel_count", 1);	×
181
182	auto opts = Loader::BinaryAnalogOptions{
183	.file_path = file_path,
184	.header_size_bytes = static_cast<size_t>(header_size),	×
185	.num_channels = static_cast<size_t>(num_channels)};	×
186
187	return opts;	×
188	}
189
190	std::vector<DataInfo> load_data_from_json_config(DataManager * dm, std::string const & json_filepath) {	×
191	std::vector<DataInfo> data_info_list;	×
192	// Open JSON file
193	std::ifstream ifs(json_filepath);	×
194	if (!ifs.is_open()) {	×
195	std::cerr << "Failed to open JSON file: " << json_filepath << std::endl;	×
196	return data_info_list;	×
197	}
198
199	// Parse JSON
200	json j;	×
201	ifs >> j;	×
202
203	// get base path of filepath
204	std::filesystem::path const base_path = std::filesystem::path(json_filepath).parent_path();	×
205
206	// Iterate through JSON array
207	for (auto const & item: j) {	×
208
209	if (!checkRequiredFields(item, {"data_type", "name", "filepath"})) {	×
210	continue;// Exit if any required field is missing	×
211	}
212
NEW 213	std::string const data_type_str = item["data_type"];	×
NEW 214	DataType data_type = stringToDataType(data_type_str);	×
NEW 215	if (data_type == DataType::Unknown) {	×
NEW 216	std::cout << "Unknown data type: " << data_type_str << std::endl;	×
NEW 217	continue;	×
218	}
219
UNCOV 220	std::string const name = item["name"];	×
221
222	auto file_exists = processFilePath(item["filepath"], base_path);	×
223	if (!file_exists) {	×
224	std::cerr << "File does not exist: " << item["filepath"] << std::endl;	×
225	continue;	×
226	}
227
228	std::string const file_path = file_exists.value();	×
229
NEW 230	switch (data_type) {	×
NEW 231	case DataType::Video: {	×
232	// Create VideoData object
NEW 233	auto video_data = load_video_into_VideoData(file_path);	×
234
235	// Add VideoData to DataManager
NEW 236	dm->setMedia(video_data);	×
237
NEW 238	data_info_list.push_back({name, "VideoData", ""});	×
NEW 239	break;	×
NEW 240	}	×
NEW 241	case DataType::Points: {	×
242
NEW 243	auto point_data = load_into_PointData(file_path, item);	×
244
NEW 245	dm->setData<PointData>(name, point_data);	×
246
NEW 247	std::string const color = item.value("color", "#0000FF");	×
NEW 248	data_info_list.push_back({name, "PointData", color});	×
NEW 249	break;	×
NEW 250	}	×
NEW 251	case DataType::Mask: {	×
252
NEW 253	std::string const frame_key = item["frame_key"];	×
NEW 254	std::string const prob_key = item["probability_key"];	×
NEW 255	std::string const x_key = item["x_key"];	×
NEW 256	std::string const y_key = item["y_key"];	×
257
NEW 258	int const height = item.value("height", -1);	×
NEW 259	int const width = item.value("width", -1);	×
260
NEW 261	std::string const color = item.value("color", "0000FF");	×
262
NEW 263	auto frames = read_array_hdf5(file_path, frame_key);	×
NEW 264	auto probs = read_ragged_hdf5(file_path, prob_key);	×
NEW 265	auto y_coords = read_ragged_hdf5(file_path, y_key);	×
NEW 266	auto x_coords = read_ragged_hdf5(file_path, x_key);	×
267
NEW 268	auto mask_data = std::make_shared<MaskData>();	×
NEW 269	mask_data->setImageSize(ImageSize{width, height});	×
270
NEW 271	for (std::size_t i = 0; i < frames.size(); i++) {	×
NEW 272	auto frame = frames[i];	×
NEW 273	auto prob = probs[i];	×
NEW 274	auto x = x_coords[i];	×
NEW 275	auto y = y_coords[i];	×
NEW 276	mask_data->addMaskAtTime(frame, x, y);	×
NEW 277	}	×
278
NEW 279	dm->setData<MaskData>(name, mask_data);	×
280
NEW 281	data_info_list.push_back({name, "MaskData", color});	×
282
NEW 283	if (item.contains("operations")) {	×
284
NEW 285	for (auto const & operation: item["operations"]) {	×
286
NEW 287	std::string const operation_type = operation["type"];	×
288
NEW 289	if (operation_type == "area") {	×
NEW 290	std::cout << "Calculating area for mask: " << name << std::endl;	×
NEW 291	auto area_data = area(dm->getData<MaskData>(name));	×
NEW 292	std::string const output_name = name + "_area";	×
NEW 293	dm->setData<AnalogTimeSeries>(output_name, area_data);	×
NEW 294	}	×
NEW 295	}	×
296	}
NEW 297	break;	×
UNCOV 298	}	×
NEW 299	case DataType::Line: {	×
300
NEW 301	auto line_map = load_line_csv(file_path);	×
302
303	//Get the whisker name from the filename using filesystem
NEW 304	auto whisker_filename = std::filesystem::path(file_path).filename().string();	×
305
306	//Remove .csv suffix from filename
NEW 307	auto whisker_name = remove_extension(whisker_filename);	×
308
NEW 309	dm->setData<LineData>(whisker_name, std::make_shared<LineData>(line_map));	×
310
NEW 311	std::string const color = item.value("color", "0000FF");	×
312
NEW 313	data_info_list.push_back({name, "LineData", color});	×
314
NEW 315	break;	×
316	}	×
NEW 317	case DataType::Analog: {	×
318
NEW 319	if (item["format"] == "int16") {	×
320
NEW 321	auto opts = createBinaryAnalogOptions(file_path, item);	×
322
NEW 323	if (opts.num_channels > 1) {	×
324
NEW 325	auto data = readBinaryFileMultiChannel<int16_t>(opts);	×
326
NEW 327	std::cout << "Read " << data.size() << " channels" << std::endl;	×
328
NEW 329	for (int channel = 0; channel < data.size(); channel++) {	×
330	// convert to float with std::transform
NEW 331	std::vector<float> data_float;	×
NEW 332	std::transform(	×
NEW 333	data[channel].begin(),	×
NEW 334	data[channel].end(),	×
NEW 335	std::back_inserter(data_float), [](int16_t i) { return i; });	×
336
NEW 337	auto analog_time_series = std::make_shared<AnalogTimeSeries>();	×
NEW 338	analog_time_series->setData(data_float);	×
339
NEW 340	std::string const channel_name = name + "_" + std::to_string(channel);	×
341
NEW 342	dm->setData<AnalogTimeSeries>(channel_name, analog_time_series);	×
343
NEW 344	if (item.contains("clock")) {	×
NEW 345	std::string const clock = item["clock"];	×
NEW 346	dm->setTimeFrame(channel_name, clock);	×
NEW 347	}	×
NEW 348	}	×
349
NEW 350	} else {	×
351
NEW 352	auto data = readBinaryFile<int16_t>(opts);	×
353
354	// convert to float with std::transform
UNCOV 355	std::vector<float> data_float;	×
356	std::transform(	×
357	data.begin(),
358	data.end(),
UNCOV 359	std::back_inserter(data_float), [](int16_t i) { return i; });	×
360
361	auto analog_time_series = std::make_shared<AnalogTimeSeries>();	×
362	analog_time_series->setData(data_float);	×
NEW 363	dm->setData<AnalogTimeSeries>(name, analog_time_series);	×
364	}	×
365
366	} else {	×
NEW 367	std::cout << "Format " << item["format"] << " not found for " << name << std::endl;	×
368	}
NEW 369	break;	×
370	}
NEW 371	case DataType::DigitalEvent: {	×
372
NEW 373	if (item["format"] == "uint16") {	×
374
NEW 375	int const channel = item["channel"];	×
NEW 376	std::string const transition = item["transition"];	×
377
NEW 378	auto opts = createBinaryAnalogOptions(file_path, item);	×
NEW 379	auto data = readBinaryFile<uint16_t>(opts);	×
380
NEW 381	auto digital_data = Loader::extractDigitalData(data, channel);	×
NEW 382	auto events = Loader::extractEvents(digital_data, transition);	×
NEW 383	std::cout << "Loaded " << events.size() << " events for " << name << std::endl;	×
384
NEW 385	auto digital_event_series = std::make_shared<DigitalEventSeries>();	×
NEW 386	digital_event_series->setData(events);	×
NEW 387	dm->setData<DigitalEventSeries>(name, digital_event_series);	×
NEW 388	} else if (item["format"] == "csv") {	×
389
NEW 390	auto opts = Loader::CSVSingleColumnOptions{.filename = file_path};	×
391
NEW 392	auto events = Loader::loadSingleColumnCSV(opts);	×
NEW 393	std::cout << "Loaded " << events.size() << " events for " << name << std::endl;	×
394
NEW 395	float const scale = item.value("scale", 1.0f);	×
NEW 396	bool const scale_divide = item.value("scale_divide", false);	×
397
NEW 398	if (scale_divide) {	×
NEW 399	for (auto & e: events) {	×
NEW 400	e /= scale;	×
401	}
402	} else {
NEW 403	for (auto & e: events) {	×
NEW 404	e *= scale;	×
405	}
406	}
407
NEW 408	auto digital_event_series = std::make_shared<DigitalEventSeries>();	×
NEW 409	digital_event_series->setData(events);	×
NEW 410	dm->setData<DigitalEventSeries>(name, digital_event_series);	×
411
NEW 412	} else {	×
NEW 413	std::cout << "Format " << item["format"] << " not found for " << name << std::endl;	×
414	}
NEW 415	break;	×
416	}
NEW 417	case DataType::DigitalInterval: {	×
418
NEW 419	if (item["format"] == "uint16") {	×
420
NEW 421	int const channel = item["channel"];	×
NEW 422	std::string const transition = item["transition"];	×
423
NEW 424	auto opts = createBinaryAnalogOptions(file_path, item);	×
NEW 425	auto data = readBinaryFile<uint16_t>(opts);	×
426
NEW 427	auto digital_data = Loader::extractDigitalData(data, channel);	×
428
NEW 429	auto intervals = Loader::extractIntervals(digital_data, transition);	×
NEW 430	std::cout << "Loaded " << intervals.size() << " intervals for " << name << std::endl;	×
431
NEW 432	auto digital_interval_series = std::make_shared<DigitalIntervalSeries>();	×
NEW 433	digital_interval_series->setData(intervals);	×
NEW 434	dm->setData<DigitalIntervalSeries>(name, digital_interval_series);	×
435
NEW 436	} else if (item["format"] == "csv") {	×
437
NEW 438	auto opts = Loader::CSVMultiColumnOptions{.filename = file_path};	×
439
NEW 440	auto intervals = Loader::loadPairColumnCSV(opts);	×
NEW 441	std::cout << "Loaded " << intervals.size() << " intervals for " << name << std::endl;	×
NEW 442	auto digital_interval_series = std::make_shared<DigitalIntervalSeries>();	×
NEW 443	digital_interval_series->setData(intervals);	×
NEW 444	dm->setData<DigitalIntervalSeries>(name, digital_interval_series);	×
NEW 445	} else {	×
NEW 446	std::cout << "Format " << item["format"] << " not found for " << name << std::endl;	×
447	}
NEW 448	break;	×
449	}
NEW 450	case DataType::Tensor: {	×
451
NEW 452	if (item["format"] == "numpy") {	×
453
NEW 454	TensorData tensor_data;	×
NEW 455	loadNpyToTensorData(file_path, tensor_data);	×
456
NEW 457	dm->setData<TensorData>(name, std::make_shared<TensorData>(tensor_data));	×
458
NEW 459	} else {	×
NEW 460	std::cout << "Format " << item["format"] << " not found for " << name << std::endl;	×
461	}
NEW 462	break;	×
463	}
NEW 464	case DataType::Time: {	×
465
NEW 466	if (item["format"] == "uint16") {	×
467
NEW 468	int const channel = item["channel"];	×
NEW 469	std::string const transition = item["transition"];	×
470
NEW 471	auto opts = createBinaryAnalogOptions(file_path, item);	×
NEW 472	auto data = readBinaryFile<uint16_t>(opts);	×
473
NEW 474	auto digital_data = Loader::extractDigitalData(data, channel);	×
NEW 475	auto events = Loader::extractEvents(digital_data, transition);	×
476
477	// convert to int with std::transform
NEW 478	std::vector<int> events_int;	×
NEW 479	events_int.reserve(events.size());	×
NEW 480	for (auto e: events) {	×
NEW 481	events_int.push_back(static_cast<int>(e));	×
482	}
NEW 483	std::cout << "Loaded " << events_int.size() << " events for " << name << std::endl;	×
484
NEW 485	auto timeframe = std::make_shared<TimeFrame>(events_int);	×
NEW 486	dm->setTime(name, timeframe);	×
487	}	×
488
NEW 489	if (item["format"] == "uint16_length") {	×
490
NEW 491	auto opts = createBinaryAnalogOptions(file_path, item);	×
NEW 492	auto data = readBinaryFile<uint16_t>(opts);	×
493
NEW 494	std::vector<int> t(data.size());	×
NEW 495	std::iota(std::begin(t), std::end(t), 0);	×
496
NEW 497	std::cout << "Total of " << t.size() << " timestamps for " << name << std::endl;	×
498
NEW 499	auto timeframe = std::make_shared<TimeFrame>(t);	×
NEW 500	dm->setTime(name, timeframe);	×
NEW 501	}	×
NEW 502	break;	×
503	}
NEW 504	default:	×
NEW 505	std::cout << "Unsupported data type: " << data_type_str << std::endl;	×
NEW 506	continue;	×
507	}	×
508	if (item.contains("clock")) {	×
509	std::string const clock = item["clock"];	×
510	std::cout << "Setting time for " << name << " to " << clock << std::endl;	×
511	dm->setTimeFrame(name, clock);	×
512	}	×
513	}	×
514
515	return data_info_list;
516	}	×
517
518	std::string DataManager::getType(std::string const & key) const {	×
519	auto it = _data.find(key);	×
520	if (it != _data.end()) {	×
521	if (std::holds_alternative<std::shared_ptr<MediaData>>(it->second)) {	×
522	return "MediaData";	×
523	} else if (std::holds_alternative<std::shared_ptr<PointData>>(it->second)) {	×
524	return "PointData";	×
525	} else if (std::holds_alternative<std::shared_ptr<LineData>>(it->second)) {	×
526	return "LineData";	×
527	} else if (std::holds_alternative<std::shared_ptr<MaskData>>(it->second)) {	×
528	return "MaskData";	×
529	} else if (std::holds_alternative<std::shared_ptr<AnalogTimeSeries>>(it->second)) {	×
530	return "AnalogTimeSeries";	×
531	} else if (std::holds_alternative<std::shared_ptr<DigitalEventSeries>>(it->second)) {	×
532	return "DigitalEventSeries";	×
533	} else if (std::holds_alternative<std::shared_ptr<DigitalIntervalSeries>>(it->second)) {	×
534	return "DigitalIntervalSeries";	×
535	} else if (std::holds_alternative<std::shared_ptr<TensorData>>(it->second)) {	×
536	return "TensorData";	×
537	}
538	return "Unknown";	×
539	}
540	}

paulmthompson / WhiskerToolbox / 14316300640

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous