18389801194

Committed 09 Oct 2025 09:35PM UTC coverage: 71.943% (+0.1%) from 71.826%

Build # 18389801194

Build Type

push

github

Committed by

paulmthompson

Commit Message

add correlation matrix to filtering interface

Run Details

207 of 337 new or added lines in 5 files covered. (61.42%)

867 existing lines in 31 files now uncovered.

49964 of 69449 relevant lines covered (71.94%)

1103.53 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

37.59

/src/DataManager/Points/IO/CSV/Point_Data_CSV.cpp

#include "Point_Data_CSV.hpp"

#include "Points/Point_Data.hpp"
#include "transforms/data_transforms.hpp"
#include "utils/string_manip.hpp"

#include <filesystem>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <sstream>

//https://stackoverflow.com/questions/4654636/how-to-determine-if-a-string-is-a-number-with-c
bool is_number(std::string const & s) {
    return !s.empty() && std::find_if(s.begin(),
                                      s.end(), [](unsigned char c) { return !std::isdigit(c); }) == s.end();
}

std::map<TimeFrameIndex, Point2D<float>> load(CSVPointLoaderOptions const & opts) {
    std::string csv_line;

    auto line_output = std::map<TimeFrameIndex, Point2D<float>>{};

    std::fstream myfile;
    myfile.open(opts.filename, std::fstream::in);

    std::string x_str;
    std::string y_str;
    std::string frame_str;
    std::string col_value;

    std::vector<std::pair<TimeFrameIndex, Point2D<float>>> csv_vector = {};

    while (getline(myfile, csv_line)) {

        std::stringstream ss(csv_line);

        int cols_read = 0;
        while (getline(ss, col_value, opts.column_delim)) {
            if (cols_read == opts.frame_column) {
                frame_str = col_value;
            } else if (cols_read == opts.x_column) {
                x_str = col_value;
            } else if (cols_read == opts.y_column) {
                y_str = col_value;
            }
            cols_read++;
        }

        if (is_number(frame_str)) {
            //line_output[TimeFrameIndex(std::stoi(frame_str))]=Point2D<float>{std::stof(x_str),std::stof(y_str)};
            csv_vector.emplace_back(TimeFrameIndex(std::stoi(frame_str)), Point2D<float>{std::stof(x_str), std::stof(y_str)});
        }
    }
    std::cout.flush();

    std::cout << "Read " << csv_vector.size() << " lines from " << opts.filename << std::endl;

    line_output.insert(csv_vector.begin(), csv_vector.end());

    return line_output;
}

std::map<std::string, std::map<TimeFrameIndex, Point2D<float>>> load_multiple_points_from_csv(std::string const & filename, int const frame_column) {
    std::fstream file;
    file.open(filename, std::fstream::in);

    std::string ln, ele;

    getline(file, ln);// skip the "scorer" row

    getline(file, ln);// bodyparts row
    std::vector<std::string> bodyparts;
    {
        std::stringstream ss(ln);
        while (getline(ss, ele, ',')) {
            bodyparts.push_back(ele);
        }
    }

    getline(file, ln);// coords row
    std::vector<std::string> dims;
    {
        std::stringstream ss(ln);
        while (getline(ss, ele, ',')) {
            dims.push_back(ele);
        }
    }

    std::map<std::string, std::map<TimeFrameIndex, Point2D<float>>> data;
    while (getline(file, ln)) {
        std::stringstream ss(ln);
        size_t col_no = 0;
        TimeFrameIndex frame_no(0);
        while (getline(ss, ele, ',')) {
            if (static_cast<int>(col_no) == frame_column) {
                frame_no = TimeFrameIndex(std::stoi(extract_numbers_from_string(ele)));
            } else if (col_no < dims.size() && dims[col_no] == "x") {
                if (col_no < bodyparts.size()) {
                    data[bodyparts[col_no]][frame_no].x = std::stof(ele);
                }
            } else if (col_no < dims.size() && dims[col_no] == "y") {
                if (col_no < bodyparts.size()) {
                    data[bodyparts[col_no]][frame_no].y = std::stof(ele);
                }
            }
            ++col_no;
        }
    }

    return data;
}

void save(PointData const * point_data, CSVPointSaverOptions const & opts)
{
    //Check if directory exists
    if (!std::filesystem::exists(opts.parent_dir)) {
        std::filesystem::create_directories(opts.parent_dir);
        std::cout << "Created directory: " << opts.parent_dir << std::endl;
    }

    std::string const filename = opts.parent_dir + "/" + opts.filename;
    std::fstream fout;

    fout.open(filename, std::fstream::out);
    if (!fout.is_open()) {
        std::cerr << "Failed to open file for saving: " << filename << std::endl;
        return;
    }

    if (opts.save_header) {
        fout << opts.header << opts.line_delim;
    }

    for (auto const& timePointsPair : point_data->GetAllPointsAsRange()) {
        fout << timePointsPair.time.getValue();
        for (size_t i = 0; i < timePointsPair.points.size(); ++i) {
            fout << opts.delimiter << timePointsPair.points[i].x << opts.delimiter << timePointsPair.points[i].y;
        }
        fout << opts.line_delim;
    }
    fout.close();
    std::cout << "Successfully saved points to " << filename << std::endl;
}

std::map<std::string, std::map<TimeFrameIndex, Point2D<float>>> load_dlc_csv(DLCPointLoaderOptions const & opts) {
    std::fstream file;
    file.open(opts.filename, std::fstream::in);
    
    if (!file.is_open()) {
        std::cerr << "Error: Could not open file " << opts.filename << std::endl;
        return {};
    }

    std::string ln, ele;

    // Skip the "scorer" row (first row)
    getline(file, ln);

    // Read bodyparts row (second row)
    getline(file, ln);
    std::vector<std::string> bodyparts;
    {
        std::stringstream ss(ln);
        while (getline(ss, ele, ',')) {
            bodyparts.push_back(ele);
        }
    }

    // Read coords row (third row)
    getline(file, ln);
    std::vector<std::string> dims;
    {
        std::stringstream ss(ln);
        while (getline(ss, ele, ',')) {
            dims.push_back(ele);
        }
    }

    // Parse data rows
    std::map<std::string, std::map<TimeFrameIndex, Point2D<float>>> data;
    
    while (getline(file, ln)) {
        std::stringstream ss(ln);
        size_t col_no = 0;
        TimeFrameIndex frame_no(0);
        
        // Temporary storage for current row's points
        std::map<std::string, Point2D<float>> temp_points;
        std::map<std::string, float> temp_likelihoods;
        
        while (getline(ss, ele, ',')) {
            if (static_cast<int>(col_no) == opts.frame_column) {
                // For DLC CSV, frame column should already be a pure number, no extraction needed
                frame_no = TimeFrameIndex(std::stoi(ele));
            } else if (col_no < dims.size() && col_no < bodyparts.size()) {
                std::string const& bodypart = bodyparts[col_no];
                std::string const& coord_type = dims[col_no];
                
                if (coord_type == "x") {
                    temp_points[bodypart].x = std::stof(ele);
                } else if (coord_type == "y") {
                    temp_points[bodypart].y = std::stof(ele);
                } else if (coord_type == "likelihood") {
                    temp_likelihoods[bodypart] = std::stof(ele);
                }
            }
            ++col_no;
        }
        
        // Only add points that meet the likelihood threshold
        for (auto const& [bodypart, point] : temp_points) {
            auto likelihood_it = temp_likelihoods.find(bodypart);
            if (likelihood_it != temp_likelihoods.end()) {
                if (likelihood_it->second >= opts.likelihood_threshold) {
                    data[bodypart][frame_no] = point;
                }
            } else {
                // If no likelihood found, add the point (backward compatibility)
                data[bodypart][frame_no] = point;
            }
        }
    }

    file.close();
    
    std::cout << "Loaded DLC CSV with " << data.size() << " bodyparts" << std::endl;
    for (auto const& [bodypart, points] : data) {
        std::cout << "  " << bodypart << ": " << points.size() << " points" << std::endl;
    }
    
    return data;
}

1	#include "Point_Data_CSV.hpp"
2
3	#include "Points/Point_Data.hpp"
4	#include "transforms/data_transforms.hpp"
5	#include "utils/string_manip.hpp"
6
7	#include <filesystem>
8	#include <fstream>
9	#include <iomanip>
10	#include <iostream>
11	#include <sstream>
12
13	//https://stackoverflow.com/questions/4654636/how-to-determine-if-a-string-is-a-number-with-c
14	bool is_number(std::string const & s) {	×
15	return !s.empty() && std::find_if(s.begin(),	×
16	s.end(), [](unsigned char c) { return !std::isdigit(c); }) == s.end();	×
17	}
18
19	std::map<TimeFrameIndex, Point2D<float>> load(CSVPointLoaderOptions const & opts) {	×
20	std::string csv_line;	×
21
22	auto line_output = std::map<TimeFrameIndex, Point2D<float>>{};	×
23
24	std::fstream myfile;	×
25	myfile.open(opts.filename, std::fstream::in);	×
26
27	std::string x_str;	×
28	std::string y_str;	×
29	std::string frame_str;	×
30	std::string col_value;	×
31
32	std::vector<std::pair<TimeFrameIndex, Point2D<float>>> csv_vector = {};	×
33
34	while (getline(myfile, csv_line)) {	×
35
36	std::stringstream ss(csv_line);	×
37
38	int cols_read = 0;	×
39	while (getline(ss, col_value, opts.column_delim)) {	×
40	if (cols_read == opts.frame_column) {	×
41	frame_str = col_value;	×
42	} else if (cols_read == opts.x_column) {	×
43	x_str = col_value;	×
44	} else if (cols_read == opts.y_column) {	×
45	y_str = col_value;	×
46	}
47	cols_read++;	×
48	}
49
50	if (is_number(frame_str)) {	×
51	//line_output[TimeFrameIndex(std::stoi(frame_str))]=Point2D<float>{std::stof(x_str),std::stof(y_str)};
52	csv_vector.emplace_back(TimeFrameIndex(std::stoi(frame_str)), Point2D<float>{std::stof(x_str), std::stof(y_str)});	×
53	}
54	}	×
55	std::cout.flush();	×
56
57	std::cout << "Read " << csv_vector.size() << " lines from " << opts.filename << std::endl;	×
58
59	line_output.insert(csv_vector.begin(), csv_vector.end());	×
60
61	return line_output;	×
62	}	×
63
64	std::map<std::string, std::map<TimeFrameIndex, Point2D<float>>> load_multiple_points_from_csv(std::string const & filename, int const frame_column) {	×
65	std::fstream file;	×
66	file.open(filename, std::fstream::in);	×
67
68	std::string ln, ele;	×
69
70	getline(file, ln);// skip the "scorer" row	×
71
72	getline(file, ln);// bodyparts row	×
73	std::vector<std::string> bodyparts;	×
74	{
75	std::stringstream ss(ln);	×
76	while (getline(ss, ele, ',')) {	×
77	bodyparts.push_back(ele);	×
78	}
79	}	×
80
81	getline(file, ln);// coords row	×
82	std::vector<std::string> dims;	×
83	{
84	std::stringstream ss(ln);	×
85	while (getline(ss, ele, ',')) {	×
86	dims.push_back(ele);	×
87	}
88	}	×
89
90	std::map<std::string, std::map<TimeFrameIndex, Point2D<float>>> data;	×
91	while (getline(file, ln)) {	×
92	std::stringstream ss(ln);	×
93	size_t col_no = 0;	×
94	TimeFrameIndex frame_no(0);	×
95	while (getline(ss, ele, ',')) {	×
96	if (static_cast<int>(col_no) == frame_column) {	×
97	frame_no = TimeFrameIndex(std::stoi(extract_numbers_from_string(ele)));	×
98	} else if (col_no < dims.size() && dims[col_no] == "x") {	×
99	if (col_no < bodyparts.size()) {	×
100	data[bodyparts[col_no]][frame_no].x = std::stof(ele);	×
101	}
102	} else if (col_no < dims.size() && dims[col_no] == "y") {	×
103	if (col_no < bodyparts.size()) {	×
104	data[bodyparts[col_no]][frame_no].y = std::stof(ele);	×
105	}
106	}
107	++col_no;	×
108	}
109	}	×
110
111	return data;	×
112	}	×
113
114	void save(PointData const * point_data, CSVPointSaverOptions const & opts)	×
115	{
116	//Check if directory exists
117	if (!std::filesystem::exists(opts.parent_dir)) {	×
118	std::filesystem::create_directories(opts.parent_dir);	×
119	std::cout << "Created directory: " << opts.parent_dir << std::endl;	×
120	}
121
122	std::string const filename = opts.parent_dir + "/" + opts.filename;	×
123	std::fstream fout;	×
124
125	fout.open(filename, std::fstream::out);	×
126	if (!fout.is_open()) {	×
127	std::cerr << "Failed to open file for saving: " << filename << std::endl;	×
128	return;	×
129	}
130
131	if (opts.save_header) {	×
132	fout << opts.header << opts.line_delim;	×
133	}
134
135	for (auto const& timePointsPair : point_data->GetAllPointsAsRange()) {	×
136	fout << timePointsPair.time.getValue();	×
137	for (size_t i = 0; i < timePointsPair.points.size(); ++i) {	×
138	fout << opts.delimiter << timePointsPair.points[i].x << opts.delimiter << timePointsPair.points[i].y;	×
139	}
140	fout << opts.line_delim;	×
UNCOV 141	}	×
142	fout.close();	×
143	std::cout << "Successfully saved points to " << filename << std::endl;	×
144	}	×
145
146	std::map<std::string, std::map<TimeFrameIndex, Point2D<float>>> load_dlc_csv(DLCPointLoaderOptions const & opts) {	8✔
147	std::fstream file;	8✔
148	file.open(opts.filename, std::fstream::in);	8✔
149
150	if (!file.is_open()) {	8✔
151	std::cerr << "Error: Could not open file " << opts.filename << std::endl;	1✔
152	return {};	1✔
153	}
154
155	std::string ln, ele;	7✔
156
157	// Skip the "scorer" row (first row)
158	getline(file, ln);	7✔
159
160	// Read bodyparts row (second row)
161	getline(file, ln);	7✔
162	std::vector<std::string> bodyparts;	7✔
163	{
164	std::stringstream ss(ln);	7✔
165	while (getline(ss, ele, ',')) {	224✔
166	bodyparts.push_back(ele);	217✔
167	}
168	}	7✔
169
170	// Read coords row (third row)
171	getline(file, ln);	7✔
172	std::vector<std::string> dims;	7✔
173	{
174	std::stringstream ss(ln);	7✔
175	while (getline(ss, ele, ',')) {	224✔
176	dims.push_back(ele);	217✔
177	}
178	}	7✔
179
180	// Parse data rows
181	std::map<std::string, std::map<TimeFrameIndex, Point2D<float>>> data;	7✔
182
183	while (getline(file, ln)) {	42✔
184	std::stringstream ss(ln);	35✔
185	size_t col_no = 0;	35✔
186	TimeFrameIndex frame_no(0);	35✔
187
188	// Temporary storage for current row's points
189	std::map<std::string, Point2D<float>> temp_points;	35✔
190	std::map<std::string, float> temp_likelihoods;	35✔
191
192	while (getline(ss, ele, ',')) {	1,120✔
193	if (static_cast<int>(col_no) == opts.frame_column) {	1,085✔
194	// For DLC CSV, frame column should already be a pure number, no extraction needed
195	frame_no = TimeFrameIndex(std::stoi(ele));	35✔
196	} else if (col_no < dims.size() && col_no < bodyparts.size()) {	1,050✔
197	std::string const& bodypart = bodyparts[col_no];	1,050✔
198	std::string const& coord_type = dims[col_no];	1,050✔
199
200	if (coord_type == "x") {	1,050✔
201	temp_points[bodypart].x = std::stof(ele);	350✔
202	} else if (coord_type == "y") {	700✔
203	temp_points[bodypart].y = std::stof(ele);	350✔
204	} else if (coord_type == "likelihood") {	350✔
205	temp_likelihoods[bodypart] = std::stof(ele);	315✔
206	}
207	}
208	++col_no;	1,085✔
209	}
210
211	// Only add points that meet the likelihood threshold
212	for (auto const& [bodypart, point] : temp_points) {	385✔
213	auto likelihood_it = temp_likelihoods.find(bodypart);	350✔
214	if (likelihood_it != temp_likelihoods.end()) {	350✔
215	if (likelihood_it->second >= opts.likelihood_threshold) {	315✔
216	data[bodypart][frame_no] = point;	295✔
217	}
218	} else {
219	// If no likelihood found, add the point (backward compatibility)
220	data[bodypart][frame_no] = point;	35✔
221	}
222	}
223	}	35✔
224
225	file.close();	7✔
226
227	std::cout << "Loaded DLC CSV with " << data.size() << " bodyparts" << std::endl;	7✔
228	for (auto const& [bodypart, points] : data) {	73✔
229	std::cout << " " << bodypart << ": " << points.size() << " points" << std::endl;	66✔
230	}
231
232	return data;	7✔
233	}	8✔

paulmthompson / WhiskerToolbox / 18389801194

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous