17270491352

Committed 27 Aug 2025 02:57PM UTC coverage: 65.333%. Remained the same

Build # 17270491352

Build Type

push

github

Committed by

paulmthompson

Commit Message

Merge branch 'main' of https://github.com/paulmthompson/WhiskerToolbox

Run Details

352 of 628 new or added lines in 92 files covered. (56.05%)

357 existing lines in 24 files now uncovered.

26429 of 40453 relevant lines covered (65.33%)

1119.34 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

31.5

/src/DataManager/Lines/IO/CSV/Line_Data_CSV.cpp

#include "Line_Data_CSV.hpp"

#include "Lines/Line_Data.hpp"
#include "utils/string_manip.hpp"

#include <chrono>
#include <filesystem>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <sstream>
#include <stdexcept>

void save_line_as_csv(Line2D const & line, std::string const & filename, int const point_precision) {
    std::fstream myfile;
    myfile.open(filename, std::fstream::out);

    myfile << std::fixed << std::setprecision(point_precision);
    for (auto & point: line) {
        myfile << point.x << "," << point.y << "\n";
    }

    myfile.close();
}

void save(
        LineData const * line_data,
        CSVSingleFileLineSaverOptions & opts) {

    //Check if directory exists
    if (!std::filesystem::exists(opts.parent_dir)) {
        std::filesystem::create_directories(opts.parent_dir);
        std::cout << "Created directory: " << opts.parent_dir << std::endl;
    }

    std::string filename = opts.parent_dir + "/" + opts.filename;

    std::ofstream file(filename);
    if (!file.is_open()) {
        throw std::runtime_error("Could not open file");
    }

    // Write the header
    if (opts.save_header) {
        file << opts.header << "\n";
    }

    // Write the data
    for (auto const & frame_and_line: line_data->GetAllLinesAsRange()) {
        for (auto const & line: frame_and_line.lines) {
            std::ostringstream x_values;
            std::ostringstream y_values;

            for (auto const & point: line) {
                x_values << std::fixed << std::setprecision(opts.precision) << point.x << opts.delimiter;
                y_values << std::fixed << std::setprecision(opts.precision) << point.y << opts.delimiter;
            }

            // Remove the trailing delimiter
            std::string x_str = x_values.str();
            std::string y_str = y_values.str();
            if (!x_str.empty()) x_str.pop_back();
            if (!y_str.empty()) y_str.pop_back();

            file << frame_and_line.time.getValue() << ",\"" << x_str << "\",\"" << y_str << "\"\n";
        }
    }

    file.close();
}

void save(
        LineData const * line_data,
        CSVMultiFileLineSaverOptions & opts) {

    // Check if directory exists
    if (!std::filesystem::exists(opts.parent_dir)) {
        std::filesystem::create_directories(opts.parent_dir);
        std::cout << "Created directory: " << opts.parent_dir << std::endl;
    }

    int files_saved = 0;
    int files_skipped = 0;

    // Iterate through all timestamps with data
    for (auto const & frame_and_line: line_data->GetAllLinesAsRange()) {
        // Only save if there are lines at this timestamp
        if (frame_and_line.lines.empty()) {
            files_skipped++;
            continue;
        }

        // Only save the first line (index 0) as documented
        Line2D const & first_line = frame_and_line.lines[0];
        
        // Generate filename with zero-padded frame number
        std::string const padded_frame = pad_frame_id(static_cast<int>(frame_and_line.time.getValue()), opts.frame_id_padding);
        std::string const filename = opts.parent_dir + "/" + padded_frame + ".csv";

        // Check if file exists and handle according to overwrite setting
        bool file_exists = std::filesystem::exists(filename);
        if (file_exists && !opts.overwrite_existing) {
            std::cout << "Skipping existing file: " << filename << std::endl;
            files_skipped++;
            continue;
        }

        // Log if we're overwriting an existing file
        if (file_exists && opts.overwrite_existing) {
            std::cout << "Overwriting existing file: " << filename << std::endl;
        }

        std::ofstream file(filename);
        if (!file.is_open()) {
            std::cerr << "Warning: Could not open file " << filename << " for writing" << std::endl;
            files_skipped++;
            continue;
        }

        // Write the header if requested
        if (opts.save_header) {
            file << opts.header << opts.line_delim;
        }

        // Write X and Y coordinates in separate columns
        file << std::fixed << std::setprecision(opts.precision);
        for (auto const & point: first_line) {
            file << point.x << opts.delimiter << point.y << opts.line_delim;
        }

        file.close();
        files_saved++;
    }

    std::cout << "Multi-file CSV save complete: " << files_saved << " files saved";
    if (files_skipped > 0) {
        std::cout << ", " << files_skipped << " timestamps skipped (no lines or file errors)";
    }
    std::cout << std::endl;
}

std::vector<float> parse_string_to_float_vector(std::string const & str, std::string const & delimiter) {
    std::vector<float> result;

    std::stringstream ss(str);
    std::string item;

    char delim_char = delimiter.empty() ? ',' : delimiter[0];
    while (std::getline(ss, item, delim_char)) {
        result.push_back(std::stof(item));
    }
    return result;
}

std::map<TimeFrameIndex, std::vector<Line2D>> load(CSVSingleFileLineLoaderOptions const & opts) {
    auto t1 = std::chrono::high_resolution_clock::now();
    std::map<TimeFrameIndex, std::vector<Line2D>> data_map;
    std::ifstream file(opts.filepath);
    if (!file.is_open()) {
        throw std::runtime_error("Could not open file: " + opts.filepath);
    }

    std::string line;
    int loaded_lines = 0;
    
    while (std::getline(file, line)) {
        std::istringstream ss(line);
        std::string frame_num_str, x_str, y_str;

        // Get frame number (first column)
        std::getline(ss, frame_num_str, opts.delimiter[0]);

        // Skip header if present
        if (opts.has_header && frame_num_str == opts.header_identifier) {
            continue;
        }

        // Get X coordinates (second column, enclosed in quotes)
        std::getline(ss, x_str, '"');
        std::getline(ss, x_str, '"');

        // Get Y coordinates (third column, enclosed in quotes)
        std::getline(ss, y_str, '"');
        std::getline(ss, y_str, '"');

        int const frame_num = std::stoi(frame_num_str);

        std::vector<float> const x_values = parse_string_to_float_vector(x_str, opts.coordinate_delimiter);
        std::vector<float> const y_values = parse_string_to_float_vector(y_str, opts.coordinate_delimiter);

        if (x_values.size() != y_values.size()) {
            std::cerr << "Mismatched x and y values at frame: " << frame_num << std::endl;
            continue;
        }

        if (data_map.find(TimeFrameIndex(frame_num)) == data_map.end()) {
            data_map[TimeFrameIndex(frame_num)] = std::vector<Line2D>();
        }

        data_map[TimeFrameIndex(frame_num)].emplace_back(create_line(x_values, y_values));
        loaded_lines += 1;
    }

    file.close();
    auto t2 = std::chrono::high_resolution_clock::now();

    auto duration = std::chrono::duration<double>(t2 - t1).count();
    std::cout << "Loaded " << loaded_lines << " lines from " << opts.filepath << " in " << duration << "s" << std::endl;
    return data_map;
}

std::map<TimeFrameIndex, std::vector<Line2D>> load_line_csv(std::string const & filepath) {
    // Wrapper function for backward compatibility
    // Uses the new options-based load function with default settings
    CSVSingleFileLineLoaderOptions opts;
    opts.filepath = filepath;
    // All other options use their default values which match the original hardcoded behavior
    return load(opts);
}

Line2D load_line_from_csv(std::string const & filename) {
    std::string csv_line;

    auto line_output = Line2D();

    std::fstream myfile;
    myfile.open(filename, std::fstream::in);

    std::string x_str;
    std::string y_str;

    while (getline(myfile, csv_line)) {

        std::stringstream ss(csv_line);

        getline(ss, x_str, ',');
        getline(ss, y_str);

        //std::cout << x_str << " , " << y_str << std::endl;

        line_output.push_back(Point2D<float>{std::stof(x_str), std::stof(y_str)});
    }

    return line_output;
}

std::map<TimeFrameIndex, std::vector<Line2D>> load(CSVMultiFileLineLoaderOptions const & opts) {
    std::map<TimeFrameIndex, std::vector<Line2D>> data_map;
    
    // Check if directory exists
    if (!std::filesystem::exists(opts.parent_dir)) {
        std::cerr << "Error: Directory does not exist: " << opts.parent_dir << std::endl;
        return data_map;
    }

    int files_loaded = 0;
    int files_skipped = 0;

    // Iterate through all files in the directory
    for (auto const & entry : std::filesystem::directory_iterator(opts.parent_dir)) {
        if (!entry.is_regular_file()) {
            continue;
        }

        std::string const filename = entry.path().filename().string();
        
        // Check if file matches the pattern (simple check for .csv extension)
        if (filename.length() < 4 || filename.substr(filename.length() - 4) != ".csv") {
            continue;
        }

        // Extract frame number from filename (remove .csv extension)
        std::string const frame_str = filename.substr(0, filename.length() - 4);
        
        // Try to parse frame number
        int frame_number;
        try {
            frame_number = std::stoi(frame_str);
        } catch (std::exception const & e) {
            std::cerr << "Warning: Could not parse frame number from filename: " << filename << std::endl;
            files_skipped++;
            continue;
        }

        // Load the CSV file
        std::ifstream file(entry.path());
        if (!file.is_open()) {
            std::cerr << "Warning: Could not open file: " << entry.path() << std::endl;
            files_skipped++;
            continue;
        }

        std::vector<Point2D<float>> line_points;
        std::string line;
        bool first_line = true;

        while (std::getline(file, line)) {
            // Skip header if present
            if (first_line && opts.has_header) {
                first_line = false;
                continue;
            }
            first_line = false;

            // Parse the line
            std::stringstream ss(line);
            std::vector<std::string> columns;
            std::string column;

            // Split by delimiter
            while (std::getline(ss, column, opts.delimiter[0])) {
                columns.push_back(column);
            }

            // Check if we have enough columns
            int max_column = std::max(opts.x_column, opts.y_column);
            if (columns.size() <= static_cast<size_t>(max_column)) {
                std::cerr << "Warning: Not enough columns in line: " << line << " (file: " << filename << ")" << std::endl;
                continue;
            }

            // Parse X and Y coordinates
            try {
                float x = std::stof(columns[static_cast<size_t>(opts.x_column)]);
                float y = std::stof(columns[static_cast<size_t>(opts.y_column)]);
                line_points.push_back(Point2D<float>{x, y});
            } catch (std::exception const & e) {
                std::cerr << "Warning: Could not parse coordinates from line: " << line << " (file: " << filename << ")" << std::endl;
                continue;
            }
        }

        file.close();

        // Add the line to the data map if we have points
        if (!line_points.empty()) {
            if (data_map.find(TimeFrameIndex(frame_number)) == data_map.end()) {
                data_map[TimeFrameIndex(frame_number)] = std::vector<Line2D>();
            }
            data_map[TimeFrameIndex(frame_number)].push_back(line_points);
            files_loaded++;
        } else {
            std::cerr << "Warning: No valid points found in file: " << filename << std::endl;
            files_skipped++;
        }
    }

    std::cout << "Multi-file CSV load complete: " << files_loaded << " files loaded";
    if (files_skipped > 0) {
        std::cout << ", " << files_skipped << " files skipped";
    }
    std::cout << std::endl;

    return data_map;
}

1	#include "Line_Data_CSV.hpp"
2
3	#include "Lines/Line_Data.hpp"
4	#include "utils/string_manip.hpp"
5
6	#include <chrono>
7	#include <filesystem>
8	#include <fstream>
9	#include <iomanip>
10	#include <iostream>
11	#include <sstream>
12	#include <stdexcept>
13
14	void save_line_as_csv(Line2D const & line, std::string const & filename, int const point_precision) {	×
15	std::fstream myfile;	×
16	myfile.open(filename, std::fstream::out);	×
17
18	myfile << std::fixed << std::setprecision(point_precision);	×
19	for (auto & point: line) {	×
20	myfile << point.x << "," << point.y << "\n";	×
21	}
22
23	myfile.close();	×
24	}	×
25
26	void save(	5✔
27	LineData const * line_data,
28	CSVSingleFileLineSaverOptions & opts) {
29
30	//Check if directory exists
31	if (!std::filesystem::exists(opts.parent_dir)) {	5✔
32	std::filesystem::create_directories(opts.parent_dir);	×
33	std::cout << "Created directory: " << opts.parent_dir << std::endl;	×
34	}
35
36	std::string filename = opts.parent_dir + "/" + opts.filename;	5✔
37
38	std::ofstream file(filename);	5✔
39	if (!file.is_open()) {	5✔
40	throw std::runtime_error("Could not open file");	×
41	}
42
43	// Write the header
44	if (opts.save_header) {	5✔
45	file << opts.header << "\n";	5✔
46	}
47
48	// Write the data
49	for (auto const & frame_and_line: line_data->GetAllLinesAsRange()) {	15✔
50	for (auto const & line: frame_and_line.lines) {	25✔
51	std::ostringstream x_values;	15✔
52	std::ostringstream y_values;	15✔
53
54	for (auto const & point: line) {	65✔
55	x_values << std::fixed << std::setprecision(opts.precision) << point.x << opts.delimiter;	50✔
56	y_values << std::fixed << std::setprecision(opts.precision) << point.y << opts.delimiter;	50✔
57	}
58
59	// Remove the trailing delimiter
60	std::string x_str = x_values.str();	15✔
61	std::string y_str = y_values.str();	15✔
62	if (!x_str.empty()) x_str.pop_back();	15✔
63	if (!y_str.empty()) y_str.pop_back();	15✔
64
65	file << frame_and_line.time.getValue() << ",\"" << x_str << "\",\"" << y_str << "\"\n";	15✔
66	}	15✔
67	}
68
69	file.close();	5✔
70	}	10✔
71
72	void save(	×
73	LineData const * line_data,
74	CSVMultiFileLineSaverOptions & opts) {
75
76	// Check if directory exists
77	if (!std::filesystem::exists(opts.parent_dir)) {	×
78	std::filesystem::create_directories(opts.parent_dir);	×
79	std::cout << "Created directory: " << opts.parent_dir << std::endl;	×
80	}
81
82	int files_saved = 0;	×
83	int files_skipped = 0;	×
84
85	// Iterate through all timestamps with data
86	for (auto const & frame_and_line: line_data->GetAllLinesAsRange()) {	×
87	// Only save if there are lines at this timestamp
88	if (frame_and_line.lines.empty()) {	×
89	files_skipped++;	×
90	continue;	×
91	}
92
93	// Only save the first line (index 0) as documented
94	Line2D const & first_line = frame_and_line.lines[0];	×
95
96	// Generate filename with zero-padded frame number
NEW 97	std::string const padded_frame = pad_frame_id(static_cast<int>(frame_and_line.time.getValue()), opts.frame_id_padding);	×
98	std::string const filename = opts.parent_dir + "/" + padded_frame + ".csv";	×
99
100	// Check if file exists and handle according to overwrite setting
101	bool file_exists = std::filesystem::exists(filename);	×
102	if (file_exists && !opts.overwrite_existing) {	×
103	std::cout << "Skipping existing file: " << filename << std::endl;	×
104	files_skipped++;	×
105	continue;	×
106	}
107
108	// Log if we're overwriting an existing file
109	if (file_exists && opts.overwrite_existing) {	×
110	std::cout << "Overwriting existing file: " << filename << std::endl;	×
111	}
112
113	std::ofstream file(filename);	×
114	if (!file.is_open()) {	×
115	std::cerr << "Warning: Could not open file " << filename << " for writing" << std::endl;	×
116	files_skipped++;	×
117	continue;	×
118	}
119
120	// Write the header if requested
121	if (opts.save_header) {	×
122	file << opts.header << opts.line_delim;	×
123	}
124
125	// Write X and Y coordinates in separate columns
126	file << std::fixed << std::setprecision(opts.precision);	×
127	for (auto const & point: first_line) {	×
128	file << point.x << opts.delimiter << point.y << opts.line_delim;	×
129	}
130
131	file.close();	×
132	files_saved++;	×
133	}	×
134
135	std::cout << "Multi-file CSV save complete: " << files_saved << " files saved";	×
136	if (files_skipped > 0) {	×
137	std::cout << ", " << files_skipped << " timestamps skipped (no lines or file errors)";	×
138	}
139	std::cout << std::endl;	×
140	}	×
141
142	std::vector<float> parse_string_to_float_vector(std::string const & str, std::string const & delimiter) {	18✔
143	std::vector<float> result;	18✔
144
145	std::stringstream ss(str);	18✔
146	std::string item;	18✔
147
148	char delim_char = delimiter.empty() ? ',' : delimiter[0];	18✔
149	while (std::getline(ss, item, delim_char)) {	78✔
150	result.push_back(std::stof(item));	60✔
151	}
152	return result;	36✔
153	}	18✔
154
155	std::map<TimeFrameIndex, std::vector<Line2D>> load(CSVSingleFileLineLoaderOptions const & opts) {	3✔
156	auto t1 = std::chrono::high_resolution_clock::now();	3✔
157	std::map<TimeFrameIndex, std::vector<Line2D>> data_map;	3✔
158	std::ifstream file(opts.filepath);	3✔
159	if (!file.is_open()) {	3✔
160	throw std::runtime_error("Could not open file: " + opts.filepath);	×
161	}
162
163	std::string line;	3✔
164	int loaded_lines = 0;	3✔
165
166	while (std::getline(file, line)) {	15✔
167	std::istringstream ss(line);	12✔
168	std::string frame_num_str, x_str, y_str;	12✔
169
170	// Get frame number (first column)
171	std::getline(ss, frame_num_str, opts.delimiter[0]);	12✔
172
173	// Skip header if present
174	if (opts.has_header && frame_num_str == opts.header_identifier) {	12✔
175	continue;	3✔
176	}
177
178	// Get X coordinates (second column, enclosed in quotes)
179	std::getline(ss, x_str, '"');	9✔
180	std::getline(ss, x_str, '"');	9✔
181
182	// Get Y coordinates (third column, enclosed in quotes)
183	std::getline(ss, y_str, '"');	9✔
184	std::getline(ss, y_str, '"');	9✔
185
186	int const frame_num = std::stoi(frame_num_str);	9✔
187
188	std::vector<float> const x_values = parse_string_to_float_vector(x_str, opts.coordinate_delimiter);	9✔
189	std::vector<float> const y_values = parse_string_to_float_vector(y_str, opts.coordinate_delimiter);	9✔
190
191	if (x_values.size() != y_values.size()) {	9✔
192	std::cerr << "Mismatched x and y values at frame: " << frame_num << std::endl;	×
193	continue;	×
194	}
195
196	if (data_map.find(TimeFrameIndex(frame_num)) == data_map.end()) {	9✔
197	data_map[TimeFrameIndex(frame_num)] = std::vector<Line2D>();	6✔
198	}
199
200	data_map[TimeFrameIndex(frame_num)].emplace_back(create_line(x_values, y_values));	9✔
201	loaded_lines += 1;	9✔
202	}	21✔
203
204	file.close();	3✔
205	auto t2 = std::chrono::high_resolution_clock::now();	3✔
206
207	auto duration = std::chrono::duration<double>(t2 - t1).count();	3✔
208	std::cout << "Loaded " << loaded_lines << " lines from " << opts.filepath << " in " << duration << "s" << std::endl;	3✔
209	return data_map;	6✔
210	}	3✔
211
212	std::map<TimeFrameIndex, std::vector<Line2D>> load_line_csv(std::string const & filepath) {	×
213	// Wrapper function for backward compatibility
214	// Uses the new options-based load function with default settings
215	CSVSingleFileLineLoaderOptions opts;	×
216	opts.filepath = filepath;	×
217	// All other options use their default values which match the original hardcoded behavior
218	return load(opts);	×
219	}	×
220
221	Line2D load_line_from_csv(std::string const & filename) {	×
222	std::string csv_line;	×
223
224	auto line_output = Line2D();	×
225
226	std::fstream myfile;	×
227	myfile.open(filename, std::fstream::in);	×
228
229	std::string x_str;	×
230	std::string y_str;	×
231
232	while (getline(myfile, csv_line)) {	×
233
234	std::stringstream ss(csv_line);	×
235
236	getline(ss, x_str, ',');	×
237	getline(ss, y_str);	×
238
239	//std::cout << x_str << " , " << y_str << std::endl;
240
241	line_output.push_back(Point2D<float>{std::stof(x_str), std::stof(y_str)});	×
242	}	×
243
244	return line_output;	×
245	}	×
246
247	std::map<TimeFrameIndex, std::vector<Line2D>> load(CSVMultiFileLineLoaderOptions const & opts) {	×
248	std::map<TimeFrameIndex, std::vector<Line2D>> data_map;	×
249
250	// Check if directory exists
251	if (!std::filesystem::exists(opts.parent_dir)) {	×
252	std::cerr << "Error: Directory does not exist: " << opts.parent_dir << std::endl;	×
253	return data_map;	×
254	}
255
256	int files_loaded = 0;	×
257	int files_skipped = 0;	×
258
259	// Iterate through all files in the directory
260	for (auto const & entry : std::filesystem::directory_iterator(opts.parent_dir)) {	×
261	if (!entry.is_regular_file()) {	×
262	continue;	×
263	}
264
265	std::string const filename = entry.path().filename().string();	×
266
267	// Check if file matches the pattern (simple check for .csv extension)
268	if (filename.length() < 4 \|\| filename.substr(filename.length() - 4) != ".csv") {	×
269	continue;	×
270	}
271
272	// Extract frame number from filename (remove .csv extension)
273	std::string const frame_str = filename.substr(0, filename.length() - 4);	×
274
275	// Try to parse frame number
276	int frame_number;
277	try {
278	frame_number = std::stoi(frame_str);	×
279	} catch (std::exception const & e) {	×
280	std::cerr << "Warning: Could not parse frame number from filename: " << filename << std::endl;	×
281	files_skipped++;	×
282	continue;	×
283	}	×
284
285	// Load the CSV file
286	std::ifstream file(entry.path());	×
287	if (!file.is_open()) {	×
288	std::cerr << "Warning: Could not open file: " << entry.path() << std::endl;	×
289	files_skipped++;	×
290	continue;	×
291	}
292
293	std::vector<Point2D<float>> line_points;	×
294	std::string line;	×
295	bool first_line = true;	×
296
297	while (std::getline(file, line)) {	×
298	// Skip header if present
299	if (first_line && opts.has_header) {	×
300	first_line = false;	×
301	continue;	×
302	}
303	first_line = false;	×
304
305	// Parse the line
306	std::stringstream ss(line);	×
307	std::vector<std::string> columns;	×
308	std::string column;	×
309
310	// Split by delimiter
311	while (std::getline(ss, column, opts.delimiter[0])) {	×
312	columns.push_back(column);	×
313	}
314
315	// Check if we have enough columns
316	int max_column = std::max(opts.x_column, opts.y_column);	×
NEW 317	if (columns.size() <= static_cast<size_t>(max_column)) {	×
318	std::cerr << "Warning: Not enough columns in line: " << line << " (file: " << filename << ")" << std::endl;	×
319	continue;	×
320	}
321
322	// Parse X and Y coordinates
323	try {
NEW 324	float x = std::stof(columns[static_cast<size_t>(opts.x_column)]);	×
NEW 325	float y = std::stof(columns[static_cast<size_t>(opts.y_column)]);	×
326	line_points.push_back(Point2D<float>{x, y});	×
327	} catch (std::exception const & e) {	×
328	std::cerr << "Warning: Could not parse coordinates from line: " << line << " (file: " << filename << ")" << std::endl;	×
329	continue;	×
330	}	×
331	}	×
332
333	file.close();	×
334
335	// Add the line to the data map if we have points
336	if (!line_points.empty()) {	×
337	if (data_map.find(TimeFrameIndex(frame_number)) == data_map.end()) {	×
338	data_map[TimeFrameIndex(frame_number)] = std::vector<Line2D>();	×
339	}
340	data_map[TimeFrameIndex(frame_number)].push_back(line_points);	×
341	files_loaded++;	×
342	} else {
343	std::cerr << "Warning: No valid points found in file: " << filename << std::endl;	×
344	files_skipped++;	×
345	}
346	}	×
347
348	std::cout << "Multi-file CSV load complete: " << files_loaded << " files loaded";	×
349	if (files_skipped > 0) {	×
350	std::cout << ", " << files_skipped << " files skipped";	×
351	}
352	std::cout << std::endl;	×
353
354	return data_map;	×
355	}	×

paulmthompson / WhiskerToolbox / 17270491352

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous