11254817533

Committed 09 Oct 2024 12:20PM UTC coverage: 77.697% (+0.08%) from 77.615%

Build # 11254817533

Build Type

Pull #476

github

Committed by

mgeplf

Commit Message

Merge remote-tracking branch 'origin/master' into efficient-swc-build

Pull Request Pull Request #476: Efficient swc build

Run Details

286 of 321 new or added lines in 10 files covered. (89.1%)

9 existing lines in 4 files now uncovered.

2125 of 2735 relevant lines covered (77.7%)

901.01 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

52.5

/src/mut/writer_hdf5.cpp

/* Copyright (c) 2013-2023, EPFL/Blue Brain Project
 *
 * SPDX-License-Identifier: Apache-2.0
 */
#include <morphio/mut/mitochondria.h>
#include <morphio/mut/morphology.h>
#include <morphio/mut/section.h>
#include <morphio/mut/writers.h>
#include <morphio/version.h>
#include <morphio/warning_handling.h>

#include <highfive/H5DataSet.hpp>
#include <highfive/H5File.hpp>
#include <highfive/H5Object.hpp>

#include "../error_message_generation.h"
#include "../shared_utils.hpp"
#include "morphio/exceptions.h"
#include "writer_utils.h"

namespace {

/**
   A structure to get the base type of nested vectors
   https://stackoverflow.com/a/30960730
 **/
template <typename T>
struct base_type {
    using type = T;
};

template <typename T>
struct base_type<std::vector<T>>: base_type<T> {};

template <typename T>
HighFive::Attribute write_attribute(HighFive::File& file,
                                    const std::string& name,
                                    const T& version) {
    HighFive::Attribute a_version =
        file.createAttribute<typename T::value_type>(name, HighFive::DataSpace::From(version));
    a_version.write(version);
    return a_version;
}

template <typename T>
HighFive::Attribute write_attribute(HighFive::Group& group,
                                    const std::string& name,
                                    const T& version) {
    HighFive::Attribute a_version =
        group.createAttribute<typename T::value_type>(name, HighFive::DataSpace::From(version));
    a_version.write(version);
    return a_version;
}

template <typename T>
void write_dataset(HighFive::File& file, const std::string& name, const T& raw) {
    HighFive::DataSet dpoints =
        file.createDataSet<typename base_type<T>::type>(name, HighFive::DataSpace::From(raw));

    dpoints.write(raw);
}

template <typename T>
void write_dataset(HighFive::Group& file, const std::string& name, const T& raw) {
    HighFive::DataSet dpoints =
        file.createDataSet<typename base_type<T>::type>(name, HighFive::DataSpace::From(raw));

    dpoints.write(raw);
}


}  // anonymous namespace

namespace morphio {
namespace mut {
namespace writer {
namespace {

void mitochondriaH5(HighFive::File& h5_file, const Mitochondria& mitochondria) {
    if (mitochondria.rootSections().empty()) {
        return;
    }

    Property::Properties properties;
    mitochondria._buildMitochondria(properties);
    auto& p = properties._mitochondriaPointLevel;
    size_t size = p._diameters.size();

    std::vector<std::vector<morphio::floatType>> points;
    std::vector<std::vector<int32_t>> structure;
    points.reserve(size);
    for (unsigned int i = 0; i < size; ++i) {
        points.push_back({static_cast<morphio::floatType>(p._sectionIds[i]),
                          p._relativePathLengths[i],
                          p._diameters[i]});
    }

    auto& s = properties._mitochondriaSectionLevel;
    structure.reserve(s._sections.size());
    for (const auto& section : s._sections) {
        structure.push_back({section[0], section[1]});
    }

    HighFive::Group g_organelles = h5_file.createGroup("organelles");
    HighFive::Group g_mitochondria = g_organelles.createGroup("mitochondria");

    write_dataset(g_mitochondria, "points", points);
    write_dataset(g_mitochondria, "structure", structure);
}


void endoplasmicReticulumH5(HighFive::File& h5_file, const EndoplasmicReticulum& reticulum) {
    if (reticulum.sectionIndices().empty()) {
        return;
    }

    HighFive::Group g_organelles = h5_file.createGroup("organelles");
    HighFive::Group g_reticulum = g_organelles.createGroup("endoplasmic_reticulum");

    write_dataset(g_reticulum, "section_index", reticulum.sectionIndices());
    write_dataset(g_reticulum, "volume", reticulum.volumes());
    write_dataset(g_reticulum, "filament_count", reticulum.filamentCounts());
    write_dataset(g_reticulum, "surface_area", reticulum.surfaceAreas());
}

void dendriticSpinePostSynapticDensityH5(HighFive::File& h5_file,
                                         const Property::DendriticSpine::Level& l) {
    const auto& psd = l._post_synaptic_density;

    HighFive::Group g_organelles = h5_file.createGroup("organelles");
    HighFive::Group g_postsynaptic_density = g_organelles.createGroup("postsynaptic_density");

    std::vector<morphio::Property::DendriticSpine::SectionId_t> sectionIds;
    sectionIds.reserve(psd.size());
    std::vector<morphio::Property::DendriticSpine::SegmentId_t> segmentIds;
    segmentIds.reserve(psd.size());
    std::vector<morphio::Property::DendriticSpine::Offset_t> offsets;
    offsets.reserve(psd.size());

    for (const auto& v : psd) {
        sectionIds.push_back(v.sectionId);
        segmentIds.push_back(v.segmentId);
        offsets.push_back(v.offset);
    }
    write_dataset(g_postsynaptic_density, "section_id", sectionIds);
    write_dataset(g_postsynaptic_density, "segment_id", segmentIds);
    write_dataset(g_postsynaptic_density, "offset", offsets);
}
}  // anonymous namespace

void h5(const Morphology& morph,
        const std::string& filename,
        std::shared_ptr<morphio::WarningHandler> handler) {
    if (details::emptyMorphology(morph, handler)) {
        throw morphio::WriterError(morphio::details::ErrorMessages().ERROR_EMPTY_MORPHOLOGY());
    }

    details::validateContourSoma(morph, handler);
    details::checkSomaHasSameNumberPointsDiameters(*morph.soma());
    details::validateRootPointsHaveTwoOrMorePoints(morph);

    HighFive::File h5_file(filename,
                           HighFive::File::ReadWrite | HighFive::File::Create |
                               HighFive::File::Truncate);

    std::unordered_map<uint32_t, int32_t> newIds;

    std::vector<std::vector<morphio::floatType>> raw_points;
    std::vector<std::vector<int32_t>> raw_structure;
    std::vector<morphio::floatType> raw_perimeters;

    const std::vector<Point>& somaPoints = morph.soma()->points();
    const auto& somaDiameters = morph.soma()->diameters();
    for (unsigned int i = 0; i < somaPoints.size(); ++i) {
        raw_points.push_back(
            {somaPoints[i][0], somaPoints[i][1], somaPoints[i][2], somaDiameters[i]});

        // If the morphology has some perimeter data, we need to fill some
        // perimeter dummy value in the soma range of the data structure to keep
        // the length matching
        if (details::hasPerimeterData(morph)) {
            raw_perimeters.push_back(0);
        }
    }

    raw_structure.push_back({0, SECTION_SOMA, -1});
    size_t offset = morph.soma()->points().size();

    int sectionIdOnDisk = 1;
    for (auto it = morph.depth_begin(); it != morph.depth_end(); ++it) {
        const std::shared_ptr<Section>& section = *it;

        const auto& points = section->points();
        const auto& diameters = section->diameters();
        const auto& perimeters = section->perimeters();

        int parentOnDisk = (section->isRoot() ? 0 : newIds[section->parent()->id()]);
        raw_structure.push_back({static_cast<int>(offset), section->type(), parentOnDisk});

        const auto numberOfPoints = points.size();
        for (unsigned int i = 0; i < numberOfPoints; ++i) {
            raw_points.push_back({points[i][0], points[i][1], points[i][2], diameters[i]});
        }

        const auto numberOfPerimeters = perimeters.size();
        if (numberOfPerimeters > 0) {
            if (numberOfPerimeters != numberOfPoints) {
                auto error = morphio::details::ErrorMessages().ERROR_VECTOR_LENGTH_MISMATCH(
                    "points", numberOfPoints, "perimeters", numberOfPerimeters);
                throw WriterError(error);
            }
            for (unsigned int i = 0; i < numberOfPerimeters; ++i) {
                raw_perimeters.push_back(perimeters[i]);
            }
        }

        newIds[section->id()] = sectionIdOnDisk++;
        offset += numberOfPoints;
    }

    write_dataset(h5_file, "/points", raw_points);
    write_dataset(h5_file, "/structure", raw_structure);

    HighFive::Group g_metadata = h5_file.createGroup("metadata");

    write_attribute(g_metadata, "version", std::array<uint32_t, 2>{1, 3});
    write_attribute(g_metadata,
                    "cell_family",
                    std::vector<uint32_t>{static_cast<uint32_t>(morph.cellFamily())});
    write_attribute(h5_file, "comment", std::vector<std::string>{details::version_string()});

    if (details::hasPerimeterData(morph)) {
        write_dataset(h5_file, "/perimeters", raw_perimeters);
    }

    mitochondriaH5(h5_file, morph.mitochondria());
    endoplasmicReticulumH5(h5_file, morph.endoplasmicReticulum());
    if (morph.cellFamily() == SPINE) {
        dendriticSpinePostSynapticDensityH5(h5_file, morph._dendriticSpineLevel);
    }
}

}  // end namespace writer
}  // end namespace mut
}  // end namespace morphio

1	/* Copyright (c) 2013-2023, EPFL/Blue Brain Project
2	*
3	* SPDX-License-Identifier: Apache-2.0
4	*/
5	#include <morphio/mut/mitochondria.h>
6	#include <morphio/mut/morphology.h>
7	#include <morphio/mut/section.h>
8	#include <morphio/mut/writers.h>
9	#include <morphio/version.h>
10	#include <morphio/warning_handling.h>
11
12	#include <highfive/H5DataSet.hpp>
13	#include <highfive/H5File.hpp>
14	#include <highfive/H5Object.hpp>
15
16	#include "../error_message_generation.h"
17	#include "../shared_utils.hpp"
18	#include "morphio/exceptions.h"
19	#include "writer_utils.h"
20
21	namespace {
22
23	/**
24	A structure to get the base type of nested vectors
25	https://stackoverflow.com/a/30960730
26	**/
27	template <typename T>
28	struct base_type {
29	using type = T;
30	};
31
32	template <typename T>
33	struct base_type<std::vector<T>>: base_type<T> {};
34
35	template <typename T>
36	HighFive::Attribute write_attribute(HighFive::File& file,	2✔
37	const std::string& name,
38	const T& version) {
39	HighFive::Attribute a_version =	2✔
40	file.createAttribute<typename T::value_type>(name, HighFive::DataSpace::From(version));
41	a_version.write(version);	2✔
42	return a_version;	2✔
43	}
44
45	template <typename T>
46	HighFive::Attribute write_attribute(HighFive::Group& group,	4✔
47	const std::string& name,
48	const T& version) {
49	HighFive::Attribute a_version =	4✔
50	group.createAttribute<typename T::value_type>(name, HighFive::DataSpace::From(version));
51	a_version.write(version);	4✔
52	return a_version;	4✔
53	}
54
55	template <typename T>
56	void write_dataset(HighFive::File& file, const std::string& name, const T& raw) {	4✔
57	HighFive::DataSet dpoints =	4✔
58	file.createDataSet<typename base_type<T>::type>(name, HighFive::DataSpace::From(raw));
59
60	dpoints.write(raw);	4✔
61	}	4✔
62
63	template <typename T>
64	void write_dataset(HighFive::Group& file, const std::string& name, const T& raw) {	×
65	HighFive::DataSet dpoints =	×
66	file.createDataSet<typename base_type<T>::type>(name, HighFive::DataSpace::From(raw));
67
68	dpoints.write(raw);	×
69	}	×
70
71
72	} // anonymous namespace
73
74	namespace morphio {
75	namespace mut {
76	namespace writer {
77	namespace {
78
79	void mitochondriaH5(HighFive::File& h5_file, const Mitochondria& mitochondria) {	2✔
80	if (mitochondria.rootSections().empty()) {	2✔
81	return;	2✔
82	}
83
84	Property::Properties properties;	×
85	mitochondria._buildMitochondria(properties);	×
86	auto& p = properties._mitochondriaPointLevel;	×
87	size_t size = p._diameters.size();	×
88
89	std::vector<std::vector<morphio::floatType>> points;	×
90	std::vector<std::vector<int32_t>> structure;	×
91	points.reserve(size);	×
92	for (unsigned int i = 0; i < size; ++i) {	×
93	points.push_back({static_cast<morphio::floatType>(p._sectionIds[i]),	×
94	p._relativePathLengths[i],	×
95	p._diameters[i]});	×
96	}
97
98	auto& s = properties._mitochondriaSectionLevel;	×
99	structure.reserve(s._sections.size());	×
100	for (const auto& section : s._sections) {	×
101	structure.push_back({section[0], section[1]});	×
102	}
103
104	HighFive::Group g_organelles = h5_file.createGroup("organelles");	×
105	HighFive::Group g_mitochondria = g_organelles.createGroup("mitochondria");	×
106
107	write_dataset(g_mitochondria, "points", points);	×
108	write_dataset(g_mitochondria, "structure", structure);	×
109	}
110
111
112	void endoplasmicReticulumH5(HighFive::File& h5_file, const EndoplasmicReticulum& reticulum) {	2✔
113	if (reticulum.sectionIndices().empty()) {	2✔
114	return;	2✔
115	}
116
117	HighFive::Group g_organelles = h5_file.createGroup("organelles");	×
118	HighFive::Group g_reticulum = g_organelles.createGroup("endoplasmic_reticulum");	×
119
120	write_dataset(g_reticulum, "section_index", reticulum.sectionIndices());	×
121	write_dataset(g_reticulum, "volume", reticulum.volumes());	×
122	write_dataset(g_reticulum, "filament_count", reticulum.filamentCounts());	×
123	write_dataset(g_reticulum, "surface_area", reticulum.surfaceAreas());	×
124	}
125
126	void dendriticSpinePostSynapticDensityH5(HighFive::File& h5_file,	×
127	const Property::DendriticSpine::Level& l) {
128	const auto& psd = l._post_synaptic_density;	×
129
130	HighFive::Group g_organelles = h5_file.createGroup("organelles");	×
131	HighFive::Group g_postsynaptic_density = g_organelles.createGroup("postsynaptic_density");	×
132
133	std::vector<morphio::Property::DendriticSpine::SectionId_t> sectionIds;	×
134	sectionIds.reserve(psd.size());	×
135	std::vector<morphio::Property::DendriticSpine::SegmentId_t> segmentIds;	×
136	segmentIds.reserve(psd.size());	×
137	std::vector<morphio::Property::DendriticSpine::Offset_t> offsets;	×
138	offsets.reserve(psd.size());	×
139
140	for (const auto& v : psd) {	×
141	sectionIds.push_back(v.sectionId);	×
142	segmentIds.push_back(v.segmentId);	×
143	offsets.push_back(v.offset);	×
144	}
145	write_dataset(g_postsynaptic_density, "section_id", sectionIds);	×
146	write_dataset(g_postsynaptic_density, "segment_id", segmentIds);	×
147	write_dataset(g_postsynaptic_density, "offset", offsets);	×
148	}	×
149	} // anonymous namespace
150
151	void h5(const Morphology& morph,	2✔
152	const std::string& filename,
153	std::shared_ptr<morphio::WarningHandler> handler) {
154	if (details::emptyMorphology(morph, handler)) {	2✔
NEW 155	throw morphio::WriterError(morphio::details::ErrorMessages().ERROR_EMPTY_MORPHOLOGY());	×
156	}
157
158	details::validateContourSoma(morph, handler);	2✔
159	details::checkSomaHasSameNumberPointsDiameters(*morph.soma());	2✔
160	details::validateRootPointsHaveTwoOrMorePoints(morph);	2✔
161
162	HighFive::File h5_file(filename,
163	HighFive::File::ReadWrite \| HighFive::File::Create \|
164	HighFive::File::Truncate);	4✔
165
166	std::unordered_map<uint32_t, int32_t> newIds;	4✔
167
168	std::vector<std::vector<morphio::floatType>> raw_points;	4✔
169	std::vector<std::vector<int32_t>> raw_structure;	4✔
170	std::vector<morphio::floatType> raw_perimeters;	4✔
171
172	const std::vector<Point>& somaPoints = morph.soma()->points();	2✔
173	const auto& somaDiameters = morph.soma()->diameters();	2✔
174	for (unsigned int i = 0; i < somaPoints.size(); ++i) {	10✔
175	raw_points.push_back(	8✔
176	{somaPoints[i][0], somaPoints[i][1], somaPoints[i][2], somaDiameters[i]});	8✔
177
178	// If the morphology has some perimeter data, we need to fill some
179	// perimeter dummy value in the soma range of the data structure to keep
180	// the length matching
181	if (details::hasPerimeterData(morph)) {	8✔
182	raw_perimeters.push_back(0);	×
183	}
184	}
185
186	raw_structure.push_back({0, SECTION_SOMA, -1});	2✔
187	size_t offset = morph.soma()->points().size();	2✔
188
189	int sectionIdOnDisk = 1;	2✔
190	for (auto it = morph.depth_begin(); it != morph.depth_end(); ++it) {	14✔
191	const std::shared_ptr<Section>& section = *it;	12✔
192
193	const auto& points = section->points();	12✔
194	const auto& diameters = section->diameters();	12✔
195	const auto& perimeters = section->perimeters();	12✔
196
197	int parentOnDisk = (section->isRoot() ? 0 : newIds[section->parent()->id()]);	12✔
198	raw_structure.push_back({static_cast<int>(offset), section->type(), parentOnDisk});	12✔
199
200	const auto numberOfPoints = points.size();	12✔
201	for (unsigned int i = 0; i < numberOfPoints; ++i) {	36✔
202	raw_points.push_back({points[i][0], points[i][1], points[i][2], diameters[i]});	24✔
203	}
204
205	const auto numberOfPerimeters = perimeters.size();	12✔
206	if (numberOfPerimeters > 0) {	12✔
207	if (numberOfPerimeters != numberOfPoints) {	×
208	auto error = morphio::details::ErrorMessages().ERROR_VECTOR_LENGTH_MISMATCH(	×
209	"points", numberOfPoints, "perimeters", numberOfPerimeters);	×
210	throw WriterError(error);	×
211	}
212	for (unsigned int i = 0; i < numberOfPerimeters; ++i) {	×
213	raw_perimeters.push_back(perimeters[i]);	×
214	}
215	}
216
217	newIds[section->id()] = sectionIdOnDisk++;	12✔
218	offset += numberOfPoints;	12✔
219	}
220
221	write_dataset(h5_file, "/points", raw_points);	2✔
222	write_dataset(h5_file, "/structure", raw_structure);	2✔
223
224	HighFive::Group g_metadata = h5_file.createGroup("metadata");	6✔
225
226	write_attribute(g_metadata, "version", std::array<uint32_t, 2>{1, 3});	2✔
227	write_attribute(g_metadata,	2✔
228	"cell_family",
229	std::vector<uint32_t>{static_cast<uint32_t>(morph.cellFamily())});	4✔
230	write_attribute(h5_file, "comment", std::vector<std::string>{details::version_string()});	4✔
231
232	if (details::hasPerimeterData(morph)) {	2✔
233	write_dataset(h5_file, "/perimeters", raw_perimeters);	×
234	}
235
236	mitochondriaH5(h5_file, morph.mitochondria());	2✔
237	endoplasmicReticulumH5(h5_file, morph.endoplasmicReticulum());	2✔
238	if (morph.cellFamily() == SPINE) {	2✔
239	dendriticSpinePostSynapticDensityH5(h5_file, morph._dendriticSpineLevel);	×
240	}
241	}	2✔
242
243	} // end namespace writer
244	} // end namespace mut
245	} // end namespace morphio

BlueBrain / MorphIO / 11254817533

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous