7044059190

Committed 30 Nov 2023 08:46AM UTC coverage: 93.582% (-0.3%) from 93.924%

Build # 7044059190

Build Type

Pull #307

github

Committed by

1uc

Commit Message

Use `#ifdef APPLE`.

Pull Request Pull Request #307: Inject dataset reading via `Hdf5Reader`.

Run Details

108 of 118 new or added lines in 11 files covered. (91.53%)

5 existing lines in 2 files now uncovered.

1881 of 2010 relevant lines covered (93.58%)

81.89 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.6

/src/population.hpp

/*************************************************************************
 * Copyright (C) 2018-2020 Blue Brain Project
 *
 * This file is part of 'libsonata', distributed under the terms
 * of the GNU Lesser General Public License version 3.
 *
 * See top-level COPYING.LESSER and COPYING files for details.
 *************************************************************************/

#pragma once

#include "hdf5_mutex.hpp"

#include <bbp/sonata/population.h>

#include <algorithm>  // stable_sort, transform
#include <iterator>   // back_inserter
#include <numeric>    // iota
#include <vector>

#include <fmt/format.h>

#include "read_bulk.hpp"
#include <highfive/H5File.hpp>

namespace bbp {
namespace sonata {

//--------------------------------------------------------------------------------------------------

namespace {

constexpr const char* const H5_DYNAMICS_PARAMS = "dynamics_params";
constexpr const char* const H5_LIBRARY = "@library";


std::set<std::string> _listChildren(const HighFive::Group& group,
                                    const std::set<std::string>& ignoreNames = {}) {
    std::set<std::string> result;
    for (const auto& name : group.listObjectNames()) {
        if (ignoreNames.count(name) > 0) {
            continue;
        }
        result.insert(name);
    }
    return result;
}

std::set<std::string> _listExplicitEnumerations(const HighFive::Group h5Group,
                                                const std::set<std::string>& attrs) {
    std::set<std::string> names{};
    const std::set<std::string> enums(_listChildren(h5Group));
    std::set_intersection(enums.begin(),
                          enums.end(),
                          attrs.begin(),
                          attrs.end(),
                          std::inserter(names, names.begin()));
    return names;
}

template <typename T>
std::vector<T> _readSelection(const HighFive::DataSet& dset,
                              const Selection& selection,
                              const Hdf5Reader& hdf5_reader) {
    if (dset.getElementCount() == 0) {
        return {};
    }

    if (bulk_read::detail::isCanonical(selection)) {
        return hdf5_reader.readSelection<T>(dset, selection);
    }

    // The fully general case:
    //
    // 1. Create a canonical selection and read into `linear_result`.
    // 2. Copy values from the canonical `linear_results` to their final
    //    destination.
    auto canonicalRanges = bulk_read::sortAndMerge(selection, 0);
    auto linear_result = hdf5_reader.readSelection<T>(dset, canonicalRanges);

    const auto ids = selection.flatten();

    std::vector<std::size_t> ids_index(ids.size());
    std::iota(ids_index.begin(), ids_index.end(), std::size_t(0));
    std::stable_sort(ids_index.begin(), ids_index.end(), [&ids](size_t i0, size_t i1) {
        return ids[i0] < ids[i1];
    });

    std::vector<T> result(ids.size());
    size_t linear_index = 0;
    result[ids_index[0]] = linear_result[0];
    for (size_t i = 1; i < ids.size(); ++i) {
        if (ids[ids_index[i - 1]] != ids[ids_index[i]]) {
            linear_index += 1;
        }

        result[ids_index[i]] = linear_result[linear_index];
    }

    return result;
}

}  // unnamed namespace


inline HighFive::File open_hdf5_file(const std::string& filename, const Hdf5Reader& hdf5_reader) {
    return hdf5_reader.openFile(filename);
}

struct Population::Impl {
    Impl(const std::string& h5FilePath,
         const std::string&,
         const std::string& _name,
         const std::string& _prefix,
         const Hdf5Reader& hdf5_reader)
        : name(_name)
        , prefix(_prefix)
        , h5File(open_hdf5_file(h5FilePath, hdf5_reader))
        , h5Root(h5File.getGroup(fmt::format("/{}s", prefix)).getGroup(name))
        , attributeNames(_listChildren(h5Root.getGroup("0"), {H5_DYNAMICS_PARAMS, H5_LIBRARY}))
        , attributeEnumNames(
              h5Root.getGroup("0").exist(H5_LIBRARY)
                  ? _listExplicitEnumerations(h5Root.getGroup("0").getGroup(H5_LIBRARY),
                                              attributeNames)
                  : std::set<std::string>{})
        , dynamicsAttributeNames(
              h5Root.getGroup("0").exist(H5_DYNAMICS_PARAMS)
                  ? _listChildren(h5Root.getGroup("0").getGroup(H5_DYNAMICS_PARAMS))
                  : std::set<std::string>{})
        , hdf5_reader(hdf5_reader) {
        if (h5Root.exist("1")) {
            throw SonataError("Only single-group populations are supported at the moment");
        }
    }

    HighFive::DataSet getAttributeDataSet(const std::string& name) const {
        if (!attributeNames.count(name)) {
            throw SonataError(fmt::format("No such attribute: '{}'", name));
        }
        return h5Root.getGroup("0").getDataSet(name);
    }

    HighFive::DataSet getLibraryDataSet(const std::string& name) const {
        if (!attributeEnumNames.count(name)) {
            throw SonataError(fmt::format("No such enumeration attribute: '{}'", name));
        }
        return h5Root.getGroup("0").getGroup(H5_LIBRARY).getDataSet(name);
    }

    HighFive::DataSet getDynamicsAttributeDataSet(const std::string& name) const {
        if (!dynamicsAttributeNames.count(name)) {
            throw SonataError(fmt::format("No such dynamics attribute: '{}'", name));
        }
        return h5Root.getGroup("0").getGroup(H5_DYNAMICS_PARAMS).getDataSet(name);
    }

    const std::string name;
    const std::string prefix;
    const HighFive::File h5File;
    const HighFive::Group h5Root;
    const std::set<std::string> attributeNames;
    const std::set<std::string> attributeEnumNames;
    const std::set<std::string> dynamicsAttributeNames;
    const Hdf5Reader hdf5_reader;
};

//--------------------------------------------------------------------------------------------------

template <typename Population>
struct PopulationStorage<Population>::Impl {
    Impl(const std::string& _h5FilePath)
        : Impl(_h5FilePath, Hdf5Reader()) {}

    Impl(const std::string& _h5FilePath, const Hdf5Reader& hdf5_reader)
        : Impl(_h5FilePath, std::string(), hdf5_reader) {}

    Impl(const std::string& _h5FilePath,
         const std::string& _csvFilePath,
         const Hdf5Reader& hdf5_reader)
        : h5FilePath(_h5FilePath)
        , csvFilePath(_csvFilePath)
        , h5File(h5FilePath)
        , h5Root(h5File.getGroup(fmt::format("/{}s", Population::ELEMENT)))
        , hdf5_reader(hdf5_reader) {
        if (!csvFilePath.empty()) {
            throw SonataError("CSV not supported at the moment");
        }
    }

    const std::string h5FilePath;
    const std::string csvFilePath;
    const HighFive::File h5File;
    const HighFive::Group h5Root;
    const Hdf5Reader hdf5_reader;
};

template <typename Population>
PopulationStorage<Population>::PopulationStorage(const std::string& h5FilePath)
    : PopulationStorage(h5FilePath, Hdf5Reader()) {}

template <typename Population>
PopulationStorage<Population>::PopulationStorage(const std::string& h5FilePath,
                                                 const Hdf5Reader& hdf5_reader)
    : PopulationStorage(h5FilePath, std::string(), hdf5_reader) {}

template <typename Population>
PopulationStorage<Population>::PopulationStorage(const std::string& h5FilePath,
                                                 const std::string& csvFilePath)
    : PopulationStorage(h5FilePath, csvFilePath, Hdf5Reader()) {}

template <typename Population>
PopulationStorage<Population>::PopulationStorage(const std::string& h5FilePath,
                                                 const std::string& csvFilePath,
                                                 const Hdf5Reader& hdf5_reader)
    : impl_([h5FilePath, csvFilePath, hdf5_reader] {
        HDF5_LOCK_GUARD
        return new PopulationStorage::Impl(h5FilePath, csvFilePath, hdf5_reader);
    }()) {}


template <typename Population>
PopulationStorage<Population>::PopulationStorage(PopulationStorage<Population>&& st) noexcept =
    default;


template <typename Population>
PopulationStorage<Population>::~PopulationStorage() noexcept = default;


template <typename Population>
std::set<std::string> PopulationStorage<Population>::populationNames() const {
    HDF5_LOCK_GUARD
    return _listChildren(impl_->h5Root);
}


template <typename Population>
std::shared_ptr<Population> PopulationStorage<Population>::openPopulation(
    const std::string& name) const {
    {
        HDF5_LOCK_GUARD
        if (!impl_->h5Root.exist(name)) {
            throw SonataError(fmt::format("No such population: '{}'", name));
        }
    }
    return std::make_shared<Population>(impl_->h5FilePath,
                                        impl_->csvFilePath,
                                        name,
                                        impl_->hdf5_reader);
}

//--------------------------------------------------------------------------------------------------

}  // namespace sonata
}  // namespace bbp

1	/*************************************************************************
2	* Copyright (C) 2018-2020 Blue Brain Project
3	*
4	* This file is part of 'libsonata', distributed under the terms
5	* of the GNU Lesser General Public License version 3.
6	*
7	* See top-level COPYING.LESSER and COPYING files for details.
8	*************************************************************************/
9
10	#pragma once
11
12	#include "hdf5_mutex.hpp"
13
14	#include <bbp/sonata/population.h>
15
16	#include <algorithm> // stable_sort, transform
17	#include <iterator> // back_inserter
18	#include <numeric> // iota
19	#include <vector>
20
21	#include <fmt/format.h>
22
23	#include "read_bulk.hpp"
24	#include <highfive/H5File.hpp>
25
26	namespace bbp {
27	namespace sonata {
28
29	//--------------------------------------------------------------------------------------------------
30
31	namespace {
32
33	constexpr const char* const H5_DYNAMICS_PARAMS = "dynamics_params";
34	constexpr const char* const H5_LIBRARY = "@library";
35
36
37	std::set<std::string> _listChildren(const HighFive::Group& group,	204✔
38	const std::set<std::string>& ignoreNames = {}) {
39	std::set<std::string> result;	204✔
40	for (const auto& name : group.listObjectNames()) {	1,884✔
41	if (ignoreNames.count(name) > 0) {	1,680✔
42	continue;	132✔
43	}
44	result.insert(name);	1,548✔
45	}
46	return result;	204✔
47	}
48
49	std::set<std::string> _listExplicitEnumerations(const HighFive::Group h5Group,	64✔
50	const std::set<std::string>& attrs) {
51	std::set<std::string> names{};	64✔
52	const std::set<std::string> enums(_listChildren(h5Group));	128✔
53	std::set_intersection(enums.begin(),
54	enums.end(),
55	attrs.begin(),
56	attrs.end(),
57	std::inserter(names, names.begin()));	64✔
58	return names;	128✔
59	}
60
61	template <typename T>
62	std::vector<T> _readSelection(const HighFive::DataSet& dset,	168✔
63	const Selection& selection,
64	const Hdf5Reader& hdf5_reader) {
65	if (dset.getElementCount() == 0) {	168✔
66	return {};	2✔
67	}
68
69	if (bulk_read::detail::isCanonical(selection)) {	166✔
70	return hdf5_reader.readSelection<T>(dset, selection);	160✔
71	}
72
73	// The fully general case:
74	//
75	// 1. Create a canonical selection and read into `linear_result`.
76	// 2. Copy values from the canonical `linear_results` to their final
77	// destination.
78	auto canonicalRanges = bulk_read::sortAndMerge(selection, 0);	12✔
79	auto linear_result = hdf5_reader.readSelection<T>(dset, canonicalRanges);	12✔
80
81	const auto ids = selection.flatten();	12✔
82
83	std::vector<std::size_t> ids_index(ids.size());	12✔
84	std::iota(ids_index.begin(), ids_index.end(), std::size_t(0));	6✔
85	std::stable_sort(ids_index.begin(), ids_index.end(), [&ids](size_t i0, size_t i1) {	94✔
86	return ids[i0] < ids[i1];	44✔
87	});
88
89	std::vector<T> result(ids.size());	12✔
90	size_t linear_index = 0;	6✔
91	result[ids_index[0]] = linear_result[0];	6✔
92	for (size_t i = 1; i < ids.size(); ++i) {	26✔
93	if (ids[ids_index[i - 1]] != ids[ids_index[i]]) {	20✔
94	linear_index += 1;	16✔
95	}
96
97	result[ids_index[i]] = linear_result[linear_index];	20✔
98	}
99
100	return result;	6✔
101	}
102
103	} // unnamed namespace
104
105
106	inline HighFive::File open_hdf5_file(const std::string& filename, const Hdf5Reader& hdf5_reader) {	70✔
107	return hdf5_reader.openFile(filename);	70✔
108	}
109
110	struct Population::Impl {
111	Impl(const std::string& h5FilePath,	70✔
112	const std::string&,
113	const std::string& _name,
114	const std::string& _prefix,
115	const Hdf5Reader& hdf5_reader)
116	: name(_name)	70✔
117	, prefix(_prefix)
118	, h5File(open_hdf5_file(h5FilePath, hdf5_reader))
119	, h5Root(h5File.getGroup(fmt::format("/{}s", prefix)).getGroup(name))	210✔
120	, attributeNames(_listChildren(h5Root.getGroup("0"), {H5_DYNAMICS_PARAMS, H5_LIBRARY}))	140✔
121	, attributeEnumNames(
122	h5Root.getGroup("0").exist(H5_LIBRARY)	140✔
123	? _listExplicitEnumerations(h5Root.getGroup("0").getGroup(H5_LIBRARY),	134✔
124	attributeNames)	64✔
125	: std::set<std::string>{})
126	, dynamicsAttributeNames(
127	h5Root.getGroup("0").exist(H5_DYNAMICS_PARAMS)	140✔
128	? _listChildren(h5Root.getGroup("0").getGroup(H5_DYNAMICS_PARAMS))	138✔
129	: std::set<std::string>{})
130	, hdf5_reader(hdf5_reader) {	988✔
131	if (h5Root.exist("1")) {	70✔
132	throw SonataError("Only single-group populations are supported at the moment");	2✔
133	}
134	}	68✔
135
136	HighFive::DataSet getAttributeDataSet(const std::string& name) const {	182✔
137	if (!attributeNames.count(name)) {	182✔
138	throw SonataError(fmt::format("No such attribute: '{}'", name));	12✔
139	}
140	return h5Root.getGroup("0").getDataSet(name);	352✔
141	}
142
143	HighFive::DataSet getLibraryDataSet(const std::string& name) const {	34✔
144	if (!attributeEnumNames.count(name)) {	34✔
145	throw SonataError(fmt::format("No such enumeration attribute: '{}'", name));	4✔
146	}
147	return h5Root.getGroup("0").getGroup(H5_LIBRARY).getDataSet(name);	64✔
148	}
149
150	HighFive::DataSet getDynamicsAttributeDataSet(const std::string& name) const {	32✔
151	if (!dynamicsAttributeNames.count(name)) {	32✔
152	throw SonataError(fmt::format("No such dynamics attribute: '{}'", name));	8✔
153	}
154	return h5Root.getGroup("0").getGroup(H5_DYNAMICS_PARAMS).getDataSet(name);	56✔
155	}
156
157	const std::string name;
158	const std::string prefix;
159	const HighFive::File h5File;
160	const HighFive::Group h5Root;
161	const std::set<std::string> attributeNames;
162	const std::set<std::string> attributeEnumNames;
163	const std::set<std::string> dynamicsAttributeNames;
164	const Hdf5Reader hdf5_reader;
165	};
166
167	//--------------------------------------------------------------------------------------------------
168
169	template <typename Population>
170	struct PopulationStorage<Population>::Impl {
NEW 171	Impl(const std::string& _h5FilePath)	×
NEW 172	: Impl(_h5FilePath, Hdf5Reader()) {}	×
173
NEW 174	Impl(const std::string& _h5FilePath, const Hdf5Reader& hdf5_reader)	×
NEW 175	: Impl(_h5FilePath, std::string(), hdf5_reader) {}	×
176
177	Impl(const std::string& _h5FilePath,	4✔
178	const std::string& _csvFilePath,
179	const Hdf5Reader& hdf5_reader)
180	: h5FilePath(_h5FilePath)
181	, csvFilePath(_csvFilePath)
182	, h5File(h5FilePath)	4✔
183	, h5Root(h5File.getGroup(fmt::format("/{}s", Population::ELEMENT)))	4✔
184	, hdf5_reader(hdf5_reader) {	18✔
185	if (!csvFilePath.empty()) {	4✔
186	throw SonataError("CSV not supported at the moment");	2✔
187	}
188	}	2✔
189
190	const std::string h5FilePath;
191	const std::string csvFilePath;
192	const HighFive::File h5File;
193	const HighFive::Group h5Root;
194	const Hdf5Reader hdf5_reader;
195	};
196
197	template <typename Population>
198	PopulationStorage<Population>::PopulationStorage(const std::string& h5FilePath)	2✔
199	: PopulationStorage(h5FilePath, Hdf5Reader()) {}	2✔
200
201	template <typename Population>
202	PopulationStorage<Population>::PopulationStorage(const std::string& h5FilePath,	2✔
203	const Hdf5Reader& hdf5_reader)
204	: PopulationStorage(h5FilePath, std::string(), hdf5_reader) {}	2✔
205
206	template <typename Population>
207	PopulationStorage<Population>::PopulationStorage(const std::string& h5FilePath,	2✔
208	const std::string& csvFilePath)
209	: PopulationStorage(h5FilePath, csvFilePath, Hdf5Reader()) {}	2✔
210
211	template <typename Population>
212	PopulationStorage<Population>::PopulationStorage(const std::string& h5FilePath,	4✔
213	const std::string& csvFilePath,
214	const Hdf5Reader& hdf5_reader)
215	: impl_([h5FilePath, csvFilePath, hdf5_reader] {	4✔
216	HDF5_LOCK_GUARD	6✔
217	return new PopulationStorage::Impl(h5FilePath, csvFilePath, hdf5_reader);	6✔
218	}()) {}	4✔
219
220
221	template <typename Population>
222	PopulationStorage<Population>::PopulationStorage(PopulationStorage<Population>&& st) noexcept =
223	default;
224
225
226	template <typename Population>
227	PopulationStorage<Population>::~PopulationStorage() noexcept = default;
228
229
230	template <typename Population>
231	std::set<std::string> PopulationStorage<Population>::populationNames() const {	2✔
232	HDF5_LOCK_GUARD	2✔
233	return _listChildren(impl_->h5Root);	4✔
234	}
235
236
237	template <typename Population>
238	std::shared_ptr<Population> PopulationStorage<Population>::openPopulation(	6✔
239	const std::string& name) const {
240	{
241	HDF5_LOCK_GUARD	12✔
242	if (!impl_->h5Root.exist(name)) {	6✔
243	throw SonataError(fmt::format("No such population: '{}'", name));	4✔
244	}
245	}
246	return std::make_shared<Population>(impl_->h5FilePath,	8✔
247	impl_->csvFilePath,	4✔
248	name,
249	impl_->hdf5_reader);	8✔
250	}
251
252	//--------------------------------------------------------------------------------------------------
253
254	} // namespace sonata
255	} // namespace bbp

BlueBrain / libsonata / 7044059190

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous