7044059190

Committed 30 Nov 2023 08:46AM UTC coverage: 93.582% (-0.3%) from 93.924%

Build # 7044059190

Build Type

Pull #307

github

Committed by

1uc

Commit Message

Use `#ifdef APPLE`.

Pull Request Pull Request #307: Inject dataset reading via `Hdf5Reader`.

Run Details

108 of 118 new or added lines in 11 files covered. (91.53%)

5 existing lines in 2 files now uncovered.

1881 of 2010 relevant lines covered (93.58%)

81.89 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.05

/src/edge_index.cpp

/*************************************************************************
 * Copyright (C) 2018-2020 Blue Brain Project
 *
 * This file is part of 'libsonata', distributed under the terms
 * of the GNU Lesser General Public License version 3.
 *
 * See top-level COPYING.LESSER and COPYING files for details.
 *************************************************************************/

#include "edge_index.h"

#include <bbp/sonata/common.h>

#include <array>
#include <cstdint>
#include <set>
#include <unordered_map>
#include <vector>

#include "read_bulk.hpp"

namespace bbp {
namespace sonata {
namespace edge_index {

namespace {

using RawIndex = std::vector<std::array<uint64_t, 2>>;

const char* const SOURCE_NODE_ID_DSET = "source_node_id";
const char* const TARGET_NODE_ID_DSET = "target_node_id";

const char* const INDEX_GROUP = "indices";
const char* const SOURCE_INDEX_GROUP = "indices/source_to_target";
const char* const TARGET_INDEX_GROUP = "indices/target_to_source";
const char* const NODE_ID_TO_RANGES_DSET = "node_id_to_ranges";
const char* const RANGE_TO_EDGE_ID_DSET = "range_to_edge_id";

}  // unnamed namespace


const HighFive::Group sourceIndex(const HighFive::Group& h5Root) {
    if (!h5Root.exist(SOURCE_INDEX_GROUP)) {
        throw SonataError("No source index group found");
    }
    return h5Root.getGroup(SOURCE_INDEX_GROUP);
}


const HighFive::Group targetIndex(const HighFive::Group& h5Root) {
    if (!h5Root.exist(TARGET_INDEX_GROUP)) {
        throw SonataError("No target index group found");
    }
    return h5Root.getGroup(TARGET_INDEX_GROUP);
}

Selection resolve(const HighFive::Group& indexGroup,
                  const std::vector<NodeID>& nodeIDs,
                  const Hdf5Reader& reader) {
    auto node2ranges_dset = indexGroup.getDataSet(NODE_ID_TO_RANGES_DSET);
    auto node_dim = node2ranges_dset.getSpace().getDimensions()[0];
    auto sortedNodeIds = nodeIDs;
    bulk_read::detail::erase_if(sortedNodeIds, [node_dim](auto id) {
        // Filter out `nodeIDs[i] >= dims`; because SYN2 used to return an
        // empty range for an out-of-range `nodeId`s.
        return id >= node_dim;
    });
    std::sort(sortedNodeIds.begin(), sortedNodeIds.end());

    auto nodeSelection = Selection::fromValues(sortedNodeIds);
    auto primaryRange = reader.readSelection<std::array<uint64_t, 2>>(node2ranges_dset,
                                                                      nodeSelection,
                                                                      Selection(RawIndex{{0, 2}}));

    bulk_read::detail::erase_if(primaryRange, [](const auto& range) {
        // Filter out any invalid ranges `start >= end`.
        return range[0] >= range[1];
    });

    primaryRange = bulk_read::sortAndMerge(primaryRange);

    auto secondaryRange = reader.readSelection<std::array<uint64_t, 2>>(
        indexGroup.getDataSet(RANGE_TO_EDGE_ID_DSET), primaryRange, RawIndex{{0, 2}});

    // Sort and eliminate empty ranges.
    secondaryRange = bulk_read::sortAndMerge(secondaryRange);

    return Selection(std::move(secondaryRange));
}

namespace {

std::unordered_map<NodeID, RawIndex> _groupNodeRanges(const std::vector<NodeID>& nodeIDs) {
    std::unordered_map<NodeID, RawIndex> result;

    if (nodeIDs.empty()) {
        return result;
    }

    uint64_t rangeStart = 0;
    NodeID lastNodeID = nodeIDs[rangeStart];
    for (uint64_t i = 1; i < nodeIDs.size(); ++i) {
        if (nodeIDs[i] != lastNodeID) {
            result[lastNodeID].push_back({rangeStart, i});
            rangeStart = i;
            lastNodeID = nodeIDs[rangeStart];
        }
    }

    result[lastNodeID].push_back({rangeStart, nodeIDs.size()});

    return result;
}


// Use only in the writing code below. General purpose reading should use the
// Hdf5Reader interface.
std::vector<NodeID> _readNodeIDs(const HighFive::Group& h5Root, const std::string& name) {
    std::vector<NodeID> result;
    h5Root.getDataSet(name).read(result);
    return result;
}


void _writeIndexDataset(const RawIndex& data, const std::string& name, HighFive::Group& h5Group) {
    auto dset = h5Group.createDataSet<uint64_t>(name, HighFive::DataSpace::From(data));
    dset.write(data);
}


void _writeIndexGroup(const std::vector<NodeID>& nodeIDs,
                      uint64_t nodeCount,
                      HighFive::Group& h5Root,
                      const std::string& name) {
    auto indexGroup = h5Root.createGroup(name);

    auto nodeToRanges = _groupNodeRanges(nodeIDs);
    const auto rangeCount =
        std::accumulate(nodeToRanges.begin(),
                        nodeToRanges.end(),
                        uint64_t{0},
                        [](uint64_t total, decltype(nodeToRanges)::const_reference item) {
                            return total + item.second.size();
                        });

    RawIndex primaryIndex;
    RawIndex secondaryIndex;

    primaryIndex.reserve(nodeCount);
    secondaryIndex.reserve(rangeCount);

    uint64_t offset = 0;
    for (NodeID nodeID = 0; nodeID < nodeCount; ++nodeID) {
        const auto it = nodeToRanges.find(nodeID);
        if (it == nodeToRanges.end()) {
            primaryIndex.push_back({offset, offset});
        } else {
            auto& ranges = it->second;
            primaryIndex.push_back({offset, offset + ranges.size()});
            offset += ranges.size();
            std::move(ranges.begin(), ranges.end(), std::back_inserter(secondaryIndex));
        }
    }

    _writeIndexDataset(primaryIndex, NODE_ID_TO_RANGES_DSET, indexGroup);
    _writeIndexDataset(secondaryIndex, RANGE_TO_EDGE_ID_DSET, indexGroup);
}

}  // unnamed namespace


void write(HighFive::Group& h5Root,
           uint64_t sourceNodeCount,
           uint64_t targetNodeCount,
           bool overwrite) {
    if (h5Root.exist(INDEX_GROUP)) {
        if (overwrite) {
            // TODO: remove INDEX_GROUP
            throw SonataError("Index overwrite not implemented yet");
        } else {
            throw SonataError("Index group already exists");
        }
    }

    try {
        _writeIndexGroup(_readNodeIDs(h5Root, SOURCE_NODE_ID_DSET),
                         sourceNodeCount,
                         h5Root,
                         SOURCE_INDEX_GROUP);
        _writeIndexGroup(_readNodeIDs(h5Root, TARGET_NODE_ID_DSET),
                         targetNodeCount,
                         h5Root,
                         TARGET_INDEX_GROUP);
    } catch (...) {
        try {
            // TODO: remove INDEX_GROUP
        } catch (...) {
        }
        throw;
    }
}

}  // namespace edge_index
}  // namespace sonata
}  // namespace bbp

1	/*************************************************************************
2	* Copyright (C) 2018-2020 Blue Brain Project
3	*
4	* This file is part of 'libsonata', distributed under the terms
5	* of the GNU Lesser General Public License version 3.
6	*
7	* See top-level COPYING.LESSER and COPYING files for details.
8	*************************************************************************/
9
10	#include "edge_index.h"
11
12	#include <bbp/sonata/common.h>
13
14	#include <array>
15	#include <cstdint>
16	#include <set>
17	#include <unordered_map>
18	#include <vector>
19
20	#include "read_bulk.hpp"
21
22	namespace bbp {
23	namespace sonata {
24	namespace edge_index {
25
26	namespace {
27
28	using RawIndex = std::vector<std::array<uint64_t, 2>>;
29
30	const char* const SOURCE_NODE_ID_DSET = "source_node_id";
31	const char* const TARGET_NODE_ID_DSET = "target_node_id";
32
33	const char* const INDEX_GROUP = "indices";
34	const char* const SOURCE_INDEX_GROUP = "indices/source_to_target";
35	const char* const TARGET_INDEX_GROUP = "indices/target_to_source";
36	const char* const NODE_ID_TO_RANGES_DSET = "node_id_to_ranges";
37	const char* const RANGE_TO_EDGE_ID_DSET = "range_to_edge_id";
38
39	} // unnamed namespace
40
41
42	const HighFive::Group sourceIndex(const HighFive::Group& h5Root) {	32✔
43	if (!h5Root.exist(SOURCE_INDEX_GROUP)) {	32✔
44	throw SonataError("No source index group found");	2✔
45	}
46	return h5Root.getGroup(SOURCE_INDEX_GROUP);	30✔
47	}
48
49
50	const HighFive::Group targetIndex(const HighFive::Group& h5Root) {	30✔
51	if (!h5Root.exist(TARGET_INDEX_GROUP)) {	30✔
52	throw SonataError("No target index group found");	2✔
53	}
54	return h5Root.getGroup(TARGET_INDEX_GROUP);	28✔
55	}
56
57	Selection resolve(const HighFive::Group& indexGroup,	58✔
58	const std::vector<NodeID>& nodeIDs,
59	const Hdf5Reader& reader) {
60	auto node2ranges_dset = indexGroup.getDataSet(NODE_ID_TO_RANGES_DSET);	174✔
61	auto node_dim = node2ranges_dset.getSpace().getDimensions()[0];	58✔
62	auto sortedNodeIds = nodeIDs;	116✔
63	bulk_read::detail::erase_if(sortedNodeIds, [node_dim](auto id) {	58✔
64	// Filter out `nodeIDs[i] >= dims`; because SYN2 used to return an
65	// empty range for an out-of-range `nodeId`s.
66	return id >= node_dim;	94✔
67	});
68	std::sort(sortedNodeIds.begin(), sortedNodeIds.end());	58✔
69
70	auto nodeSelection = Selection::fromValues(sortedNodeIds);	116✔
71	auto primaryRange = reader.readSelection<std::array<uint64_t, 2>>(node2ranges_dset,
72	nodeSelection,
73	Selection(RawIndex{{0, 2}}));	174✔
74
75	bulk_read::detail::erase_if(primaryRange, [](const auto& range) {	58✔
76	// Filter out any invalid ranges `start >= end`.
77	return range[0] >= range[1];	86✔
78	});
79
80	primaryRange = bulk_read::sortAndMerge(primaryRange);	58✔
81
82	auto secondaryRange = reader.readSelection<std::array<uint64_t, 2>>(
83	indexGroup.getDataSet(RANGE_TO_EDGE_ID_DSET), primaryRange, RawIndex{{0, 2}});	116✔
84
85	// Sort and eliminate empty ranges.
86	secondaryRange = bulk_read::sortAndMerge(secondaryRange);	58✔
87
88	return Selection(std::move(secondaryRange));	116✔
89	}
90
91	namespace {
92
93	std::unordered_map<NodeID, RawIndex> _groupNodeRanges(const std::vector<NodeID>& nodeIDs) {	4✔
94	std::unordered_map<NodeID, RawIndex> result;	4✔
95
96	if (nodeIDs.empty()) {	4✔
UNCOV 97	return result;	×
98	}
99
100	uint64_t rangeStart = 0;	4✔
101	NodeID lastNodeID = nodeIDs[rangeStart];	4✔
102	for (uint64_t i = 1; i < nodeIDs.size(); ++i) {	24✔
103	if (nodeIDs[i] != lastNodeID) {	20✔
104	result[lastNodeID].push_back({rangeStart, i});	12✔
105	rangeStart = i;	12✔
106	lastNodeID = nodeIDs[rangeStart];	12✔
107	}
108	}
109
110	result[lastNodeID].push_back({rangeStart, nodeIDs.size()});	4✔
111
112	return result;	4✔
113	}
114
115
116	// Use only in the writing code below. General purpose reading should use the
117	// Hdf5Reader interface.
118	std::vector<NodeID> _readNodeIDs(const HighFive::Group& h5Root, const std::string& name) {	4✔
119	std::vector<NodeID> result;	4✔
120	h5Root.getDataSet(name).read(result);	4✔
121	return result;	4✔
122	}
123
124
125	void _writeIndexDataset(const RawIndex& data, const std::string& name, HighFive::Group& h5Group) {	8✔
126	auto dset = h5Group.createDataSet<uint64_t>(name, HighFive::DataSpace::From(data));	8✔
127	dset.write(data);	8✔
128	}	8✔
129
130
131	void _writeIndexGroup(const std::vector<NodeID>& nodeIDs,	4✔
132	uint64_t nodeCount,
133	HighFive::Group& h5Root,
134	const std::string& name) {
135	auto indexGroup = h5Root.createGroup(name);	8✔
136
137	auto nodeToRanges = _groupNodeRanges(nodeIDs);	8✔
138	const auto rangeCount =
139	std::accumulate(nodeToRanges.begin(),	4✔
140	nodeToRanges.end(),
141	uint64_t{0},
142	[](uint64_t total, decltype(nodeToRanges)::const_reference item) {	12✔
143	return total + item.second.size();	12✔
144	});
145
146	RawIndex primaryIndex;	8✔
147	RawIndex secondaryIndex;	4✔
148
149	primaryIndex.reserve(nodeCount);	4✔
150	secondaryIndex.reserve(rangeCount);	4✔
151
152	uint64_t offset = 0;	4✔
153	for (NodeID nodeID = 0; nodeID < nodeCount; ++nodeID) {	20✔
154	const auto it = nodeToRanges.find(nodeID);	16✔
155	if (it == nodeToRanges.end()) {	16✔
156	primaryIndex.push_back({offset, offset});	4✔
157	} else {
158	auto& ranges = it->second;	12✔
159	primaryIndex.push_back({offset, offset + ranges.size()});	12✔
160	offset += ranges.size();	12✔
161	std::move(ranges.begin(), ranges.end(), std::back_inserter(secondaryIndex));	12✔
162	}
163	}
164
165	_writeIndexDataset(primaryIndex, NODE_ID_TO_RANGES_DSET, indexGroup);	4✔
166	_writeIndexDataset(secondaryIndex, RANGE_TO_EDGE_ID_DSET, indexGroup);	4✔
167	}	4✔
168
169	} // unnamed namespace
170
171
172	void write(HighFive::Group& h5Root,	6✔
173	uint64_t sourceNodeCount,
174	uint64_t targetNodeCount,
175	bool overwrite) {
176	if (h5Root.exist(INDEX_GROUP)) {	6✔
177	if (overwrite) {	4✔
178	// TODO: remove INDEX_GROUP
179	throw SonataError("Index overwrite not implemented yet");	2✔
180	} else {
181	throw SonataError("Index group already exists");	2✔
182	}
183	}
184
185	try {
186	_writeIndexGroup(_readNodeIDs(h5Root, SOURCE_NODE_ID_DSET),	2✔
187	sourceNodeCount,
188	h5Root,
189	SOURCE_INDEX_GROUP);
190	_writeIndexGroup(_readNodeIDs(h5Root, TARGET_NODE_ID_DSET),	2✔
191	targetNodeCount,
192	h5Root,
193	TARGET_INDEX_GROUP);
UNCOV 194	} catch (...) {	×
195	try {
196	// TODO: remove INDEX_GROUP
197	} catch (...) {
198	}
UNCOV 199	throw;	×
200	}
201	}	2✔
202
203	} // namespace edge_index
204	} // namespace sonata
205	} // namespace bbp

BlueBrain / libsonata / 7044059190

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous