4719722733

Build # 4719722733

Build Type

push

github

Committed by Mike Gevaert

Commit Message

Don't read the full node sets file before parsing

Run Details

13 of 13 new or added lines in 1 file covered. (100.0%)

1741 of 1788 relevant lines covered (97.37%)

79.33 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.38

/src/node_sets.cpp

#include <algorithm>  // std::find, std::transform
#include <cassert>
#include <cmath>
#include <fmt/format.h>
#include <fstream>
#include <sstream>
#include <type_traits>

#include "../extlib/filesystem.hpp"

#include <nlohmann/json.hpp>
#include <utility>

#include "utils.h"  // readFile

#include <bbp/sonata/node_sets.h>

namespace bbp {
namespace sonata {

namespace fs = ghc::filesystem;

namespace detail {

const size_t MAX_COMPOUND_RECURSION = 10;

using json = nlohmann::json;

void replace_trailing_coma(std::string& s, char c) {
    s.pop_back();
    s.pop_back();
    s.push_back(c);
}

template <typename T>
std::string toString(const std::string& key, const std::vector<T>& values) {
    return fmt::format(R"("{}": [{}])", key, fmt::join(values, ", "));
}

template <>
std::string toString(const std::string& key, const std::vector<std::string>& values) {
    // strings need to be wrapped in quotes
    return fmt::format(R"("{}": ["{}"])", key, fmt::join(values, "\", \""));
}

class NodeSets;

class NodeSetRule
{
  public:
    virtual ~NodeSetRule() = default;

    virtual Selection materialize(const NodeSets&, const NodePopulation&) const = 0;
    virtual std::string toJSON() const = 0;
    virtual bool is_compound() const {
        return false;
    }
};

using NodeSetRulePtr = std::unique_ptr<NodeSetRule>;
void parse_basic(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets);
void parse_compound(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets);

class NodeSets
{
    std::map<std::string, NodeSetRulePtr> node_sets_;

  public:
    explicit NodeSets(const json& j) {
        if (!j.is_object()) {
            throw SonataError("Top level node_set must be an object");
        }

        // Need to two pass parsing the json so that compound lookup can rely
        // on all the basic rules existing
        parse_basic(j, node_sets_);
        parse_compound(j, node_sets_);
    }

    explicit NodeSets(const fs::path& path)
        : NodeSets(json::parse(std::fstream(path))) {}

    explicit NodeSets(const std::string& content)
        : NodeSets(json::parse(content)) {}

    static std::unique_ptr<NodeSets> fromFile(const std::string& path_) {
        fs::path path(path_);
        if (!fs::exists(path)) {
            throw SonataError(fmt::format("Path does not exist: {}", std::string(path)));
        }
        return std::make_unique<detail::NodeSets>(path);
    }


    Selection materialize(const std::string& name, const NodePopulation& population) const;

    std::set<std::string> names() const {
        return getMapKeys(node_sets_);
    }

    std::string toJSON() const {
        std::string ret{"{\n"};
        for (const auto& pair : node_sets_) {
            ret += fmt::format(R"(  "{}": {{)", pair.first);
            ret += pair.second->toJSON();
            ret += "},\n";
        }
        replace_trailing_coma(ret, '\n');
        ret += "}";

        return ret;
    }
};

// { 'region': ['region1', 'region2', ...] }
template <typename T>
class NodeSetBasicRule: public NodeSetRule
{
  public:
    NodeSetBasicRule(std::string attribute, std::vector<T>& values)
        : attribute_(std::move(attribute))
        , values_(values) {}

    Selection materialize(const detail::NodeSets& /* unused */,
                          const NodePopulation& np) const final {
        return np.matchAttributeValues(attribute_, values_);
    }

    void add_attribute2rule(std::map<std::string, std::set<T>>& attribute2rule) const {
        auto& s = attribute2rule[attribute_];
        for (const auto& v : values_) {
            s.insert(v);
        }
    }

    std::string toJSON() const final {
        return toString(attribute_, values_);
    }

  private:
    std::string attribute_;
    std::vector<T> values_;
};

// { 'population': ['popA', 'popB', ] }
class NodeSetBasicPopulation: public NodeSetRule
{
  public:
    explicit NodeSetBasicPopulation(std::vector<std::string>& values)
        : values_(values) {}

    Selection materialize(const detail::NodeSets& /* unused */,
                          const NodePopulation& np) const final {
        if (std::find(values_.begin(), values_.end(), np.name()) != values_.end()) {
            return np.selectAll();
        }

        return Selection{{}};
    }

    std::string toJSON() const final {
        return toString("population", values_);
    }

  private:
    std::vector<std::string> values_;
};

// { 'node_id': [1, 2, 3, 4] }
class NodeSetBasicNodeIds: public NodeSetRule
{
  public:
    explicit NodeSetBasicNodeIds(Selection::Values&& values)
        : values_(values) {}

    Selection materialize(const detail::NodeSets& /* unused */,
                          const NodePopulation& np) const final {
        return np.selectAll() & Selection::fromValues(values_.begin(), values_.end());
    }

    std::string toJSON() const final {
        return toString("node_ids", values_);
    }

  private:
    Selection::Values values_;
};

//  {
//      "population": "biophysical",
//      "model_type": "point",
//      "node_id": [1, 2, 3, 5, 7, 9, ...]
//  }
class NodeSetBasicMultiClause: public NodeSetRule
{
  public:
    explicit NodeSetBasicMultiClause(std::vector<NodeSetRulePtr>&& clauses)
        : clauses_(std::move(clauses)) {}

    Selection materialize(const detail::NodeSets& ns, const NodePopulation& np) const final {
        Selection ret = np.selectAll();
        for (const auto& clause : clauses_) {
            ret = ret & clause->materialize(ns, np);
        }
        return ret;
    }

    std::string toJSON() const final {
        std::string ret;
        for (const auto& clause : clauses_) {
            ret += clause->toJSON();
            ret += ", ";
        }
        replace_trailing_coma(ret, ' ');
        return ret;
    }

  private:
    std::vector<NodeSetRulePtr> clauses_;
};

// "string_attr": { "$regex": "^[s][o]me value$" }
class NodeSetBasicOperatorString: public NodeSetRule
{
  public:
    explicit NodeSetBasicOperatorString(std::string attribute,
                                        const std::string& op,
                                        std::string value)
        : op_(string2op(op))
        , attribute_(std::move(attribute))
        , value_(std::move(value)) {}

    Selection materialize(const detail::NodeSets& /* unused */,
                          const NodePopulation& np) const final {
        switch (op_) {
        case Op::regex:
            return np.regexMatch(attribute_, value_);
        default:              // LCOV_EXCL_LINE
            LIBSONATA_THROW_IF_REACHED  // LCOV_EXCL_LINE
        }
    }

    std::string toJSON() const final {
        return fmt::format(R"("{}": {{ "{}": "{}" }})", attribute_, op2string(op_), value_);
    }

    enum class Op {
        regex = 1,
    };

    static Op string2op(const std::string& s) {
        if (s == "$regex") {
            return Op::regex;
        }
        throw SonataError(fmt::format("Operator '{}' not available for strings", s));
    }

    static std::string op2string(const Op op) {
        switch (op) {
        case Op::regex:
            return "$regex";
        default:              // LCOV_EXCL_LINE
            LIBSONATA_THROW_IF_REACHED  // LCOV_EXCL_LINE
        }
    }

  private:
    Op op_;
    std::string attribute_;
    std::string value_;
};

// "numeric_attribute_gt": { "$gt": 3 },
class NodeSetBasicOperatorNumeric: public NodeSetRule
{
  public:
    explicit NodeSetBasicOperatorNumeric(std::string name, const std::string& op, double value)
        : name_(std::move(name))
        , value_(value)
        , op_(string2op(op)) {}

    Selection materialize(const detail::NodeSets& /* unused */,
                          const NodePopulation& np) const final {
        switch (op_) {
        case Op::gt:
            return np.filterAttribute<double>(name_, [=](const double v) { return v > value_; });
        case Op::lt:
            return np.filterAttribute<double>(name_, [=](const double v) { return v < value_; });
        case Op::gte:
            return np.filterAttribute<double>(name_, [=](const double v) { return v >= value_; });
        case Op::lte:
            return np.filterAttribute<double>(name_, [=](const double v) { return v <= value_; });
        default:              // LCOV_EXCL_LINE
            LIBSONATA_THROW_IF_REACHED  // LCOV_EXCL_LINE
        }
    }

    std::string toJSON() const final {
        return fmt::format(R"("{}": {{ "{}": {} }})", name_, op2string(op_), value_);
    }

    enum class Op {
        gt = 1,
        lt = 2,
        gte = 3,
        lte = 4,
    };

    static Op string2op(const std::string& s) {
        if (s == "$gt") {
            return Op::gt;
        } else if (s == "$lt") {
            return Op::lt;
        } else if (s == "$gte") {
            return Op::gte;
        } else if (s == "$lte") {
            return Op::lte;
        }
        throw SonataError(fmt::format("Operator '{}' not available for numeric", s));
    }

    static std::string op2string(const Op op) {
        switch (op) {
        case Op::gt:
            return "$gt";
        case Op::lt:
            return "$lt";
        case Op::gte:
            return "$gte";
        case Op::lte:
            return "$lte";
        default:              // LCOV_EXCL_LINE
            LIBSONATA_THROW_IF_REACHED  // LCOV_EXCL_LINE
        }
    }

  private:
    std::string name_;
    double value_;
    Op op_;
};

using CompoundTargets = std::vector<std::string>;
class NodeSetCompoundRule: public NodeSetRule
{
  public:
    NodeSetCompoundRule(std::string name, CompoundTargets targets)
        : name_(std::move(name))
        , targets_(std::move(targets)) {}

    Selection materialize(const detail::NodeSets& ns, const NodePopulation& np) const final {
        Selection ret{{}};
        for (const auto& target : targets_) {
            ret = ret | ns.materialize(target, np);
        }
        return ret;
    }

    std::string toJSON() const final {
        return toString("node_ids", targets_);
    }

    bool is_compound() const override {
        return true;
    }
    const CompoundTargets& getTargets() const {
        return targets_;
    }

  private:
    std::string name_;
    CompoundTargets targets_;
};

int64_t get_int64_or_throw(const json& el) {
    auto v = el.get<double>();
    if (std::floor(v) != v) {
        throw SonataError(fmt::format("expected integer, got float {}", v));
    }
    return static_cast<int64_t>(v);
}

uint64_t get_uint64_or_throw(const json& el) {
    auto v = el.get<double>();
    if (v < 0) {
        throw SonataError(fmt::format("expected unsigned integer, got {}", v));
    }

    if (std::floor(v) != v) {
        throw SonataError(fmt::format("expected integer, got float {}", v));
    }
    return static_cast<uint64_t>(v);
}

NodeSetRulePtr _dispatch_node(const std::string& attribute, const json& value) {
    if (value.is_number()) {
        if (attribute == "population") {
            throw SonataError("'population' must be a string");
        }

        if (attribute == "node_id") {
            Selection::Values node_ids{get_uint64_or_throw(value)};
            return std::make_unique<NodeSetBasicNodeIds>(std::move(node_ids));
        } else {
            std::vector<int64_t> f = {get_int64_or_throw(value)};
            return std::make_unique<NodeSetBasicRule<int64_t>>(attribute, f);
        }
    } else if (value.is_string()) {
        if (attribute == "node_id") {
            throw SonataError("'node_id' must be numeric or a list of numbers");
        }

        if (attribute == "population") {
            std::vector<std::string> v{value.get<std::string>()};
            return std::make_unique<NodeSetBasicPopulation>(v);
        } else {
            std::vector<std::string> f = {value.get<std::string>()};
            return std::make_unique<NodeSetBasicRule<std::string>>(attribute, f);
        }
    } else if (value.is_array()) {
        const auto& array = value;

        if (array.empty()) {
            throw SonataError(fmt::format("NodeSet Array is empty for attribute: {}", attribute));
        }

        if (array[0].is_number()) {
            if (attribute == "population") {
                throw SonataError("'population' must be a string");
            }

            std::vector<int64_t> values;
            for (auto& inner_el : array.items()) {
                values.emplace_back(get_int64_or_throw(inner_el.value()));
            }

            if (attribute == "node_id") {
                Selection::Values node_ids;
                std::transform(begin(values),
                               end(values),
                               back_inserter(node_ids),
                               [](int64_t integer) {
                                   if (integer < 0) {
                                       throw SonataError("'node_id' must be positive");
                                   }
                                   return static_cast<Selection::Value>(integer);
                               });
                return std::make_unique<NodeSetBasicNodeIds>(std::move(node_ids));
            } else {
                return std::make_unique<NodeSetBasicRule<int64_t>>(attribute, values);
            }
        } else if (array[0].is_string()) {
            if (attribute == "node_id") {
                throw SonataError("'node_id' must be numeric or a list of numbers");
            }

            std::vector<std::string> values;
            for (auto& inner_el : array.items()) {
                values.emplace_back(inner_el.value().get<std::string>());
            }

            if (attribute == "population") {
                return std::make_unique<NodeSetBasicPopulation>(values);
            } else {
                return std::make_unique<NodeSetBasicRule<std::string>>(attribute, values);
            }
        } else {
            throw SonataError("Unknown array type");
        }
    } else if (value.is_object()) {
        const auto& definition = value;
        if (definition.size() != 1) {
            throw SonataError(
                fmt::format("Operator '{}' must have object with one key value pair", attribute));
        }
        const auto& key = definition.begin().key();
        const auto& value = definition.begin().value();

        if (value.is_number()) {
            return std::make_unique<NodeSetBasicOperatorNumeric>(attribute,
                                                                 key,
                                                                 value.get<double>());
        } else if (value.is_string()) {
            return std::make_unique<NodeSetBasicOperatorString>(attribute,
                                                                key,
                                                                value.get<std::string>());
        } else {
            throw SonataError("Unknown operator");
        }
    } else {
        LIBSONATA_THROW_IF_REACHED  // LCOV_EXCL_LINE
    }
}

void parse_basic(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets) {
    for (const auto& el : j.items()) {
        const auto& value = el.value();
        if (value.is_object()) {
            if (value.empty()) {
                // ignore
            } else if (value.size() == 1) {
                const auto& inner_el = value.items().begin();
                node_sets[el.key()] = _dispatch_node(inner_el.key(), inner_el.value());
            } else {
                std::vector<NodeSetRulePtr> clauses;
                for (const auto& inner_el : value.items()) {
                    clauses.push_back(_dispatch_node(inner_el.key(), inner_el.value()));
                }
                node_sets[el.key()] = std::make_unique<NodeSetBasicMultiClause>(std::move(clauses));
            }
        } else if (value.is_array()) {
            // will be parsed by the parse_compound
        } else {
            // null/boolean/number/string ?
            throw SonataError(fmt::format("Expected an array or an object, got: {}", value.dump()));
        }
    }
}

void check_compound(const std::map<std::string, NodeSetRulePtr>& node_sets,
                    const std::map<std::string, CompoundTargets>& compound_rules,
                    const std::string& name,
                    size_t depth) {
    if (node_sets.count(name) > 0) {
        return;
    }

    if (depth > MAX_COMPOUND_RECURSION) {
        throw SonataError("Compound node_set recursion depth exceeded");
    }

    const auto it = compound_rules.find(name);
    assert(it != compound_rules.end());

    for (auto const& target : it->second) {
        if (node_sets.count(target) == 0 && compound_rules.count(target) == 0) {
            throw SonataError(fmt::format("Missing '{}' from node_sets", target));
        }
        check_compound(node_sets, compound_rules, target, depth + 1);
    }
}

void parse_compound(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets) {
    std::map<std::string, CompoundTargets> compound_rules;
    for (auto& el : j.items()) {
        if (el.value().is_array()) {
            CompoundTargets targets;
            for (const auto& name : el.value()) {
                if (!name.is_string()) {
                    throw SonataError("All compound elements must be strings");
                }

                targets.emplace_back(name);
            }
            compound_rules[el.key()] = targets;
        }
    }


    for (const auto& rule : compound_rules) {
        check_compound(node_sets, compound_rules, rule.first, 0);

        NodeSetRulePtr rules = std::make_unique<NodeSetCompoundRule>(rule.first, rule.second);
        node_sets.emplace(rule.first, std::move(rules));
    }
}

Selection NodeSets::materialize(const std::string& name, const NodePopulation& population) const {
    const auto& node_set = node_sets_.find(name);
    if (node_set == node_sets_.end()) {
        throw SonataError(fmt::format("Unknown node_set {}", name));
    }
    const auto& ns = node_set->second;
    if (!ns->is_compound()) {
        return population.selectAll() & ns->materialize(*this, population);
    }

    // it's common to have a deep structure of compound statements
    // (ie: a whole hierarchy of regions), all checking the same attribute
    // rather than `materializing` them separately, we group them, and materialize
    // them all at once
    Selection ret{{}};

    std::vector<NodeSetRule*> queue{ns.get()};
    std::map<std::string, std::set<std::string>> attribute2rule_strings;
    std::map<std::string, std::set<int64_t>> attribute2rule_int64;
    while (!queue.empty()) {
        const auto* ns = queue.back();
        queue.pop_back();

        if (ns->is_compound()) {
            const auto* targets = dynamic_cast<const NodeSetCompoundRule*>(ns);
            for (const auto& target : targets->getTargets()) {
                const auto& node_set = node_sets_.find(target)->second;
                if (node_set->is_compound()) {
                    queue.push_back(node_set.get());
                    continue;
                }

                {
                    const auto* basic_int = dynamic_cast<const NodeSetBasicRule<int64_t>*>(
                        node_set.get());
                    if (basic_int != nullptr) {
                        basic_int->add_attribute2rule(attribute2rule_int64);
                        continue;
                    }
                }

                {
                    const auto* basic_string = dynamic_cast<const NodeSetBasicRule<std::string>*>(
                        node_set.get());
                    if (basic_string != nullptr) {
                        basic_string->add_attribute2rule(attribute2rule_strings);
                        continue;
                    }
                }

                ret = ret | ns->materialize(*this, population);
            }
        } else {
            ret = ret | ns->materialize(*this, population);
        }
    }

    for (const auto& it : attribute2rule_strings) {
        std::vector<std::string> values(it.second.begin(), it.second.end());
        ret = ret | population.matchAttributeValues(it.first, values);
    }

    for (const auto& it : attribute2rule_int64) {
        std::vector<int64_t> values(it.second.begin(), it.second.end());
        ret = ret | population.matchAttributeValues(it.first, values);
    }

    return ret;
}
}  // namespace detail

NodeSets::NodeSets(const std::string& content)
    : impl_(new detail::NodeSets(content)) {}

NodeSets::NodeSets(std::unique_ptr<detail::NodeSets>&& impl)
    : impl_(std::move(impl)) {}

NodeSets::NodeSets(NodeSets&&) noexcept = default;
NodeSets& NodeSets::operator=(NodeSets&&) noexcept = default;
NodeSets::~NodeSets() = default;

NodeSets NodeSets::fromFile(const std::string& path) {
    return NodeSets(detail::NodeSets::fromFile(path));
}

Selection NodeSets::materialize(const std::string& name, const NodePopulation& population) const {
    return impl_->materialize(name, population);
}

std::set<std::string> NodeSets::names() const {
    return impl_->names();
}

std::string NodeSets::toJSON() const {
    return impl_->toJSON();
}

}  // namespace sonata
}  // namespace bbp

1	#include <algorithm> // std::find, std::transform
2	#include <cassert>
3	#include <cmath>
4	#include <fmt/format.h>
5	#include <fstream>
6	#include <sstream>
7	#include <type_traits>
8
9	#include "../extlib/filesystem.hpp"
10
11	#include <nlohmann/json.hpp>
12	#include <utility>
13
14	#include "utils.h" // readFile
15
16	#include <bbp/sonata/node_sets.h>
17
18	namespace bbp {
19	namespace sonata {
20
21	namespace fs = ghc::filesystem;
22
23	namespace detail {
24
25	const size_t MAX_COMPOUND_RECURSION = 10;
26
27	using json = nlohmann::json;
28
29	void replace_trailing_coma(std::string& s, char c) {	40✔
30	s.pop_back();	40✔
31	s.pop_back();	40✔
32	s.push_back(c);	40✔
33	}	40✔
34
35	template <typename T>
36	std::string toString(const std::string& key, const std::vector<T>& values) {	10✔
37	return fmt::format(R"("{}": [{}])", key, fmt::join(values, ", "));	20✔
38	}
39
40	template <>
41	std::string toString(const std::string& key, const std::vector<std::string>& values) {	50✔
42	// strings need to be wrapped in quotes
43	return fmt::format(R"("{}": ["{}"])", key, fmt::join(values, "\", \""));	100✔
44	}
45
46	class NodeSets;
47
48	class NodeSetRule
49	{
50	public:
51	virtual ~NodeSetRule() = default;	188✔
52
53	virtual Selection materialize(const NodeSets&, const NodePopulation&) const = 0;
54	virtual std::string toJSON() const = 0;
55	virtual bool is_compound() const {	192✔
56	return false;	192✔
57	}
58	};
59
60	using NodeSetRulePtr = std::unique_ptr<NodeSetRule>;
61	void parse_basic(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets);
62	void parse_compound(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets);
63
64	class NodeSets
65	{
66	std::map<std::string, NodeSetRulePtr> node_sets_;
67
68	public:
69	explicit NodeSets(const json& j) {	118✔
70	if (!j.is_object()) {	82✔
71	throw SonataError("Top level node_set must be an object");	4✔
72	}
73
74	// Need to two pass parsing the json so that compound lookup can rely
75	// on all the basic rules existing
76	parse_basic(j, node_sets_);	78✔
77	parse_compound(j, node_sets_);	54✔
78	}	46✔
79
80	explicit NodeSets(const fs::path& path)	2✔
81	: NodeSets(json::parse(std::fstream(path))) {}	2✔
82
83	explicit NodeSets(const std::string& content)	80✔
84	: NodeSets(json::parse(content)) {}	116✔
85
86	static std::unique_ptr<NodeSets> fromFile(const std::string& path_) {	2✔
87	fs::path path(path_);	4✔
88	if (!fs::exists(path)) {	2✔
89	throw SonataError(fmt::format("Path does not exist: {}", std::string(path)));	×
90	}
91	return std::make_unique<detail::NodeSets>(path);	4✔
92	}
93
94
95	Selection materialize(const std::string& name, const NodePopulation& population) const;
96
97	std::set<std::string> names() const {	2✔
98	return getMapKeys(node_sets_);	2✔
99	}
100
101	std::string toJSON() const {	10✔
102	std::string ret{"{\n"};	10✔
103	for (const auto& pair : node_sets_) {	60✔
104	ret += fmt::format(R"( "{}": {{)", pair.first);	100✔
105	ret += pair.second->toJSON();	50✔
106	ret += "},\n";	50✔
107	}
108	replace_trailing_coma(ret, '\n');	10✔
109	ret += "}";	10✔
110
111	return ret;	10✔
112	}
113	};
114
115	// { 'region': ['region1', 'region2', ...] }
116	template <typename T>
117	class NodeSetBasicRule: public NodeSetRule
118	{
119	public:
120	NodeSetBasicRule(std::string attribute, std::vector<T>& values)	44✔
121	: attribute_(std::move(attribute))	44✔
122	, values_(values) {}	44✔
123
124	Selection materialize(const detail::NodeSets& /* unused */,	10✔
125	const NodePopulation& np) const final {
126	return np.matchAttributeValues(attribute_, values_);	10✔
127	}
128
129	void add_attribute2rule(std::map<std::string, std::set<T>>& attribute2rule) const {	20✔
130	auto& s = attribute2rule[attribute_];	20✔
131	for (const auto& v : values_) {	50✔
132	s.insert(v);	30✔
133	}
134	}	20✔
135
136	std::string toJSON() const final {	38✔
137	return toString(attribute_, values_);	38✔
138	}
139
140	private:
141	std::string attribute_;
142	std::vector<T> values_;
143	};
144
145	// { 'population': ['popA', 'popB', ] }
146	class NodeSetBasicPopulation: public NodeSetRule
147	{
148	public:
149	explicit NodeSetBasicPopulation(std::vector<std::string>& values)	14✔
150	: values_(values) {}	14✔
151
152	Selection materialize(const detail::NodeSets& /* unused */,	6✔
153	const NodePopulation& np) const final {
154	if (std::find(values_.begin(), values_.end(), np.name()) != values_.end()) {	6✔
155	return np.selectAll();	4✔
156	}
157
158	return Selection{{}};	2✔
159	}
160
161	std::string toJSON() const final {	10✔
162	return toString("population", values_);	10✔
163	}
164
165	private:
166	std::vector<std::string> values_;
167	};
168
169	// { 'node_id': [1, 2, 3, 4] }
170	class NodeSetBasicNodeIds: public NodeSetRule
171	{
172	public:
173	explicit NodeSetBasicNodeIds(Selection::Values&& values)	18✔
174	: values_(values) {}	18✔
175
176	Selection materialize(const detail::NodeSets& /* unused */,	96✔
177	const NodePopulation& np) const final {
178	return np.selectAll() & Selection::fromValues(values_.begin(), values_.end());	192✔
179	}
180
181	std::string toJSON() const final {	6✔
182	return toString("node_ids", values_);	6✔
183	}
184
185	private:
186	Selection::Values values_;
187	};
188
189	// {
190	// "population": "biophysical",
191	// "model_type": "point",
192	// "node_id": [1, 2, 3, 5, 7, 9, ...]
193	// }
194	class NodeSetBasicMultiClause: public NodeSetRule
195	{
196	public:
197	explicit NodeSetBasicMultiClause(std::vector<NodeSetRulePtr>&& clauses)	26✔
198	: clauses_(std::move(clauses)) {}	26✔
199
200	Selection materialize(const detail::NodeSets& ns, const NodePopulation& np) const final {	2✔
201	Selection ret = np.selectAll();	2✔
202	for (const auto& clause : clauses_) {	6✔
203	ret = ret & clause->materialize(ns, np);	4✔
204	}
205	return ret;	2✔
206	}
207
208	std::string toJSON() const final {	30✔
209	std::string ret;	30✔
210	for (const auto& clause : clauses_) {	120✔
211	ret += clause->toJSON();	90✔
212	ret += ", ";	90✔
213	}
214	replace_trailing_coma(ret, ' ');	30✔
215	return ret;	30✔
216	}
217
218	private:
219	std::vector<NodeSetRulePtr> clauses_;
220	};
221
222	// "string_attr": { "$regex": "^[s][o]me value$" }
223	class NodeSetBasicOperatorString: public NodeSetRule
224	{
225	public:
226	explicit NodeSetBasicOperatorString(std::string attribute,	14✔
227	const std::string& op,
228	std::string value)
229	: op_(string2op(op))	40✔
230	, attribute_(std::move(attribute))	12✔
231	, value_(std::move(value)) {}	14✔
232
233	Selection materialize(const detail::NodeSets& /* unused */,	4✔
234	const NodePopulation& np) const final {
235	switch (op_) {	4✔
236	case Op::regex:	4✔
237	return np.regexMatch(attribute_, value_);	4✔
238	default: // LCOV_EXCL_LINE
239	LIBSONATA_THROW_IF_REACHED // LCOV_EXCL_LINE
240	}
241	}
242
243	std::string toJSON() const final {	10✔
244	return fmt::format(R"("{}": {{ "{}": "{}" }})", attribute_, op2string(op_), value_);	20✔
245	}
246
247	enum class Op {
248	regex = 1,
249	};
250
251	static Op string2op(const std::string& s) {	14✔
252	if (s == "$regex") {	14✔
253	return Op::regex;	12✔
254	}
255	throw SonataError(fmt::format("Operator '{}' not available for strings", s));	4✔
256	}
257
258	static std::string op2string(const Op op) {	10✔
259	switch (op) {	10✔
260	case Op::regex:	10✔
261	return "$regex";	10✔
262	default: // LCOV_EXCL_LINE
263	LIBSONATA_THROW_IF_REACHED // LCOV_EXCL_LINE
264	}
265	}
266
267	private:
268	Op op_;
269	std::string attribute_;
270	std::string value_;
271	};
272
273	// "numeric_attribute_gt": { "$gt": 3 },
274	class NodeSetBasicOperatorNumeric: public NodeSetRule
275	{
276	public:
277	explicit NodeSetBasicOperatorNumeric(std::string name, const std::string& op, double value)	42✔
278	: name_(std::move(name))	84✔
279	, value_(value)
280	, op_(string2op(op)) {}	44✔
281
282	Selection materialize(const detail::NodeSets& /* unused */,	8✔
283	const NodePopulation& np) const final {
284	switch (op_) {	8✔
285	case Op::gt:	2✔
286	return np.filterAttribute<double>(name_, [=](const double v) { return v > value_; });	14✔
287	case Op::lt:	2✔
288	return np.filterAttribute<double>(name_, [=](const double v) { return v < value_; });	14✔
289	case Op::gte:	2✔
290	return np.filterAttribute<double>(name_, [=](const double v) { return v >= value_; });	14✔
291	case Op::lte:	2✔
292	return np.filterAttribute<double>(name_, [=](const double v) { return v <= value_; });	14✔
293	default: // LCOV_EXCL_LINE
294	LIBSONATA_THROW_IF_REACHED // LCOV_EXCL_LINE
295	}
296	}
297
298	std::string toJSON() const final {	40✔
299	return fmt::format(R"("{}": {{ "{}": {} }})", name_, op2string(op_), value_);	80✔
300	}
301
302	enum class Op {
303	gt = 1,
304	lt = 2,
305	gte = 3,
306	lte = 4,
307	};
308
309	static Op string2op(const std::string& s) {	42✔
310	if (s == "$gt") {	42✔
311	return Op::gt;	10✔
312	} else if (s == "$lt") {	32✔
313	return Op::lt;	10✔
314	} else if (s == "$gte") {	22✔
315	return Op::gte;	10✔
316	} else if (s == "$lte") {	12✔
317	return Op::lte;	10✔
318	}
319	throw SonataError(fmt::format("Operator '{}' not available for numeric", s));	4✔
320	}
321
322	static std::string op2string(const Op op) {	40✔
323	switch (op) {	40✔
324	case Op::gt:	10✔
325	return "$gt";	10✔
326	case Op::lt:	10✔
327	return "$lt";	10✔
328	case Op::gte:	10✔
329	return "$gte";	10✔
330	case Op::lte:	10✔
331	return "$lte";	10✔
332	default: // LCOV_EXCL_LINE
333	LIBSONATA_THROW_IF_REACHED // LCOV_EXCL_LINE
334	}
335	}
336
337	private:
338	std::string name_;
339	double value_;
340	Op op_;
341	};
342
343	using CompoundTargets = std::vector<std::string>;
344	class NodeSetCompoundRule: public NodeSetRule
345	{
346	public:
347	NodeSetCompoundRule(std::string name, CompoundTargets targets)	30✔
348	: name_(std::move(name))	60✔
349	, targets_(std::move(targets)) {}	30✔
350
351	Selection materialize(const detail::NodeSets& ns, const NodePopulation& np) const final {	50✔
352	Selection ret{{}};	50✔
353	for (const auto& target : targets_) {	150✔
354	ret = ret \| ns.materialize(target, np);	100✔
355	}
356	return ret;	50✔
357	}
358
359	std::string toJSON() const final {	6✔
360	return toString("node_ids", targets_);	6✔
361	}
362
363	bool is_compound() const override {	180✔
364	return true;	180✔
365	}
366	const CompoundTargets& getTargets() const {	90✔
367	return targets_;	90✔
368	}
369
370	private:
371	std::string name_;
372	CompoundTargets targets_;
373	};
374
375	int64_t get_int64_or_throw(const json& el) {	78✔
376	auto v = el.get<double>();	78✔
377	if (std::floor(v) != v) {	78✔
378	throw SonataError(fmt::format("expected integer, got float {}", v));	×
379	}
380	return static_cast<int64_t>(v);	78✔
381	}
382
383	uint64_t get_uint64_or_throw(const json& el) {	6✔
384	auto v = el.get<double>();	6✔
385	if (v < 0) {	6✔
386	throw SonataError(fmt::format("expected unsigned integer, got {}", v));	4✔
387	}
388
389	if (std::floor(v) != v) {	4✔
390	throw SonataError(fmt::format("expected integer, got float {}", v));	4✔
391	}
392	return static_cast<uint64_t>(v);	2✔
393	}
394
395	NodeSetRulePtr _dispatch_node(const std::string& attribute, const json& value) {	152✔
396	if (value.is_number()) {	152✔
397	if (attribute == "population") {	14✔
398	throw SonataError("'population' must be a string");	2✔
399	}
400
401	if (attribute == "node_id") {	12✔
402	Selection::Values node_ids{get_uint64_or_throw(value)};	6✔
403	return std::make_unique<NodeSetBasicNodeIds>(std::move(node_ids));	2✔
404	} else {
405	std::vector<int64_t> f = {get_int64_or_throw(value)};	6✔
406	return std::make_unique<NodeSetBasicRule<int64_t>>(attribute, f);	6✔
407	}
408	} else if (value.is_string()) {	138✔
409	if (attribute == "node_id") {	28✔
410	throw SonataError("'node_id' must be numeric or a list of numbers");	2✔
411	}
412
413	if (attribute == "population") {	26✔
414	std::vector<std::string> v{value.get<std::string>()};	30✔
415	return std::make_unique<NodeSetBasicPopulation>(v);	10✔
416	} else {
417	std::vector<std::string> f = {value.get<std::string>()};	48✔
418	return std::make_unique<NodeSetBasicRule<std::string>>(attribute, f);	16✔
419	}
420	} else if (value.is_array()) {	110✔
421	const auto& array = value;	50✔
422
423	if (array.empty()) {	50✔
424	throw SonataError(fmt::format("NodeSet Array is empty for attribute: {}", attribute));	×
425	}
426
427	if (array[0].is_number()) {	50✔
428	if (attribute == "population") {	24✔
429	throw SonataError("'population' must be a string");	2✔
430	}
431
432	std::vector<int64_t> values;	44✔
433	for (auto& inner_el : array.items()) {	94✔
434	values.emplace_back(get_int64_or_throw(inner_el.value()));	72✔
435	}
436
437	if (attribute == "node_id") {	22✔
438	Selection::Values node_ids;	20✔
439	std::transform(begin(values),
440	end(values),
441	back_inserter(node_ids),
442	[](int64_t integer) {	56✔
443	if (integer < 0) {	56✔
444	throw SonataError("'node_id' must be positive");	2✔
445	}
446	return static_cast<Selection::Value>(integer);	54✔
447	});	18✔
448	return std::make_unique<NodeSetBasicNodeIds>(std::move(node_ids));	16✔
449	} else {
450	return std::make_unique<NodeSetBasicRule<int64_t>>(attribute, values);	4✔
451	}
452	} else if (array[0].is_string()) {	26✔
453	if (attribute == "node_id") {	24✔
454	throw SonataError("'node_id' must be numeric or a list of numbers");	2✔
455	}
456
457	std::vector<std::string> values;	44✔
458	for (auto& inner_el : array.items()) {	60✔
459	values.emplace_back(inner_el.value().get<std::string>());	38✔
460	}
461
462	if (attribute == "population") {	22✔
463	return std::make_unique<NodeSetBasicPopulation>(values);	4✔
464	} else {
465	return std::make_unique<NodeSetBasicRule<std::string>>(attribute, values);	18✔
466	}
467	} else {
468	throw SonataError("Unknown array type");	2✔
469	}
470	} else if (value.is_object()) {	60✔
471	const auto& definition = value;	60✔
472	if (definition.size() != 1) {	60✔
473	throw SonataError(
474	fmt::format("Operator '{}' must have object with one key value pair", attribute));	4✔
475	}
476	const auto& key = definition.begin().key();	58✔
477	const auto& value = definition.begin().value();	58✔
478
479	if (value.is_number()) {	58✔
480	return std::make_unique<NodeSetBasicOperatorNumeric>(attribute,	40✔
481	key,
482	value.get<double>());	82✔
483	} else if (value.is_string()) {	16✔
484	return std::make_unique<NodeSetBasicOperatorString>(attribute,	26✔
485	key,
486	value.get<std::string>());	42✔
487	} else {
488	throw SonataError("Unknown operator");	2✔
489	}
490	} else {
491	LIBSONATA_THROW_IF_REACHED // LCOV_EXCL_LINE
492	}
493	}
494
495	void parse_basic(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets) {	78✔
496	for (const auto& el : j.items()) {	244✔
497	const auto& value = el.value();	142✔
498	if (value.is_object()) {	142✔
499	if (value.empty()) {	102✔
500	// ignore
501	} else if (value.size() == 1) {	102✔
502	const auto& inner_el = value.items().begin();	100✔
503	node_sets[el.key()] = _dispatch_node(inner_el.key(), inner_el.value());	76✔
504	} else {
505	std::vector<NodeSetRulePtr> clauses;	26✔
506	for (const auto& inner_el : value.items()) {	102✔
507	clauses.push_back(_dispatch_node(inner_el.key(), inner_el.value()));	76✔
508	}
509	node_sets[el.key()] = std::make_unique<NodeSetBasicMultiClause>(std::move(clauses));	26✔
510	}
511	} else if (value.is_array()) {	40✔
512	// will be parsed by the parse_compound
513	} else {
514	// null/boolean/number/string ?
515	throw SonataError(fmt::format("Expected an array or an object, got: {}", value.dump()));	×
516	}
517	}
518	}	54✔
519
520	void check_compound(const std::map<std::string, NodeSetRulePtr>& node_sets,	102✔
521	const std::map<std::string, CompoundTargets>& compound_rules,
522	const std::string& name,
523	size_t depth) {
524	if (node_sets.count(name) > 0) {	102✔
525	return;	42✔
526	}
527
528	if (depth > MAX_COMPOUND_RECURSION) {	60✔
529	throw SonataError("Compound node_set recursion depth exceeded");	2✔
530	}
531
532	const auto it = compound_rules.find(name);	58✔
533	assert(it != compound_rules.end());	58✔
534
535	for (auto const& target : it->second) {	100✔
536	if (node_sets.count(target) == 0 && compound_rules.count(target) == 0) {	70✔
537	throw SonataError(fmt::format("Missing '{}' from node_sets", target));	8✔
538	}
539	check_compound(node_sets, compound_rules, target, depth + 1);	66✔
540	}
541	}
542
543	void parse_compound(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets) {	54✔
544	std::map<std::string, CompoundTargets> compound_rules;	108✔
545	for (auto& el : j.items()) {	174✔
546	if (el.value().is_array()) {	118✔
547	CompoundTargets targets;	80✔
548	for (const auto& name : el.value()) {	92✔
549	if (!name.is_string()) {	54✔
550	throw SonataError("All compound elements must be strings");	2✔
551	}
552
553	targets.emplace_back(name);	52✔
554	}
555	compound_rules[el.key()] = targets;	38✔
556	}
557	}
558
559
560	for (const auto& rule : compound_rules) {	82✔
561	check_compound(node_sets, compound_rules, rule.first, 0);	36✔
562
563	NodeSetRulePtr rules = std::make_unique<NodeSetCompoundRule>(rule.first, rule.second);	60✔
564	node_sets.emplace(rule.first, std::move(rules));	30✔
565	}
566	}	46✔
567
568	Selection NodeSets::materialize(const std::string& name, const NodePopulation& population) const {	154✔
569	const auto& node_set = node_sets_.find(name);	154✔
570	if (node_set == node_sets_.end()) {	154✔
571	throw SonataError(fmt::format("Unknown node_set {}", name));	4✔
572	}
573	const auto& ns = node_set->second;	152✔
574	if (!ns->is_compound()) {	152✔
575	return population.selectAll() & ns->materialize(*this, population);	244✔
576	}
577
578	// it's common to have a deep structure of compound statements
579	// (ie: a whole hierarchy of regions), all checking the same attribute
580	// rather than `materializing` them separately, we group them, and materialize
581	// them all at once
582	Selection ret{{}};	60✔
583
584	std::vector<NodeSetRule*> queue{ns.get()};	60✔
585	std::map<std::string, std::set<std::string>> attribute2rule_strings;	60✔
586	std::map<std::string, std::set<int64_t>> attribute2rule_int64;	60✔
587	while (!queue.empty()) {	120✔
588	const auto* ns = queue.back();	90✔
589	queue.pop_back();	90✔
590
591	if (ns->is_compound()) {	90✔
592	const auto* targets = dynamic_cast<const NodeSetCompoundRule*>(ns);	90✔
593	for (const auto& target : targets->getTargets()) {	220✔
594	const auto& node_set = node_sets_.find(target)->second;	130✔
595	if (node_set->is_compound()) {	130✔
596	queue.push_back(node_set.get());	60✔
597	continue;	60✔
598	}
599
600	{
601	const auto* basic_int = dynamic_cast<const NodeSetBasicRule<int64_t>*>(	70✔
602	node_set.get());	140✔
603	if (basic_int != nullptr) {	70✔
604	basic_int->add_attribute2rule(attribute2rule_int64);	10✔
605	continue;	10✔
606	}
607	}
608
609	{
610	const auto* basic_string = dynamic_cast<const NodeSetBasicRule<std::string>*>(	60✔
611	node_set.get());	120✔
612	if (basic_string != nullptr) {	60✔
613	basic_string->add_attribute2rule(attribute2rule_strings);	10✔
614	continue;	10✔
615	}
616	}
617
618	ret = ret \| ns->materialize(*this, population);	50✔
619	}
620	} else {
621	ret = ret \| ns->materialize(*this, population);	×
622	}
623	}
624
625	for (const auto& it : attribute2rule_strings) {	40✔
626	std::vector<std::string> values(it.second.begin(), it.second.end());	10✔
627	ret = ret \| population.matchAttributeValues(it.first, values);	10✔
628	}
629
630	for (const auto& it : attribute2rule_int64) {	40✔
631	std::vector<int64_t> values(it.second.begin(), it.second.end());	10✔
632	ret = ret \| population.matchAttributeValues(it.first, values);	10✔
633	}
634
635	return ret;	30✔
636	}
637	} // namespace detail
638
639	NodeSets::NodeSets(const std::string& content)	80✔
640	: impl_(new detail::NodeSets(content)) {}	80✔
641
642	NodeSets::NodeSets(std::unique_ptr<detail::NodeSets>&& impl)	2✔
643	: impl_(std::move(impl)) {}	2✔
644
645	NodeSets::NodeSets(NodeSets&&) noexcept = default;
646	NodeSets& NodeSets::operator=(NodeSets&&) noexcept = default;
647	NodeSets::~NodeSets() = default;
648
649	NodeSets NodeSets::fromFile(const std::string& path) {	2✔
650	return NodeSets(detail::NodeSets::fromFile(path));	4✔
651	}
652
653	Selection NodeSets::materialize(const std::string& name, const NodePopulation& population) const {	54✔
654	return impl_->materialize(name, population);	54✔
655	}
656
657	std::set<std::string> NodeSets::names() const {	2✔
658	return impl_->names();	2✔
659	}
660
661	std::string NodeSets::toJSON() const {	10✔
662	return impl_->toJSON();	10✔
663	}
664
665	} // namespace sonata
666	} // namespace bbp

BlueBrain / libsonata / 4719722733

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous