4741684009

Build Type

push

github

Committed by Mike Gevaert

Commit Message

review comments

Run Details

8 of 8 new or added lines in 1 file covered. (100.0%)

1743 of 1790 relevant lines covered (97.37%)

79.24 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.39

/src/node_sets.cpp

#include <algorithm>  // std::find, std::transform
#include <cassert>
#include <cmath>
#include <fmt/format.h>
#include <fstream>
#include <sstream>
#include <type_traits>

#include "../extlib/filesystem.hpp"

#include <nlohmann/json.hpp>
#include <utility>

#include "utils.h"  // readFile

#include <bbp/sonata/node_sets.h>

namespace bbp {
namespace sonata {

namespace fs = ghc::filesystem;

namespace detail {

const size_t MAX_COMPOUND_RECURSION = 10;

using json = nlohmann::json;

void replace_trailing_coma(std::string& s, char c) {
    s.pop_back();
    s.pop_back();
    s.push_back(c);
}

template <typename T>
std::string toString(const std::string& key, const std::vector<T>& values) {
    return fmt::format(R"("{}": [{}])", key, fmt::join(values, ", "));
}

template <>
std::string toString(const std::string& key, const std::vector<std::string>& values) {
    // strings need to be wrapped in quotes
    return fmt::format(R"("{}": ["{}"])", key, fmt::join(values, "\", \""));
}

class NodeSets;

class NodeSetRule
{
  public:
    virtual ~NodeSetRule() = default;

    virtual Selection materialize(const NodeSets&, const NodePopulation&) const = 0;
    virtual std::string toJSON() const = 0;
    virtual bool is_compound() const {
        return false;
    }
};

using NodeSetRulePtr = std::unique_ptr<NodeSetRule>;
void parse_basic(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets);
void parse_compound(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets);

class NodeSets
{
    std::map<std::string, NodeSetRulePtr> node_sets_;

  public:
    explicit NodeSets(const json& j) {
        if (!j.is_object()) {
            throw SonataError("Top level node_set must be an object");
        }

        // Need to two pass parsing the json so that compound lookup can rely
        // on all the basic rules existing
        parse_basic(j, node_sets_);
        parse_compound(j, node_sets_);
    }

    static const fs::path& validate_path(const fs::path& path) {
        if (!fs::exists(path)) {
            throw SonataError(fmt::format("Path does not exist: {}", std::string(path)));
        }
        return path;
    }

    explicit NodeSets(const fs::path& path)
        : NodeSets(json::parse(std::ifstream(validate_path(path)))) {}

    static std::unique_ptr<NodeSets> fromFile(const std::string& path_) {
        fs::path path(path_);
        return std::make_unique<detail::NodeSets>(path);
    }

    explicit NodeSets(const std::string& content)
        : NodeSets(json::parse(content)) {}

    Selection materialize(const std::string& name, const NodePopulation& population) const;

    std::set<std::string> names() const {
        return getMapKeys(node_sets_);
    }

    std::string toJSON() const {
        std::string ret{"{\n"};
        for (const auto& pair : node_sets_) {
            ret += fmt::format(R"(  "{}": {{)", pair.first);
            ret += pair.second->toJSON();
            ret += "},\n";
        }
        replace_trailing_coma(ret, '\n');
        ret += "}";

        return ret;
    }
};

// { 'region': ['region1', 'region2', ...] }
template <typename T>
class NodeSetBasicRule: public NodeSetRule
{
  public:
    NodeSetBasicRule(std::string attribute, std::vector<T>& values)
        : attribute_(std::move(attribute))
        , values_(values) {}

    Selection materialize(const detail::NodeSets& /* unused */,
                          const NodePopulation& np) const final {
        return np.matchAttributeValues(attribute_, values_);
    }

    void add_attribute2rule(std::map<std::string, std::set<T>>& attribute2rule) const {
        auto& s = attribute2rule[attribute_];
        for (const auto& v : values_) {
            s.insert(v);
        }
    }

    std::string toJSON() const final {
        return toString(attribute_, values_);
    }

  private:
    std::string attribute_;
    std::vector<T> values_;
};

// { 'population': ['popA', 'popB', ] }
class NodeSetBasicPopulation: public NodeSetRule
{
  public:
    explicit NodeSetBasicPopulation(std::vector<std::string>& values)
        : values_(values) {}

    Selection materialize(const detail::NodeSets& /* unused */,
                          const NodePopulation& np) const final {
        if (std::find(values_.begin(), values_.end(), np.name()) != values_.end()) {
            return np.selectAll();
        }

        return Selection{{}};
    }

    std::string toJSON() const final {
        return toString("population", values_);
    }

  private:
    std::vector<std::string> values_;
};

// { 'node_id': [1, 2, 3, 4] }
class NodeSetBasicNodeIds: public NodeSetRule
{
  public:
    explicit NodeSetBasicNodeIds(Selection::Values&& values)
        : values_(values) {}

    Selection materialize(const detail::NodeSets& /* unused */,
                          const NodePopulation& np) const final {
        return np.selectAll() & Selection::fromValues(values_.begin(), values_.end());
    }

    std::string toJSON() const final {
        return toString("node_ids", values_);
    }

  private:
    Selection::Values values_;
};

//  {
//      "population": "biophysical",
//      "model_type": "point",
//      "node_id": [1, 2, 3, 5, 7, 9, ...]
//  }
class NodeSetBasicMultiClause: public NodeSetRule
{
  public:
    explicit NodeSetBasicMultiClause(std::vector<NodeSetRulePtr>&& clauses)
        : clauses_(std::move(clauses)) {}

    Selection materialize(const detail::NodeSets& ns, const NodePopulation& np) const final {
        Selection ret = np.selectAll();
        for (const auto& clause : clauses_) {
            ret = ret & clause->materialize(ns, np);
        }
        return ret;
    }

    std::string toJSON() const final {
        std::string ret;
        for (const auto& clause : clauses_) {
            ret += clause->toJSON();
            ret += ", ";
        }
        replace_trailing_coma(ret, ' ');
        return ret;
    }

  private:
    std::vector<NodeSetRulePtr> clauses_;
};

// "string_attr": { "$regex": "^[s][o]me value$" }
class NodeSetBasicOperatorString: public NodeSetRule
{
  public:
    explicit NodeSetBasicOperatorString(std::string attribute,
                                        const std::string& op,
                                        std::string value)
        : op_(string2op(op))
        , attribute_(std::move(attribute))
        , value_(std::move(value)) {}

    Selection materialize(const detail::NodeSets& /* unused */,
                          const NodePopulation& np) const final {
        switch (op_) {
        case Op::regex:
            return np.regexMatch(attribute_, value_);
        default:              // LCOV_EXCL_LINE
            LIBSONATA_THROW_IF_REACHED  // LCOV_EXCL_LINE
        }
    }

    std::string toJSON() const final {
        return fmt::format(R"("{}": {{ "{}": "{}" }})", attribute_, op2string(op_), value_);
    }

    enum class Op {
        regex = 1,
    };

    static Op string2op(const std::string& s) {
        if (s == "$regex") {
            return Op::regex;
        }
        throw SonataError(fmt::format("Operator '{}' not available for strings", s));
    }

    static std::string op2string(const Op op) {
        switch (op) {
        case Op::regex:
            return "$regex";
        default:              // LCOV_EXCL_LINE
            LIBSONATA_THROW_IF_REACHED  // LCOV_EXCL_LINE
        }
    }

  private:
    Op op_;
    std::string attribute_;
    std::string value_;
};

// "numeric_attribute_gt": { "$gt": 3 },
class NodeSetBasicOperatorNumeric: public NodeSetRule
{
  public:
    explicit NodeSetBasicOperatorNumeric(std::string name, const std::string& op, double value)
        : name_(std::move(name))
        , value_(value)
        , op_(string2op(op)) {}

    Selection materialize(const detail::NodeSets& /* unused */,
                          const NodePopulation& np) const final {
        switch (op_) {
        case Op::gt:
            return np.filterAttribute<double>(name_, [=](const double v) { return v > value_; });
        case Op::lt:
            return np.filterAttribute<double>(name_, [=](const double v) { return v < value_; });
        case Op::gte:
            return np.filterAttribute<double>(name_, [=](const double v) { return v >= value_; });
        case Op::lte:
            return np.filterAttribute<double>(name_, [=](const double v) { return v <= value_; });
        default:              // LCOV_EXCL_LINE
            LIBSONATA_THROW_IF_REACHED  // LCOV_EXCL_LINE
        }
    }

    std::string toJSON() const final {
        return fmt::format(R"("{}": {{ "{}": {} }})", name_, op2string(op_), value_);
    }

    enum class Op {
        gt = 1,
        lt = 2,
        gte = 3,
        lte = 4,
    };

    static Op string2op(const std::string& s) {
        if (s == "$gt") {
            return Op::gt;
        } else if (s == "$lt") {
            return Op::lt;
        } else if (s == "$gte") {
            return Op::gte;
        } else if (s == "$lte") {
            return Op::lte;
        }
        throw SonataError(fmt::format("Operator '{}' not available for numeric", s));
    }

    static std::string op2string(const Op op) {
        switch (op) {
        case Op::gt:
            return "$gt";
        case Op::lt:
            return "$lt";
        case Op::gte:
            return "$gte";
        case Op::lte:
            return "$lte";
        default:              // LCOV_EXCL_LINE
            LIBSONATA_THROW_IF_REACHED  // LCOV_EXCL_LINE
        }
    }

  private:
    std::string name_;
    double value_;
    Op op_;
};

using CompoundTargets = std::vector<std::string>;
class NodeSetCompoundRule: public NodeSetRule
{
  public:
    NodeSetCompoundRule(std::string name, CompoundTargets targets)
        : name_(std::move(name))
        , targets_(std::move(targets)) {}

    Selection materialize(const detail::NodeSets& ns, const NodePopulation& np) const final {
        Selection ret{{}};
        for (const auto& target : targets_) {
            ret = ret | ns.materialize(target, np);
        }
        return ret;
    }

    std::string toJSON() const final {
        return toString("node_ids", targets_);
    }

    bool is_compound() const override {
        return true;
    }
    const CompoundTargets& getTargets() const {
        return targets_;
    }

  private:
    std::string name_;
    CompoundTargets targets_;
};

int64_t get_int64_or_throw(const json& el) {
    auto v = el.get<double>();
    if (std::floor(v) != v) {
        throw SonataError(fmt::format("expected integer, got float {}", v));
    }
    return static_cast<int64_t>(v);
}

uint64_t get_uint64_or_throw(const json& el) {
    auto v = el.get<double>();
    if (v < 0) {
        throw SonataError(fmt::format("expected unsigned integer, got {}", v));
    }

    if (std::floor(v) != v) {
        throw SonataError(fmt::format("expected integer, got float {}", v));
    }
    return static_cast<uint64_t>(v);
}

NodeSetRulePtr _dispatch_node(const std::string& attribute, const json& value) {
    if (value.is_number()) {
        if (attribute == "population") {
            throw SonataError("'population' must be a string");
        }

        if (attribute == "node_id") {
            Selection::Values node_ids{get_uint64_or_throw(value)};
            return std::make_unique<NodeSetBasicNodeIds>(std::move(node_ids));
        } else {
            std::vector<int64_t> f = {get_int64_or_throw(value)};
            return std::make_unique<NodeSetBasicRule<int64_t>>(attribute, f);
        }
    } else if (value.is_string()) {
        if (attribute == "node_id") {
            throw SonataError("'node_id' must be numeric or a list of numbers");
        }

        if (attribute == "population") {
            std::vector<std::string> v{value.get<std::string>()};
            return std::make_unique<NodeSetBasicPopulation>(v);
        } else {
            std::vector<std::string> f = {value.get<std::string>()};
            return std::make_unique<NodeSetBasicRule<std::string>>(attribute, f);
        }
    } else if (value.is_array()) {
        const auto& array = value;

        if (array.empty()) {
            throw SonataError(fmt::format("NodeSet Array is empty for attribute: {}", attribute));
        }

        if (array[0].is_number()) {
            if (attribute == "population") {
                throw SonataError("'population' must be a string");
            }

            std::vector<int64_t> values;
            for (auto& inner_el : array.items()) {
                values.emplace_back(get_int64_or_throw(inner_el.value()));
            }

            if (attribute == "node_id") {
                Selection::Values node_ids;
                std::transform(begin(values),
                               end(values),
                               back_inserter(node_ids),
                               [](int64_t integer) {
                                   if (integer < 0) {
                                       throw SonataError("'node_id' must be positive");
                                   }
                                   return static_cast<Selection::Value>(integer);
                               });
                return std::make_unique<NodeSetBasicNodeIds>(std::move(node_ids));
            } else {
                return std::make_unique<NodeSetBasicRule<int64_t>>(attribute, values);
            }
        } else if (array[0].is_string()) {
            if (attribute == "node_id") {
                throw SonataError("'node_id' must be numeric or a list of numbers");
            }

            std::vector<std::string> values;
            for (auto& inner_el : array.items()) {
                values.emplace_back(inner_el.value().get<std::string>());
            }

            if (attribute == "population") {
                return std::make_unique<NodeSetBasicPopulation>(values);
            } else {
                return std::make_unique<NodeSetBasicRule<std::string>>(attribute, values);
            }
        } else {
            throw SonataError("Unknown array type");
        }
    } else if (value.is_object()) {
        const auto& definition = value;
        if (definition.size() != 1) {
            throw SonataError(
                fmt::format("Operator '{}' must have object with one key value pair", attribute));
        }
        const auto& key = definition.begin().key();
        const auto& value = definition.begin().value();

        if (value.is_number()) {
            return std::make_unique<NodeSetBasicOperatorNumeric>(attribute,
                                                                 key,
                                                                 value.get<double>());
        } else if (value.is_string()) {
            return std::make_unique<NodeSetBasicOperatorString>(attribute,
                                                                key,
                                                                value.get<std::string>());
        } else {
            throw SonataError("Unknown operator");
        }
    } else {
        LIBSONATA_THROW_IF_REACHED  // LCOV_EXCL_LINE
    }
}

void parse_basic(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets) {
    for (const auto& el : j.items()) {
        const auto& value = el.value();
        if (value.is_object()) {
            if (value.empty()) {
                // ignore
            } else if (value.size() == 1) {
                const auto& inner_el = value.items().begin();
                node_sets[el.key()] = _dispatch_node(inner_el.key(), inner_el.value());
            } else {
                std::vector<NodeSetRulePtr> clauses;
                for (const auto& inner_el : value.items()) {
                    clauses.push_back(_dispatch_node(inner_el.key(), inner_el.value()));
                }
                node_sets[el.key()] = std::make_unique<NodeSetBasicMultiClause>(std::move(clauses));
            }
        } else if (value.is_array()) {
            // will be parsed by the parse_compound
        } else {
            // null/boolean/number/string ?
            throw SonataError(fmt::format("Expected an array or an object, got: {}", value.dump()));
        }
    }
}

void check_compound(const std::map<std::string, NodeSetRulePtr>& node_sets,
                    const std::map<std::string, CompoundTargets>& compound_rules,
                    const std::string& name,
                    size_t depth) {
    if (node_sets.count(name) > 0) {
        return;
    }

    if (depth > MAX_COMPOUND_RECURSION) {
        throw SonataError("Compound node_set recursion depth exceeded");
    }

    const auto it = compound_rules.find(name);
    assert(it != compound_rules.end());

    for (auto const& target : it->second) {
        if (node_sets.count(target) == 0 && compound_rules.count(target) == 0) {
            throw SonataError(fmt::format("Missing '{}' from node_sets", target));
        }
        check_compound(node_sets, compound_rules, target, depth + 1);
    }
}

void parse_compound(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets) {
    std::map<std::string, CompoundTargets> compound_rules;
    for (auto& el : j.items()) {
        if (el.value().is_array()) {
            CompoundTargets targets;
            for (const auto& name : el.value()) {
                if (!name.is_string()) {
                    throw SonataError("All compound elements must be strings");
                }

                targets.emplace_back(name);
            }
            compound_rules[el.key()] = targets;
        }
    }


    for (const auto& rule : compound_rules) {
        check_compound(node_sets, compound_rules, rule.first, 0);

        NodeSetRulePtr rules = std::make_unique<NodeSetCompoundRule>(rule.first, rule.second);
        node_sets.emplace(rule.first, std::move(rules));
    }
}

Selection NodeSets::materialize(const std::string& name, const NodePopulation& population) const {
    const auto& node_set = node_sets_.find(name);
    if (node_set == node_sets_.end()) {
        throw SonataError(fmt::format("Unknown node_set {}", name));
    }
    const auto& ns = node_set->second;
    if (!ns->is_compound()) {
        return population.selectAll() & ns->materialize(*this, population);
    }

    // it's common to have a deep structure of compound statements
    // (ie: a whole hierarchy of regions), all checking the same attribute
    // rather than `materializing` them separately, we group them, and materialize
    // them all at once
    Selection ret{{}};

    std::vector<NodeSetRule*> queue{ns.get()};
    std::map<std::string, std::set<std::string>> attribute2rule_strings;
    std::map<std::string, std::set<int64_t>> attribute2rule_int64;
    while (!queue.empty()) {
        const auto* ns = queue.back();
        queue.pop_back();

        if (ns->is_compound()) {
            const auto* targets = dynamic_cast<const NodeSetCompoundRule*>(ns);
            for (const auto& target : targets->getTargets()) {
                const auto& node_set = node_sets_.find(target)->second;
                if (node_set->is_compound()) {
                    queue.push_back(node_set.get());
                    continue;
                }

                {
                    const auto* basic_int = dynamic_cast<const NodeSetBasicRule<int64_t>*>(
                        node_set.get());
                    if (basic_int != nullptr) {
                        basic_int->add_attribute2rule(attribute2rule_int64);
                        continue;
                    }
                }

                {
                    const auto* basic_string = dynamic_cast<const NodeSetBasicRule<std::string>*>(
                        node_set.get());
                    if (basic_string != nullptr) {
                        basic_string->add_attribute2rule(attribute2rule_strings);
                        continue;
                    }
                }

                ret = ret | ns->materialize(*this, population);
            }
        } else {
            ret = ret | ns->materialize(*this, population);
        }
    }

    for (const auto& it : attribute2rule_strings) {
        std::vector<std::string> values(it.second.begin(), it.second.end());
        ret = ret | population.matchAttributeValues(it.first, values);
    }

    for (const auto& it : attribute2rule_int64) {
        std::vector<int64_t> values(it.second.begin(), it.second.end());
        ret = ret | population.matchAttributeValues(it.first, values);
    }

    return ret;
}
}  // namespace detail

NodeSets::NodeSets(const std::string& content)
    : impl_(new detail::NodeSets(content)) {}

NodeSets::NodeSets(std::unique_ptr<detail::NodeSets>&& impl)
    : impl_(std::move(impl)) {}

NodeSets::NodeSets(NodeSets&&) noexcept = default;
NodeSets& NodeSets::operator=(NodeSets&&) noexcept = default;
NodeSets::~NodeSets() = default;

NodeSets NodeSets::fromFile(const std::string& path) {
    return NodeSets(detail::NodeSets::fromFile(path));
}

Selection NodeSets::materialize(const std::string& name, const NodePopulation& population) const {
    return impl_->materialize(name, population);
}

std::set<std::string> NodeSets::names() const {
    return impl_->names();
}

std::string NodeSets::toJSON() const {
    return impl_->toJSON();
}

}  // namespace sonata
}  // namespace bbp

1	#include <algorithm> // std::find, std::transform
2	#include <cassert>
3	#include <cmath>
4	#include <fmt/format.h>
5	#include <fstream>
6	#include <sstream>
7	#include <type_traits>
8
9	#include "../extlib/filesystem.hpp"
10
11	#include <nlohmann/json.hpp>
12	#include <utility>
13
14	#include "utils.h" // readFile
15
16	#include <bbp/sonata/node_sets.h>
17
18	namespace bbp {
19	namespace sonata {
20
21	namespace fs = ghc::filesystem;
22
23	namespace detail {
24
25	const size_t MAX_COMPOUND_RECURSION = 10;
26
27	using json = nlohmann::json;
28
29	void replace_trailing_coma(std::string& s, char c) {	40✔
30	s.pop_back();	40✔
31	s.pop_back();	40✔
32	s.push_back(c);	40✔
33	}	40✔
34
35	template <typename T>
36	std::string toString(const std::string& key, const std::vector<T>& values) {	10✔
37	return fmt::format(R"("{}": [{}])", key, fmt::join(values, ", "));	20✔
38	}
39
40	template <>
41	std::string toString(const std::string& key, const std::vector<std::string>& values) {	50✔
42	// strings need to be wrapped in quotes
43	return fmt::format(R"("{}": ["{}"])", key, fmt::join(values, "\", \""));	100✔
44	}
45
46	class NodeSets;
47
48	class NodeSetRule
49	{
50	public:
51	virtual ~NodeSetRule() = default;	188✔
52
53	virtual Selection materialize(const NodeSets&, const NodePopulation&) const = 0;
54	virtual std::string toJSON() const = 0;
55	virtual bool is_compound() const {	192✔
56	return false;	192✔
57	}
58	};
59
60	using NodeSetRulePtr = std::unique_ptr<NodeSetRule>;
61	void parse_basic(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets);
62	void parse_compound(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets);
63
64	class NodeSets
65	{
66	std::map<std::string, NodeSetRulePtr> node_sets_;
67
68	public:
69	explicit NodeSets(const json& j) {	118✔
70	if (!j.is_object()) {	82✔
71	throw SonataError("Top level node_set must be an object");	4✔
72	}
73
74	// Need to two pass parsing the json so that compound lookup can rely
75	// on all the basic rules existing
76	parse_basic(j, node_sets_);	78✔
77	parse_compound(j, node_sets_);	54✔
78	}	46✔
79
80	static const fs::path& validate_path(const fs::path& path) {	2✔
81	if (!fs::exists(path)) {	2✔
82	throw SonataError(fmt::format("Path does not exist: {}", std::string(path)));	×
83	}
84	return path;	2✔
85	}
86
87	explicit NodeSets(const fs::path& path)	2✔
88	: NodeSets(json::parse(std::ifstream(validate_path(path)))) {}	2✔
89
90	static std::unique_ptr<NodeSets> fromFile(const std::string& path_) {	2✔
91	fs::path path(path_);	4✔
92	return std::make_unique<detail::NodeSets>(path);	4✔
93	}
94
95	explicit NodeSets(const std::string& content)	80✔
96	: NodeSets(json::parse(content)) {}	116✔
97
98	Selection materialize(const std::string& name, const NodePopulation& population) const;
99
100	std::set<std::string> names() const {	2✔
101	return getMapKeys(node_sets_);	2✔
102	}
103
104	std::string toJSON() const {	10✔
105	std::string ret{"{\n"};	10✔
106	for (const auto& pair : node_sets_) {	60✔
107	ret += fmt::format(R"( "{}": {{)", pair.first);	100✔
108	ret += pair.second->toJSON();	50✔
109	ret += "},\n";	50✔
110	}
111	replace_trailing_coma(ret, '\n');	10✔
112	ret += "}";	10✔
113
114	return ret;	10✔
115	}
116	};
117
118	// { 'region': ['region1', 'region2', ...] }
119	template <typename T>
120	class NodeSetBasicRule: public NodeSetRule
121	{
122	public:
123	NodeSetBasicRule(std::string attribute, std::vector<T>& values)	44✔
124	: attribute_(std::move(attribute))	44✔
125	, values_(values) {}	44✔
126
127	Selection materialize(const detail::NodeSets& /* unused */,	10✔
128	const NodePopulation& np) const final {
129	return np.matchAttributeValues(attribute_, values_);	10✔
130	}
131
132	void add_attribute2rule(std::map<std::string, std::set<T>>& attribute2rule) const {	20✔
133	auto& s = attribute2rule[attribute_];	20✔
134	for (const auto& v : values_) {	50✔
135	s.insert(v);	30✔
136	}
137	}	20✔
138
139	std::string toJSON() const final {	38✔
140	return toString(attribute_, values_);	38✔
141	}
142
143	private:
144	std::string attribute_;
145	std::vector<T> values_;
146	};
147
148	// { 'population': ['popA', 'popB', ] }
149	class NodeSetBasicPopulation: public NodeSetRule
150	{
151	public:
152	explicit NodeSetBasicPopulation(std::vector<std::string>& values)	14✔
153	: values_(values) {}	14✔
154
155	Selection materialize(const detail::NodeSets& /* unused */,	6✔
156	const NodePopulation& np) const final {
157	if (std::find(values_.begin(), values_.end(), np.name()) != values_.end()) {	6✔
158	return np.selectAll();	4✔
159	}
160
161	return Selection{{}};	2✔
162	}
163
164	std::string toJSON() const final {	10✔
165	return toString("population", values_);	10✔
166	}
167
168	private:
169	std::vector<std::string> values_;
170	};
171
172	// { 'node_id': [1, 2, 3, 4] }
173	class NodeSetBasicNodeIds: public NodeSetRule
174	{
175	public:
176	explicit NodeSetBasicNodeIds(Selection::Values&& values)	18✔
177	: values_(values) {}	18✔
178
179	Selection materialize(const detail::NodeSets& /* unused */,	96✔
180	const NodePopulation& np) const final {
181	return np.selectAll() & Selection::fromValues(values_.begin(), values_.end());	192✔
182	}
183
184	std::string toJSON() const final {	6✔
185	return toString("node_ids", values_);	6✔
186	}
187
188	private:
189	Selection::Values values_;
190	};
191
192	// {
193	// "population": "biophysical",
194	// "model_type": "point",
195	// "node_id": [1, 2, 3, 5, 7, 9, ...]
196	// }
197	class NodeSetBasicMultiClause: public NodeSetRule
198	{
199	public:
200	explicit NodeSetBasicMultiClause(std::vector<NodeSetRulePtr>&& clauses)	26✔
201	: clauses_(std::move(clauses)) {}	26✔
202
203	Selection materialize(const detail::NodeSets& ns, const NodePopulation& np) const final {	2✔
204	Selection ret = np.selectAll();	2✔
205	for (const auto& clause : clauses_) {	6✔
206	ret = ret & clause->materialize(ns, np);	4✔
207	}
208	return ret;	2✔
209	}
210
211	std::string toJSON() const final {	30✔
212	std::string ret;	30✔
213	for (const auto& clause : clauses_) {	120✔
214	ret += clause->toJSON();	90✔
215	ret += ", ";	90✔
216	}
217	replace_trailing_coma(ret, ' ');	30✔
218	return ret;	30✔
219	}
220
221	private:
222	std::vector<NodeSetRulePtr> clauses_;
223	};
224
225	// "string_attr": { "$regex": "^[s][o]me value$" }
226	class NodeSetBasicOperatorString: public NodeSetRule
227	{
228	public:
229	explicit NodeSetBasicOperatorString(std::string attribute,	14✔
230	const std::string& op,
231	std::string value)
232	: op_(string2op(op))	40✔
233	, attribute_(std::move(attribute))	12✔
234	, value_(std::move(value)) {}	14✔
235
236	Selection materialize(const detail::NodeSets& /* unused */,	4✔
237	const NodePopulation& np) const final {
238	switch (op_) {	4✔
239	case Op::regex:	4✔
240	return np.regexMatch(attribute_, value_);	4✔
241	default: // LCOV_EXCL_LINE
242	LIBSONATA_THROW_IF_REACHED // LCOV_EXCL_LINE
243	}
244	}
245
246	std::string toJSON() const final {	10✔
247	return fmt::format(R"("{}": {{ "{}": "{}" }})", attribute_, op2string(op_), value_);	20✔
248	}
249
250	enum class Op {
251	regex = 1,
252	};
253
254	static Op string2op(const std::string& s) {	14✔
255	if (s == "$regex") {	14✔
256	return Op::regex;	12✔
257	}
258	throw SonataError(fmt::format("Operator '{}' not available for strings", s));	4✔
259	}
260
261	static std::string op2string(const Op op) {	10✔
262	switch (op) {	10✔
263	case Op::regex:	10✔
264	return "$regex";	10✔
265	default: // LCOV_EXCL_LINE
266	LIBSONATA_THROW_IF_REACHED // LCOV_EXCL_LINE
267	}
268	}
269
270	private:
271	Op op_;
272	std::string attribute_;
273	std::string value_;
274	};
275
276	// "numeric_attribute_gt": { "$gt": 3 },
277	class NodeSetBasicOperatorNumeric: public NodeSetRule
278	{
279	public:
280	explicit NodeSetBasicOperatorNumeric(std::string name, const std::string& op, double value)	42✔
281	: name_(std::move(name))	84✔
282	, value_(value)
283	, op_(string2op(op)) {}	44✔
284
285	Selection materialize(const detail::NodeSets& /* unused */,	8✔
286	const NodePopulation& np) const final {
287	switch (op_) {	8✔
288	case Op::gt:	2✔
289	return np.filterAttribute<double>(name_, [=](const double v) { return v > value_; });	14✔
290	case Op::lt:	2✔
291	return np.filterAttribute<double>(name_, [=](const double v) { return v < value_; });	14✔
292	case Op::gte:	2✔
293	return np.filterAttribute<double>(name_, [=](const double v) { return v >= value_; });	14✔
294	case Op::lte:	2✔
295	return np.filterAttribute<double>(name_, [=](const double v) { return v <= value_; });	14✔
296	default: // LCOV_EXCL_LINE
297	LIBSONATA_THROW_IF_REACHED // LCOV_EXCL_LINE
298	}
299	}
300
301	std::string toJSON() const final {	40✔
302	return fmt::format(R"("{}": {{ "{}": {} }})", name_, op2string(op_), value_);	80✔
303	}
304
305	enum class Op {
306	gt = 1,
307	lt = 2,
308	gte = 3,
309	lte = 4,
310	};
311
312	static Op string2op(const std::string& s) {	42✔
313	if (s == "$gt") {	42✔
314	return Op::gt;	10✔
315	} else if (s == "$lt") {	32✔
316	return Op::lt;	10✔
317	} else if (s == "$gte") {	22✔
318	return Op::gte;	10✔
319	} else if (s == "$lte") {	12✔
320	return Op::lte;	10✔
321	}
322	throw SonataError(fmt::format("Operator '{}' not available for numeric", s));	4✔
323	}
324
325	static std::string op2string(const Op op) {	40✔
326	switch (op) {	40✔
327	case Op::gt:	10✔
328	return "$gt";	10✔
329	case Op::lt:	10✔
330	return "$lt";	10✔
331	case Op::gte:	10✔
332	return "$gte";	10✔
333	case Op::lte:	10✔
334	return "$lte";	10✔
335	default: // LCOV_EXCL_LINE
336	LIBSONATA_THROW_IF_REACHED // LCOV_EXCL_LINE
337	}
338	}
339
340	private:
341	std::string name_;
342	double value_;
343	Op op_;
344	};
345
346	using CompoundTargets = std::vector<std::string>;
347	class NodeSetCompoundRule: public NodeSetRule
348	{
349	public:
350	NodeSetCompoundRule(std::string name, CompoundTargets targets)	30✔
351	: name_(std::move(name))	60✔
352	, targets_(std::move(targets)) {}	30✔
353
354	Selection materialize(const detail::NodeSets& ns, const NodePopulation& np) const final {	50✔
355	Selection ret{{}};	50✔
356	for (const auto& target : targets_) {	150✔
357	ret = ret \| ns.materialize(target, np);	100✔
358	}
359	return ret;	50✔
360	}
361
362	std::string toJSON() const final {	6✔
363	return toString("node_ids", targets_);	6✔
364	}
365
366	bool is_compound() const override {	180✔
367	return true;	180✔
368	}
369	const CompoundTargets& getTargets() const {	90✔
370	return targets_;	90✔
371	}
372
373	private:
374	std::string name_;
375	CompoundTargets targets_;
376	};
377
378	int64_t get_int64_or_throw(const json& el) {	78✔
379	auto v = el.get<double>();	78✔
380	if (std::floor(v) != v) {	78✔
381	throw SonataError(fmt::format("expected integer, got float {}", v));	×
382	}
383	return static_cast<int64_t>(v);	78✔
384	}
385
386	uint64_t get_uint64_or_throw(const json& el) {	6✔
387	auto v = el.get<double>();	6✔
388	if (v < 0) {	6✔
389	throw SonataError(fmt::format("expected unsigned integer, got {}", v));	4✔
390	}
391
392	if (std::floor(v) != v) {	4✔
393	throw SonataError(fmt::format("expected integer, got float {}", v));	4✔
394	}
395	return static_cast<uint64_t>(v);	2✔
396	}
397
398	NodeSetRulePtr _dispatch_node(const std::string& attribute, const json& value) {	152✔
399	if (value.is_number()) {	152✔
400	if (attribute == "population") {	14✔
401	throw SonataError("'population' must be a string");	2✔
402	}
403
404	if (attribute == "node_id") {	12✔
405	Selection::Values node_ids{get_uint64_or_throw(value)};	6✔
406	return std::make_unique<NodeSetBasicNodeIds>(std::move(node_ids));	2✔
407	} else {
408	std::vector<int64_t> f = {get_int64_or_throw(value)};	6✔
409	return std::make_unique<NodeSetBasicRule<int64_t>>(attribute, f);	6✔
410	}
411	} else if (value.is_string()) {	138✔
412	if (attribute == "node_id") {	28✔
413	throw SonataError("'node_id' must be numeric or a list of numbers");	2✔
414	}
415
416	if (attribute == "population") {	26✔
417	std::vector<std::string> v{value.get<std::string>()};	30✔
418	return std::make_unique<NodeSetBasicPopulation>(v);	10✔
419	} else {
420	std::vector<std::string> f = {value.get<std::string>()};	48✔
421	return std::make_unique<NodeSetBasicRule<std::string>>(attribute, f);	16✔
422	}
423	} else if (value.is_array()) {	110✔
424	const auto& array = value;	50✔
425
426	if (array.empty()) {	50✔
427	throw SonataError(fmt::format("NodeSet Array is empty for attribute: {}", attribute));	×
428	}
429
430	if (array[0].is_number()) {	50✔
431	if (attribute == "population") {	24✔
432	throw SonataError("'population' must be a string");	2✔
433	}
434
435	std::vector<int64_t> values;	44✔
436	for (auto& inner_el : array.items()) {	94✔
437	values.emplace_back(get_int64_or_throw(inner_el.value()));	72✔
438	}
439
440	if (attribute == "node_id") {	22✔
441	Selection::Values node_ids;	20✔
442	std::transform(begin(values),
443	end(values),
444	back_inserter(node_ids),
445	[](int64_t integer) {	56✔
446	if (integer < 0) {	56✔
447	throw SonataError("'node_id' must be positive");	2✔
448	}
449	return static_cast<Selection::Value>(integer);	54✔
450	});	18✔
451	return std::make_unique<NodeSetBasicNodeIds>(std::move(node_ids));	16✔
452	} else {
453	return std::make_unique<NodeSetBasicRule<int64_t>>(attribute, values);	4✔
454	}
455	} else if (array[0].is_string()) {	26✔
456	if (attribute == "node_id") {	24✔
457	throw SonataError("'node_id' must be numeric or a list of numbers");	2✔
458	}
459
460	std::vector<std::string> values;	44✔
461	for (auto& inner_el : array.items()) {	60✔
462	values.emplace_back(inner_el.value().get<std::string>());	38✔
463	}
464
465	if (attribute == "population") {	22✔
466	return std::make_unique<NodeSetBasicPopulation>(values);	4✔
467	} else {
468	return std::make_unique<NodeSetBasicRule<std::string>>(attribute, values);	18✔
469	}
470	} else {
471	throw SonataError("Unknown array type");	2✔
472	}
473	} else if (value.is_object()) {	60✔
474	const auto& definition = value;	60✔
475	if (definition.size() != 1) {	60✔
476	throw SonataError(
477	fmt::format("Operator '{}' must have object with one key value pair", attribute));	4✔
478	}
479	const auto& key = definition.begin().key();	58✔
480	const auto& value = definition.begin().value();	58✔
481
482	if (value.is_number()) {	58✔
483	return std::make_unique<NodeSetBasicOperatorNumeric>(attribute,	40✔
484	key,
485	value.get<double>());	82✔
486	} else if (value.is_string()) {	16✔
487	return std::make_unique<NodeSetBasicOperatorString>(attribute,	26✔
488	key,
489	value.get<std::string>());	42✔
490	} else {
491	throw SonataError("Unknown operator");	2✔
492	}
493	} else {
494	LIBSONATA_THROW_IF_REACHED // LCOV_EXCL_LINE
495	}
496	}
497
498	void parse_basic(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets) {	78✔
499	for (const auto& el : j.items()) {	244✔
500	const auto& value = el.value();	142✔
501	if (value.is_object()) {	142✔
502	if (value.empty()) {	102✔
503	// ignore
504	} else if (value.size() == 1) {	102✔
505	const auto& inner_el = value.items().begin();	100✔
506	node_sets[el.key()] = _dispatch_node(inner_el.key(), inner_el.value());	76✔
507	} else {
508	std::vector<NodeSetRulePtr> clauses;	26✔
509	for (const auto& inner_el : value.items()) {	102✔
510	clauses.push_back(_dispatch_node(inner_el.key(), inner_el.value()));	76✔
511	}
512	node_sets[el.key()] = std::make_unique<NodeSetBasicMultiClause>(std::move(clauses));	26✔
513	}
514	} else if (value.is_array()) {	40✔
515	// will be parsed by the parse_compound
516	} else {
517	// null/boolean/number/string ?
518	throw SonataError(fmt::format("Expected an array or an object, got: {}", value.dump()));	×
519	}
520	}
521	}	54✔
522
523	void check_compound(const std::map<std::string, NodeSetRulePtr>& node_sets,	102✔
524	const std::map<std::string, CompoundTargets>& compound_rules,
525	const std::string& name,
526	size_t depth) {
527	if (node_sets.count(name) > 0) {	102✔
528	return;	42✔
529	}
530
531	if (depth > MAX_COMPOUND_RECURSION) {	60✔
532	throw SonataError("Compound node_set recursion depth exceeded");	2✔
533	}
534
535	const auto it = compound_rules.find(name);	58✔
536	assert(it != compound_rules.end());	58✔
537
538	for (auto const& target : it->second) {	100✔
539	if (node_sets.count(target) == 0 && compound_rules.count(target) == 0) {	70✔
540	throw SonataError(fmt::format("Missing '{}' from node_sets", target));	8✔
541	}
542	check_compound(node_sets, compound_rules, target, depth + 1);	66✔
543	}
544	}
545
546	void parse_compound(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets) {	54✔
547	std::map<std::string, CompoundTargets> compound_rules;	108✔
548	for (auto& el : j.items()) {	174✔
549	if (el.value().is_array()) {	118✔
550	CompoundTargets targets;	80✔
551	for (const auto& name : el.value()) {	92✔
552	if (!name.is_string()) {	54✔
553	throw SonataError("All compound elements must be strings");	2✔
554	}
555
556	targets.emplace_back(name);	52✔
557	}
558	compound_rules[el.key()] = targets;	38✔
559	}
560	}
561
562
563	for (const auto& rule : compound_rules) {	82✔
564	check_compound(node_sets, compound_rules, rule.first, 0);	36✔
565
566	NodeSetRulePtr rules = std::make_unique<NodeSetCompoundRule>(rule.first, rule.second);	60✔
567	node_sets.emplace(rule.first, std::move(rules));	30✔
568	}
569	}	46✔
570
571	Selection NodeSets::materialize(const std::string& name, const NodePopulation& population) const {	154✔
572	const auto& node_set = node_sets_.find(name);	154✔
573	if (node_set == node_sets_.end()) {	154✔
574	throw SonataError(fmt::format("Unknown node_set {}", name));	4✔
575	}
576	const auto& ns = node_set->second;	152✔
577	if (!ns->is_compound()) {	152✔
578	return population.selectAll() & ns->materialize(*this, population);	244✔
579	}
580
581	// it's common to have a deep structure of compound statements
582	// (ie: a whole hierarchy of regions), all checking the same attribute
583	// rather than `materializing` them separately, we group them, and materialize
584	// them all at once
585	Selection ret{{}};	60✔
586
587	std::vector<NodeSetRule*> queue{ns.get()};	60✔
588	std::map<std::string, std::set<std::string>> attribute2rule_strings;	60✔
589	std::map<std::string, std::set<int64_t>> attribute2rule_int64;	60✔
590	while (!queue.empty()) {	120✔
591	const auto* ns = queue.back();	90✔
592	queue.pop_back();	90✔
593
594	if (ns->is_compound()) {	90✔
595	const auto* targets = dynamic_cast<const NodeSetCompoundRule*>(ns);	90✔
596	for (const auto& target : targets->getTargets()) {	220✔
597	const auto& node_set = node_sets_.find(target)->second;	130✔
598	if (node_set->is_compound()) {	130✔
599	queue.push_back(node_set.get());	60✔
600	continue;	60✔
601	}
602
603	{
604	const auto* basic_int = dynamic_cast<const NodeSetBasicRule<int64_t>*>(	70✔
605	node_set.get());	140✔
606	if (basic_int != nullptr) {	70✔
607	basic_int->add_attribute2rule(attribute2rule_int64);	10✔
608	continue;	10✔
609	}
610	}
611
612	{
613	const auto* basic_string = dynamic_cast<const NodeSetBasicRule<std::string>*>(	60✔
614	node_set.get());	120✔
615	if (basic_string != nullptr) {	60✔
616	basic_string->add_attribute2rule(attribute2rule_strings);	10✔
617	continue;	10✔
618	}
619	}
620
621	ret = ret \| ns->materialize(*this, population);	50✔
622	}
623	} else {
624	ret = ret \| ns->materialize(*this, population);	×
625	}
626	}
627
628	for (const auto& it : attribute2rule_strings) {	40✔
629	std::vector<std::string> values(it.second.begin(), it.second.end());	10✔
630	ret = ret \| population.matchAttributeValues(it.first, values);	10✔
631	}
632
633	for (const auto& it : attribute2rule_int64) {	40✔
634	std::vector<int64_t> values(it.second.begin(), it.second.end());	10✔
635	ret = ret \| population.matchAttributeValues(it.first, values);	10✔
636	}
637
638	return ret;	30✔
639	}
640	} // namespace detail
641
642	NodeSets::NodeSets(const std::string& content)	80✔
643	: impl_(new detail::NodeSets(content)) {}	80✔
644
645	NodeSets::NodeSets(std::unique_ptr<detail::NodeSets>&& impl)	2✔
646	: impl_(std::move(impl)) {}	2✔
647
648	NodeSets::NodeSets(NodeSets&&) noexcept = default;
649	NodeSets& NodeSets::operator=(NodeSets&&) noexcept = default;
650	NodeSets::~NodeSets() = default;
651
652	NodeSets NodeSets::fromFile(const std::string& path) {	2✔
653	return NodeSets(detail::NodeSets::fromFile(path));	4✔
654	}
655
656	Selection NodeSets::materialize(const std::string& name, const NodePopulation& population) const {	54✔
657	return impl_->materialize(name, population);	54✔
658	}
659
660	std::set<std::string> NodeSets::names() const {	2✔
661	return impl_->names();	2✔
662	}
663
664	std::string NodeSets::toJSON() const {	10✔
665	return impl_->toJSON();	10✔
666	}
667
668	} // namespace sonata
669	} // namespace bbp

BlueBrain / libsonata / 4741684009

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous