5131875325

pending completion

Build # 5131875325

Build Type

push

github

Committed by

mgeplf

Commit Message

Allow empty arrays in nodesets

Run Details

5 of 5 new or added lines in 1 file covered. (100.0%)

1747 of 1793 relevant lines covered (97.43%)

76.28 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.1

/src/node_sets.cpp

#include <algorithm>  // std::find, std::transform
#include <cassert>
#include <cmath>
#include <fmt/format.h>
#include <fstream>
#include <sstream>
#include <type_traits>

#include "../extlib/filesystem.hpp"

#include <nlohmann/json.hpp>
#include <utility>

#include "utils.h"  // readFile

#include <bbp/sonata/node_sets.h>

namespace bbp {
namespace sonata {

namespace fs = ghc::filesystem;

namespace detail {

const size_t MAX_COMPOUND_RECURSION = 10;

using json = nlohmann::json;

void replace_trailing_coma(std::string& s, char c) {
    s.pop_back();
    s.pop_back();
    s.push_back(c);
}

template <typename T>
std::string toString(const std::string& key, const std::vector<T>& values) {
    return fmt::format(R"("{}": [{}])", key, fmt::join(values, ", "));
}

template <>
std::string toString(const std::string& key, const std::vector<std::string>& values) {
    // strings need to be wrapped in quotes
    return fmt::format(R"("{}": ["{}"])", key, fmt::join(values, "\", \""));
}

class NodeSets;

class NodeSetRule
{
  public:
    virtual ~NodeSetRule() = default;

    virtual Selection materialize(const NodeSets&, const NodePopulation&) const = 0;
    virtual std::string toJSON() const = 0;
    virtual bool is_compound() const {
        return false;
    }
};

using NodeSetRulePtr = std::unique_ptr<NodeSetRule>;
void parse_basic(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets);
void parse_compound(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets);

class NodeSets
{
    std::map<std::string, NodeSetRulePtr> node_sets_;

  public:
    explicit NodeSets(const json& j) {
        if (!j.is_object()) {
            throw SonataError("Top level node_set must be an object");
        }

        // Need to two pass parsing the json so that compound lookup can rely
        // on all the basic rules existing
        parse_basic(j, node_sets_);
        parse_compound(j, node_sets_);
    }

    static const fs::path& validate_path(const fs::path& path) {
        if (!fs::exists(path)) {
            throw SonataError(fmt::format("Path does not exist: {}", std::string(path)));
        }
        return path;
    }

    explicit NodeSets(const fs::path& path)
        : NodeSets(json::parse(std::ifstream(validate_path(path)))) {}

    static std::unique_ptr<NodeSets> fromFile(const std::string& path_) {
        fs::path path(path_);
        return std::make_unique<detail::NodeSets>(path);
    }

    explicit NodeSets(const std::string& content)
        : NodeSets(json::parse(content)) {}

    Selection materialize(const std::string& name, const NodePopulation& population) const;

    std::set<std::string> names() const {
        return getMapKeys(node_sets_);
    }

    std::string toJSON() const {
        std::string ret{"{\n"};
        for (const auto& pair : node_sets_) {
            ret += fmt::format(R"(  "{}": {{)", pair.first);
            ret += pair.second->toJSON();
            ret += "},\n";
        }
        replace_trailing_coma(ret, '\n');
        ret += "}";

        return ret;
    }
};

class NodeSetNullRule: public NodeSetRule
{
    Selection materialize(const detail::NodeSets& /* unused */,
                          const NodePopulation& /* unused */) const final {
        return Selection({});
    }

    std::string toJSON() const final {
        return {};
    }
};

// { 'region': ['region1', 'region2', ...] }
template <typename T>
class NodeSetBasicRule: public NodeSetRule
{
  public:
    NodeSetBasicRule(std::string attribute, std::vector<T>& values)
        : attribute_(std::move(attribute))
        , values_(values) {}

    Selection materialize(const detail::NodeSets& /* unused */,
                          const NodePopulation& np) const final {
        return np.matchAttributeValues(attribute_, values_);
    }

    void add_attribute2rule(std::map<std::string, std::set<T>>& attribute2rule) const {
        auto& s = attribute2rule[attribute_];
        for (const auto& v : values_) {
            s.insert(v);
        }
    }

    std::string toJSON() const final {
        return toString(attribute_, values_);
    }

  private:
    std::string attribute_;
    std::vector<T> values_;
};

// { 'population': ['popA', 'popB', ] }
class NodeSetBasicPopulation: public NodeSetRule
{
  public:
    explicit NodeSetBasicPopulation(std::vector<std::string>& values)
        : values_(values) {}

    Selection materialize(const detail::NodeSets& /* unused */,
                          const NodePopulation& np) const final {
        if (std::find(values_.begin(), values_.end(), np.name()) != values_.end()) {
            return np.selectAll();
        }

        return Selection{{}};
    }

    std::string toJSON() const final {
        return toString("population", values_);
    }

  private:
    std::vector<std::string> values_;
};

// { 'node_id': [1, 2, 3, 4] }
class NodeSetBasicNodeIds: public NodeSetRule
{
  public:
    explicit NodeSetBasicNodeIds(Selection::Values&& values)
        : values_(values) {}

    Selection materialize(const detail::NodeSets& /* unused */,
                          const NodePopulation& np) const final {
        return np.selectAll() & Selection::fromValues(values_.begin(), values_.end());
    }

    std::string toJSON() const final {
        return toString("node_ids", values_);
    }

  private:
    Selection::Values values_;
};

//  {
//      "population": "biophysical",
//      "model_type": "point",
//      "node_id": [1, 2, 3, 5, 7, 9, ...]
//  }
class NodeSetBasicMultiClause: public NodeSetRule
{
  public:
    explicit NodeSetBasicMultiClause(std::vector<NodeSetRulePtr>&& clauses)
        : clauses_(std::move(clauses)) {}

    Selection materialize(const detail::NodeSets& ns, const NodePopulation& np) const final {
        Selection ret = np.selectAll();
        for (const auto& clause : clauses_) {
            ret = ret & clause->materialize(ns, np);
        }
        return ret;
    }

    std::string toJSON() const final {
        std::string ret;
        for (const auto& clause : clauses_) {
            ret += clause->toJSON();
            ret += ", ";
        }
        replace_trailing_coma(ret, ' ');
        return ret;
    }

  private:
    std::vector<NodeSetRulePtr> clauses_;
};

// "string_attr": { "$regex": "^[s][o]me value$" }
class NodeSetBasicOperatorString: public NodeSetRule
{
  public:
    explicit NodeSetBasicOperatorString(std::string attribute,
                                        const std::string& op,
                                        std::string value)
        : op_(string2op(op))
        , attribute_(std::move(attribute))
        , value_(std::move(value)) {}

    Selection materialize(const detail::NodeSets& /* unused */,
                          const NodePopulation& np) const final {
        switch (op_) {
        case Op::regex:
            return np.regexMatch(attribute_, value_);
        default:              // LCOV_EXCL_LINE
            LIBSONATA_THROW_IF_REACHED  // LCOV_EXCL_LINE
        }
    }

    std::string toJSON() const final {
        return fmt::format(R"("{}": {{ "{}": "{}" }})", attribute_, op2string(op_), value_);
    }

    enum class Op {
        regex = 1,
    };

    static Op string2op(const std::string& s) {
        if (s == "$regex") {
            return Op::regex;
        }
        throw SonataError(fmt::format("Operator '{}' not available for strings", s));
    }

    static std::string op2string(const Op op) {
        switch (op) {
        case Op::regex:
            return "$regex";
        default:              // LCOV_EXCL_LINE
            LIBSONATA_THROW_IF_REACHED  // LCOV_EXCL_LINE
        }
    }

  private:
    Op op_;
    std::string attribute_;
    std::string value_;
};

// "numeric_attribute_gt": { "$gt": 3 },
class NodeSetBasicOperatorNumeric: public NodeSetRule
{
  public:
    explicit NodeSetBasicOperatorNumeric(std::string name, const std::string& op, double value)
        : name_(std::move(name))
        , value_(value)
        , op_(string2op(op)) {}

    Selection materialize(const detail::NodeSets& /* unused */,
                          const NodePopulation& np) const final {
        switch (op_) {
        case Op::gt:
            return np.filterAttribute<double>(name_, [=](const double v) { return v > value_; });
        case Op::lt:
            return np.filterAttribute<double>(name_, [=](const double v) { return v < value_; });
        case Op::gte:
            return np.filterAttribute<double>(name_, [=](const double v) { return v >= value_; });
        case Op::lte:
            return np.filterAttribute<double>(name_, [=](const double v) { return v <= value_; });
        default:              // LCOV_EXCL_LINE
            LIBSONATA_THROW_IF_REACHED  // LCOV_EXCL_LINE
        }
    }

    std::string toJSON() const final {
        return fmt::format(R"("{}": {{ "{}": {} }})", name_, op2string(op_), value_);
    }

    enum class Op {
        gt = 1,
        lt = 2,
        gte = 3,
        lte = 4,
    };

    static Op string2op(const std::string& s) {
        if (s == "$gt") {
            return Op::gt;
        } else if (s == "$lt") {
            return Op::lt;
        } else if (s == "$gte") {
            return Op::gte;
        } else if (s == "$lte") {
            return Op::lte;
        }
        throw SonataError(fmt::format("Operator '{}' not available for numeric", s));
    }

    static std::string op2string(const Op op) {
        switch (op) {
        case Op::gt:
            return "$gt";
        case Op::lt:
            return "$lt";
        case Op::gte:
            return "$gte";
        case Op::lte:
            return "$lte";
        default:              // LCOV_EXCL_LINE
            LIBSONATA_THROW_IF_REACHED  // LCOV_EXCL_LINE
        }
    }

  private:
    std::string name_;
    double value_;
    Op op_;
};

using CompoundTargets = std::vector<std::string>;
class NodeSetCompoundRule: public NodeSetRule
{
  public:
    NodeSetCompoundRule(std::string name, CompoundTargets targets)
        : name_(std::move(name))
        , targets_(std::move(targets)) {}

    Selection materialize(const detail::NodeSets& ns, const NodePopulation& np) const final {
        Selection ret{{}};
        for (const auto& target : targets_) {
            ret = ret | ns.materialize(target, np);
        }
        return ret;
    }

    std::string toJSON() const final {
        return toString("node_ids", targets_);
    }

    bool is_compound() const override {
        return true;
    }
    const CompoundTargets& getTargets() const {
        return targets_;
    }

  private:
    std::string name_;
    CompoundTargets targets_;
};

int64_t get_int64_or_throw(const json& el) {
    auto v = el.get<double>();
    if (std::floor(v) != v) {
        throw SonataError(fmt::format("expected integer, got float {}", v));
    }
    return static_cast<int64_t>(v);
}

uint64_t get_uint64_or_throw(const json& el) {
    auto v = el.get<double>();
    if (v < 0) {
        throw SonataError(fmt::format("expected unsigned integer, got {}", v));
    }

    if (std::floor(v) != v) {
        throw SonataError(fmt::format("expected integer, got float {}", v));
    }
    return static_cast<uint64_t>(v);
}

NodeSetRulePtr _dispatch_node(const std::string& attribute, const json& value) {
    if (value.is_number()) {
        if (attribute == "population") {
            throw SonataError("'population' must be a string");
        }

        if (attribute == "node_id") {
            Selection::Values node_ids{get_uint64_or_throw(value)};
            return std::make_unique<NodeSetBasicNodeIds>(std::move(node_ids));
        } else {
            std::vector<int64_t> f = {get_int64_or_throw(value)};
            return std::make_unique<NodeSetBasicRule<int64_t>>(attribute, f);
        }
    } else if (value.is_string()) {
        if (attribute == "node_id") {
            throw SonataError("'node_id' must be numeric or a list of numbers");
        }

        if (attribute == "population") {
            std::vector<std::string> v{value.get<std::string>()};
            return std::make_unique<NodeSetBasicPopulation>(v);
        } else {
            std::vector<std::string> f = {value.get<std::string>()};
            return std::make_unique<NodeSetBasicRule<std::string>>(attribute, f);
        }
    } else if (value.is_array()) {
        const auto& array = value;

        if (array.empty()) {
            return std::make_unique<NodeSetNullRule>();
        }

        if (array[0].is_number()) {
            if (attribute == "population") {
                throw SonataError("'population' must be a string");
            }

            std::vector<int64_t> values;
            for (auto& inner_el : array.items()) {
                values.emplace_back(get_int64_or_throw(inner_el.value()));
            }

            if (attribute == "node_id") {
                Selection::Values node_ids;
                std::transform(begin(values),
                               end(values),
                               back_inserter(node_ids),
                               [](int64_t integer) {
                                   if (integer < 0) {
                                       throw SonataError("'node_id' must be positive");
                                   }
                                   return static_cast<Selection::Value>(integer);
                               });
                return std::make_unique<NodeSetBasicNodeIds>(std::move(node_ids));
            } else {
                return std::make_unique<NodeSetBasicRule<int64_t>>(attribute, values);
            }
        } else if (array[0].is_string()) {
            if (attribute == "node_id") {
                throw SonataError("'node_id' must be numeric or a list of numbers");
            }

            std::vector<std::string> values;
            for (auto& inner_el : array.items()) {
                values.emplace_back(inner_el.value().get<std::string>());
            }

            if (attribute == "population") {
                return std::make_unique<NodeSetBasicPopulation>(values);
            } else {
                return std::make_unique<NodeSetBasicRule<std::string>>(attribute, values);
            }
        } else {
            throw SonataError("Unknown array type");
        }
    } else if (value.is_object()) {
        const auto& definition = value;
        if (definition.size() != 1) {
            throw SonataError(
                fmt::format("Operator '{}' must have object with one key value pair", attribute));
        }
        const auto& key = definition.begin().key();
        const auto& value = definition.begin().value();

        if (value.is_number()) {
            return std::make_unique<NodeSetBasicOperatorNumeric>(attribute,
                                                                 key,
                                                                 value.get<double>());
        } else if (value.is_string()) {
            return std::make_unique<NodeSetBasicOperatorString>(attribute,
                                                                key,
                                                                value.get<std::string>());
        } else {
            throw SonataError("Unknown operator");
        }
    } else {
        LIBSONATA_THROW_IF_REACHED  // LCOV_EXCL_LINE
    }
}

void parse_basic(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets) {
    for (const auto& el : j.items()) {
        const auto& value = el.value();
        if (value.is_object()) {
            if (value.empty()) {
                // ignore
            } else if (value.size() == 1) {
                const auto& inner_el = value.items().begin();
                node_sets[el.key()] = _dispatch_node(inner_el.key(), inner_el.value());
            } else {
                std::vector<NodeSetRulePtr> clauses;
                for (const auto& inner_el : value.items()) {
                    clauses.push_back(_dispatch_node(inner_el.key(), inner_el.value()));
                }
                node_sets[el.key()] = std::make_unique<NodeSetBasicMultiClause>(std::move(clauses));
            }
        } else if (value.is_array()) {
            // will be parsed by the parse_compound
        } else {
            // null/boolean/number/string ?
            throw SonataError(fmt::format("Expected an array or an object, got: {}", value.dump()));
        }
    }
}

void check_compound(const std::map<std::string, NodeSetRulePtr>& node_sets,
                    const std::map<std::string, CompoundTargets>& compound_rules,
                    const std::string& name,
                    size_t depth) {
    if (node_sets.count(name) > 0) {
        return;
    }

    if (depth > MAX_COMPOUND_RECURSION) {
        throw SonataError("Compound node_set recursion depth exceeded");
    }

    const auto it = compound_rules.find(name);
    assert(it != compound_rules.end());

    for (auto const& target : it->second) {
        if (node_sets.count(target) == 0 && compound_rules.count(target) == 0) {
            throw SonataError(fmt::format("Missing '{}' from node_sets", target));
        }
        check_compound(node_sets, compound_rules, target, depth + 1);
    }
}

void parse_compound(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets) {
    std::map<std::string, CompoundTargets> compound_rules;
    for (auto& el : j.items()) {
        if (el.value().is_array()) {
            CompoundTargets targets;
            for (const auto& name : el.value()) {
                if (!name.is_string()) {
                    throw SonataError("All compound elements must be strings");
                }

                targets.emplace_back(name);
            }
            compound_rules[el.key()] = targets;
        }
    }


    for (const auto& rule : compound_rules) {
        check_compound(node_sets, compound_rules, rule.first, 0);

        NodeSetRulePtr rules = std::make_unique<NodeSetCompoundRule>(rule.first, rule.second);
        node_sets.emplace(rule.first, std::move(rules));
    }
}

Selection NodeSets::materialize(const std::string& name, const NodePopulation& population) const {
    const auto& node_set = node_sets_.find(name);
    if (node_set == node_sets_.end()) {
        throw SonataError(fmt::format("Unknown node_set {}", name));
    }
    const auto& ns = node_set->second;
    if (!ns->is_compound()) {
        return population.selectAll() & ns->materialize(*this, population);
    }

    // it's common to have a deep structure of compound statements
    // (ie: a whole hierarchy of regions), all checking the same attribute
    // rather than `materializing` them separately, we group them, and materialize
    // them all at once
    Selection ret{{}};

    std::vector<NodeSetRule*> queue{ns.get()};
    std::map<std::string, std::set<std::string>> attribute2rule_strings;
    std::map<std::string, std::set<int64_t>> attribute2rule_int64;
    while (!queue.empty()) {
        const auto* ns = queue.back();
        queue.pop_back();

        if (ns->is_compound()) {
            const auto* targets = dynamic_cast<const NodeSetCompoundRule*>(ns);
            for (const auto& target : targets->getTargets()) {
                const auto& node_set = node_sets_.find(target)->second;
                if (node_set->is_compound()) {
                    queue.push_back(node_set.get());
                    continue;
                }

                {
                    const auto* basic_int = dynamic_cast<const NodeSetBasicRule<int64_t>*>(
                        node_set.get());
                    if (basic_int != nullptr) {
                        basic_int->add_attribute2rule(attribute2rule_int64);
                        continue;
                    }
                }

                {
                    const auto* basic_string = dynamic_cast<const NodeSetBasicRule<std::string>*>(
                        node_set.get());
                    if (basic_string != nullptr) {
                        basic_string->add_attribute2rule(attribute2rule_strings);
                        continue;
                    }
                }

                ret = ret | ns->materialize(*this, population);
            }
        } else {
            ret = ret | ns->materialize(*this, population);
        }
    }

    for (const auto& it : attribute2rule_strings) {
        std::vector<std::string> values(it.second.begin(), it.second.end());
        ret = ret | population.matchAttributeValues(it.first, values);
    }

    for (const auto& it : attribute2rule_int64) {
        std::vector<int64_t> values(it.second.begin(), it.second.end());
        ret = ret | population.matchAttributeValues(it.first, values);
    }

    return ret;
}
}  // namespace detail

NodeSets::NodeSets(const std::string& content)
    : impl_(new detail::NodeSets(content)) {}

NodeSets::NodeSets(std::unique_ptr<detail::NodeSets>&& impl)
    : impl_(std::move(impl)) {}

NodeSets::NodeSets(NodeSets&&) noexcept = default;
NodeSets& NodeSets::operator=(NodeSets&&) noexcept = default;
NodeSets::~NodeSets() = default;

NodeSets NodeSets::fromFile(const std::string& path) {
    return NodeSets(detail::NodeSets::fromFile(path));
}

Selection NodeSets::materialize(const std::string& name, const NodePopulation& population) const {
    return impl_->materialize(name, population);
}

std::set<std::string> NodeSets::names() const {
    return impl_->names();
}

std::string NodeSets::toJSON() const {
    return impl_->toJSON();
}

}  // namespace sonata
}  // namespace bbp

1	#include <algorithm> // std::find, std::transform
2	#include <cassert>
3	#include <cmath>
4	#include <fmt/format.h>
5	#include <fstream>
6	#include <sstream>
7	#include <type_traits>
8
9	#include "../extlib/filesystem.hpp"
10
11	#include <nlohmann/json.hpp>
12	#include <utility>
13
14	#include "utils.h" // readFile
15
16	#include <bbp/sonata/node_sets.h>
17
18	namespace bbp {
19	namespace sonata {
20
21	namespace fs = ghc::filesystem;
22
23	namespace detail {
24
25	const size_t MAX_COMPOUND_RECURSION = 10;
26
27	using json = nlohmann::json;
28
29	void replace_trailing_coma(std::string& s, char c) {	40✔
30	s.pop_back();	40✔
31	s.pop_back();	40✔
32	s.push_back(c);	40✔
33	}	40✔
34
35	template <typename T>
36	std::string toString(const std::string& key, const std::vector<T>& values) {	10✔
37	return fmt::format(R"("{}": [{}])", key, fmt::join(values, ", "));	20✔
38	}
39
40	template <>
41	std::string toString(const std::string& key, const std::vector<std::string>& values) {	50✔
42	// strings need to be wrapped in quotes
43	return fmt::format(R"("{}": ["{}"])", key, fmt::join(values, "\", \""));	100✔
44	}
45
46	class NodeSets;
47
48	class NodeSetRule
49	{
50	public:
51	virtual ~NodeSetRule() = default;	194✔
52
53	virtual Selection materialize(const NodeSets&, const NodePopulation&) const = 0;
54	virtual std::string toJSON() const = 0;
55	virtual bool is_compound() const {	194✔
56	return false;	194✔
57	}
58	};
59
60	using NodeSetRulePtr = std::unique_ptr<NodeSetRule>;
61	void parse_basic(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets);
62	void parse_compound(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets);
63
64	class NodeSets
65	{
66	std::map<std::string, NodeSetRulePtr> node_sets_;
67
68	public:
69	explicit NodeSets(const json& j) {	122✔
70	if (!j.is_object()) {	86✔
71	throw SonataError("Top level node_set must be an object");	4✔
72	}
73
74	// Need to two pass parsing the json so that compound lookup can rely
75	// on all the basic rules existing
76	parse_basic(j, node_sets_);	82✔
77	parse_compound(j, node_sets_);	58✔
78	}	50✔
79
80	static const fs::path& validate_path(const fs::path& path) {	2✔
81	if (!fs::exists(path)) {	2✔
82	throw SonataError(fmt::format("Path does not exist: {}", std::string(path)));	×
83	}
84	return path;	2✔
85	}
86
87	explicit NodeSets(const fs::path& path)	2✔
88	: NodeSets(json::parse(std::ifstream(validate_path(path)))) {}	2✔
89
90	static std::unique_ptr<NodeSets> fromFile(const std::string& path_) {	2✔
91	fs::path path(path_);	4✔
92	return std::make_unique<detail::NodeSets>(path);	4✔
93	}
94
95	explicit NodeSets(const std::string& content)	84✔
96	: NodeSets(json::parse(content)) {}	120✔
97
98	Selection materialize(const std::string& name, const NodePopulation& population) const;
99
100	std::set<std::string> names() const {	2✔
101	return getMapKeys(node_sets_);	2✔
102	}
103
104	std::string toJSON() const {	10✔
105	std::string ret{"{\n"};	10✔
106	for (const auto& pair : node_sets_) {	60✔
107	ret += fmt::format(R"( "{}": {{)", pair.first);	100✔
108	ret += pair.second->toJSON();	50✔
109	ret += "},\n";	50✔
110	}
111	replace_trailing_coma(ret, '\n');	10✔
112	ret += "}";	10✔
113
114	return ret;	10✔
115	}
116	};
117
118	class NodeSetNullRule: public NodeSetRule
119	{
120	Selection materialize(const detail::NodeSets& /* unused */,	2✔
121	const NodePopulation& /* unused */) const final {
122	return Selection({});	2✔
123	}
124
125	std::string toJSON() const final {	×
126	return {};	×
127	}
128	};
129
130	// { 'region': ['region1', 'region2', ...] }
131	template <typename T>
132	class NodeSetBasicRule: public NodeSetRule
133	{
134	public:
135	NodeSetBasicRule(std::string attribute, std::vector<T>& values)	44✔
136	: attribute_(std::move(attribute))	44✔
137	, values_(values) {}	44✔
138
139	Selection materialize(const detail::NodeSets& /* unused */,	10✔
140	const NodePopulation& np) const final {
141	return np.matchAttributeValues(attribute_, values_);	10✔
142	}
143
144	void add_attribute2rule(std::map<std::string, std::set<T>>& attribute2rule) const {	20✔
145	auto& s = attribute2rule[attribute_];	20✔
146	for (const auto& v : values_) {	50✔
147	s.insert(v);	30✔
148	}
149	}	20✔
150
151	std::string toJSON() const final {	38✔
152	return toString(attribute_, values_);	38✔
153	}
154
155	private:
156	std::string attribute_;
157	std::vector<T> values_;
158	};
159
160	// { 'population': ['popA', 'popB', ] }
161	class NodeSetBasicPopulation: public NodeSetRule
162	{
163	public:
164	explicit NodeSetBasicPopulation(std::vector<std::string>& values)	14✔
165	: values_(values) {}	14✔
166
167	Selection materialize(const detail::NodeSets& /* unused */,	6✔
168	const NodePopulation& np) const final {
169	if (std::find(values_.begin(), values_.end(), np.name()) != values_.end()) {	6✔
170	return np.selectAll();	4✔
171	}
172
173	return Selection{{}};	2✔
174	}
175
176	std::string toJSON() const final {	10✔
177	return toString("population", values_);	10✔
178	}
179
180	private:
181	std::vector<std::string> values_;
182	};
183
184	// { 'node_id': [1, 2, 3, 4] }
185	class NodeSetBasicNodeIds: public NodeSetRule
186	{
187	public:
188	explicit NodeSetBasicNodeIds(Selection::Values&& values)	18✔
189	: values_(values) {}	18✔
190
191	Selection materialize(const detail::NodeSets& /* unused */,	96✔
192	const NodePopulation& np) const final {
193	return np.selectAll() & Selection::fromValues(values_.begin(), values_.end());	192✔
194	}
195
196	std::string toJSON() const final {	6✔
197	return toString("node_ids", values_);	6✔
198	}
199
200	private:
201	Selection::Values values_;
202	};
203
204	// {
205	// "population": "biophysical",
206	// "model_type": "point",
207	// "node_id": [1, 2, 3, 5, 7, 9, ...]
208	// }
209	class NodeSetBasicMultiClause: public NodeSetRule
210	{
211	public:
212	explicit NodeSetBasicMultiClause(std::vector<NodeSetRulePtr>&& clauses)	26✔
213	: clauses_(std::move(clauses)) {}	26✔
214
215	Selection materialize(const detail::NodeSets& ns, const NodePopulation& np) const final {	2✔
216	Selection ret = np.selectAll();	2✔
217	for (const auto& clause : clauses_) {	6✔
218	ret = ret & clause->materialize(ns, np);	4✔
219	}
220	return ret;	2✔
221	}
222
223	std::string toJSON() const final {	30✔
224	std::string ret;	30✔
225	for (const auto& clause : clauses_) {	120✔
226	ret += clause->toJSON();	90✔
227	ret += ", ";	90✔
228	}
229	replace_trailing_coma(ret, ' ');	30✔
230	return ret;	30✔
231	}
232
233	private:
234	std::vector<NodeSetRulePtr> clauses_;
235	};
236
237	// "string_attr": { "$regex": "^[s][o]me value$" }
238	class NodeSetBasicOperatorString: public NodeSetRule
239	{
240	public:
241	explicit NodeSetBasicOperatorString(std::string attribute,	14✔
242	const std::string& op,
243	std::string value)
244	: op_(string2op(op))	40✔
245	, attribute_(std::move(attribute))	12✔
246	, value_(std::move(value)) {}	14✔
247
248	Selection materialize(const detail::NodeSets& /* unused */,	4✔
249	const NodePopulation& np) const final {
250	switch (op_) {	4✔
251	case Op::regex:	4✔
252	return np.regexMatch(attribute_, value_);	4✔
253	default: // LCOV_EXCL_LINE
254	LIBSONATA_THROW_IF_REACHED // LCOV_EXCL_LINE
255	}
256	}
257
258	std::string toJSON() const final {	10✔
259	return fmt::format(R"("{}": {{ "{}": "{}" }})", attribute_, op2string(op_), value_);	20✔
260	}
261
262	enum class Op {
263	regex = 1,
264	};
265
266	static Op string2op(const std::string& s) {	14✔
267	if (s == "$regex") {	14✔
268	return Op::regex;	12✔
269	}
270	throw SonataError(fmt::format("Operator '{}' not available for strings", s));	4✔
271	}
272
273	static std::string op2string(const Op op) {	10✔
274	switch (op) {	10✔
275	case Op::regex:	10✔
276	return "$regex";	10✔
277	default: // LCOV_EXCL_LINE
278	LIBSONATA_THROW_IF_REACHED // LCOV_EXCL_LINE
279	}
280	}
281
282	private:
283	Op op_;
284	std::string attribute_;
285	std::string value_;
286	};
287
288	// "numeric_attribute_gt": { "$gt": 3 },
289	class NodeSetBasicOperatorNumeric: public NodeSetRule
290	{
291	public:
292	explicit NodeSetBasicOperatorNumeric(std::string name, const std::string& op, double value)	42✔
293	: name_(std::move(name))	84✔
294	, value_(value)
295	, op_(string2op(op)) {}	44✔
296
297	Selection materialize(const detail::NodeSets& /* unused */,	8✔
298	const NodePopulation& np) const final {
299	switch (op_) {	8✔
300	case Op::gt:	2✔
301	return np.filterAttribute<double>(name_, [=](const double v) { return v > value_; });	14✔
302	case Op::lt:	2✔
303	return np.filterAttribute<double>(name_, [=](const double v) { return v < value_; });	14✔
304	case Op::gte:	2✔
305	return np.filterAttribute<double>(name_, [=](const double v) { return v >= value_; });	14✔
306	case Op::lte:	2✔
307	return np.filterAttribute<double>(name_, [=](const double v) { return v <= value_; });	14✔
308	default: // LCOV_EXCL_LINE
309	LIBSONATA_THROW_IF_REACHED // LCOV_EXCL_LINE
310	}
311	}
312
313	std::string toJSON() const final {	40✔
314	return fmt::format(R"("{}": {{ "{}": {} }})", name_, op2string(op_), value_);	80✔
315	}
316
317	enum class Op {
318	gt = 1,
319	lt = 2,
320	gte = 3,
321	lte = 4,
322	};
323
324	static Op string2op(const std::string& s) {	42✔
325	if (s == "$gt") {	42✔
326	return Op::gt;	10✔
327	} else if (s == "$lt") {	32✔
328	return Op::lt;	10✔
329	} else if (s == "$gte") {	22✔
330	return Op::gte;	10✔
331	} else if (s == "$lte") {	12✔
332	return Op::lte;	10✔
333	}
334	throw SonataError(fmt::format("Operator '{}' not available for numeric", s));	4✔
335	}
336
337	static std::string op2string(const Op op) {	40✔
338	switch (op) {	40✔
339	case Op::gt:	10✔
340	return "$gt";	10✔
341	case Op::lt:	10✔
342	return "$lt";	10✔
343	case Op::gte:	10✔
344	return "$gte";	10✔
345	case Op::lte:	10✔
346	return "$lte";	10✔
347	default: // LCOV_EXCL_LINE
348	LIBSONATA_THROW_IF_REACHED // LCOV_EXCL_LINE
349	}
350	}
351
352	private:
353	std::string name_;
354	double value_;
355	Op op_;
356	};
357
358	using CompoundTargets = std::vector<std::string>;
359	class NodeSetCompoundRule: public NodeSetRule
360	{
361	public:
362	NodeSetCompoundRule(std::string name, CompoundTargets targets)	32✔
363	: name_(std::move(name))	64✔
364	, targets_(std::move(targets)) {}	32✔
365
366	Selection materialize(const detail::NodeSets& ns, const NodePopulation& np) const final {	50✔
367	Selection ret{{}};	50✔
368	for (const auto& target : targets_) {	150✔
369	ret = ret \| ns.materialize(target, np);	100✔
370	}
371	return ret;	50✔
372	}
373
374	std::string toJSON() const final {	6✔
375	return toString("node_ids", targets_);	6✔
376	}
377
378	bool is_compound() const override {	184✔
379	return true;	184✔
380	}
381	const CompoundTargets& getTargets() const {	92✔
382	return targets_;	92✔
383	}
384
385	private:
386	std::string name_;
387	CompoundTargets targets_;
388	};
389
390	int64_t get_int64_or_throw(const json& el) {	78✔
391	auto v = el.get<double>();	78✔
392	if (std::floor(v) != v) {	78✔
393	throw SonataError(fmt::format("expected integer, got float {}", v));	×
394	}
395	return static_cast<int64_t>(v);	78✔
396	}
397
398	uint64_t get_uint64_or_throw(const json& el) {	6✔
399	auto v = el.get<double>();	6✔
400	if (v < 0) {	6✔
401	throw SonataError(fmt::format("expected unsigned integer, got {}", v));	4✔
402	}
403
404	if (std::floor(v) != v) {	4✔
405	throw SonataError(fmt::format("expected integer, got float {}", v));	4✔
406	}
407	return static_cast<uint64_t>(v);	2✔
408	}
409
410	NodeSetRulePtr _dispatch_node(const std::string& attribute, const json& value) {	156✔
411	if (value.is_number()) {	156✔
412	if (attribute == "population") {	14✔
413	throw SonataError("'population' must be a string");	2✔
414	}
415
416	if (attribute == "node_id") {	12✔
417	Selection::Values node_ids{get_uint64_or_throw(value)};	6✔
418	return std::make_unique<NodeSetBasicNodeIds>(std::move(node_ids));	2✔
419	} else {
420	std::vector<int64_t> f = {get_int64_or_throw(value)};	6✔
421	return std::make_unique<NodeSetBasicRule<int64_t>>(attribute, f);	6✔
422	}
423	} else if (value.is_string()) {	142✔
424	if (attribute == "node_id") {	28✔
425	throw SonataError("'node_id' must be numeric or a list of numbers");	2✔
426	}
427
428	if (attribute == "population") {	26✔
429	std::vector<std::string> v{value.get<std::string>()};	30✔
430	return std::make_unique<NodeSetBasicPopulation>(v);	10✔
431	} else {
432	std::vector<std::string> f = {value.get<std::string>()};	48✔
433	return std::make_unique<NodeSetBasicRule<std::string>>(attribute, f);	16✔
434	}
435	} else if (value.is_array()) {	114✔
436	const auto& array = value;	54✔
437
438	if (array.empty()) {	54✔
439	return std::make_unique<NodeSetNullRule>();	4✔
440	}
441
442	if (array[0].is_number()) {	50✔
443	if (attribute == "population") {	24✔
444	throw SonataError("'population' must be a string");	2✔
445	}
446
447	std::vector<int64_t> values;	44✔
448	for (auto& inner_el : array.items()) {	94✔
449	values.emplace_back(get_int64_or_throw(inner_el.value()));	72✔
450	}
451
452	if (attribute == "node_id") {	22✔
453	Selection::Values node_ids;	20✔
454	std::transform(begin(values),
455	end(values),
456	back_inserter(node_ids),
457	[](int64_t integer) {	56✔
458	if (integer < 0) {	56✔
459	throw SonataError("'node_id' must be positive");	2✔
460	}
461	return static_cast<Selection::Value>(integer);	54✔
462	});	18✔
463	return std::make_unique<NodeSetBasicNodeIds>(std::move(node_ids));	16✔
464	} else {
465	return std::make_unique<NodeSetBasicRule<int64_t>>(attribute, values);	4✔
466	}
467	} else if (array[0].is_string()) {	26✔
468	if (attribute == "node_id") {	24✔
469	throw SonataError("'node_id' must be numeric or a list of numbers");	2✔
470	}
471
472	std::vector<std::string> values;	44✔
473	for (auto& inner_el : array.items()) {	60✔
474	values.emplace_back(inner_el.value().get<std::string>());	38✔
475	}
476
477	if (attribute == "population") {	22✔
478	return std::make_unique<NodeSetBasicPopulation>(values);	4✔
479	} else {
480	return std::make_unique<NodeSetBasicRule<std::string>>(attribute, values);	18✔
481	}
482	} else {
483	throw SonataError("Unknown array type");	2✔
484	}
485	} else if (value.is_object()) {	60✔
486	const auto& definition = value;	60✔
487	if (definition.size() != 1) {	60✔
488	throw SonataError(
489	fmt::format("Operator '{}' must have object with one key value pair", attribute));	4✔
490	}
491	const auto& key = definition.begin().key();	58✔
492	const auto& value = definition.begin().value();	58✔
493
494	if (value.is_number()) {	58✔
495	return std::make_unique<NodeSetBasicOperatorNumeric>(attribute,	40✔
496	key,
497	value.get<double>());	82✔
498	} else if (value.is_string()) {	16✔
499	return std::make_unique<NodeSetBasicOperatorString>(attribute,	26✔
500	key,
501	value.get<std::string>());	42✔
502	} else {
503	throw SonataError("Unknown operator");	2✔
504	}
505	} else {
506	LIBSONATA_THROW_IF_REACHED // LCOV_EXCL_LINE
507	}
508	}
509
510	void parse_basic(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets) {	82✔
511	for (const auto& el : j.items()) {	254✔
512	const auto& value = el.value();	148✔
513	if (value.is_object()) {	148✔
514	if (value.empty()) {	106✔
515	// ignore
516	} else if (value.size() == 1) {	106✔
517	const auto& inner_el = value.items().begin();	104✔
518	node_sets[el.key()] = _dispatch_node(inner_el.key(), inner_el.value());	80✔
519	} else {
520	std::vector<NodeSetRulePtr> clauses;	26✔
521	for (const auto& inner_el : value.items()) {	102✔
522	clauses.push_back(_dispatch_node(inner_el.key(), inner_el.value()));	76✔
523	}
524	node_sets[el.key()] = std::make_unique<NodeSetBasicMultiClause>(std::move(clauses));	26✔
525	}
526	} else if (value.is_array()) {	42✔
527	// will be parsed by the parse_compound
528	} else {
529	// null/boolean/number/string ?
530	throw SonataError(fmt::format("Expected an array or an object, got: {}", value.dump()));	×
531	}
532	}
533	}	58✔
534
535	void check_compound(const std::map<std::string, NodeSetRulePtr>& node_sets,	104✔
536	const std::map<std::string, CompoundTargets>& compound_rules,
537	const std::string& name,
538	size_t depth) {
539	if (node_sets.count(name) > 0) {	104✔
540	return;	42✔
541	}
542
543	if (depth > MAX_COMPOUND_RECURSION) {	62✔
544	throw SonataError("Compound node_set recursion depth exceeded");	2✔
545	}
546
547	const auto it = compound_rules.find(name);	60✔
548	assert(it != compound_rules.end());	60✔
549
550	for (auto const& target : it->second) {	102✔
551	if (node_sets.count(target) == 0 && compound_rules.count(target) == 0) {	70✔
552	throw SonataError(fmt::format("Missing '{}' from node_sets", target));	8✔
553	}
554	check_compound(node_sets, compound_rules, target, depth + 1);	66✔
555	}
556	}
557
558	void parse_compound(const json& j, std::map<std::string, NodeSetRulePtr>& node_sets) {	58✔
559	std::map<std::string, CompoundTargets> compound_rules;	116✔
560	for (auto& el : j.items()) {	184✔
561	if (el.value().is_array()) {	124✔
562	CompoundTargets targets;	84✔
563	for (const auto& name : el.value()) {	94✔
564	if (!name.is_string()) {	54✔
565	throw SonataError("All compound elements must be strings");	2✔
566	}
567
568	targets.emplace_back(name);	52✔
569	}
570	compound_rules[el.key()] = targets;	40✔
571	}
572	}
573
574
575	for (const auto& rule : compound_rules) {	88✔
576	check_compound(node_sets, compound_rules, rule.first, 0);	38✔
577
578	NodeSetRulePtr rules = std::make_unique<NodeSetCompoundRule>(rule.first, rule.second);	64✔
579	node_sets.emplace(rule.first, std::move(rules));	32✔
580	}
581	}	50✔
582
583	Selection NodeSets::materialize(const std::string& name, const NodePopulation& population) const {	158✔
584	const auto& node_set = node_sets_.find(name);	158✔
585	if (node_set == node_sets_.end()) {	158✔
586	throw SonataError(fmt::format("Unknown node_set {}", name));	4✔
587	}
588	const auto& ns = node_set->second;	156✔
589	if (!ns->is_compound()) {	156✔
590	return population.selectAll() & ns->materialize(*this, population);	248✔
591	}
592
593	// it's common to have a deep structure of compound statements
594	// (ie: a whole hierarchy of regions), all checking the same attribute
595	// rather than `materializing` them separately, we group them, and materialize
596	// them all at once
597	Selection ret{{}};	64✔
598
599	std::vector<NodeSetRule*> queue{ns.get()};	64✔
600	std::map<std::string, std::set<std::string>> attribute2rule_strings;	64✔
601	std::map<std::string, std::set<int64_t>> attribute2rule_int64;	64✔
602	while (!queue.empty()) {	124✔
603	const auto* ns = queue.back();	92✔
604	queue.pop_back();	92✔
605
606	if (ns->is_compound()) {	92✔
607	const auto* targets = dynamic_cast<const NodeSetCompoundRule*>(ns);	92✔
608	for (const auto& target : targets->getTargets()) {	222✔
609	const auto& node_set = node_sets_.find(target)->second;	130✔
610	if (node_set->is_compound()) {	130✔
611	queue.push_back(node_set.get());	60✔
612	continue;	60✔
613	}
614
615	{
616	const auto* basic_int = dynamic_cast<const NodeSetBasicRule<int64_t>*>(	70✔
617	node_set.get());	140✔
618	if (basic_int != nullptr) {	70✔
619	basic_int->add_attribute2rule(attribute2rule_int64);	10✔
620	continue;	10✔
621	}
622	}
623
624	{
625	const auto* basic_string = dynamic_cast<const NodeSetBasicRule<std::string>*>(	60✔
626	node_set.get());	120✔
627	if (basic_string != nullptr) {	60✔
628	basic_string->add_attribute2rule(attribute2rule_strings);	10✔
629	continue;	10✔
630	}
631	}
632
633	ret = ret \| ns->materialize(*this, population);	50✔
634	}
635	} else {
636	ret = ret \| ns->materialize(*this, population);	×
637	}
638	}
639
640	for (const auto& it : attribute2rule_strings) {	42✔
641	std::vector<std::string> values(it.second.begin(), it.second.end());	10✔
642	ret = ret \| population.matchAttributeValues(it.first, values);	10✔
643	}
644
645	for (const auto& it : attribute2rule_int64) {	42✔
646	std::vector<int64_t> values(it.second.begin(), it.second.end());	10✔
647	ret = ret \| population.matchAttributeValues(it.first, values);	10✔
648	}
649
650	return ret;	32✔
651	}
652	} // namespace detail
653
654	NodeSets::NodeSets(const std::string& content)	84✔
655	: impl_(new detail::NodeSets(content)) {}	84✔
656
657	NodeSets::NodeSets(std::unique_ptr<detail::NodeSets>&& impl)	2✔
658	: impl_(std::move(impl)) {}	2✔
659
660	NodeSets::NodeSets(NodeSets&&) noexcept = default;
661	NodeSets& NodeSets::operator=(NodeSets&&) noexcept = default;
662	NodeSets::~NodeSets() = default;
663
664	NodeSets NodeSets::fromFile(const std::string& path) {	2✔
665	return NodeSets(detail::NodeSets::fromFile(path));	4✔
666	}
667
668	Selection NodeSets::materialize(const std::string& name, const NodePopulation& population) const {	58✔
669	return impl_->materialize(name, population);	58✔
670	}
671
672	std::set<std::string> NodeSets::names() const {	2✔
673	return impl_->names();	2✔
674	}
675
676	std::string NodeSets::toJSON() const {	10✔
677	return impl_->toJSON();	10✔
678	}
679
680	} // namespace sonata
681	} // namespace bbp

BlueBrain / libsonata / 5131875325

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous