30554808706

Committed 30 Jul 2026 03:03PM UTC coverage: 64.737% (-0.001%) from 64.738%

Build # 30554808706

Build Type

Pull #917

github

Committed by

web-flow

Commit Message

Merge 784206e93 into 4a519a416

Pull Request Pull Request #917: Add scalar tensor

Coverage Stats

28 of 38 new or added lines in 3 files covered. (73.68%)

3 existing lines in 1 file now uncovered.

44367 of 68534 relevant lines covered (64.74%)

716.46 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.77

/sdfg/src/data_flow/library_nodes/math/tensor/elementwise_node.cpp

#include "sdfg/data_flow/library_nodes/math/tensor/elementwise_node.h"

#include "sdfg/analysis/analysis.h"
#include "sdfg/builder/structured_sdfg_builder.h"
#include "sdfg/data_flow/tasklet.h"
#include "sdfg/types/type.h"

namespace sdfg {
namespace math {
namespace tensor {

ElementWiseDataflowTensorNode::ElementWiseDataflowTensorNode(
    size_t element_id,
    const DebugInfo& debug_info,
    const graph::Vertex vertex,
    data_flow::DataFlowGraph& parent,
    const data_flow::LibraryNodeCode& code,
    const std::vector<symbolic::Expression>& shape,
    const std::string& modified_tensor_conn,
    const std::vector<std::string>& tensor_inputs,
    QuantizationType quantization,
    const data_flow::ImplementationType& impl_type
)
    : TensorNode(
          element_id,
          debug_info,
          vertex,
          parent,
          code,
          {},
          build_input_conns(modified_tensor_conn, tensor_inputs),
          impl_type
      ),
      fixed_quantization_(quantization), shape_(shape) {}

std::vector<std::string> ElementWiseDataflowTensorNode::
    build_input_conns(const std::string& modified_tensor_conn, const std::vector<std::string>& tensor_inputs) {
    std::vector<std::string> input_conns;
    input_conns.reserve(1 + input_conns.size());
    input_conns.push_back(modified_tensor_conn);
    input_conns.insert(input_conns.end(), tensor_inputs.begin(), tensor_inputs.end());
    return input_conns;
}

types::PrimitiveType ElementWiseDataflowTensorNode::fixed_quantization() const { return fixed_quantization_; }

void ElementWiseDataflowTensorNode::set_fixed_quantization(const QuantizationType quant) {
    fixed_quantization_ = quant;
}

types::PrimitiveType ElementWiseDataflowTensorNode::quantization(const data_flow::DataFlowGraph& data_flow_graph
) const {
    if (fixed_quantization_ != QUANTIZATION_MATCH_INPUTS) {
        return fixed_quantization_;
    } else {
        return this->primitive_type(data_flow_graph);
    }
}

std::optional<types::PrimitiveType> ElementWiseDataflowTensorNode::uniform_quantization(const data_flow::DataFlowGraph&
                                                                                            data_flow_graph) const {
    if (fixed_quantization_ != QUANTIZATION_MATCH_INPUTS) {
        auto inferred = this->primitive_type(data_flow_graph);
        if (inferred == fixed_quantization_) {
            return fixed_quantization_;
        } else {
            return std::nullopt;
        }
    } else {
        return this->primitive_type(data_flow_graph);
    }
}

void ElementWiseDataflowTensorNode::validate_target_tensor(const data_flow::DataFlowGraph& graph) const {
    auto* target_ptr_edge = graph.in_edge_for_connector(*this, inputs_.at(0));
    if (!target_ptr_edge) {
        throw InvalidSDFGException(
            "On libNode #" + std::to_string(element_id()) + ": input " + inputs_.at(0) + " is not connected"
        );
    }
    auto& tensor_output = static_cast<const types::Tensor&>(target_ptr_edge->base_type());

    validate_shape_matches(shape_, tensor_output.layout(), "output tensor");
}

void ElementWiseDataflowTensorNode::validate_all_input_tensors(const data_flow::DataFlowGraph& graph) const {
    for (int i = 1; i < tensor_input_count(); ++i) {
        auto* iedge = graph.in_edge_for_connector(*this, inputs_.at(i));
        if (!iedge) {
            throw InvalidSDFGException(
                "On libNode #" + std::to_string(element_id()) + ": input " + inputs_.at(i) + " is not connected"
            );
        }
        if (iedge->base_type().type_id() == types::TypeID::Scalar) {
            continue;
        }
        auto& tensor_input = static_cast<const types::Tensor&>(iedge->base_type());
        // Case 1: Scalar input is allowed as secondary input
        if (tensor_input.is_scalar()) {
            continue;
        }

        // currently no arbitrary broadcast support! but could be added
        validate_shape_matches(shape_, tensor_input.layout(), "input " + inputs_.at(i));
    }
}

void ElementWiseDataflowTensorNode::validate_non_tensor_inputs(const data_flow::DataFlowGraph& graph) const {
    for (int i = tensor_input_count(); i < inputs_.size(); ++i) {
        auto* iedge = graph.in_edge_for_connector(*this, inputs_.at(i));
        if (!iedge) {
            if (i < mandatory_input_count()) {
                throw InvalidSDFGException(
                    "On libNode #" + std::to_string(element_id()) + ": input " + inputs_.at(i) + " is not connected"
                );
            } else {
                continue;
            }
        }
        if (iedge->base_type().type_id() != types::TypeID::Scalar) {
            throw InvalidSDFGException(
                "On libNode #" + std::to_string(element_id()) + ": input " + inputs_.at(i) + " is not scalar"
            );
        }
    }
}

void ElementWiseDataflowTensorNode::validate(const Function& function) const {
    TensorNode::validate(function);

    auto& graph = this->get_parent();

    validate_target_tensor(graph);

    validate_all_input_tensors(graph);

    validate_non_tensor_inputs(graph);
}

symbolic::SymbolSet ElementWiseDataflowTensorNode::symbols() const {
    symbolic::SymbolSet syms;
    for (const auto& dim : shape_) {
        for (auto& atom : symbolic::atoms(dim)) {
            syms.insert(atom);
        }
    }
    return syms;
}

void ElementWiseDataflowTensorNode::
    replace(const symbolic::Expression old_expression, const symbolic::Expression new_expression) {
    for (auto& dim : shape_) {
        dim = symbolic::subs(dim, old_expression, new_expression);
    }
}

void ElementWiseDataflowTensorNode::replace(const symbolic::ExpressionMapping& replacements) {
    for (auto& dim : shape_) {
        dim = symbolic::subs(dim, replacements);
    }
}

std::pair<structured_control_flow::Sequence*, std::vector<symbolic::Expression>> ElementWiseDataflowTensorNode::
    add_eltwise_scope(
        builder::StructuredSDFGBuilder& builder,
        const DebugInfo& scope_deb_info,
        Sequence& parent,
        const std::vector<symbolic::Expression>& shape
    ) {
    // Add maps
    data_flow::Subset new_subset;
    std::vector<symbolic::Expression> loop_vars;
    structured_control_flow::Sequence* last_scope = &parent;
    structured_control_flow::Map* last_map = nullptr;

    for (size_t i = 0; i < shape.size(); i++) {
        std::string indvar_str = builder.find_new_name("_i");
        builder.add_container(indvar_str, types::Scalar(types::PrimitiveType::UInt64));

        auto indvar = symbolic::symbol(indvar_str);
        auto init = symbolic::zero();
        auto update = symbolic::add(indvar, symbolic::one());
        auto condition = symbolic::Lt(indvar, shape.at(i));
        last_map = &builder.add_map(
            *last_scope,
            indvar,
            condition,
            init,
            update,
            structured_control_flow::ScheduleType_Sequential::create(),
            scope_deb_info
        );
        last_scope = &last_map->root();

        loop_vars.push_back(indvar);
    }
    return {last_scope, loop_vars};
}

std::unique_ptr<types::IType> ElementWiseDataflowTensorNode::access_type(const std::pair<
                                                                         types::PrimitiveType,
                                                                         const TensorLayout*>& pair) {
    if (pair.second) {
        return std::make_unique<types::Tensor>(pair.first, *pair.second);
    } else {
        return std::make_unique<types::Scalar>(pair.first);
    }
}

bool ElementWiseDataflowTensorNode::create_input(
    builder::StructuredSDFGBuilder& builder,
    structured_control_flow::Block& block,
    const data_flow::AccessNode& org_src,
    const std::pair<types::PrimitiveType, const TensorLayout*>& src_type,
    const ElementInput& needed_input,
    const std::vector<symbolic::Expression>& eltwise_subset,
    std::unordered_map<const data_flow::AccessNode*, data_flow::AccessNode*>& new_node_mapping
) {
    auto* new_consumer = needed_input.consumer;
    if (new_consumer) {
        if (src_type.first != needed_input.required_type) {
            throw InvalidSDFGException(
                "Input " + std::to_string(needed_input.input_conn_index) + " on node #" +
                std::to_string(new_consumer->element_id()) + " is required as " +
                types::primitive_type_to_string(needed_input.required_type) + " but provided as " +
                types::primitive_type_to_string(src_type.first)
            );
        }
        auto existing_input_it = new_node_mapping.find(&org_src);
        data_flow::AccessNode* input_node;
        std::vector<symbolic::Expression> empty_subset;
        const std::vector<symbolic::Expression>* memlet_subset;
        if (src_type.second && !src_type.second->is_scalar()) {
            memlet_subset = &eltwise_subset;
        } else {
            memlet_subset = &empty_subset;
        }
        auto new_type = access_type(src_type);
        if (existing_input_it != new_node_mapping.end()) {
            input_node = existing_input_it->second;
        } else {
            if (org_src.is_constant()) {
                types::Scalar const_type(src_type.first);
                input_node = &builder.add_constant(block, org_src.data(), const_type);
            } else {
                input_node = &builder.add_access(block, org_src.data());
            }
            new_node_mapping.emplace(&org_src, input_node);
        }

        builder.add_computational_memlet(
            block,
            *input_node,
            *new_consumer,
            new_consumer->input(needed_input.input_conn_index),
            *memlet_subset,
            *new_type
        );
        return true;
    } else {
        return false;
    }
}

void ElementWiseDataflowTensorNode::create_output(
    builder::StructuredSDFGBuilder& builder,
    structured_control_flow::Block& block,
    const data_flow::AccessNode& org_dst,
    const types::Tensor& dst_type,
    const ElementOutput& provided_output,
    const std::vector<symbolic::Expression>& eltwise_subset
) {
    auto* producer = provided_output.producer;
    if (dst_type.primitive_type() != provided_output.type) {
        throw InvalidSDFGException(
            "Output " + std::to_string(provided_output.output_conn_index) + " on node #" +
            std::to_string(producer->element_id()) + " is provided as " +
            types::primitive_type_to_string(provided_output.type) + " but required as " +
            types::primitive_type_to_string(dst_type.primitive_type())
        );
    }
    auto& output_node = builder.add_access(block, org_dst.data());
    builder.add_computational_memlet(
        block, *producer, producer->output(provided_output.output_conn_index), output_node, eltwise_subset, dst_type
    );
}

passes::LibNodeExpander::ExpandOutcome ElementWiseDataflowTensorNode::
    expand(passes::LibNodeExpander::ExpandContext& context, Block& block) {
    auto& dataflow = this->get_parent();

    auto* output_tensor_iedge = dataflow.in_edge_for_connector(*this, inputs_.at(0));
    if (!output_tensor_iedge) {
        return context.unable();
    }
    auto& target_tensor = static_cast<const types::Tensor&>(output_tensor_iedge->base_type());
    std::vector<const data_flow::Memlet*> iedges;
    std::vector<const data_flow::AccessNode*> inputs_sa;
    std::vector<std::pair<types::PrimitiveType, const TensorLayout*>> input_types;
    iedges.reserve(inputs_.size() - 1);
    using Dir = passes::LibNodeExpander::InputUse;
    std::vector<Dir> access_dirs{Dir::IndirectWrite};
    for (int i = 1; i < this->inputs_.size(); ++i) {
        auto* iedge = dataflow.in_edge_for_connector(*this, inputs_.at(i));
        if (!iedge) {
            if (i < mandatory_input_count()) {
                return context.unable();
            } else {
                continue;
            }
        }
        iedges.push_back(iedge);
        auto* input_sa = dataflow.find_standalone_entry(iedge);
        if (!input_sa) {
            return context.unable();
        }
        inputs_sa.push_back(input_sa);
        auto& input_type = iedge->base_type();
        if (input_type.type_id() == types::TypeID::Scalar) {
            access_dirs.push_back(Dir::Scalar);
            input_types.emplace_back(input_type.primitive_type(), nullptr);
        } else {
            access_dirs.push_back(Dir::IndirectRead);
            auto& tensor_type = static_cast<const types::Tensor&>(iedge->base_type());
            input_types.emplace_back(input_type.primitive_type(), &tensor_type.layout());
        }
    }

    auto* output_tensor_sa = dataflow.find_standalone_entry(output_tensor_iedge);
    if (!output_tensor_sa) {
        return context.unable();
    }

    auto standalone = context.replacement_requires_access_nodes(access_dirs);

    if (standalone) {
        auto& builder = standalone->builder();

        // Add new graph after the current block
        auto& new_sequence = standalone->replace_with_sequence();

        auto [eltw_scope, loop_vars] = add_eltwise_scope(builder, block.debug_info(), new_sequence, shape_);

        std::vector<tensor::ElementWiseDataflowTensorNode::ElementInput> eltwise_inputs;
        eltwise_inputs.reserve(inputs_.size() - 1);
        for (int i = 0; i < input_types.size(); ++i) {
            eltwise_inputs.push_back({.required_type = input_types.at(i).first});
        }

        auto& new_block = builder.add_block(*eltw_scope);

        auto produced_output =
            expand_operation_dataflow(builder, new_block, eltwise_inputs, target_tensor.primitive_type());
        if (!produced_output.producer) {
            return context.unable();
        }

        std::unordered_map<const data_flow::AccessNode*, data_flow::AccessNode*> new_node_mapping;

        // for all old input edge, remove old, create new
        for (int i = 0; i < iedges.size(); ++i) {
            create_input(
                builder, new_block, *inputs_sa.at(i), input_types.at(i), eltwise_inputs.at(i), loop_vars, new_node_mapping
            );
        }
        create_output(builder, new_block, *output_tensor_sa, target_tensor, produced_output, loop_vars);

        return standalone->successfully_expanded();
    } else {
        return context.unable();
    }
}

data_flow::PointerAccessType ElementWiseDataflowTensorNode::pointer_access_type(int input_idx) const {
    if (input_idx == 0) {
        return data_flow::PointerAccessMeta::create_full_write_only(symbolic::__nullptr__(), true);
    } else if (input_idx < tensor_input_count()) {
        return data_flow::PointerAccessMeta::create_read_only(symbolic::__nullptr__(), true);
    } else {
        return TensorNode::pointer_access_type(input_idx);
    }
}

data_flow::AccessNode& ElementWiseDataflowTensorNode::create_tmp_access_node(
    builder::StructuredSDFGBuilder& builder,
    structured_control_flow::Block& block,
    const std::string& prefix,
    const types::IType& type
) const {
    auto cont = builder.find_new_name(prefix);
    builder.add_container(cont, type);
    auto& output_node_add = builder.add_access(block, cont);
    return output_node_add;
}

nlohmann::json BaseElementWiseDataflowTensorNodeSerializer::serialize(const data_flow::LibraryNode& library_node) {
    const ElementWiseDataflowTensorNode& elem_node = static_cast<const ElementWiseDataflowTensorNode&>(library_node);
    nlohmann::json j;

    j["code"] = elem_node.code().value();

    serializer::JSONSerializer serializer;
    j["shape"] = nlohmann::json::array();
    for (auto& dim : elem_node.shape()) {
        j["shape"].push_back(serializer.expression(dim));
    }

    j["result_quant"] = elem_node.fixed_quantization();

    return j;
}

BaseElementWiseDataflowTensorNodeSerializer::BaseDeser BaseElementWiseDataflowTensorNodeSerializer::
    deserialize_base_values(const nlohmann::json& j) {
    assert(j.contains("element_id"));
    assert(j.contains("code"));
    assert(j.contains("debug_info"));

    std::vector<symbolic::Expression> shape;
    if (j.contains("shape")) {
        for (const auto& dim : j["shape"]) {
            shape.push_back(symbolic::parse(dim.get<std::string>()));
        }
    }

    serializer::JSONSerializer serializer;
    auto debug_info = serializer.json_to_debug_info(j["debug_info"]);
    return {
        .shape = shape,
        .quantization = deserialize_quantization(j, "result_quant", QUANTIZATION_MATCH_INPUTS),
        .debug_info = debug_info
    };
}

} // namespace tensor
} // namespace math
} // namespace sdfg

1	#include "sdfg/data_flow/library_nodes/math/tensor/elementwise_node.h"
2
3	#include "sdfg/analysis/analysis.h"
4	#include "sdfg/builder/structured_sdfg_builder.h"
5	#include "sdfg/data_flow/tasklet.h"
6	#include "sdfg/types/type.h"
7
8	namespace sdfg {
9	namespace math {
10	namespace tensor {
11
12	ElementWiseDataflowTensorNode::ElementWiseDataflowTensorNode(
13	size_t element_id,
14	const DebugInfo& debug_info,
15	const graph::Vertex vertex,
16	data_flow::DataFlowGraph& parent,
17	const data_flow::LibraryNodeCode& code,
18	const std::vector<symbolic::Expression>& shape,
19	const std::string& modified_tensor_conn,
20	const std::vector<std::string>& tensor_inputs,
21	QuantizationType quantization,
22	const data_flow::ImplementationType& impl_type
23	)
24	: TensorNode(	595✔
25	element_id,	595✔
26	debug_info,	595✔
27	vertex,	595✔
28	parent,	595✔
29	code,	595✔
30	{},	595✔
31	build_input_conns(modified_tensor_conn, tensor_inputs),	595✔
32	impl_type	595✔
33	),	595✔
34	fixed_quantization_(quantization), shape_(shape) {}	595✔
35
36	std::vector<std::string> ElementWiseDataflowTensorNode::
37	build_input_conns(const std::string& modified_tensor_conn, const std::vector<std::string>& tensor_inputs) {	595✔
38	std::vector<std::string> input_conns;	595✔
39	input_conns.reserve(1 + input_conns.size());	595✔
40	input_conns.push_back(modified_tensor_conn);	595✔
41	input_conns.insert(input_conns.end(), tensor_inputs.begin(), tensor_inputs.end());	595✔
42	return input_conns;	595✔
43	}	595✔
44
45	types::PrimitiveType ElementWiseDataflowTensorNode::fixed_quantization() const { return fixed_quantization_; }	3✔
46
47	void ElementWiseDataflowTensorNode::set_fixed_quantization(const QuantizationType quant) {	×
48	fixed_quantization_ = quant;	×
49	}	×
50
51	types::PrimitiveType ElementWiseDataflowTensorNode::quantization(const data_flow::DataFlowGraph& data_flow_graph
52	) const {	×
53	if (fixed_quantization_ != QUANTIZATION_MATCH_INPUTS) {	×
54	return fixed_quantization_;	×
55	} else {	×
56	return this->primitive_type(data_flow_graph);	×
57	}	×
58	}	×
59
60	std::optional<types::PrimitiveType> ElementWiseDataflowTensorNode::uniform_quantization(const data_flow::DataFlowGraph&
61	data_flow_graph) const {	×
62	if (fixed_quantization_ != QUANTIZATION_MATCH_INPUTS) {	×
63	auto inferred = this->primitive_type(data_flow_graph);	×
64	if (inferred == fixed_quantization_) {	×
65	return fixed_quantization_;	×
66	} else {	×
67	return std::nullopt;	×
68	}	×
69	} else {	×
70	return this->primitive_type(data_flow_graph);	×
71	}	×
72	}	×
73
74	void ElementWiseDataflowTensorNode::validate_target_tensor(const data_flow::DataFlowGraph& graph) const {	590✔
75	auto* target_ptr_edge = graph.in_edge_for_connector(*this, inputs_.at(0));	590✔
76	if (!target_ptr_edge) {	590✔
NEW 77	throw InvalidSDFGException(	×
NEW 78	"On libNode #" + std::to_string(element_id()) + ": input " + inputs_.at(0) + " is not connected"	×
NEW 79	);	×
NEW 80	}	×
81	auto& tensor_output = static_cast<const types::Tensor&>(target_ptr_edge->base_type());	590✔
82
83	validate_shape_matches(shape_, tensor_output.layout(), "output tensor");	590✔
84	}	590✔
85
86	void ElementWiseDataflowTensorNode::validate_all_input_tensors(const data_flow::DataFlowGraph& graph) const {	618✔
87	for (int i = 1; i < tensor_input_count(); ++i) {	1,526✔
88	auto* iedge = graph.in_edge_for_connector(*this, inputs_.at(i));	908✔
89	if (!iedge) {	908✔
90	throw InvalidSDFGException(	×
91	"On libNode #" + std::to_string(element_id()) + ": input " + inputs_.at(i) + " is not connected"	×
92	);	×
93	}	×
94	if (iedge->base_type().type_id() == types::TypeID::Scalar) {	908✔
95	continue;	×
96	}	×
97	auto& tensor_input = static_cast<const types::Tensor&>(iedge->base_type());	908✔
98	// Case 1: Scalar input is allowed as secondary input
99	if (tensor_input.is_scalar()) {	908✔
100	continue;	1✔
101	}	1✔
102
103	// currently no arbitrary broadcast support! but could be added
104	validate_shape_matches(shape_, tensor_input.layout(), "input " + inputs_.at(i));	907✔
105	}	907✔
106	}	618✔
107
108	void ElementWiseDataflowTensorNode::validate_non_tensor_inputs(const data_flow::DataFlowGraph& graph) const {	362✔
109	for (int i = tensor_input_count(); i < inputs_.size(); ++i) {	362✔
110	auto* iedge = graph.in_edge_for_connector(*this, inputs_.at(i));	×
111	if (!iedge) {	×
112	if (i < mandatory_input_count()) {	×
113	throw InvalidSDFGException(	×
114	"On libNode #" + std::to_string(element_id()) + ": input " + inputs_.at(i) + " is not connected"	×
115	);	×
116	} else {	×
117	continue;	×
118	}	×
119	}	×
120	if (iedge->base_type().type_id() != types::TypeID::Scalar) {	×
121	throw InvalidSDFGException(	×
122	"On libNode #" + std::to_string(element_id()) + ": input " + inputs_.at(i) + " is not scalar"	×
123	);	×
124	}	×
125	}	×
126	}	362✔
127
128	void ElementWiseDataflowTensorNode::validate(const Function& function) const {	140✔
129	TensorNode::validate(function);	140✔
130
131	auto& graph = this->get_parent();	140✔
132
133	validate_target_tensor(graph);	140✔
134
135	validate_all_input_tensors(graph);	140✔
136
137	validate_non_tensor_inputs(graph);	140✔
138	}	140✔
139
140	symbolic::SymbolSet ElementWiseDataflowTensorNode::symbols() const {	17✔
141	symbolic::SymbolSet syms;	17✔
142	for (const auto& dim : shape_) {	68✔
143	for (auto& atom : symbolic::atoms(dim)) {	68✔
144	syms.insert(atom);	×
145	}	×
146	}	68✔
147	return syms;	17✔
148	}	17✔
149
150	void ElementWiseDataflowTensorNode::
151	replace(const symbolic::Expression old_expression, const symbolic::Expression new_expression) {	×
152	for (auto& dim : shape_) {	×
153	dim = symbolic::subs(dim, old_expression, new_expression);	×
154	}	×
155	}	×
156
157	void ElementWiseDataflowTensorNode::replace(const symbolic::ExpressionMapping& replacements) {	×
158	for (auto& dim : shape_) {	×
159	dim = symbolic::subs(dim, replacements);	×
160	}	×
161	}	×
162
163	std::pair<structured_control_flow::Sequence*, std::vector<symbolic::Expression>> ElementWiseDataflowTensorNode::
164	add_eltwise_scope(
165	builder::StructuredSDFGBuilder& builder,
166	const DebugInfo& scope_deb_info,
167	Sequence& parent,
168	const std::vector<symbolic::Expression>& shape
169	) {	560✔
170	// Add maps
171	data_flow::Subset new_subset;	560✔
172	std::vector<symbolic::Expression> loop_vars;	560✔
173	structured_control_flow::Sequence* last_scope = &parent;	560✔
174	structured_control_flow::Map* last_map = nullptr;	560✔
175
176	for (size_t i = 0; i < shape.size(); i++) {	1,902✔
177	std::string indvar_str = builder.find_new_name("_i");	1,342✔
178	builder.add_container(indvar_str, types::Scalar(types::PrimitiveType::UInt64));	1,342✔
179
180	auto indvar = symbolic::symbol(indvar_str);	1,342✔
181	auto init = symbolic::zero();	1,342✔
182	auto update = symbolic::add(indvar, symbolic::one());	1,342✔
183	auto condition = symbolic::Lt(indvar, shape.at(i));	1,342✔
184	last_map = &builder.add_map(	1,342✔
185	*last_scope,	1,342✔
186	indvar,	1,342✔
187	condition,	1,342✔
188	init,	1,342✔
189	update,	1,342✔
190	structured_control_flow::ScheduleType_Sequential::create(),	1,342✔
191	scope_deb_info	1,342✔
192	);	1,342✔
193	last_scope = &last_map->root();	1,342✔
194
195	loop_vars.push_back(indvar);	1,342✔
196	}	1,342✔
197	return {last_scope, loop_vars};	560✔
198	}	560✔
199
200	std::unique_ptr<types::IType> ElementWiseDataflowTensorNode::access_type(const std::pair<
201	types::PrimitiveType,
202	const TensorLayout*>& pair) {	862✔
203	if (pair.second) {	862✔
204	return std::make_unique<types::Tensor>(pair.first, *pair.second);	862✔
205	} else {	862✔
206	return std::make_unique<types::Scalar>(pair.first);	×
207	}	×
208	}	862✔
209
210	bool ElementWiseDataflowTensorNode::create_input(
211	builder::StructuredSDFGBuilder& builder,
212	structured_control_flow::Block& block,
213	const data_flow::AccessNode& org_src,
214	const std::pair<types::PrimitiveType, const TensorLayout*>& src_type,
215	const ElementInput& needed_input,
216	const std::vector<symbolic::Expression>& eltwise_subset,
217	std::unordered_map<const data_flow::AccessNode, data_flow::AccessNode>& new_node_mapping
218	) {	862✔
219	auto* new_consumer = needed_input.consumer;	862✔
220	if (new_consumer) {	862✔
221	if (src_type.first != needed_input.required_type) {	862✔
222	throw InvalidSDFGException(	×
223	"Input " + std::to_string(needed_input.input_conn_index) + " on node #" +	×
224	std::to_string(new_consumer->element_id()) + " is required as " +	×
225	types::primitive_type_to_string(needed_input.required_type) + " but provided as " +	×
226	types::primitive_type_to_string(src_type.first)	×
227	);	×
228	}	×
229	auto existing_input_it = new_node_mapping.find(&org_src);	862✔
230	data_flow::AccessNode* input_node;	862✔
231	std::vector<symbolic::Expression> empty_subset;	862✔
232	const std::vector<symbolic::Expression>* memlet_subset;	862✔
233	if (src_type.second && !src_type.second->is_scalar()) {	862✔
234	memlet_subset = &eltwise_subset;	843✔
235	} else {	843✔
236	memlet_subset = &empty_subset;	19✔
237	}	19✔
238	auto new_type = access_type(src_type);	862✔
239	if (existing_input_it != new_node_mapping.end()) {	862✔
240	input_node = existing_input_it->second;	4✔
241	} else {	858✔
242	if (org_src.is_constant()) {	858✔
243	types::Scalar const_type(src_type.first);	×
244	input_node = &builder.add_constant(block, org_src.data(), const_type);	×
245	} else {	858✔
246	input_node = &builder.add_access(block, org_src.data());	858✔
247	}	858✔
248	new_node_mapping.emplace(&org_src, input_node);	858✔
249	}	858✔
250
251	builder.add_computational_memlet(	862✔
252	block,	862✔
253	*input_node,	862✔
254	*new_consumer,	862✔
255	new_consumer->input(needed_input.input_conn_index),	862✔
256	*memlet_subset,	862✔
257	*new_type	862✔
258	);	862✔
259	return true;	862✔
260	} else {	862✔
261	return false;	×
262	}	×
263	}	862✔
264
265	void ElementWiseDataflowTensorNode::create_output(
266	builder::StructuredSDFGBuilder& builder,
267	structured_control_flow::Block& block,
268	const data_flow::AccessNode& org_dst,
269	const types::Tensor& dst_type,
270	const ElementOutput& provided_output,
271	const std::vector<symbolic::Expression>& eltwise_subset
272	) {	560✔
273	auto* producer = provided_output.producer;	560✔
274	if (dst_type.primitive_type() != provided_output.type) {	560✔
275	throw InvalidSDFGException(	×
276	"Output " + std::to_string(provided_output.output_conn_index) + " on node #" +	×
277	std::to_string(producer->element_id()) + " is provided as " +	×
278	types::primitive_type_to_string(provided_output.type) + " but required as " +	×
279	types::primitive_type_to_string(dst_type.primitive_type())	×
280	);	×
281	}	×
282	auto& output_node = builder.add_access(block, org_dst.data());	560✔
283	builder.add_computational_memlet(	560✔
284	block, *producer, producer->output(provided_output.output_conn_index), output_node, eltwise_subset, dst_type	560✔
285	);	560✔
286	}	560✔
287
288	passes::LibNodeExpander::ExpandOutcome ElementWiseDataflowTensorNode::
289	expand(passes::LibNodeExpander::ExpandContext& context, Block& block) {	560✔
290	auto& dataflow = this->get_parent();	560✔
291
292	auto* output_tensor_iedge = dataflow.in_edge_for_connector(*this, inputs_.at(0));	560✔
293	if (!output_tensor_iedge) {	560✔
294	return context.unable();	×
295	}	×
296	auto& target_tensor = static_cast<const types::Tensor&>(output_tensor_iedge->base_type());	560✔
297	std::vector<const data_flow::Memlet*> iedges;	560✔
298	std::vector<const data_flow::AccessNode*> inputs_sa;	560✔
299	std::vector<std::pair<types::PrimitiveType, const TensorLayout*>> input_types;	560✔
300	iedges.reserve(inputs_.size() - 1);	560✔
301	using Dir = passes::LibNodeExpander::InputUse;	560✔
302	std::vector<Dir> access_dirs{Dir::IndirectWrite};	560✔
303	for (int i = 1; i < this->inputs_.size(); ++i) {	1,422✔
304	auto* iedge = dataflow.in_edge_for_connector(*this, inputs_.at(i));	862✔
305	if (!iedge) {	862✔
306	if (i < mandatory_input_count()) {	×
307	return context.unable();	×
308	} else {	×
309	continue;	×
310	}	×
311	}	×
312	iedges.push_back(iedge);	862✔
313	auto* input_sa = dataflow.find_standalone_entry(iedge);	862✔
314	if (!input_sa) {	862✔
315	return context.unable();	×
316	}	×
317	inputs_sa.push_back(input_sa);	862✔
318	auto& input_type = iedge->base_type();	862✔
319	if (input_type.type_id() == types::TypeID::Scalar) {	862✔
320	access_dirs.push_back(Dir::Scalar);	×
321	input_types.emplace_back(input_type.primitive_type(), nullptr);	×
322	} else {	862✔
323	access_dirs.push_back(Dir::IndirectRead);	862✔
324	auto& tensor_type = static_cast<const types::Tensor&>(iedge->base_type());	862✔
325	input_types.emplace_back(input_type.primitive_type(), &tensor_type.layout());	862✔
326	}	862✔
327	}	862✔
328
329	auto* output_tensor_sa = dataflow.find_standalone_entry(output_tensor_iedge);	560✔
330	if (!output_tensor_sa) {	560✔
331	return context.unable();	×
332	}	×
333
334	auto standalone = context.replacement_requires_access_nodes(access_dirs);	560✔
335
336	if (standalone) {	560✔
337	auto& builder = standalone->builder();	560✔
338
339	// Add new graph after the current block
340	auto& new_sequence = standalone->replace_with_sequence();	560✔
341
342	auto [eltw_scope, loop_vars] = add_eltwise_scope(builder, block.debug_info(), new_sequence, shape_);	560✔
343
344	std::vector<tensor::ElementWiseDataflowTensorNode::ElementInput> eltwise_inputs;	560✔
345	eltwise_inputs.reserve(inputs_.size() - 1);	560✔
346	for (int i = 0; i < input_types.size(); ++i) {	1,422✔
347	eltwise_inputs.push_back({.required_type = input_types.at(i).first});	862✔
348	}	862✔
349
350	auto& new_block = builder.add_block(*eltw_scope);	560✔
351
352	auto produced_output =	560✔
353	expand_operation_dataflow(builder, new_block, eltwise_inputs, target_tensor.primitive_type());	560✔
354	if (!produced_output.producer) {	560✔
355	return context.unable();	×
356	}	×
357
358	std::unordered_map<const data_flow::AccessNode, data_flow::AccessNode> new_node_mapping;	560✔
359
360	// for all old input edge, remove old, create new
361	for (int i = 0; i < iedges.size(); ++i) {	1,422✔
362	create_input(	862✔
363	builder, new_block, *inputs_sa.at(i), input_types.at(i), eltwise_inputs.at(i), loop_vars, new_node_mapping	862✔
364	);	862✔
365	}	862✔
366	create_output(builder, new_block, *output_tensor_sa, target_tensor, produced_output, loop_vars);	560✔
367
368	return standalone->successfully_expanded();	560✔
369	} else {	560✔
370	return context.unable();	×
371	}	×
372	}	560✔
373
374	data_flow::PointerAccessType ElementWiseDataflowTensorNode::pointer_access_type(int input_idx) const {	×
375	if (input_idx == 0) {	×
376	return data_flow::PointerAccessMeta::create_full_write_only(symbolic::__nullptr__(), true);	×
377	} else if (input_idx < tensor_input_count()) {	×
378	return data_flow::PointerAccessMeta::create_read_only(symbolic::__nullptr__(), true);	×
379	} else {	×
380	return TensorNode::pointer_access_type(input_idx);	×
381	}	×
382	}	×
383
384	data_flow::AccessNode& ElementWiseDataflowTensorNode::create_tmp_access_node(
385	builder::StructuredSDFGBuilder& builder,
386	structured_control_flow::Block& block,
387	const std::string& prefix,
388	const types::IType& type
389	) const {	50✔
390	auto cont = builder.find_new_name(prefix);	50✔
391	builder.add_container(cont, type);	50✔
392	auto& output_node_add = builder.add_access(block, cont);	50✔
393	return output_node_add;	50✔
394	}	50✔
395
396	nlohmann::json BaseElementWiseDataflowTensorNodeSerializer::serialize(const data_flow::LibraryNode& library_node) {	3✔
397	const ElementWiseDataflowTensorNode& elem_node = static_cast<const ElementWiseDataflowTensorNode&>(library_node);	3✔
398	nlohmann::json j;	3✔
399
400	j["code"] = elem_node.code().value();	3✔
401
402	serializer::JSONSerializer serializer;	3✔
403	j["shape"] = nlohmann::json::array();	3✔
404	for (auto& dim : elem_node.shape()) {	4✔
405	j["shape"].push_back(serializer.expression(dim));	4✔
406	}	4✔
407
408	j["result_quant"] = elem_node.fixed_quantization();	3✔
409
410	return j;	3✔
411	}	3✔
412
413	BaseElementWiseDataflowTensorNodeSerializer::BaseDeser BaseElementWiseDataflowTensorNodeSerializer::
414	deserialize_base_values(const nlohmann::json& j) {	3✔
415	assert(j.contains("element_id"));	3✔
416	assert(j.contains("code"));	3✔
417	assert(j.contains("debug_info"));	3✔
418
419	std::vector<symbolic::Expression> shape;	3✔
420	if (j.contains("shape")) {	3✔
421	for (const auto& dim : j["shape"]) {	4✔
422	shape.push_back(symbolic::parse(dim.get<std::string>()));	4✔
423	}	4✔
424	}	3✔
425
426	serializer::JSONSerializer serializer;	3✔
427	auto debug_info = serializer.json_to_debug_info(j["debug_info"]);	3✔
428	return {	3✔
429	.shape = shape,	3✔
430	.quantization = deserialize_quantization(j, "result_quant", QUANTIZATION_MATCH_INPUTS),	3✔
431	.debug_info = debug_info	3✔
432	};	3✔
433	}	3✔
434
435	} // namespace tensor
436	} // namespace math
437	} // namespace sdfg

daisytuner / docc / 30554808706

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous