27981272983

Committed 22 Jun 2026 08:18PM UTC coverage: 61.754% (-0.03%) from 61.782%

Build # 27981272983

Build Type

Pull #781

github

Committed by

web-flow

Commit Message

Merge bddaa3724 into fe87d162b

Pull Request Pull Request #781: Extend Segformer benchmarks setup

Coverage Stats

987 of 1432 new or added lines in 62 files covered. (68.92%)

9 existing lines in 7 files now uncovered.

38121 of 61730 relevant lines covered (61.75%)

993.19 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

57.46

/sdfg/src/data_flow/library_nodes/math/tensor/elementwise_node.cpp

#include "sdfg/data_flow/library_nodes/math/tensor/elementwise_node.h"

#include "sdfg/analysis/analysis.h"
#include "sdfg/builder/structured_sdfg_builder.h"
#include "sdfg/data_flow/tasklet.h"
#include "sdfg/types/type.h"

namespace sdfg {
namespace math {
namespace tensor {

ElementWiseDataflowTensorNode::ElementWiseDataflowTensorNode(
    size_t element_id,
    const DebugInfo& debug_info,
    const graph::Vertex vertex,
    data_flow::DataFlowGraph& parent,
    const data_flow::LibraryNodeCode& code,
    const std::vector<symbolic::Expression>& shape,
    const std::string& modified_tensor_conn,
    const std::vector<std::string>& tensor_inputs,
    QuantizationType quantization,
    const data_flow::ImplementationType& impl_type
)
    : TensorNode(
          element_id,
          debug_info,
          vertex,
          parent,
          code,
          {},
          build_input_conns(modified_tensor_conn, tensor_inputs),
          impl_type
      ),
      fixed_quantization_(quantization), shape_(shape) {}

std::vector<std::string> ElementWiseDataflowTensorNode::
    build_input_conns(const std::string& modified_tensor_conn, const std::vector<std::string>& tensor_inputs) {
    std::vector<std::string> input_conns;
    input_conns.reserve(1 + input_conns.size());
    input_conns.push_back(modified_tensor_conn);
    input_conns.insert(input_conns.end(), tensor_inputs.begin(), tensor_inputs.end());
    return input_conns;
}

types::PrimitiveType ElementWiseDataflowTensorNode::fixed_quantization() const { return fixed_quantization_; }

void ElementWiseDataflowTensorNode::set_fixed_quantization(const QuantizationType quant) {
    fixed_quantization_ = quant;
}

types::PrimitiveType ElementWiseDataflowTensorNode::quantization(const data_flow::DataFlowGraph& data_flow_graph
) const {
    if (fixed_quantization_ != QUANTIZATION_MATCH_INPUTS) {
        return fixed_quantization_;
    } else {
        return this->primitive_type(data_flow_graph);
    }
}

std::optional<types::PrimitiveType> ElementWiseDataflowTensorNode::uniform_quantization(const data_flow::DataFlowGraph&
                                                                                            data_flow_graph) const {
    if (fixed_quantization_ != QUANTIZATION_MATCH_INPUTS) {
        auto inferred = this->primitive_type(data_flow_graph);
        if (inferred == fixed_quantization_) {
            return fixed_quantization_;
        } else {
            return std::nullopt;
        }
    } else {
        return this->primitive_type(data_flow_graph);
    }
}

void ElementWiseDataflowTensorNode::validate_target_tensor(const data_flow::DataFlowGraph& graph) const {
    auto* target_ptr_edge = graph.in_edge_for_connector(*this, inputs_.at(0));
    auto& tensor_output = static_cast<const types::Tensor&>(target_ptr_edge->base_type());

    validate_shape_matches(shape_, tensor_output.layout(), "output tensor");
}

void ElementWiseDataflowTensorNode::validate_all_input_tensors(const data_flow::DataFlowGraph& graph) const {
    for (int i = 1; i < tensor_input_count(); ++i) {
        auto* iedge = graph.in_edge_for_connector(*this, inputs_.at(i));
        if (!iedge) {
            throw InvalidSDFGException(
                "On libNode #" + std::to_string(element_id()) + ": input " + inputs_.at(i) + " is not connected"
            );
        }
        if (iedge->base_type().type_id() == types::TypeID::Scalar) {
            continue;
        }
        auto& tensor_input = static_cast<const types::Tensor&>(iedge->base_type());
        // Case 1: Scalar input is allowed as secondary input
        if (tensor_input.is_scalar()) {
            continue;
        }

        // currently no arbitrary broadcast support! but could be added
        validate_shape_matches(shape_, tensor_input.layout(), "input " + inputs_.at(i));
    }
}

void ElementWiseDataflowTensorNode::validate_non_tensor_inputs(const data_flow::DataFlowGraph& graph) const {
    for (int i = tensor_input_count(); i < inputs_.size(); ++i) {
        auto* iedge = graph.in_edge_for_connector(*this, inputs_.at(i));
        if (!iedge) {
            if (i < mandatory_input_count()) {
                throw InvalidSDFGException(
                    "On libNode #" + std::to_string(element_id()) + ": input " + inputs_.at(i) + " is not connected"
                );
            } else {
                continue;
            }
        }
        if (iedge->base_type().type_id() != types::TypeID::Scalar) {
            throw InvalidSDFGException(
                "On libNode #" + std::to_string(element_id()) + ": input " + inputs_.at(i) + " is not scalar"
            );
        }
    }
}

void ElementWiseDataflowTensorNode::validate(const Function& function) const {
    TensorNode::validate(function);

    auto& graph = this->get_parent();

    validate_target_tensor(graph);

    validate_all_input_tensors(graph);

    validate_non_tensor_inputs(graph);
}

symbolic::SymbolSet ElementWiseDataflowTensorNode::symbols() const {
    symbolic::SymbolSet syms;
    for (const auto& dim : shape_) {
        for (auto& atom : symbolic::atoms(dim)) {
            syms.insert(atom);
        }
    }
    return syms;
}

void ElementWiseDataflowTensorNode::
    replace(const symbolic::Expression old_expression, const symbolic::Expression new_expression) {
    for (auto& dim : shape_) {
        dim = symbolic::subs(dim, old_expression, new_expression);
    }
}

void ElementWiseDataflowTensorNode::replace(const symbolic::ExpressionMapping& replacements) {
    for (auto& dim : shape_) {
        dim = symbolic::subs(dim, replacements);
    }
}

std::pair<structured_control_flow::Sequence*, std::vector<symbolic::Expression>> ElementWiseDataflowTensorNode::
    add_eltwise_scope(
        builder::StructuredSDFGBuilder& builder,
        const DebugInfo& scope_deb_info,
        Sequence& parent,
        const std::vector<symbolic::Expression>& shape
    ) {
    // Add maps
    data_flow::Subset new_subset;
    std::vector<symbolic::Expression> loop_vars;
    structured_control_flow::Sequence* last_scope = &parent;
    structured_control_flow::Map* last_map = nullptr;

    for (size_t i = 0; i < shape.size(); i++) {
        std::string indvar_str = builder.find_new_name("_i");
        builder.add_container(indvar_str, types::Scalar(types::PrimitiveType::UInt64));

        auto indvar = symbolic::symbol(indvar_str);
        auto init = symbolic::zero();
        auto update = symbolic::add(indvar, symbolic::one());
        auto condition = symbolic::Lt(indvar, shape.at(i));
        last_map = &builder.add_map(
            *last_scope,
            indvar,
            condition,
            init,
            update,
            structured_control_flow::ScheduleType_Sequential::create(),
            {},
            scope_deb_info
        );
        last_scope = &last_map->root();

        loop_vars.push_back(indvar);
    }
    return {last_scope, loop_vars};
}

std::unique_ptr<types::IType> ElementWiseDataflowTensorNode::access_type(const std::pair<
                                                                         types::PrimitiveType,
                                                                         const TensorLayout*>& pair) {
    if (pair.second) {
        return std::make_unique<types::Tensor>(pair.first, *pair.second);
    } else {
        return std::make_unique<types::Scalar>(pair.first);
    }
}

bool ElementWiseDataflowTensorNode::create_input(
    builder::StructuredSDFGBuilder& builder,
    structured_control_flow::Block& block,
    const data_flow::AccessNode& org_src,
    const std::pair<types::PrimitiveType, const TensorLayout*>& src_type,
    const ElementInput& needed_input,
    const std::vector<symbolic::Expression>& eltwise_subset,
    std::unordered_map<const data_flow::AccessNode*, data_flow::AccessNode*>& new_node_mapping
) {
    auto* new_consumer = needed_input.consumer;
    if (new_consumer) {
        if (src_type.first != needed_input.required_type) {
            throw InvalidSDFGException(
                "Input " + std::to_string(needed_input.input_conn_index) + " on node #" +
                std::to_string(new_consumer->element_id()) + " is required as " +
                types::primitive_type_to_string(needed_input.required_type) + " but provided as " +
                types::primitive_type_to_string(src_type.first)
            );
        }
        auto existing_input_it = new_node_mapping.find(&org_src);
        data_flow::AccessNode* input_node;
        std::vector<symbolic::Expression> empty_subset;
        const std::vector<symbolic::Expression>* memlet_subset;
        if (src_type.second && !src_type.second->is_scalar()) {
            memlet_subset = &eltwise_subset;
        } else {
            memlet_subset = &empty_subset;
        }
        auto new_type = access_type(src_type);
        if (existing_input_it != new_node_mapping.end()) {
            input_node = existing_input_it->second;
        } else {
            if (org_src.is_constant()) {
                types::Scalar const_type(src_type.first);
                input_node = &builder.add_constant(block, org_src.data(), const_type);
            } else {
                input_node = &builder.add_access(block, org_src.data());
            }
            new_node_mapping.emplace(&org_src, input_node);
        }

        builder.add_computational_memlet(
            block,
            *input_node,
            *new_consumer,
            new_consumer->input(needed_input.input_conn_index),
            *memlet_subset,
            *new_type
        );
        return true;
    } else {
        return false;
    }
}

void ElementWiseDataflowTensorNode::create_output(
    builder::StructuredSDFGBuilder& builder,
    structured_control_flow::Block& block,
    const data_flow::AccessNode& org_dst,
    const types::Tensor& dst_type,
    const ElementOutput& provided_output,
    const std::vector<symbolic::Expression>& eltwise_subset
) {
    auto* producer = provided_output.producer;
    if (dst_type.primitive_type() != provided_output.type) {
        throw InvalidSDFGException(
            "Output " + std::to_string(provided_output.output_conn_index) + " on node #" +
            std::to_string(producer->element_id()) + " is provided as " +
            types::primitive_type_to_string(provided_output.type) + " but required as " +
            types::primitive_type_to_string(dst_type.primitive_type())
        );
    }
    auto& output_node = builder.add_access(block, org_dst.data());
    builder.add_computational_memlet(
        block, *producer, producer->output(provided_output.output_conn_index), output_node, eltwise_subset, dst_type
    );
}

bool ElementWiseDataflowTensorNode::
    expand(builder::StructuredSDFGBuilder& builder, analysis::AnalysisManager& analysis_manager) {
    auto& dataflow = this->get_parent();
    auto& org_block = static_cast<structured_control_flow::Block&>(*dataflow.get_parent());

    auto* output_tensor_iedge = dataflow.in_edge_for_connector(*this, inputs_.at(0));
    if (!output_tensor_iedge) {
        return false;
    }
    auto& target_tensor = static_cast<const types::Tensor&>(output_tensor_iedge->base_type());
    std::vector<const data_flow::Memlet*> iedges;
    std::vector<const data_flow::AccessNode*> inputs_sa;
    std::vector<std::pair<types::PrimitiveType, const TensorLayout*>> input_types;
    iedges.reserve(inputs_.size() - 1);
    for (int i = 1; i < this->inputs_.size(); ++i) {
        auto* iedge = dataflow.in_edge_for_connector(*this, inputs_.at(i));
        if (!iedge) {
            if (i < mandatory_input_count()) {
                return false;
            } else {
                continue;
            }
        }
        iedges.push_back(iedge);
        auto* input_sa = dataflow.find_standalone_entry(iedge);
        if (!input_sa) {
            return false;
        }
        inputs_sa.push_back(input_sa);
        auto& input_type = iedge->base_type();
        if (input_type.type_id() == types::TypeID::Scalar) {
            input_types.emplace_back(input_type.primitive_type(), nullptr);
        } else {
            auto& tensor_type = static_cast<const types::Tensor&>(iedge->base_type());
            input_types.emplace_back(input_type.primitive_type(), &tensor_type.layout());
        }
    }

    auto* output_tensor_sa = dataflow.find_standalone_entry(output_tensor_iedge);
    if (!output_tensor_sa) {
        return false;
    }

    auto& parent = static_cast<structured_control_flow::Sequence&>(*org_block.get_parent());
    int index = parent.index(org_block);
    auto& transition = parent.at(index).second;

    // Add new graph after the current block
    auto& new_sequence =
        builder.add_sequence_before(parent, org_block, transition.assignments(), org_block.debug_info());

    auto [eltw_scope, loop_vars] = add_eltwise_scope(builder, org_block.debug_info(), new_sequence, shape_);

    std::vector<tensor::ElementWiseDataflowTensorNode::ElementInput> eltwise_inputs;
    eltwise_inputs.reserve(inputs_.size() - 1);
    for (int i = 0; i < input_types.size(); ++i) {
        eltwise_inputs.push_back({.required_type = input_types.at(i).first});
    }

    auto& new_block = builder.add_block(*eltw_scope);

    auto produced_output =
        expand_operation_dataflow(builder, analysis_manager, new_block, eltwise_inputs, target_tensor.primitive_type());
    if (!produced_output.producer) {
        return false;
    }

    std::unordered_map<const data_flow::AccessNode*, data_flow::AccessNode*> new_node_mapping;

    // for all old input edge, remove old, create new
    for (int i = 0; i < iedges.size(); ++i) {
        create_input(
            builder, new_block, *inputs_sa.at(i), input_types.at(i), eltwise_inputs.at(i), loop_vars, new_node_mapping
        );
    }
    create_output(builder, new_block, *output_tensor_sa, target_tensor, produced_output, loop_vars);
    builder.clear_code_node_legacy(org_block, *this);
    // WARNING: this has been deallocated at this point!!
    builder.remove_child(parent, index + 1);

    return true;
}

data_flow::PointerAccessType ElementWiseDataflowTensorNode::pointer_access_type(int input_idx) const {
    if (input_idx == 0) {
        return data_flow::PointerAccessMeta::create_full_write_only(symbolic::__nullptr__(), true);
    } else if (input_idx < tensor_input_count()) {
        return data_flow::PointerAccessMeta::create_read_only(symbolic::__nullptr__(), true);
    } else {
        return TensorNode::pointer_access_type(input_idx);
    }
}

data_flow::AccessNode& ElementWiseDataflowTensorNode::create_tmp_access_node(
    builder::StructuredSDFGBuilder& builder,
    structured_control_flow::Block& block,
    const std::string& prefix,
    const types::IType& type
) const {
    auto cont = builder.find_new_name(prefix);
    builder.add_container(cont, type);
    auto& output_node_add = builder.add_access(block, cont);
    return output_node_add;
}

nlohmann::json BaseElementWiseDataflowTensorNodeSerializer::serialize(const data_flow::LibraryNode& library_node) {
    const ElementWiseDataflowTensorNode& elem_node = static_cast<const ElementWiseDataflowTensorNode&>(library_node);
    nlohmann::json j;

    j["code"] = elem_node.code().value();

    serializer::JSONSerializer serializer;
    j["shape"] = nlohmann::json::array();
    for (auto& dim : elem_node.shape()) {
        j["shape"].push_back(serializer.expression(dim));
    }

    j["result_quant"] = elem_node.fixed_quantization();

    return j;
}

BaseElementWiseDataflowTensorNodeSerializer::BaseDeser BaseElementWiseDataflowTensorNodeSerializer::
    deserialize_base_values(const nlohmann::json& j) {
    assert(j.contains("element_id"));
    assert(j.contains("code"));
    assert(j.contains("debug_info"));

    std::vector<symbolic::Expression> shape;
    if (j.contains("shape")) {
        for (const auto& dim : j["shape"]) {
            shape.push_back(symbolic::parse(dim.get<std::string>()));
        }
    }

    serializer::JSONSerializer serializer;
    auto debug_info = serializer.json_to_debug_info(j["debug_info"]);
    return {
        .shape = shape,
        .quantization = deserialize_quantization(j, "result_quant", QUANTIZATION_MATCH_INPUTS),
        .debug_info = debug_info
    };
}

} // namespace tensor
} // namespace math
} // namespace sdfg

1	#include "sdfg/data_flow/library_nodes/math/tensor/elementwise_node.h"
2
3	#include "sdfg/analysis/analysis.h"
4	#include "sdfg/builder/structured_sdfg_builder.h"
5	#include "sdfg/data_flow/tasklet.h"
6	#include "sdfg/types/type.h"
7
8	namespace sdfg {
9	namespace math {
10	namespace tensor {
11
12	ElementWiseDataflowTensorNode::ElementWiseDataflowTensorNode(
13	size_t element_id,
14	const DebugInfo& debug_info,
15	const graph::Vertex vertex,
16	data_flow::DataFlowGraph& parent,
17	const data_flow::LibraryNodeCode& code,
18	const std::vector<symbolic::Expression>& shape,
19	const std::string& modified_tensor_conn,
20	const std::vector<std::string>& tensor_inputs,
21	QuantizationType quantization,
22	const data_flow::ImplementationType& impl_type
23	)
24	: TensorNode(	550✔
25	element_id,	550✔
26	debug_info,	550✔
27	vertex,	550✔
28	parent,	550✔
29	code,	550✔
30	{},	550✔
31	build_input_conns(modified_tensor_conn, tensor_inputs),	550✔
32	impl_type	550✔
33	),	550✔
34	fixed_quantization_(quantization), shape_(shape) {}	550✔
35
36	std::vector<std::string> ElementWiseDataflowTensorNode::
37	build_input_conns(const std::string& modified_tensor_conn, const std::vector<std::string>& tensor_inputs) {	550✔
38	std::vector<std::string> input_conns;	550✔
39	input_conns.reserve(1 + input_conns.size());	550✔
40	input_conns.push_back(modified_tensor_conn);	550✔
41	input_conns.insert(input_conns.end(), tensor_inputs.begin(), tensor_inputs.end());	550✔
42	return input_conns;	550✔
43	}	550✔
44
45	types::PrimitiveType ElementWiseDataflowTensorNode::fixed_quantization() const { return fixed_quantization_; }	×
46
47	void ElementWiseDataflowTensorNode::set_fixed_quantization(const QuantizationType quant) {	×
48	fixed_quantization_ = quant;	×
49	}	×
50
51	types::PrimitiveType ElementWiseDataflowTensorNode::quantization(const data_flow::DataFlowGraph& data_flow_graph
52	) const {	×
53	if (fixed_quantization_ != QUANTIZATION_MATCH_INPUTS) {	×
54	return fixed_quantization_;	×
55	} else {	×
56	return this->primitive_type(data_flow_graph);	×
57	}	×
58	}	×
59
60	std::optional<types::PrimitiveType> ElementWiseDataflowTensorNode::uniform_quantization(const data_flow::DataFlowGraph&
61	data_flow_graph) const {	×
62	if (fixed_quantization_ != QUANTIZATION_MATCH_INPUTS) {	×
63	auto inferred = this->primitive_type(data_flow_graph);	×
64	if (inferred == fixed_quantization_) {	×
65	return fixed_quantization_;	×
66	} else {	×
67	return std::nullopt;	×
68	}	×
69	} else {	×
70	return this->primitive_type(data_flow_graph);	×
71	}	×
72	}	×
73
74	void ElementWiseDataflowTensorNode::validate_target_tensor(const data_flow::DataFlowGraph& graph) const {	556✔
75	auto* target_ptr_edge = graph.in_edge_for_connector(*this, inputs_.at(0));	556✔
76	auto& tensor_output = static_cast<const types::Tensor&>(target_ptr_edge->base_type());	556✔
77
78	validate_shape_matches(shape_, tensor_output.layout(), "output tensor");	556✔
79	}	556✔
80
81	void ElementWiseDataflowTensorNode::validate_all_input_tensors(const data_flow::DataFlowGraph& graph) const {	584✔
82	for (int i = 1; i < tensor_input_count(); ++i) {	1,457✔
83	auto* iedge = graph.in_edge_for_connector(*this, inputs_.at(i));	873✔
84	if (!iedge) {	873✔
85	throw InvalidSDFGException(	×
86	"On libNode #" + std::to_string(element_id()) + ": input " + inputs_.at(i) + " is not connected"	×
87	);	×
88	}	×
89	if (iedge->base_type().type_id() == types::TypeID::Scalar) {	873✔
90	continue;	×
91	}	×
92	auto& tensor_input = static_cast<const types::Tensor&>(iedge->base_type());	873✔
93	// Case 1: Scalar input is allowed as secondary input
94	if (tensor_input.is_scalar()) {	873✔
95	continue;	7✔
96	}	7✔
97
98	// currently no arbitrary broadcast support! but could be added
99	validate_shape_matches(shape_, tensor_input.layout(), "input " + inputs_.at(i));	866✔
100	}	866✔
101	}	584✔
102
103	void ElementWiseDataflowTensorNode::validate_non_tensor_inputs(const data_flow::DataFlowGraph& graph) const {	328✔
104	for (int i = tensor_input_count(); i < inputs_.size(); ++i) {	328✔
105	auto* iedge = graph.in_edge_for_connector(*this, inputs_.at(i));	×
106	if (!iedge) {	×
107	if (i < mandatory_input_count()) {	×
108	throw InvalidSDFGException(	×
109	"On libNode #" + std::to_string(element_id()) + ": input " + inputs_.at(i) + " is not connected"	×
110	);	×
111	} else {	×
112	continue;	×
113	}	×
114	}	×
115	if (iedge->base_type().type_id() != types::TypeID::Scalar) {	×
116	throw InvalidSDFGException(	×
117	"On libNode #" + std::to_string(element_id()) + ": input " + inputs_.at(i) + " is not scalar"	×
118	);	×
119	}	×
120	}	×
121	}	328✔
122
123	void ElementWiseDataflowTensorNode::validate(const Function& function) const {	330✔
124	TensorNode::validate(function);	330✔
125
126	auto& graph = this->get_parent();	330✔
127
128	validate_target_tensor(graph);	330✔
129
130	validate_all_input_tensors(graph);	330✔
131
132	validate_non_tensor_inputs(graph);	330✔
133	}	330✔
134
135	symbolic::SymbolSet ElementWiseDataflowTensorNode::symbols() const {	17✔
136	symbolic::SymbolSet syms;	17✔
137	for (const auto& dim : shape_) {	68✔
138	for (auto& atom : symbolic::atoms(dim)) {	68✔
139	syms.insert(atom);	×
140	}	×
141	}	68✔
142	return syms;	17✔
143	}	17✔
144
145	void ElementWiseDataflowTensorNode::
146	replace(const symbolic::Expression old_expression, const symbolic::Expression new_expression) {	×
147	for (auto& dim : shape_) {	×
148	dim = symbolic::subs(dim, old_expression, new_expression);	×
149	}	×
150	}	×
151
NEW 152	void ElementWiseDataflowTensorNode::replace(const symbolic::ExpressionMapping& replacements) {	×
NEW 153	for (auto& dim : shape_) {	×
NEW 154	dim = symbolic::subs(dim, replacements);	×
NEW 155	}	×
NEW 156	}	×
157
158	std::pair<structured_control_flow::Sequence*, std::vector<symbolic::Expression>> ElementWiseDataflowTensorNode::
159	add_eltwise_scope(
160	builder::StructuredSDFGBuilder& builder,
161	const DebugInfo& scope_deb_info,
162	Sequence& parent,
163	const std::vector<symbolic::Expression>& shape
164	) {	521✔
165	// Add maps
166	data_flow::Subset new_subset;	521✔
167	std::vector<symbolic::Expression> loop_vars;	521✔
168	structured_control_flow::Sequence* last_scope = &parent;	521✔
169	structured_control_flow::Map* last_map = nullptr;	521✔
170
171	for (size_t i = 0; i < shape.size(); i++) {	1,798✔
172	std::string indvar_str = builder.find_new_name("_i");	1,277✔
173	builder.add_container(indvar_str, types::Scalar(types::PrimitiveType::UInt64));	1,277✔
174
175	auto indvar = symbolic::symbol(indvar_str);	1,277✔
176	auto init = symbolic::zero();	1,277✔
177	auto update = symbolic::add(indvar, symbolic::one());	1,277✔
178	auto condition = symbolic::Lt(indvar, shape.at(i));	1,277✔
179	last_map = &builder.add_map(	1,277✔
180	*last_scope,	1,277✔
181	indvar,	1,277✔
182	condition,	1,277✔
183	init,	1,277✔
184	update,	1,277✔
185	structured_control_flow::ScheduleType_Sequential::create(),	1,277✔
186	{},	1,277✔
187	scope_deb_info	1,277✔
188	);	1,277✔
189	last_scope = &last_map->root();	1,277✔
190
191	loop_vars.push_back(indvar);	1,277✔
192	}	1,277✔
193	return {last_scope, loop_vars};	521✔
194	}	521✔
195
196	std::unique_ptr<types::IType> ElementWiseDataflowTensorNode::access_type(const std::pair<
197	types::PrimitiveType,
198	const TensorLayout*>& pair) {	812✔
199	if (pair.second) {	812✔
200	return std::make_unique<types::Tensor>(pair.first, *pair.second);	812✔
201	} else {	812✔
202	return std::make_unique<types::Scalar>(pair.first);	×
203	}	×
204	}	812✔
205
206	bool ElementWiseDataflowTensorNode::create_input(
207	builder::StructuredSDFGBuilder& builder,
208	structured_control_flow::Block& block,
209	const data_flow::AccessNode& org_src,
210	const std::pair<types::PrimitiveType, const TensorLayout*>& src_type,
211	const ElementInput& needed_input,
212	const std::vector<symbolic::Expression>& eltwise_subset,
213	std::unordered_map<const data_flow::AccessNode, data_flow::AccessNode>& new_node_mapping
214	) {	812✔
215	auto* new_consumer = needed_input.consumer;	812✔
216	if (new_consumer) {	812✔
217	if (src_type.first != needed_input.required_type) {	812✔
218	throw InvalidSDFGException(	×
219	"Input " + std::to_string(needed_input.input_conn_index) + " on node #" +	×
220	std::to_string(new_consumer->element_id()) + " is required as " +	×
221	types::primitive_type_to_string(needed_input.required_type) + " but provided as " +	×
222	types::primitive_type_to_string(src_type.first)	×
223	);	×
224	}	×
225	auto existing_input_it = new_node_mapping.find(&org_src);	812✔
226	data_flow::AccessNode* input_node;	812✔
227	std::vector<symbolic::Expression> empty_subset;	812✔
228	const std::vector<symbolic::Expression>* memlet_subset;	812✔
229	if (src_type.second && !src_type.second->is_scalar()) {	812✔
230	memlet_subset = &eltwise_subset;	812✔
231	} else {	812✔
232	memlet_subset = &empty_subset;	×
233	}	×
234	auto new_type = access_type(src_type);	812✔
235	if (existing_input_it != new_node_mapping.end()) {	812✔
236	input_node = existing_input_it->second;	×
237	} else {	812✔
238	if (org_src.is_constant()) {	812✔
239	types::Scalar const_type(src_type.first);	×
240	input_node = &builder.add_constant(block, org_src.data(), const_type);	×
241	} else {	812✔
242	input_node = &builder.add_access(block, org_src.data());	812✔
243	}	812✔
244	new_node_mapping.emplace(&org_src, input_node);	812✔
245	}	812✔
246
247	builder.add_computational_memlet(	812✔
248	block,	812✔
249	*input_node,	812✔
250	*new_consumer,	812✔
251	new_consumer->input(needed_input.input_conn_index),	812✔
252	*memlet_subset,	812✔
253	*new_type	812✔
254	);	812✔
255	return true;	812✔
256	} else {	812✔
257	return false;	×
258	}	×
259	}	812✔
260
261	void ElementWiseDataflowTensorNode::create_output(
262	builder::StructuredSDFGBuilder& builder,
263	structured_control_flow::Block& block,
264	const data_flow::AccessNode& org_dst,
265	const types::Tensor& dst_type,
266	const ElementOutput& provided_output,
267	const std::vector<symbolic::Expression>& eltwise_subset
268	) {	521✔
269	auto* producer = provided_output.producer;	521✔
270	if (dst_type.primitive_type() != provided_output.type) {	521✔
271	throw InvalidSDFGException(	×
272	"Output " + std::to_string(provided_output.output_conn_index) + " on node #" +	×
273	std::to_string(producer->element_id()) + " is provided as " +	×
274	types::primitive_type_to_string(provided_output.type) + " but required as " +	×
275	types::primitive_type_to_string(dst_type.primitive_type())	×
276	);	×
277	}	×
278	auto& output_node = builder.add_access(block, org_dst.data());	521✔
279	builder.add_computational_memlet(	521✔
280	block, *producer, producer->output(provided_output.output_conn_index), output_node, eltwise_subset, dst_type	521✔
281	);	521✔
282	}	521✔
283
284	bool ElementWiseDataflowTensorNode::
285	expand(builder::StructuredSDFGBuilder& builder, analysis::AnalysisManager& analysis_manager) {	521✔
286	auto& dataflow = this->get_parent();	521✔
287	auto& org_block = static_cast<structured_control_flow::Block&>(*dataflow.get_parent());	521✔
288
289	auto* output_tensor_iedge = dataflow.in_edge_for_connector(*this, inputs_.at(0));	521✔
290	if (!output_tensor_iedge) {	521✔
291	return false;	×
292	}	×
293	auto& target_tensor = static_cast<const types::Tensor&>(output_tensor_iedge->base_type());	521✔
294	std::vector<const data_flow::Memlet*> iedges;	521✔
295	std::vector<const data_flow::AccessNode*> inputs_sa;	521✔
296	std::vector<std::pair<types::PrimitiveType, const TensorLayout*>> input_types;	521✔
297	iedges.reserve(inputs_.size() - 1);	521✔
298	for (int i = 1; i < this->inputs_.size(); ++i) {	1,333✔
299	auto* iedge = dataflow.in_edge_for_connector(*this, inputs_.at(i));	812✔
300	if (!iedge) {	812✔
301	if (i < mandatory_input_count()) {	×
302	return false;	×
303	} else {	×
304	continue;	×
305	}	×
306	}	×
307	iedges.push_back(iedge);	812✔
308	auto* input_sa = dataflow.find_standalone_entry(iedge);	812✔
309	if (!input_sa) {	812✔
310	return false;	×
311	}	×
312	inputs_sa.push_back(input_sa);	812✔
313	auto& input_type = iedge->base_type();	812✔
314	if (input_type.type_id() == types::TypeID::Scalar) {	812✔
315	input_types.emplace_back(input_type.primitive_type(), nullptr);	×
316	} else {	812✔
317	auto& tensor_type = static_cast<const types::Tensor&>(iedge->base_type());	812✔
318	input_types.emplace_back(input_type.primitive_type(), &tensor_type.layout());	812✔
319	}	812✔
320	}	812✔
321
322	auto* output_tensor_sa = dataflow.find_standalone_entry(output_tensor_iedge);	521✔
323	if (!output_tensor_sa) {	521✔
324	return false;	×
325	}	×
326
327	auto& parent = static_cast<structured_control_flow::Sequence&>(*org_block.get_parent());	521✔
328	int index = parent.index(org_block);	521✔
329	auto& transition = parent.at(index).second;	521✔
330
331	// Add new graph after the current block
332	auto& new_sequence =	521✔
333	builder.add_sequence_before(parent, org_block, transition.assignments(), org_block.debug_info());	521✔
334
335	auto [eltw_scope, loop_vars] = add_eltwise_scope(builder, org_block.debug_info(), new_sequence, shape_);	521✔
336
337	std::vector<tensor::ElementWiseDataflowTensorNode::ElementInput> eltwise_inputs;	521✔
338	eltwise_inputs.reserve(inputs_.size() - 1);	521✔
339	for (int i = 0; i < input_types.size(); ++i) {	1,333✔
340	eltwise_inputs.push_back({.required_type = input_types.at(i).first});	812✔
341	}	812✔
342
343	auto& new_block = builder.add_block(*eltw_scope);	521✔
344
345	auto produced_output =	521✔
346	expand_operation_dataflow(builder, analysis_manager, new_block, eltwise_inputs, target_tensor.primitive_type());	521✔
347	if (!produced_output.producer) {	521✔
348	return false;	×
349	}	×
350
351	std::unordered_map<const data_flow::AccessNode, data_flow::AccessNode> new_node_mapping;	521✔
352
353	// for all old input edge, remove old, create new
354	for (int i = 0; i < iedges.size(); ++i) {	1,333✔
355	create_input(	812✔
356	builder, new_block, *inputs_sa.at(i), input_types.at(i), eltwise_inputs.at(i), loop_vars, new_node_mapping	812✔
357	);	812✔
358	}	812✔
359	create_output(builder, new_block, *output_tensor_sa, target_tensor, produced_output, loop_vars);	521✔
360	builder.clear_code_node_legacy(org_block, *this);	521✔
361	// WARNING: this has been deallocated at this point!!
362	builder.remove_child(parent, index + 1);	521✔
363
364	return true;	521✔
365	}	521✔
366
367	data_flow::PointerAccessType ElementWiseDataflowTensorNode::pointer_access_type(int input_idx) const {	×
368	if (input_idx == 0) {	×
369	return data_flow::PointerAccessMeta::create_full_write_only(symbolic::__nullptr__(), true);	×
370	} else if (input_idx < tensor_input_count()) {	×
371	return data_flow::PointerAccessMeta::create_read_only(symbolic::__nullptr__(), true);	×
372	} else {	×
373	return TensorNode::pointer_access_type(input_idx);	×
374	}	×
375	}	×
376
377	data_flow::AccessNode& ElementWiseDataflowTensorNode::create_tmp_access_node(
378	builder::StructuredSDFGBuilder& builder,
379	structured_control_flow::Block& block,
380	const std::string& prefix,
381	const types::IType& type
382	) const {	12✔
383	auto cont = builder.find_new_name(prefix);	12✔
384	builder.add_container(cont, type);	12✔
385	auto& output_node_add = builder.add_access(block, cont);	12✔
386	return output_node_add;	12✔
387	}	12✔
388
389	nlohmann::json BaseElementWiseDataflowTensorNodeSerializer::serialize(const data_flow::LibraryNode& library_node) {	×
390	const ElementWiseDataflowTensorNode& elem_node = static_cast<const ElementWiseDataflowTensorNode&>(library_node);	×
391	nlohmann::json j;	×
392
393	j["code"] = elem_node.code().value();	×
394
395	serializer::JSONSerializer serializer;	×
396	j["shape"] = nlohmann::json::array();	×
397	for (auto& dim : elem_node.shape()) {	×
398	j["shape"].push_back(serializer.expression(dim));	×
399	}	×
400
401	j["result_quant"] = elem_node.fixed_quantization();	×
402
403	return j;	×
404	}	×
405
406	BaseElementWiseDataflowTensorNodeSerializer::BaseDeser BaseElementWiseDataflowTensorNodeSerializer::
407	deserialize_base_values(const nlohmann::json& j) {	×
408	assert(j.contains("element_id"));	×
409	assert(j.contains("code"));	×
410	assert(j.contains("debug_info"));	×
411
412	std::vector<symbolic::Expression> shape;	×
413	if (j.contains("shape")) {	×
414	for (const auto& dim : j["shape"]) {	×
415	shape.push_back(symbolic::parse(dim.get<std::string>()));	×
416	}	×
417	}	×
418
419	serializer::JSONSerializer serializer;	×
420	auto debug_info = serializer.json_to_debug_info(j["debug_info"]);	×
421	return {	×
422	.shape = shape,	×
423	.quantization = deserialize_quantization(j, "result_quant", QUANTIZATION_MATCH_INPUTS),	×
424	.debug_info = debug_info	×
425	};	×
426	}	×
427
428	} // namespace tensor
429	} // namespace math
430	} // namespace sdfg

daisytuner / docc / 27981272983

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous