24517813654

Committed 16 Apr 2026 03:05PM UTC coverage: 64.286% (-0.2%) from 64.503%

Build # 24517813654

Build Type

Pull #683

github

Committed by

web-flow

Commit Message

Merge 3803436ab into e165e5156

Pull Request Pull Request #683: Tuned ROCM Gemm

Coverage Stats

7 of 174 new or added lines in 3 files covered. (4.02%)

129 existing lines in 6 files now uncovered.

30845 of 47981 relevant lines covered (64.29%)

578.73 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.36

/opt/src/analysis/data_transfer_elimination_analysis.cpp

#include "sdfg/analysis/data_transfer_elimination_analysis.h"


#include "sdfg/targets/cuda/cuda.h"
#include "sdfg/targets/offloading/data_offloading_node.h"

namespace sdfg::analysis {

void OffloadHolder::remove_h2d_parts() {
    host_data = nullptr;
    host_access = nullptr;
    updates_on_dev = false;
}

void OffloadHolder::remove_d2h_parts() {
    host_data = nullptr;
    host_access = nullptr;
    updates_on_host = false;
}

OffloadState::OffloadState(DataTransferEliminationCandidateCollector& collector) : collector_(collector) {}

void OffloadState::found_escape(const std::string& container) { kills_containers_.insert(container); }

void OffloadState::found_ptr_write(const std::string& container, const data_flow::Memlet* memlet) {
    kills_containers_.insert(container);
}

void OffloadState::found_ptr_read(const std::string& container, const data_flow::Memlet* memlet) {
    // todo check with generated set if its ever possible for it to contain the transfer and a use
    reads_[container].push_back(memlet);
}

void OffloadState::found_full_barrier(ControlFlowNode& node) {
    generated_.clear(); // all generateds are from before and now wiped
    full_kill_ = true;
}

/**
 * @return { type, killing node }
 */
std::pair<OffloadState::KillingType, OffloadHolder*> OffloadState::find_killing_entry_node(const ExposedOffload&
                                                                                               in_flight) const {
    auto& holder = *in_flight.offload;
    static const types::Scalar void_type(types::Void);
    auto& host_access_type = holder.host_access ? holder.host_access->base_type() : void_type;

    auto* offload_node = holder.offload_node;
    auto* malloc_node = holder.malloc_node;

    if (holder.ends_dev_lifetime || holder.updates_on_host || malloc_node) {
        for (const auto& entry : h2d_nodes_ | std::views::values) {
            auto& entry_holder = *entry;
            auto& entry_host_access_type = entry_holder.host_access ? entry_holder.host_access->base_type() : void_type;
            bool host_ptr_matches = entry_holder.host_data && in_flight.container == entry_holder.host_data->data();

            if (host_ptr_matches) {
                if (offload_node && (holder.updates_on_host || holder.ends_dev_lifetime)) {
                    // D2H -> H2D
                    bool is_elim_candidate = holder.offload_node->is_compatible_with(*entry_holder.offload_node) &&
                                             entry_holder.updates_on_dev;
                    return {is_elim_candidate ? KillingType::DeviceReuse : KillingType::Basic, &entry_holder};
                } else if (malloc_node) {
                    // Malloc -> H2D
                    // mallocs should only be in flight as long as they are untouched
                    bool can_be_removed = entry_holder.offload_node->is_alloc() && entry_holder.offload_node->is_h2d();
                    return {can_be_removed ? KillingType::EmptyHostMalloc : KillingType::Basic, &entry_holder};
                } else {
                    return {KillingType::Basic, &entry_holder};
                }
            } else if (host_access_type == entry_host_access_type) { // aliases
                // any -> any with aliasing types
                // return &entry_node; // TODO left unhandled for now, because then most situations like a matmul could
                // never be eliminated
            }
        }
    } else if (holder.starts_dev_lifetime) {
        for (const auto& entry : generated_ | std::views::values) {
            auto& entry_holder = *entry;

            bool dev_ptr_matches = false;
            if (holder.dev_data && entry_holder.dev_data) {
                dev_ptr_matches = in_flight.container == entry_holder.dev_data->data();
            }

            if (dev_ptr_matches) {
                // D_ALLOC -> D_FREE is the expected case, but kill for any match
                KillingType killType = KillingType::Basic;
                if (holder.offload_node->is_compatible_with(*entry_holder.offload_node) &&
                    entry_holder.ends_dev_lifetime) {
                    if (entry_holder.updates_on_host) {
                        killType = KillingType::RedundantD2H;
                    } else {
                        killType = KillingType::DeviceCleanFree;
                    }
                }

                return {killType, &entry_holder};
            }
        }
    }

    return {KillingType::None, nullptr};
}

void OffloadState::found_malloc(Block& block, stdlib::MallocNode& malloc) {
    auto& dflow = block.dataflow();

    auto out_edges = dflow.out_edges_for_connector(malloc, malloc.output(0));
    if (out_edges.size() != 1) {
        throw std::runtime_error(
            "Unsupported: malloc node " + std::to_string(malloc.element_id()) + " with other than 1 output"
        );
    }
    auto* memlet = out_edges.at(0);
    auto* access_node = dynamic_cast<const data_flow::AccessNode*>(&memlet->dst());
    generated_.emplace(malloc.element_id(), std::make_unique<OffloadHolder>(&malloc, access_node, memlet));
}

void OffloadState::found_offload_node(Block& block, offloading::DataOffloadingNode& offload) {
    auto& dflow = block.dataflow();

    bool src_is_dev = false;
    bool src_is_host = false;
    bool dst_is_dev = false;
    bool dst_is_host = false;

    bool starts_dev_lifetime = false;
    bool ends_dev_lifetime = false;
    bool updates_on_dev = false;
    bool updates_on_host = false;

    auto transfer_direction = offload.transfer_direction();
    auto lifecycle = offload.buffer_lifecycle();
    if (transfer_direction == offloading::DataTransferDirection::D2H) {
        src_is_dev = true;
        dst_is_host = true;
        updates_on_host = true;
        if (lifecycle == offloading::BufferLifecycle::FREE) {
            ends_dev_lifetime = true;
        }
    } else if (transfer_direction == offloading::DataTransferDirection::H2D) {
        src_is_host = true;
        dst_is_dev = true;
        updates_on_dev = true;
        if (lifecycle == offloading::BufferLifecycle::ALLOC) {
            starts_dev_lifetime = true;
        }
    } else if (offloading::is_NONE(transfer_direction)) {
        if (lifecycle == offloading::BufferLifecycle::ALLOC) {
            starts_dev_lifetime = true;
            dst_is_dev = true;
        } else if (lifecycle == offloading::BufferLifecycle::FREE) {
            ends_dev_lifetime = true;
            src_is_dev = true;
        }
    }

    const data_flow::AccessNode* found_dev_access = nullptr;
    const data_flow::AccessNode* found_host_access = nullptr;
    const data_flow::Memlet* found_host_memlet = nullptr;

    for (auto& conn : offload.inputs()) {
        auto* memlet = dflow.in_edge_for_connector(offload, conn);
        auto* access_node = dynamic_cast<const data_flow::AccessNode*>(&memlet->src());

        if (src_is_host) {
            found_host_access = access_node;
            found_host_memlet = memlet;
        }
        if (src_is_dev) {
            found_dev_access = access_node;
        }
    }

    for (auto& conn : offload.outputs()) {
        auto edges = dflow.out_edges_for_connector(offload, conn);
        if (edges.size() > 1) {
            throw std::runtime_error(
                "Unsupported: offload node " + std::to_string(offload.element_id()) +
                " with multiple outputs edges on " + conn
            );
        }
        auto* memlet = edges.at(0);
        auto* access_node = dynamic_cast<const data_flow::AccessNode*>(&memlet->dst());

        if (dst_is_host) {
            found_host_access = access_node;
            found_host_memlet = memlet;
        }
        if (dst_is_dev) {
            found_dev_access = access_node;
        }
    }

    if (ends_dev_lifetime || updates_on_host) {
        generated_.emplace(
            offload.element_id(),
            std::make_unique<OffloadHolder>(
                &offload,
                found_host_access,
                found_host_memlet,
                found_dev_access,
                starts_dev_lifetime,
                ends_dev_lifetime,
                updates_on_dev,
                updates_on_host
            )
        );
    } else if (starts_dev_lifetime || updates_on_dev) {
        add_h2d_entry(OffloadHolder{
            &offload,
            found_host_access,
            found_host_memlet,
            found_dev_access,
            starts_dev_lifetime,
            ends_dev_lifetime,
            updates_on_dev,
            updates_on_host
        });
    }
}

void OffloadState::add_h2d_entry(const OffloadHolder& entry) {
    h2d_nodes_.emplace(entry.offload_node->element_id(), std::make_unique<OffloadHolder>(entry));
    // todo also need to remove generated ones killed by this. But right now, only max 1 per block anyway
}

void OffloadState::apply_kills_and_changes(ExposedType& exposed) const {
    if (full_kill_) {
        exposed.clear();
        return;
    }
    std::list<ExposedOffload> dynamic_inserts;
    for (auto it = exposed.begin(); it != exposed.end();) {
        auto& [id, exposedOffload] = *it;
        auto& holder = *exposedOffload.offload;


        auto container_reads = reads_.find(exposedOffload.container);
        if (container_reads != reads_.end() && !container_reads->second.empty()) {
            if (holder.malloc_node) { // mallocs are just killed on first use
                DEBUG_PRINTLN(
                    "In-flight malloc area of #" << holder.malloc_node->element_id()
                                                 << " is read without being initialized!"
                );
                it = exposed.erase(it);
                continue;
            } else { // track if a live var is read
                ++exposedOffload.read_count;
            }
        }

        if (kills_containers_.contains(exposedOffload.container)) {
            it = exposed.erase(it);
            continue;
        }

        auto [kill_type, killing_entry] = find_killing_entry_node(exposedOffload);
        if (kill_type != KillingType::None) {
            if (kill_type == KillingType::DeviceReuse) {
                collector_.found_transfer_reuse_pair(exposedOffload, *killing_entry);
            } else if (kill_type == KillingType::EmptyHostMalloc) {
                collector_.found_empty_host_malloc(exposedOffload, *killing_entry);
            } else if (kill_type == KillingType::DeviceCleanFree) {
                // we have a on-device-alloc that survived without kills to the on-device-free
                // -> promote this to a host-relevant D2H point, that might be reused

                // replace the current H2D with the "D2H" that would allow it to live on
                // this creates a D2H-like exposedOffload, despite us not knowing the host-var at this point
                auto* host_data = holder.host_data;
                if (host_data) {
                    dynamic_inserts.emplace_back(killing_entry, host_data->data(), 0);
                    it = exposed.erase(it);
                    continue;
                }
            } else if (kill_type == KillingType::RedundantD2H) {
                collector_.found_redundant_d2h_pair(exposedOffload, *killing_entry);
            }
            it = exposed.erase(it);
            continue;
        }
        ++it;
    }

    for (auto& [id, gen] : generated_) {
        auto* holder = gen.get();
        if (holder->updates_on_host || holder->malloc_node) { // block unidentified host-container ones from being
                                                              // exposed. If we could reconstruct, it will be a
                                                              // dynamic_insert
            exposed.insert({id, ExposedOffload{holder, holder->host_data->data(), 0}});
        }
    }
    for (auto& gen : dynamic_inserts) {
        exposed.insert({gen.offload->offload_node->element_id(), gen});
    }
    for (auto& [id, gen] : h2d_nodes_) {
        auto* holder = gen.get();
        if (holder->starts_dev_lifetime) {
            exposed.insert({id, ExposedOffload{holder, holder->dev_data->data(), 0}});
        }
    }
}


void DataTransferEliminationAnalysis::handle_lib_node(Block& block, data_flow::LibraryNode& node) {
    BaseUserVisitor::handle_lib_node(block, node);

    if (auto* offload_node = dynamic_cast<offloading::DataOffloadingNode*>(&node)) {
        get_or_create_state(block).found_offload_node(block, *offload_node);
    } else if (auto* malloc_node = dynamic_cast<stdlib::MallocNode*>(&node)) {
        get_or_create_state(block).found_malloc(block, *malloc_node);
    }
}

void DataTransferEliminationAnalysis::handle_structured_loop_before_body(StructuredLoop& loop) {
    BaseUserVisitor::handle_structured_loop_before_body(loop);

    // auto* map = dynamic_cast<sdfg::structured_control_flow::Map*>(&loop);

    // if (map && map->schedule_type().category() == ScheduleTypeCategory::Offloader) {
    //     get_or_create_state(loop).found_offloaded_kernel(*map);
    // }
}

void DataTransferEliminationAnalysis::
    on_escape(const std::string& container, const ControlFlowNode* node, const Element* user) {
    if (dynamic_cast<const Block*>(node)) {
        if (auto* memlet = dynamic_cast<const data_flow::Memlet*>(user)) {
            if (auto* offload = dynamic_cast<const offloading::DataOffloadingNode*>(&memlet->dst())) {
                // accesses of offloading nodes are handled more intelligently in found_offload_node, so ignore them
                // here
                return;
            }
        }
    }
    get_or_create_state(*node).found_escape(container);
}

void DataTransferEliminationAnalysis::
    on_read_via(const std::string& container, const ControlFlowNode* node, const data_flow::Memlet* user) {
    if (!dynamic_cast<const offloading::DataOffloadingNode*>(&user->dst())) {
        get_or_create_state(*node).found_ptr_read(container, user);
    }
}

void DataTransferEliminationAnalysis::
    on_write_via(const std::string& container, const ControlFlowNode* node, const data_flow::Memlet* user) {
    if (!dynamic_cast<const offloading::DataOffloadingNode*>(&user->dst())) {
        get_or_create_state(*node).found_ptr_write(container, user);
    }
}

std::unique_ptr<OffloadState> DataTransferEliminationAnalysis::
    create_initial_state(const structured_control_flow::ControlFlowNode& node) {
    return std::make_unique<OffloadState>(*this);
}

void DataTransferEliminationAnalysis::run() {
    dispatch(sdfg_.root());

    run_forward(sdfg_.root());
}

} // namespace sdfg::analysis

1	#include "sdfg/analysis/data_transfer_elimination_analysis.h"
2
3
4	#include "sdfg/targets/cuda/cuda.h"
5	#include "sdfg/targets/offloading/data_offloading_node.h"
6
7	namespace sdfg::analysis {
8
9	void OffloadHolder::remove_h2d_parts() {	3✔
10	host_data = nullptr;	3✔
11	host_access = nullptr;	3✔
12	updates_on_dev = false;	3✔
13	}	3✔
14
15	void OffloadHolder::remove_d2h_parts() {	5✔
16	host_data = nullptr;	5✔
17	host_access = nullptr;	5✔
18	updates_on_host = false;	5✔
19	}	5✔
20
21	OffloadState::OffloadState(DataTransferEliminationCandidateCollector& collector) : collector_(collector) {}	50✔
22
23	void OffloadState::found_escape(const std::string& container) { kills_containers_.insert(container); }	4✔
24
25	void OffloadState::found_ptr_write(const std::string& container, const data_flow::Memlet* memlet) {	3✔
26	kills_containers_.insert(container);	3✔
27	}	3✔
28
29	void OffloadState::found_ptr_read(const std::string& container, const data_flow::Memlet* memlet) {	3✔
30	// todo check with generated set if its ever possible for it to contain the transfer and a use
31	reads_[container].push_back(memlet);	3✔
32	}	3✔
33
UNCOV 34	void OffloadState::found_full_barrier(ControlFlowNode& node) {	×
UNCOV 35	generated_.clear(); // all generateds are from before and now wiped	×
UNCOV 36	full_kill_ = true;	×
UNCOV 37	}	×
38
39	/**
40	* @return { type, killing node }
41	*/
42	std::pair<OffloadState::KillingType, OffloadHolder*> OffloadState::find_killing_entry_node(const ExposedOffload&
43	in_flight) const {	20✔
44	auto& holder = *in_flight.offload;	20✔
45	static const types::Scalar void_type(types::Void);	20✔
46	auto& host_access_type = holder.host_access ? holder.host_access->base_type() : void_type;	20✔
47
48	auto* offload_node = holder.offload_node;	20✔
49	auto* malloc_node = holder.malloc_node;	20✔
50
51	if (holder.ends_dev_lifetime \|\| holder.updates_on_host \|\| malloc_node) {	20✔
52	for (const auto& entry : h2d_nodes_ \| std::views::values) {	10✔
53	auto& entry_holder = *entry;	10✔
54	auto& entry_host_access_type = entry_holder.host_access ? entry_holder.host_access->base_type() : void_type;	10✔
55	bool host_ptr_matches = entry_holder.host_data && in_flight.container == entry_holder.host_data->data();	10✔
56
57	if (host_ptr_matches) {	10✔
58	if (offload_node && (holder.updates_on_host \|\| holder.ends_dev_lifetime)) {	10✔
59	// D2H -> H2D
60	bool is_elim_candidate = holder.offload_node->is_compatible_with(*entry_holder.offload_node) &&	7✔
61	entry_holder.updates_on_dev;	7✔
62	return {is_elim_candidate ? KillingType::DeviceReuse : KillingType::Basic, &entry_holder};	7✔
63	} else if (malloc_node) {	7✔
64	// Malloc -> H2D
65	// mallocs should only be in flight as long as they are untouched
66	bool can_be_removed = entry_holder.offload_node->is_alloc() && entry_holder.offload_node->is_h2d();	3✔
67	return {can_be_removed ? KillingType::EmptyHostMalloc : KillingType::Basic, &entry_holder};	3✔
68	} else {	3✔
UNCOV 69	return {KillingType::Basic, &entry_holder};	×
UNCOV 70	}	×
71	} else if (host_access_type == entry_host_access_type) { // aliases	10✔
72	// any -> any with aliasing types
73	// return &entry_node; // TODO left unhandled for now, because then most situations like a matmul could
74	// never be eliminated
UNCOV 75	}	×
76	}	10✔
77	} else if (holder.starts_dev_lifetime) {	10✔
78	for (const auto& entry : generated_ \| std::views::values) {	10✔
79	auto& entry_holder = *entry;	9✔
80
81	bool dev_ptr_matches = false;	9✔
82	if (holder.dev_data && entry_holder.dev_data) {	9✔
83	dev_ptr_matches = in_flight.container == entry_holder.dev_data->data();	9✔
84	}	9✔
85
86	if (dev_ptr_matches) {	9✔
87	// D_ALLOC -> D_FREE is the expected case, but kill for any match
88	KillingType killType = KillingType::Basic;	9✔
89	if (holder.offload_node->is_compatible_with(*entry_holder.offload_node) &&	9✔
90	entry_holder.ends_dev_lifetime) {	9✔
91	if (entry_holder.updates_on_host) {	9✔
92	killType = KillingType::RedundantD2H;	7✔
93	} else {	7✔
94	killType = KillingType::DeviceCleanFree;	2✔
95	}	2✔
96	}	9✔
97
98	return {killType, &entry_holder};	9✔
99	}	9✔
100	}	9✔
101	}	10✔
102
103	return {KillingType::None, nullptr};	1✔
104	}	20✔
105
106	void OffloadState::found_malloc(Block& block, stdlib::MallocNode& malloc) {	4✔
107	auto& dflow = block.dataflow();	4✔
108
109	auto out_edges = dflow.out_edges_for_connector(malloc, malloc.output(0));	4✔
110	if (out_edges.size() != 1) {	4✔
UNCOV 111	throw std::runtime_error(	×
UNCOV 112	"Unsupported: malloc node " + std::to_string(malloc.element_id()) + " with other than 1 output"	×
UNCOV 113	);	×
UNCOV 114	}	×
115	auto* memlet = out_edges.at(0);	4✔
116	auto* access_node = dynamic_cast<const data_flow::AccessNode*>(&memlet->dst());	4✔
117	generated_.emplace(malloc.element_id(), std::make_unique<OffloadHolder>(&malloc, access_node, memlet));	4✔
118	}	4✔
119
120	void OffloadState::found_offload_node(Block& block, offloading::DataOffloadingNode& offload) {	27✔
121	auto& dflow = block.dataflow();	27✔
122
123	bool src_is_dev = false;	27✔
124	bool src_is_host = false;	27✔
125	bool dst_is_dev = false;	27✔
126	bool dst_is_host = false;	27✔
127
128	bool starts_dev_lifetime = false;	27✔
129	bool ends_dev_lifetime = false;	27✔
130	bool updates_on_dev = false;	27✔
131	bool updates_on_host = false;	27✔
132
133	auto transfer_direction = offload.transfer_direction();	27✔
134	auto lifecycle = offload.buffer_lifecycle();	27✔
135	if (transfer_direction == offloading::DataTransferDirection::D2H) {	27✔
136	src_is_dev = true;	11✔
137	dst_is_host = true;	11✔
138	updates_on_host = true;	11✔
139	if (lifecycle == offloading::BufferLifecycle::FREE) {	11✔
140	ends_dev_lifetime = true;	10✔
141	}	10✔
142	} else if (transfer_direction == offloading::DataTransferDirection::H2D) {	16✔
143	src_is_host = true;	13✔
144	dst_is_dev = true;	13✔
145	updates_on_dev = true;	13✔
146	if (lifecycle == offloading::BufferLifecycle::ALLOC) {	13✔
147	starts_dev_lifetime = true;	13✔
148	}	13✔
149	} else if (offloading::is_NONE(transfer_direction)) {	13✔
150	if (lifecycle == offloading::BufferLifecycle::ALLOC) {	3✔
UNCOV 151	starts_dev_lifetime = true;	×
UNCOV 152	dst_is_dev = true;	×
153	} else if (lifecycle == offloading::BufferLifecycle::FREE) {	3✔
154	ends_dev_lifetime = true;	3✔
155	src_is_dev = true;	3✔
156	}	3✔
157	}	3✔
158
159	const data_flow::AccessNode* found_dev_access = nullptr;	27✔
160	const data_flow::AccessNode* found_host_access = nullptr;	27✔
161	const data_flow::Memlet* found_host_memlet = nullptr;	27✔
162
163	for (auto& conn : offload.inputs()) {	27✔
164	auto* memlet = dflow.in_edge_for_connector(offload, conn);	27✔
165	auto* access_node = dynamic_cast<const data_flow::AccessNode*>(&memlet->src());	27✔
166
167	if (src_is_host) {	27✔
168	found_host_access = access_node;	13✔
169	found_host_memlet = memlet;	13✔
170	}	13✔
171	if (src_is_dev) {	27✔
172	found_dev_access = access_node;	14✔
173	}	14✔
174	}	27✔
175
176	for (auto& conn : offload.outputs()) {	27✔
177	auto edges = dflow.out_edges_for_connector(offload, conn);	27✔
178	if (edges.size() > 1) {	27✔
UNCOV 179	throw std::runtime_error(	×
UNCOV 180	"Unsupported: offload node " + std::to_string(offload.element_id()) +	×
UNCOV 181	" with multiple outputs edges on " + conn	×
UNCOV 182	);	×
UNCOV 183	}	×
184	auto* memlet = edges.at(0);	27✔
185	auto* access_node = dynamic_cast<const data_flow::AccessNode*>(&memlet->dst());	27✔
186
187	if (dst_is_host) {	27✔
188	found_host_access = access_node;	11✔
189	found_host_memlet = memlet;	11✔
190	}	11✔
191	if (dst_is_dev) {	27✔
192	found_dev_access = access_node;	13✔
193	}	13✔
194	}	27✔
195
196	if (ends_dev_lifetime \|\| updates_on_host) {	27✔
197	generated_.emplace(	14✔
198	offload.element_id(),	14✔
199	std::make_unique<OffloadHolder>(	14✔
200	&offload,	14✔
201	found_host_access,	14✔
202	found_host_memlet,	14✔
203	found_dev_access,	14✔
204	starts_dev_lifetime,	14✔
205	ends_dev_lifetime,	14✔
206	updates_on_dev,	14✔
207	updates_on_host	14✔
208	)	14✔
209	);	14✔
210	} else if (starts_dev_lifetime \|\| updates_on_dev) {	14✔
211	add_h2d_entry(OffloadHolder{	13✔
212	&offload,	13✔
213	found_host_access,	13✔
214	found_host_memlet,	13✔
215	found_dev_access,	13✔
216	starts_dev_lifetime,	13✔
217	ends_dev_lifetime,	13✔
218	updates_on_dev,	13✔
219	updates_on_host	13✔
220	});	13✔
221	}	13✔
222	}	27✔
223
224	void OffloadState::add_h2d_entry(const OffloadHolder& entry) {	13✔
225	h2d_nodes_.emplace(entry.offload_node->element_id(), std::make_unique<OffloadHolder>(entry));	13✔
226	// todo also need to remove generated ones killed by this. But right now, only max 1 per block anyway
227	}	13✔
228
229	void OffloadState::apply_kills_and_changes(ExposedType& exposed) const {	38✔
230	if (full_kill_) {	38✔
231	exposed.clear();	×
232	return;	×
233	}	×
234	std::list<ExposedOffload> dynamic_inserts;	38✔
235	for (auto it = exposed.begin(); it != exposed.end();) {	64✔
236	auto& [id, exposedOffload] = *it;	26✔
237	auto& holder = *exposedOffload.offload;	26✔
238
239
240	auto container_reads = reads_.find(exposedOffload.container);	26✔
241	if (container_reads != reads_.end() && !container_reads->second.empty()) {	26✔
242	if (holder.malloc_node) { // mallocs are just killed on first use	2✔
UNCOV 243	DEBUG_PRINTLN(	×
UNCOV 244	"In-flight malloc area of #" << holder.malloc_node->element_id()	×
UNCOV 245	<< " is read without being initialized!"	×
UNCOV 246	);	×
UNCOV 247	it = exposed.erase(it);	×
UNCOV 248	continue;	×
249	} else { // track if a live var is read	2✔
250	++exposedOffload.read_count;	2✔
251	}	2✔
252	}	2✔
253
254	if (kills_containers_.contains(exposedOffload.container)) {	26✔
255	it = exposed.erase(it);	6✔
256	continue;	6✔
257	}	6✔
258
259	auto [kill_type, killing_entry] = find_killing_entry_node(exposedOffload);	20✔
260	if (kill_type != KillingType::None) {	20✔
261	if (kill_type == KillingType::DeviceReuse) {	19✔
262	collector_.found_transfer_reuse_pair(exposedOffload, *killing_entry);	7✔
263	} else if (kill_type == KillingType::EmptyHostMalloc) {	12✔
264	collector_.found_empty_host_malloc(exposedOffload, *killing_entry);	3✔
265	} else if (kill_type == KillingType::DeviceCleanFree) {	9✔
266	// we have a on-device-alloc that survived without kills to the on-device-free
267	// -> promote this to a host-relevant D2H point, that might be reused
268
269	// replace the current H2D with the "D2H" that would allow it to live on
270	// this creates a D2H-like exposedOffload, despite us not knowing the host-var at this point
271	auto* host_data = holder.host_data;	2✔
272	if (host_data) {	2✔
273	dynamic_inserts.emplace_back(killing_entry, host_data->data(), 0);	2✔
274	it = exposed.erase(it);	2✔
275	continue;	2✔
276	}	2✔
277	} else if (kill_type == KillingType::RedundantD2H) {	7✔
278	collector_.found_redundant_d2h_pair(exposedOffload, *killing_entry);	7✔
279	}	7✔
280	it = exposed.erase(it);	17✔
281	continue;	17✔
282	}	19✔
283	++it;	1✔
284	}	1✔
285
286	for (auto& [id, gen] : generated_) {	38✔
287	auto* holder = gen.get();	18✔
288	if (holder->updates_on_host \|\| holder->malloc_node) { // block unidentified host-container ones from being	18✔
289	// exposed. If we could reconstruct, it will be a
290	// dynamic_insert
291	exposed.insert({id, ExposedOffload{holder, holder->host_data->data(), 0}});	15✔
292	}	15✔
293	}	18✔
294	for (auto& gen : dynamic_inserts) {	38✔
295	exposed.insert({gen.offload->offload_node->element_id(), gen});	2✔
296	}	2✔
297	for (auto& [id, gen] : h2d_nodes_) {	38✔
298	auto* holder = gen.get();	13✔
299	if (holder->starts_dev_lifetime) {	13✔
300	exposed.insert({id, ExposedOffload{holder, holder->dev_data->data(), 0}});	13✔
301	}	13✔
302	}	13✔
303	}	38✔
304
305
306	void DataTransferEliminationAnalysis::handle_lib_node(Block& block, data_flow::LibraryNode& node) {	35✔
307	BaseUserVisitor::handle_lib_node(block, node);	35✔
308
309	if (auto* offload_node = dynamic_cast<offloading::DataOffloadingNode*>(&node)) {	35✔
310	get_or_create_state(block).found_offload_node(block, *offload_node);	27✔
311	} else if (auto* malloc_node = dynamic_cast<stdlib::MallocNode*>(&node)) {	27✔
312	get_or_create_state(block).found_malloc(block, *malloc_node);	4✔
313	}	4✔
314	}	35✔
315
316	void DataTransferEliminationAnalysis::handle_structured_loop_before_body(StructuredLoop& loop) {	2✔
317	BaseUserVisitor::handle_structured_loop_before_body(loop);	2✔
318
319	// auto* map = dynamic_cast<sdfg::structured_control_flow::Map*>(&loop);
320
321	// if (map && map->schedule_type().category() == ScheduleTypeCategory::Offloader) {
322	// get_or_create_state(loop).found_offloaded_kernel(*map);
323	// }
324	}	2✔
325
326	void DataTransferEliminationAnalysis::
327	on_escape(const std::string& container, const ControlFlowNode* node, const Element* user) {	31✔
328	if (dynamic_cast<const Block*>(node)) {	31✔
329	if (auto* memlet = dynamic_cast<const data_flow::Memlet*>(user)) {	31✔
330	if (auto* offload = dynamic_cast<const offloading::DataOffloadingNode*>(&memlet->dst())) {	31✔
331	// accesses of offloading nodes are handled more intelligently in found_offload_node, so ignore them
332	// here
333	return;	27✔
334	}	27✔
335	}	31✔
336	}	31✔
337	get_or_create_state(*node).found_escape(container);	4✔
338	}	4✔
339
340	void DataTransferEliminationAnalysis::
341	on_read_via(const std::string& container, const ControlFlowNode* node, const data_flow::Memlet* user) {	3✔
342	if (!dynamic_cast<const offloading::DataOffloadingNode*>(&user->dst())) {	3✔
343	get_or_create_state(*node).found_ptr_read(container, user);	3✔
344	}	3✔
345	}	3✔
346
347	void DataTransferEliminationAnalysis::
348	on_write_via(const std::string& container, const ControlFlowNode* node, const data_flow::Memlet* user) {	3✔
349	if (!dynamic_cast<const offloading::DataOffloadingNode*>(&user->dst())) {	3✔
350	get_or_create_state(*node).found_ptr_write(container, user);	3✔
351	}	3✔
352	}	3✔
353
354	std::unique_ptr<OffloadState> DataTransferEliminationAnalysis::
355	create_initial_state(const structured_control_flow::ControlFlowNode& node) {	50✔
356	return std::make_unique<OffloadState>(*this);	50✔
357	}	50✔
358
359	void DataTransferEliminationAnalysis::run() {	8✔
360	dispatch(sdfg_.root());	8✔
361
362	run_forward(sdfg_.root());	8✔
363	}	8✔
364
365	} // namespace sdfg::analysis

daisytuner / docc / 24517813654

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous