27325952678

Committed 10 Jun 2026 03:49PM UTC coverage: 61.108% (-0.08%) from 61.191%

Build # 27325952678

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #747 from daisytuner/memcpyOnDevice

Translate CPU memcpy to GPU memcpy

Coverage Stats

41 of 149 new or added lines in 6 files covered. (27.52%)

2 existing lines in 2 files now uncovered.

36048 of 58991 relevant lines covered (61.11%)

747.87 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

57.53

/opt/src/transformations/offloading/cuda_stdlib_data_transfer_extraction.cpp

#include "sdfg/transformations/offloading/cuda_stdlib_data_transfer_extraction.h"

#include <cassert>
#include <string>
#include <unordered_map>

#include "sdfg/analysis/analysis.h"
#include "sdfg/analysis/scope_analysis.h"
#include "sdfg/builder/structured_sdfg_builder.h"
#include "sdfg/data_flow/access_node.h"
#include "sdfg/data_flow/library_nodes/stdlib/memcpy.h"
#include "sdfg/data_flow/library_nodes/stdlib/memset.h"
#include "sdfg/exceptions.h"
#include "sdfg/structured_control_flow/block.h"
#include "sdfg/structured_control_flow/sequence.h"
#include "sdfg/symbolic/symbolic.h"
#include "sdfg/targets/cuda/cuda.h"
#include "sdfg/targets/cuda/cuda_data_offloading_node.h"
#include "sdfg/types/type.h"
#include "sdfg/types/utils.h"
#include "symengine/symengine_rcp.h"

namespace sdfg {
namespace cuda {

std::string CUDAStdlibDataTransferExtraction::create_device_container(
    builder::StructuredSDFGBuilder& builder, const types::Pointer& type, const symbolic::Expression& size
) {
    auto new_type = type.clone();
    new_type->storage_type(types::StorageType(
        "NV_Generic", size, types::StorageType::AllocationType::Unmanaged, types::StorageType::AllocationType::Unmanaged
    ));
    auto device_container = builder.find_new_name(CUDA_DEVICE_PREFIX);
    builder.add_container(device_container, *new_type);
    return device_container;
}

void CUDAStdlibDataTransferExtraction::create_allocate(
    builder::StructuredSDFGBuilder& builder,
    structured_control_flow::Sequence& sequence,
    structured_control_flow::Block& block,
    const std::string& device_container,
    const symbolic::Expression& size,
    const types::Pointer& type
) {
    auto& alloc_block = builder.add_block_before(sequence, block, {}, block.debug_info());
    offloading::add_offloading_node<CUDADataOffloadingNode>(
        builder,
        alloc_block,
        device_container,
        device_container,
        offloading::DataTransferDirection::NONE,
        offloading::BufferLifecycle::ALLOC,
        type,
        type,
        this->lib_node_.debug_info(),
        size,
        symbolic::zero()
    );
}

void CUDAStdlibDataTransferExtraction::create_deallocate(
    builder::StructuredSDFGBuilder& builder,
    structured_control_flow::Sequence& sequence,
    structured_control_flow::Block& block,
    const std::string& device_container,
    const types::Pointer& type
) {
    auto& dealloc_block = builder.add_block_after(sequence, block, {}, block.debug_info());
    offloading::add_offloading_node<CUDADataOffloadingNode>(
        builder,
        dealloc_block,
        device_container,
        device_container,
        offloading::DataTransferDirection::NONE,
        offloading::BufferLifecycle::FREE,
        type,
        type,
        this->lib_node_.debug_info(),
        SymEngine::null,
        symbolic::zero()
    );
}

void CUDAStdlibDataTransferExtraction::create_copy_from_device_with_deallocation(
    builder::StructuredSDFGBuilder& builder,
    structured_control_flow::Sequence& sequence,
    structured_control_flow::Block& block,
    const std::string& host_container,
    const std::string& device_container,
    const symbolic::Expression& size,
    const types::Pointer& type
) {
    auto& copy_block = builder.add_block_after(sequence, block, {}, block.debug_info());
    offloading::add_offloading_node<CUDADataOffloadingNode>(
        builder,
        copy_block,
        host_container,
        device_container,
        offloading::DataTransferDirection::D2H,
        offloading::BufferLifecycle::FREE,
        type,
        type,
        this->lib_node_.debug_info(),
        size,
        symbolic::zero()
    );
}

void CUDAStdlibDataTransferExtraction::create_copy_to_device_with_allocation(
    builder::StructuredSDFGBuilder& builder,
    structured_control_flow::Sequence& sequence,
    structured_control_flow::Block& block,
    const std::string& host_container,
    const std::string& device_container,
    const symbolic::Expression& size,
    const types::Pointer& type
) {
    auto& copy_block = builder.add_block_before(sequence, block, {}, block.debug_info());
    offloading::add_offloading_node<CUDADataOffloadingNode>(
        builder,
        copy_block,
        host_container,
        device_container,
        offloading::DataTransferDirection::H2D,
        offloading::BufferLifecycle::ALLOC,
        type,
        type,
        this->lib_node_.debug_info(),
        size,
        symbolic::zero()
    );
}

CUDAStdlibDataTransferExtraction::CUDAStdlibDataTransferExtraction(data_flow::LibraryNode& lib_node)
    : lib_node_(lib_node) {}

std::string CUDAStdlibDataTransferExtraction::name() const { return "CUDAStdlibDataTransferExtraction"; }

bool CUDAStdlibDataTransferExtraction::
    can_be_applied(builder::StructuredSDFGBuilder& builder, analysis::AnalysisManager& analysis_manager) {
    if (this->lib_node_.implementation_type().value() != cuda::ImplementationType_CUDAWithTransfers.value()) {
        return false;
    }

    // Restrict to nodes in their own block
    auto& dfg = this->lib_node_.get_parent();
    if (dfg.nodes().size() != dfg.in_degree(this->lib_node_) + dfg.out_degree(this->lib_node_) + 1) {
        return false;
    }

    // Supported stdlib nodes
    if (dynamic_cast<stdlib::MemsetNode*>(&this->lib_node_)) {
        return true;
    } else if (dynamic_cast<stdlib::MemcpyNode*>(&this->lib_node_)) {
        return true;
    } else {
        return false;
    }
}

void CUDAStdlibDataTransferExtraction::
    apply(builder::StructuredSDFGBuilder& builder, analysis::AnalysisManager& analysis_manager) {
    // Get data flow graph and block
    auto& dfg = this->lib_node_.get_parent();
    auto* block = dynamic_cast<structured_control_flow::Block*>(dfg.get_parent());
    assert(block);

    // Get sequence
    auto& scope_analysis = analysis_manager.get<analysis::ScopeAnalysis>();
    auto* sequence = dynamic_cast<structured_control_flow::Sequence*>(scope_analysis.parent_scope(block));
    assert(sequence);

    if (dynamic_cast<stdlib::MemsetNode*>(&this->lib_node_)) {
        this->apply_memset(builder, analysis_manager, dfg, *sequence, *block);
    } else if (dynamic_cast<stdlib::MemcpyNode*>(&this->lib_node_)) {
        this->apply_memcpy(builder, analysis_manager, dfg, *sequence, *block);
    }

    // Change the implementation type to without transfers
    this->lib_node_.implementation_type() = cuda::ImplementationType_CUDAWithoutTransfers;
}

void CUDAStdlibDataTransferExtraction::apply_memset(
    builder::StructuredSDFGBuilder& builder,
    analysis::AnalysisManager& analysis_manager,
    data_flow::DataFlowGraph& dfg,
    structured_control_flow::Sequence& sequence,
    structured_control_flow::Block& block
) {
    auto& memset_node = static_cast<stdlib::MemsetNode&>(this->lib_node_);

    // Capture output accesses
    auto ptr_edge = dfg.in_edge_for_connector(memset_node, "_ptr");
    auto& host_access_node =
        const_cast<data_flow::AccessNode&>(static_cast<const data_flow::AccessNode&>(ptr_edge->src()));
    auto& host_container_name = host_access_node.data();

    // Use the host container's actual type to avoid type mismatches
    auto& host_type = builder.subject().type(host_container_name);
    auto& type = static_cast<const types::Pointer&>(host_type);

    auto ptr_size = memset_node.num();
    auto dPtr = this->create_device_container(builder, type, ptr_size);

    // Allocate device buffer
    this->create_allocate(builder, sequence, block, dPtr, ptr_size, type);

    // Copy from device to host and deallocate
    this->create_copy_from_device_with_deallocation(builder, sequence, block, host_container_name, dPtr, ptr_size, type);

    // Redirect output to device container
    host_access_node.data(dPtr);
}

void CUDAStdlibDataTransferExtraction::apply_memcpy(
    builder::StructuredSDFGBuilder& builder,
    analysis::AnalysisManager& analysis_manager,
    data_flow::DataFlowGraph& dfg,
    structured_control_flow::Sequence& sequence,
    structured_control_flow::Block& block
) {
    auto& memcpy_node = static_cast<stdlib::MemcpyNode&>(this->lib_node_);
    auto ptr_size = memcpy_node.count();

    // Handle _src (read) - need H2D transfer
    auto src_edge = dfg.in_edge_for_connector(memcpy_node, "_src");
    auto& src_access_node = const_cast<data_flow::AccessNode&>(static_cast<const data_flow::AccessNode&>(src_edge->src()
    ));
    auto& src_container_name = src_access_node.data();
    auto& src_type = static_cast<const types::Pointer&>(builder.subject().type(src_container_name));

    auto dSrc = this->create_device_container(builder, src_type, ptr_size);
    this->create_copy_to_device_with_allocation(builder, sequence, block, src_container_name, dSrc, ptr_size, src_type);
    this->create_deallocate(builder, sequence, block, dSrc, src_type);
    src_access_node.data(dSrc);

    // Handle _dst (write) - need D2H transfer
    auto dst_edge = dfg.in_edge_for_connector(memcpy_node, "_dst");
    auto& dst_access_node = const_cast<data_flow::AccessNode&>(static_cast<const data_flow::AccessNode&>(dst_edge->src()
    ));
    auto& dst_container_name = dst_access_node.data();
    auto& dst_type = static_cast<const types::Pointer&>(builder.subject().type(dst_container_name));

    auto dDst = this->create_device_container(builder, dst_type, ptr_size);
    this->create_allocate(builder, sequence, block, dDst, ptr_size, dst_type);
    this->create_copy_from_device_with_deallocation(builder, sequence, block, dst_container_name, dDst, ptr_size, dst_type);
    dst_access_node.data(dDst);
}

void CUDAStdlibDataTransferExtraction::to_json(nlohmann::json& j) const {
    j["transformation_type"] = this->name();
    j["subgraph"] = {{"0", {{"element_id", this->lib_node_.element_id()}, {"type", "unknown"}}}};
    j["lib_node_element_id"] = this->lib_node_.element_id();
}

} // namespace cuda
} // namespace sdfg

1	#include "sdfg/transformations/offloading/cuda_stdlib_data_transfer_extraction.h"
2
3	#include <cassert>
4	#include <string>
5	#include <unordered_map>
6
7	#include "sdfg/analysis/analysis.h"
8	#include "sdfg/analysis/scope_analysis.h"
9	#include "sdfg/builder/structured_sdfg_builder.h"
10	#include "sdfg/data_flow/access_node.h"
11	#include "sdfg/data_flow/library_nodes/stdlib/memcpy.h"
12	#include "sdfg/data_flow/library_nodes/stdlib/memset.h"
13	#include "sdfg/exceptions.h"
14	#include "sdfg/structured_control_flow/block.h"
15	#include "sdfg/structured_control_flow/sequence.h"
16	#include "sdfg/symbolic/symbolic.h"
17	#include "sdfg/targets/cuda/cuda.h"
18	#include "sdfg/targets/cuda/cuda_data_offloading_node.h"
19	#include "sdfg/types/type.h"
20	#include "sdfg/types/utils.h"
21	#include "symengine/symengine_rcp.h"
22
23	namespace sdfg {
24	namespace cuda {
25
26	std::string CUDAStdlibDataTransferExtraction::create_device_container(
27	builder::StructuredSDFGBuilder& builder, const types::Pointer& type, const symbolic::Expression& size
28	) {	2✔
29	auto new_type = type.clone();	2✔
30	new_type->storage_type(types::StorageType(	2✔
31	"NV_Generic", size, types::StorageType::AllocationType::Unmanaged, types::StorageType::AllocationType::Unmanaged	2✔
32	));	2✔
33	auto device_container = builder.find_new_name(CUDA_DEVICE_PREFIX);	2✔
34	builder.add_container(device_container, *new_type);	2✔
35	return device_container;	2✔
36	}	2✔
37
38	void CUDAStdlibDataTransferExtraction::create_allocate(
39	builder::StructuredSDFGBuilder& builder,
40	structured_control_flow::Sequence& sequence,
41	structured_control_flow::Block& block,
42	const std::string& device_container,
43	const symbolic::Expression& size,
44	const types::Pointer& type
45	) {	2✔
46	auto& alloc_block = builder.add_block_before(sequence, block, {}, block.debug_info());	2✔
47	offloading::add_offloading_node<CUDADataOffloadingNode>(	2✔
48	builder,	2✔
49	alloc_block,	2✔
50	device_container,	2✔
51	device_container,	2✔
52	offloading::DataTransferDirection::NONE,	2✔
53	offloading::BufferLifecycle::ALLOC,	2✔
54	type,	2✔
55	type,	2✔
56	this->lib_node_.debug_info(),	2✔
57	size,	2✔
58	symbolic::zero()	2✔
59	);	2✔
60	}	2✔
61
62	void CUDAStdlibDataTransferExtraction::create_deallocate(
63	builder::StructuredSDFGBuilder& builder,
64	structured_control_flow::Sequence& sequence,
65	structured_control_flow::Block& block,
66	const std::string& device_container,
67	const types::Pointer& type
68	) {	×
69	auto& dealloc_block = builder.add_block_after(sequence, block, {}, block.debug_info());	×
70	offloading::add_offloading_node<CUDADataOffloadingNode>(	×
71	builder,	×
72	dealloc_block,	×
73	device_container,	×
74	device_container,	×
75	offloading::DataTransferDirection::NONE,	×
76	offloading::BufferLifecycle::FREE,	×
77	type,	×
78	type,	×
NEW 79	this->lib_node_.debug_info(),	×
80	SymEngine::null,	×
81	symbolic::zero()	×
82	);	×
83	}	×
84
85	void CUDAStdlibDataTransferExtraction::create_copy_from_device_with_deallocation(
86	builder::StructuredSDFGBuilder& builder,
87	structured_control_flow::Sequence& sequence,
88	structured_control_flow::Block& block,
89	const std::string& host_container,
90	const std::string& device_container,
91	const symbolic::Expression& size,
92	const types::Pointer& type
93	) {	2✔
94	auto& copy_block = builder.add_block_after(sequence, block, {}, block.debug_info());	2✔
95	offloading::add_offloading_node<CUDADataOffloadingNode>(	2✔
96	builder,	2✔
97	copy_block,	2✔
98	host_container,	2✔
99	device_container,	2✔
100	offloading::DataTransferDirection::D2H,	2✔
101	offloading::BufferLifecycle::FREE,	2✔
102	type,	2✔
103	type,	2✔
104	this->lib_node_.debug_info(),	2✔
105	size,	2✔
106	symbolic::zero()	2✔
107	);	2✔
108	}	2✔
109
110	void CUDAStdlibDataTransferExtraction::create_copy_to_device_with_allocation(
111	builder::StructuredSDFGBuilder& builder,
112	structured_control_flow::Sequence& sequence,
113	structured_control_flow::Block& block,
114	const std::string& host_container,
115	const std::string& device_container,
116	const symbolic::Expression& size,
117	const types::Pointer& type
NEW 118	) {	×
NEW 119	auto& copy_block = builder.add_block_before(sequence, block, {}, block.debug_info());	×
NEW 120	offloading::add_offloading_node<CUDADataOffloadingNode>(	×
NEW 121	builder,	×
NEW 122	copy_block,	×
NEW 123	host_container,	×
NEW 124	device_container,	×
NEW 125	offloading::DataTransferDirection::H2D,	×
NEW 126	offloading::BufferLifecycle::ALLOC,	×
NEW 127	type,	×
NEW 128	type,	×
NEW 129	this->lib_node_.debug_info(),	×
NEW 130	size,	×
NEW 131	symbolic::zero()	×
NEW 132	);	×
NEW 133	}	×
134
135	CUDAStdlibDataTransferExtraction::CUDAStdlibDataTransferExtraction(data_flow::LibraryNode& lib_node)
136	: lib_node_(lib_node) {}	8✔
137
138	std::string CUDAStdlibDataTransferExtraction::name() const { return "CUDAStdlibDataTransferExtraction"; }	1✔
139
140	bool CUDAStdlibDataTransferExtraction::
141	can_be_applied(builder::StructuredSDFGBuilder& builder, analysis::AnalysisManager& analysis_manager) {	7✔
142	if (this->lib_node_.implementation_type().value() != cuda::ImplementationType_CUDAWithTransfers.value()) {	7✔
143	return false;	4✔
144	}	4✔
145
146	// Restrict to nodes in their own block
147	auto& dfg = this->lib_node_.get_parent();	3✔
148	if (dfg.nodes().size() != dfg.in_degree(this->lib_node_) + dfg.out_degree(this->lib_node_) + 1) {	3✔
149	return false;	×
150	}	×
151
152	// Supported stdlib nodes
153	if (dynamic_cast<stdlib::MemsetNode*>(&this->lib_node_)) {	3✔
154	return true;	3✔
155	} else if (dynamic_cast<stdlib::MemcpyNode*>(&this->lib_node_)) {	3✔
NEW 156	return true;	×
NEW 157	} else {	×
NEW 158	return false;	×
NEW 159	}	×
160	}	3✔
161
162	void CUDAStdlibDataTransferExtraction::
163	apply(builder::StructuredSDFGBuilder& builder, analysis::AnalysisManager& analysis_manager) {	2✔
164	// Get data flow graph and block
165	auto& dfg = this->lib_node_.get_parent();	2✔
166	auto* block = dynamic_cast<structured_control_flow::Block*>(dfg.get_parent());	2✔
167	assert(block);	2✔
168
169	// Get sequence
170	auto& scope_analysis = analysis_manager.get<analysis::ScopeAnalysis>();	2✔
171	auto* sequence = dynamic_cast<structured_control_flow::Sequence*>(scope_analysis.parent_scope(block));	2✔
172	assert(sequence);	2✔
173
174	if (dynamic_cast<stdlib::MemsetNode*>(&this->lib_node_)) {	2✔
175	this->apply_memset(builder, analysis_manager, dfg, sequence, block);	2✔
176	} else if (dynamic_cast<stdlib::MemcpyNode*>(&this->lib_node_)) {	2✔
NEW 177	this->apply_memcpy(builder, analysis_manager, dfg, sequence, block);	×
NEW 178	}	×
179
180	// Change the implementation type to without transfers
181	this->lib_node_.implementation_type() = cuda::ImplementationType_CUDAWithoutTransfers;	2✔
182	}	2✔
183
184	void CUDAStdlibDataTransferExtraction::apply_memset(
185	builder::StructuredSDFGBuilder& builder,
186	analysis::AnalysisManager& analysis_manager,
187	data_flow::DataFlowGraph& dfg,
188	structured_control_flow::Sequence& sequence,
189	structured_control_flow::Block& block
190	) {	2✔
191	auto& memset_node = static_cast<stdlib::MemsetNode&>(this->lib_node_);	2✔
192
193	// Capture output accesses
194	auto ptr_edge = dfg.in_edge_for_connector(memset_node, "_ptr");	2✔
195	auto& host_access_node =	2✔
196	const_cast<data_flow::AccessNode&>(static_cast<const data_flow::AccessNode&>(ptr_edge->src()));	2✔
197	auto& host_container_name = host_access_node.data();	2✔
198
199	// Use the host container's actual type to avoid type mismatches
200	auto& host_type = builder.subject().type(host_container_name);	2✔
201	auto& type = static_cast<const types::Pointer&>(host_type);	2✔
202
203	auto ptr_size = memset_node.num();	2✔
204	auto dPtr = this->create_device_container(builder, type, ptr_size);	2✔
205
206	// Allocate device buffer
207	this->create_allocate(builder, sequence, block, dPtr, ptr_size, type);	2✔
208
209	// Copy from device to host and deallocate
210	this->create_copy_from_device_with_deallocation(builder, sequence, block, host_container_name, dPtr, ptr_size, type);	2✔
211
212	// Redirect output to device container
213	host_access_node.data(dPtr);	2✔
214	}	2✔
215
216	void CUDAStdlibDataTransferExtraction::apply_memcpy(
217	builder::StructuredSDFGBuilder& builder,
218	analysis::AnalysisManager& analysis_manager,
219	data_flow::DataFlowGraph& dfg,
220	structured_control_flow::Sequence& sequence,
221	structured_control_flow::Block& block
NEW 222	) {	×
NEW 223	auto& memcpy_node = static_cast<stdlib::MemcpyNode&>(this->lib_node_);	×
NEW 224	auto ptr_size = memcpy_node.count();	×
225
226	// Handle _src (read) - need H2D transfer
NEW 227	auto src_edge = dfg.in_edge_for_connector(memcpy_node, "_src");	×
NEW 228	auto& src_access_node = const_cast<data_flow::AccessNode&>(static_cast<const data_flow::AccessNode&>(src_edge->src()	×
NEW 229	));	×
NEW 230	auto& src_container_name = src_access_node.data();	×
NEW 231	auto& src_type = static_cast<const types::Pointer&>(builder.subject().type(src_container_name));	×
232
NEW 233	auto dSrc = this->create_device_container(builder, src_type, ptr_size);	×
NEW 234	this->create_copy_to_device_with_allocation(builder, sequence, block, src_container_name, dSrc, ptr_size, src_type);	×
NEW 235	this->create_deallocate(builder, sequence, block, dSrc, src_type);	×
NEW 236	src_access_node.data(dSrc);	×
237
238	// Handle _dst (write) - need D2H transfer
NEW 239	auto dst_edge = dfg.in_edge_for_connector(memcpy_node, "_dst");	×
NEW 240	auto& dst_access_node = const_cast<data_flow::AccessNode&>(static_cast<const data_flow::AccessNode&>(dst_edge->src()	×
NEW 241	));	×
NEW 242	auto& dst_container_name = dst_access_node.data();	×
NEW 243	auto& dst_type = static_cast<const types::Pointer&>(builder.subject().type(dst_container_name));	×
244
NEW 245	auto dDst = this->create_device_container(builder, dst_type, ptr_size);	×
NEW 246	this->create_allocate(builder, sequence, block, dDst, ptr_size, dst_type);	×
NEW 247	this->create_copy_from_device_with_deallocation(builder, sequence, block, dst_container_name, dDst, ptr_size, dst_type);	×
NEW 248	dst_access_node.data(dDst);	×
UNCOV 249	}	×
250
251	void CUDAStdlibDataTransferExtraction::to_json(nlohmann::json& j) const {	1✔
252	j["transformation_type"] = this->name();	1✔
253	j["subgraph"] = {{"0", {{"element_id", this->lib_node_.element_id()}, {"type", "unknown"}}}};	1✔
254	j["lib_node_element_id"] = this->lib_node_.element_id();	1✔
255	}	1✔
256
257	} // namespace cuda
258	} // namespace sdfg

daisytuner / docc / 27325952678

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous