24517813654

Committed 16 Apr 2026 03:05PM UTC coverage: 64.286% (-0.2%) from 64.503%

Build # 24517813654

Build Type

Pull #683

github

Committed by

web-flow

Commit Message

Merge 3803436ab into e165e5156

Pull Request Pull Request #683: Tuned ROCM Gemm

Coverage Stats

7 of 174 new or added lines in 3 files covered. (4.02%)

129 existing lines in 6 files now uncovered.

30845 of 47981 relevant lines covered (64.29%)

578.73 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

30.48

/opt/src/passes/offloading/data_transfer_minimization_pass.cpp

#include "sdfg/passes/offloading/data_transfer_minimization_pass.h"

#include <cstddef>
#include <string>
#include <unordered_set>
#include <utility>

#include "sdfg/analysis/analysis.h"
#include "sdfg/analysis/data_transfer_elimination_analysis.h"
#include "sdfg/analysis/scope_analysis.h"
#include "sdfg/analysis/users.h"
#include "sdfg/data_flow/access_node.h"
#include "sdfg/data_flow/code_node.h"
#include "sdfg/data_flow/data_flow_graph.h"
#include "sdfg/data_flow/library_node.h"
#include "sdfg/data_flow/memlet.h"
#include "sdfg/data_flow/tasklet.h"
#include "sdfg/element.h"
#include "sdfg/exceptions.h"
#include "sdfg/helpers/helpers.h"
#include "sdfg/structured_control_flow/block.h"
#include "sdfg/structured_control_flow/control_flow_node.h"
#include "sdfg/structured_control_flow/sequence.h"
#include "sdfg/symbolic/symbolic.h"
#include "sdfg/targets/cuda/cuda_data_offloading_node.h"
#include "sdfg/targets/offloading/data_offloading_node.h"
#include "sdfg/targets/rocm/rocm_data_offloading_node.h"
#include "sdfg/types/pointer.h"
#include "sdfg/visitor/structured_sdfg_visitor.h"

namespace sdfg {
namespace passes {

DataTransferMinimizationPass::DataTransferMinimizationPass() {}

bool DataTransferMinimizationPass::eliminate_malloc_first_transfer(
    builder::StructuredSDFGBuilder& builder, analysis::OffloadHolder& malloc_holder, analysis::OffloadHolder& copy_in
) {
    // Get all relevant information
    std::string copy_in_device_container = copy_in.dev_data->data();
    DebugInfo copy_in_dst_debinfo = copy_in.dev_data->debug_info();

    // leave the malloc itself, because we have not proven yet that there is no more d2H transfer that needs it
    // DDE needs to be able to find it

    auto* h2d_block = dynamic_cast<structured_control_flow::Block*>(copy_in.offload_node->get_parent().get_parent());
    builder.remove_memlet(*h2d_block, *copy_in.host_access);
    builder.remove_node(*h2d_block, *copy_in.host_data);
    copy_in.remove_h2d_parts();
    copy_in.offload_node->remove_h2d();

    return true;
}

bool DataTransferMinimizationPass::eliminate_redundant_d2h(
    builder::StructuredSDFGBuilder& builder, analysis::OffloadHolder& h2d, analysis::OffloadHolder& d2h
) {
    // Get all relevant information
    std::string copy_out_device_container = d2h.dev_data->data();
    DebugInfo copy_out_dst_debinfo = d2h.dev_data->debug_info();

    // leave the malloc itself, because we have not proven yet that there is no more d2H transfer that needs it
    // DDE needs to be able to find it

    auto* d2h_block = dynamic_cast<structured_control_flow::Block*>(d2h.offload_node->get_parent().get_parent());
    if (d2h.offload_node->is_d2h()) {
        if (d2h.offload_node->is_free()) {
            std::string out_conn = d2h.host_access->src_conn();
            auto access_type = d2h.host_access->base_type().clone();
            builder.remove_memlet(*d2h_block, *d2h.host_access);
            builder.remove_node(*d2h_block, *d2h.host_data);
            d2h.offload_node->remove_d2h();
            auto& free_write = builder.add_access(*d2h_block, copy_out_device_container, copy_out_dst_debinfo);
            builder.add_computational_memlet(
                *d2h_block, *d2h.offload_node, out_conn, free_write, {}, *access_type, copy_out_dst_debinfo
            );
        } else { // remove it entirely
            builder.clear_code_node_legacy(*d2h_block, *d2h.offload_node);
        }
        d2h.remove_d2h_parts();
        return true;
    } else {
        return false;
    }
}

bool DataTransferMinimizationPass::eliminate_transfer_pair(
    builder::StructuredSDFGBuilder& builder,
    analysis::OffloadHolder& copy_out,
    analysis::OffloadHolder& copy_in,
    bool remove_d2h
) {
    // Get all relevant information
    std::string copy_out_device_container = copy_out.dev_data->data();
    std::string copy_in_device_container = copy_in.dev_data->data();
    DebugInfo copy_out_src_debinfo = copy_out.dev_data->debug_info();
    DebugInfo copy_in_dst_debinfo = copy_in.dev_data->debug_info();

    bool remove_entirely = false;
    // Remove what you can remove
    if (!remove_d2h && copy_out.offload_node->is_free()) {
        if (copy_out.offload_node->is_d2h()) {
            copy_out.offload_node->remove_free();
        } else {
            remove_entirely = true;
        }
    } else if (remove_d2h) {
        remove_entirely = true;
    }
    if (remove_entirely) {
        auto* copy_out_block =
            dynamic_cast<structured_control_flow::Block*>(copy_out.offload_node->get_parent().get_parent());
        builder.clear_code_node_legacy(*copy_out_block, *copy_out.offload_node);
    }

    auto* copy_in_block = dynamic_cast<structured_control_flow::Block*>(copy_in.offload_node->get_parent().get_parent()
    );
    builder.clear_code_node_legacy(*copy_in_block, *copy_in.offload_node);

    // Maps the device pointers if necessary
    if (copy_out_device_container != copy_in_device_container) {
        auto& container_type = builder.subject().type(copy_out_device_container);
        auto ref_type = container_type.clone();
        auto& in_access = builder.add_access(*copy_in_block, copy_out_device_container, copy_out_src_debinfo);
        auto& out_access = builder.add_access(*copy_in_block, copy_in_device_container, copy_in_dst_debinfo);
        builder.add_reference_memlet(
            *copy_in_block,
            in_access,
            out_access,
            {symbolic::zero()},
            *ref_type,
            DebugInfo::merge(copy_out.offload_node->debug_info(), copy_in.offload_node->debug_info())
        );
    }

    return true;
}

bool DataTransferMinimizationPass::
    run_pass(builder::StructuredSDFGBuilder& builder, analysis::AnalysisManager& analysis_manager) {
    analysis::DataTransferEliminationAnalysis transfer_analysis(builder.subject(), analysis_manager);
    transfer_analysis.run();

    int removed = 0;

    for (auto& [malloc_cand, first_h2d] : transfer_analysis.empty_malloc_candidates()) {
        auto& malloc_holder = *malloc_cand.offload;

        DEBUG_PRINTLN(
            "  Elim malloc: " << "#" << malloc_holder.malloc_node->element_id() << " -> "
                              << (malloc_holder.host_data ? malloc_holder.host_data->data() : "-") << " / "
                              << "h2d+malloc: #" << first_h2d.offload_node->element_id() << " "
                              << (first_h2d.host_data ? first_h2d.host_data->data() : "-") << " -> "
                              << first_h2d.dev_data->data()
        );

        bool success = eliminate_malloc_first_transfer(builder, malloc_holder, first_h2d);

        if (success) {
            ++removed;
        }
    }

    for (auto& [h2d_cand, redundant_d2h] : transfer_analysis.redundant_d2h_candidates()) {
        auto& h2d_holder = *h2d_cand.offload;

        if (h2d_holder.updates_on_dev) {
            DEBUG_PRINTLN(
                "  Elim h2d: " << "#" << h2d_holder.offload_node->element_id() << " -> " << h2d_holder.host_data->data()
                               << " -> " << h2d_holder.dev_data->data() << " / "
                               << "clean d2h: #" << redundant_d2h.offload_node->element_id() << " "
                               << redundant_d2h.dev_data->data() << " -> " << redundant_d2h.host_data->data()
            );
            bool success = eliminate_redundant_d2h(builder, h2d_holder, redundant_d2h);

            if (success) {
                ++removed;
            }
        }
    }

    for (auto& candidate : transfer_analysis.transfer_reuse_candidates()) {
        auto reads = candidate.first.read_count;
        auto& copy_out = *candidate.first.offload;
        auto& copy_in = candidate.second;
        auto& copy_in_container = copy_in.host_data->data();

        DEBUG_PRINTLN(
            "  Elim hd2: #" << copy_out.offload_node->element_id() << " " << copy_out.dev_data->data() << " -> "
                            << (copy_out.host_data ? copy_out.host_data->data() : "-") << " / "
                            << "d2h: #" << copy_in.offload_node->element_id() << " "
                            << (copy_in.host_data ? copy_in.host_data->data() : "-") << " -> "
                            << copy_in.dev_data->data()
        );

        bool success = eliminate_transfer_pair(builder, copy_out, copy_in, false);

        if (success) {
            ++removed;
        }
    }

    return removed > 0;
}

DataTransferMinimizationLegacy::
    DataTransferMinimizationLegacy(builder::StructuredSDFGBuilder& builder, analysis::AnalysisManager& analysis_manager)
    : visitor::NonStoppingStructuredSDFGVisitor(builder, analysis_manager) {}

bool DataTransferMinimizationLegacy::visit() {
    DEBUG_PRINTLN("Running DataTransferMinimizationPass on " << this->builder_.subject().name());
    return visitor::NonStoppingStructuredSDFGVisitor::visit();
}

bool DataTransferMinimizationLegacy::accept(structured_control_flow::Sequence& sequence) {
    bool applied = false;
    offloading::DataOffloadingNode* copy_out = nullptr;
    structured_control_flow::Block* copy_out_block = nullptr;
    size_t copy_out_index = 0;

    // While a copy-out can be found:
    while (copy_out_index < sequence.size()) {
        // Find a new copy-out
        for (; copy_out_index < sequence.size(); copy_out_index++) {
            if (auto* block = dynamic_cast<structured_control_flow::Block*>(&sequence.at(copy_out_index).first)) {
                if (block->dataflow().library_nodes().size() == 1 && block->dataflow().tasklets().size() == 0) {
                    auto* libnode = *block->dataflow().library_nodes().begin();
                    if (auto* offloading_node = dynamic_cast<offloading::DataOffloadingNode*>(libnode)) {
                        if (offloading_node->is_d2h()) {
                            copy_out = offloading_node;
                            copy_out_block = block;
                            break;
                        }
                    }
                }
            }
        }

        // Find a matching copy-in
        size_t i;
        for (i = copy_out_index; i < sequence.size(); i++) {
            // Child must be a block
            auto* copy_in_block = dynamic_cast<structured_control_flow::Block*>(&sequence.at(i).first);
            if (!copy_in_block) {
                continue;
            }

            // Block must contain exactly one library node
            if (copy_in_block->dataflow().library_nodes().size() != 1 ||
                copy_in_block->dataflow().tasklets().size() != 0) {
                continue;
            }

            // Library node must be an offloading node
            auto* copy_in =
                dynamic_cast<offloading::DataOffloadingNode*>(*copy_in_block->dataflow().library_nodes().begin());
            if (!copy_in) {
                continue;
            }

            // Offloading node must be a copy-in
            if (!copy_in->is_h2d()) {
                continue;
            }

            // Copy-in and copy-out must be redundant
            if (!copy_out->redundant_with(*copy_in)) {
                continue;
            }

            // Get src and dst access nodes for copy-in & -out
            auto [copy_out_src, copy_out_dst] = this->get_src_and_dst(copy_out_block->dataflow(), copy_out);
            auto [copy_in_src, copy_in_dst] = this->get_src_and_dst(copy_in_block->dataflow(), copy_in);

            // Get the write and read users
            auto& users = this->analysis_manager_.get<analysis::Users>();
            analysis::User* write = users.get_user(copy_out_dst->data(), copy_out_dst, analysis::Use::WRITE);
            if (!write) {
                continue;
            }
            analysis::User* read = users.get_user(copy_in_src->data(), copy_in_src, analysis::Use::READ);
            if (!read) {
                continue;
            }

            if (copy_out_dst->data() == copy_in_src->data()) {
                // Ensure that the container is not written between the data transfer nodes
                bool used_between = false;
                for (auto* user : users.all_uses_between(*write, *read)) {
                    if (user->container() == copy_out_dst->data() && user->use() != analysis::Use::READ) {
                        used_between = true;
                        break;
                    }
                }
                if (used_between) {
                    continue;
                }
            } else {
                if (!this->check_container_dependency(
                        copy_out_block, copy_out_dst->data(), copy_in_block, copy_in_src->data()
                    )) {
                    continue;
                }
            }

            // Check that the container is not written after the data transfer nodes
            bool read_after = false;
            for (auto* user : users.all_uses_after(*write)) {
                if (user->container() == copy_out_dst->data() && user->use() == analysis::Use::READ && user != read) {
                    read_after = true;
                    break;
                }
            }

            // Debug output
            DEBUG_PRINTLN(
                "  Eliminating " << (read_after ? "(" : "") << "copy-out: #" << copy_out->element_id() << " "
                                 << copy_out_src->data() << " -> " << copy_out_dst->data() << (read_after ? ")" : "")
                                 << " / copy-in: #" << copy_in->element_id() << " " << copy_in_src->data() << " -> "
                                 << copy_in_dst->data()
            );

            // Get all relevant information
            std::string copy_out_device_container = copy_out_src->data();
            std::string copy_in_device_container = copy_in_dst->data();
            DebugInfo copy_out_src_debinfo = copy_out_src->debug_info();
            DebugInfo copy_in_dst_debinfo = copy_in_dst->debug_info();

            // Remove the data tranfers
            if (read_after && copy_out->is_free()) {
                copy_out->remove_free();
            } else if (!read_after) {
                this->builder_.clear_code_node_legacy(*copy_out_block, *copy_out);
            }
            this->builder_.clear_code_node_legacy(*copy_in_block, *copy_in);

            // Maps the device pointers if necessary
            if (copy_out_device_container != copy_in_device_container) {
                auto& container_type = this->builder_.subject().type(copy_out_device_container);
                auto ref_type = container_type.clone();
                auto& in_access =
                    this->builder_.add_access(*copy_in_block, copy_out_device_container, copy_out_src_debinfo);
                auto& out_access =
                    this->builder_.add_access(*copy_in_block, copy_in_device_container, copy_in_dst_debinfo);
                this->builder_.add_reference_memlet(
                    *copy_in_block,
                    in_access,
                    out_access,
                    {symbolic::zero()},
                    *ref_type,
                    DebugInfo::merge(copy_out->debug_info(), copy_in->debug_info())
                );
            }

            // Invalidate users analysis
            this->analysis_manager_.invalidate<analysis::Users>();
            applied = true;
            break;
        }

        // Skip if no matching copy-in was found
        if (i >= sequence.size()) {
            copy_out_index++;
        }
    }

    return applied;
}

std::pair<data_flow::AccessNode*, data_flow::AccessNode*> DataTransferMinimizationLegacy::
    get_src_and_dst(data_flow::DataFlowGraph& dfg, offloading::DataOffloadingNode* offloading_node) {
    if (!offloading_node->has_transfer()) {
        throw InvalidSDFGException(
            "DataTransferMinimization: Cannot get copy access nodes for offloading node without data transfers"
        );
    }
    data_flow::AccessNode *src, *dst;
    if (dynamic_cast<cuda::CUDADataOffloadingNode*>(offloading_node)) {
        src = this->get_in_access(offloading_node, "_src");
        dst = this->get_out_access(offloading_node, "_dst");
    } else if (dynamic_cast<rocm::ROCMDataOffloadingNode*>(offloading_node)) {
        src = this->get_in_access(offloading_node, "_src");
        dst = this->get_out_access(offloading_node, "_dst");
    } else {
        throw InvalidSDFGException(
            "DataTransferMinimization: Unknown offloading node encountered: " + offloading_node->code().value()
        );
    }
    return {src, dst};
}

data_flow::AccessNode* DataTransferMinimizationLegacy::
    get_in_access(data_flow::CodeNode* node, const std::string& dst_conn) {
    auto& dfg = node->get_parent();
    for (auto& iedge : dfg.in_edges(*node)) {
        if (iedge.dst_conn() == dst_conn) {
            return dynamic_cast<data_flow::AccessNode*>(&iedge.src());
        }
    }
    return nullptr;
}

data_flow::AccessNode* DataTransferMinimizationLegacy::
    get_out_access(data_flow::CodeNode* node, const std::string& src_conn) {
    auto& dfg = node->get_parent();
    for (auto& oedge : dfg.out_edges(*node)) {
        if (oedge.src_conn() == src_conn) {
            return static_cast<data_flow::AccessNode*>(&oedge.dst());
        }
    }
    return nullptr;
}

bool DataTransferMinimizationLegacy::check_container_dependency(
    structured_control_flow::Block* copy_out_block,
    const std::string& copy_out_container,
    structured_control_flow::Block* copy_in_block,
    const std::string& copy_in_container
) {
    // Simplification: Assume blocks are in the same sequence
    auto& scope_analysis = this->analysis_manager_.get<analysis::ScopeAnalysis>();
    auto* copy_out_block_parent = scope_analysis.parent_scope(copy_out_block);
    auto* copy_in_block_parent = scope_analysis.parent_scope(copy_in_block);
    auto* sequence = dynamic_cast<structured_control_flow::Sequence*>(copy_out_block_parent);
    if (copy_out_block_parent != copy_in_block_parent || !sequence) {
        return false;
    }

    std::unordered_set<std::string> copy_out_container_captures, copy_in_container_parts;
    size_t start = sequence->index(*copy_out_block);
    size_t stop = sequence->index(*copy_in_block);
    for (size_t i = start + 1; i < stop; i++) {
        auto* block = dynamic_cast<structured_control_flow::Block*>(&sequence->at(i).first);
        if (!block) {
            continue;
        }

        auto& dfg = block->dataflow();
        for (auto* access_node : dfg.data_nodes()) {
            if (access_node->data() == copy_in_container) {
                // Only allow constant assignments
                for (auto& iedge : dfg.in_edges(*access_node)) {
                    auto* tasklet = dynamic_cast<data_flow::Tasklet*>(&iedge.src());
                    if (!tasklet || tasklet->code() != data_flow::TaskletCode::assign) {
                        continue;
                    }

                    auto& iedge2 = *dfg.in_edges(*tasklet).begin();
                    if (!dynamic_cast<data_flow::ConstantNode*>(&iedge2.src())) {
                        return false;
                    }
                }

                // Collect H2D container parts
                for (auto& oedge : dfg.out_edges(*access_node)) {
                    if (oedge.type() != data_flow::MemletType::Reference) {
                        continue;
                    }

                    auto* access_node2 = dynamic_cast<data_flow::AccessNode*>(&oedge.dst());
                    if (!access_node2) {
                        continue;
                    }

                    copy_in_container_parts.insert(access_node2->data());
                }
            } else if (access_node->data() == copy_out_container) {
                // Collect D2H container captures
                for (auto& oedge : dfg.out_edges(*access_node)) {
                    if (oedge.type() != data_flow::MemletType::Dereference_Dst) {
                        continue;
                    }

                    auto* access_node2 = dynamic_cast<data_flow::AccessNode*>(&oedge.dst());
                    if (!access_node2) {
                        continue;
                    }

                    copy_out_container_captures.insert(access_node2->data());
                }
            }
        }
    }

    // Find all matches between captures and parts
    size_t matches = 0;
    for (auto& capture : copy_out_container_captures) {
        for (auto& part : copy_in_container_parts) {
            if (capture == part) {
                matches++;
            }
        }
    }

    return (matches == 1);
}

} // namespace passes
} // namespace sdfg

1	#include "sdfg/passes/offloading/data_transfer_minimization_pass.h"
2
3	#include <cstddef>
4	#include <string>
5	#include <unordered_set>
6	#include <utility>
7
8	#include "sdfg/analysis/analysis.h"
9	#include "sdfg/analysis/data_transfer_elimination_analysis.h"
10	#include "sdfg/analysis/scope_analysis.h"
11	#include "sdfg/analysis/users.h"
12	#include "sdfg/data_flow/access_node.h"
13	#include "sdfg/data_flow/code_node.h"
14	#include "sdfg/data_flow/data_flow_graph.h"
15	#include "sdfg/data_flow/library_node.h"
16	#include "sdfg/data_flow/memlet.h"
17	#include "sdfg/data_flow/tasklet.h"
18	#include "sdfg/element.h"
19	#include "sdfg/exceptions.h"
20	#include "sdfg/helpers/helpers.h"
21	#include "sdfg/structured_control_flow/block.h"
22	#include "sdfg/structured_control_flow/control_flow_node.h"
23	#include "sdfg/structured_control_flow/sequence.h"
24	#include "sdfg/symbolic/symbolic.h"
25	#include "sdfg/targets/cuda/cuda_data_offloading_node.h"
26	#include "sdfg/targets/offloading/data_offloading_node.h"
27	#include "sdfg/targets/rocm/rocm_data_offloading_node.h"
28	#include "sdfg/types/pointer.h"
29	#include "sdfg/visitor/structured_sdfg_visitor.h"
30
31	namespace sdfg {
32	namespace passes {
33
34	DataTransferMinimizationPass::DataTransferMinimizationPass() {}	8✔
35
36	bool DataTransferMinimizationPass::eliminate_malloc_first_transfer(
37	builder::StructuredSDFGBuilder& builder, analysis::OffloadHolder& malloc_holder, analysis::OffloadHolder& copy_in
38	) {	3✔
39	// Get all relevant information
40	std::string copy_in_device_container = copy_in.dev_data->data();	3✔
41	DebugInfo copy_in_dst_debinfo = copy_in.dev_data->debug_info();	3✔
42
43	// leave the malloc itself, because we have not proven yet that there is no more d2H transfer that needs it
44	// DDE needs to be able to find it
45
46	auto* h2d_block = dynamic_cast<structured_control_flow::Block*>(copy_in.offload_node->get_parent().get_parent());	3✔
47	builder.remove_memlet(h2d_block, copy_in.host_access);	3✔
48	builder.remove_node(h2d_block, copy_in.host_data);	3✔
49	copy_in.remove_h2d_parts();	3✔
50	copy_in.offload_node->remove_h2d();	3✔
51
52	return true;	3✔
53	}	3✔
54
55	bool DataTransferMinimizationPass::eliminate_redundant_d2h(
56	builder::StructuredSDFGBuilder& builder, analysis::OffloadHolder& h2d, analysis::OffloadHolder& d2h
57	) {	5✔
58	// Get all relevant information
59	std::string copy_out_device_container = d2h.dev_data->data();	5✔
60	DebugInfo copy_out_dst_debinfo = d2h.dev_data->debug_info();	5✔
61
62	// leave the malloc itself, because we have not proven yet that there is no more d2H transfer that needs it
63	// DDE needs to be able to find it
64
65	auto* d2h_block = dynamic_cast<structured_control_flow::Block*>(d2h.offload_node->get_parent().get_parent());	5✔
66	if (d2h.offload_node->is_d2h()) {	5✔
67	if (d2h.offload_node->is_free()) {	5✔
68	std::string out_conn = d2h.host_access->src_conn();	5✔
69	auto access_type = d2h.host_access->base_type().clone();	5✔
70	builder.remove_memlet(d2h_block, d2h.host_access);	5✔
71	builder.remove_node(d2h_block, d2h.host_data);	5✔
72	d2h.offload_node->remove_d2h();	5✔
73	auto& free_write = builder.add_access(*d2h_block, copy_out_device_container, copy_out_dst_debinfo);	5✔
74	builder.add_computational_memlet(	5✔
75	d2h_block, d2h.offload_node, out_conn, free_write, {}, *access_type, copy_out_dst_debinfo	5✔
76	);	5✔
77	} else { // remove it entirely	5✔
UNCOV 78	builder.clear_code_node_legacy(d2h_block, d2h.offload_node);	×
UNCOV 79	}	×
80	d2h.remove_d2h_parts();	5✔
81	return true;	5✔
82	} else {	5✔
UNCOV 83	return false;	×
UNCOV 84	}	×
85	}	5✔
86
87	bool DataTransferMinimizationPass::eliminate_transfer_pair(
88	builder::StructuredSDFGBuilder& builder,
89	analysis::OffloadHolder& copy_out,
90	analysis::OffloadHolder& copy_in,
91	bool remove_d2h
92	) {	7✔
93	// Get all relevant information
94	std::string copy_out_device_container = copy_out.dev_data->data();	7✔
95	std::string copy_in_device_container = copy_in.dev_data->data();	7✔
96	DebugInfo copy_out_src_debinfo = copy_out.dev_data->debug_info();	7✔
97	DebugInfo copy_in_dst_debinfo = copy_in.dev_data->debug_info();	7✔
98
99	bool remove_entirely = false;	7✔
100	// Remove what you can remove
101	if (!remove_d2h && copy_out.offload_node->is_free()) {	7✔
102	if (copy_out.offload_node->is_d2h()) {	7✔
103	copy_out.offload_node->remove_free();	3✔
104	} else {	4✔
105	remove_entirely = true;	4✔
106	}	4✔
107	} else if (remove_d2h) {	7✔
UNCOV 108	remove_entirely = true;	×
UNCOV 109	}	×
110	if (remove_entirely) {	7✔
111	auto* copy_out_block =	4✔
112	dynamic_cast<structured_control_flow::Block*>(copy_out.offload_node->get_parent().get_parent());	4✔
113	builder.clear_code_node_legacy(copy_out_block, copy_out.offload_node);	4✔
114	}	4✔
115
116	auto* copy_in_block = dynamic_cast<structured_control_flow::Block*>(copy_in.offload_node->get_parent().get_parent()	7✔
117	);	7✔
118	builder.clear_code_node_legacy(copy_in_block, copy_in.offload_node);	7✔
119
120	// Maps the device pointers if necessary
121	if (copy_out_device_container != copy_in_device_container) {	7✔
UNCOV 122	auto& container_type = builder.subject().type(copy_out_device_container);	×
UNCOV 123	auto ref_type = container_type.clone();	×
UNCOV 124	auto& in_access = builder.add_access(*copy_in_block, copy_out_device_container, copy_out_src_debinfo);	×
UNCOV 125	auto& out_access = builder.add_access(*copy_in_block, copy_in_device_container, copy_in_dst_debinfo);	×
UNCOV 126	builder.add_reference_memlet(	×
UNCOV 127	*copy_in_block,	×
UNCOV 128	in_access,	×
UNCOV 129	out_access,	×
UNCOV 130	{symbolic::zero()},	×
UNCOV 131	*ref_type,	×
UNCOV 132	DebugInfo::merge(copy_out.offload_node->debug_info(), copy_in.offload_node->debug_info())	×
UNCOV 133	);	×
UNCOV 134	}	×
135
136	return true;	7✔
137	}	7✔
138
139	bool DataTransferMinimizationPass::
140	run_pass(builder::StructuredSDFGBuilder& builder, analysis::AnalysisManager& analysis_manager) {	8✔
141	analysis::DataTransferEliminationAnalysis transfer_analysis(builder.subject(), analysis_manager);	8✔
142	transfer_analysis.run();	8✔
143
144	int removed = 0;	8✔
145
146	for (auto& [malloc_cand, first_h2d] : transfer_analysis.empty_malloc_candidates()) {	8✔
147	auto& malloc_holder = *malloc_cand.offload;	3✔
148
149	DEBUG_PRINTLN(	3✔
150	" Elim malloc: " << "#" << malloc_holder.malloc_node->element_id() << " -> "	3✔
151	<< (malloc_holder.host_data ? malloc_holder.host_data->data() : "-") << " / "	3✔
152	<< "h2d+malloc: #" << first_h2d.offload_node->element_id() << " "	3✔
153	<< (first_h2d.host_data ? first_h2d.host_data->data() : "-") << " -> "	3✔
154	<< first_h2d.dev_data->data()	3✔
155	);	3✔
156
157	bool success = eliminate_malloc_first_transfer(builder, malloc_holder, first_h2d);	3✔
158
159	if (success) {	3✔
160	++removed;	3✔
161	}	3✔
162	}	3✔
163
164	for (auto& [h2d_cand, redundant_d2h] : transfer_analysis.redundant_d2h_candidates()) {	8✔
165	auto& h2d_holder = *h2d_cand.offload;	7✔
166
167	if (h2d_holder.updates_on_dev) {	7✔
168	DEBUG_PRINTLN(	5✔
169	" Elim h2d: " << "#" << h2d_holder.offload_node->element_id() << " -> " << h2d_holder.host_data->data()	5✔
170	<< " -> " << h2d_holder.dev_data->data() << " / "	5✔
171	<< "clean d2h: #" << redundant_d2h.offload_node->element_id() << " "	5✔
172	<< redundant_d2h.dev_data->data() << " -> " << redundant_d2h.host_data->data()	5✔
173	);	5✔
174	bool success = eliminate_redundant_d2h(builder, h2d_holder, redundant_d2h);	5✔
175
176	if (success) {	5✔
177	++removed;	5✔
178	}	5✔
179	}	5✔
180	}	7✔
181
182	for (auto& candidate : transfer_analysis.transfer_reuse_candidates()) {	8✔
183	auto reads = candidate.first.read_count;	7✔
184	auto& copy_out = *candidate.first.offload;	7✔
185	auto& copy_in = candidate.second;	7✔
186	auto& copy_in_container = copy_in.host_data->data();	7✔
187
188	DEBUG_PRINTLN(	7✔
189	" Elim hd2: #" << copy_out.offload_node->element_id() << " " << copy_out.dev_data->data() << " -> "	7✔
190	<< (copy_out.host_data ? copy_out.host_data->data() : "-") << " / "	7✔
191	<< "d2h: #" << copy_in.offload_node->element_id() << " "	7✔
192	<< (copy_in.host_data ? copy_in.host_data->data() : "-") << " -> "	7✔
193	<< copy_in.dev_data->data()	7✔
194	);	7✔
195
196	bool success = eliminate_transfer_pair(builder, copy_out, copy_in, false);	7✔
197
198	if (success) {	7✔
199	++removed;	7✔
200	}	7✔
201	}	7✔
202
203	return removed > 0;	8✔
204	}	8✔
205
206	DataTransferMinimizationLegacy::
207	DataTransferMinimizationLegacy(builder::StructuredSDFGBuilder& builder, analysis::AnalysisManager& analysis_manager)
208	: visitor::NonStoppingStructuredSDFGVisitor(builder, analysis_manager) {}	×
209
210	bool DataTransferMinimizationLegacy::visit() {	×
UNCOV 211	DEBUG_PRINTLN("Running DataTransferMinimizationPass on " << this->builder_.subject().name());	×
UNCOV 212	return visitor::NonStoppingStructuredSDFGVisitor::visit();	×
213	}	×
214
UNCOV 215	bool DataTransferMinimizationLegacy::accept(structured_control_flow::Sequence& sequence) {	×
216	bool applied = false;	×
217	offloading::DataOffloadingNode* copy_out = nullptr;	×
218	structured_control_flow::Block* copy_out_block = nullptr;	×
219	size_t copy_out_index = 0;	×
220
221	// While a copy-out can be found:
222	while (copy_out_index < sequence.size()) {	×
223	// Find a new copy-out
224	for (; copy_out_index < sequence.size(); copy_out_index++) {	×
225	if (auto* block = dynamic_cast<structured_control_flow::Block*>(&sequence.at(copy_out_index).first)) {	×
UNCOV 226	if (block->dataflow().library_nodes().size() == 1 && block->dataflow().tasklets().size() == 0) {	×
UNCOV 227	auto* libnode = *block->dataflow().library_nodes().begin();	×
228	if (auto* offloading_node = dynamic_cast<offloading::DataOffloadingNode*>(libnode)) {	×
229	if (offloading_node->is_d2h()) {	×
230	copy_out = offloading_node;	×
231	copy_out_block = block;	×
232	break;	×
UNCOV 233	}	×
UNCOV 234	}	×
235	}	×
236	}	×
237	}	×
238
239	// Find a matching copy-in
240	size_t i;	×
241	for (i = copy_out_index; i < sequence.size(); i++) {	×
242	// Child must be a block
UNCOV 243	auto* copy_in_block = dynamic_cast<structured_control_flow::Block*>(&sequence.at(i).first);	×
UNCOV 244	if (!copy_in_block) {	×
245	continue;	×
246	}	×
247
248	// Block must contain exactly one library node
249	if (copy_in_block->dataflow().library_nodes().size() != 1 \|\|	×
250	copy_in_block->dataflow().tasklets().size() != 0) {	×
251	continue;	×
252	}	×
253
254	// Library node must be an offloading node
255	auto* copy_in =	×
256	dynamic_cast<offloading::DataOffloadingNode>(copy_in_block->dataflow().library_nodes().begin());	×
257	if (!copy_in) {	×
UNCOV 258	continue;	×
259	}	×
260
261	// Offloading node must be a copy-in
262	if (!copy_in->is_h2d()) {	×
263	continue;	×
264	}	×
265
266	// Copy-in and copy-out must be redundant
267	if (!copy_out->redundant_with(*copy_in)) {	×
268	continue;	×
269	}	×
270
271	// Get src and dst access nodes for copy-in & -out
272	auto [copy_out_src, copy_out_dst] = this->get_src_and_dst(copy_out_block->dataflow(), copy_out);	×
273	auto [copy_in_src, copy_in_dst] = this->get_src_and_dst(copy_in_block->dataflow(), copy_in);	×
274
275	// Get the write and read users
276	auto& users = this->analysis_manager_.get<analysis::Users>();	×
277	analysis::User* write = users.get_user(copy_out_dst->data(), copy_out_dst, analysis::Use::WRITE);	×
UNCOV 278	if (!write) {	×
UNCOV 279	continue;	×
280	}	×
281	analysis::User* read = users.get_user(copy_in_src->data(), copy_in_src, analysis::Use::READ);	×
282	if (!read) {	×
283	continue;	×
284	}	×
285
286	if (copy_out_dst->data() == copy_in_src->data()) {	×
287	// Ensure that the container is not written between the data transfer nodes
UNCOV 288	bool used_between = false;	×
289	for (auto* user : users.all_uses_between(write, read)) {	×
290	if (user->container() == copy_out_dst->data() && user->use() != analysis::Use::READ) {	×
291	used_between = true;	×
292	break;	×
293	}	×
294	}	×
UNCOV 295	if (used_between) {	×
UNCOV 296	continue;	×
297	}	×
298	} else {	×
299	if (!this->check_container_dependency(	×
300	copy_out_block, copy_out_dst->data(), copy_in_block, copy_in_src->data()	×
UNCOV 301	)) {	×
UNCOV 302	continue;	×
303	}	×
304	}	×
305
306	// Check that the container is not written after the data transfer nodes
307	bool read_after = false;	×
308	for (auto* user : users.all_uses_after(*write)) {	×
UNCOV 309	if (user->container() == copy_out_dst->data() && user->use() == analysis::Use::READ && user != read) {	×
UNCOV 310	read_after = true;	×
311	break;	×
312	}	×
313	}	×
314
315	// Debug output
316	DEBUG_PRINTLN(	×
317	" Eliminating " << (read_after ? "(" : "") << "copy-out: #" << copy_out->element_id() << " "	×
318	<< copy_out_src->data() << " -> " << copy_out_dst->data() << (read_after ? ")" : "")	×
319	<< " / copy-in: #" << copy_in->element_id() << " " << copy_in_src->data() << " -> "	×
320	<< copy_in_dst->data()	×
321	);	×
322
323	// Get all relevant information
324	std::string copy_out_device_container = copy_out_src->data();	×
325	std::string copy_in_device_container = copy_in_dst->data();	×
326	DebugInfo copy_out_src_debinfo = copy_out_src->debug_info();	×
UNCOV 327	DebugInfo copy_in_dst_debinfo = copy_in_dst->debug_info();	×
328
329	// Remove the data tranfers
330	if (read_after && copy_out->is_free()) {	×
331	copy_out->remove_free();	×
332	} else if (!read_after) {	×
UNCOV 333	this->builder_.clear_code_node_legacy(copy_out_block, copy_out);	×
UNCOV 334	}	×
335	this->builder_.clear_code_node_legacy(copy_in_block, copy_in);	×
336
337	// Maps the device pointers if necessary
338	if (copy_out_device_container != copy_in_device_container) {	×
UNCOV 339	auto& container_type = this->builder_.subject().type(copy_out_device_container);	×
340	auto ref_type = container_type.clone();	×
341	auto& in_access =	×
UNCOV 342	this->builder_.add_access(*copy_in_block, copy_out_device_container, copy_out_src_debinfo);	×
UNCOV 343	auto& out_access =	×
344	this->builder_.add_access(*copy_in_block, copy_in_device_container, copy_in_dst_debinfo);	×
345	this->builder_.add_reference_memlet(	×
346	*copy_in_block,	×
347	in_access,	×
348	out_access,	×
349	{symbolic::zero()},	×
350	*ref_type,	×
351	DebugInfo::merge(copy_out->debug_info(), copy_in->debug_info())	×
352	);	×
353	}	×
354
355	// Invalidate users analysis
356	this->analysis_manager_.invalidate<analysis::Users>();	×
357	applied = true;	×
358	break;	×
359	}	×
360
361	// Skip if no matching copy-in was found
362	if (i >= sequence.size()) {	×
363	copy_out_index++;	×
UNCOV 364	}	×
UNCOV 365	}	×
366
367	return applied;	×
368	}	×
369
370	std::pair<data_flow::AccessNode, data_flow::AccessNode> DataTransferMinimizationLegacy::
371	get_src_and_dst(data_flow::DataFlowGraph& dfg, offloading::DataOffloadingNode* offloading_node) {	×
372	if (!offloading_node->has_transfer()) {	×
373	throw InvalidSDFGException(	×
374	"DataTransferMinimization: Cannot get copy access nodes for offloading node without data transfers"	×
UNCOV 375	);	×
UNCOV 376	}	×
377	data_flow::AccessNode src, dst;	×
378	if (dynamic_cast<cuda::CUDADataOffloadingNode*>(offloading_node)) {	×
379	src = this->get_in_access(offloading_node, "_src");	×
380	dst = this->get_out_access(offloading_node, "_dst");	×
381	} else if (dynamic_cast<rocm::ROCMDataOffloadingNode*>(offloading_node)) {	×
382	src = this->get_in_access(offloading_node, "_src");	×
383	dst = this->get_out_access(offloading_node, "_dst");	×
384	} else {	×
385	throw InvalidSDFGException(	×
UNCOV 386	"DataTransferMinimization: Unknown offloading node encountered: " + offloading_node->code().value()	×
UNCOV 387	);	×
UNCOV 388	}	×
UNCOV 389	return {src, dst};	×
UNCOV 390	}	×
391
392	data_flow::AccessNode* DataTransferMinimizationLegacy::
UNCOV 393	get_in_access(data_flow::CodeNode* node, const std::string& dst_conn) {	×
394	auto& dfg = node->get_parent();	×
395	for (auto& iedge : dfg.in_edges(*node)) {	×
396	if (iedge.dst_conn() == dst_conn) {	×
397	return dynamic_cast<data_flow::AccessNode*>(&iedge.src());	×
398	}	×
399	}	×
400	return nullptr;	×
UNCOV 401	}	×
402
403	data_flow::AccessNode* DataTransferMinimizationLegacy::
404	get_out_access(data_flow::CodeNode* node, const std::string& src_conn) {	×
405	auto& dfg = node->get_parent();	×
406	for (auto& oedge : dfg.out_edges(*node)) {	×
407	if (oedge.src_conn() == src_conn) {	×
408	return static_cast<data_flow::AccessNode*>(&oedge.dst());	×
409	}	×
UNCOV 410	}	×
411	return nullptr;	×
412	}	×
413
414	bool DataTransferMinimizationLegacy::check_container_dependency(
415	structured_control_flow::Block* copy_out_block,
416	const std::string& copy_out_container,
417	structured_control_flow::Block* copy_in_block,
418	const std::string& copy_in_container
419	) {	×
420	// Simplification: Assume blocks are in the same sequence
421	auto& scope_analysis = this->analysis_manager_.get<analysis::ScopeAnalysis>();	×
422	auto* copy_out_block_parent = scope_analysis.parent_scope(copy_out_block);	×
423	auto* copy_in_block_parent = scope_analysis.parent_scope(copy_in_block);	×
424	auto* sequence = dynamic_cast<structured_control_flow::Sequence*>(copy_out_block_parent);	×
425	if (copy_out_block_parent != copy_in_block_parent \|\| !sequence) {	×
UNCOV 426	return false;	×
UNCOV 427	}	×
428
429	std::unordered_set<std::string> copy_out_container_captures, copy_in_container_parts;	×
430	size_t start = sequence->index(*copy_out_block);	×
431	size_t stop = sequence->index(*copy_in_block);	×
UNCOV 432	for (size_t i = start + 1; i < stop; i++) {	×
433	auto* block = dynamic_cast<structured_control_flow::Block*>(&sequence->at(i).first);	×
434	if (!block) {	×
435	continue;	×
436	}	×
437
438	auto& dfg = block->dataflow();	×
439	for (auto* access_node : dfg.data_nodes()) {	×
440	if (access_node->data() == copy_in_container) {	×
441	// Only allow constant assignments
442	for (auto& iedge : dfg.in_edges(*access_node)) {	×
443	auto* tasklet = dynamic_cast<data_flow::Tasklet*>(&iedge.src());	×
444	if (!tasklet \|\| tasklet->code() != data_flow::TaskletCode::assign) {	×
445	continue;	×
UNCOV 446	}	×
447
448	auto& iedge2 = dfg.in_edges(tasklet).begin();	×
449	if (!dynamic_cast<data_flow::ConstantNode*>(&iedge2.src())) {	×
450	return false;	×
UNCOV 451	}	×
452	}	×
453
454	// Collect H2D container parts
455	for (auto& oedge : dfg.out_edges(*access_node)) {	×
456	if (oedge.type() != data_flow::MemletType::Reference) {	×
UNCOV 457	continue;	×
UNCOV 458	}	×
459
460	auto* access_node2 = dynamic_cast<data_flow::AccessNode*>(&oedge.dst());	×
461	if (!access_node2) {	×
462	continue;	×
463	}	×
464
465	copy_in_container_parts.insert(access_node2->data());	×
466	}	×
UNCOV 467	} else if (access_node->data() == copy_out_container) {	×
468	// Collect D2H container captures
469	for (auto& oedge : dfg.out_edges(*access_node)) {	×
UNCOV 470	if (oedge.type() != data_flow::MemletType::Dereference_Dst) {	×
UNCOV 471	continue;	×
UNCOV 472	}	×
473
UNCOV 474	auto* access_node2 = dynamic_cast<data_flow::AccessNode*>(&oedge.dst());	×
UNCOV 475	if (!access_node2) {	×
UNCOV 476	continue;	×
UNCOV 477	}	×
478
UNCOV 479	copy_out_container_captures.insert(access_node2->data());	×
UNCOV 480	}	×
UNCOV 481	}	×
UNCOV 482	}	×
UNCOV 483	}	×
484
485	// Find all matches between captures and parts
UNCOV 486	size_t matches = 0;	×
UNCOV 487	for (auto& capture : copy_out_container_captures) {	×
UNCOV 488	for (auto& part : copy_in_container_parts) {	×
UNCOV 489	if (capture == part) {	×
UNCOV 490	matches++;	×
UNCOV 491	}	×
UNCOV 492	}	×
UNCOV 493	}	×
494
UNCOV 495	return (matches == 1);	×
UNCOV 496	}	×
497
498	} // namespace passes
499	} // namespace sdfg

daisytuner / docc / 24517813654

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous