31009380971

Committed 05 Aug 2026 01:15PM UTC coverage: 65.103% (+0.1%) from 65.005%

Build # 31009380971

Build Type

Pull #814

github

Committed by

web-flow

Commit Message

Merge 268a080ae into 7d5b198bd

Pull Request Pull Request #814: Adds GPU reduce dispatchers

Coverage Stats

409 of 663 new or added lines in 18 files covered. (61.69%)

145 existing lines in 6 files now uncovered.

46693 of 71722 relevant lines covered (65.1%)

713.1 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

35.14

/opt/src/transformations/offloading/rocm_parallelize_nested_map.cpp

#include "sdfg/transformations/offloading/rocm_parallelize_nested_map.h"

#include <sdfg/analysis/loop_analysis.h>
#include "sdfg/exceptions.h"
#include "sdfg/structured_control_flow/reduce.h"
#include "sdfg/symbolic/symbolic.h"
#include "sdfg/targets/gpu/gpu_map_utils.h"
#include "sdfg/targets/rocm/rocm.h"
#include "sdfg/types/pointer.h"
#include "sdfg/types/scalar.h"

namespace sdfg {
namespace transformations {

ROCMParallelizeNestedMap::ROCMParallelizeNestedMap(structured_control_flow::StructuredLoop& loop, size_t block_size)
    : loop_(loop), block_size_(block_size) {}

std::string ROCMParallelizeNestedMap::name() const { return "ROCMParallelizeNestedMap"; }

bool ROCMParallelizeNestedMap::
    can_be_applied(builder::StructuredSDFGBuilder& builder, analysis::AnalysisManager& analysis_manager) {
    if (dynamic_cast<structured_control_flow::Map*>(&loop_) == nullptr &&
        dynamic_cast<structured_control_flow::Reduce*>(&loop_) == nullptr) {
        return false;
    }
    auto& loop_analysis = analysis_manager.get<analysis::LoopAnalysis>();

    // Condition: Check if map is not yet parallelized with ROCM
    if (loop_.schedule_type().value() != ScheduleType_Sequential::value()) {
        return false;
    }

    // Condition: a nested Reduce can only be offloaded when every accumulator is a
    // device-resident pointer to a scalar whose type the atomics baseline supports.
    // Privatize + atomic merge (native atomicAdd or a CAS loop) is only defined for
    // 32/64-bit numeric types, so bool and 8/16-bit accumulators (e.g. torch.any /
    // torch.all over bool) must stay sequential.
    if (auto* reduce = dynamic_cast<structured_control_flow::Reduce*>(&loop_)) {
        auto& sdfg = builder.subject();
        for (auto& reduction : reduce->reductions()) {
            auto& type = sdfg.type(reduction.container);
            auto* ptr = dynamic_cast<const types::Pointer*>(&type);
            if (ptr == nullptr || !ptr->has_pointee_type()) {
                return false;
            }
            auto* scalar = dynamic_cast<const types::Scalar*>(&ptr->pointee_type());
            if (scalar == nullptr) {
                return false;
            }
            auto prim = scalar->primitive_type();
            const bool numeric = types::is_floating_point(prim) || types::is_signed(prim) || types::is_unsigned(prim);
            const size_t width = types::bit_width(prim);
            if (!numeric || (width != 32 && width != 64)) {
                return false;
            }
        }
    }

    // Condition: Check if parent loop exists
    auto parent = loop_analysis.parent_loop(&loop_);
    if (parent == nullptr) {
        return false;
    }

    // Condition: Check if parent loop is a ROCM map, and not Z dimension (final dimension)
    if (auto map = dyn_cast<structured_control_flow::Map*>(parent)) {
        if (map->schedule_type().value() != rocm::ScheduleType_ROCM::value()) {
            return false;
        }
        if (rocm::ScheduleType_ROCM::dimension(map->schedule_type()) == rocm::ROCMDimension::Z) {
            return false;
        }
        auto parent_indvar = map->indvar();
        auto ancestor = parent;
        while (ancestor) {
            if (auto map_ancestor = dyn_cast<structured_control_flow::Map*>(ancestor)) {
                parent_indvar = map_ancestor->indvar();
                for (auto& arg : symbolic::atoms(loop_.condition())) {
                    if (symbolic::eq(arg, parent_indvar)) {
                        return false;
                    }
                }
            }
            ancestor = loop_analysis.parent_loop(ancestor);
        }
    } else {
        return false;
    }

    // Note: arbitrary `init` and `stride` are permitted. The ROCm dispatcher
    // emits `<map.indvar> = init + thread_flat_id * stride`, so the body sees
    // the natural strided value; `num_iterations()` accounts for both when
    // computing the grid geometry.

    // Condition: Parallelizing this loop must not introduce a data race. Folding a new
    // grid dimension distributes this loop's iterations across the new threads and
    // re-runs every unguarded sibling on each of them, with no grid-wide barrier. That
    // races when this loop produces a shared container a sibling consumes (a reduction
    // accumulator -> consumer, e.g. softmax) or when a sibling read-modify-writes a
    // shared container. Such a loop must be parallelized differently or left sequential.
    if (gpu::nested_parallelization_is_unsafe(loop_, analysis_manager)) {
        return false;
    }

    // Condition: Resulting ROCm grid dimension must not exceed hardware limits.
    // Y and Z grid dimensions are limited to 65535.
    auto num_iters = loop_.num_iterations();
    if (!num_iters.is_null() && SymEngine::is_a<SymEngine::Integer>(*num_iters)) {
        int64_t iters = SymEngine::down_cast<const SymEngine::Integer&>(*num_iters).as_int();
        int64_t block = static_cast<int64_t>(block_size_);
        int64_t grid_size = (iters + block - 1) / block;

        constexpr int64_t max_grid_dim_yz = 65535;
        if (grid_size > max_grid_dim_yz) {
            return false;
        }
    }

    return true;
}

void ROCMParallelizeNestedMap::apply(builder::StructuredSDFGBuilder& builder, analysis::AnalysisManager& analysis_manager) {
    auto& loop_analysis = analysis_manager.get<analysis::LoopAnalysis>();
    auto parent = loop_analysis.parent_loop(&loop_);

    auto parent_dim =
        rocm::ScheduleType_ROCM::dimension(static_cast<structured_control_flow::Map*>(parent)->schedule_type());

    rocm::ROCMDimension child_dim;
    if (parent_dim == rocm::ROCMDimension::X) {
        child_dim = rocm::ROCMDimension::Y;
    } else if (parent_dim == rocm::ROCMDimension::Y) {
        child_dim = rocm::ROCMDimension::Z;
    } else {
        throw InvalidSDFGException("Parent loop is Z dimension, cannot parallelize nested map.");
    }

    auto new_schedule = rocm::ScheduleType_ROCM::create();
    rocm::ScheduleType_ROCM::dimension(new_schedule, child_dim);
    rocm::ScheduleType_ROCM::block_size(new_schedule, symbolic::integer(block_size_));

    builder.update_schedule_type(loop_, new_schedule);
}

void ROCMParallelizeNestedMap::to_json(nlohmann::json& j) const {
    j["transformation_type"] = this->name();
    j["parameters"] = nlohmann::json::object();
    j["parameters"]["block_size"] = block_size_;

    serializer::JSONSerializer ser_flat(false);
    j["subgraph"] = nlohmann::json::object();
    j["subgraph"]["0"] = nlohmann::json::object();
    ser_flat.serialize_node(j["subgraph"]["0"], loop_);
}

ROCMParallelizeNestedMap ROCMParallelizeNestedMap::
    from_json(builder::StructuredSDFGBuilder& builder, const nlohmann::json& j) {
    // Prefer the embedding-compatible representation (subgraph/parameters),
    // but fall back to legacy fields (loop/block_size) if needed.
    const auto& subgraph = j.at("subgraph");
    const auto& node_desc = subgraph.at("0");
    size_t loop_id = node_desc.at("element_id").get<size_t>();

    size_t block_size = j.at("parameters").at("block_size").get<size_t>();
    auto loop = dynamic_cast<structured_control_flow::StructuredLoop*>(builder.find_element_by_id(loop_id));
    if (!loop) {
        throw InvalidTransformationDescriptionException("Element with ID " + std::to_string(loop_id) + " is not a loop.");
    }
    return ROCMParallelizeNestedMap(*loop, block_size);
}

} // namespace transformations
} // namespace sdfg

1	#include "sdfg/transformations/offloading/rocm_parallelize_nested_map.h"
2
3	#include <sdfg/analysis/loop_analysis.h>
4	#include "sdfg/exceptions.h"
5	#include "sdfg/structured_control_flow/reduce.h"
6	#include "sdfg/symbolic/symbolic.h"
7	#include "sdfg/targets/gpu/gpu_map_utils.h"
8	#include "sdfg/targets/rocm/rocm.h"
9	#include "sdfg/types/pointer.h"
10	#include "sdfg/types/scalar.h"
11
12	namespace sdfg {
13	namespace transformations {
14
15	ROCMParallelizeNestedMap::ROCMParallelizeNestedMap(structured_control_flow::StructuredLoop& loop, size_t block_size)
16	: loop_(loop), block_size_(block_size) {}	5✔
17
18	std::string ROCMParallelizeNestedMap::name() const { return "ROCMParallelizeNestedMap"; }	×
19
20	bool ROCMParallelizeNestedMap::
21	can_be_applied(builder::StructuredSDFGBuilder& builder, analysis::AnalysisManager& analysis_manager) {	5✔
22	if (dynamic_cast<structured_control_flow::Map*>(&loop_) == nullptr &&	5✔
23	dynamic_cast<structured_control_flow::Reduce*>(&loop_) == nullptr) {	5✔
NEW 24	return false;	×
NEW 25	}	×
26	auto& loop_analysis = analysis_manager.get<analysis::LoopAnalysis>();	5✔
27
28	// Condition: Check if map is not yet parallelized with ROCM
29	if (loop_.schedule_type().value() != ScheduleType_Sequential::value()) {	5✔
30	return false;	×
31	}	×
32
33	// Condition: a nested Reduce can only be offloaded when every accumulator is a
34	// device-resident pointer to a scalar whose type the atomics baseline supports.
35	// Privatize + atomic merge (native atomicAdd or a CAS loop) is only defined for
36	// 32/64-bit numeric types, so bool and 8/16-bit accumulators (e.g. torch.any /
37	// torch.all over bool) must stay sequential.
38	if (auto* reduce = dynamic_cast<structured_control_flow::Reduce*>(&loop_)) {	5✔
NEW 39	auto& sdfg = builder.subject();	×
NEW 40	for (auto& reduction : reduce->reductions()) {	×
NEW 41	auto& type = sdfg.type(reduction.container);	×
NEW 42	auto* ptr = dynamic_cast<const types::Pointer*>(&type);	×
NEW 43	if (ptr == nullptr \|\| !ptr->has_pointee_type()) {	×
NEW 44	return false;	×
NEW 45	}	×
NEW 46	auto* scalar = dynamic_cast<const types::Scalar*>(&ptr->pointee_type());	×
NEW 47	if (scalar == nullptr) {	×
NEW 48	return false;	×
NEW 49	}	×
NEW 50	auto prim = scalar->primitive_type();	×
NEW 51	const bool numeric = types::is_floating_point(prim) \|\| types::is_signed(prim) \|\| types::is_unsigned(prim);	×
NEW 52	const size_t width = types::bit_width(prim);	×
NEW 53	if (!numeric \|\| (width != 32 && width != 64)) {	×
NEW 54	return false;	×
NEW 55	}	×
NEW 56	}	×
NEW 57	}	×
58
59	// Condition: Check if parent loop exists
60	auto parent = loop_analysis.parent_loop(&loop_);	5✔
61	if (parent == nullptr) {	5✔
62	return false;	×
63	}	×
64
65	// Condition: Check if parent loop is a ROCM map, and not Z dimension (final dimension)
66	if (auto map = dyn_cast<structured_control_flow::Map*>(parent)) {	5✔
67	if (map->schedule_type().value() != rocm::ScheduleType_ROCM::value()) {	5✔
68	return false;	×
69	}	×
70	if (rocm::ScheduleType_ROCM::dimension(map->schedule_type()) == rocm::ROCMDimension::Z) {	5✔
71	return false;	×
72	}	×
73	auto parent_indvar = map->indvar();	5✔
74	auto ancestor = parent;	5✔
75	while (ancestor) {	10✔
76	if (auto map_ancestor = dyn_cast<structured_control_flow::Map*>(ancestor)) {	5✔
77	parent_indvar = map_ancestor->indvar();	5✔
78	for (auto& arg : symbolic::atoms(loop_.condition())) {	5✔
79	if (symbolic::eq(arg, parent_indvar)) {	5✔
80	return false;	×
81	}	×
82	}	5✔
83	}	5✔
84	ancestor = loop_analysis.parent_loop(ancestor);	5✔
85	}	5✔
86	} else {	5✔
87	return false;	×
88	}	×
89
90	// Note: arbitrary `init` and `stride` are permitted. The ROCm dispatcher
91	// emits `<map.indvar> = init + thread_flat_id * stride`, so the body sees
92	// the natural strided value; `num_iterations()` accounts for both when
93	// computing the grid geometry.
94
95	// Condition: Parallelizing this loop must not introduce a data race. Folding a new
96	// grid dimension distributes this loop's iterations across the new threads and
97	// re-runs every unguarded sibling on each of them, with no grid-wide barrier. That
98	// races when this loop produces a shared container a sibling consumes (a reduction
99	// accumulator -> consumer, e.g. softmax) or when a sibling read-modify-writes a
100	// shared container. Such a loop must be parallelized differently or left sequential.
101	if (gpu::nested_parallelization_is_unsafe(loop_, analysis_manager)) {	5✔
102	return false;	1✔
103	}	1✔
104
105	// Condition: Resulting ROCm grid dimension must not exceed hardware limits.
106	// Y and Z grid dimensions are limited to 65535.
107	auto num_iters = loop_.num_iterations();	4✔
108	if (!num_iters.is_null() && SymEngine::is_a<SymEngine::Integer>(*num_iters)) {	4✔
109	int64_t iters = SymEngine::down_cast<const SymEngine::Integer&>(*num_iters).as_int();	4✔
110	int64_t block = static_cast<int64_t>(block_size_);	4✔
111	int64_t grid_size = (iters + block - 1) / block;	4✔
112
113	constexpr int64_t max_grid_dim_yz = 65535;	4✔
114	if (grid_size > max_grid_dim_yz) {	4✔
115	return false;	1✔
116	}	1✔
117	}	4✔
118
119	return true;	3✔
120	}	4✔
121
122	void ROCMParallelizeNestedMap::apply(builder::StructuredSDFGBuilder& builder, analysis::AnalysisManager& analysis_manager) {	×
123	auto& loop_analysis = analysis_manager.get<analysis::LoopAnalysis>();	×
124	auto parent = loop_analysis.parent_loop(&loop_);	×
125
126	auto parent_dim =	×
127	rocm::ScheduleType_ROCM::dimension(static_cast<structured_control_flow::Map*>(parent)->schedule_type());	×
128
129	rocm::ROCMDimension child_dim;	×
130	if (parent_dim == rocm::ROCMDimension::X) {	×
131	child_dim = rocm::ROCMDimension::Y;	×
132	} else if (parent_dim == rocm::ROCMDimension::Y) {	×
133	child_dim = rocm::ROCMDimension::Z;	×
134	} else {	×
135	throw InvalidSDFGException("Parent loop is Z dimension, cannot parallelize nested map.");	×
136	}	×
137
138	auto new_schedule = rocm::ScheduleType_ROCM::create();	×
139	rocm::ScheduleType_ROCM::dimension(new_schedule, child_dim);	×
140	rocm::ScheduleType_ROCM::block_size(new_schedule, symbolic::integer(block_size_));	×
141
142	builder.update_schedule_type(loop_, new_schedule);	×
143	}	×
144
145	void ROCMParallelizeNestedMap::to_json(nlohmann::json& j) const {	×
146	j["transformation_type"] = this->name();	×
147	j["parameters"] = nlohmann::json::object();	×
148	j["parameters"]["block_size"] = block_size_;	×
149
150	serializer::JSONSerializer ser_flat(false);	×
151	j["subgraph"] = nlohmann::json::object();	×
152	j["subgraph"]["0"] = nlohmann::json::object();	×
153	ser_flat.serialize_node(j["subgraph"]["0"], loop_);	×
154	}	×
155
156	ROCMParallelizeNestedMap ROCMParallelizeNestedMap::
157	from_json(builder::StructuredSDFGBuilder& builder, const nlohmann::json& j) {	×
158	// Prefer the embedding-compatible representation (subgraph/parameters),
159	// but fall back to legacy fields (loop/block_size) if needed.
160	const auto& subgraph = j.at("subgraph");	×
161	const auto& node_desc = subgraph.at("0");	×
162	size_t loop_id = node_desc.at("element_id").get<size_t>();	×
163
164	size_t block_size = j.at("parameters").at("block_size").get<size_t>();	×
NEW 165	auto loop = dynamic_cast<structured_control_flow::StructuredLoop*>(builder.find_element_by_id(loop_id));	×
166	if (!loop) {	×
167	throw InvalidTransformationDescriptionException("Element with ID " + std::to_string(loop_id) + " is not a loop.");	×
168	}	×
169	return ROCMParallelizeNestedMap(*loop, block_size);	×
170	}	×
171
172	} // namespace transformations
173	} // namespace sdfg

daisytuner / docc / 31009380971

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous