11854130628

Committed 15 Nov 2024 09:58AM UTC coverage: 95.102% (-0.06%) from 95.163%

Build # 11854130628

Build Type

push

github

Committed by

psalz

Commit Message

Update benchmark results for buffer_access_map refactor

Run Details

2992 of 3394 branches covered (88.16%)

Branch coverage included in aggregate %.

6677 of 6773 relevant lines covered (98.58%)

1294452.81 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.0

/include/task.h

#pragma once

#include <cassert>
#include <memory>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>

#include "graph.h"
#include "grid.h"
#include "hint.h"
#include "intrusive_graph.h"
#include "launcher.h"
#include "range_mapper.h"
#include "ranges.h"
#include "reduction.h"
#include "sycl_wrappers.h"
#include "types.h"

namespace celerity {

class handler;

namespace detail {

        struct task_geometry {
                int dimensions = 0;
                range<3> global_size{1, 1, 1};
                id<3> global_offset;
                range<3> granularity{1, 1, 1};
        };

        struct buffer_access {
                buffer_id bid = -1;
                access_mode mode = access_mode::atomic;
                std::unique_ptr<range_mapper_base> range_mapper;
        };

        class buffer_access_map {
          public:
                /// Default ctor for tasks w/o buffer accesses
                buffer_access_map() = default;

                buffer_access_map(std::vector<buffer_access>&& accesses, const task_geometry& geometry);

                const std::unordered_set<buffer_id>& get_accessed_buffers() const& { return m_accessed_buffers; }

                size_t get_num_accesses() const { return m_accesses.size(); }

                std::pair<buffer_id, access_mode> get_nth_access(const size_t n) const {
                        const auto& [bid, mode, _] = m_accesses[n];
                        return {bid, mode};
                }

                region<3> get_requirements_for_nth_access(const size_t n, const box<3>& execution_range) const;

                /// Returns the union of all consumer accesses made across the entire task (conceptually, the
                /// union of the set of regions obtained by calling get_consumed_region for each chunk).
                region<3> get_task_consumed_region(const buffer_id bid) const {
                        if(auto it = m_task_consumed_regions.find(bid); it != m_task_consumed_regions.end()) { return it->second; }
                        return {};
                }

                /// Returns the union of all producer accesses made across the entire task (conceptually, the
                /// union of the set of regions obtained by calling get_produced_region for each chunk).
                region<3> get_task_produced_region(const buffer_id bid) const {
                        if(auto it = m_task_produced_regions.find(bid); it != m_task_produced_regions.end()) { return it->second; }
                        return {};
                };

                /// Computes the union of all consumed regions (across multiple accesses) for a given execution range.
                region<3> compute_consumed_region(const buffer_id bid, const box<3>& execution_range) const;

                /// Computes the union of all produced regions (across multiple accesses) for a given execution range.
                region<3> compute_produced_region(const buffer_id bid, const box<3>& execution_range) const;

                /// Returns a set of bounding boxes, one for each accessed region, that must be allocated contiguously.
                box_vector<3> compute_required_contiguous_boxes(const buffer_id bid, const box<3>& execution_range) const;

          private:
                std::vector<buffer_access> m_accesses;
                std::unordered_set<buffer_id> m_accessed_buffers; ///< Cached set of buffer ids found in m_accesses
                task_geometry m_task_geometry;
                std::unordered_map<buffer_id, region<3>> m_task_consumed_regions;
                std::unordered_map<buffer_id, region<3>> m_task_produced_regions;
        };

        using reduction_set = std::vector<reduction_info>;

        class side_effect_map : private std::unordered_map<host_object_id, experimental::side_effect_order> {
          private:
                using map_base = std::unordered_map<host_object_id, experimental::side_effect_order>;

          public:
                using typename map_base::const_iterator, map_base::value_type, map_base::key_type, map_base::mapped_type, map_base::const_reference,
                    map_base::const_pointer;
                using iterator = const_iterator;
                using reference = const_reference;
                using pointer = const_pointer;

                using map_base::size, map_base::count, map_base::empty, map_base::cbegin, map_base::cend, map_base::at;

                iterator begin() const { return cbegin(); }
                iterator end() const { return cend(); }
                iterator find(host_object_id key) const { return map_base::find(key); }

                void add_side_effect(host_object_id hoid, experimental::side_effect_order order);
        };

        class task_promise {
          public:
                task_promise() = default;
                task_promise(const task_promise&) = delete;
                task_promise(task_promise&&) = delete;
                task_promise& operator=(const task_promise&) = delete;
                task_promise& operator=(task_promise&&) = delete;
                virtual ~task_promise() = default;

                virtual void fulfill() = 0;
                virtual allocation_id get_user_allocation_id() = 0; // TODO move to struct task instead
        };

        class task : public intrusive_graph_node<task> {
          public:
                task_type get_type() const { return m_type; }

                task_id get_id() const { return m_tid; }

                collective_group_id get_collective_group_id() const { return m_cgid; }

                const buffer_access_map& get_buffer_access_map() const { return m_access_map; }

                const side_effect_map& get_side_effect_map() const { return m_side_effects; }

                const task_geometry& get_geometry() const { return m_geometry; }

                int get_dimensions() const { return m_geometry.dimensions; }

                range<3> get_global_size() const { return m_geometry.global_size; }

                id<3> get_global_offset() const { return m_geometry.global_offset; }

                range<3> get_granularity() const { return m_geometry.granularity; }

                void set_debug_name(const std::string& debug_name) { m_debug_name = debug_name; }
                const std::string& get_debug_name() const { return m_debug_name; }

                bool has_variable_split() const { return m_type == task_type::host_compute || m_type == task_type::device_compute; }

                execution_target get_execution_target() const {
                        switch(m_type) {
                        case task_type::epoch: return execution_target::none;
                        case task_type::device_compute: return execution_target::device;
                        case task_type::host_compute:
                        case task_type::collective:
                        case task_type::master_node: return execution_target::host;
                        case task_type::horizon:
                        case task_type::fence: return execution_target::none;
                        default: utils::unreachable(); // LCOV_EXCL_LINE
                        }
                }

                const reduction_set& get_reductions() const { return m_reductions; }

                epoch_action get_epoch_action() const { return m_epoch_action; }

                task_promise* get_task_promise() const { return m_promise.get(); }

                template <typename Launcher>
                Launcher get_launcher() const {
                        return std::get<Launcher>(m_launcher);
                }

                void add_hint(std::unique_ptr<hint_base>&& h) { m_hints.emplace_back(std::move(h)); }

                template <typename Hint>
                const Hint* get_hint() const {
                        static_assert(std::is_base_of_v<hint_base, Hint>, "Hint must extend hint_base");
                        for(auto& h : m_hints) {
                                if(auto* ptr = dynamic_cast<Hint*>(h.get()); ptr != nullptr) { return ptr; }
                        }
                        return nullptr;
                }

                static std::unique_ptr<task> make_epoch(task_id tid, detail::epoch_action action, std::unique_ptr<task_promise> promise) {
                        return std::unique_ptr<task>(new task(tid, task_type::epoch, non_collective_group_id, task_geometry{}, {}, {}, {}, {}, action, std::move(promise)));
                }

                static std::unique_ptr<task> make_host_compute(task_id tid, task_geometry geometry, host_task_launcher launcher, buffer_access_map access_map,
                    side_effect_map side_effect_map, reduction_set reductions) {
                        return std::unique_ptr<task>(new task(tid, task_type::host_compute, non_collective_group_id, geometry, std::move(launcher), std::move(access_map),
                            std::move(side_effect_map), std::move(reductions), {}, nullptr));
                }

                static std::unique_ptr<task> make_device_compute(
                    task_id tid, task_geometry geometry, device_kernel_launcher launcher, buffer_access_map access_map, reduction_set reductions) {
                        return std::unique_ptr<task>(new task(tid, task_type::device_compute, non_collective_group_id, geometry, std::move(launcher), std::move(access_map),
                            {}, std::move(reductions), {}, nullptr));
                }

                static std::unique_ptr<task> make_collective(task_id tid, task_geometry geometry, collective_group_id cgid, size_t num_collective_nodes,
                    host_task_launcher launcher, buffer_access_map access_map, side_effect_map side_effect_map) {
                        // The geometry is required to construct the buffer_access_map, so we pass it in here even though it has to have a specific shape
                        assert(geometry.dimensions == 1 && geometry.global_size == detail::range_cast<3>(range(num_collective_nodes)) && geometry.global_offset == zeros);
                        return std::unique_ptr<task>(
                            new task(tid, task_type::collective, cgid, geometry, std::move(launcher), std::move(access_map), std::move(side_effect_map), {}, {}, nullptr));
                }

                static std::unique_ptr<task> make_master_node(task_id tid, host_task_launcher launcher, buffer_access_map access_map, side_effect_map side_effect_map) {
                        return std::unique_ptr<task>(new task(tid, task_type::master_node, non_collective_group_id, task_geometry{}, std::move(launcher),
                            std::move(access_map), std::move(side_effect_map), {}, {}, nullptr));
                }

                static std::unique_ptr<task> make_horizon(task_id tid) {
                        return std::unique_ptr<task>(new task(tid, task_type::horizon, non_collective_group_id, task_geometry{}, {}, {}, {}, {}, {}, nullptr));
                }

                static std::unique_ptr<task> make_fence(
                    task_id tid, buffer_access_map access_map, side_effect_map side_effect_map, std::unique_ptr<task_promise> promise) {
                        return std::unique_ptr<task>(new task(tid, task_type::fence, non_collective_group_id, task_geometry{}, {}, std::move(access_map),
                            std::move(side_effect_map), {}, {}, std::move(promise)));
                }

          private:
                task_id m_tid;
                task_type m_type;
                collective_group_id m_cgid;
                task_geometry m_geometry;
                command_group_launcher m_launcher;
                buffer_access_map m_access_map;
                detail::side_effect_map m_side_effects;
                reduction_set m_reductions;
                std::string m_debug_name;
                detail::epoch_action m_epoch_action;
                std::unique_ptr<task_promise> m_promise; // TODO keep user_allocation_id in struct task instead of inside task_promise
                std::vector<std::unique_ptr<hint_base>> m_hints;

                task(task_id tid, task_type type, collective_group_id cgid, task_geometry geometry, command_group_launcher launcher, buffer_access_map access_map,
                    detail::side_effect_map side_effects, reduction_set reductions, detail::epoch_action epoch_action, std::unique_ptr<task_promise> promise)
                    : m_tid(tid), m_type(type), m_cgid(cgid), m_geometry(geometry), m_launcher(std::move(launcher)), m_access_map(std::move(access_map)),
                      m_side_effects(std::move(side_effects)), m_reductions(std::move(reductions)), m_epoch_action(epoch_action), m_promise(std::move(promise)) {
                        assert(type == task_type::host_compute || type == task_type::device_compute || get_granularity().size() == 1);
                        // Only host tasks can have side effects
                        assert(this->m_side_effects.empty() || type == task_type::host_compute || type == task_type::collective || type == task_type::master_node
                               || type == task_type::fence);
                }
        };

        [[nodiscard]] std::string print_task_debug_label(const task& tsk, bool title_case = false);

        /// Determines which overlapping regions appear between write accesses when the iteration space of `tsk` is split into `chunks`.
        std::unordered_map<buffer_id, region<3>> detect_overlapping_writes(const task& tsk, const box_vector<3>& chunks);

        /// The task graph (TDAG) represents all cluster-wide operations, such as command group submissions and fences, and their interdependencies.
        class task_graph : public graph<task> {}; // inheritance instead of type alias so we can forward declare task_graph

} // namespace detail
} // namespace celerity

1	#pragma once
2
3	#include <cassert>
4	#include <memory>
5	#include <unordered_map>
6	#include <unordered_set>
7	#include <utility>
8	#include <vector>
9
10	#include "graph.h"
11	#include "grid.h"
12	#include "hint.h"
13	#include "intrusive_graph.h"
14	#include "launcher.h"
15	#include "range_mapper.h"
16	#include "ranges.h"
17	#include "reduction.h"
18	#include "sycl_wrappers.h"
19	#include "types.h"
20
21	namespace celerity {
22
23	class handler;
24
25	namespace detail {
26
27	struct task_geometry {
28	int dimensions = 0;
29	range<3> global_size{1, 1, 1};
30	id<3> global_offset;
31	range<3> granularity{1, 1, 1};
32	};
33
34	struct buffer_access {
35	buffer_id bid = -1;
36	access_mode mode = access_mode::atomic;
37	std::unique_ptr<range_mapper_base> range_mapper;
38	};
39
40	class buffer_access_map {
41	public:
42	/// Default ctor for tasks w/o buffer accesses
43	buffer_access_map() = default;	2,129✔
44
45	buffer_access_map(std::vector<buffer_access>&& accesses, const task_geometry& geometry);
46
47	const std::unordered_set<buffer_id>& get_accessed_buffers() const& { return m_accessed_buffers; }	24,947✔
48
49	size_t get_num_accesses() const { return m_accesses.size(); }	16,768✔
50
51	std::pair<buffer_id, access_mode> get_nth_access(const size_t n) const {	5,656✔
52	const auto& [bid, mode, _] = m_accesses[n];	5,656✔
53	return {bid, mode};	5,656✔
54	}
55
56	region<3> get_requirements_for_nth_access(const size_t n, const box<3>& execution_range) const;
57
58	/// Returns the union of all consumer accesses made across the entire task (conceptually, the
59	/// union of the set of regions obtained by calling get_consumed_region for each chunk).
60	region<3> get_task_consumed_region(const buffer_id bid) const {	7,370✔
61	if(auto it = m_task_consumed_regions.find(bid); it != m_task_consumed_regions.end()) { return it->second; }	7,370✔
62	return {};	729✔
63	}
64
65	/// Returns the union of all producer accesses made across the entire task (conceptually, the
66	/// union of the set of regions obtained by calling get_produced_region for each chunk).
67	region<3> get_task_produced_region(const buffer_id bid) const {	6,381✔
68	if(auto it = m_task_produced_regions.find(bid); it != m_task_produced_regions.end()) { return it->second; }	6,381✔
69	return {};	133✔
70	};
71
72	/// Computes the union of all consumed regions (across multiple accesses) for a given execution range.
73	region<3> compute_consumed_region(const buffer_id bid, const box<3>& execution_range) const;
74
75	/// Computes the union of all produced regions (across multiple accesses) for a given execution range.
76	region<3> compute_produced_region(const buffer_id bid, const box<3>& execution_range) const;
77
78	/// Returns a set of bounding boxes, one for each accessed region, that must be allocated contiguously.
79	box_vector<3> compute_required_contiguous_boxes(const buffer_id bid, const box<3>& execution_range) const;
80
81	private:
82	std::vector<buffer_access> m_accesses;
83	std::unordered_set<buffer_id> m_accessed_buffers; ///< Cached set of buffer ids found in m_accesses
84	task_geometry m_task_geometry;
85	std::unordered_map<buffer_id, region<3>> m_task_consumed_regions;
86	std::unordered_map<buffer_id, region<3>> m_task_produced_regions;
87	};
88
89	using reduction_set = std::vector<reduction_info>;
90
91	class side_effect_map : private std::unordered_map<host_object_id, experimental::side_effect_order> {
92	private:
93	using map_base = std::unordered_map<host_object_id, experimental::side_effect_order>;
94
95	public:
96	using typename map_base::const_iterator, map_base::value_type, map_base::key_type, map_base::mapped_type, map_base::const_reference,
97	map_base::const_pointer;
98	using iterator = const_iterator;
99	using reference = const_reference;
100	using pointer = const_pointer;
101
102	using map_base::size, map_base::count, map_base::empty, map_base::cbegin, map_base::cend, map_base::at;
103
104	iterator begin() const { return cbegin(); }	7,744✔
105	iterator end() const { return cend(); }	7,725✔
106	iterator find(host_object_id key) const { return map_base::find(key); }
107
108	void add_side_effect(host_object_id hoid, experimental::side_effect_order order);
109	};
110
111	class task_promise {
112	public:
113	task_promise() = default;	589✔
114	task_promise(const task_promise&) = delete;
115	task_promise(task_promise&&) = delete;
116	task_promise& operator=(const task_promise&) = delete;
117	task_promise& operator=(task_promise&&) = delete;
118	virtual ~task_promise() = default;	589✔
119
120	virtual void fulfill() = 0;
121	virtual allocation_id get_user_allocation_id() = 0; // TODO move to struct task instead
122	};
123
124	class task : public intrusive_graph_node<task> {
125	public:
126	task_type get_type() const { return m_type; }	35,958✔
127
128	task_id get_id() const { return m_tid; }	37,821✔
129
130	collective_group_id get_collective_group_id() const { return m_cgid; }	14,953✔
131
132	const buffer_access_map& get_buffer_access_map() const { return m_access_map; }	40,520✔
133
134	const side_effect_map& get_side_effect_map() const { return m_side_effects; }	13,562✔
135
136	const task_geometry& get_geometry() const { return m_geometry; }	1,931✔
137
138	int get_dimensions() const { return m_geometry.dimensions; }	1✔
139
140	range<3> get_global_size() const { return m_geometry.global_size; }	19,888✔
141
142	id<3> get_global_offset() const { return m_geometry.global_offset; }	10,382✔
143
144	range<3> get_granularity() const { return m_geometry.granularity; }	6,903✔
145
146	void set_debug_name(const std::string& debug_name) { m_debug_name = debug_name; }	2,705✔
147	const std::string& get_debug_name() const { return m_debug_name; }	7,198✔
148
149	bool has_variable_split() const { return m_type == task_type::host_compute \|\| m_type == task_type::device_compute; }	4,963✔
150
151	execution_target get_execution_target() const {	9,411✔
152	switch(m_type) {	9,411!
153	case task_type::epoch: return execution_target::none;	×
154	case task_type::device_compute: return execution_target::device;	4,118✔
155	case task_type::host_compute:	5,293✔
156	case task_type::collective:
157	case task_type::master_node: return execution_target::host;	5,293✔
158	case task_type::horizon:	×
159	case task_type::fence: return execution_target::none;	×
160	default: utils::unreachable(); // LCOV_EXCL_LINE
161	}
162	}
163
164	const reduction_set& get_reductions() const { return m_reductions; }	68,299✔
165
166	epoch_action get_epoch_action() const { return m_epoch_action; }	3,960✔
167
168	task_promise* get_task_promise() const { return m_promise.get(); }	1,260✔
169
170	template <typename Launcher>
171	Launcher get_launcher() const {	2,606✔
172	return std::get<Launcher>(m_launcher);	2,606✔
173	}
174
175	void add_hint(std::unique_ptr<hint_base>&& h) { m_hints.emplace_back(std::move(h)); }	82✔
176
177	template <typename Hint>
178	const Hint* get_hint() const {	7,183✔
179	static_assert(std::is_base_of_v<hint_base, Hint>, "Hint must extend hint_base");
180	for(auto& h : m_hints) {	7,386✔
181	if(auto* ptr = dynamic_cast<Hint*>(h.get()); ptr != nullptr) { return ptr; }	319!
182	}
183	return nullptr;	7,067✔
184	}
185
186	static std::unique_ptr<task> make_epoch(task_id tid, detail::epoch_action action, std::unique_ptr<task_promise> promise) {	1,326✔
187	return std::unique_ptr<task>(new task(tid, task_type::epoch, non_collective_group_id, task_geometry{}, {}, {}, {}, {}, action, std::move(promise)));	1,326!
188	}
189
190	static std::unique_ptr<task> make_host_compute(task_id tid, task_geometry geometry, host_task_launcher launcher, buffer_access_map access_map,	336✔
191	side_effect_map side_effect_map, reduction_set reductions) {
192	return std::unique_ptr<task>(new task(tid, task_type::host_compute, non_collective_group_id, geometry, std::move(launcher), std::move(access_map),	672✔
193	std::move(side_effect_map), std::move(reductions), {}, nullptr));	1,008!
194	}
195
196	static std::unique_ptr<task> make_device_compute(	1,045✔
197	task_id tid, task_geometry geometry, device_kernel_launcher launcher, buffer_access_map access_map, reduction_set reductions) {
198	return std::unique_ptr<task>(new task(tid, task_type::device_compute, non_collective_group_id, geometry, std::move(launcher), std::move(access_map),	2,090✔
199	{}, std::move(reductions), {}, nullptr));	3,135!
200	}
201
202	static std::unique_ptr<task> make_collective(task_id tid, task_geometry geometry, collective_group_id cgid, size_t num_collective_nodes,	61✔
203	host_task_launcher launcher, buffer_access_map access_map, side_effect_map side_effect_map) {
204	// The geometry is required to construct the buffer_access_map, so we pass it in here even though it has to have a specific shape
205	assert(geometry.dimensions == 1 && geometry.global_size == detail::range_cast<3>(range(num_collective_nodes)) && geometry.global_offset == zeros);	61✔
206	return std::unique_ptr<task>(
207	new task(tid, task_type::collective, cgid, geometry, std::move(launcher), std::move(access_map), std::move(side_effect_map), {}, {}, nullptr));	61!
208	}
209
210	static std::unique_ptr<task> make_master_node(task_id tid, host_task_launcher launcher, buffer_access_map access_map, side_effect_map side_effect_map) {	1,264✔
211	return std::unique_ptr<task>(new task(tid, task_type::master_node, non_collective_group_id, task_geometry{}, std::move(launcher),	1,264✔
212	std::move(access_map), std::move(side_effect_map), {}, {}, nullptr));	2,528!
213	}
214
215	static std::unique_ptr<task> make_horizon(task_id tid) {	814✔
216	return std::unique_ptr<task>(new task(tid, task_type::horizon, non_collective_group_id, task_geometry{}, {}, {}, {}, {}, {}, nullptr));	814!
217	}
218
219	static std::unique_ptr<task> make_fence(	85✔
220	task_id tid, buffer_access_map access_map, side_effect_map side_effect_map, std::unique_ptr<task_promise> promise) {
221	return std::unique_ptr<task>(new task(tid, task_type::fence, non_collective_group_id, task_geometry{}, {}, std::move(access_map),	85✔
222	std::move(side_effect_map), {}, {}, std::move(promise)));	170!
223	}
224
225	private:
226	task_id m_tid;
227	task_type m_type;
228	collective_group_id m_cgid;
229	task_geometry m_geometry;
230	command_group_launcher m_launcher;
231	buffer_access_map m_access_map;
232	detail::side_effect_map m_side_effects;
233	reduction_set m_reductions;
234	std::string m_debug_name;
235	detail::epoch_action m_epoch_action;
236	std::unique_ptr<task_promise> m_promise; // TODO keep user_allocation_id in struct task instead of inside task_promise
237	std::vector<std::unique_ptr<hint_base>> m_hints;
238
239	task(task_id tid, task_type type, collective_group_id cgid, task_geometry geometry, command_group_launcher launcher, buffer_access_map access_map,	4,693✔
240	detail::side_effect_map side_effects, reduction_set reductions, detail::epoch_action epoch_action, std::unique_ptr<task_promise> promise)
241	: m_tid(tid), m_type(type), m_cgid(cgid), m_geometry(geometry), m_launcher(std::move(launcher)), m_access_map(std::move(access_map)),	4,693✔
242	m_side_effects(std::move(side_effects)), m_reductions(std::move(reductions)), m_epoch_action(epoch_action), m_promise(std::move(promise)) {	9,386✔
243	assert(type == task_type::host_compute \|\| type == task_type::device_compute \|\| get_granularity().size() == 1);	4,693✔
244	// Only host tasks can have side effects
245	assert(this->m_side_effects.empty() \|\| type == task_type::host_compute \|\| type == task_type::collective \|\| type == task_type::master_node	4,693✔
246	\|\| type == task_type::fence);
247	}	4,693✔
248	};
249
250	[[nodiscard]] std::string print_task_debug_label(const task& tsk, bool title_case = false);
251
252	/// Determines which overlapping regions appear between write accesses when the iteration space of `tsk` is split into `chunks`.
253	std::unordered_map<buffer_id, region<3>> detect_overlapping_writes(const task& tsk, const box_vector<3>& chunks);
254
255	/// The task graph (TDAG) represents all cluster-wide operations, such as command group submissions and fences, and their interdependencies.
256	class task_graph : public graph<task> {}; // inheritance instead of type alias so we can forward declare task_graph
257
258	} // namespace detail
259	} // namespace celerity

celerity / celerity-runtime / 11854130628

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous