12009876465

Committed 25 Nov 2024 12:19PM UTC coverage: 94.911%. Remained the same

Build # 12009876465

Build Type

push

github

Committed by

fknorr

Commit Message

[RM] fixup includes

Run Details

3189 of 3626 branches covered (87.95%)

Branch coverage included in aggregate %.

7049 of 7161 relevant lines covered (98.44%)

1541661.11 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.0

/include/task.h

#pragma once

#include "graph.h"
#include "grid.h"
#include "hint.h"
#include "intrusive_graph.h"
#include "launcher.h"
#include "range_mapper.h"
#include "ranges.h"
#include "reduction.h"
#include "sycl_wrappers.h"
#include "types.h"
#include "utils.h"

#include <cassert>
#include <cstddef>
#include <memory>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>


namespace celerity {

class handler;

namespace detail {

        struct task_geometry {
                int dimensions = 0;
                range<3> global_size{1, 1, 1};
                id<3> global_offset;
                range<3> granularity{1, 1, 1};
        };

        struct buffer_access {
                buffer_id bid = -1;
                access_mode mode = access_mode::atomic;
                std::unique_ptr<range_mapper_base> range_mapper;
        };

        class buffer_access_map {
          public:
                /// Default ctor for tasks w/o buffer accesses
                buffer_access_map() = default;

                buffer_access_map(std::vector<buffer_access>&& accesses, const task_geometry& geometry);

                const std::unordered_set<buffer_id>& get_accessed_buffers() const& { return m_accessed_buffers; }

                size_t get_num_accesses() const { return m_accesses.size(); }

                std::pair<buffer_id, access_mode> get_nth_access(const size_t n) const {
                        const auto& [bid, mode, _] = m_accesses[n];
                        return {bid, mode};
                }

                region<3> get_requirements_for_nth_access(const size_t n, const box<3>& execution_range) const;

                /// Returns the union of all consumer accesses made across the entire task (conceptually, the
                /// union of the set of regions obtained by calling get_consumed_region for each chunk).
                region<3> get_task_consumed_region(const buffer_id bid) const {
                        if(auto it = m_task_consumed_regions.find(bid); it != m_task_consumed_regions.end()) { return it->second; }
                        return {};
                }

                /// Returns the union of all producer accesses made across the entire task (conceptually, the
                /// union of the set of regions obtained by calling get_produced_region for each chunk).
                region<3> get_task_produced_region(const buffer_id bid) const {
                        if(auto it = m_task_produced_regions.find(bid); it != m_task_produced_regions.end()) { return it->second; }
                        return {};
                };

                /// Computes the union of all consumed regions (across multiple accesses) for a given execution range.
                region<3> compute_consumed_region(const buffer_id bid, const box<3>& execution_range) const;

                /// Computes the union of all produced regions (across multiple accesses) for a given execution range.
                region<3> compute_produced_region(const buffer_id bid, const box<3>& execution_range) const;

                /// Returns a set of bounding boxes, one for each accessed region, that must be allocated contiguously.
                box_vector<3> compute_required_contiguous_boxes(const buffer_id bid, const box<3>& execution_range) const;

          private:
                std::vector<buffer_access> m_accesses;
                std::unordered_set<buffer_id> m_accessed_buffers; ///< Cached set of buffer ids found in m_accesses
                task_geometry m_task_geometry;
                std::unordered_map<buffer_id, region<3>> m_task_consumed_regions;
                std::unordered_map<buffer_id, region<3>> m_task_produced_regions;
        };

        using reduction_set = std::vector<reduction_info>;

        class side_effect_map : private std::unordered_map<host_object_id, experimental::side_effect_order> {
          private:
                using map_base = std::unordered_map<host_object_id, experimental::side_effect_order>;

          public:
                using typename map_base::const_iterator, map_base::value_type, map_base::key_type, map_base::mapped_type, map_base::const_reference,
                    map_base::const_pointer;
                using iterator = const_iterator;
                using reference = const_reference;
                using pointer = const_pointer;

                using map_base::size, map_base::count, map_base::empty, map_base::cbegin, map_base::cend, map_base::at;

                iterator begin() const { return cbegin(); }
                iterator end() const { return cend(); }
                iterator find(host_object_id key) const { return map_base::find(key); }

                void add_side_effect(host_object_id hoid, experimental::side_effect_order order);
        };

        class task_promise {
          public:
                task_promise() = default;
                task_promise(const task_promise&) = delete;
                task_promise(task_promise&&) = delete;
                task_promise& operator=(const task_promise&) = delete;
                task_promise& operator=(task_promise&&) = delete;
                virtual ~task_promise() = default;

                virtual void fulfill() = 0;
                virtual allocation_id get_user_allocation_id() = 0; // TODO move to struct task instead
        };

        class task : public intrusive_graph_node<task> {
          public:
                task_type get_type() const { return m_type; }

                task_id get_id() const { return m_tid; }

                collective_group_id get_collective_group_id() const { return m_cgid; }

                const buffer_access_map& get_buffer_access_map() const { return m_access_map; }

                const side_effect_map& get_side_effect_map() const { return m_side_effects; }

                const task_geometry& get_geometry() const { return m_geometry; }

                int get_dimensions() const { return m_geometry.dimensions; }

                range<3> get_global_size() const { return m_geometry.global_size; }

                id<3> get_global_offset() const { return m_geometry.global_offset; }

                range<3> get_granularity() const { return m_geometry.granularity; }

                void set_debug_name(const std::string& debug_name) { m_debug_name = debug_name; }
                const std::string& get_debug_name() const { return m_debug_name; }

                bool has_variable_split() const { return m_type == task_type::host_compute || m_type == task_type::device_compute; }

                execution_target get_execution_target() const {
                        switch(m_type) {
                        case task_type::epoch: return execution_target::none;
                        case task_type::device_compute: return execution_target::device;
                        case task_type::host_compute:
                        case task_type::collective:
                        case task_type::master_node: return execution_target::host;
                        case task_type::horizon:
                        case task_type::fence: return execution_target::none;
                        default: utils::unreachable(); // LCOV_EXCL_LINE
                        }
                }

                const reduction_set& get_reductions() const { return m_reductions; }

                epoch_action get_epoch_action() const { return m_epoch_action; }

                task_promise* get_task_promise() const { return m_promise.get(); }

                template <typename Launcher>
                Launcher get_launcher() const {
                        return std::get<Launcher>(m_launcher);
                }

                void add_hint(std::unique_ptr<hint_base>&& h) { m_hints.emplace_back(std::move(h)); }

                template <typename Hint>
                const Hint* get_hint() const {
                        static_assert(std::is_base_of_v<hint_base, Hint>, "Hint must extend hint_base");
                        for(auto& h : m_hints) {
                                if(auto* ptr = dynamic_cast<Hint*>(h.get()); ptr != nullptr) { return ptr; }
                        }
                        return nullptr;
                }

                static std::unique_ptr<task> make_epoch(task_id tid, detail::epoch_action action, std::unique_ptr<task_promise> promise) {
                        return std::unique_ptr<task>(new task(tid, task_type::epoch, non_collective_group_id, task_geometry{}, {}, {}, {}, {}, action, std::move(promise)));
                }

                static std::unique_ptr<task> make_host_compute(task_id tid, task_geometry geometry, host_task_launcher launcher, buffer_access_map access_map,
                    side_effect_map side_effect_map, reduction_set reductions) {
                        return std::unique_ptr<task>(new task(tid, task_type::host_compute, non_collective_group_id, geometry, std::move(launcher), std::move(access_map),
                            std::move(side_effect_map), std::move(reductions), {}, nullptr));
                }

                static std::unique_ptr<task> make_device_compute(
                    task_id tid, task_geometry geometry, device_kernel_launcher launcher, buffer_access_map access_map, reduction_set reductions) {
                        return std::unique_ptr<task>(new task(tid, task_type::device_compute, non_collective_group_id, geometry, std::move(launcher), std::move(access_map),
                            {}, std::move(reductions), {}, nullptr));
                }

                static std::unique_ptr<task> make_collective(task_id tid, task_geometry geometry, collective_group_id cgid, size_t num_collective_nodes,
                    host_task_launcher launcher, buffer_access_map access_map, side_effect_map side_effect_map) {
                        // The geometry is required to construct the buffer_access_map, so we pass it in here even though it has to have a specific shape
                        assert(geometry.dimensions == 1 && geometry.global_size == detail::range_cast<3>(range(num_collective_nodes)) && geometry.global_offset == zeros);
                        return std::unique_ptr<task>(
                            new task(tid, task_type::collective, cgid, geometry, std::move(launcher), std::move(access_map), std::move(side_effect_map), {}, {}, nullptr));
                }

                static std::unique_ptr<task> make_master_node(task_id tid, host_task_launcher launcher, buffer_access_map access_map, side_effect_map side_effect_map) {
                        return std::unique_ptr<task>(new task(tid, task_type::master_node, non_collective_group_id, task_geometry{}, std::move(launcher),
                            std::move(access_map), std::move(side_effect_map), {}, {}, nullptr));
                }

                static std::unique_ptr<task> make_horizon(task_id tid) {
                        return std::unique_ptr<task>(new task(tid, task_type::horizon, non_collective_group_id, task_geometry{}, {}, {}, {}, {}, {}, nullptr));
                }

                static std::unique_ptr<task> make_fence(
                    task_id tid, buffer_access_map access_map, side_effect_map side_effect_map, std::unique_ptr<task_promise> promise) {
                        return std::unique_ptr<task>(new task(tid, task_type::fence, non_collective_group_id, task_geometry{}, {}, std::move(access_map),
                            std::move(side_effect_map), {}, {}, std::move(promise)));
                }

          private:
                task_id m_tid;
                task_type m_type;
                collective_group_id m_cgid;
                task_geometry m_geometry;
                command_group_launcher m_launcher;
                buffer_access_map m_access_map;
                detail::side_effect_map m_side_effects;
                reduction_set m_reductions;
                std::string m_debug_name;
                detail::epoch_action m_epoch_action;
                std::unique_ptr<task_promise> m_promise; // TODO keep user_allocation_id in struct task instead of inside task_promise
                std::vector<std::unique_ptr<hint_base>> m_hints;

                task(task_id tid, task_type type, collective_group_id cgid, task_geometry geometry, command_group_launcher launcher, buffer_access_map access_map,
                    detail::side_effect_map side_effects, reduction_set reductions, detail::epoch_action epoch_action, std::unique_ptr<task_promise> promise)
                    : m_tid(tid), m_type(type), m_cgid(cgid), m_geometry(geometry), m_launcher(std::move(launcher)), m_access_map(std::move(access_map)),
                      m_side_effects(std::move(side_effects)), m_reductions(std::move(reductions)), m_epoch_action(epoch_action), m_promise(std::move(promise)) {
                        assert(type == task_type::host_compute || type == task_type::device_compute || get_granularity().size() == 1);
                        // Only host tasks can have side effects
                        assert(this->m_side_effects.empty() || type == task_type::host_compute || type == task_type::collective || type == task_type::master_node
                               || type == task_type::fence);
                }
        };

        [[nodiscard]] std::string print_task_debug_label(const task& tsk, bool title_case = false);

        /// Determines which overlapping regions appear between write accesses when the iteration space of `tsk` is split into `chunks`.
        std::unordered_map<buffer_id, region<3>> detect_overlapping_writes(const task& tsk, const box_vector<3>& chunks);

        /// The task graph (TDAG) represents all cluster-wide operations, such as command group submissions and fences, and their interdependencies.
        class task_graph : public graph<task> {}; // inheritance instead of type alias so we can forward declare task_graph

} // namespace detail
} // namespace celerity

1	#pragma once
2
3	#include "graph.h"
4	#include "grid.h"
5	#include "hint.h"
6	#include "intrusive_graph.h"
7	#include "launcher.h"
8	#include "range_mapper.h"
9	#include "ranges.h"
10	#include "reduction.h"
11	#include "sycl_wrappers.h"
12	#include "types.h"
13	#include "utils.h"
14
15	#include <cassert>
16	#include <cstddef>
17	#include <memory>
18	#include <string>
19	#include <unordered_map>
20	#include <unordered_set>
21	#include <utility>
22	#include <vector>
23
24
25	namespace celerity {
26
27	class handler;
28
29	namespace detail {
30
31	struct task_geometry {
32	int dimensions = 0;
33	range<3> global_size{1, 1, 1};
34	id<3> global_offset;
35	range<3> granularity{1, 1, 1};
36	};
37
38	struct buffer_access {
39	buffer_id bid = -1;
40	access_mode mode = access_mode::atomic;
41	std::unique_ptr<range_mapper_base> range_mapper;
42	};
43
44	class buffer_access_map {
45	public:
46	/// Default ctor for tasks w/o buffer accesses
47	buffer_access_map() = default;	2,225✔
48
49	buffer_access_map(std::vector<buffer_access>&& accesses, const task_geometry& geometry);
50
51	const std::unordered_set<buffer_id>& get_accessed_buffers() const& { return m_accessed_buffers; }	29,177✔
52
53	size_t get_num_accesses() const { return m_accesses.size(); }	21,047✔
54
55	std::pair<buffer_id, access_mode> get_nth_access(const size_t n) const {	7,157✔
56	const auto& [bid, mode, _] = m_accesses[n];	7,157✔
57	return {bid, mode};	7,157✔
58	}
59
60	region<3> get_requirements_for_nth_access(const size_t n, const box<3>& execution_range) const;
61
62	/// Returns the union of all consumer accesses made across the entire task (conceptually, the
63	/// union of the set of regions obtained by calling get_consumed_region for each chunk).
64	region<3> get_task_consumed_region(const buffer_id bid) const {	8,006✔
65	if(auto it = m_task_consumed_regions.find(bid); it != m_task_consumed_regions.end()) { return it->second; }	8,006✔
66	return {};	753✔
67	}
68
69	/// Returns the union of all producer accesses made across the entire task (conceptually, the
70	/// union of the set of regions obtained by calling get_produced_region for each chunk).
71	region<3> get_task_produced_region(const buffer_id bid) const {	6,953✔
72	if(auto it = m_task_produced_regions.find(bid); it != m_task_produced_regions.end()) { return it->second; }	6,953✔
73	return {};	133✔
74	};
75
76	/// Computes the union of all consumed regions (across multiple accesses) for a given execution range.
77	region<3> compute_consumed_region(const buffer_id bid, const box<3>& execution_range) const;
78
79	/// Computes the union of all produced regions (across multiple accesses) for a given execution range.
80	region<3> compute_produced_region(const buffer_id bid, const box<3>& execution_range) const;
81
82	/// Returns a set of bounding boxes, one for each accessed region, that must be allocated contiguously.
83	box_vector<3> compute_required_contiguous_boxes(const buffer_id bid, const box<3>& execution_range) const;
84
85	private:
86	std::vector<buffer_access> m_accesses;
87	std::unordered_set<buffer_id> m_accessed_buffers; ///< Cached set of buffer ids found in m_accesses
88	task_geometry m_task_geometry;
89	std::unordered_map<buffer_id, region<3>> m_task_consumed_regions;
90	std::unordered_map<buffer_id, region<3>> m_task_produced_regions;
91	};
92
93	using reduction_set = std::vector<reduction_info>;
94
95	class side_effect_map : private std::unordered_map<host_object_id, experimental::side_effect_order> {
96	private:
97	using map_base = std::unordered_map<host_object_id, experimental::side_effect_order>;
98
99	public:
100	using typename map_base::const_iterator, map_base::value_type, map_base::key_type, map_base::mapped_type, map_base::const_reference,
101	map_base::const_pointer;
102	using iterator = const_iterator;
103	using reference = const_reference;
104	using pointer = const_pointer;
105
106	using map_base::size, map_base::count, map_base::empty, map_base::cbegin, map_base::cend, map_base::at;
107
108	iterator begin() const { return cbegin(); }	8,382✔
109	iterator end() const { return cend(); }	8,363✔
110	iterator find(host_object_id key) const { return map_base::find(key); }
111
112	void add_side_effect(host_object_id hoid, experimental::side_effect_order order);
113	};
114
115	class task_promise {
116	public:
117	task_promise() = default;	609✔
118	task_promise(const task_promise&) = delete;
119	task_promise(task_promise&&) = delete;
120	task_promise& operator=(const task_promise&) = delete;
121	task_promise& operator=(task_promise&&) = delete;
122	virtual ~task_promise() = default;	609✔
123
124	virtual void fulfill() = 0;
125	virtual allocation_id get_user_allocation_id() = 0; // TODO move to struct task instead
126	};
127
128	class task : public intrusive_graph_node<task> {
129	public:
130	task_type get_type() const { return m_type; }	39,663✔
131
132	task_id get_id() const { return m_tid; }	41,809✔
133
134	collective_group_id get_collective_group_id() const { return m_cgid; }	17,512✔
135
136	const buffer_access_map& get_buffer_access_map() const { return m_access_map; }	46,915✔
137
138	const side_effect_map& get_side_effect_map() const { return m_side_effects; }	16,063✔
139
140	const task_geometry& get_geometry() const { return m_geometry; }	2,227✔
141
142	int get_dimensions() const { return m_geometry.dimensions; }	1✔
143
144	range<3> get_global_size() const { return m_geometry.global_size; }	23,992✔
145
146	id<3> get_global_offset() const { return m_geometry.global_offset; }	11,708✔
147
148	range<3> get_granularity() const { return m_geometry.granularity; }	10,233✔
149
150	void set_debug_name(const std::string& debug_name) { m_debug_name = debug_name; }	2,989✔
151	const std::string& get_debug_name() const { return m_debug_name; }	9,078✔
152
153	bool has_variable_split() const { return m_type == task_type::host_compute \|\| m_type == task_type::device_compute; }	7,447✔
154
155	execution_target get_execution_target() const {	16,665✔
156	switch(m_type) {	16,665!
157	case task_type::epoch: return execution_target::none;	×
158	case task_type::device_compute: return execution_target::device;	8,975✔
159	case task_type::host_compute:	7,690✔
160	case task_type::collective:
161	case task_type::master_node: return execution_target::host;	7,690✔
162	case task_type::horizon:	×
163	case task_type::fence: return execution_target::none;	×
164	default: utils::unreachable(); // LCOV_EXCL_LINE
165	}
166	}
167
168	const reduction_set& get_reductions() const { return m_reductions; }	78,691✔
169
170	epoch_action get_epoch_action() const { return m_epoch_action; }	4,167✔
171
172	task_promise* get_task_promise() const { return m_promise.get(); }	1,330✔
173
174	template <typename Launcher>
175	Launcher get_launcher() const {	3,280✔
176	return std::get<Launcher>(m_launcher);	3,280✔
177	}
178
179	void add_hint(std::unique_ptr<hint_base>&& h) { m_hints.emplace_back(std::move(h)); }	142✔
180
181	template <typename Hint>
182	const Hint* get_hint() const {	12,151✔
183	static_assert(std::is_base_of_v<hint_base, Hint>, "Hint must extend hint_base");
184	for(auto& h : m_hints) {	12,538✔
185	if(auto* ptr = dynamic_cast<Hint*>(h.get()); ptr != nullptr) { return ptr; }	667!
186	}
187	return nullptr;	11,871✔
188	}
189
190	static std::unique_ptr<task> make_epoch(task_id tid, detail::epoch_action action, std::unique_ptr<task_promise> promise) {	1,386✔
191	return std::unique_ptr<task>(new task(tid, task_type::epoch, non_collective_group_id, task_geometry{}, {}, {}, {}, {}, action, std::move(promise)));	1,386!
192	}
193
194	static std::unique_ptr<task> make_host_compute(task_id tid, task_geometry geometry, host_task_launcher launcher, buffer_access_map access_map,	336✔
195	side_effect_map side_effect_map, reduction_set reductions) {
196	return std::unique_ptr<task>(new task(tid, task_type::host_compute, non_collective_group_id, geometry, std::move(launcher), std::move(access_map),	672✔
197	std::move(side_effect_map), std::move(reductions), {}, nullptr));	1,008!
198	}
199
200	static std::unique_ptr<task> make_device_compute(	1,329✔
201	task_id tid, task_geometry geometry, device_kernel_launcher launcher, buffer_access_map access_map, reduction_set reductions) {
202	return std::unique_ptr<task>(new task(tid, task_type::device_compute, non_collective_group_id, geometry, std::move(launcher), std::move(access_map),	2,658✔
203	{}, std::move(reductions), {}, nullptr));	3,987!
204	}
205
206	static std::unique_ptr<task> make_collective(task_id tid, task_geometry geometry, collective_group_id cgid, size_t num_collective_nodes,	61✔
207	host_task_launcher launcher, buffer_access_map access_map, side_effect_map side_effect_map) {
208	// The geometry is required to construct the buffer_access_map, so we pass it in here even though it has to have a specific shape
209	assert(geometry.dimensions == 1 && geometry.global_size == detail::range_cast<3>(range(num_collective_nodes)) && geometry.global_offset == zeros);	61✔
210	return std::unique_ptr<task>(
211	new task(tid, task_type::collective, cgid, geometry, std::move(launcher), std::move(access_map), std::move(side_effect_map), {}, {}, nullptr));	61!
212	}
213
214	static std::unique_ptr<task> make_master_node(task_id tid, host_task_launcher launcher, buffer_access_map access_map, side_effect_map side_effect_map) {	1,264✔
215	return std::unique_ptr<task>(new task(tid, task_type::master_node, non_collective_group_id, task_geometry{}, std::move(launcher),	1,264✔
216	std::move(access_map), std::move(side_effect_map), {}, {}, nullptr));	2,528!
217	}
218
219	static std::unique_ptr<task> make_horizon(task_id tid) {	850✔
220	return std::unique_ptr<task>(new task(tid, task_type::horizon, non_collective_group_id, task_geometry{}, {}, {}, {}, {}, {}, nullptr));	850!
221	}
222
223	static std::unique_ptr<task> make_fence(	90✔
224	task_id tid, buffer_access_map access_map, side_effect_map side_effect_map, std::unique_ptr<task_promise> promise) {
225	return std::unique_ptr<task>(new task(tid, task_type::fence, non_collective_group_id, task_geometry{}, {}, std::move(access_map),	90✔
226	std::move(side_effect_map), {}, {}, std::move(promise)));	180!
227	}
228
229	private:
230	task_id m_tid;
231	task_type m_type;
232	collective_group_id m_cgid;
233	task_geometry m_geometry;
234	command_group_launcher m_launcher;
235	buffer_access_map m_access_map;
236	detail::side_effect_map m_side_effects;
237	reduction_set m_reductions;
238	std::string m_debug_name;
239	detail::epoch_action m_epoch_action;
240	std::unique_ptr<task_promise> m_promise; // TODO keep user_allocation_id in struct task instead of inside task_promise
241	std::vector<std::unique_ptr<hint_base>> m_hints;
242
243	task(task_id tid, task_type type, collective_group_id cgid, task_geometry geometry, command_group_launcher launcher, buffer_access_map access_map,	5,078✔
244	detail::side_effect_map side_effects, reduction_set reductions, detail::epoch_action epoch_action, std::unique_ptr<task_promise> promise)
245	: m_tid(tid), m_type(type), m_cgid(cgid), m_geometry(geometry), m_launcher(std::move(launcher)), m_access_map(std::move(access_map)),	5,078✔
246	m_side_effects(std::move(side_effects)), m_reductions(std::move(reductions)), m_epoch_action(epoch_action), m_promise(std::move(promise)) {	10,156✔
247	assert(type == task_type::host_compute \|\| type == task_type::device_compute \|\| get_granularity().size() == 1);	5,078✔
248	// Only host tasks can have side effects
249	assert(this->m_side_effects.empty() \|\| type == task_type::host_compute \|\| type == task_type::collective \|\| type == task_type::master_node	5,078✔
250	\|\| type == task_type::fence);
251	}	5,078✔
252	};
253
254	[[nodiscard]] std::string print_task_debug_label(const task& tsk, bool title_case = false);
255
256	/// Determines which overlapping regions appear between write accesses when the iteration space of `tsk` is split into `chunks`.
257	std::unordered_map<buffer_id, region<3>> detect_overlapping_writes(const task& tsk, const box_vector<3>& chunks);
258
259	/// The task graph (TDAG) represents all cluster-wide operations, such as command group submissions and fences, and their interdependencies.
260	class task_graph : public graph<task> {}; // inheritance instead of type alias so we can forward declare task_graph
261
262	} // namespace detail
263	} // namespace celerity

celerity / celerity-runtime / 12009876465

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous