• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

daisytuner / sdfglib / 21469091322

29 Jan 2026 07:12AM UTC coverage: 65.843% (+0.1%) from 65.732%
21469091322

push

github

web-flow
Merge pull request #484 from daisytuner/python-gather

adds support for Python gather operations

190 of 240 new or added lines in 4 files covered. (79.17%)

1 existing line in 1 file now uncovered.

22407 of 34031 relevant lines covered (65.84%)

383.67 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.18
/python/docc/compiled_sdfg.py
1
import ctypes
4✔
2
from ._sdfg import Scalar, Array, Pointer, Structure, PrimitiveType
4✔
3

4
try:
4✔
5
    import numpy as np
4✔
6
except ImportError:
×
7
    np = None
×
8

9
_CTYPES_MAP = {
4✔
10
    PrimitiveType.Bool: ctypes.c_bool,
11
    PrimitiveType.Int8: ctypes.c_int8,
12
    PrimitiveType.Int16: ctypes.c_int16,
13
    PrimitiveType.Int32: ctypes.c_int32,
14
    PrimitiveType.Int64: ctypes.c_int64,
15
    PrimitiveType.UInt8: ctypes.c_uint8,
16
    PrimitiveType.UInt16: ctypes.c_uint16,
17
    PrimitiveType.UInt32: ctypes.c_uint32,
18
    PrimitiveType.UInt64: ctypes.c_uint64,
19
    PrimitiveType.Float: ctypes.c_float,
20
    PrimitiveType.Double: ctypes.c_double,
21
}
22

23

24
class CompiledSDFG:
4✔
25
    def __init__(
4✔
26
        self,
27
        lib_path,
28
        sdfg,
29
        shape_sources=None,
30
        structure_member_info=None,
31
        output_args=None,
32
        output_shapes=None,
33
    ):
34
        self.lib_path = lib_path
4✔
35
        self.sdfg = sdfg
4✔
36
        self.shape_sources = shape_sources or []
4✔
37
        self.structure_member_info = structure_member_info or {}
4✔
38
        self.lib = ctypes.CDLL(lib_path)
4✔
39
        self.func = getattr(self.lib, sdfg.name)
4✔
40

41
        # Check for output args
42
        self.output_args = output_args or []
4✔
43
        if not self.output_args and hasattr(sdfg, "metadata"):
4✔
44
            out_args_str = sdfg.metadata("output_args")
4✔
45
            if out_args_str:
4✔
46
                self.output_args = out_args_str.split(",")
×
47

48
        self.output_shapes = output_shapes or {}
4✔
49

50
        # Cache for ctypes structure definitions
51
        self._ctypes_structures = {}
4✔
52

53
        # Set up argument types
54
        self.arg_names = sdfg.arguments
4✔
55
        self.arg_types = []
4✔
56
        self.arg_sdfg_types = []  # Keep track of original sdfg types
4✔
57
        for arg_name in sdfg.arguments:
4✔
58
            arg_type = sdfg.type(arg_name)
4✔
59
            self.arg_sdfg_types.append(arg_type)
4✔
60
            ct_type = self._get_ctypes_type(arg_type)
4✔
61
            self.arg_types.append(ct_type)
4✔
62

63
        self.func.argtypes = self.arg_types
4✔
64

65
        # Set up return type
66
        self.func.restype = self._get_ctypes_type(sdfg.return_type)
4✔
67

68
    def _convert_to_python_syntax(self, expr_str):
4✔
69
        """Convert SDFG parentheses notation to Python bracket notation.
70

71
        SDFG uses parentheses for array indexing (e.g., "A_row(0)") while Python
72
        uses brackets (e.g., "A_row[0]"). This method converts the notation for
73
        runtime evaluation.
74

75
        Examples:
76
            "A_row(0)" -> "A_row[0]"
77
            "(A_row(1) - A_row(0))" -> "(A_row[1] - A_row[0])"
78
        """
79
        import re
4✔
80

81
        result = expr_str
4✔
82

83
        while True:
4✔
84
            pattern = r"([a-zA-Z_][a-zA-Z0-9_]*)\(([^()]+)\)"
4✔
85
            match = re.search(pattern, result)
4✔
86
            if not match:
4✔
87
                break
4✔
88

NEW
89
            name = match.group(1)
×
NEW
90
            index = match.group(2)
×
91

92
            # Skip known function names
NEW
93
            known_functions = {"int", "float", "abs", "min", "max", "sum", "len"}
×
NEW
94
            if name.lower() in known_functions:
×
NEW
95
                placeholder = f"__FUNC__{name}__{index}__"
×
NEW
96
                result = result[: match.start()] + placeholder + result[match.end() :]
×
97
            else:
NEW
98
                result = (
×
99
                    result[: match.start()] + f"{name}[{index}]" + result[match.end() :]
100
                )
101

102
        result = re.sub(
4✔
103
            r"__FUNC__([a-zA-Z_][a-zA-Z0-9_]*)__([^_]+)__", r"\1(\2)", result
104
        )
105

106
        return result
4✔
107

108
    def _create_ctypes_structure(self, struct_name):
4✔
109
        """Create a ctypes Structure class for the given structure name."""
110
        if struct_name in self._ctypes_structures:
4✔
111
            return self._ctypes_structures[struct_name]
×
112

113
        if struct_name not in self.structure_member_info:
4✔
114
            raise ValueError(f"Structure '{struct_name}' not found in member info")
×
115

116
        # Get member info: {member_name: (index, type)}
117
        members = self.structure_member_info[struct_name]
4✔
118
        # Sort by index to get correct order
119
        sorted_members = sorted(members.items(), key=lambda x: x[1][0])
4✔
120

121
        # Build _fields_ for ctypes.Structure
122
        fields = []
4✔
123
        for member_name, (index, member_type) in sorted_members:
4✔
124
            ct_type = self._get_ctypes_type(member_type)
4✔
125
            fields.append((member_name, ct_type))
4✔
126

127
        # Create the ctypes Structure class dynamically
128
        class CStructure(ctypes.Structure):
4✔
129
            _fields_ = fields
4✔
130

131
        self._ctypes_structures[struct_name] = CStructure
4✔
132
        return CStructure
4✔
133

134
    def _get_ctypes_type(self, sdfg_type):
4✔
135
        if isinstance(sdfg_type, Scalar):
4✔
136
            return _CTYPES_MAP.get(sdfg_type.primitive_type, ctypes.c_void_p)
4✔
137
        elif isinstance(sdfg_type, Array):
4✔
138
            # Arrays are passed as pointers
139
            elem_type = _CTYPES_MAP.get(sdfg_type.primitive_type, ctypes.c_void_p)
×
140
            return ctypes.POINTER(elem_type)
×
141
        elif isinstance(sdfg_type, Pointer):
4✔
142
            # Check if pointee is a Structure
143
            # Note: has_pointee_type() is guaranteed to exist on Pointer instances from C++ bindings
144
            if sdfg_type.has_pointee_type():
4✔
145
                pointee = sdfg_type.pointee_type
4✔
146
                if isinstance(pointee, Structure):
4✔
147
                    # Create ctypes structure and return pointer to it
148
                    struct_class = self._create_ctypes_structure(pointee.name)
4✔
149
                    return ctypes.POINTER(struct_class)
4✔
150
                elif isinstance(pointee, Scalar):
4✔
151
                    elem_type = _CTYPES_MAP.get(pointee.primitive_type, ctypes.c_void_p)
4✔
152
                    return ctypes.POINTER(elem_type)
4✔
153
            return ctypes.c_void_p
×
154
        return ctypes.c_void_p
×
155

156
    def __call__(self, *args):
4✔
157
        # Identify user arguments vs implicit arguments (shapes, return values)
158

159
        # 1. Compute shape symbol values from user args input
160
        shape_symbol_values = {}
4✔
161
        for u_idx, dim_idx in self.shape_sources:
4✔
162
            if u_idx < len(args):
4✔
163
                val = args[u_idx].shape[dim_idx]
4✔
164
                s_idx = self.shape_sources.index((u_idx, dim_idx))
4✔
165
                shape_symbol_values[f"_s{s_idx}"] = val
4✔
166

167
        # Add input arrays to the shape context for expressions with indirect access
168
        # This allows evaluating expressions like A_row[0] at runtime
169
        user_arg_idx = 0
4✔
170
        for name in self.arg_names:
4✔
171
            if name in self.output_args:
4✔
172
                continue
4✔
173
            if name.startswith("_s") and name[2:].isdigit():
4✔
174
                continue
4✔
175

176
            # Must be a user parameter - add it to shape context if it's an array
177
            if user_arg_idx < len(args):
4✔
178
                val = args[user_arg_idx]
4✔
179
                if isinstance(val, (int, float, np.integer, np.floating)):
4✔
180
                    shape_symbol_values[name] = val
4✔
181
                elif np is not None and isinstance(val, np.ndarray):
4✔
182
                    # Add numpy arrays to context for indirect access shape evaluation
183
                    shape_symbol_values[name] = val
4✔
184
                user_arg_idx += 1
4✔
185

186
        param_arg_idx = 0
4✔
187
        for name in self.arg_names:
4✔
188
            if name in self.output_args:
4✔
189
                continue
4✔
190
            if name.startswith("_s") and name[2:].isdigit():
4✔
191
                continue
4✔
192

193
            # Must be a user parameter
194
            if param_arg_idx < len(args):
4✔
195
                val = args[param_arg_idx]
4✔
196
                if isinstance(val, (int, float, np.integer, np.floating)):
4✔
197
                    shape_symbol_values[name] = val
4✔
198
                param_arg_idx += 1
4✔
199

200
        converted_args = []
4✔
201
        structure_refs = []
4✔
202
        return_buffers = {}
4✔
203

204
        next_user_arg_idx = 0
4✔
205

206
        for i, arg_name in enumerate(self.arg_names):
4✔
207
            target_type = self.arg_types[i]
4✔
208

209
            if arg_name in self.output_args:
4✔
210
                base_type = target_type._type_
4✔
211

212
                # If array (pointer type) and we have shape info, we need to allocate array.
213
                # If not in output_shapes, assume scalar return (pointer to single value).
214
                if arg_name in self.output_shapes:
4✔
215
                    size = 1
4✔
216
                    dims = self.output_shapes[arg_name]
4✔
217
                    # Evaluate
218
                    for dim_str in dims:
4✔
219
                        try:
4✔
220
                            # Convert SDFG parentheses notation to Python bracket notation
221
                            # e.g., "A_row(0)" -> "A_row[0]"
222
                            eval_str = self._convert_to_python_syntax(str(dim_str))
4✔
223
                            val = eval(eval_str, {}, shape_symbol_values)
4✔
224
                            size *= int(val)
4✔
225
                        except Exception as e:
×
226
                            raise RuntimeError(
×
227
                                f"Could not evaluate shape {dim_str} for {arg_name}: {e}"
228
                            )
229

230
                    buf_type = base_type * size
4✔
231
                    buf = buf_type()
4✔
232
                    return_buffers[arg_name] = (buf, size, dims)
4✔
233
                    converted_args.append(
4✔
234
                        ctypes.cast(ctypes.addressof(buf), target_type)
235
                    )
236
                    continue
4✔
237

238
                # Scalar Return (Pointer(Scalar))
239
                buf = base_type()
4✔
240
                return_buffers[arg_name] = (buf, 1, None)
4✔
241
                converted_args.append(ctypes.byref(buf))
4✔
242
                continue
4✔
243

244
            if arg_name.startswith("_s") and arg_name[2:].isdigit():
4✔
245
                s_idx = int(arg_name[2:])
4✔
246
                if f"_s{s_idx}" in shape_symbol_values:
4✔
247
                    val = shape_symbol_values[f"_s{s_idx}"]
4✔
248
                    converted_args.append(ctypes.c_int64(val))
4✔
249
                else:
250
                    converted_args.append(ctypes.c_int64(0))
×
251
                continue
4✔
252

253
            # User Argument
254
            if next_user_arg_idx >= len(args):
4✔
255
                raise ValueError("Not enough arguments provided")
×
256

257
            arg = args[next_user_arg_idx]
4✔
258
            next_user_arg_idx += 1
4✔
259

260
            # ... Conversion logic (numpy to ctypes) ...
261
            sdfg_type = self.arg_sdfg_types[i]
4✔
262

263
            if np is not None and isinstance(arg, np.ndarray):
4✔
264
                if hasattr(target_type, "contents"):
4✔
265
                    converted_args.append(arg.ctypes.data_as(target_type))
4✔
266
                else:
267
                    converted_args.append(arg)
×
268
            elif (
4✔
269
                sdfg_type
270
                and isinstance(sdfg_type, Pointer)
271
                and sdfg_type.has_pointee_type()
272
                and isinstance(sdfg_type.pointee_type, Structure)
273
            ):
274
                # Struct logic
275
                struct_name = sdfg_type.pointee_type.name
4✔
276
                struct_class = self._ctypes_structures.get(struct_name)
4✔
277
                members = self.structure_member_info[struct_name]
4✔
278
                sorted_members = sorted(members.items(), key=lambda x: x[1][0])
4✔
279
                struct_values = {}
4✔
280
                for member_name, (index, member_type) in sorted_members:
4✔
281
                    if hasattr(arg, member_name):
4✔
282
                        struct_values[member_name] = getattr(arg, member_name)
4✔
283
                c_struct = struct_class(**struct_values)
4✔
284
                structure_refs.append(c_struct)
4✔
285
                converted_args.append(ctypes.pointer(c_struct))
4✔
286
            else:
287
                converted_args.append(
4✔
288
                    target_type(arg)
289
                )  # Explicit cast to ensure int stays int
290

291
        self.func(*converted_args)
4✔
292

293
        # Process returns
294
        results = []
4✔
295
        sorted_ret_names = sorted(
4✔
296
            return_buffers.keys(), key=lambda x: int(x.split("_")[-1])
297
        )
298

299
        for name in sorted_ret_names:
4✔
300
            buf, size, dims = return_buffers[name]
4✔
301
            if size == 1 and dims is None:
4✔
302
                # Scalar
303
                # buf is c_double / c_int instance
304
                results.append(buf.value)
4✔
305
            else:
306
                # Array
307
                # buf is (c_double * size) instance.
308
                # Convert to numpy
309
                if np is not None:
4✔
310
                    # Create numpy array from buffer
311
                    arr = np.ctypeslib.as_array(buf)  # 1D
4✔
312
                    if dims:
4✔
313
                        # Reshape
314
                        try:
4✔
315
                            shape = []
4✔
316
                            for dim_str in dims:
4✔
317
                                eval_str = self._convert_to_python_syntax(str(dim_str))
4✔
318
                                val = eval(eval_str, {}, shape_symbol_values)
4✔
319
                                shape.append(int(val))
4✔
320
                            arr = arr.reshape(shape)
4✔
321
                        except:
×
322
                            pass
×
323
                    results.append(arr)
4✔
324
                else:
325
                    # fallback list
326
                    results.append(list(buf))
×
327

328
        if len(results) == 1:
4✔
329
            return results[0]
4✔
330
        elif len(results) > 1:
4✔
331
            return tuple(results)
4✔
332

333
        return None
4✔
334

335
    def get_return_shape(self, *args):
4✔
336
        shape_str = self.sdfg.metadata("return_shape")
4✔
337
        if not shape_str:
4✔
338
            return None
4✔
339

340
        shape_exprs = shape_str.split(",")
×
341

342
        # Reconstruct shape values
343
        shape_values = {}
×
344
        for i, (arg_idx, dim_idx) in enumerate(self.shape_sources):
×
345
            arg = args[arg_idx]
×
346
            if np is not None and isinstance(arg, np.ndarray):
×
347
                val = arg.shape[dim_idx]
×
348
                shape_values[f"_s{i}"] = val
×
349

350
        # Add scalar arguments to shape_values
351
        # We assume the first len(args) arguments in sdfg.arguments correspond to the user arguments
352
        if hasattr(self.sdfg, "arguments"):
×
353
            for arg_name, arg_val in zip(self.sdfg.arguments, args):
×
354
                if isinstance(arg_val, (int, np.integer)):
×
355
                    shape_values[arg_name] = int(arg_val)
×
356

357
        evaluated_shape = []
×
358
        for expr in shape_exprs:
×
359
            # Simple evaluation using eval with shape_values
360
            # Warning: eval is unsafe, but here expressions come from our compiler
361
            try:
×
362
                val = eval(expr, {}, shape_values)
×
363
                evaluated_shape.append(int(val))
×
364
            except Exception:
×
365
                return None
×
366

367
        return tuple(evaluated_shape)
×
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc