24845644334

Committed 23 Apr 2026 04:05PM UTC coverage: 64.104% (-0.06%) from 64.167%

Build # 24845644334

Build Type

push

github

Committed by

web-flow

Commit Message

Modular Compile process (#690)

 + DoccTarget to allow modularly adding support for snippets that need to be compiled differently (standalone, cross-compiled kernels or just with special compilers)
 + Builder infrastructure to define C/C++ build options with inheritance, overrides and ability to modify by plugins, extensions and targets as needed
 + The redirected handling of snippets based on the extension can contribute link-options when they have been used (the generic solution to highway support)
 + Model to handle any parallism for compile in its own class without having to touch the definitions and compile options
 + DoccPaths extended to resolve default plugin dirs (as used by et-plugin)
 + PrettyPrinter can write directly to another (file) stream instead of a local buffer
 * extended CodeGenerator to allow generating the header first and direct-to-file (its needed for all snippet & main compile, which could be parallel otherwise)
 + Defaults to parallel builds with as many cores as present
 + DOCC_DEBUG is more defined. A list of key-value pairs separated by ; or : Supported options: "dump" dump sdfgs, "build" add debug symbols, "build_threads=x" override thread count. less then 1 is the default auto-selection.
 + CodegenStats is integrated into the new Compiler concept. To reduce the impact of the asynchronicity, times are always measured, but only recored with the CodegenStats if enabled and later, in sequential places. The items recorded now include the sdfg & snippet names to identify slow parts
 + Switched MacOS tests over to inplace-python with only 1 build

Coverage Stats

224 of 347 new or added lines in 13 files covered. (64.55%)

2 existing lines in 2 files now uncovered.

30800 of 48047 relevant lines covered (64.1%)

573.39 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

72.12

/python/docc/compiler/docc_program.py

from abc import ABC, abstractmethod
import sys
from typing import Any, Optional
import os
import re

from docc.sdfg import StructuredSDFG
from docc.sdfg._sdfg import (
    _enable_statistics,
    _statistics_enabled_by_env,
    _statistics_summary,
)
from docc.compiler.compiled_sdfg import CompiledSDFG
from docc.compiler.target_registry import (
    get_target_schedule_fn,
    get_target_compile_fn,
    get_target_expand_fn,
)


def _parse_docc_debug() -> dict[str, str]:
    debug_env = os.environ.get("DOCC_DEBUG", "")
    debug_dict = {}
    if debug_env:
        for entry in re.split(r"[;:]", debug_env):
            if not entry:
                continue
            parts = entry.split("=", 1)
            key = parts[0].strip()
            value = parts[1].strip() if len(parts) > 1 else ""
            debug_dict[key] = value
    return debug_dict


def _is_debug_dump(flags: dict[str, str]) -> bool:
    return "dump" in flags


def _is_debug_compile(flags: dict[str, str]) -> bool:
    return "build" in flags


def _get_build_thread_count(flags: dict[str, str]) -> int:
    return int(flags.get("build_threads", "0"))


class DoccProgram(ABC):

    def __init__(
        self,
        name: str,
        target: str = "none",
        category: str = "server",
        instrumentation_mode: Optional[str] = None,
        capture_args: Optional[bool] = None,
        remote_tuning: bool = False,
    ):
        self.name = name
        self.target = target
        self.category = category
        self.remote_tuning = remote_tuning
        self.last_sdfg: Optional[StructuredSDFG] = None
        self.cache: dict = {}
        debug_flags = _parse_docc_debug()
        self.debug_dump: bool = _is_debug_dump(debug_flags)
        self.debug_build: bool = _is_debug_compile(debug_flags)
        self.build_thread_count: int = _get_build_thread_count(debug_flags)

        # Check environment variable DOCC_CI
        docc_ci = os.environ.get("DOCC_CI", "")
        if docc_ci:
            if docc_ci == "regions":
                if instrumentation_mode is None:
                    instrumentation_mode = "ols"
            elif docc_ci == "arg-capture":
                if capture_args is None:
                    capture_args = True
            else:
                # Full mode (or unknown value treated as full)
                if instrumentation_mode is None:
                    instrumentation_mode = "ols"
                if capture_args is None:
                    capture_args = True

        self.instrumentation_mode = instrumentation_mode
        self.capture_args = capture_args

    @abstractmethod
    def __call__(self, *args: Any) -> Any:
        pass

    @abstractmethod
    def compile(self, *args: Any, output_folder: Optional[str] = None) -> CompiledSDFG:
        pass

    def sdfg_pipe(
        self,
        sdfg: StructuredSDFG,
        output_folder: Optional[str],
        instrumentation_mode: str,
        capture_args: bool,
    ) -> str:

        if self.debug_dump:
            sdfg.dump(output_folder, "py0.parsed", dump_dot=True)

        # Enable statistics if envvar is set
        if _statistics_enabled_by_env():
            _enable_statistics()

        sdfg.validate()

        # Tensor targets keep tensor nodes
        custom_expand_fn = get_target_expand_fn(self.target)
        if custom_expand_fn is not None:
            custom_expand_fn(sdfg, self.category, {})
        else:
            sdfg.expand()
        if self.debug_dump:
            sdfg.dump(output_folder, "py1.expanded", dump_dot=True)

        # Simplify pipelines
        sdfg.simplify()
        if self.debug_dump:
            sdfg.dump(output_folder, "py2.opt", dump_dot=True)

        # Normalization for scheduling
        if self.target != "none":
            sdfg.normalize()

        if self.debug_dump or instrumentation_mode or capture_args:
            sdfg.dump(
                output_folder,
                "py3.norm",
                dump_dot=self.debug_dump,
                dump_json=True,
                record_for_instrumentation=True,
            )

        # Schedule if target is specified

        if self.target != "none":
            # Check for custom registered target first
            custom_schedule_fn = get_target_schedule_fn(self.target)
            if custom_schedule_fn is not None:
                custom_schedule_fn(
                    sdfg, self.category, {"remote_tuning": self.remote_tuning}
                )
            else:
                sdfg.schedule(self.target, self.category, self.remote_tuning)

            if self.debug_dump:
                sdfg.dump(output_folder, "py4.post_sched", dump_dot=True)

        self.last_sdfg = sdfg

        custom_compile_fn = get_target_compile_fn(self.target)
        if custom_compile_fn is not None:
            lib_path = custom_compile_fn(
                sdfg,
                output_folder,
                instrumentation_mode,
                capture_args,
                {"debug_build": self.debug_build, "threads": self.build_thread_count},
            )
        else:
            lib_path = sdfg._compile(
                output_folder=output_folder,
                target=self.target,
                instrumentation_mode=instrumentation_mode,
                capture_args=capture_args,
                debug_build=self.debug_build,
                threads=self.build_thread_count,
            )

        # Dump statistics after compile
        if _statistics_enabled_by_env():
            print(_statistics_summary(), file=sys.stderr)

        return lib_path

    @abstractmethod
    def to_sdfg(self, *args: Any) -> StructuredSDFG:
        pass

    @abstractmethod
    def _convert_inputs(self, args: tuple) -> tuple:
        pass

    @abstractmethod
    def _convert_outputs(self, result: Any, original_args: tuple) -> Any:
        pass

    def _get_cache_key(self, *args: Any) -> str:
        return ""

1	from abc import ABC, abstractmethod	4✔
2	import sys	4✔
3	from typing import Any, Optional	4✔
4	import os	4✔
5	import re	4✔
6
7	from docc.sdfg import StructuredSDFG	4✔
8	from docc.sdfg._sdfg import (	4✔
9	_enable_statistics,
10	_statistics_enabled_by_env,
11	_statistics_summary,
12	)
13	from docc.compiler.compiled_sdfg import CompiledSDFG	4✔
14	from docc.compiler.target_registry import (	4✔
15	get_target_schedule_fn,
16	get_target_compile_fn,
17	get_target_expand_fn,
18	)
19
20
21	def _parse_docc_debug() -> dict[str, str]:	4✔
22	debug_env = os.environ.get("DOCC_DEBUG", "")	4✔
23	debug_dict = {}	4✔
24	if debug_env:	4✔
NEW 25	for entry in re.split(r"[;:]", debug_env):	×
NEW 26	if not entry:	×
NEW 27	continue	×
NEW 28	parts = entry.split("=", 1)	×
NEW 29	key = parts[0].strip()	×
NEW 30	value = parts[1].strip() if len(parts) > 1 else ""	×
NEW 31	debug_dict[key] = value	×
32	return debug_dict	4✔
33
34
35	def _is_debug_dump(flags: dict[str, str]) -> bool:	4✔
36	return "dump" in flags	4✔
37
38
39	def _is_debug_compile(flags: dict[str, str]) -> bool:	4✔
40	return "build" in flags	4✔
41
42
43	def _get_build_thread_count(flags: dict[str, str]) -> int:	4✔
44	return int(flags.get("build_threads", "0"))	4✔
45
46
47	class DoccProgram(ABC):	4✔
48
49	def __init__(	4✔
50	self,
51	name: str,
52	target: str = "none",
53	category: str = "server",
54	instrumentation_mode: Optional[str] = None,
55	capture_args: Optional[bool] = None,
56	remote_tuning: bool = False,
57	):
58	self.name = name	4✔
59	self.target = target	4✔
60	self.category = category	4✔
61	self.remote_tuning = remote_tuning	4✔
62	self.last_sdfg: Optional[StructuredSDFG] = None	4✔
63	self.cache: dict = {}	4✔
64	debug_flags = _parse_docc_debug()	4✔
65	self.debug_dump: bool = _is_debug_dump(debug_flags)	4✔
66	self.debug_build: bool = _is_debug_compile(debug_flags)	4✔
67	self.build_thread_count: int = _get_build_thread_count(debug_flags)	4✔
68
69	# Check environment variable DOCC_CI
70	docc_ci = os.environ.get("DOCC_CI", "")	4✔
71	if docc_ci:	4✔
72	if docc_ci == "regions":	×
73	if instrumentation_mode is None:	×
74	instrumentation_mode = "ols"	×
75	elif docc_ci == "arg-capture":	×
76	if capture_args is None:	×
77	capture_args = True	×
78	else:
79	# Full mode (or unknown value treated as full)
80	if instrumentation_mode is None:	×
81	instrumentation_mode = "ols"	×
82	if capture_args is None:	×
83	capture_args = True	×
84
85	self.instrumentation_mode = instrumentation_mode	4✔
86	self.capture_args = capture_args	4✔
87
88	@abstractmethod	4✔
89	def __call__(self, *args: Any) -> Any:	4✔
90	pass	×
91
92	@abstractmethod	4✔
93	def compile(self, *args: Any, output_folder: Optional[str] = None) -> CompiledSDFG:	4✔
94	pass	×
95
96	def sdfg_pipe(	4✔
97	self,
98	sdfg: StructuredSDFG,
99	output_folder: Optional[str],
100	instrumentation_mode: str,
101	capture_args: bool,
102	) -> str:
103
104	if self.debug_dump:	4✔
105	sdfg.dump(output_folder, "py0.parsed", dump_dot=True)	×
106
107	# Enable statistics if envvar is set
108	if _statistics_enabled_by_env():	4✔
109	_enable_statistics()	×
110
111	sdfg.validate()	4✔
112
113	# Tensor targets keep tensor nodes
114	custom_expand_fn = get_target_expand_fn(self.target)	4✔
115	if custom_expand_fn is not None:	4✔
116	custom_expand_fn(sdfg, self.category, {})	4✔
117	else:
118	sdfg.expand()	4✔
119	if self.debug_dump:	4✔
120	sdfg.dump(output_folder, "py1.expanded", dump_dot=True)	×
121
122	# Simplify pipelines
123	sdfg.simplify()	4✔
124	if self.debug_dump:	4✔
125	sdfg.dump(output_folder, "py2.opt", dump_dot=True)	×
126
127	# Normalization for scheduling
128	if self.target != "none":	4✔
129	sdfg.normalize()	4✔
130
131	if self.debug_dump or instrumentation_mode or capture_args:	4✔
132	sdfg.dump(	4✔
133	output_folder,
134	"py3.norm",
135	dump_dot=self.debug_dump,
136	dump_json=True,
137	record_for_instrumentation=True,
138	)
139
140	# Schedule if target is specified
141
142	if self.target != "none":	4✔
143	# Check for custom registered target first
144	custom_schedule_fn = get_target_schedule_fn(self.target)	4✔
145	if custom_schedule_fn is not None:	4✔
146	custom_schedule_fn(	4✔
147	sdfg, self.category, {"remote_tuning": self.remote_tuning}
148	)
149	else:
150	sdfg.schedule(self.target, self.category, self.remote_tuning)	4✔
151
152	if self.debug_dump:	4✔
153	sdfg.dump(output_folder, "py4.post_sched", dump_dot=True)	×
154
155	self.last_sdfg = sdfg	4✔
156
157	custom_compile_fn = get_target_compile_fn(self.target)	4✔
158	if custom_compile_fn is not None:	4✔
159	lib_path = custom_compile_fn(	4✔
160	sdfg,
161	output_folder,
162	instrumentation_mode,
163	capture_args,
164	{"debug_build": self.debug_build, "threads": self.build_thread_count},
165	)
166	else:
167	lib_path = sdfg._compile(	4✔
168	output_folder=output_folder,
169	target=self.target,
170	instrumentation_mode=instrumentation_mode,
171	capture_args=capture_args,
172	debug_build=self.debug_build,
173	threads=self.build_thread_count,
174	)
175
176	# Dump statistics after compile
177	if _statistics_enabled_by_env():	4✔
178	print(_statistics_summary(), file=sys.stderr)	×
179
180	return lib_path	4✔
181
182	@abstractmethod	4✔
183	def to_sdfg(self, *args: Any) -> StructuredSDFG:	4✔
184	pass	×
185
186	@abstractmethod	4✔
187	def _convert_inputs(self, args: tuple) -> tuple:	4✔
188	pass	×
189
190	@abstractmethod	4✔
191	def _convert_outputs(self, result: Any, original_args: tuple) -> Any:	4✔
192	pass	×
193
194	def _get_cache_key(self, *args: Any) -> str:	4✔
195	return ""	×

daisytuner / docc / 24845644334

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous