22699554347

Committed 05 Mar 2026 02:33AM UTC coverage: 19.922% (-50.1%) from 70.041%

Build Type

push

github

Committed by Rui Hu

Commit Message

Merge github/main, resolve two version conflicts by keeping gitlab local

Coverage Stats

6 of 2484 new or added lines in 35 files covered. (0.24%)

871 of 4372 relevant lines covered (19.92%)

0.2 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/pipa/service/call_graph/func_node.py

NEW 1	from typing import Dict, List, Optional, Tuple	×
NEW 2	import json	×
NEW 3	import pandas as pd	×
NEW 4	import os	×
NEW 5	import time	×
NEW 6	from pipa.common.logger import logger	×
NEW 7	from pipa.common.utils import process_compression, FileFormat, check_file_format	×
NEW 8	from pipa.common.hardware.cpu import NUM_CORES_PHYSICAL	×
NEW 9	from collections import defaultdict	×
NEW 10	from tempfile import mkdtemp	×
11
12	# multi processing
NEW 13	from multiprocessing import Pool	×
14
15	# pyelftools
NEW 16	from elftools.elf.elffile import ELFFile	×
17
NEW 18	from pipa.service.call_graph.addr import (	×
19	DEFAULT_BUILD_ID_DIR,
20	disassemble_func,
21	addr2lines,
22	get_arch_mode,
23	get_symbol_addresses,
24	get_text_section,
25	)
26
NEW 27	from pipa.service.call_graph.node import Node, NodeEncoder, NodeTable	×
28
29
NEW 30	class FunctionNode:	×
31	"""
32	Represents a function node in the call graph.
33
34	Attributes:
35	func_name (str): The name of the function.
36	module_name (str): The name of the module containing the function.
37	nodes (list[Node] \| None): A list of child nodes, if any.
38	"""
39
NEW 40	def __init__(	×
41	self,
42	func_name: str,
43	module_name: str,
44	nodes: list[Node] \| None,
45	node_infos: Optional[List[Tuple]] = None,
46	source_codes: Optional[List[str]] = None,
47	):
48	"""
49	Initialize a CallGraph object.
50
51	Args:
52	func_name (str): The name of the function.
53	module_name (str): The name of the module.
54	nodes (list[Node] \| None): A list of Node objects representing the nodes in the call graph.
55	If None, the call graph is empty.
56	node_infos (list[Tuple], optional): A list of tuples containing information about the instructions / events and source codes.
57	source_codes (list[str], optional): Full source codes corresponding to the function node.
58	"""
NEW 59	self.func_name = func_name	×
NEW 60	self.module_name = module_name	×
NEW 61	self.nodes = nodes	×
NEW 62	self.node_infos = node_infos	×
NEW 63	self.source_codes = source_codes	×
NEW 64	self._cycles = sum([node.cycles for node in nodes]) if nodes else 0	×
NEW 65	self._instructions = sum([node.instructions for node in nodes]) if nodes else 0	×
66
NEW 67	def __str__(self):	×
NEW 68	return f"{self.func_name} {self.module_name}"	×
69
NEW 70	def __hash__(self) -> int:	×
NEW 71	return hash(str(self))	×
72
NEW 73	def set_node_infos(self, node_infos: List[Tuple]):	×
NEW 74	self.node_infos = node_infos	×
75
NEW 76	def set_source_codes(self, source_codes: List[str]):	×
NEW 77	self.source_codes = source_codes	×
78
NEW 79	def extend_node_infos(self, node_infos: List[Tuple]):	×
NEW 80	if not self.node_infos:	×
NEW 81	self.node_infos = node_infos	×
82	else:
NEW 83	self.node_infos.extend(node_infos)	×
84
NEW 85	def extend_source_codes(self, source_codes: List[str]):	×
NEW 86	if not self.source_codes:	×
NEW 87	self.source_codes = source_codes	×
88	else:
NEW 89	self.source_codes.extend(source_codes)	×
90
NEW 91	def get_cycles(self):	×
NEW 92	cycles_cur = sum([node.cycles for node in self.nodes]) if self.nodes else 0	×
NEW 93	self._cycles = cycles_cur	×
NEW 94	return cycles_cur	×
95
NEW 96	def get_instructions(self):	×
NEW 97	instructions_cur = (	×
98	sum([node.instructions for node in self.nodes]) if self.nodes else 0
99	)
NEW 100	self._instructions = instructions_cur	×
NEW 101	return instructions_cur	×
102
NEW 103	def get_events_values(self):	×
NEW 104	ev: Dict[str, int] = defaultdict(lambda: 0)	×
NEW 105	for n in self.nodes:	×
NEW 106	for e, v in n.events.items():	×
NEW 107	ev[e] += v	×
NEW 108	return ev	×
109
110
NEW 111	class FunctionNodeTable:	×
112	"""
113	A class representing a table of function nodes.
114
115	This class provides a dictionary-like interface to store and manipulate function nodes.
116
117	Attributes:
118	function_nodes (dict[str, FunctionNode]): A dictionary that stores function nodes.
119
120	Methods:
121	__getitem__(self, key: str) -> FunctionNode: Returns the function node associated with the given key.
122	__setitem__(self, key: str, value: FunctionNode): Sets the function node associated with the given key.
123	__iter__(self): Returns an iterator over the function nodes.
124	__len__(self): Returns the number of function nodes in the table.
125	__str__(self): Returns a string representation of the function node table.
126	__repr__(self): Returns a string representation of the function node table.
127	__contains__(self, key): Checks if the table contains a function node with the given key.
128	__delitem__(self, key): Deletes the function node associated with the given key.
129	__add__(self, other): Returns a new function node table that is the union of this table and another table.
130	__sub__(self, other): Returns a new function node table that contains the function nodes in this table but not in another table.
131	__and__(self, other): Returns a new function node table that contains the function nodes that are common to both this table and another table.
132	__or__(self, other): Returns a new function node table that is the union of this table and another table.
133	__xor__(self, other): Returns a new function node table that contains the function nodes that are in either this table or another table, but not in both.
134	__eq__(self, other): Checks if this function node table is equal to another function node table.
135	__ne__(self, other): Checks if this function node table is not equal to another function node table.
136	__lt__(self, other): Checks if this function node table is less than another function node table.
137	__le__(self, other): Checks if this function node table is less than or equal to another function node table.
138	__gt__(self, other): Checks if this function node table is greater than another function node table.
139	__ge__(self, other): Checks if this function node table is greater than or equal to another function node table.
140	from_node_table(cls, node_table: NodeTable): Creates a new function node table from a node table.
141
142	"""
143
NEW 144	def __init__(self, function_nodes: dict[str, FunctionNode] \| None = None):	×
NEW 145	self.function_nodes = function_nodes if function_nodes else {}	×
146
NEW 147	def __getitem__(self, key: str) -> FunctionNode:	×
NEW 148	return self.function_nodes[key]	×
149
NEW 150	def __setitem__(self, key: str, value: FunctionNode):	×
NEW 151	self.function_nodes[key] = value	×
152
NEW 153	def __iter__(self):	×
NEW 154	return iter(self.function_nodes)	×
155
NEW 156	def __len__(self):	×
NEW 157	return len(self.function_nodes)	×
158
NEW 159	def __str__(self):	×
NEW 160	return str(self.function_nodes)	×
161
NEW 162	def __repr__(self):	×
NEW 163	return repr(self.function_nodes)	×
164
NEW 165	def __contains__(self, key):	×
NEW 166	return key in self.function_nodes	×
167
NEW 168	def __delitem__(self, key):	×
NEW 169	del self.function_nodes[key]	×
170
NEW 171	def __add__(self, other):	×
NEW 172	return FunctionNodeTable({self.function_nodes, other.function_nodes})	×
173
NEW 174	def __sub__(self, other):	×
NEW 175	return FunctionNodeTable(	×
176	{
177	k: v
178	for k, v in self.function_nodes.items()
179	if k not in other.function_nodes
180	}
181	)
182
NEW 183	def __and__(self, other):	×
NEW 184	return FunctionNodeTable(	×
185	{k: v for k, v in self.function_nodes.items() if k in other.function_nodes}
186	)
187
NEW 188	def __or__(self, other):	×
NEW 189	return FunctionNodeTable({self.function_nodes, other.function_nodes})	×
190
NEW 191	def __xor__(self, other):	×
NEW 192	return FunctionNodeTable(	×
193	{
194	k: v
195	for k, v in self.function_nodes.items()
196	if k not in other.function_nodes
197	}
198	\| {
199	k: v
200	for k, v in other.function_nodes.items()
201	if k not in self.function_nodes
202	}
203	)
204
NEW 205	def __eq__(self, other):	×
NEW 206	return self.function_nodes == other.function_nodes	×
207
NEW 208	def __ne__(self, other):	×
NEW 209	return self.function_nodes != other.function_nodes	×
210
NEW 211	def __lt__(self, other):	×
NEW 212	return self.function_nodes < other.function_nodes	×
213
NEW 214	def __le__(self, other):	×
NEW 215	return self.function_nodes <= other.function_nodes	×
216
NEW 217	def __gt__(self, other):	×
NEW 218	return self.function_nodes > other.function_nodes	×
219
NEW 220	def __ge__(self, other):	×
NEW 221	return self.function_nodes >= other.function_nodes	×
222
NEW 223	@classmethod	×
NEW 224	def from_node_table(	×
225	cls,
226	node_table: NodeTable,
227	gen_epm: bool = False,
228	buildid_list: Dict[str, str] = {},
229	source_file_prefix: Optional[str] = None,
230	addr2lines_processes: int = NUM_CORES_PHYSICAL,
231	):
232	"""
233	Create a CallGraph object from a NodeTable.
234
235	Args:
236	node_table (NodeTable): The NodeTable object containing the nodes.
237	gen_epm (bool, optional): If True, generate EPM for each function node. Defaults to False.
238	buildid_list (Dict[str, str], optional): A dictionary containing build IDs. Not provided as default.
239	source_file_prefix (Optional[str], optional): The prefix of all source files. Useful when analysis machine is different from the collected machine. Defaults to None.
240	addr2lines_processes (int, optional): The number of processes used for addr2lines. Defaults to NUM_CORES_PHYSICAL.
241
242	Returns:
243	CallGraph: The CallGraph object created from the NodeTable.
244	"""
NEW 245	res: Dict[str, FunctionNode] = {}	×
NEW 246	for node in node_table._nodes.values():	×
NEW 247	method_name = node.get_function_name()	×
NEW 248	module_name = node.caller	×
NEW 249	k = f"{method_name} {module_name}"	×
NEW 250	if k not in res:	×
NEW 251	res[k] = FunctionNode(	×
252	func_name=method_name, module_name=module_name, nodes=[node]
253	)
254	else:
NEW 255	res[k].nodes.append(node) # type: ignore	×
NEW 256	if not gen_epm:	×
NEW 257	return cls(function_nodes=res)	×
258	# generate EPM
NEW 259	logger.debug("Start generate Extended Performance Metrics")	×
260	# RalESL: module -> function -> (offset, addr, cycles, more events metrics)
NEW 261	RawESL: Dict[str, Dict[str, List[Tuple[int, str, int, Dict[str, int]]]]] = (	×
262	defaultdict(lambda: defaultdict(lambda: []))
263	)
NEW 264	for func_node in res.values():	×
NEW 265	for n in func_node.nodes: # type: ignore	×
NEW 266	if n.function_offset is None:	×
NEW 267	continue	×
NEW 268	RawESL[func_node.module_name][func_node.func_name].append(	×
269	(n.function_offset, n.addr, n.cycles, n.events.copy())
270	)
271	# Extended Performance Metrics
NEW 272	EPM: Dict[Tuple[str, str], Dict[str, List[Tuple]]] = defaultdict(	×
273	lambda: defaultdict(lambda: [])
274	)
NEW 275	pool = Pool(addr2lines_processes)	×
NEW 276	for module, funcs in RawESL.items():	×
277	# determine the module for analysis
NEW 278	esl_module = module	×
NEW 279	buildid = buildid_list.get(module)	×
NEW 280	debug_module = None	×
NEW 281	if buildid:	×
NEW 282	debug_module = os.path.join(	×
283	DEFAULT_BUILD_ID_DIR, module.strip("/"), buildid, "elf"
284	)
NEW 285	if debug_module is not None and os.path.exists(debug_module):	×
NEW 286	logger.debug(	×
287	f"Found module {module}'s seperated debuginfo file: {debug_module}"
288	)
NEW 289	module = debug_module	×
NEW 290	elif not os.path.exists(module):	×
NEW 291	logger.warning(f"Not found ELF File {module}")	×
NEW 292	continue	×
NEW 293	logger.debug(f"Start analyze ELF File {module}")	×
294	# check if it's an elf file with debuginfo
295	# if it's a compress file, extract to a tmpdir and will use the extracted elf file (if it contains) for further processing
296	# if it's not a compress file or elf file, pass
NEW 297	fformat = check_file_format(module)	×
NEW 298	if fformat == FileFormat.xz:	×
299	# buildid will generate a xz compressed file named like drm_vram_helper.ko.xz
300	# it contains debuginfo elf, named like drm_vram_helper.ko
NEW 301	tmpd = mkdtemp()	×
NEW 302	extracted, _ = os.path.splitext(os.path.basename(module))	×
NEW 303	extracted = os.path.join(tmpd, extracted)	×
NEW 304	process_compression(	×
305	compressed=module,
306	decompressed=extracted,
307	format=FileFormat.xz,
308	decompress=True,
309	)
NEW 310	if not os.path.exists(extracted):	×
NEW 311	logger.warning(	×
312	f"Extract {module} to {tmpd}. But expected {extracted} not found"
313	)
NEW 314	continue	×
NEW 315	module = extracted	×
NEW 316	elif fformat != FileFormat.elf:	×
NEW 317	continue	×
318	# open elf file
NEW 319	f = open(module, "rb")	×
NEW 320	elffile = ELFFile(f)	×
NEW 321	if not elffile.has_dwarf_info():	×
NEW 322	logger.warning(f"{module} has no dwarf info, please provide debuginfo")	×
NEW 323	f.close()	×
NEW 324	continue	×
325	# judge arch and mod
NEW 326	try:	×
NEW 327	arch, mode = get_arch_mode(elffile)	×
NEW 328	except NotImplementedError as e:	×
NEW 329	logger.warning(f"{module} has unsupported arch or mode: {e}")	×
NEW 330	f.close()	×
NEW 331	continue	×
332	# get dwarf info
NEW 333	dwarfinfo = elffile.get_dwarf_info()	×
334	# check dwarf info has debug info
NEW 335	if not dwarfinfo.has_debug_info:	×
NEW 336	logger.warning(	×
337	f"{module}'s dwarf lost debuginfo, source codes may not be found. Please check your compile methods"
338	)
339	# get symbol table info
NEW 340	symtable = elffile.get_section_by_name(".symtab")	×
NEW 341	if symtable is None:	×
NEW 342	logger.warning(	×
343	f"Not found symtable in {module}, please provide debuginfo."
344	)
NEW 345	f.close()	×
NEW 346	continue	×
347	# get all functions' start address / name / size in the elffile
NEW 348	function_address_size_info = get_symbol_addresses(	×
349	elffile=elffile, func_name=None
350	)
351	# get text section's info to calculate the offset of each function
NEW 352	try:	×
NEW 353	text_data, text_addr = get_text_section(elffile=elffile)	×
NEW 354	text_data_len = len(text_data)	×
NEW 355	except ValueError as e:	×
NEW 356	logger.warning(f"{module} has no text section: {e}")	×
NEW 357	f.close()	×
NEW 358	continue	×
359	# get module/function info (start addr, func size in bytes)
360	# ip_perfs: list of (address, ip, cycles, more events metrics) (detected performance metrics)
361	# func_asm: key: address, dict list of (mnemonic, op_str)
362	# func_sourcelines: list of (source file, address, line, column)
363	# combine info to FunctionNode
NEW 364	for func_n, ip_perfs in funcs.items():	×
365	# get function info by function name
NEW 366	func_info = function_address_size_info.get(func_n)	×
NEW 367	if func_info is None:	×
NEW 368	continue	×
369
370	# calculate function offset
NEW 371	func_addr, func_size = func_info	×
NEW 372	func_offset = func_addr - text_addr	×
NEW 373	if func_offset < 0 or func_offset + func_size > text_data_len:	×
NEW 374	logger.warning(	×
375	f"function {func_n}'s offset is out of the .text section's range"
376	)
NEW 377	continue	×
378
379	# start disassemble function
NEW 380	func_asm = disassemble_func(	×
381	text_data[func_offset : func_offset + func_size],
382	func_addr,
383	arch,
384	mode,
385	)
386
387	# get function's addresses
NEW 388	func_addrs = [k for k in func_asm.keys()]	×
389
390	# start addr to source lines
NEW 391	stime = time.perf_counter()	×
NEW 392	func_sourcelines = addr2lines(dwarfinfo, func_addrs, pool=pool)	×
NEW 393	etime = time.perf_counter()	×
NEW 394	logger.debug(	×
395	f"End symbolize {func_n} in {module} within {etime - stime} seconds"
396	)
397
398	# func_node_k should be equal to what in ESL.
NEW 399	func_node_k = f"{func_n} {esl_module}"	×
NEW 400	for addr, asm in func_asm.items():	×
NEW 401	addr_ips = []	×
NEW 402	addr_cycles = 0	×
NEW 403	addr_source_file = ""	×
NEW 404	addr_relative_dir = ""	×
NEW 405	addr_line = -1	×
NEW 406	addr_column = -1	×
NEW 407	addr_mnemonic = asm[0]	×
NEW 408	addr_op_str = asm[1]	×
NEW 409	addr_other_events: Dict[str, int] = defaultdict(lambda: 0)	×
410	# TODO seems like we can use addr as func_sourcelines's key, as it may be same as the key in func_asm, each key can store multi data
NEW 411	for func_source_mapping_info in func_sourcelines:	×
NEW 412	source_file, lsaddr, lsline, lscolumn, lsrelative_dir = (	×
413	func_source_mapping_info
414	)
NEW 415	if lsaddr == addr:	×
NEW 416	addr_source_file = source_file	×
NEW 417	addr_relative_dir = lsrelative_dir	×
NEW 418	addr_line = lsline	×
NEW 419	addr_column = lscolumn	×
NEW 420	break	×
NEW 421	for iperf in ip_perfs:	×
NEW 422	ioffset, ip, ic, other_events = iperf	×
NEW 423	if ioffset + func_addr == addr:	×
NEW 424	addr_ips.append(ip)	×
NEW 425	addr_cycles += ic	×
NEW 426	for e, v in other_events.items():	×
NEW 427	addr_other_events[e] += v	×
428	# when source codes (debuginfo) lost, the key will be ("", "")
429	# key is (source_file, relative_dir)
430	# some func / module may share same sourcefiles, use this kind of key to reduce times of reading sourcecodes
NEW 431	EPM[(addr_source_file, addr_relative_dir)][func_node_k].append(	×
432	(
433	addr,
434	addr_ips,
435	addr_cycles,
436	addr_line,
437	addr_column,
438	addr_mnemonic,
439	addr_op_str,
440	addr_other_events,
441	)
442	)
443	# at last close module file
NEW 444	f.close()	×
NEW 445	pool.close()	×
NEW 446	pool.join()	×
447	# start get source codes
NEW 448	for (source_file_raw, relative_dir), func_nodes in EPM.items():	×
449	# source_file_prefix ignores /
450	# get source file's contents
NEW 451	source_file_with_prefix = (	×
452	f"{source_file_prefix}/{source_file_raw}"
453	if source_file_prefix
454	else source_file_raw
455	)
NEW 456	source_file = source_file_with_prefix	×
NEW 457	if not os.path.exists(source_file_with_prefix):	×
NEW 458	relative_source_file_with_prefix = os.path.join(	×
459	relative_dir, source_file_raw
460	)
NEW 461	if source_file_prefix:	×
NEW 462	relative_source_file_with_prefix = (	×
463	f"{source_file_prefix}/{relative_source_file_with_prefix}"
464	)
NEW 465	source_file = relative_source_file_with_prefix	×
NEW 466	if os.path.isdir(source_file):	×
NEW 467	logger.warning(f"{source_file} is directory, can't read")	×
NEW 468	continue	×
NEW 469	if not os.path.exists(source_file):	×
NEW 470	logger.warning(	×
471	f"source file {source_file} couldn't found, please check"
472	)
NEW 473	continue	×
NEW 474	with open(source_file, "r") as f:	×
NEW 475	source_conts = f.readlines()	×
476	# for each function node, get its info and stored it in its node_info attrs
NEW 477	for func_node_k, addr_infos in func_nodes.items():	×
NEW 478	lines = []	×
NEW 479	func_node_infos = []	×
NEW 480	for addr_info in addr_infos:	×
NEW 481	addr_line, addr_column = addr_info[3], addr_info[4]	×
NEW 482	if addr_line > 0 and addr_column > 0:	×
NEW 483	lines.append(addr_line - 1)	×
NEW 484	addr_conts = source_conts[addr_line - 1]	×
485	else:
NEW 486	continue	×
NEW 487	node_info = (*addr_info, addr_conts, source_file)	×
NEW 488	func_node_infos.append(node_info)	×
NEW 489	lines.sort()	×
490	# a function may has multiple corresponding source files
491	# a sourcefile may has multiple corresponding functions
NEW 492	res[func_node_k].extend_source_codes(	×
493	source_conts[lines[0] : lines[-1] + 1]
494	)
NEW 495	res[func_node_k].extend_node_infos(func_node_infos)	×
NEW 496	return cls(function_nodes=res)	×
497
NEW 498	def to_dataframe(self):	×
NEW 499	data = [	×
500	{
501	"function_name": v.func_name,
502	"module_name": v.module_name,
503	"cycles": v.get_cycles(),
504	"instructions": v.get_instructions(),
505	}
506	for v in self.function_nodes.values()
507	]
NEW 508	return pd.DataFrame(data)	×
509
NEW 510	def copy(self):	×
NEW 511	return FunctionNodeTable({k: v for k, v in self.function_nodes.items()})	×
512
513
NEW 514	class ClusterEncoder(json.JSONEncoder):	×
NEW 515	def default(self, obj):	×
NEW 516	if isinstance(obj, Node):	×
NEW 517	return NodeEncoder().default(obj)	×
NEW 518	return super().default(obj)	×

ZJU-SPAIL / pipa / 22699554347

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous