22699554347

Committed 05 Mar 2026 02:33AM UTC coverage: 19.922% (-50.1%) from 70.041%

Build Type

push

github

Committed by Rui Hu

Commit Message

Merge github/main, resolve two version conflicts by keeping gitlab local

Coverage Stats

6 of 2484 new or added lines in 35 files covered. (0.24%)

871 of 4372 relevant lines covered (19.92%)

0.2 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/pipa/parser/perf_script_call.py

NEW 1	import re	×
NEW 2	import pandas as pd	×
NEW 3	from multiprocessing import Pool	×
NEW 4	from typing import Optional, Dict, Set, List	×
NEW 5	from decimal import Decimal, InvalidOperation	×
NEW 6	from pipa.common.hardware.cpu import NUM_CORES_PHYSICAL	×
NEW 7	from pipa.common.logger import logger	×
NEW 8	from collections import defaultdict	×
9
10
11	# Data classes for perf script parsing and processing
NEW 12	class PerfScriptCall:	×
13	"""
14	Represents a single performance script call.
15
16	Attributes:
17	addr (str): The address of the call.
18	symbol (str): The symbol associated with the call.
19	caller (str): The caller of the call.
20	"""
21
NEW 22	def __init__(self, addr: str, symbol: str, caller: str):	×
NEW 23	self.addr: str = addr	×
NEW 24	self.symbol: str = symbol	×
NEW 25	self.caller: str = caller	×
26
NEW 27	def __str__(self):	×
NEW 28	return f"{self.addr} {self.symbol} ({self.caller})"	×
29
30
NEW 31	class PerfScriptHeader:	×
32	"""
33	Represents a header in a perf script block.
34
35	Attributes:
36	command (str): The command associated with the record.
37	pid (int): The process ID associated with the record.
38	cpu (int): The CPU number associated with the record.
39	time (str): The time associated with the record.
40	value (int): The value associated with the record.
41	event (str): The event associated with the record.
42	"""
43
NEW 44	def __init__(	×
45	self, command: str, pid: int, cpu: int, time: str, value: int, event: str
46	):
NEW 47	self.command: str = command	×
NEW 48	self.pid: int = pid	×
NEW 49	self.cpu: int = cpu	×
50	# the perf script time field format is x.y
51	# default unit is seconds.microseconds
52	# when append --ns param, unit is seconds.nanoseconds
53	# when append --reltime param, it starts from 0.0 at the begining
NEW 54	self.time: str = time	×
NEW 55	self.xytime: Decimal = Decimal(time)	×
NEW 56	self.value: int = value	×
NEW 57	self.event: str = event	×
58
NEW 59	def __str__(self):	×
NEW 60	return f"{self.command} {self.pid} {self.cpu} {self.time} {self.value} {self.event}"	×
61
62
NEW 63	class PerfScriptBlock:	×
64	"""
65	Represents a block of performance script.
66
67	Attributes:
68	header (PerfScriptHeader): The header of the performance script block.
69	calls (list[PerfScriptCall]): The list of performance script calls.
70
71	Methods:
72	__str__(): Returns a string representation of the PerfScriptBlock object.
73	to_record(): Converts the PerfScriptBlock object to a record.
74	"""
75
NEW 76	def __init__(self, header: PerfScriptHeader, calls: List[PerfScriptCall]):	×
NEW 77	self.header: PerfScriptHeader = header	×
NEW 78	self.calls: List[PerfScriptCall] = calls	×
79
NEW 80	def __str__(self):	×
NEW 81	return f"{self.header}\n{self.calls}"	×
82
NEW 83	def to_record(self):	×
84	"""
85	Converts the PerfScriptBlock object to a record.
86
87	Returns:
88	dict: A dictionary containing the record data.
89	"""
NEW 90	return {	×
91	"command": self.header.command,
92	"pid": self.header.pid,
93	"cpu": self.header.cpu,
94	"time": self.header.time,
95	"value": self.header.value,
96	"event": self.header.event,
97	"calls": [str(c) for c in self.calls],
98	}
99
100
101	# Parser class for performance script data
NEW 102	class PerfScriptParser:	×
103	"""
104	A class responsible for parsing performance script data.
105	"""
106
NEW 107	@staticmethod	×
NEW 108	def parse_one_call(line: str) -> Optional[List[str]]:	×
109	"""
110	Parses a single line of a performance script call and returns the parsed values.
111
112	Args:
113	line (str): The line to parse.
114
115	Returns:
116	list: A list containing the parsed values [addr, symbol, caller], or None if parsing fails.
117	"""
NEW 118	pattern = re.compile(r"([0-9a-f]+)\s+(.+?)\s+\((.+)\)")	×
NEW 119	matches = pattern.findall(line)	×
NEW 120	if not matches:	×
NEW 121	logger.warning(f"script one call '{line}' parse failed")	×
NEW 122	return None	×
NEW 123	addr, symbol, caller = matches[0]	×
NEW 124	return [addr, symbol, caller]	×
125
NEW 126	@staticmethod	×
NEW 127	def parse_one_header(line: str) -> Optional[List]:	×
128	"""
129	Parses a single header line from a perf script block.
130
131	Args:
132	line (str): The header line to parse.
133
134	Returns:
135	list: A list containing the parsed values [command, pid, cpu, time, value, event].
136	Returns None if the line cannot be parsed.
137	"""
NEW 138	try:	×
NEW 139	try:	×
NEW 140	pattern = r"(\S+\|\:-\d+)\s+(\d+\|-\d+)\s+\[(\d+)]\s+(\d+\.\d+):\s+(\d+)\s+(\S+):"	×
NEW 141	command, pid, cpu, time, value, event = re.match(	×
142	pattern, line.strip()
143	).groups()
NEW 144	except Exception:	×
NEW 145	try:	×
NEW 146	pattern = r"(\d+\|-\d+)\s+\[(\d+)]\s+(\d+\.\d+):\s+(\d+)\s+(\S+):"	×
NEW 147	pid, cpu, time, value, event = re.match(	×
148	pattern, line[15:].strip()
149	).groups()
NEW 150	command = line[:15].strip()	×
NEW 151	except Exception:	×
NEW 152	pattern = r"(\d+\|-\d+)\s+\[(\d+)]\s+(\d+\.\d+):\s+(\d+)\s+(\S+):"	×
NEW 153	pid, cpu, time, value, event = re.match(	×
154	pattern, line[10:].strip()
155	).groups()
NEW 156	command = line[:10].strip()	×
NEW 157	except Exception as e:	×
NEW 158	logger.warning(f"script header '{line}' parse failed due to: {e}")	×
NEW 159	return None	×
160
NEW 161	return [	×
162	command,
163	int(pid),
164	int(cpu),
165	time,
166	int(value),
167	event,
168	]
169
NEW 170	@staticmethod	×
NEW 171	def parse_block(lines: List[str]) -> Optional[tuple]:	×
172	"""
173	Parses the lines of the performance script block.
174
175	Args:
176	lines (list): The lines of the performance script block.
177
178	Returns:
179	tuple: A tuple containing the parsed header and calls.
180	"""
181	# TODO: There may be some other info like brstackinsn at first
182	# TODO: First we just ignore, need to handle it at further stage
183	# perf script -F comm,pid,cpu,time,period,event,ip,sym,dso -I --header
NEW 184	start_index = -1	×
NEW 185	line_len = len(lines)	×
NEW 186	header = None	×
NEW 187	while header is None:	×
NEW 188	start_index += 1	×
NEW 189	if start_index >= line_len:	×
NEW 190	break	×
NEW 191	header_data = PerfScriptParser.parse_one_header(lines[start_index])	×
NEW 192	if header_data:	×
NEW 193	header = PerfScriptHeader(*header_data)	×
NEW 194	if header is None:	×
NEW 195	logger.warning(f"{lines} can't be parsed by perf script")	×
NEW 196	return None	×
NEW 197	calls = []	×
NEW 198	for line in lines[start_index + 1 :]:	×
NEW 199	call_data = PerfScriptParser.parse_one_call(line)	×
NEW 200	if call_data:	×
NEW 201	calls.append(PerfScriptCall(*call_data))	×
NEW 202	return header, calls	×
203
NEW 204	@staticmethod	×
NEW 205	def divid_into_blocks(lines: List[str]) -> List[List[str]]:	×
206	"""
207	Divides the lines into blocks based on empty lines.
208
209	Args:
210	lines (list): The list of lines to divide into blocks.
211
212	Returns:
213	list: A list of blocks, where each block is a list of lines.
214
215	"""
NEW 216	blocks, cur = [], []	×
NEW 217	for l in lines:	×
NEW 218	if l:	×
NEW 219	cur.append(l)	×
NEW 220	elif cur:	×
NEW 221	blocks.append(cur)	×
NEW 222	cur = []	×
NEW 223	if cur:	×
NEW 224	blocks.append(cur)	×
NEW 225	return blocks	×
226
227
228	# Data processor class for performance script data
NEW 229	class PerfScriptDataProcessor:	×
230	"""
231	A class responsible for processing performance script data.
232	"""
233
NEW 234	def __init__(self, blocks: List[PerfScriptBlock]):	×
NEW 235	self.blocks = blocks	×
236
NEW 237	def summary_all_cmds(self) -> Dict[str, Dict[str, Set]]:	×
238	"""
239	Returns a dictionary of commands, each containing a set of CPUs and modules relative to the command.
240
241	Returns:
242	Dict[Dict[Set]]: A dictionary of commands, each command contains a dict of 'cpus' and 'modules'. 'cpus' are the set of CPUs, 'modules' are the set of modules.
243	"""
NEW 244	cmds = defaultdict(lambda: defaultdict(lambda: set()))	×
NEW 245	for b in self.blocks:	×
NEW 246	cmd = b.header.command	×
NEW 247	cmds[cmd]["cpus"].add(b.header.cpu)	×
NEW 248	for s in b.calls:	×
NEW 249	cmds[cmd]["modules"].add(s.caller)	×
NEW 250	return cmds	×
251
NEW 252	def filter_by_time_window(	×
253	self,
254	start: Optional[Decimal \| str \| float] = None,
255	end: Optional[Decimal \| str \| float] = None,
256	deltatime: Optional[Decimal \| str \| float] = None,
257	) -> "PerfScriptDataProcessor":
258	"""
259	Filters the performance script data by a given time window.
260
261	Will generate start and end time by params start / end / deltatime
262	If deltatime is None, start / end will be block start / block end if set to none
263	If deltatime not None, following is used:
264	If start not None and end is None then start = start; end = start + deltatime;
265	If start is None and end not None then end = end; start = end - deltatime;
266	If start not None and end not None then start = start; end = start + deltatime;
267	If start is None and end is None then start = block start; end = start + deltatime
268
269	Args:
270	start (Optional[str \| float]): The start time of the window.
271	end (Optional[str \| float]): The end time of the window.
272	deltatime (Optional[str \| float]): The time duration of the window.
273
274	Returns:
275	PerfScriptDataProcessor: A new PerfScriptDataProcessor object containing only the blocks in given time window.
276	"""
NEW 277	if len(self.blocks) < 2:	×
NEW 278	return PerfScriptDataProcessor(self.blocks)	×
NEW 279	block_start = self.blocks[0].header.xytime	×
NEW 280	block_end = self.blocks[-1].header.xytime	×
NEW 281	logger.debug(f"Script start time: {block_start}")	×
NEW 282	logger.debug(f"Script end time: {block_end}")	×
NEW 283	try:	×
NEW 284	tstart = Decimal(str(start)) if start else block_start	×
NEW 285	tend = Decimal(str(end)) if end else block_end	×
NEW 286	if deltatime:	×
NEW 287	deltatime = Decimal(str(deltatime))	×
NEW 288	if start is None and end:	×
NEW 289	tstart = Decimal(str(end)) - deltatime	×
290	else:
NEW 291	tend = tstart + deltatime	×
NEW 292	except InvalidOperation:	×
NEW 293	logger.warning("The time window should be format of float")	×
NEW 294	return PerfScriptDataProcessor(self.blocks)	×
NEW 295	return PerfScriptDataProcessor(	×
296	[
297	b
298	for b in self.blocks
299	if b.header.xytime >= tstart and b.header.xytime <= tend
300	]
301	)
302
NEW 303	def filter_by_command(self, command: str) -> "PerfScriptDataProcessor":	×
304	"""
305	Filters the performance script data by a command.
306
307	Args:
308	command (str): The command to filter by.
309
310	Returns:
311	PerfScriptDataProcessor: A new PerfScriptDataProcessor object containing only the blocks with matching command.
312	"""
NEW 313	return PerfScriptDataProcessor(	×
314	[b for b in self.blocks if b.header.command == command]
315	)
316
NEW 317	def filter_by_commands(self, commands: List[str]) -> "PerfScriptDataProcessor":	×
318	"""
319	Filters the performance script data by a list of commands.
320
321	Args:
322	commands (list[str]): A list of commands to filter by.
323
324	Returns:
325	PerfScriptDataProcessor: A new PerfScriptDataProcessor object containing only the blocks with matching commands.
326	"""
NEW 327	return PerfScriptDataProcessor(	×
328	[b for b in self.blocks if b.header.command in commands]
329	)
330
NEW 331	def filter_by_pid(self, pid: int) -> "PerfScriptDataProcessor":	×
332	"""
333	Filters the performance script data by process ID.
334
335	Args:
336	pid (int): The process ID to filter by.
337
338	Returns:
339	PerfScriptDataProcessor: A new PerfScriptDataProcessor object containing the filtered blocks.
340
341	"""
NEW 342	return PerfScriptDataProcessor([b for b in self.blocks if b.header.pid == pid])	×
343
NEW 344	def filter_by_pids(self, pids: List[int]) -> "PerfScriptDataProcessor":	×
345	"""
346	Filters the performance script data by a list of process IDs (pids).
347
348	Args:
349	pids (list[int]): A list of process IDs to filter by.
350
351	Returns:
352	PerfScriptDataProcessor: A new PerfScriptDataProcessor object containing only the blocks with matching pids.
353	"""
NEW 354	return PerfScriptDataProcessor([b for b in self.blocks if b.header.pid in pids])	×
355
NEW 356	def filter_by_cpu(self, cpu: int) -> "PerfScriptDataProcessor":	×
357	"""
358	Filters the performance script data by CPU.
359
360	Args:
361	cpu (int): The CPU to filter by.
362
363	Returns:
364	PerfScriptDataProcessor: A new PerfScriptDataProcessor object containing the filtered blocks.
365
366	"""
NEW 367	return PerfScriptDataProcessor([b for b in self.blocks if b.header.cpu == cpu])	×
368
NEW 369	def filter_by_cpus(self, cpus: List[int]) -> "PerfScriptDataProcessor":	×
370	"""
371	Filters the performance script data by the given list of CPUs.
372
373	Args:
374	cpus (list[int]): A list of CPUs to filter by.
375
376	Returns:
377	PerfScriptDataProcessor: A new PerfScriptDataProcessor object containing only the blocks
378	that match the specified CPUs.
379	"""
NEW 380	return PerfScriptDataProcessor([b for b in self.blocks if b.header.cpu in cpus])	×
381
NEW 382	def to_raw_dataframe(self) -> pd.DataFrame:	×
383	"""
384	Converts the blocks to a raw dataframe.
385	Returns:
386	pd.DataFrame: A pandas DataFrame containing the records from the blocks.
387	"""
NEW 388	return pd.DataFrame([b.to_record() for b in self.blocks])	×
389
NEW 390	def to_flat_dataframe(self) -> pd.DataFrame:	×
391	"""
392	Converts the blocks to a flat dataframe. Can be used for FlameGraph.
393	Returns:
394	pd.DataFrame: A pandas DataFrame containing the records from the blocks.
395	"""
NEW 396	return self.to_raw_dataframe().explode("calls")	×
397
NEW 398	def to_callee_dataframe(self) -> pd.DataFrame:	×
399	"""
400	Converts the blocks to a callee dataframe. Can be used for metrics analysis.
401	Returns:
402	pd.DataFrame: A pandas DataFrame containing the records from the blocks.
403	"""
NEW 404	df = self.to_raw_dataframe()	×
NEW 405	df["callee"] = df["calls"].apply(lambda x: x[0] if x else None)	×
NEW 406	df[["addr", "symbol", "caller"]] = df["callee"].apply(	×
407	lambda callee: pd.Series(PerfScriptParser.parse_one_call(callee))
408	)
NEW 409	df = df.drop(columns=["calls", "callee", "caller"])	×
NEW 410	df[["symbol", "offset"]] = df["symbol"].str.rsplit("+", n=1, expand=True)	×
NEW 411	return df	×
412
NEW 413	@staticmethod	×
NEW 414	def transfer_callee_to_metric_dataframe(	×
415	df: pd.DataFrame,
416	numerator: str,
417	denominator: str,
418	metric_name: Optional[str] = None,
419	) -> pd.DataFrame:
420	"""
421	This method is a class method that can be used to convert a DataFrame containing perf script data into a metric DataFrame.
422	It filters the DataFrame based on the numerator and denominator events, pivots the data to create a metric ratio,
423	and returns a new DataFrame with the calculated metric.
424	It is recommended to use the filtered DataFrame for metric calculation.
425	This method is useful for analyzing performance metrics in a structured way.
426
427	Args:
428	df (pd.DataFrame): The DataFrame to use for the metric calculation.
429	numerator (str): The numerator of the metric, eg. "ll_cache_miss:S".
430	denominator (str): The denominator of the metric, eg. "ll_cache:S".
431	metric_name (str): The name of the metric, eg. "ll_cache_miss_ratio".
432	Returns:
433	pd.DataFrame: A pandas DataFrame containing the records from the blocks.
434	"""
NEW 435	df_pivot = (	×
436	df[df["event"].isin([numerator, denominator])]
437	.pivot_table(
438	index=["time", "symbol", "cpu"],
439	columns="event",
440	values="value",
441	)
442	.reset_index()
443	)
NEW 444	df_grouped = (	×
445	df_pivot.drop(columns=["cpu", "time"])
446	.groupby(["symbol"])
447	.sum()
448	.reset_index()
449	)
NEW 450	if metric_name is None:	×
NEW 451	metric_name = f"{numerator}_ratio"	×
NEW 452	df_grouped[metric_name] = df_grouped[numerator] / df_grouped[denominator]	×
NEW 453	return df_grouped.sort_values(by=metric_name, ascending=False)	×
454
NEW 455	def to_metric_dataframe(	×
456	self, numerator: str, denominator: str, metric_name: Optional[str] = None
457	) -> pd.DataFrame:
458	"""
459	Converts the blocks to a metric dataframe. Can be used for metrics analysis.
460	It's recommended to use the filtered DataFrame for metric calculation.
461
462	Args:
463	numerator (str): The numerator of the metric, eg. "ll_cache_miss:S".
464	denominator (str): The denominator of the metric, eg. "ll_cache:S".
465	metric_name (str): The name of the metric, eg. "ll_cache_miss_ratio".
466	Returns:
467	pd.DataFrame: A pandas DataFrame containing the records from the blocks.
468	"""
NEW 469	return self.transfer_callee_to_metric_dataframe(	×
470	self.to_callee_dataframe(), numerator, denominator, metric_name
471	)
472
473
NEW 474	def _mp_parse_block(block_lines):	×
475	"""Top-level helper function for multiprocessing pool.map."""
NEW 476	return PerfScriptParser.parse_block(block_lines)	×
477
478
479	# Main class for handling perf script data
NEW 480	class PerfScriptData:	×
481	"""
482	Represents a collection of perf script blocks.
483	It represents the data obtained from a perf script file.
484
485	Args:
486	blocks (list[PerfScriptBlock]): The list of PerfScriptBlock objects.
487
488	Attributes:
489	blocks (list[PerfScriptBlock]): The list of PerfScriptBlock objects.
490	processor (PerfScriptDataProcessor): The data processor for the blocks.
491
492	Methods:
493	__str__(): Returns a string representation of the PerfScriptData object.
494	__iter__(): Returns an iterator for iterating over the PerfScriptData object.
495	__getitem__(index): Returns the PerfScriptBlock object at the specified index.
496	__len__(): Returns the number of PerfScriptBlock objects in the PerfScriptData object.
497	from_file(file_path, processes_num=NUM_CORES_PHYSICAL): Creates a PerfScriptData object from a file.
498	"""
499
NEW 500	def __init__(self, blocks: List[PerfScriptBlock]):	×
NEW 501	self.blocks: List[PerfScriptBlock] = blocks	×
NEW 502	self.processor = PerfScriptDataProcessor(blocks)	×
503
NEW 504	def __str__(self):	×
NEW 505	return f"{self.blocks}"	×
506
NEW 507	def __iter__(self):	×
NEW 508	return iter(self.blocks)	×
509
NEW 510	def __getitem__(self, index):	×
NEW 511	return self.blocks[index]	×
512
NEW 513	def __len__(self):	×
NEW 514	return len(self.blocks)	×
515
NEW 516	def __getattribute__(self, name):	×
NEW 517	if hasattr(self.processor, name):	×
NEW 518	return getattr(self.processor, name)	×
519
NEW 520	@classmethod	×
NEW 521	def from_file(	×
522	cls, file_path: str, processes_num=NUM_CORES_PHYSICAL
523	) -> "PerfScriptData":
524	"""
525	Creates a PerfScriptData object from a file.
526
527	Args:
528	file_path (str): The path to the file.
529	processes_num (int, optional): The number of processes to use for parallel processing. Defaults to NUM_CORES_PHYSICAL.
530
531	Returns:
532	PerfScriptData: A new PerfScriptData object created from the file.
533
534	"""
NEW 535	with open(file_path, "r") as f:	×
NEW 536	lines = [	×
537	l.strip()
538	for l in f.readlines()
539	if not (l.startswith("#") or l.startswith("[") or l.startswith("\|"))
540	]
541
NEW 542	with Pool(processes=processes_num) as pool:	×
NEW 543	blocks = pool.map(	×
544	_mp_parse_block,
545	PerfScriptParser.divid_into_blocks(lines),
546	)
547
548	# remove None in blocks
NEW 549	blocks = [block for block in blocks if block is not None]	×
NEW 550	blocks = [PerfScriptBlock(*block) for block in blocks]	×
551
NEW 552	return cls(blocks)	×

ZJU-SPAIL / pipa / 22699554347

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous