374f7459-4bbd-421d-9b8d-94fbf13e60d3

Committed 12 Jan 2025 10:13AM UTC coverage: 42.723% (-43.1%) from 85.817%

Build # 374f7459-4bbd-421d-9b8d-94fbf13e60d3

Build Type

Pull #250

circleci

Committed by

my8100

Commit Message

Update test_schedule.py

Pull Request Pull Request #250: Fix bug for scrapyd v1.5.0

Run Details

1 of 3 new or added lines in 2 files covered. (33.33%)

1746 existing lines in 29 files now uncovered.

1726 of 4040 relevant lines covered (42.72%)

4.98 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/scrapydweb/utils/sub_process.py

1	# coding: utf-8
UNCOV 2	import atexit	×
UNCOV 3	from ctypes import cdll	×
UNCOV 4	import logging	×
UNCOV 5	import os	×
UNCOV 6	import platform	×
UNCOV 7	import signal	×
UNCOV 8	from subprocess import Popen	×
UNCOV 9	import sys	×
10
UNCOV 11	from ..common import json_dumps	×
12
13
UNCOV 14	logger = logging.getLogger(__name__)	×
15
UNCOV 16	CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))	×
17
18
19	# https://stackoverflow.com/a/13256908/10517783
20	# https://stackoverflow.com/a/23587108/10517783
21	# http://evans.io/legacy/posts/killing-child-processes-on-parent-exit-prctl/
UNCOV 22	class PrCtlError(Exception):	×
UNCOV 23	pass	×
24
25
UNCOV 26	def on_parent_exit(signame):	×
27	"""
28	Return a function to be run in a child process which will trigger
29	SIGNAME to be sent when the parent process dies
30	"""
31	# On Windows, signal() can only be called with SIGABRT, SIGFPE, SIGILL, SIGINT, SIGSEGV, or SIGTERM.
UNCOV 32	signum = getattr(signal, signame) # SIGTERM 15 SIGKILL 9	×
33
UNCOV 34	def set_parent_exit_signal():	×
35	# Constant taken from http://linux.die.net/include/linux/prctl.h
36	PR_SET_PDEATHSIG = 1	×
37	# http://linux.die.net/man/2/prctl
38	result = cdll['libc.so.6'].prctl(PR_SET_PDEATHSIG, signum)	×
39	if result != 0:	×
40	raise PrCtlError('prctl failed with error code %s' % result)	×
41
UNCOV 42	return set_parent_exit_signal	×
43
44
45	# https://stackoverflow.com/a/19448255/10517783
UNCOV 46	def kill_child(proc, title=''):	×
47	proc.kill()	×
48	# A None value indicates that the process has not terminated yet.
49	# A negative value -N indicates that the child was terminated by signal N (Unix only).
50	logger.warning('%s subprocess (pid: %s) killed with returncode: %s', title, proc.pid, proc.wait())	×
51
52
UNCOV 53	def init_logparser(config):	×
UNCOV 54	logparser_subprocess = start_logparser(config)	×
UNCOV 55	logparser_pid = logparser_subprocess.pid	×
UNCOV 56	logger.info("Running LogParser in the background with pid: %s", logparser_pid)	×
UNCOV 57	atexit.register(kill_child, logparser_subprocess, 'LogParser')	×
UNCOV 58	return logparser_pid	×
59
60
UNCOV 61	def start_logparser(config):	×
UNCOV 62	args = [	×
63	sys.executable,
64	'-m',
65	'logparser.run',
66	'-dir',
67	config['LOCAL_SCRAPYD_LOGS_DIR'],
68	'--main_pid',
69	str(config['MAIN_PID']),
70	]
71
UNCOV 72	if platform.system() == 'Linux':	×
UNCOV 73	kwargs = dict(preexec_fn=on_parent_exit('SIGKILL')) # 'SIGTERM' 'SIGKILL'	×
UNCOV 74	try:	×
UNCOV 75	logparser_subprocess = Popen(args, **kwargs)	×
76	except Exception as err:	×
77	logger.error(err)	×
78	logparser_subprocess = Popen(args)	×
79	else:
80	logparser_subprocess = Popen(args)	×
81
UNCOV 82	return logparser_subprocess	×
83
84
UNCOV 85	def init_poll(config):	×
UNCOV 86	poll_subprocess = start_poll(config)	×
UNCOV 87	poll_pid = poll_subprocess.pid	×
UNCOV 88	logger.info("Start polling job stats for monitor & alert in the background with pid: %s", poll_pid)	×
UNCOV 89	atexit.register(kill_child, poll_subprocess, 'Poll')	×
UNCOV 90	return poll_pid	×
91
92
UNCOV 93	def start_poll(config):	×
UNCOV 94	args = [	×
95	sys.executable,
96	os.path.join(CURRENT_DIR, 'poll.py'),
97
98	config['URL_SCRAPYDWEB'],
99	config.get('USERNAME', '') if config.get('ENABLE_AUTH', False) else '',
100	config.get('PASSWORD', '') if config.get('ENABLE_AUTH', False) else '',
101	json_dumps(config.get('SCRAPYD_SERVERS', ['127.0.0.1'])),
102	json_dumps(config.get('SCRAPYD_SERVERS_AUTHS', [None])),
103	str(config.get('POLL_ROUND_INTERVAL', 300)),
104	str(config.get('POLL_REQUEST_INTERVAL', 10)),
105	str(config['MAIN_PID']),
106	str(config.get('VERBOSE', False))
107	]
108
109	# 'Windows':
110	# AttributeError: module 'signal' has no attribute 'SIGKILL'
111	# ValueError: preexec_fn is not supported on Windows platforms
112	# macOS('Darwin'):
113	# subprocess.SubprocessError: Exception occurred in preexec_fn.
114	# OSError: dlopen(libc.so.6, 6): image not found
UNCOV 115	if platform.system() == 'Linux':	×
UNCOV 116	kwargs = dict(preexec_fn=on_parent_exit('SIGKILL')) # 'SIGTERM' 'SIGKILL'	×
UNCOV 117	try:	×
UNCOV 118	poll_subprocess = Popen(args, **kwargs)	×
119	except Exception as err:	×
120	logger.error(err)	×
121	poll_subprocess = Popen(args)	×
122	else:
123	poll_subprocess = Popen(args)	×
124
UNCOV 125	return poll_subprocess	×

my8100 / scrapydweb / 374f7459-4bbd-421d-9b8d-94fbf13e60d3

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous