1c1050ed-56a8-4fb6-acbf-584e4fa7fadb

Committed 19 Feb 2025 02:41PM UTC coverage: 85.586% (-0.3%) from 85.836%

Build # 1c1050ed-56a8-4fb6-acbf-584e4fa7fadb

Build Type

push

circleci

Committed by

web-flow

Commit Message

Update README.md (#257)

* Update README.md

* Update README.md

Coverage Stats

3527 of 4121 relevant lines covered (85.59%)

10.14 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

56.9

/scrapydweb/utils/sub_process.py

# coding: utf-8
import atexit
from ctypes import cdll
import logging
import os
import platform
import signal
from subprocess import Popen
import sys

from ..common import json_dumps


logger = logging.getLogger(__name__)

CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))


# https://stackoverflow.com/a/13256908/10517783
# https://stackoverflow.com/a/23587108/10517783
# http://evans.io/legacy/posts/killing-child-processes-on-parent-exit-prctl/
class PrCtlError(Exception):
    pass


def on_parent_exit(signame):
    """
    Return a function to be run in a child process which will trigger
    SIGNAME to be sent when the parent process dies
    """
    # On Windows, signal() can only be called with SIGABRT, SIGFPE, SIGILL, SIGINT, SIGSEGV, or SIGTERM.
    signum = getattr(signal, signame)  # SIGTERM 15  SIGKILL 9

    def set_parent_exit_signal():
        # Constant taken from http://linux.die.net/include/linux/prctl.h
        PR_SET_PDEATHSIG = 1
        # http://linux.die.net/man/2/prctl
        result = cdll['libc.so.6'].prctl(PR_SET_PDEATHSIG, signum)
        if result != 0:
            raise PrCtlError('prctl failed with error code %s' % result)

    return set_parent_exit_signal


# https://stackoverflow.com/a/19448255/10517783
def kill_child(proc, title=''):
    proc.kill()
    # A None value indicates that the process has not terminated yet.
    # A negative value -N indicates that the child was terminated by signal N (Unix only).
    logger.warning('%s subprocess (pid: %s) killed with returncode: %s', title, proc.pid, proc.wait())


def init_logparser(config):
    logparser_subprocess = start_logparser(config)
    logparser_pid = logparser_subprocess.pid
    logger.info("Running LogParser in the background with pid: %s", logparser_pid)
    atexit.register(kill_child, logparser_subprocess, 'LogParser')
    return logparser_pid


def start_logparser(config):
    args = [
        sys.executable,
        '-m',
        'logparser.run',
        '-dir',
        config['LOCAL_SCRAPYD_LOGS_DIR'],
        '--main_pid',
        str(config['MAIN_PID']),
    ]

    if platform.system() == 'Linux':
        kwargs = dict(preexec_fn=on_parent_exit('SIGKILL'))  # 'SIGTERM' 'SIGKILL'
        try:
            logparser_subprocess = Popen(args, **kwargs)
        except Exception as err:
            logger.error(err)
            logparser_subprocess = Popen(args)
    else:
        logparser_subprocess = Popen(args)

    return logparser_subprocess


def init_poll(config):
    poll_subprocess = start_poll(config)
    poll_pid = poll_subprocess.pid
    logger.info("Start polling job stats for monitor & alert in the background with pid: %s", poll_pid)
    atexit.register(kill_child, poll_subprocess, 'Poll')
    return poll_pid


def start_poll(config):
    args = [
        sys.executable,
        os.path.join(CURRENT_DIR, 'poll.py'),

        config['URL_SCRAPYDWEB'],
        config.get('USERNAME', '') if config.get('ENABLE_AUTH', False) else '',
        config.get('PASSWORD', '') if config.get('ENABLE_AUTH', False) else '',
        json_dumps(config.get('SCRAPYD_SERVERS', ['127.0.0.1'])),
        json_dumps(config.get('SCRAPYD_SERVERS_AUTHS', [None])),
        str(config.get('POLL_ROUND_INTERVAL', 300)),
        str(config.get('POLL_REQUEST_INTERVAL', 10)),
        str(config['MAIN_PID']),
        str(config.get('VERBOSE', False))
    ]

    # 'Windows':
    # AttributeError: module 'signal' has no attribute 'SIGKILL'
    # ValueError: preexec_fn is not supported on Windows platforms
    # macOS('Darwin'):
    # subprocess.SubprocessError: Exception occurred in preexec_fn.
    # OSError: dlopen(libc.so.6, 6): image not found
    if platform.system() == 'Linux':
        kwargs = dict(preexec_fn=on_parent_exit('SIGKILL'))  # 'SIGTERM' 'SIGKILL'
        try:
            poll_subprocess = Popen(args, **kwargs)
        except Exception as err:
            logger.error(err)
            poll_subprocess = Popen(args)
    else:
        poll_subprocess = Popen(args)

    return poll_subprocess

1	# coding: utf-8
2	import atexit	12✔
3	from ctypes import cdll	12✔
4	import logging	12✔
5	import os	12✔
6	import platform	12✔
7	import signal	12✔
8	from subprocess import Popen	12✔
9	import sys	12✔
10
11	from ..common import json_dumps	12✔
12
13
14	logger = logging.getLogger(__name__)	12✔
15
16	CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))	12✔
17
18
19	# https://stackoverflow.com/a/13256908/10517783
20	# https://stackoverflow.com/a/23587108/10517783
21	# http://evans.io/legacy/posts/killing-child-processes-on-parent-exit-prctl/
22	class PrCtlError(Exception):	12✔
23	pass	12✔
24
25
26	def on_parent_exit(signame):	12✔
27	"""
28	Return a function to be run in a child process which will trigger
29	SIGNAME to be sent when the parent process dies
30	"""
31	# On Windows, signal() can only be called with SIGABRT, SIGFPE, SIGILL, SIGINT, SIGSEGV, or SIGTERM.
32	signum = getattr(signal, signame) # SIGTERM 15 SIGKILL 9	12✔
33
34	def set_parent_exit_signal():	12✔
35	# Constant taken from http://linux.die.net/include/linux/prctl.h
36	PR_SET_PDEATHSIG = 1	×
37	# http://linux.die.net/man/2/prctl
38	result = cdll['libc.so.6'].prctl(PR_SET_PDEATHSIG, signum)	×
39	if result != 0:	×
40	raise PrCtlError('prctl failed with error code %s' % result)	×
41
42	return set_parent_exit_signal	12✔
43
44
45	# https://stackoverflow.com/a/19448255/10517783
46	def kill_child(proc, title=''):	12✔
47	proc.kill()	×
48	# A None value indicates that the process has not terminated yet.
49	# A negative value -N indicates that the child was terminated by signal N (Unix only).
50	logger.warning('%s subprocess (pid: %s) killed with returncode: %s', title, proc.pid, proc.wait())	×
51
52
53	def init_logparser(config):	12✔
54	logparser_subprocess = start_logparser(config)	12✔
55	logparser_pid = logparser_subprocess.pid	12✔
56	logger.info("Running LogParser in the background with pid: %s", logparser_pid)	12✔
57	atexit.register(kill_child, logparser_subprocess, 'LogParser')	12✔
58	return logparser_pid	12✔
59
60
61	def start_logparser(config):	12✔
62	args = [	12✔
63	sys.executable,
64	'-m',
65	'logparser.run',
66	'-dir',
67	config['LOCAL_SCRAPYD_LOGS_DIR'],
68	'--main_pid',
69	str(config['MAIN_PID']),
70	]
71
72	if platform.system() == 'Linux':	12✔
73	kwargs = dict(preexec_fn=on_parent_exit('SIGKILL')) # 'SIGTERM' 'SIGKILL'	12✔
74	try:	12✔
75	logparser_subprocess = Popen(args, **kwargs)	12✔
76	except Exception as err:	×
77	logger.error(err)	×
78	logparser_subprocess = Popen(args)	×
79	else:
80	logparser_subprocess = Popen(args)	×
81
82	return logparser_subprocess	12✔
83
84
85	def init_poll(config):	12✔
86	poll_subprocess = start_poll(config)	×
87	poll_pid = poll_subprocess.pid	×
88	logger.info("Start polling job stats for monitor & alert in the background with pid: %s", poll_pid)	×
89	atexit.register(kill_child, poll_subprocess, 'Poll')	×
90	return poll_pid	×
91
92
93	def start_poll(config):	12✔
94	args = [	×
95	sys.executable,
96	os.path.join(CURRENT_DIR, 'poll.py'),
97
98	config['URL_SCRAPYDWEB'],
99	config.get('USERNAME', '') if config.get('ENABLE_AUTH', False) else '',
100	config.get('PASSWORD', '') if config.get('ENABLE_AUTH', False) else '',
101	json_dumps(config.get('SCRAPYD_SERVERS', ['127.0.0.1'])),
102	json_dumps(config.get('SCRAPYD_SERVERS_AUTHS', [None])),
103	str(config.get('POLL_ROUND_INTERVAL', 300)),
104	str(config.get('POLL_REQUEST_INTERVAL', 10)),
105	str(config['MAIN_PID']),
106	str(config.get('VERBOSE', False))
107	]
108
109	# 'Windows':
110	# AttributeError: module 'signal' has no attribute 'SIGKILL'
111	# ValueError: preexec_fn is not supported on Windows platforms
112	# macOS('Darwin'):
113	# subprocess.SubprocessError: Exception occurred in preexec_fn.
114	# OSError: dlopen(libc.so.6, 6): image not found
115	if platform.system() == 'Linux':	×
116	kwargs = dict(preexec_fn=on_parent_exit('SIGKILL')) # 'SIGTERM' 'SIGKILL'	×
117	try:	×
118	poll_subprocess = Popen(args, **kwargs)	×
119	except Exception as err:	×
120	logger.error(err)	×
121	poll_subprocess = Popen(args)	×
122	else:
123	poll_subprocess = Popen(args)	×
124
125	return poll_subprocess	×

my8100 / scrapydweb / 1c1050ed-56a8-4fb6-acbf-584e4fa7fadb

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous