3685840692

Build Type

push

github

Committed by viacheslav.kirichenko

Commit Message

ci: update test matrix

Run Details

749 of 1514 branches covered (49.47%)

Branch coverage included in aggregate %.

2893 of 4242 relevant lines covered (68.2%)

0.68 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

53.33

/lib/utils.py

import errno
import os
import sys
import collections
import signal
import fcntl
import difflib
import time
import json
import subprocess
from lib.colorer import color_stdout

try:
    # Python3.5 or above
    from signal import Signals
except ImportError:
    # Python2
    Signals = None

try:
    # Python 3.3+.
    from shlex import quote as _shlex_quote
except ImportError:
    # Python 2.7.
    from pipes import quote as _shlex_quote


UNIX_SOCKET_LEN_LIMIT = 107

# Useful for very coarse version differentiation.
PY3 = sys.version_info[0] == 3
PY2 = sys.version_info[0] == 2

if PY2:
    FileNotFoundError = IOError

if PY3:
    string_types = str,
    integer_types = int,
else:
    string_types = basestring,      # noqa: F821
    integer_types = (int, long)     # noqa: F821


def check_libs():
    deps = [
        ('msgpack', 'msgpack-python'),
        ('tarantool', 'tarantool-python')
    ]
    base_path = os.path.dirname(os.path.abspath(__file__))

    for (mod_name, mod_dir) in deps:
        mod_path = os.path.join(base_path, mod_dir)
        if mod_path not in sys.path:
            sys.path = [mod_path] + sys.path

    for (mod_name, _mod_dir) in deps:
        try:
            __import__(mod_name)
        except ImportError as e:
            color_stdout("\n\nNo %s library found\n" % mod_name,
                         schema='error')
            print(e)
            sys.exit(1)


def non_empty_valgrind_logs(paths_to_log):
    """ Check that there were no warnings in the log."""
    non_empty_logs = []
    for path_to_log in paths_to_log:
        if os.path.exists(path_to_log) and os.path.getsize(path_to_log) != 0:
            non_empty_logs.append(path_to_log)
    return non_empty_logs


def print_tail_n(filename, num_lines=None):
    """ Print N last lines of a file. If num_lines is not set,
    prints the whole file.
    """
    with open(filename, "r") as logfile:
        tail_n = collections.deque(logfile, num_lines)
        for line in tail_n:
            color_stdout(line, schema='tail')


def find_in_path(name):
    path = os.curdir + os.pathsep + os.environ["PATH"]
    for _dir in path.split(os.pathsep):
        exe = os.path.join(_dir, name)
        if os.access(exe, os.X_OK):
            return exe
    return ''


# http://stackoverflow.com/a/2549950
SIGNAMES = dict((int(v), k) for k, v in reversed(sorted(
    signal.__dict__.items())) if k.startswith('SIG') and
    not k.startswith('SIG_'))
SIGNUMS = dict((k, int(v)) for k, v in reversed(sorted(
    signal.__dict__.items())) if k.startswith('SIG') and
    not k.startswith('SIG_'))


def signame(signal):
    if isinstance(signal, integer_types):
        return SIGNAMES[signal]
    if Signals and isinstance(signal, Signals):
        return SIGNAMES[int(signal)]
    if isinstance(signal, string_types):
        return signal
    raise TypeError('signame(): signal argument of unexpected type: {}'.format(
                    str(type(signal))))


def signum(signal):
    if isinstance(signal, integer_types):
        return signal
    if Signals and isinstance(signal, Signals):
        return int(signal)
    if isinstance(signal, string_types):
        if not signal.startswith('SIG'):
            signal = 'SIG' + signal
        return SIGNUMS[signal]
    raise TypeError('signum(): signal argument of unexpected type: {}'.format(
                    str(type(signal))))


def warn_unix_sockets_at_start(vardir):
    max_unix_socket_rel = '???_replication/autobootstrap_guest3.control'
    real_vardir = os.path.realpath(vardir)
    max_unix_socket_abs = os.path.join(real_vardir, max_unix_socket_rel)
    max_unix_socket_real = os.path.realpath(max_unix_socket_abs)
    if len(max_unix_socket_real) > UNIX_SOCKET_LEN_LIMIT:
        color_stdout(
            'WARGING: unix sockets can become longer than %d symbols:\n'
            % UNIX_SOCKET_LEN_LIMIT,
            schema='error')
        color_stdout('WARNING: for example: "%s" has length %d\n' %
                     (max_unix_socket_real, len(max_unix_socket_real)),
                     schema='error')


def warn_unix_socket(path):
    real_path = os.path.realpath(path)
    if len(real_path) <= UNIX_SOCKET_LEN_LIMIT or \
            real_path in warn_unix_socket.warned:
        return
    color_stdout(
        '\nWARGING: unix socket\'s "%s" path has length %d symbols that is '
        'longer than %d. That likely will cause failing of tests.\n' %
        (real_path, len(real_path), UNIX_SOCKET_LEN_LIMIT), schema='error')
    warn_unix_socket.warned.add(real_path)


warn_unix_socket.warned = set()


def safe_makedirs(directory):
    if os.path.isdir(directory):
        return
    # try-except to prevent races btw processes
    try:
        os.makedirs(directory)
    except OSError:
        pass


def format_process(pid):
    cmdline = 'unknown'
    try:
        with open('/proc/%d/cmdline' % pid, 'r') as f:
            cmdline = ' '.join(f.read().split('\0')).strip() or cmdline
    except (OSError, IOError):
        pass
    status = 'unknown'
    try:
        with open('/proc/%d/status' % pid, 'r') as f:
            for line in f:
                if ':' not in line:
                    continue
                key, value = line.split(':', 1)
                if key == 'State':
                    status = value.strip()
    except (OSError, IOError):
        pass
    return 'process %d [%s; %s]' % (pid, status, cmdline)


def proc_stat_rss_supported():
    return os.path.isfile('/proc/%d/status' % os.getpid())


def get_proc_stat_rss(pid):
    rss = 0
    try:
        with open('/proc/%d/status' % pid, 'r') as f:
            for line in f:
                if ':' not in line:
                    continue
                key, value = line.split(':', 1)
                if key == 'VmRSS':
                    rss = int(value.strip().split()[0])
    except (OSError, IOError):
        pass
    return rss


def set_fd_cloexec(socket):
    flags = fcntl.fcntl(socket, fcntl.F_GETFD)
    fcntl.fcntl(socket, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC)


def print_unidiff(filepath_a, filepath_b):
    def process_file(filepath):
        fh = None
        try:
            fh = open(filepath, 'r')
            lines = fh.readlines()
            ctime = time.ctime(os.stat(filepath).st_mtime)
        except Exception:
            if not os.path.exists(filepath):
                color_stdout('[File does not exist: {}]\n'.format(filepath),
                             schema='error')
            lines = []
            ctime = time.ctime()
        if fh:
            fh.close()
        return lines, ctime

    lines_a, time_a = process_file(filepath_a)
    lines_b, time_b = process_file(filepath_b)
    diff = difflib.unified_diff(lines_a,
                                lines_b,
                                filepath_a,
                                filepath_b,
                                time_a,
                                time_b)
    color_stdout.writeout_unidiff(diff)


def prefix_each_line(prefix, data):
    data = data.rstrip('\n')
    lines = [(line + '\n') for line in data.split('\n')]
    return prefix + prefix.join(lines)


def just_and_trim(src, width):
    if len(src) > width:
        return src[:width - 1] + '>'
    return src.ljust(width)


def xlog_rows(xlog_path):
    """ Parse xlog / snapshot file.

        Assume tarantool and tarantoolctl is in PATH.
    """
    if not os.path.exists(xlog_path):
        raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), xlog_path)
    cmd = ['tarantoolctl', 'cat', xlog_path, '--format=json', '--show-system']
    with open(os.devnull, 'w') as devnull:
        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=devnull)
    for line in process.stdout.readlines():
        yield json.loads(bytes_to_str(line))


def extract_schema_from_snapshot(snapshot_path):
    """
    Extract schema version from snapshot.

    Assume tarantool and tarantoolctl is in PATH.

    Example of record:

     {
       "HEADER": {"lsn":2, "type": "INSERT", "timestamp": 1584694286.0031},
       "BODY": {"space_id": 272, "tuple": ["version", 2, 3, 1]}
     }

    :returns: (2, 3, 1)
    """
    BOX_SCHEMA_ID = 272
    for row in xlog_rows(snapshot_path):
        if row['HEADER']['type'] == 'INSERT' and \
           row['BODY']['space_id'] == BOX_SCHEMA_ID:
            res = row['BODY']['tuple']
            if res[0] == 'version':
                return tuple(res[1:])
    return None


def assert_bytes(b):
    """ Ensure given value is <bytes>.
    """
    if type(b) != bytes:
        raise ValueError('Internal error: expected {}, got {}: {}'.format(
            str(bytes), str(type(b)), repr(b)))


def assert_str(s):
    """ Ensure given value is <str>.
    """
    if type(s) != str:
        raise ValueError('Internal error: expected {}, got {}: {}'.format(
            str(str), str(type(s)), repr(s)))


def bytes_to_str(b):
    """ Convert <bytes> to <str>.

        No-op on Python 2.
    """
    assert_bytes(b)
    if PY2:
        return b
    return b.decode('utf-8')


def str_to_bytes(s):
    """ Convert <str> to <bytes>.

        No-op on Python 2.
    """
    assert_str(s)
    if PY2:
        return s
    return s.encode('utf-8')


def parse_tag_line(line):
    tags_str = line.split(':', 1)[1].strip()
    return [tag.strip() for tag in tags_str.split(',')]


def find_tags(filename):
    """ Extract tags from a first comment in the file.
    """
    # TODO: Support multiline comments. See exclude_tests() in
    # lib/server.py.
    if filename.endswith('.lua') or filename.endswith('.sql'):
        singleline_comment = '--'
    elif filename.endswith('.py'):
        singleline_comment = '#'
    else:
        return []

    tags = []
    with open(filename, 'r') as f:
        for line in f:
            line = line.rstrip('\n')
            if line.startswith('#!'):
                pass
            elif line == '':
                pass
            elif line.startswith(singleline_comment + ' tags:'):
                tags.extend(parse_tag_line(line))
            elif line.startswith(singleline_comment):
                pass
            else:
                break
    return tags


def prepend_path(p):
    """ Add an absolute path into PATH (at start) if it is not already there.
    """
    p = os.path.abspath(p)
    if p in os.environ['PATH'].split(os.pathsep):
        return
    os.environ['PATH'] = os.pathsep.join((p, os.environ['PATH']))


def shlex_quote(s):
    return _shlex_quote(s)

1	import errno	1✔
2	import os	1✔
3	import sys	1✔
4	import collections	1✔
5	import signal	1✔
6	import fcntl	1✔
7	import difflib	1✔
8	import time	1✔
9	import json	1✔
10	import subprocess	1✔
11	from lib.colorer import color_stdout	1✔
12
13	try:	1✔
14	# Python3.5 or above
15	from signal import Signals	1✔
16	except ImportError:	×
17	# Python2
18	Signals = None	×
19
20	try:	1✔
21	# Python 3.3+.
22	from shlex import quote as _shlex_quote	1✔
23	except ImportError:	×
24	# Python 2.7.
25	from pipes import quote as _shlex_quote	×
26
27
28	UNIX_SOCKET_LEN_LIMIT = 107	1✔
29
30	# Useful for very coarse version differentiation.
31	PY3 = sys.version_info[0] == 3	1✔
32	PY2 = sys.version_info[0] == 2	1✔
33
34	if PY2:	1!
35	FileNotFoundError = IOError	×
36
37	if PY3:	1!
38	string_types = str,	1✔
39	integer_types = int,	1✔
40	else:
41	string_types = basestring, # noqa: F821	×
42	integer_types = (int, long) # noqa: F821	×
43
44
45	def check_libs():	1✔
46	deps = [	1✔
47	('msgpack', 'msgpack-python'),
48	('tarantool', 'tarantool-python')
49	]
50	base_path = os.path.dirname(os.path.abspath(__file__))	1✔
51
52	for (mod_name, mod_dir) in deps:	1✔
53	mod_path = os.path.join(base_path, mod_dir)	1✔
54	if mod_path not in sys.path:	1!
55	sys.path = [mod_path] + sys.path	1✔
56
57	for (mod_name, _mod_dir) in deps:	1✔
58	try:	1✔
59	__import__(mod_name)	1✔
60	except ImportError as e:	×
61	color_stdout("\n\nNo %s library found\n" % mod_name,	×
62	schema='error')
63	print(e)	×
64	sys.exit(1)	×
65
66
67	def non_empty_valgrind_logs(paths_to_log):	1✔
68	""" Check that there were no warnings in the log."""
69	non_empty_logs = []	×
70	for path_to_log in paths_to_log:	×
71	if os.path.exists(path_to_log) and os.path.getsize(path_to_log) != 0:	×
72	non_empty_logs.append(path_to_log)	×
73	return non_empty_logs	×
74
75
76	def print_tail_n(filename, num_lines=None):	1✔
77	""" Print N last lines of a file. If num_lines is not set,
78	prints the whole file.
79	"""
80	with open(filename, "r") as logfile:	×
81	tail_n = collections.deque(logfile, num_lines)	×
82	for line in tail_n:	×
83	color_stdout(line, schema='tail')	×
84
85
86	def find_in_path(name):	1✔
87	path = os.curdir + os.pathsep + os.environ["PATH"]	×
88	for _dir in path.split(os.pathsep):	×
89	exe = os.path.join(_dir, name)	×
90	if os.access(exe, os.X_OK):	×
91	return exe	×
92	return ''	×
93
94
95	# http://stackoverflow.com/a/2549950
96	SIGNAMES = dict((int(v), k) for k, v in reversed(sorted(	1✔
97	signal.__dict__.items())) if k.startswith('SIG') and
98	not k.startswith('SIG_'))
99	SIGNUMS = dict((k, int(v)) for k, v in reversed(sorted(	1✔
100	signal.__dict__.items())) if k.startswith('SIG') and
101	not k.startswith('SIG_'))
102
103
104	def signame(signal):	1✔
105	if isinstance(signal, integer_types):	1!
106	return SIGNAMES[signal]	1✔
107	if Signals and isinstance(signal, Signals):	×
108	return SIGNAMES[int(signal)]	×
109	if isinstance(signal, string_types):	×
110	return signal	×
111	raise TypeError('signame(): signal argument of unexpected type: {}'.format(	×
112	str(type(signal))))
113
114
115	def signum(signal):	1✔
116	if isinstance(signal, integer_types):	×
117	return signal	×
118	if Signals and isinstance(signal, Signals):	×
119	return int(signal)	×
120	if isinstance(signal, string_types):	×
121	if not signal.startswith('SIG'):	×
122	signal = 'SIG' + signal	×
123	return SIGNUMS[signal]	×
124	raise TypeError('signum(): signal argument of unexpected type: {}'.format(	×
125	str(type(signal))))
126
127
128	def warn_unix_sockets_at_start(vardir):	1✔
129	max_unix_socket_rel = '???_replication/autobootstrap_guest3.control'	1✔
130	real_vardir = os.path.realpath(vardir)	1✔
131	max_unix_socket_abs = os.path.join(real_vardir, max_unix_socket_rel)	1✔
132	max_unix_socket_real = os.path.realpath(max_unix_socket_abs)	1✔
133	if len(max_unix_socket_real) > UNIX_SOCKET_LEN_LIMIT:	1!
134	color_stdout(	×
135	'WARGING: unix sockets can become longer than %d symbols:\n'
136	% UNIX_SOCKET_LEN_LIMIT,
137	schema='error')
138	color_stdout('WARNING: for example: "%s" has length %d\n' %	×
139	(max_unix_socket_real, len(max_unix_socket_real)),
140	schema='error')
141
142
143	def warn_unix_socket(path):	1✔
144	real_path = os.path.realpath(path)	1✔
145	if len(real_path) <= UNIX_SOCKET_LEN_LIMIT or \	1!
146	real_path in warn_unix_socket.warned:
147	return	1✔
148	color_stdout(	×
149	'\nWARGING: unix socket\'s "%s" path has length %d symbols that is '
150	'longer than %d. That likely will cause failing of tests.\n' %
151	(real_path, len(real_path), UNIX_SOCKET_LEN_LIMIT), schema='error')
152	warn_unix_socket.warned.add(real_path)	×
153
154
155	warn_unix_socket.warned = set()	1✔
156
157
158	def safe_makedirs(directory):	1✔
159	if os.path.isdir(directory):	1✔
160	return	1✔
161	# try-except to prevent races btw processes
162	try:	1✔
163	os.makedirs(directory)	1✔
164	except OSError:	×
165	pass	×
166
167
168	def format_process(pid):	1✔
169	cmdline = 'unknown'	1✔
170	try:	1✔
171	with open('/proc/%d/cmdline' % pid, 'r') as f:	1✔
172	cmdline = ' '.join(f.read().split('\0')).strip() or cmdline	1✔
173	except (OSError, IOError):	1✔
174	pass	1✔
175	status = 'unknown'	1✔
176	try:	1✔
177	with open('/proc/%d/status' % pid, 'r') as f:	1✔
178	for line in f:	1✔
179	if ':' not in line:	1!
180	continue	×
181	key, value = line.split(':', 1)	1✔
182	if key == 'State':	1✔
183	status = value.strip()	1✔
184	except (OSError, IOError):	1✔
185	pass	1✔
186	return 'process %d [%s; %s]' % (pid, status, cmdline)	1✔
187
188
189	def proc_stat_rss_supported():	1✔
190	return os.path.isfile('/proc/%d/status' % os.getpid())	1✔
191
192
193	def get_proc_stat_rss(pid):	1✔
194	rss = 0	1✔
195	try:	1✔
196	with open('/proc/%d/status' % pid, 'r') as f:	1✔
197	for line in f:	1✔
198	if ':' not in line:	1!
199	continue	×
200	key, value = line.split(':', 1)	1✔
201	if key == 'VmRSS':	1✔
202	rss = int(value.strip().split()[0])	1✔
203	except (OSError, IOError):	×
204	pass	×
205	return rss	1✔
206
207
208	def set_fd_cloexec(socket):	1✔
209	flags = fcntl.fcntl(socket, fcntl.F_GETFD)	1✔
210	fcntl.fcntl(socket, fcntl.F_SETFD, flags \| fcntl.FD_CLOEXEC)	1✔
211
212
213	def print_unidiff(filepath_a, filepath_b):	1✔
214	def process_file(filepath):	×
215	fh = None	×
216	try:	×
217	fh = open(filepath, 'r')	×
218	lines = fh.readlines()	×
219	ctime = time.ctime(os.stat(filepath).st_mtime)	×
220	except Exception:	×
221	if not os.path.exists(filepath):	×
222	color_stdout('[File does not exist: {}]\n'.format(filepath),	×
223	schema='error')
224	lines = []	×
225	ctime = time.ctime()	×
226	if fh:	×
227	fh.close()	×
228	return lines, ctime	×
229
230	lines_a, time_a = process_file(filepath_a)	×
231	lines_b, time_b = process_file(filepath_b)	×
232	diff = difflib.unified_diff(lines_a,	×
233	lines_b,
234	filepath_a,
235	filepath_b,
236	time_a,
237	time_b)
238	color_stdout.writeout_unidiff(diff)	×
239
240
241	def prefix_each_line(prefix, data):	1✔
242	data = data.rstrip('\n')	1✔
243	lines = [(line + '\n') for line in data.split('\n')]	1✔
244	return prefix + prefix.join(lines)	1✔
245
246
247	def just_and_trim(src, width):	1✔
248	if len(src) > width:	1✔
249	return src[:width - 1] + '>'	1✔
250	return src.ljust(width)	1✔
251
252
253	def xlog_rows(xlog_path):	1✔
254	""" Parse xlog / snapshot file.
255
256	Assume tarantool and tarantoolctl is in PATH.
257	"""
258	if not os.path.exists(xlog_path):	1!
259	raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), xlog_path)	×
260	cmd = ['tarantoolctl', 'cat', xlog_path, '--format=json', '--show-system']	1✔
261	with open(os.devnull, 'w') as devnull:	1✔
262	process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=devnull)	1✔
263	for line in process.stdout.readlines():	1!
264	yield json.loads(bytes_to_str(line))	1✔
265
266
267	def extract_schema_from_snapshot(snapshot_path):	1✔
268	"""
269	Extract schema version from snapshot.
270
271	Assume tarantool and tarantoolctl is in PATH.
272
273	Example of record:
274
275	{
276	"HEADER": {"lsn":2, "type": "INSERT", "timestamp": 1584694286.0031},
277	"BODY": {"space_id": 272, "tuple": ["version", 2, 3, 1]}
278	}
279
280	:returns: (2, 3, 1)
281	"""
282	BOX_SCHEMA_ID = 272	1✔
283	for row in xlog_rows(snapshot_path):	1!
284	if row['HEADER']['type'] == 'INSERT' and \	1!
285	row['BODY']['space_id'] == BOX_SCHEMA_ID:
286	res = row['BODY']['tuple']	1✔
287	if res[0] == 'version':	1✔
288	return tuple(res[1:])	1✔
289	return None	×
290
291
292	def assert_bytes(b):	1✔
293	""" Ensure given value is <bytes>.
294	"""
295	if type(b) != bytes:	1!
296	raise ValueError('Internal error: expected {}, got {}: {}'.format(	×
297	str(bytes), str(type(b)), repr(b)))
298
299
300	def assert_str(s):	1✔
301	""" Ensure given value is <str>.
302	"""
303	if type(s) != str:	1!
304	raise ValueError('Internal error: expected {}, got {}: {}'.format(	×
305	str(str), str(type(s)), repr(s)))
306
307
308	def bytes_to_str(b):	1✔
309	""" Convert <bytes> to <str>.
310
311	No-op on Python 2.
312	"""
313	assert_bytes(b)	1✔
314	if PY2:	1!
315	return b	×
316	return b.decode('utf-8')	1✔
317
318
319	def str_to_bytes(s):	1✔
320	""" Convert <str> to <bytes>.
321
322	No-op on Python 2.
323	"""
324	assert_str(s)	1✔
325	if PY2:	1!
326	return s	×
327	return s.encode('utf-8')	1✔
328
329
330	def parse_tag_line(line):	1✔
331	tags_str = line.split(':', 1)[1].strip()	×
332	return [tag.strip() for tag in tags_str.split(',')]	×
333
334
335	def find_tags(filename):	1✔
336	""" Extract tags from a first comment in the file.
337	"""
338	# TODO: Support multiline comments. See exclude_tests() in
339	# lib/server.py.
340	if filename.endswith('.lua') or filename.endswith('.sql'):	×
341	singleline_comment = '--'	×
342	elif filename.endswith('.py'):	×
343	singleline_comment = '#'	×
344	else:
345	return []	×
346
347	tags = []	×
348	with open(filename, 'r') as f:	×
349	for line in f:	×
350	line = line.rstrip('\n')	×
351	if line.startswith('#!'):	×
352	pass	×
353	elif line == '':	×
354	pass	×
355	elif line.startswith(singleline_comment + ' tags:'):	×
356	tags.extend(parse_tag_line(line))	×
357	elif line.startswith(singleline_comment):	×
358	pass	×
359	else:
360	break	×
361	return tags	×
362
363
364	def prepend_path(p):	1✔
365	""" Add an absolute path into PATH (at start) if it is not already there.
366	"""
367	p = os.path.abspath(p)	1✔
368	if p in os.environ['PATH'].split(os.pathsep):	1✔
369	return	1✔
370	os.environ['PATH'] = os.pathsep.join((p, os.environ['PATH']))	1✔
371
372
373	def shlex_quote(s):	1✔
374	return _shlex_quote(s)	×

tarantool / test-run / 3685840692

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous