5132620618

pending completion

Build # 5132620618

Build Type

push

github

Committed by

ylobankov

Commit Message

Prettify messages about log files

The ideas behind the change:

* The log is sometimes from a tarantool instance, but a luatest based
  test may print logs from another process to stderr. Use more general
  term 'log' to don't confuse anyone.
* Replace term 'instance' with 'test-run server' due to the same
  reasons.
* Make zero size and no file situations more explicit.
* Catch and report 'no logfile property' situation (shouldn't occur, but
  was semi-handled by this code previously for some reason).
* Reduce code reusability for the sake of readability and to ease future
  modifications.

Run Details

758 of 1554 branches covered (48.78%)

Branch coverage included in aggregate %.

22 of 22 new or added lines in 1 file covered. (100.0%)

2929 of 4328 relevant lines covered (67.68%)

0.68 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.67

/lib/server.py

import glob
import os
import shutil
import subprocess
from itertools import product

from lib.server_mixins import ValgrindMixin
from lib.server_mixins import GdbMixin
from lib.server_mixins import GdbServerMixin
from lib.server_mixins import LLdbMixin
from lib.server_mixins import StraceMixin
from lib.server_mixins import LuacovMixin
from lib.colorer import color_stdout
from lib.options import Options
from lib.utils import print_tail_n
from lib.utils import bytes_to_str
from lib.utils import find_tags

DEFAULT_CHECKPOINT_PATTERNS = ["*.snap", "*.xlog", "*.vylog", "*.inprogress",
                               "[0-9]*/"]

DEFAULT_SNAPSHOT_NAME = "00000000000000000000.snap"


class Server(object):
    """Server represents a single server instance. Normally, the
    program operates with only one server, but in future we may add
    replication slaves. The server is started once at the beginning
    of each suite, and stopped at the end."""
    DEFAULT_INSPECTOR = 0
    TEST_RUN_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                ".."))
    # assert(false) hangs due to gh-4983, added fiber.sleep(0) to workaround it
    DISABLE_AUTO_UPGRADE = "require('fiber').sleep(0) \
        assert(box.error.injection.set('ERRINJ_AUTO_UPGRADE', true) == 'ok', \
        'no such errinj')"

    @property
    def vardir(self):
        if not hasattr(self, '_vardir'):
            raise ValueError("No vardir specified")
        return self._vardir

    @vardir.setter
    def vardir(self, path):
        if path is None:
            return
        self._vardir = os.path.abspath(path)

    @staticmethod
    def get_mixed_class(cls, ini):
        if ini is None:
            return cls

        conflict_options = ('valgrind', 'gdb', 'gdbserver', 'lldb', 'strace')
        for op1, op2 in product(conflict_options, repeat=2):
            if op1 != op2 and \
                    (op1 in ini and ini[op1]) and \
                    (op2 in ini and ini[op2]):
                format_str = 'Can\'t run under {} and {} simultaniously'
                raise OSError(format_str.format(op1, op2))

        lname = cls.__name__.lower()

        if ini.get('valgrind') and 'valgrind' not in lname:
            cls = type('Valgrind' + cls.__name__, (ValgrindMixin, cls), {})
        elif ini.get('gdbserver') and 'gdbserver' not in lname:
            cls = type('GdbServer' + cls.__name__, (GdbServerMixin, cls), {})
        elif ini.get('gdb') and 'gdb' not in lname:
            cls = type('Gdb' + cls.__name__, (GdbMixin, cls), {})
        elif ini.get('lldb') and 'lldb' not in lname:
            cls = type('LLdb' + cls.__name__, (LLdbMixin, cls), {})
        elif 'strace' in ini and ini['strace']:
            cls = type('Strace' + cls.__name__, (StraceMixin, cls), {})
        elif 'luacov' in ini and ini['luacov']:
            cls = type('Luacov' + cls.__name__, (LuacovMixin, cls), {})

        return cls

    def __new__(cls, ini=None, *args, **kwargs):
        if ini is None or 'core' not in ini or ini['core'] is None:
            return object.__new__(cls)
        core = ini['core'].lower().strip()
        cls.mdlname = "lib.{0}_server".format(core.replace(' ', '_'))
        cls.clsname = "{0}Server".format(core.title().replace(' ', ''))
        corecls = __import__(cls.mdlname,
                             fromlist=cls.clsname).__dict__[cls.clsname]
        return corecls.__new__(corecls, ini, *args, **kwargs)

    def __init__(self, ini, test_suite=None):
        self.core = ini['core']
        self.ini = ini
        self.vardir = ini['vardir']
        self.inspector_port = int(ini.get(
            'inspector_port', self.DEFAULT_INSPECTOR
        ))
        self.disable_schema_upgrade = Options().args.disable_schema_upgrade
        self.snapshot_path = Options().args.snapshot_path

        # filled in {Test,AppTest,LuaTest,PythonTest}.execute()
        # or passed through execfile() for PythonTest (see
        # TarantoolServer.__init__).
        self.current_test = None

        # Used in valgrind_log property. 'test_suite' is not None only for
        # default servers running in TestSuite.run_all()
        self.test_suite = test_suite

    @classmethod
    def version(cls):
        p = subprocess.Popen([cls.binary, "--version"], stdout=subprocess.PIPE)
        version = bytes_to_str(p.stdout.read()).rstrip()
        p.wait()
        return version

    def prepare_args(self, args=[]):
        return args

    def pretest_clean(self):
        self.cleanup()

    def cleanup(self, dirname='.'):
        waldir = os.path.join(self.vardir, dirname)
        for pattern in DEFAULT_CHECKPOINT_PATTERNS:
            for f in glob.glob(os.path.join(waldir, pattern)):
                if os.path.isdir(f):
                    shutil.rmtree(f)
                else:
                    os.remove(f)

    def install(self, binary=None, vardir=None, mem=None, silent=True):
        pass

    def init(self):
        pass

    def start(self, silent=True):
        pass

    def stop(self, silent=True):
        pass

    def restart(self):
        pass

    def print_log(self, num_lines=None):
        """ Show information from the given log file.
        """
        prefix = '\n[test-run server "{instance}"] '.format(instance=self.name)
        if not self.logfile:
            msg = 'No log file is set (internal test-run error)\n'
            color_stdout(prefix + msg, schema='error')
        elif not os.path.exists(self.logfile):
            fmt_str = 'The log file {logfile} does not exist\n'
            msg = fmt_str.format(logfile=self.logfile)
            color_stdout(prefix + msg, schema='error')
        elif os.path.getsize(self.logfile) == 0:
            fmt_str = 'The log file {logfile} has zero size\n'
            msg = fmt_str.format(logfile=self.logfile)
            color_stdout(prefix + msg, schema='error')
        elif num_lines:
            fmt_str = 'Last {num_lines} lines of the log file {logfile}:\n'
            msg = fmt_str.format(logfile=self.logfile, num_lines=num_lines)
            color_stdout(prefix + msg, schema='error')
            print_tail_n(self.logfile, num_lines)
        else:
            fmt_str = 'The log file {logfile}:\n'
            msg = fmt_str.format(logfile=self.logfile)
            color_stdout(msg, schema='error')
            print_tail_n(self.logfile, num_lines)

    @staticmethod
    def exclude_tests(test_names, exclude_patterns):
        """ Filter out test files by several criteria:

            exclude_patters: a list of strings. If a string is
            a substring of the test full name, exclude it.

            If --tags option is provided, exclude tests, which
            have no a tag from the provided list.
        """

        # TODO: Support filtering by another file (for unit
        # tests). Transform a test name into a test source name.
        # Don't forget about out of source build.
        # TODO: Support multiline comments (mainly for unit
        # tests).

        def match_any_pattern(test_name, patterns):
            for pattern in patterns:
                if pattern in test_name:
                    return True
            return False

        def match_any_tag(test_name, accepted_tags):
            tags = find_tags(test_name)
            for tag in tags:
                if tag in accepted_tags:
                    return True
            return False

        accepted_tags = Options().args.tags

        res = []
        for test_name in test_names:
            if match_any_pattern(test_name, exclude_patterns):
                continue
            if accepted_tags is None or match_any_tag(test_name, accepted_tags):
                res.append(test_name)
        return res

1	import glob	1✔
2	import os	1✔
3	import shutil	1✔
4	import subprocess	1✔
5	from itertools import product	1✔
6
7	from lib.server_mixins import ValgrindMixin	1✔
8	from lib.server_mixins import GdbMixin	1✔
9	from lib.server_mixins import GdbServerMixin	1✔
10	from lib.server_mixins import LLdbMixin	1✔
11	from lib.server_mixins import StraceMixin	1✔
12	from lib.server_mixins import LuacovMixin	1✔
13	from lib.colorer import color_stdout	1✔
14	from lib.options import Options	1✔
15	from lib.utils import print_tail_n	1✔
16	from lib.utils import bytes_to_str	1✔
17	from lib.utils import find_tags	1✔
18
19	DEFAULT_CHECKPOINT_PATTERNS = [".snap", ".xlog", ".vylog", ".inprogress",	1✔
20	"[0-9]*/"]
21
22	DEFAULT_SNAPSHOT_NAME = "00000000000000000000.snap"	1✔
23
24
25	class Server(object):	1✔
26	"""Server represents a single server instance. Normally, the
27	program operates with only one server, but in future we may add
28	replication slaves. The server is started once at the beginning
29	of each suite, and stopped at the end."""
30	DEFAULT_INSPECTOR = 0	1✔
31	TEST_RUN_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__),	1✔
32	".."))
33	# assert(false) hangs due to gh-4983, added fiber.sleep(0) to workaround it
34	DISABLE_AUTO_UPGRADE = "require('fiber').sleep(0) \	1✔
35	assert(box.error.injection.set('ERRINJ_AUTO_UPGRADE', true) == 'ok', \
36	'no such errinj')"
37
38	@property	1✔
39	def vardir(self):	1✔
40	if not hasattr(self, '_vardir'):	1!
41	raise ValueError("No vardir specified")	×
42	return self._vardir	1✔
43
44	@vardir.setter	1✔
45	def vardir(self, path):	1✔
46	if path is None:	1✔
47	return	1✔
48	self._vardir = os.path.abspath(path)	1✔
49
50	@staticmethod	1✔
51	def get_mixed_class(cls, ini):	1✔
52	if ini is None:	1✔
53	return cls	1✔
54
55	conflict_options = ('valgrind', 'gdb', 'gdbserver', 'lldb', 'strace')	1✔
56	for op1, op2 in product(conflict_options, repeat=2):	1✔
57	if op1 != op2 and \	1!
58	(op1 in ini and ini[op1]) and \
59	(op2 in ini and ini[op2]):
60	format_str = 'Can\'t run under {} and {} simultaniously'	×
61	raise OSError(format_str.format(op1, op2))	×
62
63	lname = cls.__name__.lower()	1✔
64
65	if ini.get('valgrind') and 'valgrind' not in lname:	1!
66	cls = type('Valgrind' + cls.__name__, (ValgrindMixin, cls), {})	×
67	elif ini.get('gdbserver') and 'gdbserver' not in lname:	1!
68	cls = type('GdbServer' + cls.__name__, (GdbServerMixin, cls), {})	×
69	elif ini.get('gdb') and 'gdb' not in lname:	1!
70	cls = type('Gdb' + cls.__name__, (GdbMixin, cls), {})	×
71	elif ini.get('lldb') and 'lldb' not in lname:	1!
72	cls = type('LLdb' + cls.__name__, (LLdbMixin, cls), {})	×
73	elif 'strace' in ini and ini['strace']:	1!
74	cls = type('Strace' + cls.__name__, (StraceMixin, cls), {})	×
75	elif 'luacov' in ini and ini['luacov']:	1!
76	cls = type('Luacov' + cls.__name__, (LuacovMixin, cls), {})	×
77
78	return cls	1✔
79
80	def __new__(cls, ini=None, args, *kwargs):	1✔
81	if ini is None or 'core' not in ini or ini['core'] is None:	1!
82	return object.__new__(cls)	×
83	core = ini['core'].lower().strip()	1✔
84	cls.mdlname = "lib.{0}_server".format(core.replace(' ', '_'))	1✔
85	cls.clsname = "{0}Server".format(core.title().replace(' ', ''))	1✔
86	corecls = __import__(cls.mdlname,	1✔
87	fromlist=cls.clsname).__dict__[cls.clsname]
88	return corecls.__new__(corecls, ini, args, *kwargs)	1✔
89
90	def __init__(self, ini, test_suite=None):	1✔
91	self.core = ini['core']	1✔
92	self.ini = ini	1✔
93	self.vardir = ini['vardir']	1✔
94	self.inspector_port = int(ini.get(	1✔
95	'inspector_port', self.DEFAULT_INSPECTOR
96	))
97	self.disable_schema_upgrade = Options().args.disable_schema_upgrade	1✔
98	self.snapshot_path = Options().args.snapshot_path	1✔
99
100	# filled in {Test,AppTest,LuaTest,PythonTest}.execute()
101	# or passed through execfile() for PythonTest (see
102	# TarantoolServer.__init__).
103	self.current_test = None	1✔
104
105	# Used in valgrind_log property. 'test_suite' is not None only for
106	# default servers running in TestSuite.run_all()
107	self.test_suite = test_suite	1✔
108
109	@classmethod	1✔
110	def version(cls):	1✔
111	p = subprocess.Popen([cls.binary, "--version"], stdout=subprocess.PIPE)	1✔
112	version = bytes_to_str(p.stdout.read()).rstrip()	1✔
113	p.wait()	1✔
114	return version	1✔
115
116	def prepare_args(self, args=[]):	1✔
117	return args	×
118
119	def pretest_clean(self):	1✔
120	self.cleanup()	1✔
121
122	def cleanup(self, dirname='.'):	1✔
123	waldir = os.path.join(self.vardir, dirname)	1✔
124	for pattern in DEFAULT_CHECKPOINT_PATTERNS:	1✔
125	for f in glob.glob(os.path.join(waldir, pattern)):	1✔
126	if os.path.isdir(f):	1!
127	shutil.rmtree(f)	×
128	else:
129	os.remove(f)	1✔
130
131	def install(self, binary=None, vardir=None, mem=None, silent=True):	1✔
132	pass	×
133
134	def init(self):	1✔
135	pass	×
136
137	def start(self, silent=True):	1✔
138	pass	×
139
140	def stop(self, silent=True):	1✔
141	pass	1✔
142
143	def restart(self):	1✔
144	pass	×
145
146	def print_log(self, num_lines=None):	1✔
147	""" Show information from the given log file.
148	"""
149	prefix = '\n[test-run server "{instance}"] '.format(instance=self.name)	×
150	if not self.logfile:	×
151	msg = 'No log file is set (internal test-run error)\n'	×
152	color_stdout(prefix + msg, schema='error')	×
153	elif not os.path.exists(self.logfile):	×
154	fmt_str = 'The log file {logfile} does not exist\n'	×
155	msg = fmt_str.format(logfile=self.logfile)	×
156	color_stdout(prefix + msg, schema='error')	×
157	elif os.path.getsize(self.logfile) == 0:	×
158	fmt_str = 'The log file {logfile} has zero size\n'	×
159	msg = fmt_str.format(logfile=self.logfile)	×
160	color_stdout(prefix + msg, schema='error')	×
161	elif num_lines:	×
162	fmt_str = 'Last {num_lines} lines of the log file {logfile}:\n'	×
163	msg = fmt_str.format(logfile=self.logfile, num_lines=num_lines)	×
164	color_stdout(prefix + msg, schema='error')	×
165	print_tail_n(self.logfile, num_lines)	×
166	else:
167	fmt_str = 'The log file {logfile}:\n'	×
168	msg = fmt_str.format(logfile=self.logfile)	×
169	color_stdout(msg, schema='error')	×
170	print_tail_n(self.logfile, num_lines)	×
171
172	@staticmethod	1✔
173	def exclude_tests(test_names, exclude_patterns):	1✔
174	""" Filter out test files by several criteria:
175
176	exclude_patters: a list of strings. If a string is
177	a substring of the test full name, exclude it.
178
179	If --tags option is provided, exclude tests, which
180	have no a tag from the provided list.
181	"""
182
183	# TODO: Support filtering by another file (for unit
184	# tests). Transform a test name into a test source name.
185	# Don't forget about out of source build.
186	# TODO: Support multiline comments (mainly for unit
187	# tests).
188
189	def match_any_pattern(test_name, patterns):	1✔
190	for pattern in patterns:	1✔
191	if pattern in test_name:	1✔
192	return True	1✔
193	return False	1✔
194
195	def match_any_tag(test_name, accepted_tags):	1✔
196	tags = find_tags(test_name)	×
197	for tag in tags:	×
198	if tag in accepted_tags:	×
199	return True	×
200	return False	×
201
202	accepted_tags = Options().args.tags	1✔
203
204	res = []	1✔
205	for test_name in test_names:	1✔
206	if match_any_pattern(test_name, exclude_patterns):	1✔
207	continue	1✔
208	if accepted_tags is None or match_any_tag(test_name, accepted_tags):	1!
209	res.append(test_name)	1✔
210	return res	1✔

tarantool / test-run / 5132620618

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous