9430798702

Committed 08 Jun 2024 06:38PM UTC coverage: 62.625% (+0.05%) from 62.574%

Build # 9430798702

Build Type

push

github

Committed by

ylobankov

Commit Message

luatest: allow to run test cases in parallel

In order to use the feature, add the following comment to a beginning of
the test file:

```lua
-- tags: parallel
```

Run Details

765 of 1578 branches covered (48.48%)

Branch coverage included in aggregate %.

8 of 20 new or added lines in 1 file covered. (40.0%)

11 existing lines in 1 file now uncovered.

2980 of 4402 relevant lines covered (67.7%)

0.68 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

71.43

/lib/luatest_server.py

import glob
import os
import re
import sys

from subprocess import PIPE
from subprocess import Popen
from threading import Timer

from lib.colorer import color_stdout
from lib.error import TestRunInitError
from lib.options import Options
from lib.sampler import sampler
from lib.server import Server
from lib.tarantool_server import Test
from lib.tarantool_server import TestExecutionError
from lib.tarantool_server import TarantoolServer
from lib.utils import bytes_to_str
from lib.utils import find_tags


def timeout_handler(process, test_timeout):
    color_stdout("Test timeout of %d secs reached\t" % test_timeout, schema='error')
    process.kill()


class LuatestTest(Test):
    """ Handle *_test.lua.

    Provide method for executing luatest <name>_test.lua test.
    """

    def __init__(self, *args, **kwargs):
        super(LuatestTest, self).__init__(*args, **kwargs)
        self.valgrind = kwargs.get('valgrind', False)

    def execute(self, server):
        """Execute test by luatest command

        Execute `luatest -c --no-clean --verbose <name>_test.lua --output tap`
        command. Disable capture mode and deletion of the var directory.
        Provide a verbose output in the tap format. Extend the command by
        `--pattern <pattern>` if the corresponding option is provided.
        """
        server.current_test = self
        script = os.path.join(os.path.basename(server.testdir), self.name)

        # Disable stdout buffering.
        command = [server.binary, '-e', "io.stdout:setvbuf('no')"]
        # Add luatest as the script.
        command.extend([server.luatest])
        # Add luatest command-line options.
        command.extend(['-c', '--no-clean', '--verbose', script, '--output', 'tap'])
        if Options().args.pattern:
            for p in Options().args.pattern:
                command.extend(['--pattern', p])

        # Run a specific test case. See find_tests() for details.
        if 'test_case' in self.run_params:
            command.extend(['--run-test-case', self.run_params['test_case']])

        # We start luatest from the project source directory, it
        # is the usual way to use luatest.
        #
        # VARDIR (${BUILDDIR}/test/var/001_foo) will be used for
        # write ahead logs, snapshots, logs, unix domain sockets
        # and so on.
        os.environ['VARDIR'] = server.vardir
        project_dir = os.environ['SOURCEDIR']

        with open(server.logfile, 'ab') as f:
            proc = Popen(command, cwd=project_dir, stdout=sys.stdout, stderr=f)
        sampler.register_process(proc.pid, self.id, server.name)
        test_timeout = Options().args.test_timeout
        timer = Timer(test_timeout, timeout_handler, (proc, test_timeout))
        timer.start()
        proc.wait()
        timer.cancel()
        if proc.returncode != 0:
            raise TestExecutionError


class LuatestServer(Server):
    """A dummy server implementation for luatest server tests"""

    def __new__(cls, ini=None, *args, **kwargs):
        cls = Server.get_mixed_class(cls, ini)
        return object.__new__(cls)

    def __init__(self, _ini=None, test_suite=None):
        if _ini is None:
            _ini = {}
        ini = {'vardir': None}
        ini.update(_ini)
        super(LuatestServer, self).__init__(ini, test_suite)
        self.testdir = os.path.abspath(os.curdir)
        self.vardir = ini['vardir']
        self.builddir = ini['builddir']
        self.name = 'luatest_server'

    @property
    def logfile(self):
        # Remove the suite name using basename().
        test_name = os.path.basename(self.current_test.name)
        # Strip '.lua' from the end.
        #
        # The '_test' postfix is kept to ease distinguish this
        # log file from luatest.server instance logs.
        test_name = test_name[:-len('.lua')]
        # Add '.log'.
        file_name = test_name + '.log'
        # Put into vardir.
        return os.path.join(self.vardir, file_name)

    def deploy(self, vardir=None, silent=True, wait=True):
        self.vardir = vardir
        if not os.access(self.vardir, os.F_OK):
            os.makedirs(self.vardir)

    @classmethod
    def find_exe(cls, builddir):
        cls.builddir = builddir
        cls.binary = TarantoolServer.binary
        cls.debug = bool(re.findall(r'^Target:.*-Debug$', str(cls.version()),
                                    re.M))
        cls.luatest = os.environ['TEST_RUN_DIR'] + '/lib/luatest/bin/luatest'

    @classmethod
    def verify_luatest_exe(cls):
        """Verify that luatest executable is available."""
        try:
            # Just check that the command returns zero exit code.
            with open(os.devnull, 'w') as devnull:
                returncode = Popen([cls.luatest, '--version'],
                                   stdout=devnull,
                                   stderr=devnull).wait()
            if returncode != 0:
                raise TestRunInitError('Unable to run `luatest --version`',
                                       {'returncode': returncode})
        except OSError as e:
            # Python 2 raises OSError if the executable is not
            # found or if it has no executable bit. Python 3
            # raises FileNotFoundError and PermissionError in
            # those cases, which are childs of OSError anyway.
            raise TestRunInitError('Unable to find luatest executable', e)

    @classmethod
    def test_cases(cls, test_name):
        p = Popen([cls.luatest, test_name, '--list-test-cases'], stdout=PIPE)
        output = bytes_to_str(p.stdout.read()).rstrip()
        p.wait()

        # Exclude the first line if it is a tarantool version
        # report.
        res = output.split('\n')
        if len(res) > 0 and res[0].startswith('Tarantool version is'):
            return res[1:]

        return res

    @staticmethod
    def find_tests(test_suite, suite_path):
        """Looking for *_test.lua, which are can be executed by luatest."""

        # TODO: Investigate why this old hack is needed and drop
        # it if possible (move the assignment to test_suite.py).
        #
        # cdc70f94701f suggests that it is related to the out of
        # source build.
        test_suite.ini['suite'] = suite_path

        # A pattern here means just a substring to find in a test
        # name.
        include_patterns = Options().args.tests
        exclude_patterns = Options().args.exclude

        accepted_tags = Options().args.tags

        tests = []
        for test_name in glob.glob(os.path.join(suite_path, '*_test.lua')):
            # If neither of the include patterns are substrings of
            # the given test name, skip the test.
            if not any(p in test_name for p in include_patterns):
                continue

            # If at least one of the exclude patterns is a
            # substring of the given test name, skip the test.
            if any(p in test_name for p in exclude_patterns):
                continue

            tags = find_tags(test_name)

            # If --tags <...> CLI option is provided...
            if accepted_tags:
                # ...and the test has neither of the given tags,
                # skip the test.
                if not any(t in accepted_tags for t in tags):
                    continue

            # Add the test to the execution list otherwise.
            if 'parallel' in tags:
                # If the test has the 'parallel' tag, split the
                # test to test cases to run in separate tasks in
                # parallel.
                test_cases = LuatestServer.test_cases(test_name)

                # Display shorter test case names on the screen:
                # strip the common prefix.
                prefix_len = len(os.path.commonprefix(test_cases))

                for test_case in test_cases:
                    tests.append(LuatestTest(test_name, test_suite.args, test_suite.ini,
                                             params={"test_case": test_case},
                                             conf_name=test_case[prefix_len:]))
            else:
                # If the test has no 'parallel' tag, run all the
                # test cases as one task.
                tests.append(LuatestTest(test_name, test_suite.args, test_suite.ini))

        tests.sort(key=lambda t: t.name)

        # TODO: Don't modify a test suite object's field from
        # another object directly. It is much better to just
        # return a list of tests from this method.
        test_suite.tests = tests

1	import glob	1✔
2	import os	1✔
3	import re	1✔
4	import sys	1✔
5
6	from subprocess import PIPE	1✔
7	from subprocess import Popen	1✔
8	from threading import Timer	1✔
9
10	from lib.colorer import color_stdout	1✔
11	from lib.error import TestRunInitError	1✔
12	from lib.options import Options	1✔
13	from lib.sampler import sampler	1✔
14	from lib.server import Server	1✔
15	from lib.tarantool_server import Test	1✔
16	from lib.tarantool_server import TestExecutionError	1✔
17	from lib.tarantool_server import TarantoolServer	1✔
18	from lib.utils import bytes_to_str	1✔
19	from lib.utils import find_tags	1✔
20
21
22	def timeout_handler(process, test_timeout):	1✔
23	color_stdout("Test timeout of %d secs reached\t" % test_timeout, schema='error')	×
UNCOV 24	process.kill()	×
25
26
27	class LuatestTest(Test):	1✔
28	""" Handle *_test.lua.
29
30	Provide method for executing luatest <name>_test.lua test.
31	"""
32
33	def __init__(self, args, *kwargs):	1✔
34	super(LuatestTest, self).__init__(args, *kwargs)	1✔
35	self.valgrind = kwargs.get('valgrind', False)	1✔
36
37	def execute(self, server):	1✔
38	"""Execute test by luatest command
39
40	Execute `luatest -c --no-clean --verbose <name>_test.lua --output tap`
41	command. Disable capture mode and deletion of the var directory.
42	Provide a verbose output in the tap format. Extend the command by
43	`--pattern <pattern>` if the corresponding option is provided.
44	"""
45	server.current_test = self	1✔
46	script = os.path.join(os.path.basename(server.testdir), self.name)	1✔
47
48	# Disable stdout buffering.
49	command = [server.binary, '-e', "io.stdout:setvbuf('no')"]	1✔
50	# Add luatest as the script.
51	command.extend([server.luatest])	1✔
52	# Add luatest command-line options.
53	command.extend(['-c', '--no-clean', '--verbose', script, '--output', 'tap'])	1✔
54	if Options().args.pattern:	1!
55	for p in Options().args.pattern:	×
UNCOV 56	command.extend(['--pattern', p])	×
57
58	# Run a specific test case. See find_tests() for details.
59	if 'test_case' in self.run_params:	1!
NEW 60	command.extend(['--run-test-case', self.run_params['test_case']])	×
61
62	# We start luatest from the project source directory, it
63	# is the usual way to use luatest.
64	#
65	# VARDIR (${BUILDDIR}/test/var/001_foo) will be used for
66	# write ahead logs, snapshots, logs, unix domain sockets
67	# and so on.
68	os.environ['VARDIR'] = server.vardir	1✔
69	project_dir = os.environ['SOURCEDIR']	1✔
70
71	with open(server.logfile, 'ab') as f:	1✔
72	proc = Popen(command, cwd=project_dir, stdout=sys.stdout, stderr=f)	1✔
73	sampler.register_process(proc.pid, self.id, server.name)	1✔
74	test_timeout = Options().args.test_timeout	1✔
75	timer = Timer(test_timeout, timeout_handler, (proc, test_timeout))	1✔
76	timer.start()	1✔
77	proc.wait()	1✔
78	timer.cancel()	1✔
79	if proc.returncode != 0:	1!
UNCOV 80	raise TestExecutionError	×
81
82
83	class LuatestServer(Server):	1✔
84	"""A dummy server implementation for luatest server tests"""
85
86	def __new__(cls, ini=None, args, *kwargs):	1✔
87	cls = Server.get_mixed_class(cls, ini)	1✔
88	return object.__new__(cls)	1✔
89
90	def __init__(self, _ini=None, test_suite=None):	1✔
91	if _ini is None:	1!
UNCOV 92	_ini = {}	×
93	ini = {'vardir': None}	1✔
94	ini.update(_ini)	1✔
95	super(LuatestServer, self).__init__(ini, test_suite)	1✔
96	self.testdir = os.path.abspath(os.curdir)	1✔
97	self.vardir = ini['vardir']	1✔
98	self.builddir = ini['builddir']	1✔
99	self.name = 'luatest_server'	1✔
100
101	@property	1✔
102	def logfile(self):	1✔
103	# Remove the suite name using basename().
104	test_name = os.path.basename(self.current_test.name)	1✔
105	# Strip '.lua' from the end.
106	#
107	# The '_test' postfix is kept to ease distinguish this
108	# log file from luatest.server instance logs.
109	test_name = test_name[:-len('.lua')]	1✔
110	# Add '.log'.
111	file_name = test_name + '.log'	1✔
112	# Put into vardir.
113	return os.path.join(self.vardir, file_name)	1✔
114
115	def deploy(self, vardir=None, silent=True, wait=True):	1✔
116	self.vardir = vardir	1✔
117	if not os.access(self.vardir, os.F_OK):	1✔
118	os.makedirs(self.vardir)	1✔
119
120	@classmethod	1✔
121	def find_exe(cls, builddir):	1✔
122	cls.builddir = builddir	1✔
123	cls.binary = TarantoolServer.binary	1✔
124	cls.debug = bool(re.findall(r'^Target:.*-Debug$', str(cls.version()),	1✔
125	re.M))
126	cls.luatest = os.environ['TEST_RUN_DIR'] + '/lib/luatest/bin/luatest'	1✔
127
128	@classmethod	1✔
129	def verify_luatest_exe(cls):	1✔
130	"""Verify that luatest executable is available."""
131	try:	1✔
132	# Just check that the command returns zero exit code.
133	with open(os.devnull, 'w') as devnull:	1✔
134	returncode = Popen([cls.luatest, '--version'],	1✔
135	stdout=devnull,
136	stderr=devnull).wait()
137	if returncode != 0:	1!
UNCOV 138	raise TestRunInitError('Unable to run `luatest --version`',	×
139	{'returncode': returncode})
UNCOV 140	except OSError as e:	×
141	# Python 2 raises OSError if the executable is not
142	# found or if it has no executable bit. Python 3
143	# raises FileNotFoundError and PermissionError in
144	# those cases, which are childs of OSError anyway.
UNCOV 145	raise TestRunInitError('Unable to find luatest executable', e)	×
146
147	@classmethod	1✔
148	def test_cases(cls, test_name):	1✔
NEW 149	p = Popen([cls.luatest, test_name, '--list-test-cases'], stdout=PIPE)	×
NEW 150	output = bytes_to_str(p.stdout.read()).rstrip()	×
NEW 151	p.wait()	×
152
153	# Exclude the first line if it is a tarantool version
154	# report.
NEW 155	res = output.split('\n')	×
NEW 156	if len(res) > 0 and res[0].startswith('Tarantool version is'):	×
NEW 157	return res[1:]	×
158
NEW 159	return res	×
160
161	@staticmethod	1✔
162	def find_tests(test_suite, suite_path):	1✔
163	"""Looking for *_test.lua, which are can be executed by luatest."""
164
165	# TODO: Investigate why this old hack is needed and drop
166	# it if possible (move the assignment to test_suite.py).
167	#
168	# cdc70f94701f suggests that it is related to the out of
169	# source build.
170	test_suite.ini['suite'] = suite_path	1✔
171
172	# A pattern here means just a substring to find in a test
173	# name.
174	include_patterns = Options().args.tests	1✔
175	exclude_patterns = Options().args.exclude	1✔
176
177	accepted_tags = Options().args.tags	1✔
178
179	tests = []	1✔
180	for test_name in glob.glob(os.path.join(suite_path, '*_test.lua')):	1✔
181	# If neither of the include patterns are substrings of
182	# the given test name, skip the test.
183	if not any(p in test_name for p in include_patterns):	1!
UNCOV 184	continue	×
185
186	# If at least one of the exclude patterns is a
187	# substring of the given test name, skip the test.
188	if any(p in test_name for p in exclude_patterns):	1!
UNCOV 189	continue	×
190
191	tags = find_tags(test_name)	1✔
192
193	# If --tags <...> CLI option is provided...
194	if accepted_tags:	1!
195	# ...and the test has neither of the given tags,
196	# skip the test.
UNCOV 197	if not any(t in accepted_tags for t in tags):	×
UNCOV 198	continue	×
199
200	# Add the test to the execution list otherwise.
201	if 'parallel' in tags:	1!
202	# If the test has the 'parallel' tag, split the
203	# test to test cases to run in separate tasks in
204	# parallel.
NEW 205	test_cases = LuatestServer.test_cases(test_name)	×
206
207	# Display shorter test case names on the screen:
208	# strip the common prefix.
NEW 209	prefix_len = len(os.path.commonprefix(test_cases))	×
210
NEW 211	for test_case in test_cases:	×
NEW 212	tests.append(LuatestTest(test_name, test_suite.args, test_suite.ini,	×
213	params={"test_case": test_case},
214	conf_name=test_case[prefix_len:]))
215	else:
216	# If the test has no 'parallel' tag, run all the
217	# test cases as one task.
218	tests.append(LuatestTest(test_name, test_suite.args, test_suite.ini))	1✔
219
220	tests.sort(key=lambda t: t.name)	1✔
221
222	# TODO: Don't modify a test suite object's field from
223	# another object directly. It is much better to just
224	# return a list of tests from this method.
225	test_suite.tests = tests	1✔

tarantool / test-run / 9430798702

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous