1476

pending completion

Build # 1476

Build Type

Pull #402

travis-ci

Committed by

web-flow

Commit Message

<a href="https://github.com/zodb/relstorage/commit/<a class=hub.com/zodb/relstorage/commit/<a class="double-link" href="https://git"><a class=hub.com/zodb/relstorage/commit/<a class="double-link" href="https://git"><a class=hub.com/zodb/relstorage/commit/<a class="double-link" href="https://git"><a class=hub.com/zodb/relstorage/commit/75eefa8cf4a2b6ab6ab0e4b4bf29d26e2542b7af">75eefa8cf">&lt;a href=&quot;https://github.com/zodb/relstorage/commit/</a><a class="double-link" href="https://github.com/zodb/relstorage/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/zodb/relstorage/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/zodb/relstorage/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/zodb/relstorage/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/zodb/relstorage/commit/75eefa8cf4a2b6ab6ab0e4b4bf29d26e2542b7af">75eefa8cf</a><a href="https://github.com/zodb/relstorage/commit/75eefa8cf4a2b6ab6ab0e4b4bf29d26e2542b7af">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/zodb/relstorage/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/zodb/relstorage/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/zodb/relstorage/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/zodb/relstorage/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/zodb/relstorage/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/zodb/relstorage/commit/75eefa8cf4a2b6ab6ab0e4b4bf29d26e2542b7af&quot;&gt;75eefa8cf&lt;/a&gt;&lt;a href=&quot;https://github.com/zodb/relstorage/commit/75eefa8cf4a2b6ab6ab0e4b4bf29d26e2542b7af&quot;&gt;&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/zodb/relstorage/commit/&amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/zodb/relstorage/commit/&amp;amp;lt;a class=&amp;amp;quot;double-link&am... (continued)

Pull Request Pull Request #402: SQLite: Make wal_autocheckpoint more reliable.

Run Details

70 of 70 new or added lines in 9 files covered. (100.0%)

18440 of 19883 relevant lines covered (92.74%)

4.57 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.67

/src/relstorage/adapters/sqlite/tests/test_sqlite.py

# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2019 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################

"""Integration tests for sqlite support."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tempfile
import os.path
import unittest

from relstorage.adapters.sqlite.adapter import Sqlite3Adapter
from relstorage.options import Options

from relstorage.tests import reltestbase
from relstorage.tests.util import AbstractTestSuiteBuilder
from relstorage.tests.util import RUNNING_ON_TRAVIS
from relstorage.tests.util import RUNNING_ON_APPVEYOR
from relstorage._compat import PYPY
from relstorage._compat import PY3

class Sqlite3AdapterMixin(object):

    def __get_db_name(self):
        if self.keep_history:
            db = self.base_dbname
        else:
            db = self.base_dbname + '_hf'
        return db

    def _get_data_dir(self):
        # Our layers tend to change the temporary directory,
        # and then destroy it when the layer is torn down.
        # So our files don't persist. This can show a different set of
        # bugs than re-using an existing schema that gets zapped.
        # return "/tmp"
        return tempfile.gettempdir()

    DEFAULT_EXTRA_PRAGMAS = {}

    def __get_adapter_options(self, dbname=None):
        dbname = dbname or self.__get_db_name()
        assert isinstance(dbname, str), (dbname, type(dbname))
        data_dir = os.path.join(
            self._get_data_dir(),
            dbname
        )

        return {
            'data_dir': data_dir,
            'pragmas': self.DEFAULT_EXTRA_PRAGMAS.copy(),
        }

    def make_adapter(self, options, db=None):

        return self.get_adapter_class()(
            options=options,
            **self.__get_adapter_options(db)
        )

    def get_adapter_class(self):
        return Sqlite3Adapter

    def get_adapter_zconfig(self):
        options = self.__get_adapter_options()
        options['data-dir'] = options['data_dir']
        del options['data_dir']
        del options['pragmas']
        options['driver'] = self.driver_name
        formatted_options = '\n'.join(
            '     %s %s' % (k, v)
            for k, v in options.items()
        )

        return u"""
        <sqlite3>
            %s
            gevent_yield_interval 42
            <pragmas>
               journal_mode memory
               cache_size 8mb
            </pragmas>
        </sqlite3>
        """ % (formatted_options)

    def verify_adapter_from_zconfig(self, adapter):
        self.assertEqual(adapter.connmanager.path,
                         os.path.join(
                             self.__get_adapter_options()['data_dir'], 'main.sqlite3'))
        if 'gevent' in adapter.connmanager.driver.__name__:
            self.assertEqual(adapter.connmanager.driver.yield_to_gevent_instruction_interval,
                             42)
        conn, _ = adapter.connmanager.open()
        try:
            cur = conn.execute('pragma cache_size')
            size, = cur.fetchall()[0]
            self.assertEqual(size, 8388608)
            # But we weren't allowed to change journal_mode
            cur = conn.execute('pragma journal_mode')
            journal, = cur.fetchall()[0]
            self.assertEqual(journal, 'wal')
        finally:
            conn.close()


class TestSqlite3(Sqlite3AdapterMixin, reltestbase.RelStorageTestBase):

    DEFAULT_EXTRA_PRAGMAS = {
        # Checkpoint frequently so that we don't have to iterate too much.
        # The default checkpoint is usually 4MB.
        'wal_autocheckpoint': 2
    }

    def test_wal_does_not_grow(self):
        # issue 401 https://github.com/zodb/relstorage/issues/401
        import transaction
        from ZODB import DB

        def size_wal():
            d = self._storage._adapter.data_dir
            wal = os.path.join(d, 'main.sqlite3-wal')

            #os.system('ls -lh ' + wal)
            size = os.stat(wal).st_size
            return size


        def run_transaction(db):
            tx = transaction.begin()
            conn = db.open()
            try:
                root = conn.root()
                root['key'] = 'abcd' * 1000
                tx.commit()
            except:
                tx.abort()
                raise
            finally:
                conn.close()

        old_explicit = transaction.manager.explicit
        transaction.manager.explicit = True
        try:
            db = self._closing(DB(self._storage))
            # Open an isolated connection. If the transaction
            # manager isn't in explicit mode, it really will talk to
            # the database now, which will cause WAL to start accumulating.
            # Use explicit transaction managers to prevent that from happening
            # and keep a tight reign on transaction duration.
            isolated_txm = transaction.TransactionManager()
            isolated_txm.explicit = True # If this is commented out, the WAL grows without bound
            c1 = db.open(isolated_txm)

            try:
                TX_COUNT = 100
                RUNS = 3
                # Prime it.
                for _ in range(TX_COUNT):
                    run_transaction(db)
                wal_size = size_wal()

                for _ in range(RUNS):
                    for _ in range(TX_COUNT):
                        run_transaction(db)
                    self.assertEqual(wal_size, size_wal())
            finally:
                c1.close()
                db.close()

        finally:
            transaction.manager.explicit = old_explicit

class Sqlite3TestSuiteBuilder(AbstractTestSuiteBuilder):

    __name__ = 'Sqlite3'
    RAISED_EXCEPTIONS = Exception

    def __init__(self):
        from relstorage.adapters.sqlite import drivers
        super(Sqlite3TestSuiteBuilder, self).__init__(
            drivers,
            Sqlite3AdapterMixin,
            extra_test_classes=(TestSqlite3,)
        )

    def _compute_large_blob_size(self, use_small_blobs):
        return Options().blob_chunk_size

    __BASE_SKIPPED_TESTS = (
        # These were both seen on Travis with PyPy3.6 7.1.1, sqlite 3.11.
        # I can't reproduce locally.
        ('checkAutoReconnect', PYPY and PY3 and RUNNING_ON_TRAVIS,
         "Somehow still winds up closed"),
        ('checkAutoReconnectOnSync', PYPY and PY3 and RUNNING_ON_TRAVIS,
         "Somehow still winds up closed"),
    )

    def __add_skips(self, klass, extra_skips=()):
        for mname, skip, message in self.__BASE_SKIPPED_TESTS + extra_skips:
            meth = getattr(klass, mname)
            meth = unittest.skipIf(skip, message)(meth)
            setattr(klass, mname, meth)

    def _make_check_class_HistoryPreservingRelStorageTests(self, bases,
                                                           name, klass_dict=None):
        klass = self._default_make_check_class(bases, name, klass_dict=klass_dict)
        skips = (
            # For some reason this fails to get the undo log. Something
            # to do with the way we manage the connection? Seen on Python 3.7
            # with sqlite 3.28
            ('checkPackUnlinkedFromRoot', RUNNING_ON_APPVEYOR,
             "Fails to get undo log"),
            # Ditto, but on Python 2.7 with sqlite 3.14
            ('checkTransactionalUndoAfterPackWithObjectUnlinkFromRoot', RUNNING_ON_APPVEYOR,
             "Fails to get undo log"),
        )
        self.__add_skips(klass, skips)
        return klass

    def _make_check_class_HistoryFreeRelStorageTests(self, bases,
                                                     name, klass_dict=None):
        klass = self._default_make_check_class(bases, name, klass_dict=klass_dict)
        self.__add_skips(klass)
        return klass

def test_suite():
    return Sqlite3TestSuiteBuilder().test_suite()

1	# -- coding: utf-8 --
2	##############################################################################
3	#
4	# Copyright (c) 2019 Zope Foundation and Contributors.
5	# All Rights Reserved.
6	#
7	# This software is subject to the provisions of the Zope Public License,
8	# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
9	# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
10	# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
11	# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
12	# FOR A PARTICULAR PURPOSE.
13	#
14	##############################################################################
15
16	"""Integration tests for sqlite support."""	5✔
17
18	from __future__ import absolute_import	5✔
19	from __future__ import division	5✔
20	from __future__ import print_function	5✔
21	import tempfile	5✔
22	import os.path	5✔
23	import unittest	5✔
24
25	from relstorage.adapters.sqlite.adapter import Sqlite3Adapter	5✔
26	from relstorage.options import Options	5✔
27
28	from relstorage.tests import reltestbase	5✔
29	from relstorage.tests.util import AbstractTestSuiteBuilder	5✔
30	from relstorage.tests.util import RUNNING_ON_TRAVIS	5✔
31	from relstorage.tests.util import RUNNING_ON_APPVEYOR	5✔
32	from relstorage._compat import PYPY	5✔
33	from relstorage._compat import PY3	5✔
34
35	class Sqlite3AdapterMixin(object):	5✔
36
37	def __get_db_name(self):	5✔
38	if self.keep_history:	4✔
39	db = self.base_dbname	4✔
40	else:
41	db = self.base_dbname + '_hf'	4✔
42	return db	4✔
43
44	def _get_data_dir(self):	5✔
45	# Our layers tend to change the temporary directory,
46	# and then destroy it when the layer is torn down.
47	# So our files don't persist. This can show a different set of
48	# bugs than re-using an existing schema that gets zapped.
49	# return "/tmp"
50	return tempfile.gettempdir()	4✔
51
52	DEFAULT_EXTRA_PRAGMAS = {}	5✔
53
54	def __get_adapter_options(self, dbname=None):	5✔
55	dbname = dbname or self.__get_db_name()	4✔
56	assert isinstance(dbname, str), (dbname, type(dbname))	4✔
57	data_dir = os.path.join(	4✔
58	self._get_data_dir(),
59	dbname
60	)
61
62	return {	4✔
63	'data_dir': data_dir,
64	'pragmas': self.DEFAULT_EXTRA_PRAGMAS.copy(),
65	}
66
67	def make_adapter(self, options, db=None):	5✔
68
69	return self.get_adapter_class()(	4✔
70	options=options,
71	**self.__get_adapter_options(db)
72	)
73
74	def get_adapter_class(self):	5✔
75	return Sqlite3Adapter	4✔
76
77	def get_adapter_zconfig(self):	5✔
78	options = self.__get_adapter_options()	4✔
79	options['data-dir'] = options['data_dir']	4✔
80	del options['data_dir']	4✔
81	del options['pragmas']	4✔
82	options['driver'] = self.driver_name	4✔
83	formatted_options = '\n'.join(	4✔
84	' %s %s' % (k, v)
85	for k, v in options.items()
86	)
87
88	return u"""	4✔
89	<sqlite3>
90	%s
91	gevent_yield_interval 42
92	<pragmas>
93	journal_mode memory
94	cache_size 8mb
95	</pragmas>
96	</sqlite3>
97	""" % (formatted_options)
98
99	def verify_adapter_from_zconfig(self, adapter):	5✔
100	self.assertEqual(adapter.connmanager.path,	4✔
101	os.path.join(
102	self.__get_adapter_options()['data_dir'], 'main.sqlite3'))
103	if 'gevent' in adapter.connmanager.driver.__name__:	4✔
104	self.assertEqual(adapter.connmanager.driver.yield_to_gevent_instruction_interval,	×
105	42)
106	conn, _ = adapter.connmanager.open()	4✔
107	try:	4✔
108	cur = conn.execute('pragma cache_size')	4✔
109	size, = cur.fetchall()[0]	4✔
110	self.assertEqual(size, 8388608)	4✔
111	# But we weren't allowed to change journal_mode
112	cur = conn.execute('pragma journal_mode')	4✔
113	journal, = cur.fetchall()[0]	4✔
114	self.assertEqual(journal, 'wal')	4✔
115	finally:
116	conn.close()	4✔
117
118
119	class TestSqlite3(Sqlite3AdapterMixin, reltestbase.RelStorageTestBase):	5✔
120
121	DEFAULT_EXTRA_PRAGMAS = {	5✔
122	# Checkpoint frequently so that we don't have to iterate too much.
123	# The default checkpoint is usually 4MB.
124	'wal_autocheckpoint': 2
125	}
126
127	def test_wal_does_not_grow(self):	5✔
128	# issue 401 https://github.com/zodb/relstorage/issues/401
129	import transaction	4✔
130	from ZODB import DB	4✔
131
132	def size_wal():	4✔
133	d = self._storage._adapter.data_dir	4✔
134	wal = os.path.join(d, 'main.sqlite3-wal')	4✔
135
136	#os.system('ls -lh ' + wal)
137	size = os.stat(wal).st_size	4✔
138	return size	4✔
139
140
141	def run_transaction(db):	4✔
142	tx = transaction.begin()	4✔
143	conn = db.open()	4✔
144	try:	4✔
145	root = conn.root()	4✔
146	root['key'] = 'abcd' * 1000	4✔
147	tx.commit()	4✔
148	except:	×
149	tx.abort()	×
150	raise	×
151	finally:
152	conn.close()	4✔
153
154	old_explicit = transaction.manager.explicit	4✔
155	transaction.manager.explicit = True	4✔
156	try:	4✔
157	db = self._closing(DB(self._storage))	4✔
158	# Open an isolated connection. If the transaction
159	# manager isn't in explicit mode, it really will talk to
160	# the database now, which will cause WAL to start accumulating.
161	# Use explicit transaction managers to prevent that from happening
162	# and keep a tight reign on transaction duration.
163	isolated_txm = transaction.TransactionManager()	4✔
164	isolated_txm.explicit = True # If this is commented out, the WAL grows without bound	4✔
165	c1 = db.open(isolated_txm)	4✔
166
167	try:	4✔
168	TX_COUNT = 100	4✔
169	RUNS = 3	4✔
170	# Prime it.
171	for _ in range(TX_COUNT):	4✔
172	run_transaction(db)	4✔
173	wal_size = size_wal()	4✔
174
175	for _ in range(RUNS):	4✔
176	for _ in range(TX_COUNT):	4✔
177	run_transaction(db)	4✔
178	self.assertEqual(wal_size, size_wal())	4✔
179	finally:
180	c1.close()	4✔
181	db.close()	4✔
182
183	finally:
184	transaction.manager.explicit = old_explicit	4✔
185
186	class Sqlite3TestSuiteBuilder(AbstractTestSuiteBuilder):	5✔
187
188	__name__ = 'Sqlite3'	5✔
189	RAISED_EXCEPTIONS = Exception	5✔
190
191	def __init__(self):	5✔
192	from relstorage.adapters.sqlite import drivers	5✔
193	super(Sqlite3TestSuiteBuilder, self).__init__(	5✔
194	drivers,
195	Sqlite3AdapterMixin,
196	extra_test_classes=(TestSqlite3,)
197	)
198
199	def _compute_large_blob_size(self, use_small_blobs):	5✔
200	return Options().blob_chunk_size	5✔
201
202	__BASE_SKIPPED_TESTS = (	5✔
203	# These were both seen on Travis with PyPy3.6 7.1.1, sqlite 3.11.
204	# I can't reproduce locally.
205	('checkAutoReconnect', PYPY and PY3 and RUNNING_ON_TRAVIS,
206	"Somehow still winds up closed"),
207	('checkAutoReconnectOnSync', PYPY and PY3 and RUNNING_ON_TRAVIS,
208	"Somehow still winds up closed"),
209	)
210
211	def __add_skips(self, klass, extra_skips=()):	5✔
212	for mname, skip, message in self.__BASE_SKIPPED_TESTS + extra_skips:	4✔
213	meth = getattr(klass, mname)	4✔
214	meth = unittest.skipIf(skip, message)(meth)	4✔
215	setattr(klass, mname, meth)	4✔
216
217	def _make_check_class_HistoryPreservingRelStorageTests(self, bases,	5✔
218	name, klass_dict=None):
219	klass = self._default_make_check_class(bases, name, klass_dict=klass_dict)	4✔
220	skips = (	4✔
221	# For some reason this fails to get the undo log. Something
222	# to do with the way we manage the connection? Seen on Python 3.7
223	# with sqlite 3.28
224	('checkPackUnlinkedFromRoot', RUNNING_ON_APPVEYOR,
225	"Fails to get undo log"),
226	# Ditto, but on Python 2.7 with sqlite 3.14
227	('checkTransactionalUndoAfterPackWithObjectUnlinkFromRoot', RUNNING_ON_APPVEYOR,
228	"Fails to get undo log"),
229	)
230	self.__add_skips(klass, skips)	4✔
231	return klass	4✔
232
233	def _make_check_class_HistoryFreeRelStorageTests(self, bases,	5✔
234	name, klass_dict=None):
235	klass = self._default_make_check_class(bases, name, klass_dict=klass_dict)	4✔
236	self.__add_skips(klass)	4✔
237	return klass	4✔
238
239	def test_suite():	5✔
240	return Sqlite3TestSuiteBuilder().test_suite()	5✔

zodb / relstorage / 1476

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous