1018

pending completion

Build # 1018

Build Type

Pull #307

travis-ci

Committed by

web-flow

Commit Message

mvccadapter: check if the last TID changed without invalidation

Since commit b5895a5c2 ("mvccadapter:
fix race with invalidations when starting a new transaction"),
a ZEO test fails as follows:

    File "src/ZEO/tests/drop_cache_rather_than_verify.txt", line 114, in drop_cache_rather_than_verify.txt
    Failed example:
        conn.root()[1].x
    Expected:
        6
    Got:
        1

Earlier in the test, the ZEO server is restarted and then another
client commits. When disconnected, the first client does not receive
invalidations anymore and the connection gets stuck in the past until
there's a new commit after it reconnected. It was possible to make the
test pass with the following patch:

--- a/src/ZEO/ClientStorage.py
+++ b/src/ZEO/ClientStorage.py
@@ -357,6 +357,7 @@ def notify_connected(self, conn, info):

         # invalidate our db cache
         if self._db is not None:
+            self._db.invalidate(self.lastTransaction(), ())
             self._db.invalidateCache()

         logger.info("%s %s to storage: %s",

Other implementations like NEO are probably affected the same way.

Rather than changing interfaces in a backward-incompatible way,
this commit revert to the original behaviour, and all the changes
that were done in existing tests are reverted.

Pull Request Pull Request #307: mvccadapter: check if the last TID changed without invalidation

Run Details

3 of 3 new or added lines in 1 file covered. (100.0%)

6387 of 8572 relevant lines covered (74.51%)

0.75 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

76.79

/src/ZODB/POSException.py

##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
"""ZODB-defined exceptions

$Id$"""

from ZODB.utils import oid_repr, readable_tid_repr

# BBB: We moved the two transactions to the transaction package
from transaction.interfaces import TransactionError, TransactionFailedError

import transaction.interfaces

def _fmt_undo(oid, reason):
    s = reason and (": %s" % reason) or ""
    return "Undo error %s%s" % (oid_repr(oid), s)

def _recon(class_, state):
    err = class_.__new__(class_)
    err.__setstate__(state)
    return err
_recon.__no_side_effects__ = True

class POSError(Exception):
    """Persistent object system error."""

    def __reduce__(self):
        # Copy extra data from internal structures
        state = self.__dict__.copy()
        state['args'] = self.args

        return (_recon, (self.__class__, state))

    def __setstate__(self, state):
        # PyPy doesn't store the 'args' attribute in an instance's
        # __dict__; instead, it uses what amounts to a slot. Because
        # we customize the pickled representation to just be a dictionary,
        # the args would then get lost, leading to unprintable exceptions
        # and worse. Manually assign to args from the state to be sure
        # this doesn't happen.
        super(POSError,self).__setstate__(state)
        self.args = state['args']

class POSKeyError(POSError, KeyError):
    """Key not found in database."""

    def __str__(self):
        return oid_repr(self.args[0])


class ConflictError(POSError, transaction.interfaces.TransientError):
    """Two transactions tried to modify the same object at once.

    This transaction should be resubmitted.

    Instance attributes:
      oid : string
        the OID (8-byte packed string) of the object in conflict
      class_name : string
        the fully-qualified name of that object's class
      message : string
        a human-readable explanation of the error
      serials : (string, string)
        a pair of 8-byte packed strings; these are the serial numbers
        related to conflict.  The first is the revision of object that
        is in conflict, the currently committed serial.  The second is
        the revision the current transaction read when it started.
      data : string
        The database record that failed to commit, used to put the
        class name in the error message.

    The caller should pass either object or oid as a keyword argument,
    but not both of them.  If object is passed, it should be a
    persistent object with an _p_oid attribute.
    """

    def __init__(self, message=None, object=None, oid=None, serials=None,
                 data=None):
        if message is None:
            self.message = "database conflict error"
        else:
            self.message = message

        if object is None:
            self.oid = None
            self.class_name = None
        else:
            self.oid = object._p_oid
            klass = object.__class__
            self.class_name = klass.__module__ + "." + klass.__name__

        if oid is not None:
            assert self.oid is None
            self.oid = oid

        if data is not None:
            # avoid circular import chain
            from ZODB.utils import get_pickle_metadata
            self.class_name = '.'.join(get_pickle_metadata(data))

        self.serials = serials

    def __str__(self):
        extras = []
        if self.oid:
            extras.append("oid %s" % oid_repr(self.oid))
        if self.class_name:
            extras.append("class %s" % self.class_name)
        if self.serials:
            current, old = self.serials
            extras.append("serial this txn started with %s" %
                          readable_tid_repr(old))
            extras.append("serial currently committed %s" %
                          readable_tid_repr(current))
        if extras:
            return "%s (%s)" % (self.message, ", ".join(extras))
        else:
            return self.message

    def get_oid(self):
        return self.oid

    def get_class_name(self):
        return self.class_name

    def get_old_serial(self):
        return self.serials[1]

    def get_new_serial(self):
        return self.serials[0]

    def get_serials(self):
        return self.serials

class ReadConflictError(ConflictError):
    """Conflict detected when object was loaded.

    An attempt was made to read an object that has changed in another
    transaction (eg. another thread or process).
    """
    def __init__(self, message=None, object=None, serials=None, **kw):
        if message is None:
            message = "database read conflict error"
        ConflictError.__init__(self, message=message, object=object,
                               serials=serials, **kw)

class BTreesConflictError(ConflictError):
    """A special subclass for BTrees conflict errors."""

    msgs = [# 0; i2 or i3 bucket split; positions are all -1
            'Conflicting bucket split',

            # 1; keys the same, but i2 and i3 values differ, and both values
            # differ from i1's value
            'Conflicting changes',

            # 2; i1's value changed in i2, but key+value deleted in i3
            'Conflicting delete and change',

            # 3; i1's value changed in i3, but key+value deleted in i2
            'Conflicting delete and change',

            # 4; i1 and i2 both added the same key, or both deleted the
            # same key
            'Conflicting inserts or deletes',

            # 5;  i2 and i3 both deleted the same key
            'Conflicting deletes',

            # 6; i2 and i3 both added the same key
            'Conflicting inserts',

            # 7; i2 and i3 both deleted the same key, or i2 changed the value
            # associated with a key and i3 deleted that key
            'Conflicting deletes, or delete and change',

            # 8; i2 and i3 both deleted the same key, or i3 changed the value
            # associated with a key and i2 deleted that key
            'Conflicting deletes, or delete and change',

            # 9; i2 and i3 both deleted the same key
            'Conflicting deletes',

            # 10; i2 and i3 deleted all the keys, and didn't insert any,
            # leaving an empty bucket; conflict resolution doesn't have
            # enough info to unlink an empty bucket from its containing
            # BTree correctly
            'Empty bucket from deleting all keys',

            # 11; conflicting changes in an internal BTree node
            'Conflicting changes in an internal BTree node',

            # 12; i2 or i3 was empty
            'Empty bucket in a transaction',

            # 13; delete of first key, which causes change to parent node
            'Delete of first key',
            ]

    def __init__(self, p1, p2, p3, reason):
        self.p1 = p1
        self.p2 = p2
        self.p3 = p3
        self.reason = reason

    def __repr__(self):
        return "BTreesConflictError(%d, %d, %d, %d)" % (self.p1,
                                                        self.p2,
                                                        self.p3,
                                                        self.reason)
    def __str__(self):
        return "BTrees conflict error at %d/%d/%d: %s" % (
            self.p1, self.p2, self.p3, self.msgs[self.reason])

class DanglingReferenceError(POSError, transaction.interfaces.TransactionError):
    """An object has a persistent reference to a missing object.

    If an object is stored and it has a reference to another object
    that does not exist (for example, it was deleted by pack), this
    exception may be raised.  Whether a storage supports this feature,
    it a quality of implementation issue.

    Instance attributes:
    referer: oid of the object being written
    missing: referenced oid that does not have a corresponding object
    """

    def __init__(self, Aoid, Boid):
        self.referer = Aoid
        self.missing = Boid

    def __str__(self):
        return "from %s to %s" % (oid_repr(self.referer),
                                  oid_repr(self.missing))


############################################################################
# Only used in storages; versions are no longer supported.

class VersionError(POSError):
    """An error in handling versions occurred."""

class VersionCommitError(VersionError):
    """An invalid combination of versions was used in a version commit."""

class VersionLockError(VersionError, transaction.interfaces.TransactionError):
    """Modification to an object modified in an unsaved version.

    An attempt was made to modify an object that has been modified in an
    unsaved version.
    """
############################################################################

class UndoError(POSError):
    """An attempt was made to undo a non-undoable transaction."""

    def __init__(self, reason, oid=None):
        self._reason = reason
        self._oid = oid

    def __str__(self):
        return _fmt_undo(self._oid, self._reason)

class MultipleUndoErrors(UndoError):
    """Several undo errors occurred during a single transaction."""

    def __init__(self, errs):
        # provide a reason and oid for clients that only look at that
        UndoError.__init__(self, *errs[0])
        self._errs = errs

    def __str__(self):
        return "\n".join([_fmt_undo(*pair) for pair in self._errs])

class StorageError(POSError):
    """Base class for storage based exceptions."""

class StorageTransactionError(StorageError):
    """An operation was invoked for an invalid transaction or state."""

class StorageSystemError(StorageError):
    """Panic! Internal storage error!"""

class MountedStorageError(StorageError):
    """Unable to access mounted storage."""

class ReadOnlyError(StorageError):
    """Unable to modify objects in a read-only storage."""

class TransactionTooLargeError(StorageTransactionError):
    """The transaction exhausted some finite storage resource."""

class ExportError(POSError):
    """An export file doesn't have the right format."""

class Unsupported(POSError):
    """A feature was used that is not supported by the storage."""

class ReadOnlyHistoryError(POSError):
    """Unable to add or modify objects in an historical connection."""

class InvalidObjectReference(POSError):
    """An object contains an invalid reference to another object.

    An invalid reference may be one of:

    o A reference to a wrapped persistent object.

    o A reference to an object in a different database connection.

    TODO:  The exception ought to have a member that is the invalid object.
    """

class ConnectionStateError(POSError):
    """A Connection isn't in the required state for an operation.

    o An operation such as a load is attempted on a closed connection.

    o An attempt to close a connection is made while the connection is
      still joined to a transaction (for example, a transaction is in
      progress, with uncommitted modifications in the connection).
    """

1	##############################################################################
2	#
3	# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
4	# All Rights Reserved.
5	#
6	# This software is subject to the provisions of the Zope Public License,
7	# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
8	# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
9	# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
10	# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
11	# FOR A PARTICULAR PURPOSE
12	#
13	##############################################################################
14	"""ZODB-defined exceptions	1✔
15
16	$Id$"""
17
18	from ZODB.utils import oid_repr, readable_tid_repr	1✔
19
20	# BBB: We moved the two transactions to the transaction package
21	from transaction.interfaces import TransactionError, TransactionFailedError	1✔
22
23	import transaction.interfaces	1✔
24
25	def _fmt_undo(oid, reason):	1✔
26	s = reason and (": %s" % reason) or ""	×
27	return "Undo error %s%s" % (oid_repr(oid), s)	×
28
29	def _recon(class_, state):	1✔
30	err = class_.__new__(class_)	×
31	err.__setstate__(state)	×
32	return err	×
33	_recon.__no_side_effects__ = True	1✔
34
35	class POSError(Exception):	1✔
36	"""Persistent object system error."""
37
38	def __reduce__(self):	1✔
39	# Copy extra data from internal structures
40	state = self.__dict__.copy()	×
41	state['args'] = self.args	×
42
43	return (_recon, (self.__class__, state))	×
44
45	def __setstate__(self, state):	1✔
46	# PyPy doesn't store the 'args' attribute in an instance's
47	# __dict__; instead, it uses what amounts to a slot. Because
48	# we customize the pickled representation to just be a dictionary,
49	# the args would then get lost, leading to unprintable exceptions
50	# and worse. Manually assign to args from the state to be sure
51	# this doesn't happen.
52	super(POSError,self).__setstate__(state)	×
53	self.args = state['args']	×
54
55	class POSKeyError(POSError, KeyError):	1✔
56	"""Key not found in database."""
57
58	def __str__(self):	1✔
59	return oid_repr(self.args[0])	1✔
60
61
62	class ConflictError(POSError, transaction.interfaces.TransientError):	1✔
63	"""Two transactions tried to modify the same object at once.
64
65	This transaction should be resubmitted.
66
67	Instance attributes:
68	oid : string
69	the OID (8-byte packed string) of the object in conflict
70	class_name : string
71	the fully-qualified name of that object's class
72	message : string
73	a human-readable explanation of the error
74	serials : (string, string)
75	a pair of 8-byte packed strings; these are the serial numbers
76	related to conflict. The first is the revision of object that
77	is in conflict, the currently committed serial. The second is
78	the revision the current transaction read when it started.
79	data : string
80	The database record that failed to commit, used to put the
81	class name in the error message.
82
83	The caller should pass either object or oid as a keyword argument,
84	but not both of them. If object is passed, it should be a
85	persistent object with an _p_oid attribute.
86	"""
87
88	def __init__(self, message=None, object=None, oid=None, serials=None,	1✔
89	data=None):
90	if message is None:	1✔
91	self.message = "database conflict error"	1✔
92	else:
93	self.message = message	1✔
94
95	if object is None:	1✔
96	self.oid = None	1✔
97	self.class_name = None	1✔
98	else:
99	self.oid = object._p_oid	1✔
100	klass = object.__class__	1✔
101	self.class_name = klass.__module__ + "." + klass.__name__	1✔
102
103	if oid is not None:	1✔
104	assert self.oid is None	1✔
105	self.oid = oid	1✔
106
107	if data is not None:	1✔
108	# avoid circular import chain
109	from ZODB.utils import get_pickle_metadata	1✔
110	self.class_name = '.'.join(get_pickle_metadata(data))	1✔
111
112	self.serials = serials	1✔
113
114	def __str__(self):	1✔
115	extras = []	1✔
116	if self.oid:	1✔
117	extras.append("oid %s" % oid_repr(self.oid))	1✔
118	if self.class_name:	1✔
119	extras.append("class %s" % self.class_name)	1✔
120	if self.serials:	1✔
121	current, old = self.serials	1✔
122	extras.append("serial this txn started with %s" %	1✔
123	readable_tid_repr(old))
124	extras.append("serial currently committed %s" %	1✔
125	readable_tid_repr(current))
126	if extras:	1✔
127	return "%s (%s)" % (self.message, ", ".join(extras))	1✔
128	else:
129	return self.message	1✔
130
131	def get_oid(self):	1✔
132	return self.oid	×
133
134	def get_class_name(self):	1✔
135	return self.class_name	×
136
137	def get_old_serial(self):	1✔
138	return self.serials[1]	×
139
140	def get_new_serial(self):	1✔
141	return self.serials[0]	×
142
143	def get_serials(self):	1✔
144	return self.serials	×
145
146	class ReadConflictError(ConflictError):	1✔
147	"""Conflict detected when object was loaded.
148
149	An attempt was made to read an object that has changed in another
150	transaction (eg. another thread or process).
151	"""
152	def __init__(self, message=None, object=None, serials=None, **kw):	1✔
153	if message is None:	1✔
154	message = "database read conflict error"	1✔
155	ConflictError.__init__(self, message=message, object=object,	1✔
156	serials=serials, **kw)
157
158	class BTreesConflictError(ConflictError):	1✔
159	"""A special subclass for BTrees conflict errors."""
160
161	msgs = [# 0; i2 or i3 bucket split; positions are all -1	1✔
162	'Conflicting bucket split',
163
164	# 1; keys the same, but i2 and i3 values differ, and both values
165	# differ from i1's value
166	'Conflicting changes',
167
168	# 2; i1's value changed in i2, but key+value deleted in i3
169	'Conflicting delete and change',
170
171	# 3; i1's value changed in i3, but key+value deleted in i2
172	'Conflicting delete and change',
173
174	# 4; i1 and i2 both added the same key, or both deleted the
175	# same key
176	'Conflicting inserts or deletes',
177
178	# 5; i2 and i3 both deleted the same key
179	'Conflicting deletes',
180
181	# 6; i2 and i3 both added the same key
182	'Conflicting inserts',
183
184	# 7; i2 and i3 both deleted the same key, or i2 changed the value
185	# associated with a key and i3 deleted that key
186	'Conflicting deletes, or delete and change',
187
188	# 8; i2 and i3 both deleted the same key, or i3 changed the value
189	# associated with a key and i2 deleted that key
190	'Conflicting deletes, or delete and change',
191
192	# 9; i2 and i3 both deleted the same key
193	'Conflicting deletes',
194
195	# 10; i2 and i3 deleted all the keys, and didn't insert any,
196	# leaving an empty bucket; conflict resolution doesn't have
197	# enough info to unlink an empty bucket from its containing
198	# BTree correctly
199	'Empty bucket from deleting all keys',
200
201	# 11; conflicting changes in an internal BTree node
202	'Conflicting changes in an internal BTree node',
203
204	# 12; i2 or i3 was empty
205	'Empty bucket in a transaction',
206
207	# 13; delete of first key, which causes change to parent node
208	'Delete of first key',
209	]
210
211	def __init__(self, p1, p2, p3, reason):	1✔
212	self.p1 = p1	×
213	self.p2 = p2	×
214	self.p3 = p3	×
215	self.reason = reason	×
216
217	def __repr__(self):	1✔
218	return "BTreesConflictError(%d, %d, %d, %d)" % (self.p1,	×
219	self.p2,
220	self.p3,
221	self.reason)
222	def __str__(self):	1✔
223	return "BTrees conflict error at %d/%d/%d: %s" % (	×
224	self.p1, self.p2, self.p3, self.msgs[self.reason])
225
226	class DanglingReferenceError(POSError, transaction.interfaces.TransactionError):	1✔
227	"""An object has a persistent reference to a missing object.
228
229	If an object is stored and it has a reference to another object
230	that does not exist (for example, it was deleted by pack), this
231	exception may be raised. Whether a storage supports this feature,
232	it a quality of implementation issue.
233
234	Instance attributes:
235	referer: oid of the object being written
236	missing: referenced oid that does not have a corresponding object
237	"""
238
239	def __init__(self, Aoid, Boid):	1✔
240	self.referer = Aoid	×
241	self.missing = Boid	×
242
243	def __str__(self):	1✔
244	return "from %s to %s" % (oid_repr(self.referer),	×
245	oid_repr(self.missing))
246
247
248	############################################################################
249	# Only used in storages; versions are no longer supported.
250
251	class VersionError(POSError):	1✔
252	"""An error in handling versions occurred."""
253
254	class VersionCommitError(VersionError):	1✔
255	"""An invalid combination of versions was used in a version commit."""
256
257	class VersionLockError(VersionError, transaction.interfaces.TransactionError):	1✔
258	"""Modification to an object modified in an unsaved version.
259
260	An attempt was made to modify an object that has been modified in an
261	unsaved version.
262	"""
263	############################################################################
264
265	class UndoError(POSError):	1✔
266	"""An attempt was made to undo a non-undoable transaction."""
267
268	def __init__(self, reason, oid=None):	1✔
269	self._reason = reason	1✔
270	self._oid = oid	1✔
271
272	def __str__(self):	1✔
273	return _fmt_undo(self._oid, self._reason)	×
274
275	class MultipleUndoErrors(UndoError):	1✔
276	"""Several undo errors occurred during a single transaction."""
277
278	def __init__(self, errs):	1✔
279	# provide a reason and oid for clients that only look at that
280	UndoError.__init__(self, *errs[0])	1✔
281	self._errs = errs	1✔
282
283	def __str__(self):	1✔
284	return "\n".join([_fmt_undo(*pair) for pair in self._errs])	×
285
286	class StorageError(POSError):	1✔
287	"""Base class for storage based exceptions."""
288
289	class StorageTransactionError(StorageError):	1✔
290	"""An operation was invoked for an invalid transaction or state."""
291
292	class StorageSystemError(StorageError):	1✔
293	"""Panic! Internal storage error!"""
294
295	class MountedStorageError(StorageError):	1✔
296	"""Unable to access mounted storage."""
297
298	class ReadOnlyError(StorageError):	1✔
299	"""Unable to modify objects in a read-only storage."""
300
301	class TransactionTooLargeError(StorageTransactionError):	1✔
302	"""The transaction exhausted some finite storage resource."""
303
304	class ExportError(POSError):	1✔
305	"""An export file doesn't have the right format."""
306
307	class Unsupported(POSError):	1✔
308	"""A feature was used that is not supported by the storage."""
309
310	class ReadOnlyHistoryError(POSError):	1✔
311	"""Unable to add or modify objects in an historical connection."""
312
313	class InvalidObjectReference(POSError):	1✔
314	"""An object contains an invalid reference to another object.
315
316	An invalid reference may be one of:
317
318	o A reference to a wrapped persistent object.
319
320	o A reference to an object in a different database connection.
321
322	TODO: The exception ought to have a member that is the invalid object.
323	"""
324
325	class ConnectionStateError(POSError):	1✔
326	"""A Connection isn't in the required state for an operation.
327
328	o An operation such as a load is attempted on a closed connection.
329
330	o An attempt to close a connection is made while the connection is
331	still joined to a transaction (for example, a transaction is in
332	progress, with uncommitted modifications in the connection).
333	"""

zopefoundation / ZODB / 1018

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous