9258565736

Committed 27 May 2024 05:41PM UTC coverage: 93.83% (-0.05%) from 93.88%

Build # 9258565736

Build Type

Pull #204

github

Committed by

tseaver

Commit Message

fix: 'tox -e lint' passes

Pull Request Pull Request #204: fix: 'tox -e lint' passes

Run Details

1551 of 1653 branches covered (93.83%)

Branch coverage included in aggregate %.

212 of 220 new or added lines in 14 files covered. (96.36%)

2 existing lines in 2 files now uncovered.

7649 of 8152 relevant lines covered (93.83%)

7.49 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

15.23

/src/BTrees/tests/testConflict.py

##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
import unittest

from .common import ConflictTestBase
from .common import _skip_wo_ZODB


class NastyConfictFunctionalTests(ConflictTestBase, unittest.TestCase):
    # FUNCTESTS: Provoke various conflict scenarios using ZODB + transaction

    def _getTargetClass(self):
        from BTrees.OOBTree import OOBTree
        return OOBTree

    def openDB(self):
        # The conflict tests tend to open two or more connections
        # and then try to commit them. A standard FileStorage
        # is not MVCC aware, and so each connection would have the same
        # instance of the storage, leading to the error
        # "Duplicate tpc_begin calls for same transaction" on commit;
        # thus we use a MVCCMappingStorage for these tests, ensuring each
        # connection has its own storage.
        #
        # Unfortunately, it wants to acquire the identically same
        # non-recursive lock in each of its *its* tpc_* methods, which
        # deadlocks.
        #
        # The solution is to give each instance its own lock, and trust in the
        # serialization (ordering) of the datamanager, and the fact that these
        # tests are single-threaded.
        import threading

        from ZODB.DB import DB
        from ZODB.tests.MVCCMappingStorage import MVCCMappingStorage

        class _MVCCMappingStorage(MVCCMappingStorage):
            def new_instance(self):
                inst = MVCCMappingStorage.new_instance(self)
                inst._commit_lock = threading.Lock()
                return inst

        self.storage = _MVCCMappingStorage()
        self.db = DB(self.storage)
        return self.db

    @_skip_wo_ZODB
    def testSimpleConflict(self):
        # Invoke conflict resolution by committing a transaction and
        # catching a conflict in the storage.
        import transaction
        self.openDB()

        r1 = self.db.open().root()
        r1["t"] = t = self._makeOne()
        transaction.commit()

        r2 = self.db.open().root()
        copy = r2["t"]
        list(copy)    # unghostify

        self.assertEqual(t._p_serial, copy._p_serial)

        t.update({1: 2, 2: 3})
        transaction.commit()

        copy.update({3: 4})
        transaction.commit()

    # This tests a problem that cropped up while trying to write
    # testBucketSplitConflict (below):  conflict resolution wasn't
    # working at all in non-trivial cases.  Symptoms varied from
    # strange complaints about pickling (despite that the test isn't
    # doing any *directly*), thru SystemErrors from Python and
    # AssertionErrors inside the BTree code.
    @_skip_wo_ZODB
    def testResolutionBlowsUp(self):
        import transaction
        b = self._makeOne()
        for i in range(0, 200, 4):
            b[i] = i
        # bucket 0 has 15 values: 0, 4 .. 56
        # bucket 1 has 15 values: 60, 64 .. 116
        # bucket 2 has 20 values: 120, 124 .. 196
        state = b.__getstate__()
        # Looks like:  ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
        # If these fail, the *preconditions* for running the test aren't
        # satisfied -- the test itself hasn't been run yet.
        self.assertEqual(len(state), 2)
        self.assertEqual(len(state[0]), 5)
        self.assertEqual(state[0][1], 60)
        self.assertEqual(state[0][3], 120)

        # Invoke conflict resolution by committing a transaction.
        self.openDB()

        r1 = self.db.open().root()
        r1["t"] = b
        transaction.commit()

        r2 = self.db.open().root()
        copy = r2["t"]
        # Make sure all of copy is loaded.
        list(copy.values())

        self.assertEqual(b._p_serial, copy._p_serial)

        b.update({1: 2, 2: 3})
        transaction.commit()

        copy.update({3: 4})
        transaction.commit()  # if this doesn't blow up
        list(copy.values())         # and this doesn't either, then fine

    @_skip_wo_ZODB
    def testBucketSplitConflict(self):
        # Tests that a bucket split is viewed as a conflict.
        # It's (almost necessarily) a white-box test, and sensitive to
        # implementation details.
        import transaction
        from ZODB.POSException import ConflictError
        b = orig = self._makeOne()
        for i in range(0, 200, 4):
            b[i] = i
        # bucket 0 has 15 values: 0, 4 .. 56
        # bucket 1 has 15 values: 60, 64 .. 116
        # bucket 2 has 20 values: 120, 124 .. 196
        state = b.__getstate__()
        # Looks like:  ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
        # If these fail, the *preconditions* for running the test aren't
        # satisfied -- the test itself hasn't been run yet.
        self.assertEqual(len(state), 2)
        self.assertEqual(len(state[0]), 5)
        self.assertEqual(state[0][1], 60)
        self.assertEqual(state[0][3], 120)

        # Invoke conflict resolution by committing a transaction.
        self.openDB()

        tm1 = transaction.TransactionManager()
        r1 = self.db.open(transaction_manager=tm1).root()
        r1["t"] = b
        tm1.commit()

        tm2 = transaction.TransactionManager()
        r2 = self.db.open(transaction_manager=tm2).root()
        copy = r2["t"]
        # Make sure all of copy is loaded.
        list(copy.values())

        self.assertEqual(orig._p_serial, copy._p_serial)

        # In one transaction, add 16 new keys to bucket1, to force a bucket
        # split.
        b = orig
        numtoadd = 16
        candidate = 60
        while numtoadd:
            if candidate not in b:
                b[candidate] = candidate
                numtoadd -= 1
            candidate += 1
        # bucket 0 has 15 values: 0, 4 .. 56
        # bucket 1 has 15 values: 60, 61 .. 74
        # bucket 2 has 16 values: [75, 76 .. 81] + [84, 88 ..116]
        # bucket 3 has 20 values: 120, 124 .. 196
        state = b.__getstate__()
        # Looks like:  ((b0, 60, b1, 75, b2, 120, b3), firstbucket)
        # The next block is still verifying preconditions.
        self.assertEqual(len(state), 2)
        self.assertEqual(len(state[0]), 7)
        self.assertEqual(state[0][1], 60)
        self.assertEqual(state[0][3], 75)
        self.assertEqual(state[0][5], 120)

        tm1.commit()

        # In the other transaction, add 3 values near the tail end of bucket1.
        # This doesn't cause a split.
        b = copy
        for i in range(112, 116):
            b[i] = i
        # bucket 0 has 15 values: 0, 4 .. 56
        # bucket 1 has 18 values: 60, 64 .. 112, 113, 114, 115, 116
        # bucket 2 has 20 values: 120, 124 .. 196
        state = b.__getstate__()
        # Looks like:  ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
        # The next block is still verifying preconditions.
        self.assertEqual(len(state), 2)
        self.assertEqual(len(state[0]), 5)
        self.assertEqual(state[0][1], 60)
        self.assertEqual(state[0][3], 120)

        self.assertRaises(ConflictError, tm2.commit)

    @_skip_wo_ZODB
    def testEmptyBucketConflict(self):
        # Tests that an emptied bucket *created by* conflict resolution is
        # viewed as a conflict:  conflict resolution doesn't have enough
        # info to unlink the empty bucket from the BTree correctly.
        import transaction
        from ZODB.POSException import ConflictError
        b = orig = self._makeOne()
        for i in range(0, 200, 4):
            b[i] = i
        # bucket 0 has 15 values: 0, 4 .. 56
        # bucket 1 has 15 values: 60, 64 .. 116
        # bucket 2 has 20 values: 120, 124 .. 196
        state = b.__getstate__()
        # Looks like:  ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
        # If these fail, the *preconditions* for running the test aren't
        # satisfied -- the test itself hasn't been run yet.
        self.assertEqual(len(state), 2)
        self.assertEqual(len(state[0]), 5)
        self.assertEqual(state[0][1], 60)
        self.assertEqual(state[0][3], 120)

        # Invoke conflict resolution by committing a transaction.
        self.openDB()

        tm1 = transaction.TransactionManager()
        r1 = self.db.open(transaction_manager=tm1).root()
        r1["t"] = b
        tm1.commit()

        tm2 = transaction.TransactionManager()
        r2 = self.db.open(transaction_manager=tm2).root()
        copy = r2["t"]
        # Make sure all of copy is loaded.
        list(copy.values())

        self.assertEqual(orig._p_serial, copy._p_serial)

        # In one transaction, delete half of bucket 1.
        b = orig
        for k in 60, 64, 68, 72, 76, 80, 84, 88:
            del b[k]
        # bucket 0 has 15 values: 0, 4 .. 56
        # bucket 1 has 7 values: 92, 96, 100, 104, 108, 112, 116
        # bucket 2 has 20 values: 120, 124 .. 196
        state = b.__getstate__()
        # Looks like:  ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
        # The next block is still verifying preconditions.
        self.assertEqual(len(state), 2)
        self.assertEqual(len(state[0]), 5)
        self.assertEqual(state[0][1], 92)
        self.assertEqual(state[0][3], 120)

        tm1.commit()

        # In the other transaction, delete the other half of bucket 1.
        b = copy
        for k in 92, 96, 100, 104, 108, 112, 116:
            del b[k]
        # bucket 0 has 15 values: 0, 4 .. 56
        # bucket 1 has 8 values: 60, 64, 68, 72, 76, 80, 84, 88
        # bucket 2 has 20 values: 120, 124 .. 196
        state = b.__getstate__()
        # Looks like:  ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
        # The next block is still verifying preconditions.
        self.assertEqual(len(state), 2)
        self.assertEqual(len(state[0]), 5)
        self.assertEqual(state[0][1], 60)
        self.assertEqual(state[0][3], 120)

        # Conflict resolution empties bucket1 entirely.  This used to
        # create an "insane" BTree (a legit BTree cannot contain an empty
        # bucket -- it contains NULL pointers the BTree code doesn't
        # expect, and segfaults result).
        self.assertRaises(ConflictError, tm2.commit)

    @_skip_wo_ZODB
    def testEmptyBucketNoConflict(self):
        # Tests that a plain empty bucket (on input) is not viewed as a
        # conflict.
        import transaction
        b = orig = self._makeOne()
        for i in range(0, 200, 4):
            b[i] = i
        # bucket 0 has 15 values: 0, 4 .. 56
        # bucket 1 has 15 values: 60, 64 .. 116
        # bucket 2 has 20 values: 120, 124 .. 196
        state = b.__getstate__()
        # Looks like:  ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
        # If these fail, the *preconditions* for running the test aren't
        # satisfied -- the test itself hasn't been run yet.
        self.assertEqual(len(state), 2)
        self.assertEqual(len(state[0]), 5)
        self.assertEqual(state[0][1], 60)
        self.assertEqual(state[0][3], 120)

        # Invoke conflict resolution by committing a transaction.
        self.openDB()

        r1 = self.db.open().root()
        r1["t"] = orig
        transaction.commit()

        r2 = self.db.open().root()
        copy = r2["t"]
        # Make sure all of copy is loaded.
        list(copy.values())

        self.assertEqual(orig._p_serial, copy._p_serial)

        # In one transaction, just add a key.
        b = orig
        b[1] = 1
        # bucket 0 has 16 values: [0, 1] + [4, 8 .. 56]
        # bucket 1 has 15 values: 60, 64 .. 116
        # bucket 2 has 20 values: 120, 124 .. 196
        state = b.__getstate__()
        # Looks like:  ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
        # The next block is still verifying preconditions.
        self.assertEqual(len(state), 2)
        self.assertEqual(len(state[0]), 5)
        self.assertEqual(state[0][1], 60)
        self.assertEqual(state[0][3], 120)

        transaction.commit()

        # In the other transaction, delete bucket 2.
        b = copy
        for k in range(120, 200, 4):
            del b[k]
        # bucket 0 has 15 values: 0, 4 .. 56
        # bucket 1 has 15 values: 60, 64 .. 116
        state = b.__getstate__()
        # Looks like:  ((bucket0, 60, bucket1), firstbucket)
        # The next block is still verifying preconditions.
        self.assertEqual(len(state), 2)
        self.assertEqual(len(state[0]), 3)
        self.assertEqual(state[0][1], 60)

        # This shouldn't create a ConflictError.
        transaction.commit()
        # And the resulting BTree shouldn't have internal damage.
        b._check()

    # The snaky control flow in _bucket__p_resolveConflict ended up trying
    # to decref a NULL pointer if conflict resolution was fed 3 empty
    # buckets.  http://collector.zope.org/Zope/553
    def testThreeEmptyBucketsNoSegfault(self):
        # Note that the conflict is raised by our C extension, rather than
        # indirectly via the storage, and hence is a more specialized type.
        # This test therefore does not require ZODB.
        from BTrees.Interfaces import BTreesConflictError
        t = self._makeOne()
        t[1] = 1
        bucket = t._firstbucket
        del t[1]
        state1 = bucket.__getstate__()
        state2 = bucket.__getstate__()
        state3 = bucket.__getstate__()
        self.assertTrue(state2 is not state1 and
                        state2 is not state3 and
                        state3 is not state1)
        self.assertTrue(state2 == state1 and
                        state3 == state1)
        self.assertRaises(BTreesConflictError, bucket._p_resolveConflict,
                          state1, state2, state3)
        # When an empty BTree resolves conflicts, it computes the
        # bucket state as None, so...
        self.assertRaises(BTreesConflictError, bucket._p_resolveConflict,
                          None, None, None)

    @_skip_wo_ZODB
    def testCantResolveBTreeConflict(self):
        # Test that a conflict involving two different changes to
        # an internal BTree node is unresolvable.  An internal node
        # only changes when there are enough additions or deletions
        # to a child bucket that the bucket is split or removed.
        # It's (almost necessarily) a white-box test, and sensitive to
        # implementation details.
        import transaction
        from ZODB.POSException import ConflictError
        b = orig = self._makeOne()
        for i in range(0, 200, 4):
            b[i] = i
        # bucket 0 has 15 values: 0, 4 .. 56
        # bucket 1 has 15 values: 60, 64 .. 116
        # bucket 2 has 20 values: 120, 124 .. 196
        state = b.__getstate__()
        # Looks like:  ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
        # If these fail, the *preconditions* for running the test aren't
        # satisfied -- the test itself hasn't been run yet.
        self.assertEqual(len(state), 2)
        self.assertEqual(len(state[0]), 5)
        self.assertEqual(state[0][1], 60)
        self.assertEqual(state[0][3], 120)

        # Set up database connections to provoke conflict.
        self.openDB()
        tm1 = transaction.TransactionManager()
        r1 = self.db.open(transaction_manager=tm1).root()
        r1["t"] = orig
        tm1.commit()

        tm2 = transaction.TransactionManager()
        r2 = self.db.open(transaction_manager=tm2).root()
        copy = r2["t"]
        # Make sure all of copy is loaded.
        list(copy.values())

        self.assertEqual(orig._p_serial, copy._p_serial)

        # Now one transaction should add enough keys to cause a split,
        # and another should remove all the keys in one bucket.

        for k in range(200, 300, 4):
            orig[k] = k
        tm1.commit()

        for k in range(0, 60, 4):
            del copy[k]

        self.assertRaises(ConflictError, tm2.commit)

    @_skip_wo_ZODB
    def testConflictWithOneEmptyBucket(self):
        # If one transaction empties a bucket, while another adds an item
        # to the bucket, all the changes "look resolvable":  bucket conflict
        # resolution returns a bucket containing (only) the item added by
        # the latter transaction, but changes from the former transaction
        # removing the bucket are uncontested:  the bucket is removed from
        # the BTree despite that resolution thinks it's non-empty!  This
        # was first reported by Dieter Maurer, to zodb-dev on 22 Mar 2005.
        import transaction
        from ZODB.POSException import ConflictError
        b = orig = self._makeOne()
        for i in range(0, 200, 4):
            b[i] = i
        # bucket 0 has 15 values: 0, 4 .. 56
        # bucket 1 has 15 values: 60, 64 .. 116
        # bucket 2 has 20 values: 120, 124 .. 196
        state = b.__getstate__()
        # Looks like:  ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
        # If these fail, the *preconditions* for running the test aren't
        # satisfied -- the test itself hasn't been run yet.
        self.assertEqual(len(state), 2)
        self.assertEqual(len(state[0]), 5)
        self.assertEqual(state[0][1], 60)
        self.assertEqual(state[0][3], 120)

        # Set up database connections to provoke conflict.
        self.openDB()
        tm1 = transaction.TransactionManager()
        r1 = self.db.open(transaction_manager=tm1).root()
        r1["t"] = orig
        tm1.commit()

        tm2 = transaction.TransactionManager()
        r2 = self.db.open(transaction_manager=tm2).root()
        copy = r2["t"]
        # Make sure all of copy is loaded.
        list(copy.values())

        self.assertEqual(orig._p_serial, copy._p_serial)

        # Now one transaction empties the first bucket, and another adds a
        # key to the first bucket.

        for k in range(0, 60, 4):
            del orig[k]
        tm1.commit()

        copy[1] = 1

        self.assertRaises(ConflictError, tm2.commit)

        # Same thing, except commit the transactions in the opposite order.
        b = self._makeOne()
        for i in range(0, 200, 4):
            b[i] = i

        tm1 = transaction.TransactionManager()
        r1 = self.db.open(transaction_manager=tm1).root()
        r1["t"] = b
        tm1.commit()

        tm2 = transaction.TransactionManager()
        r2 = self.db.open(transaction_manager=tm2).root()
        copy = r2["t"]
        # Make sure all of copy is loaded.
        list(copy.values())

        self.assertEqual(b._p_serial, copy._p_serial)

        # Now one transaction empties the first bucket, and another adds a
        # key to the first bucket.
        b[1] = 1
        tm1.commit()

        for k in range(0, 60, 4):
            del copy[k]

        self.assertRaises(ConflictError, tm2.commit)

    @_skip_wo_ZODB
    def testConflictOfInsertAndDeleteOfFirstBucketItem(self):
        # Recently, BTrees became careful about removing internal keys
        # (keys in internal aka BTree nodes) when they were deleted from
        # buckets. This poses a problem for conflict resolution.

        # We want to guard against a case in which the first key in a
        # bucket is removed in one transaction while a key is added
        # after that key but before the next key in another transaction
        # with the result that the added key is unreachable.

        # original:

        #   Bucket(...), k1, Bucket((k1, v1), (k3, v3), ...)

        # tran1

        #   Bucket(...), k3, Bucket(k3, v3), ...)

        # tran2

        #   Bucket(...), k1, Bucket((k1, v1), (k2, v2), (k3, v3), ...)

        #   where k1 < k2 < k3

        # We don't want:

        #   Bucket(...), k3, Bucket((k2, v2), (k3, v3), ...)

        #   as k2 would be unfindable, so we want a conflict.

        import transaction
        from ZODB.POSException import ConflictError
        mytype = self._getTargetClass()
        db = self.openDB()
        tm1 = transaction.TransactionManager()
        conn1 = db.open(tm1)
        conn1.root.t = t = mytype()
        for i in range(0, 200, 2):
            t[i] = i
        tm1.commit()
        k = t.__getstate__()[0][1]
        assert t.__getstate__()[0][2].keys()[0] == k

        tm2 = transaction.TransactionManager()
        conn2 = db.open(tm2)

        t[k+1] = k+1
        del conn2.root.t[k]
        for i in range(200, 300):
            conn2.root.t[i] = i

        tm1.commit()
        self.assertRaises(ConflictError, tm2.commit)
        tm2.abort()

        k = t.__getstate__()[0][1]
        t[k+1] = k+1
        del conn2.root.t[k]

        tm2.commit()
        self.assertRaises(ConflictError, tm1.commit)
        tm1.abort()

1	##############################################################################
2	#
3	# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
4	# All Rights Reserved.
5	#
6	# This software is subject to the provisions of the Zope Public License,
7	# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
8	# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
9	# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
10	# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
11	# FOR A PARTICULAR PURPOSE
12	#
13	##############################################################################
14	import unittest	8✔
15
16	from .common import ConflictTestBase	8✔
17	from .common import _skip_wo_ZODB	8✔
18
19
20	class NastyConfictFunctionalTests(ConflictTestBase, unittest.TestCase):	8✔
21	# FUNCTESTS: Provoke various conflict scenarios using ZODB + transaction
22
23	def _getTargetClass(self):	8✔
24	from BTrees.OOBTree import OOBTree	8✔
25	return OOBTree	8✔
26
27	def openDB(self):	8✔
28	# The conflict tests tend to open two or more connections
29	# and then try to commit them. A standard FileStorage
30	# is not MVCC aware, and so each connection would have the same
31	# instance of the storage, leading to the error
32	# "Duplicate tpc_begin calls for same transaction" on commit;
33	# thus we use a MVCCMappingStorage for these tests, ensuring each
34	# connection has its own storage.
35	#
36	# Unfortunately, it wants to acquire the identically same
37	# non-recursive lock in each of its its tpc_* methods, which
38	# deadlocks.
39	#
40	# The solution is to give each instance its own lock, and trust in the
41	# serialization (ordering) of the datamanager, and the fact that these
42	# tests are single-threaded.
UNCOV 43	import threading	×
44
NEW 45	from ZODB.DB import DB	×
46	from ZODB.tests.MVCCMappingStorage import MVCCMappingStorage	×
47
48	class _MVCCMappingStorage(MVCCMappingStorage):	×
49	def new_instance(self):	×
50	inst = MVCCMappingStorage.new_instance(self)	×
51	inst._commit_lock = threading.Lock()	×
52	return inst	×
53
54	self.storage = _MVCCMappingStorage()	×
55	self.db = DB(self.storage)	×
56	return self.db	×
57
58	@_skip_wo_ZODB	8✔
59	def testSimpleConflict(self):	7✔
60	# Invoke conflict resolution by committing a transaction and
61	# catching a conflict in the storage.
62	import transaction	×
63	self.openDB()	×
64
65	r1 = self.db.open().root()	×
66	r1["t"] = t = self._makeOne()	×
67	transaction.commit()	×
68
69	r2 = self.db.open().root()	×
70	copy = r2["t"]	×
71	list(copy) # unghostify	×
72
73	self.assertEqual(t._p_serial, copy._p_serial)	×
74
NEW 75	t.update({1: 2, 2: 3})	×
76	transaction.commit()	×
77
NEW 78	copy.update({3: 4})	×
79	transaction.commit()	×
80
81	# This tests a problem that cropped up while trying to write
82	# testBucketSplitConflict (below): conflict resolution wasn't
83	# working at all in non-trivial cases. Symptoms varied from
84	# strange complaints about pickling (despite that the test isn't
85	# doing any directly), thru SystemErrors from Python and
86	# AssertionErrors inside the BTree code.
87	@_skip_wo_ZODB	8✔
88	def testResolutionBlowsUp(self):	7✔
89	import transaction	×
90	b = self._makeOne()	×
91	for i in range(0, 200, 4):	×
92	b[i] = i	×
93	# bucket 0 has 15 values: 0, 4 .. 56
94	# bucket 1 has 15 values: 60, 64 .. 116
95	# bucket 2 has 20 values: 120, 124 .. 196
96	state = b.__getstate__()	×
97	# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
98	# If these fail, the preconditions for running the test aren't
99	# satisfied -- the test itself hasn't been run yet.
100	self.assertEqual(len(state), 2)	×
101	self.assertEqual(len(state[0]), 5)	×
102	self.assertEqual(state[0][1], 60)	×
103	self.assertEqual(state[0][3], 120)	×
104
105	# Invoke conflict resolution by committing a transaction.
106	self.openDB()	×
107
108	r1 = self.db.open().root()	×
109	r1["t"] = b	×
110	transaction.commit()	×
111
112	r2 = self.db.open().root()	×
113	copy = r2["t"]	×
114	# Make sure all of copy is loaded.
115	list(copy.values())	×
116
117	self.assertEqual(b._p_serial, copy._p_serial)	×
118
NEW 119	b.update({1: 2, 2: 3})	×
120	transaction.commit()	×
121
NEW 122	copy.update({3: 4})	×
123	transaction.commit() # if this doesn't blow up	×
124	list(copy.values()) # and this doesn't either, then fine	×
125
126	@_skip_wo_ZODB	8✔
127	def testBucketSplitConflict(self):	7✔
128	# Tests that a bucket split is viewed as a conflict.
129	# It's (almost necessarily) a white-box test, and sensitive to
130	# implementation details.
131	import transaction	×
132	from ZODB.POSException import ConflictError	×
133	b = orig = self._makeOne()	×
134	for i in range(0, 200, 4):	×
135	b[i] = i	×
136	# bucket 0 has 15 values: 0, 4 .. 56
137	# bucket 1 has 15 values: 60, 64 .. 116
138	# bucket 2 has 20 values: 120, 124 .. 196
139	state = b.__getstate__()	×
140	# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
141	# If these fail, the preconditions for running the test aren't
142	# satisfied -- the test itself hasn't been run yet.
143	self.assertEqual(len(state), 2)	×
144	self.assertEqual(len(state[0]), 5)	×
145	self.assertEqual(state[0][1], 60)	×
146	self.assertEqual(state[0][3], 120)	×
147
148	# Invoke conflict resolution by committing a transaction.
149	self.openDB()	×
150
151	tm1 = transaction.TransactionManager()	×
152	r1 = self.db.open(transaction_manager=tm1).root()	×
153	r1["t"] = b	×
154	tm1.commit()	×
155
156	tm2 = transaction.TransactionManager()	×
157	r2 = self.db.open(transaction_manager=tm2).root()	×
158	copy = r2["t"]	×
159	# Make sure all of copy is loaded.
160	list(copy.values())	×
161
162	self.assertEqual(orig._p_serial, copy._p_serial)	×
163
164	# In one transaction, add 16 new keys to bucket1, to force a bucket
165	# split.
166	b = orig	×
167	numtoadd = 16	×
168	candidate = 60	×
169	while numtoadd:	×
170	if candidate not in b:	×
171	b[candidate] = candidate	×
172	numtoadd -= 1	×
173	candidate += 1	×
174	# bucket 0 has 15 values: 0, 4 .. 56
175	# bucket 1 has 15 values: 60, 61 .. 74
176	# bucket 2 has 16 values: [75, 76 .. 81] + [84, 88 ..116]
177	# bucket 3 has 20 values: 120, 124 .. 196
178	state = b.__getstate__()	×
179	# Looks like: ((b0, 60, b1, 75, b2, 120, b3), firstbucket)
180	# The next block is still verifying preconditions.
NEW 181	self.assertEqual(len(state), 2)	×
182	self.assertEqual(len(state[0]), 7)	×
183	self.assertEqual(state[0][1], 60)	×
184	self.assertEqual(state[0][3], 75)	×
185	self.assertEqual(state[0][5], 120)	×
186
187	tm1.commit()	×
188
189	# In the other transaction, add 3 values near the tail end of bucket1.
190	# This doesn't cause a split.
191	b = copy	×
192	for i in range(112, 116):	×
193	b[i] = i	×
194	# bucket 0 has 15 values: 0, 4 .. 56
195	# bucket 1 has 18 values: 60, 64 .. 112, 113, 114, 115, 116
196	# bucket 2 has 20 values: 120, 124 .. 196
197	state = b.__getstate__()	×
198	# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
199	# The next block is still verifying preconditions.
200	self.assertEqual(len(state), 2)	×
201	self.assertEqual(len(state[0]), 5)	×
202	self.assertEqual(state[0][1], 60)	×
203	self.assertEqual(state[0][3], 120)	×
204
205	self.assertRaises(ConflictError, tm2.commit)	×
206
207	@_skip_wo_ZODB	8✔
208	def testEmptyBucketConflict(self):	7✔
209	# Tests that an emptied bucket created by conflict resolution is
210	# viewed as a conflict: conflict resolution doesn't have enough
211	# info to unlink the empty bucket from the BTree correctly.
212	import transaction	×
213	from ZODB.POSException import ConflictError	×
214	b = orig = self._makeOne()	×
215	for i in range(0, 200, 4):	×
216	b[i] = i	×
217	# bucket 0 has 15 values: 0, 4 .. 56
218	# bucket 1 has 15 values: 60, 64 .. 116
219	# bucket 2 has 20 values: 120, 124 .. 196
220	state = b.__getstate__()	×
221	# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
222	# If these fail, the preconditions for running the test aren't
223	# satisfied -- the test itself hasn't been run yet.
224	self.assertEqual(len(state), 2)	×
225	self.assertEqual(len(state[0]), 5)	×
226	self.assertEqual(state[0][1], 60)	×
227	self.assertEqual(state[0][3], 120)	×
228
229	# Invoke conflict resolution by committing a transaction.
230	self.openDB()	×
231
232	tm1 = transaction.TransactionManager()	×
233	r1 = self.db.open(transaction_manager=tm1).root()	×
234	r1["t"] = b	×
235	tm1.commit()	×
236
237	tm2 = transaction.TransactionManager()	×
238	r2 = self.db.open(transaction_manager=tm2).root()	×
239	copy = r2["t"]	×
240	# Make sure all of copy is loaded.
241	list(copy.values())	×
242
243	self.assertEqual(orig._p_serial, copy._p_serial)	×
244
245	# In one transaction, delete half of bucket 1.
246	b = orig	×
247	for k in 60, 64, 68, 72, 76, 80, 84, 88:	×
248	del b[k]	×
249	# bucket 0 has 15 values: 0, 4 .. 56
250	# bucket 1 has 7 values: 92, 96, 100, 104, 108, 112, 116
251	# bucket 2 has 20 values: 120, 124 .. 196
252	state = b.__getstate__()	×
253	# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
254	# The next block is still verifying preconditions.
NEW 255	self.assertEqual(len(state), 2)	×
256	self.assertEqual(len(state[0]), 5)	×
257	self.assertEqual(state[0][1], 92)	×
258	self.assertEqual(state[0][3], 120)	×
259
260	tm1.commit()	×
261
262	# In the other transaction, delete the other half of bucket 1.
263	b = copy	×
264	for k in 92, 96, 100, 104, 108, 112, 116:	×
265	del b[k]	×
266	# bucket 0 has 15 values: 0, 4 .. 56
267	# bucket 1 has 8 values: 60, 64, 68, 72, 76, 80, 84, 88
268	# bucket 2 has 20 values: 120, 124 .. 196
269	state = b.__getstate__()	×
270	# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
271	# The next block is still verifying preconditions.
272	self.assertEqual(len(state), 2)	×
273	self.assertEqual(len(state[0]), 5)	×
274	self.assertEqual(state[0][1], 60)	×
275	self.assertEqual(state[0][3], 120)	×
276
277	# Conflict resolution empties bucket1 entirely. This used to
278	# create an "insane" BTree (a legit BTree cannot contain an empty
279	# bucket -- it contains NULL pointers the BTree code doesn't
280	# expect, and segfaults result).
281	self.assertRaises(ConflictError, tm2.commit)	×
282
283	@_skip_wo_ZODB	8✔
284	def testEmptyBucketNoConflict(self):	7✔
285	# Tests that a plain empty bucket (on input) is not viewed as a
286	# conflict.
287	import transaction	×
288	b = orig = self._makeOne()	×
289	for i in range(0, 200, 4):	×
290	b[i] = i	×
291	# bucket 0 has 15 values: 0, 4 .. 56
292	# bucket 1 has 15 values: 60, 64 .. 116
293	# bucket 2 has 20 values: 120, 124 .. 196
294	state = b.__getstate__()	×
295	# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
296	# If these fail, the preconditions for running the test aren't
297	# satisfied -- the test itself hasn't been run yet.
298	self.assertEqual(len(state), 2)	×
299	self.assertEqual(len(state[0]), 5)	×
300	self.assertEqual(state[0][1], 60)	×
301	self.assertEqual(state[0][3], 120)	×
302
303	# Invoke conflict resolution by committing a transaction.
304	self.openDB()	×
305
306	r1 = self.db.open().root()	×
307	r1["t"] = orig	×
308	transaction.commit()	×
309
310	r2 = self.db.open().root()	×
311	copy = r2["t"]	×
312	# Make sure all of copy is loaded.
313	list(copy.values())	×
314
315	self.assertEqual(orig._p_serial, copy._p_serial)	×
316
317	# In one transaction, just add a key.
318	b = orig	×
319	b[1] = 1	×
320	# bucket 0 has 16 values: [0, 1] + [4, 8 .. 56]
321	# bucket 1 has 15 values: 60, 64 .. 116
322	# bucket 2 has 20 values: 120, 124 .. 196
323	state = b.__getstate__()	×
324	# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
325	# The next block is still verifying preconditions.
326	self.assertEqual(len(state), 2)	×
327	self.assertEqual(len(state[0]), 5)	×
328	self.assertEqual(state[0][1], 60)	×
329	self.assertEqual(state[0][3], 120)	×
330
331	transaction.commit()	×
332
333	# In the other transaction, delete bucket 2.
334	b = copy	×
335	for k in range(120, 200, 4):	×
336	del b[k]	×
337	# bucket 0 has 15 values: 0, 4 .. 56
338	# bucket 1 has 15 values: 60, 64 .. 116
339	state = b.__getstate__()	×
340	# Looks like: ((bucket0, 60, bucket1), firstbucket)
341	# The next block is still verifying preconditions.
342	self.assertEqual(len(state), 2)	×
343	self.assertEqual(len(state[0]), 3)	×
344	self.assertEqual(state[0][1], 60)	×
345
346	# This shouldn't create a ConflictError.
347	transaction.commit()	×
348	# And the resulting BTree shouldn't have internal damage.
349	b._check()	×
350
351	# The snaky control flow in _bucket__p_resolveConflict ended up trying
352	# to decref a NULL pointer if conflict resolution was fed 3 empty
353	# buckets. http://collector.zope.org/Zope/553
354	def testThreeEmptyBucketsNoSegfault(self):	8✔
355	# Note that the conflict is raised by our C extension, rather than
356	# indirectly via the storage, and hence is a more specialized type.
357	# This test therefore does not require ZODB.
358	from BTrees.Interfaces import BTreesConflictError	8✔
359	t = self._makeOne()	8✔
360	t[1] = 1	8✔
361	bucket = t._firstbucket	8✔
362	del t[1]	8✔
363	state1 = bucket.__getstate__()	8✔
364	state2 = bucket.__getstate__()	8✔
365	state3 = bucket.__getstate__()	8✔
366	self.assertTrue(state2 is not state1 and	8✔
367	state2 is not state3 and
368	state3 is not state1)
369	self.assertTrue(state2 == state1 and	8✔
370	state3 == state1)
371	self.assertRaises(BTreesConflictError, bucket._p_resolveConflict,	8✔
372	state1, state2, state3)
373	# When an empty BTree resolves conflicts, it computes the
374	# bucket state as None, so...
375	self.assertRaises(BTreesConflictError, bucket._p_resolveConflict,	8✔
376	None, None, None)
377
378	@_skip_wo_ZODB	8✔
379	def testCantResolveBTreeConflict(self):	7✔
380	# Test that a conflict involving two different changes to
381	# an internal BTree node is unresolvable. An internal node
382	# only changes when there are enough additions or deletions
383	# to a child bucket that the bucket is split or removed.
384	# It's (almost necessarily) a white-box test, and sensitive to
385	# implementation details.
386	import transaction	×
387	from ZODB.POSException import ConflictError	×
388	b = orig = self._makeOne()	×
389	for i in range(0, 200, 4):	×
390	b[i] = i	×
391	# bucket 0 has 15 values: 0, 4 .. 56
392	# bucket 1 has 15 values: 60, 64 .. 116
393	# bucket 2 has 20 values: 120, 124 .. 196
394	state = b.__getstate__()	×
395	# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
396	# If these fail, the preconditions for running the test aren't
397	# satisfied -- the test itself hasn't been run yet.
398	self.assertEqual(len(state), 2)	×
399	self.assertEqual(len(state[0]), 5)	×
400	self.assertEqual(state[0][1], 60)	×
401	self.assertEqual(state[0][3], 120)	×
402
403	# Set up database connections to provoke conflict.
404	self.openDB()	×
405	tm1 = transaction.TransactionManager()	×
406	r1 = self.db.open(transaction_manager=tm1).root()	×
407	r1["t"] = orig	×
408	tm1.commit()	×
409
410	tm2 = transaction.TransactionManager()	×
411	r2 = self.db.open(transaction_manager=tm2).root()	×
412	copy = r2["t"]	×
413	# Make sure all of copy is loaded.
414	list(copy.values())	×
415
416	self.assertEqual(orig._p_serial, copy._p_serial)	×
417
418	# Now one transaction should add enough keys to cause a split,
419	# and another should remove all the keys in one bucket.
420
421	for k in range(200, 300, 4):	×
422	orig[k] = k	×
423	tm1.commit()	×
424
425	for k in range(0, 60, 4):	×
426	del copy[k]	×
427
428	self.assertRaises(ConflictError, tm2.commit)	×
429
430	@_skip_wo_ZODB	8✔
431	def testConflictWithOneEmptyBucket(self):	7✔
432	# If one transaction empties a bucket, while another adds an item
433	# to the bucket, all the changes "look resolvable": bucket conflict
434	# resolution returns a bucket containing (only) the item added by
435	# the latter transaction, but changes from the former transaction
436	# removing the bucket are uncontested: the bucket is removed from
437	# the BTree despite that resolution thinks it's non-empty! This
438	# was first reported by Dieter Maurer, to zodb-dev on 22 Mar 2005.
439	import transaction	×
440	from ZODB.POSException import ConflictError	×
441	b = orig = self._makeOne()	×
442	for i in range(0, 200, 4):	×
443	b[i] = i	×
444	# bucket 0 has 15 values: 0, 4 .. 56
445	# bucket 1 has 15 values: 60, 64 .. 116
446	# bucket 2 has 20 values: 120, 124 .. 196
447	state = b.__getstate__()	×
448	# Looks like: ((bucket0, 60, bucket1, 120, bucket2), firstbucket)
449	# If these fail, the preconditions for running the test aren't
450	# satisfied -- the test itself hasn't been run yet.
451	self.assertEqual(len(state), 2)	×
452	self.assertEqual(len(state[0]), 5)	×
453	self.assertEqual(state[0][1], 60)	×
454	self.assertEqual(state[0][3], 120)	×
455
456	# Set up database connections to provoke conflict.
457	self.openDB()	×
458	tm1 = transaction.TransactionManager()	×
459	r1 = self.db.open(transaction_manager=tm1).root()	×
460	r1["t"] = orig	×
461	tm1.commit()	×
462
463	tm2 = transaction.TransactionManager()	×
464	r2 = self.db.open(transaction_manager=tm2).root()	×
465	copy = r2["t"]	×
466	# Make sure all of copy is loaded.
467	list(copy.values())	×
468
469	self.assertEqual(orig._p_serial, copy._p_serial)	×
470
471	# Now one transaction empties the first bucket, and another adds a
472	# key to the first bucket.
473
474	for k in range(0, 60, 4):	×
475	del orig[k]	×
476	tm1.commit()	×
477
478	copy[1] = 1	×
479
480	self.assertRaises(ConflictError, tm2.commit)	×
481
482	# Same thing, except commit the transactions in the opposite order.
483	b = self._makeOne()	×
484	for i in range(0, 200, 4):	×
485	b[i] = i	×
486
487	tm1 = transaction.TransactionManager()	×
488	r1 = self.db.open(transaction_manager=tm1).root()	×
489	r1["t"] = b	×
490	tm1.commit()	×
491
492	tm2 = transaction.TransactionManager()	×
493	r2 = self.db.open(transaction_manager=tm2).root()	×
494	copy = r2["t"]	×
495	# Make sure all of copy is loaded.
496	list(copy.values())	×
497
498	self.assertEqual(b._p_serial, copy._p_serial)	×
499
500	# Now one transaction empties the first bucket, and another adds a
501	# key to the first bucket.
502	b[1] = 1	×
503	tm1.commit()	×
504
505	for k in range(0, 60, 4):	×
506	del copy[k]	×
507
508	self.assertRaises(ConflictError, tm2.commit)	×
509
510	@_skip_wo_ZODB	8✔
511	def testConflictOfInsertAndDeleteOfFirstBucketItem(self):	7✔
512	# Recently, BTrees became careful about removing internal keys
513	# (keys in internal aka BTree nodes) when they were deleted from
514	# buckets. This poses a problem for conflict resolution.
515
516	# We want to guard against a case in which the first key in a
517	# bucket is removed in one transaction while a key is added
518	# after that key but before the next key in another transaction
519	# with the result that the added key is unreachable.
520
521	# original:
522
523	# Bucket(...), k1, Bucket((k1, v1), (k3, v3), ...)
524
525	# tran1
526
527	# Bucket(...), k3, Bucket(k3, v3), ...)
528
529	# tran2
530
531	# Bucket(...), k1, Bucket((k1, v1), (k2, v2), (k3, v3), ...)
532
533	# where k1 < k2 < k3
534
535	# We don't want:
536
537	# Bucket(...), k3, Bucket((k2, v2), (k3, v3), ...)
538
539	# as k2 would be unfindable, so we want a conflict.
540
541	import transaction	×
542	from ZODB.POSException import ConflictError	×
543	mytype = self._getTargetClass()	×
544	db = self.openDB()	×
545	tm1 = transaction.TransactionManager()	×
546	conn1 = db.open(tm1)	×
547	conn1.root.t = t = mytype()	×
548	for i in range(0, 200, 2):	×
549	t[i] = i	×
550	tm1.commit()	×
551	k = t.__getstate__()[0][1]	×
552	assert t.__getstate__()[0][2].keys()[0] == k	×
553
554	tm2 = transaction.TransactionManager()	×
555	conn2 = db.open(tm2)	×
556
557	t[k+1] = k+1	×
558	del conn2.root.t[k]	×
NEW 559	for i in range(200, 300):	×
560	conn2.root.t[i] = i	×
561
562	tm1.commit()	×
563	self.assertRaises(ConflictError, tm2.commit)	×
564	tm2.abort()	×
565
566	k = t.__getstate__()[0][1]	×
567	t[k+1] = k+1	×
568	del conn2.root.t[k]	×
569
570	tm2.commit()	×
571	self.assertRaises(ConflictError, tm1.commit)	×
572	tm1.abort()	×

zopefoundation / BTrees / 9258565736

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous