zopefoundation / ZODB / 805

pending completion

Build # 805

Build Type

Pull #198

travis-ci

Committed by

web-flow

Commit Message

Fix a spelling error

prefect -> prefetch

Pull Request Pull Request #198: Fix a spelling error

Run Details

6424 of 8536 relevant lines covered (75.26%)

2.99 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.38

/src/ZODB/utils.py

##############################################################################
#
# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE
#
##############################################################################
from __future__ import print_function
import os
import struct
import sys
import time
import threading
from binascii import hexlify, unhexlify
from struct import pack, unpack
from tempfile import mkstemp

from persistent.TimeStamp import TimeStamp

from ZODB._compat import Unpickler
from ZODB._compat import BytesIO
from ZODB._compat import ascii_bytes

from six import PY2

__all__ = ['z64',
           'p64',
           'u64',
           'U64',
           'cp',
           'maxtid',
           'newTid',
           'oid_repr',
           'serial_repr',
           'tid_repr',
           'positive_id',
           'readable_tid_repr',
           'get_pickle_metadata',
           'locked',
          ]


if PY2:
    def as_bytes(obj):
        "Convert obj into bytes"
        return str(obj)

    def as_text(bytes):
        "Convert bytes into string"
        return bytes

    # Convert an element of a bytes object into an int
    byte_ord = ord
    byte_chr = chr

else:
    def as_bytes(obj):
        if isinstance(obj, bytes):
            # invoking str on a bytes object gives its repr()
            return obj
        return str(obj).encode("ascii")

    def as_text(bytes):
        return bytes.decode("ascii")

    def byte_ord(byte):
        return byte # elements of bytes are already ints

    def byte_chr(int):
        return bytes((int,))

z64 = b'\0' * 8

maxtid = b'\x7f\xff\xff\xff\xff\xff\xff\xff'

assert sys.hexversion >= 0x02030000

# The distinction between ints and longs is blurred in Python 2.2,
# so u64() are U64() really the same.

def p64(v):
    """Pack an integer or long into a 8-byte string"""
    return pack(">Q", v)

def u64(v):
    """Unpack an 8-byte string into a 64-bit long integer."""
    return unpack(">Q", v)[0]

U64 = u64


def cp(f1, f2, length=None, bufsize=64 * 1024):
    """Copy all data from one file to another.

    It copies the data from the current position of the input file (f1)
    appending it to the current position of the output file (f2).

    It copies at most 'length' bytes. If 'length' isn't given, it copies
    until the end of the input file.
    """
    read = f1.read
    write = f2.write
    n = bufsize

    if length is None:
        old_pos = f1.tell()
        f1.seek(0,2)
        length = f1.tell()
        f1.seek(old_pos)

    while length > 0:
        if n > length:
            n = length
        data = read(n)
        if not data:
            break
        write(data)
        length -= len(data)

def newTid(old):
    t = time.time()
    ts = TimeStamp(*time.gmtime(t)[:5]+(t%60,))
    if old is not None:
        ts = ts.laterThan(TimeStamp(old))
    return ts.raw()


def oid_repr(oid):
    if isinstance(oid, bytes) and len(oid) == 8:
        # Convert to hex and strip leading zeroes.
        as_hex = hexlify(oid).lstrip(b'0')
        # Ensure two characters per input byte.
        if len(as_hex) & 1:
            as_hex = b'0' + as_hex
        elif as_hex == b'':
            as_hex = b'00'
        return '0x' + as_hex.decode()
    else:
        return repr(oid)

def repr_to_oid(repr):
    repr = ascii_bytes(repr)
    if repr.startswith(b"0x"):
        repr = repr[2:]
    as_bin = unhexlify(repr)
    as_bin = b"\x00"*(8-len(as_bin)) + as_bin
    return as_bin

serial_repr = oid_repr
tid_repr = serial_repr

# For example, produce
#     '0x03441422948b4399 2002-04-14 20:50:34.815000'
# for 8-byte string tid b'\x03D\x14"\x94\x8bC\x99'.
def readable_tid_repr(tid):
    result = tid_repr(tid)
    if isinstance(tid, bytes) and len(tid) == 8:
        result = "%s %s" % (result, TimeStamp(tid))
    return result

# Addresses can "look negative" on some boxes, some of the time.  If you
# feed a "negative address" to an %x format, Python 2.3 displays it as
# unsigned, but produces a FutureWarning, because Python 2.4 will display
# it as signed.  So when you want to prodce an address, use positive_id() to
# obtain it.
# _ADDRESS_MASK is 2**(number_of_bits_in_a_native_pointer).  Adding this to
# a negative address gives a positive int with the same hex representation as
# the significant bits in the original.

_ADDRESS_MASK = 256 ** struct.calcsize('P')
def positive_id(obj):
    """Return id(obj) as a non-negative integer."""

    result = id(obj)
    if result < 0:
        result += _ADDRESS_MASK
        assert result > 0
    return result

# Given a ZODB pickle, return pair of strings (module_name, class_name).
# Do this without importing the module or class object.
# See ZODB/serialize.py's module docstring for the only docs that exist about
# ZODB pickle format.  If the code here gets smarter, please update those
# docs to be at least as smart.  The code here doesn't appear to make sense
# for what serialize.py calls formats 5 and 6.

def get_pickle_metadata(data):
    # Returns a 2-tuple of strings.

    # ZODB's data records contain two pickles.  The first is the class
    # of the object, the second is the object.  We're only trying to
    # pick apart the first here, to extract the module and class names.
    if data[0] in (0x80,    # Py3k indexes bytes -> int
                   b'\x80'  # Python2 indexes bytes -> bytes
                  ): # protocol marker, protocol > 1
        data = data[2:]
    if data.startswith(b'(c'):   # pickle MARK GLOBAL opcode sequence
        global_prefix = 2
    elif data.startswith(b'c'):  # pickle GLOBAL opcode
        global_prefix = 1
    else:
        global_prefix = 0

    if global_prefix:
        # Formats 1 and 2.
        # Don't actually unpickle a class, because it will attempt to
        # load the class.  Just break open the pickle and get the
        # module and class from it.  The module and class names are given by
        # newline-terminated strings following the GLOBAL opcode.
        modname, classname, rest = data.split(b'\n', 2)
        modname = modname[global_prefix:]   # strip GLOBAL opcode
        return modname.decode(), classname.decode()

    # Else there are a bunch of other possible formats.
    f = BytesIO(data)
    u = Unpickler(f)
    try:
        class_info = u.load()
    except Exception as err:
        return '', ''
    if isinstance(class_info, tuple):
        if isinstance(class_info[0], tuple):
            # Formats 3 and 4.
            modname, classname = class_info[0]
        else:
            # Formats 5 and 6 (probably) end up here.
            modname, classname = class_info
    else:
        # This isn't a known format.
        modname = repr(class_info)
        classname = ''
    return modname, classname

def mktemp(dir=None, prefix='tmp'):
    """Create a temp file, known by name, in a semi-secure manner."""
    handle, filename = mkstemp(dir=dir, prefix=prefix)
    os.close(handle)
    return filename

def check_precondition(precondition):
    if not precondition():
        raise AssertionError(
            "Failed precondition: ",
            precondition.__doc__.strip())

class Locked(object):

    def __init__(self, func, inst=None, class_=None, preconditions=()):
        self.__func__ = func
        self.__self__ = inst
        self.__self_class__ = class_
        self.preconditions = preconditions

    def __get__(self, inst, class_):
        return self.__class__(
            self.__func__, inst, class_, self.preconditions)

    def __call__(self, *args, **kw):
        inst = self.__self__
        if inst is None:
            inst = args[0]
        func = self.__func__.__get__(self.__self__, self.__self_class__)

        with inst._lock:
            for precondition in self.preconditions:
                if not precondition(inst):
                    raise AssertionError(
                        "Failed precondition: ",
                        precondition.__doc__.strip())

            return func(*args, **kw)

class locked(object):

    def __init__(self, *preconditions):
        self.preconditions = preconditions

    def __get__(self, inst, class_):
        # We didn't get any preconditions, so we have a single "precondition",
        # which is actually the function to call.
        func, = self.preconditions
        return Locked(func, inst, class_)

    def __call__(self, func):
        return Locked(func, preconditions=self.preconditions)


if os.environ.get('DEBUG_LOCKING'): # pragma: no cover
    # NOTE: This only works on Python 3.
    class Lock(object):

        lock_class = threading.Lock

        def __init__(self):
            self._lock = self.lock_class()

        def pr(self, name, a=None, kw=None):
            f = sys._getframe(2)
            if f.f_code.co_filename.endswith('ZODB/utils.py'):
                f = sys._getframe(3)
            f = '%s:%s' % (f.f_code.co_filename, f.f_lineno)
            print(id(self), self._lock, threading.get_ident(), f, name,
                  a if a else '', kw if kw else '')

        def acquire(self, *a, **kw):
            self.pr('acquire', a, kw)
            return self._lock.acquire(*a, **kw)

        def release(self):
            self.pr('release')
            return self._lock.release()

        def __enter__(self):
            self.pr('acquire')
            return self._lock.acquire()

        def __exit__(self, *ignored):
            self.pr('release')
            return self._lock.release()

    class RLock(Lock):

        lock_class = threading.RLock

    class Condition(Lock):

        lock_class = threading.Condition

        def wait(self, *a, **kw):
            self.pr('wait', a, kw)
            return self._lock.wait(*a, **kw)

        def wait_for(self, *a, **kw):
            self.pr('wait_for', a, kw)
            return self._lock.wait_for(*a, **kw)

        def notify(self, *a, **kw):
            self.pr('notify', a, kw)
            return self._lock.notify(*a, **kw)

        def notify_all(self):
            self.pr('notify_all')
            return self._lock.notify_all()

        notifyAll = notify_all

else:

    from threading import Condition, Lock, RLock


import ZODB.POSException

def load_current(storage, oid, version=''):
    """Load the most recent revision of an object by calling loadBefore

    Starting in ZODB 5, it's no longer necessary for storages to
    provide a load method.

    This function is mainly intended to facilitate transitioning from
    load to loadBefore.  It's mainly useful for tests that are meant
    to test storages, but do so by calling load on the storages.

    This function will likely become unnecessary and be deprecated
    some time in the future.
    """
    assert not version
    r = storage.loadBefore(oid, maxtid)
    if r is None:
        raise ZODB.POSException.POSKeyError(oid)
    assert r[2] is None
    return r[:2]

1	##############################################################################
2	#
3	# Copyright (c) 2001, 2002 Zope Foundation and Contributors.
4	# All Rights Reserved.
5	#
6	# This software is subject to the provisions of the Zope Public License,
7	# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
8	# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
9	# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
10	# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
11	# FOR A PARTICULAR PURPOSE
12	#
13	##############################################################################
14	from __future__ import print_function	4✔
15	import os	4✔
16	import struct	4✔
17	import sys	4✔
18	import time	4✔
19	import threading	4✔
20	from binascii import hexlify, unhexlify	4✔
21	from struct import pack, unpack	4✔
22	from tempfile import mkstemp	4✔
23
24	from persistent.TimeStamp import TimeStamp	4✔
25
26	from ZODB._compat import Unpickler	4✔
27	from ZODB._compat import BytesIO	4✔
28	from ZODB._compat import ascii_bytes	4✔
29
30	from six import PY2	4✔
31
32	__all__ = ['z64',	4✔
33	'p64',
34	'u64',
35	'U64',
36	'cp',
37	'maxtid',
38	'newTid',
39	'oid_repr',
40	'serial_repr',
41	'tid_repr',
42	'positive_id',
43	'readable_tid_repr',
44	'get_pickle_metadata',
45	'locked',
46	]
47
48
49	if PY2:	4✔
50	def as_bytes(obj):	1 only 805.3 ✔
51	"Convert obj into bytes"
52	return str(obj)	1 only 805.3 ✔
53
54	def as_text(bytes):	1 only 805.3 ✔
55	"Convert bytes into string"
56	return bytes	1 only 805.3 ✔
57
58	# Convert an element of a bytes object into an int
59	byte_ord = ord	1 only 805.3 ✔
60	byte_chr = chr	1 only 805.3 ✔
61
62	else:
63	def as_bytes(obj):	3 all except 805.3 ✔
64	if isinstance(obj, bytes):	3 all except 805.3 ✔
65	# invoking str on a bytes object gives its repr()
66	return obj	3 all except 805.3 ✔
67	return str(obj).encode("ascii")	3 all except 805.3 ✔
68
69	def as_text(bytes):	3 all except 805.3 ✔
70	return bytes.decode("ascii")	3 all except 805.3 ✔
71
72	def byte_ord(byte):	3 all except 805.3 ✔
73	return byte # elements of bytes are already ints	3 all except 805.3 ✔
74
75	def byte_chr(int):	3 all except 805.3 ✔
76	return bytes((int,))	3 all except 805.3 ✔
77
78	z64 = b'\0' * 8	4✔
79
80	maxtid = b'\x7f\xff\xff\xff\xff\xff\xff\xff'	4✔
81
82	assert sys.hexversion >= 0x02030000	4✔
83
84	# The distinction between ints and longs is blurred in Python 2.2,
85	# so u64() are U64() really the same.
86
87	def p64(v):	4✔
88	"""Pack an integer or long into a 8-byte string"""
89	return pack(">Q", v)	4✔
90
91	def u64(v):	4✔
92	"""Unpack an 8-byte string into a 64-bit long integer."""
93	return unpack(">Q", v)[0]	4✔
94
95	U64 = u64	4✔
96
97
98	def cp(f1, f2, length=None, bufsize=64 * 1024):	4✔
99	"""Copy all data from one file to another.
100
101	It copies the data from the current position of the input file (f1)
102	appending it to the current position of the output file (f2).
103
104	It copies at most 'length' bytes. If 'length' isn't given, it copies
105	until the end of the input file.
106	"""
107	read = f1.read	4✔
108	write = f2.write	4✔
109	n = bufsize	4✔
110
111	if length is None:	4✔
112	old_pos = f1.tell()	4✔
113	f1.seek(0,2)	4✔
114	length = f1.tell()	4✔
115	f1.seek(old_pos)	4✔
116
117	while length > 0:	4✔
118	if n > length:	4✔
119	n = length	4✔
120	data = read(n)	4✔
121	if not data:	4✔
122	break	×
123	write(data)	4✔
124	length -= len(data)	4✔
125
126	def newTid(old):	4✔
127	t = time.time()	4✔
128	ts = TimeStamp(*time.gmtime(t)[:5]+(t%60,))	4✔
129	if old is not None:	4✔
130	ts = ts.laterThan(TimeStamp(old))	4✔
131	return ts.raw()	4✔
132
133
134	def oid_repr(oid):	4✔
135	if isinstance(oid, bytes) and len(oid) == 8:	4✔
136	# Convert to hex and strip leading zeroes.
137	as_hex = hexlify(oid).lstrip(b'0')	4✔
138	# Ensure two characters per input byte.
139	if len(as_hex) & 1:	4✔
140	as_hex = b'0' + as_hex	4✔
141	elif as_hex == b'':	4✔
142	as_hex = b'00'	4✔
143	return '0x' + as_hex.decode()	4✔
144	else:
145	return repr(oid)	4✔
146
147	def repr_to_oid(repr):	4✔
148	repr = ascii_bytes(repr)	4✔
149	if repr.startswith(b"0x"):	4✔
150	repr = repr[2:]	4✔
151	as_bin = unhexlify(repr)	4✔
152	as_bin = b"\x00"*(8-len(as_bin)) + as_bin	4✔
153	return as_bin	4✔
154
155	serial_repr = oid_repr	4✔
156	tid_repr = serial_repr	4✔
157
158	# For example, produce
159	# '0x03441422948b4399 2002-04-14 20:50:34.815000'
160	# for 8-byte string tid b'\x03D\x14"\x94\x8bC\x99'.
161	def readable_tid_repr(tid):	4✔
162	result = tid_repr(tid)	4✔
163	if isinstance(tid, bytes) and len(tid) == 8:	4✔
164	result = "%s %s" % (result, TimeStamp(tid))	4✔
165	return result	4✔
166
167	# Addresses can "look negative" on some boxes, some of the time. If you
168	# feed a "negative address" to an %x format, Python 2.3 displays it as
169	# unsigned, but produces a FutureWarning, because Python 2.4 will display
170	# it as signed. So when you want to prodce an address, use positive_id() to
171	# obtain it.
172	# _ADDRESS_MASK is 2**(number_of_bits_in_a_native_pointer). Adding this to
173	# a negative address gives a positive int with the same hex representation as
174	# the significant bits in the original.
175
176	_ADDRESS_MASK = 256 ** struct.calcsize('P')	4✔
177	def positive_id(obj):	4✔
178	"""Return id(obj) as a non-negative integer."""
179
180	result = id(obj)	4✔
181	if result < 0:	4✔
182	result += _ADDRESS_MASK	×
183	assert result > 0	×
184	return result	4✔
185
186	# Given a ZODB pickle, return pair of strings (module_name, class_name).
187	# Do this without importing the module or class object.
188	# See ZODB/serialize.py's module docstring for the only docs that exist about
189	# ZODB pickle format. If the code here gets smarter, please update those
190	# docs to be at least as smart. The code here doesn't appear to make sense
191	# for what serialize.py calls formats 5 and 6.
192
193	def get_pickle_metadata(data):	4✔
194	# Returns a 2-tuple of strings.
195
196	# ZODB's data records contain two pickles. The first is the class
197	# of the object, the second is the object. We're only trying to
198	# pick apart the first here, to extract the module and class names.
199	if data[0] in (0x80, # Py3k indexes bytes -> int	4✔
200	b'\x80' # Python2 indexes bytes -> bytes
201	): # protocol marker, protocol > 1
202	data = data[2:]	4✔
203	if data.startswith(b'(c'): # pickle MARK GLOBAL opcode sequence	4✔
204	global_prefix = 2	×
205	elif data.startswith(b'c'): # pickle GLOBAL opcode	4✔
206	global_prefix = 1	4✔
207	else:
208	global_prefix = 0	4✔
209
210	if global_prefix:	4✔
211	# Formats 1 and 2.
212	# Don't actually unpickle a class, because it will attempt to
213	# load the class. Just break open the pickle and get the
214	# module and class from it. The module and class names are given by
215	# newline-terminated strings following the GLOBAL opcode.
216	modname, classname, rest = data.split(b'\n', 2)	4✔
217	modname = modname[global_prefix:] # strip GLOBAL opcode	4✔
218	return modname.decode(), classname.decode()	4✔
219
220	# Else there are a bunch of other possible formats.
221	f = BytesIO(data)	4✔
222	u = Unpickler(f)	4✔
223	try:	4✔
224	class_info = u.load()	4✔
225	except Exception as err:	4✔
226	return '', ''	4✔
227	if isinstance(class_info, tuple):	4✔
228	if isinstance(class_info[0], tuple):	4✔
229	# Formats 3 and 4.
230	modname, classname = class_info[0]	4✔
231	else:
232	# Formats 5 and 6 (probably) end up here.
233	modname, classname = class_info	×
234	else:
235	# This isn't a known format.
236	modname = repr(class_info)	×
237	classname = ''	×
238	return modname, classname	4✔
239
240	def mktemp(dir=None, prefix='tmp'):	4✔
241	"""Create a temp file, known by name, in a semi-secure manner."""
242	handle, filename = mkstemp(dir=dir, prefix=prefix)	4✔
243	os.close(handle)	4✔
244	return filename	4✔
245
246	def check_precondition(precondition):	4✔
247	if not precondition():	4✔
248	raise AssertionError(	×
249	"Failed precondition: ",
250	precondition.__doc__.strip())
251
252	class Locked(object):	4✔
253
254	def __init__(self, func, inst=None, class_=None, preconditions=()):	4✔
255	self.__func__ = func	4✔
256	self.__self__ = inst	4✔
257	self.__self_class__ = class_	4✔
258	self.preconditions = preconditions	4✔
259
260	def __get__(self, inst, class_):	4✔
261	return self.__class__(	4✔
262	self.__func__, inst, class_, self.preconditions)
263
264	def __call__(self, args, *kw):	4✔
265	inst = self.__self__	4✔
266	if inst is None:	4✔
267	inst = args[0]	4✔
268	func = self.__func__.__get__(self.__self__, self.__self_class__)	4✔
269
270	with inst._lock:	4✔
271	for precondition in self.preconditions:	4✔
272	if not precondition(inst):	4✔
273	raise AssertionError(	4✔
274	"Failed precondition: ",
275	precondition.__doc__.strip())
276
277	return func(args, *kw)	4✔
278
279	class locked(object):	4✔
280
281	def __init__(self, *preconditions):	4✔
282	self.preconditions = preconditions	4✔
283
284	def __get__(self, inst, class_):	4✔
285	# We didn't get any preconditions, so we have a single "precondition",
286	# which is actually the function to call.
287	func, = self.preconditions	4✔
288	return Locked(func, inst, class_)	4✔
289
290	def __call__(self, func):	4✔
291	return Locked(func, preconditions=self.preconditions)	4✔
292
293
294	if os.environ.get('DEBUG_LOCKING'): # pragma: no cover
295	# NOTE: This only works on Python 3.
296	class Lock(object):
297
298	lock_class = threading.Lock
299
300	def __init__(self):
301	self._lock = self.lock_class()
302
303	def pr(self, name, a=None, kw=None):
304	f = sys._getframe(2)
305	if f.f_code.co_filename.endswith('ZODB/utils.py'):
306	f = sys._getframe(3)
307	f = '%s:%s' % (f.f_code.co_filename, f.f_lineno)
308	print(id(self), self._lock, threading.get_ident(), f, name,
309	a if a else '', kw if kw else '')
310
311	def acquire(self, a, *kw):
312	self.pr('acquire', a, kw)
313	return self._lock.acquire(a, *kw)
314
315	def release(self):
316	self.pr('release')
317	return self._lock.release()
318
319	def __enter__(self):
320	self.pr('acquire')
321	return self._lock.acquire()
322
323	def __exit__(self, *ignored):
324	self.pr('release')
325	return self._lock.release()
326
327	class RLock(Lock):
328
329	lock_class = threading.RLock
330
331	class Condition(Lock):
332
333	lock_class = threading.Condition
334
335	def wait(self, a, *kw):
336	self.pr('wait', a, kw)
337	return self._lock.wait(a, *kw)
338
339	def wait_for(self, a, *kw):
340	self.pr('wait_for', a, kw)
341	return self._lock.wait_for(a, *kw)
342
343	def notify(self, a, *kw):
344	self.pr('notify', a, kw)
345	return self._lock.notify(a, *kw)
346
347	def notify_all(self):
348	self.pr('notify_all')
349	return self._lock.notify_all()
350
351	notifyAll = notify_all
352
353	else:
354
355	from threading import Condition, Lock, RLock	4✔
356
357
358	import ZODB.POSException	4✔
359
360	def load_current(storage, oid, version=''):	4✔
361	"""Load the most recent revision of an object by calling loadBefore
362
363	Starting in ZODB 5, it's no longer necessary for storages to
364	provide a load method.
365
366	This function is mainly intended to facilitate transitioning from
367	load to loadBefore. It's mainly useful for tests that are meant
368	to test storages, but do so by calling load on the storages.
369
370	This function will likely become unnecessary and be deprecated
371	some time in the future.
372	"""
373	assert not version	4✔
374	r = storage.loadBefore(oid, maxtid)	4✔
375	if r is None:	4✔
376	raise ZODB.POSException.POSKeyError(oid)	×
377	assert r[2] is None	4✔
378	return r[:2]	4✔

zopefoundation / ZODB / 805

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous