• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

zopefoundation / BTrees / 26437399251

26 May 2026 06:59AM UTC coverage: 93.877% (-0.4%) from 94.228%
26437399251

Pull #224

github

icemac
Configuring for c-code
Pull Request #224: Configuring for c-code: Update to latest zope.meta

627 of 659 branches covered (95.14%)

Branch coverage included in aggregate %.

7652 of 8160 relevant lines covered (93.77%)

6.56 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.76
/src/BTrees/_datatypes.py
1
##############################################################################
2
#
3
# Copyright Zope Foundation and Contributors.
4
# All Rights Reserved.
5
#
6
# This software is subject to the provisions of the Zope Public License,
7
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
8
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
9
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
10
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
11
# FOR A PARTICULAR PURPOSE.
12
#
13
##############################################################################
14
"""
15
Descriptions of the datatypes supported by this package.
16
"""
17

18
import abc
7✔
19
import operator
7✔
20
import struct
7✔
21

22
from .utils import Lazy
7✔
23

24

25
class DataType:
7✔
26
    """
27
    Describes a data type used as a value.
28

29
    Subclasses will be defined for each particular
30
    supported type.
31
    """
32

33
    # The name for this datatype as used in interface names.
34
    long_name = None
7✔
35

36
    # The prefix code for this data type. Usually a single letter.
37
    prefix_code = None
7✔
38

39
    # The multiplication identity for this data type. Used in
40
    # combining (merging) data types. Leave undefined if this is
41
    # not a valid operation.
42
    multiplication_identity = None
7✔
43

44
    # Does the data take up 64-bits? Currently only relevant for the
45
    # integer key types.
46
    using64bits = False
7✔
47

48
    def __init__(self):
7✔
49
        if not self.prefix_code:
7✔
50
            self.prefix_code = type(self).__name__
7✔
51

52
    def __call__(self, item):
7✔
53
        """
54
        Verify *item* is in the correct format (or "close" enough)
55
        and return the item or its suitable conversion.
56

57
        If this cannot be done, raise a :exc:`TypeError`.
58

59
        The definition of "close" varies according to the datatypes.
60
        For example, integer datatypes will accept anything that can
61
        be converted into an integer using normal python coercion
62
        rules (calling ``__index__``) and where the integer fits into
63
        the required native type size (e.g., 4 bytes).
64
        """
65
        raise NotImplementedError
66

67
    def coerce(self, item):
7✔
68
        """
69
        Coerce *item* into something that can be used with
70
        ``__call__`` and return it.
71

72
        The coercion rules will vary by datatype. This exists only
73
        for test cases. The default is to perform the same validation
74
        as ``__call__``.
75
        """
76
        return self(item)
7✔
77

78
    def apply_weight(self, item, weight):
7✔
79
        """
80
        Apply a *weight* multiplier to *item*.
81

82
        Used when merging data structures. The *item* will be a
83
        value.
84
        """
85
        return item
×
86

87
    def as_value_type(self):
7✔
88
        # Because ``O'`` is used for both key and value,
89
        # we can override this to get the less restrictive value type.
90
        return self
7✔
91

92
    def supports_value_union(self):
7✔
93
        raise NotImplementedError
94

95
    def getTwoExamples(self):
7✔
96
        """
97
        Provide two distinct (non equal) examples acceptable to `__call__`.
98

99
        This is for testing.
100
        """
101
        return "object1", "object2"
7✔
102

103
    def get_lower_bound(self):
7✔
104
        """
105
        If there is a lower bound (inclusive) on the data type, return
106
        it. Otherwise, return ``None``.
107

108
        For integer types, this will only depend on whether it
109
        supports signed or unsigned values, and the answer will be 0
110
        or a negative number. For object types, ``None`` is always
111
        defined to sort as the lowest bound.
112

113
        This can be relevant for both key and value types.
114
        """
115
        return None
7✔
116

117
    def get_upper_bound(self):
7✔
118
        """
119
        If there is an upper bound (inclusive) on the data type,
120
        return it. Otherwise, return ``None``.
121

122
        Remarks are as for `get_lower_bound`.
123
        """
124
        return None
7✔
125

126
    def add_extra_methods(self, base_name, cls):
7✔
127
        """
128
        Hook method called on the key datatype to add zero or more
129
        desired arbitrary additional, non-standard, methods to the
130
        *cls* being constructed.
131

132
        *base_name* will be a string identifying the particular family
133
        of class being constructed, such as 'Bucket' or 'BTree'.
134
        """
135

136

137
class KeyDataType(DataType):
7✔
138
    """
139
    Describes a data type that has additional restrictions allowing it
140
    to be used as a key.
141
    """
142

143
    # When used as the key, this number determines the
144
    # max_internal_size.
145
    tree_size = 500
7✔
146

147
    default_bucket_size = 120
7✔
148

149
    def __call__(self, item):
7✔
150
        raise NotImplementedError
151

152
    def bucket_size_for_value(self, value_type):
7✔
153
        """
154
        What should the bucket (``max_leaf_size``) be when
155
        this data type is used with the given *value_type*?
156
        """
157
        if isinstance(value_type, Any):
7✔
158
            return self.default_bucket_size // 2
7✔
159
        return self.default_bucket_size
7✔
160

161

162
class Any(DataType):
7✔
163
    """
164
    Arbitrary Python objects.
165
    """
166
    prefix_code = 'O'
7✔
167
    long_name = 'Object'
7✔
168

169
    def __call__(self, item):
7✔
170
        return item
7✔
171

172
    def supports_value_union(self):
7✔
173
        return False
7✔
174

175

176
class _HasDefaultComparison(abc.ABC):
7✔
177
    """
178
    An `abc.ABC <https://docs.python.org/3/library/abc.html>_` for
179
    checking whether an item has default comparison.
180

181
    All we have to do is override ``__subclasshook__`` to implement an
182
    algorithm determining whether a class has default comparison.
183
    Python and the ABC machinery will take care of translating
184
    ``isinstance(thing, _HasDefaultComparison)`` into something like
185
    ``_HasDefaultComparison.__subclasshook__(type(thing))``. The ABC
186
    handles caching the answer (based on exact classes, no MRO), and
187
    getting the type from ``thing`` (including mostly dealing with
188
    old-style) classes on Python 2.
189
    """
190

191
    # Comparisons only use special methods defined on the
192
    # type, not instance variables.
193

194
    # On CPython 3, classes inherit __lt__ with ``__objclass__`` of ``object``.
195
    # On PyPy3, they do.
196
    #
197
    # Test these conditions at runtime and define the method variant
198
    # appropriately.
199
    #
200
    # Remember the method is checking if the object has default comparison
201
    assert '__lt__' not in DataType.__dict__
7✔
202
    if getattr(DataType.__lt__, '__objclass__', None) is object:
7✔
203
        # CPython 3
204
        @classmethod
7✔
205
        def __subclasshook__(cls, C, _NoneType=type(None)):
7✔
206
            if C is _NoneType:
7✔
207
                return False
7✔
208
            defining_class = getattr(C.__lt__, '__objclass__', None)
7✔
209
            if defining_class is None:
7✔
210
                # Implemented in Python
211
                return False
7✔
212
            return C.__lt__.__objclass__ is object
7✔
213
    else:
214
        # PyPy3
215
        @classmethod
×
216
        def __subclasshook__(
×
217
            cls, C, _object_lt=object.__lt__, _NoneType=type(None)
218
        ):
219
            if C is _NoneType:
×
220
                return False
×
221
            return C.__lt__ is _object_lt
×
222

223

224
class O(KeyDataType):  # noqa E742
7✔
225
    """
226
    Arbitrary (sortable) Python objects.
227
    """
228
    long_name = 'Object'
7✔
229
    tree_size = 250
7✔
230
    default_bucket_size = 60
7✔
231

232
    def as_value_type(self):
7✔
233
        return Any()
7✔
234

235
    def supports_value_union(self):
7✔
236
        return False
7✔
237

238
    def __call__(self, item):
7✔
239
        if isinstance(item, _HasDefaultComparison):
7✔
240
            raise TypeError(
7✔
241
                "Object of class {} has default comparison".format(
242
                    type(item).__name__
243
                )
244
            )
245
        return item
7✔
246

247

248
class _AbstractNativeDataType(KeyDataType):
7✔
249
    """
250
    Uses `struct.Struct` to verify that the data can fit into a native
251
    type.
252
    """
253

254
    _struct_format = None
7✔
255
    _as_python_type = NotImplementedError
7✔
256
    _required_python_type = object
7✔
257
    _error_description = None
7✔
258
    _as_packable = operator.index
7✔
259

260
    @Lazy
7✔
261
    def _check_native(self):
7✔
262
        return struct.Struct(self._struct_format).pack
7✔
263

264
    def __call__(self, item):
7✔
265
        try:
7✔
266
            self._check_native(self._as_packable(item))
7✔
267
        except (struct.error, TypeError, ValueError):
7✔
268
            # PyPy can raise ValueError converting a negative number to a
269
            # unsigned value.
270
            if isinstance(item, int):
7✔
271
                raise TypeError("Value out of range", item)
7✔
272
            raise TypeError(self._error_description)
7✔
273

274
        return self._as_python_type(item)
7✔
275

276
    def apply_weight(self, item, weight):
7✔
277
        return item * weight
7✔
278

279
    def supports_value_union(self):
7✔
280
        return True
7✔
281

282

283
class _AbstractIntDataType(_AbstractNativeDataType):
7✔
284
    _as_python_type = int
7✔
285
    _required_python_type = int
7✔
286
    multiplication_identity = 1
7✔
287
    long_name = "Integer"
7✔
288

289
    def getTwoExamples(self):
7✔
290
        return 1, 2
7✔
291

292
    def get_lower_bound(self):
7✔
293
        exp = 64 if self.using64bits else 32
7✔
294
        exp -= 1
7✔
295
        return int(-(2 ** exp))
7✔
296

297
    def get_upper_bound(self):
7✔
298
        exp = 64 if self.using64bits else 32
7✔
299
        exp -= 1
7✔
300
        return int(2 ** exp - 1)
7✔
301

302

303
class _AbstractUIntDataType(_AbstractIntDataType):
7✔
304
    long_name = 'Unsigned'
7✔
305

306
    def get_lower_bound(self):
7✔
307
        return 0
7✔
308

309
    def get_upper_bound(self):
7✔
310
        exp = 64 if self.using64bits else 32
7✔
311
        return int(2 ** exp - 1)
7✔
312

313

314
class I(_AbstractIntDataType):  # noqa E742
7✔
315
    _struct_format = 'i'
7✔
316
    _error_description = "32-bit integer expected"
7✔
317

318

319
class U(_AbstractUIntDataType):
7✔
320
    _struct_format = 'I'
7✔
321
    _error_description = 'non-negative 32-bit integer expected'
7✔
322

323

324
class F(_AbstractNativeDataType):
7✔
325
    _struct_format = 'f'
7✔
326
    _as_python_type = float
7✔
327
    _error_description = 'float expected'
7✔
328
    multiplication_identity = 1.0
7✔
329
    long_name = 'Float'
7✔
330

331
    def _as_packable(self, k):  # identity
7✔
332
        return k
7✔
333

334
    def getTwoExamples(self):
7✔
335
        return 0.5, 1.5
7✔
336

337

338
class L(_AbstractIntDataType):
7✔
339
    _struct_format = 'q'
7✔
340
    _error_description = '64-bit integer expected'
7✔
341
    using64bits = True
7✔
342

343

344
class Q(_AbstractUIntDataType):
7✔
345
    _struct_format = 'Q'
7✔
346
    _error_description = 'non-negative 64-bit integer expected'
7✔
347
    using64bits = True
7✔
348

349

350
class _AbstractBytes(KeyDataType):
7✔
351
    """
352
    An exact-length byte string type.
353

354
    This must be subclassed to provide the actual byte length.
355
    """
356
    tree_size = 500
7✔
357
    default_bucket_size = 500
7✔
358
    _length = None
7✔
359

360
    def __call__(self, item):
7✔
361
        if not isinstance(item, bytes) or len(item) != self._length:
7✔
362
            raise TypeError(
7✔
363
                f"{self._length}-byte array expected, not {item!r}"
364
            )
365
        return item
7✔
366

367
    def supports_value_union(self):
7✔
368
        # We don't implement 'multiunion' for fsBTree.
369
        return False
7✔
370

371

372
class f(_AbstractBytes):
7✔
373
    """
374
    The key type for an ``fs`` tree.
375

376
    This is a two-byte prefix of an overall 8-byte value
377
    like a ZODB object ID or transaction ID.
378
    """
379

380
    # Our keys are treated like integers; the module
381
    # implements IIntegerObjectBTreeModule
382
    long_name = 'Integer'
7✔
383
    prefix_code = 'f'
7✔
384
    _length = 2
7✔
385

386
    # Check it can be converted to a two-byte
387
    # value. Note that even though we allow negative values
388
    # that can break test assumptions: -1 < 0 < 1, but the byte
389
    # values for those are \xff\xff > \x00\x00 < \x00\x01.
390
    _as_2_bytes = struct.Struct('>h').pack
7✔
391

392
    def coerce(self, item):
7✔
393
        try:
7✔
394
            return self(item)
7✔
395
        except TypeError:
7✔
396
            try:
7✔
397
                return self._as_2_bytes(operator.index(item))
7✔
398
            except struct.error as e:
7✔
399
                raise TypeError(e)
7✔
400

401
    @staticmethod
7✔
402
    def _make_Bucket_toString():
7✔
403
        def toString(self):
7✔
404
            return b''.join(self._keys) + b''.join(self._values)
7✔
405
        return toString
7✔
406

407
    @staticmethod
7✔
408
    def _make_Bucket_fromString():
7✔
409
        def fromString(self, v):
7✔
410
            length = len(v)
7✔
411
            if length % 8 != 0:
7✔
412
                raise ValueError()
7✔
413
            count = length // 8
7✔
414
            keys, values = v[:count * 2], v[count * 2:]
7✔
415
            self.clear()
7✔
416
            while keys and values:
7✔
417
                key, keys = keys[:2], keys[2:]
7✔
418
                value, values = values[:6], values[6:]
7✔
419
                self._keys.append(key)
7✔
420
                self._values.append(value)
7✔
421
            return self
7✔
422
        return fromString
7✔
423

424
    def add_extra_methods(self, base_name, cls):
7✔
425
        if base_name == 'Bucket':
7✔
426
            cls.toString = self._make_Bucket_toString()
7✔
427
            cls.fromString = self._make_Bucket_fromString()
7✔
428

429

430
class s(_AbstractBytes):
7✔
431
    """
432
    The value type for an ``fs`` tree.
433

434
    This is a 6-byte suffix of an overall 8-byte value
435
    like a ZODB object ID or transaction ID.
436
    """
437

438
    # Our values are treated like objects; the
439
    # module implements IIntegerObjectBTreeModule
440
    long_name = 'Object'
7✔
441
    prefix_code = 's'
7✔
442
    _length = 6
7✔
443

444
    def get_lower_bound(self):
7✔
445
        # Negative values have the high bit set, which is incompatible
446
        # with our transformation.
447
        return 0
7✔
448

449
    # To coerce an integer, as used in tests, first convert to 8 bytes
450
    # in big-endian order, then ensure the first two
451
    # are 0 and cut them off.
452
    _as_8_bytes = struct.Struct('>q').pack
7✔
453

454
    def coerce(self, item):
7✔
455
        try:
7✔
456
            return self(item)
7✔
457
        except TypeError:
7✔
458
            try:
7✔
459
                as_bytes = self._as_8_bytes(operator.index(item))
7✔
460
            except struct.error as e:
7✔
461
                raise TypeError(e)
7✔
462

463
            if as_bytes[:2] != b'\x00\x00':
7✔
464
                raise TypeError(
7✔
465
                    "Cannot convert {!r} to 6 bytes ({!r})".format(
466
                        item, as_bytes
467
                    )
468
                )
469
            return as_bytes[2:]
7✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc