2062341675

Build # 2062341675

Build Type

github

Committed by GitHub

Commit Message

Merge 9e4038166 into dad95fe4b

Pull Request Pull Request #177: WIP: begin adding user-defined types

Run Details

4436 of 4479 branches covered (99.04%)

Branch coverage included in aggregate %.

676 of 735 new or added lines in 14 files covered. (91.97%)

31 existing lines in 1 file now uncovered.

14203 of 14270 relevant lines covered (99.53%)

14.25 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.24

/grblas/dtypes.py

import numba
import numpy as np
from numpy import find_common_type, promote_types

from . import ffi, lib

# Default assumption unless FC32/FC64 are found in lib
_supports_complex = hasattr(lib, "GrB_FC64") or hasattr(lib, "GxB_FC64")


class DataType:
    __slots__ = "name", "gb_obj", "gb_name", "c_type", "numba_type", "np_type", "__weakref__"

    def __init__(self, name, gb_obj, gb_name, c_type, numba_type, np_type):
        self.name = name
        self.gb_obj = gb_obj
        self.gb_name = gb_name
        self.c_type = c_type
        self.numba_type = numba_type
        self.np_type = np_type

    def __repr__(self):
        return self.name

    def __eq__(self, other):
        if type(other) is DataType:
            return self is other
        # Attempt to use `other` as a lookup key
        try:
            other = lookup_dtype(other)
            return self is other
        except ValueError:
            raise TypeError(f"Invalid or unknown datatype: {other}") from None

    def __hash__(self):
        return hash(self.gb_obj)

    def __lt__(self, other):
        # Let us sort for prettier error reporting
        if type(other) is DataType:
            return str(self.np_type) < str(other.np_type)
        # Attempt to use `other` as a lookup key
        try:
            other = lookup_dtype(other)
            return str(self.np_type) < str(other.np_type)
        except ValueError:
            raise TypeError(f"Invalid or unknown datatype: {other}") from None

    def __reduce__(self):
        if self._is_udt:
            return (self._deserialize, (self.name, self.np_type, self._is_anonymous))
        if self.gb_name == "GrB_Index":
            return "_INDEX"
        return self.name

    @property
    def _carg(self):
        return self.gb_obj[0] if self.gb_name is None else self.gb_obj

    @property
    def _is_anonymous(self):
        return globals().get(self.name) is not self

    @property
    def _is_udt(self):
        return self.gb_name is None

    @staticmethod
    def _deserialize(name, dtype, is_anonymous):
        if is_anonymous:
            return register_anonymous(dtype, name)
        if name in _registry:
            return _registry[name]
        return register_new(name, dtype)


def register_new(name, dtype):
    if not name.isidentifier():
        raise ValueError(f"`name` argument must be a valid Python identifier; got: {name!r}")
    if name in _registry or name in globals():
        raise ValueError(f"{name!r} name for dtype is unavailable")
    rv = register_anonymous(dtype, name)
    _registry[name] = rv
    globals()[name] = rv
    return rv


def register_anonymous(dtype, name=None):
    from .exceptions import check_status_carg

    dtype = np.dtype(dtype)
    if dtype.hasobject:
        raise ValueError("dtype must not allow Python objects")
    if dtype.isbuiltin != 0:
        raise ValueError("dtype must not be a builtin type")
    if name is None:
        name = repr(dtype)
    numba_type = numba.typeof(dtype).dtype

    if dtype in _registry:
        gb_obj = _registry[dtype].gb_obj
    else:
        gb_obj = ffi.new("GrB_Type*")
        status = lib.GrB_Type_new(gb_obj, dtype.itemsize)
        check_status_carg(status, "Type", gb_obj[0])
    # For now, let's use "opaque" unsigned bytes for the c type.
    # grb_name probably isn't useful, right?
    rv = DataType(name, gb_obj, None, f"uint8_t[{dtype.itemsize}]", numba_type, dtype)
    if dtype not in _registry:
        _registry[gb_obj] = rv
        _registry[dtype] = rv
        _registry[numba_type] = rv
        _registry[numba_type.name] = rv
    return rv


BOOL = DataType("BOOL", lib.GrB_BOOL, "GrB_BOOL", "_Bool", numba.types.bool_, np.bool_)
INT8 = DataType("INT8", lib.GrB_INT8, "GrB_INT8", "int8_t", numba.types.int8, np.int8)
UINT8 = DataType("UINT8", lib.GrB_UINT8, "GrB_UINT8", "uint8_t", numba.types.uint8, np.uint8)
INT16 = DataType("INT16", lib.GrB_INT16, "GrB_INT16", "int16_t", numba.types.int16, np.int16)
UINT16 = DataType("UINT16", lib.GrB_UINT16, "GrB_UINT16", "uint16_t", numba.types.uint16, np.uint16)
INT32 = DataType("INT32", lib.GrB_INT32, "GrB_INT32", "int32_t", numba.types.int32, np.int32)
UINT32 = DataType("UINT32", lib.GrB_UINT32, "GrB_UINT32", "uint32_t", numba.types.uint32, np.uint32)
INT64 = DataType("INT64", lib.GrB_INT64, "GrB_INT64", "int64_t", numba.types.int64, np.int64)
# _Index (like UINT64) is for internal use only and shouldn't be exposed to the user
_INDEX = DataType("UINT64", lib.GrB_UINT64, "GrB_Index", "GrB_Index", numba.types.uint64, np.uint64)
UINT64 = DataType("UINT64", lib.GrB_UINT64, "GrB_UINT64", "uint64_t", numba.types.uint64, np.uint64)
FP32 = DataType("FP32", lib.GrB_FP32, "GrB_FP32", "float", numba.types.float32, np.float32)
FP64 = DataType("FP64", lib.GrB_FP64, "GrB_FP64", "double", numba.types.float64, np.float64)

if _supports_complex and hasattr(lib, "GxB_FC32"):
    FC32 = DataType(
        "FC32", lib.GxB_FC32, "GxB_FC32", "float _Complex", numba.types.complex64, np.complex64
    )
if _supports_complex and hasattr(lib, "GrB_FC32"):  # pragma: no coverage
    FC32 = DataType(
        "FC32", lib.GrB_FC32, "GrB_FC32", "float _Complex", numba.types.complex64, np.complex64
    )
if _supports_complex and hasattr(lib, "GxB_FC64"):
    FC64 = DataType(
        "FC64", lib.GxB_FC64, "GxB_FC64", "double _Complex", numba.types.complex128, np.complex128
    )
if _supports_complex and hasattr(lib, "GrB_FC64"):  # pragma: no coverage
    FC64 = DataType(
        "FC64", lib.GrB_FC64, "GrB_FC64", "double _Complex", numba.types.complex128, np.complex128
    )

# Used for testing user-defined functions
_sample_values = {
    INT8.name: np.int8(1),
    UINT8.name: np.uint8(1),
    INT16.name: np.int16(1),
    UINT16.name: np.uint16(1),
    INT32.name: np.int32(1),
    UINT32.name: np.uint32(1),
    INT64.name: np.int64(1),
    UINT64.name: np.uint64(1),
    FP32.name: np.float32(0.5),
    FP64.name: np.float64(0.5),
    BOOL.name: np.bool_(True),
}
if _supports_complex:
    _sample_values.update(
        {
            FC32.name: np.complex64(complex(0, 0.5)),
            FC64.name: np.complex128(complex(0, 0.5)),
        }
    )

# Create register to easily lookup types by name, gb_obj, or c_type
_registry = {}
_dtypes_to_register = [
    BOOL,
    INT8,
    UINT8,
    INT16,
    UINT16,
    INT32,
    UINT32,
    INT64,
    UINT64,
    FP32,
    FP64,
]
if _supports_complex:
    _dtypes_to_register.extend([FC32, FC64])

for dtype in _dtypes_to_register:
    _registry[dtype.name] = dtype
    _registry[dtype.name.lower()] = dtype
    _registry[dtype.gb_obj] = dtype
    _registry[dtype.gb_name] = dtype
    _registry[dtype.gb_name.lower()] = dtype
    _registry[dtype.c_type] = dtype
    _registry[dtype.c_type.upper()] = dtype
    _registry[dtype.numba_type] = dtype
    _registry[dtype.numba_type.name] = dtype
    val = _sample_values[dtype.name]
    _registry[val.dtype] = dtype
    _registry[val.dtype.name] = dtype
# Upcast numpy float16 to float32
_registry[np.dtype(np.float16)] = FP32
_registry["float16"] = FP32

# Add some common Python types as lookup keys
_registry[bool] = BOOL
_registry[int] = INT64
_registry[float] = FP64
_registry["bool"] = BOOL
_registry["int"] = INT64
_registry["float"] = FP64  # Choose 'float' to match numpy/Python; c_type 'float' would be FP32
if _supports_complex:
    _registry[complex] = FC64
    _registry["complex"] = FC64


def lookup_dtype(key, value=None):
    # Check for silly lookup where key is already a DataType
    if type(key) is DataType:
        return key
    try:
        return _registry[key]
    except KeyError:
        pass
    if hasattr(key, "name"):
        try:
            return _registry[key.name]
        except KeyError:
            pass
    if value is not None and hasattr(value, "dtype") and value.dtype in _registry:
        return _registry[value.dtype]
    try:
        return lookup_dtype(np.dtype(key))
    except Exception:
        pass
    raise ValueError(f"Unknown dtype: {key}")


def unify(type1, type2, *, is_left_scalar=False, is_right_scalar=False):
    """
    Returns a type that can hold both type1 and type2

    For example:
    unify(INT32, INT64) -> INT64
    unify(INT8, UINT16) -> INT32
    unify(BOOL, UINT16) -> UINT16
    unify(FP32, INT32) -> FP64
    """
    if type1 is type2:
        return type1
    if is_left_scalar:
        scalar_types = [type1.np_type]
        array_types = []
    elif not is_right_scalar:
        # Using `promote_types` is faster than `find_common_type`
        return lookup_dtype(promote_types(type1.np_type, type2.np_type))
    else:
        scalar_types = []
        array_types = [type1.np_type]
    if is_right_scalar:
        scalar_types.append(type2.np_type)
    else:
        array_types.append(type2.np_type)
    return lookup_dtype(find_common_type(array_types, scalar_types))

1	import numba	15✔
2	import numpy as np	15✔
3	from numpy import find_common_type, promote_types	15✔
4
5	from . import ffi, lib	15✔
6
7	# Default assumption unless FC32/FC64 are found in lib
8	_supports_complex = hasattr(lib, "GrB_FC64") or hasattr(lib, "GxB_FC64")	15✔
9
10
11	class DataType:	15✔
12	__slots__ = "name", "gb_obj", "gb_name", "c_type", "numba_type", "np_type", "__weakref__"	15✔
13
14	def __init__(self, name, gb_obj, gb_name, c_type, numba_type, np_type):	15✔
15	self.name = name	15✔
16	self.gb_obj = gb_obj	15✔
17	self.gb_name = gb_name	15✔
18	self.c_type = c_type	15✔
19	self.numba_type = numba_type	15✔
20	self.np_type = np_type	15✔
21
22	def __repr__(self):	15✔
23	return self.name	15✔
24
25	def __eq__(self, other):	15✔
26	if type(other) is DataType:	15✔
27	return self is other	15✔
28	# Attempt to use `other` as a lookup key
29	try:	15✔
30	other = lookup_dtype(other)	15✔
31	return self is other	15✔
32	except ValueError:	15✔
33	raise TypeError(f"Invalid or unknown datatype: {other}") from None	15✔
34
35	def __hash__(self):	15✔
36	return hash(self.gb_obj)	15✔
37
38	def __lt__(self, other):	15✔
39	# Let us sort for prettier error reporting
NEW 40	if type(other) is DataType:	×
NEW 41	return str(self.np_type) < str(other.np_type)	×
42	# Attempt to use `other` as a lookup key
NEW 43	try:	×
NEW 44	other = lookup_dtype(other)	×
NEW 45	return str(self.np_type) < str(other.np_type)	×
NEW 46	except ValueError:	×
NEW 47	raise TypeError(f"Invalid or unknown datatype: {other}") from None	×
48
49	def __reduce__(self):	15✔
50	if self._is_udt:	15!
NEW 51	return (self._deserialize, (self.name, self.np_type, self._is_anonymous))	×
52	if self.gb_name == "GrB_Index":	15✔
53	return "_INDEX"	15✔
54	return self.name	15✔
55
56	@property	15✔
57	def _carg(self):	15✔
58	return self.gb_obj[0] if self.gb_name is None else self.gb_obj	15✔
59
60	@property	15✔
61	def _is_anonymous(self):	15✔
62	return globals().get(self.name) is not self	15✔
63
64	@property	15✔
65	def _is_udt(self):	15✔
66	return self.gb_name is None	15✔
67
68	@staticmethod	15✔
69	def _deserialize(name, dtype, is_anonymous):	15✔
NEW 70	if is_anonymous:	×
NEW 71	return register_anonymous(dtype, name)	×
NEW 72	if name in _registry:	×
NEW 73	return _registry[name]	×
NEW 74	return register_new(name, dtype)	×
75
76
77	def register_new(name, dtype):	15✔
78	if not name.isidentifier():	15!
NEW 79	raise ValueError(f"`name` argument must be a valid Python identifier; got: {name!r}")	×
80	if name in _registry or name in globals():	15!
NEW 81	raise ValueError(f"{name!r} name for dtype is unavailable")	×
82	rv = register_anonymous(dtype, name)	15✔
83	_registry[name] = rv	15✔
84	globals()[name] = rv	15✔
85	return rv	15✔
86
87
88	def register_anonymous(dtype, name=None):	15✔
89	from .exceptions import check_status_carg	15✔
90
91	dtype = np.dtype(dtype)	15✔
92	if dtype.hasobject:	15!
NEW 93	raise ValueError("dtype must not allow Python objects")	×
94	if dtype.isbuiltin != 0:	15!
NEW 95	raise ValueError("dtype must not be a builtin type")	×
96	if name is None:	15!
NEW 97	name = repr(dtype)	×
98	numba_type = numba.typeof(dtype).dtype	15✔
99
100	if dtype in _registry:	15✔
101	gb_obj = _registry[dtype].gb_obj	15✔
102	else:
103	gb_obj = ffi.new("GrB_Type*")	15✔
104	status = lib.GrB_Type_new(gb_obj, dtype.itemsize)	15✔
105	check_status_carg(status, "Type", gb_obj[0])	15✔
106	# For now, let's use "opaque" unsigned bytes for the c type.
107	# grb_name probably isn't useful, right?
108	rv = DataType(name, gb_obj, None, f"uint8_t[{dtype.itemsize}]", numba_type, dtype)	15✔
109	if dtype not in _registry:	15✔
110	_registry[gb_obj] = rv	15✔
111	_registry[dtype] = rv	15✔
112	_registry[numba_type] = rv	15✔
113	_registry[numba_type.name] = rv	15✔
114	return rv	15✔
115
116
117	BOOL = DataType("BOOL", lib.GrB_BOOL, "GrB_BOOL", "_Bool", numba.types.bool_, np.bool_)	15✔
118	INT8 = DataType("INT8", lib.GrB_INT8, "GrB_INT8", "int8_t", numba.types.int8, np.int8)	15✔
119	UINT8 = DataType("UINT8", lib.GrB_UINT8, "GrB_UINT8", "uint8_t", numba.types.uint8, np.uint8)	15✔
120	INT16 = DataType("INT16", lib.GrB_INT16, "GrB_INT16", "int16_t", numba.types.int16, np.int16)	15✔
121	UINT16 = DataType("UINT16", lib.GrB_UINT16, "GrB_UINT16", "uint16_t", numba.types.uint16, np.uint16)	15✔
122	INT32 = DataType("INT32", lib.GrB_INT32, "GrB_INT32", "int32_t", numba.types.int32, np.int32)	15✔
123	UINT32 = DataType("UINT32", lib.GrB_UINT32, "GrB_UINT32", "uint32_t", numba.types.uint32, np.uint32)	15✔
124	INT64 = DataType("INT64", lib.GrB_INT64, "GrB_INT64", "int64_t", numba.types.int64, np.int64)	15✔
125	# _Index (like UINT64) is for internal use only and shouldn't be exposed to the user
126	_INDEX = DataType("UINT64", lib.GrB_UINT64, "GrB_Index", "GrB_Index", numba.types.uint64, np.uint64)	15✔
127	UINT64 = DataType("UINT64", lib.GrB_UINT64, "GrB_UINT64", "uint64_t", numba.types.uint64, np.uint64)	15✔
128	FP32 = DataType("FP32", lib.GrB_FP32, "GrB_FP32", "float", numba.types.float32, np.float32)	15✔
129	FP64 = DataType("FP64", lib.GrB_FP64, "GrB_FP64", "double", numba.types.float64, np.float64)	15✔
130
131	if _supports_complex and hasattr(lib, "GxB_FC32"):	15✔
132	FC32 = DataType(	11✔
133	"FC32", lib.GxB_FC32, "GxB_FC32", "float _Complex", numba.types.complex64, np.complex64
134	)
135	if _supports_complex and hasattr(lib, "GrB_FC32"): # pragma: no coverage
136	FC32 = DataType(
137	"FC32", lib.GrB_FC32, "GrB_FC32", "float _Complex", numba.types.complex64, np.complex64
138	)
139	if _supports_complex and hasattr(lib, "GxB_FC64"):	15✔
140	FC64 = DataType(	11✔
141	"FC64", lib.GxB_FC64, "GxB_FC64", "double _Complex", numba.types.complex128, np.complex128
142	)
143	if _supports_complex and hasattr(lib, "GrB_FC64"): # pragma: no coverage
144	FC64 = DataType(
145	"FC64", lib.GrB_FC64, "GrB_FC64", "double _Complex", numba.types.complex128, np.complex128
146	)
147
148	# Used for testing user-defined functions
149	_sample_values = {	15✔
150	INT8.name: np.int8(1),
151	UINT8.name: np.uint8(1),
152	INT16.name: np.int16(1),
153	UINT16.name: np.uint16(1),
154	INT32.name: np.int32(1),
155	UINT32.name: np.uint32(1),
156	INT64.name: np.int64(1),
157	UINT64.name: np.uint64(1),
158	FP32.name: np.float32(0.5),
159	FP64.name: np.float64(0.5),
160	BOOL.name: np.bool_(True),
161	}
162	if _supports_complex:	15✔
163	_sample_values.update(	11✔
164	{
165	FC32.name: np.complex64(complex(0, 0.5)),
166	FC64.name: np.complex128(complex(0, 0.5)),
167	}
168	)
169
170	# Create register to easily lookup types by name, gb_obj, or c_type
171	_registry = {}	15✔
172	_dtypes_to_register = [	15✔
173	BOOL,
174	INT8,
175	UINT8,
176	INT16,
177	UINT16,
178	INT32,
179	UINT32,
180	INT64,
181	UINT64,
182	FP32,
183	FP64,
184	]
185	if _supports_complex:	15✔
186	_dtypes_to_register.extend([FC32, FC64])	11✔
187
188	for dtype in _dtypes_to_register:	15✔
189	_registry[dtype.name] = dtype	15✔
190	_registry[dtype.name.lower()] = dtype	15✔
191	_registry[dtype.gb_obj] = dtype	15✔
192	_registry[dtype.gb_name] = dtype	15✔
193	_registry[dtype.gb_name.lower()] = dtype	15✔
194	_registry[dtype.c_type] = dtype	15✔
195	_registry[dtype.c_type.upper()] = dtype	15✔
196	_registry[dtype.numba_type] = dtype	15✔
197	_registry[dtype.numba_type.name] = dtype	15✔
198	val = _sample_values[dtype.name]	15✔
199	_registry[val.dtype] = dtype	15✔
200	_registry[val.dtype.name] = dtype	15✔
201	# Upcast numpy float16 to float32
202	_registry[np.dtype(np.float16)] = FP32	15✔
203	_registry["float16"] = FP32	15✔
204
205	# Add some common Python types as lookup keys
206	_registry[bool] = BOOL	15✔
207	_registry[int] = INT64	15✔
208	_registry[float] = FP64	15✔
209	_registry["bool"] = BOOL	15✔
210	_registry["int"] = INT64	15✔
211	_registry["float"] = FP64 # Choose 'float' to match numpy/Python; c_type 'float' would be FP32	15✔
212	if _supports_complex:	15✔
213	_registry[complex] = FC64	11✔
214	_registry["complex"] = FC64	11✔
215
216
217	def lookup_dtype(key, value=None):	15✔
218	# Check for silly lookup where key is already a DataType
219	if type(key) is DataType:	15✔
220	return key	15✔
221	try:	15✔
222	return _registry[key]	15✔
223	except KeyError:	15✔
224	pass	15✔
225	if hasattr(key, "name"):	15✔
226	try:	15✔
227	return _registry[key.name]	15✔
228	except KeyError:	15✔
229	pass	15✔
230	if value is not None and hasattr(value, "dtype") and value.dtype in _registry:	15✔
231	return _registry[value.dtype]	15✔
232	try:	15✔
233	return lookup_dtype(np.dtype(key))	15✔
234	except Exception:	15✔
235	pass	15✔
236	raise ValueError(f"Unknown dtype: {key}")	15✔
237
238
239	def unify(type1, type2, *, is_left_scalar=False, is_right_scalar=False):	15✔
240	"""
241	Returns a type that can hold both type1 and type2
242
243	For example:
244	unify(INT32, INT64) -> INT64
245	unify(INT8, UINT16) -> INT32
246	unify(BOOL, UINT16) -> UINT16
247	unify(FP32, INT32) -> FP64
248	"""
249	if type1 is type2:	15✔
250	return type1	15✔
251	if is_left_scalar:	15✔
252	scalar_types = [type1.np_type]	15✔
253	array_types = []	15✔
254	elif not is_right_scalar:	15✔
255	# Using `promote_types` is faster than `find_common_type`
256	return lookup_dtype(promote_types(type1.np_type, type2.np_type))	15✔
257	else:
258	scalar_types = []	15✔
259	array_types = [type1.np_type]	15✔
260	if is_right_scalar:	15✔
261	scalar_types.append(type2.np_type)	15✔
262	else:
263	array_types.append(type2.np_type)	15✔
264	return lookup_dtype(find_common_type(array_types, scalar_types))	15✔

metagraph-dev / grblas / 2062341675

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous