10762260192

Committed 08 Sep 2024 06:40PM UTC coverage: 90.033% (-0.2%) from 90.226%

Build # 10762260192

Build Type

push

github

Committed by

Giorgi

Commit Message

Merge branch 'nightly-varint' into nightly-build

Run Details

886 of 1013 branches covered (87.46%)

Branch coverage included in aggregate %.

45 of 48 new or added lines in 2 files covered. (93.75%)

32 existing lines in 5 files now uncovered.

1833 of 2007 relevant lines covered (91.33%)

876732.29 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.24

/DuckDB.NET.Data/Internal/Reader/NumericVectorDataReader.cs

﻿using System;
using System.Collections.Generic;
using System.Numerics;
using System.Runtime.CompilerServices;
using DuckDB.NET.Data.Extensions;
using DuckDB.NET.Native;

namespace DuckDB.NET.Data.Internal.Reader;

internal sealed class NumericVectorDataReader : VectorDataReaderBase
{
    private const int VarIntHeaderSize = 3;

    internal unsafe NumericVectorDataReader(void* dataPointer, ulong* validityMaskPointer, DuckDBType columnType, string columnName) : base(dataPointer, validityMaskPointer, columnType, columnName)
    {
    }

    protected override T GetValidValue<T>(ulong offset, Type targetType)
    {
        var isFloatingNumericType = TypeExtensions.IsFloatingNumericType<T>();
        var isIntegralNumericType = TypeExtensions.IsIntegralNumericType<T>();

        if (!(isIntegralNumericType || isFloatingNumericType))
        {
            return base.GetValidValue<T>(offset, targetType);
        }

        //If T is integral type and column is also integral read the data and use Unsafe.As<> or Convert.ChangeType to change type
        //If T is floating and column is floating too, read data and cast to T
        //Otherwise use the non-generic path
        if (isIntegralNumericType)
        {
            return DuckDBType switch
            {
                DuckDBType.TinyInt => GetUnmanagedTypeValue<sbyte, T>(offset),
                DuckDBType.SmallInt => GetUnmanagedTypeValue<short, T>(offset),
                DuckDBType.Integer => GetUnmanagedTypeValue<int, T>(offset),
                DuckDBType.BigInt => GetUnmanagedTypeValue<long, T>(offset),
                DuckDBType.UnsignedTinyInt => GetUnmanagedTypeValue<byte, T>(offset),
                DuckDBType.UnsignedSmallInt => GetUnmanagedTypeValue<ushort, T>(offset),
                DuckDBType.UnsignedInteger => GetUnmanagedTypeValue<uint, T>(offset),
                DuckDBType.UnsignedBigInt => GetUnmanagedTypeValue<ulong, T>(offset),
                DuckDBType.HugeInt => GetBigInteger<T>(offset, false),
                DuckDBType.UnsignedHugeInt => GetBigInteger<T>(offset, true),
                DuckDBType.VarInt => GetBigInteger<T>(offset),
                _ => base.GetValidValue<T>(offset, targetType)
            };
        }

        return DuckDBType switch
        {
            DuckDBType.Float => (T)(object)GetFieldData<float>(offset),
            DuckDBType.Double => (T)(object)GetFieldData<double>(offset),
            _ => base.GetValidValue<T>(offset, targetType)
        };
    }

    internal override object GetValue(ulong offset, Type targetType)
    {
        var value = DuckDBType switch
        {
            DuckDBType.TinyInt => GetFieldData<sbyte>(offset),
            DuckDBType.SmallInt => GetFieldData<short>(offset),
            DuckDBType.Integer => GetFieldData<int>(offset),
            DuckDBType.BigInt => GetFieldData<long>(offset),
            DuckDBType.UnsignedTinyInt => GetFieldData<byte>(offset),
            DuckDBType.UnsignedSmallInt => GetFieldData<ushort>(offset),
            DuckDBType.UnsignedInteger => GetFieldData<uint>(offset),
            DuckDBType.UnsignedBigInt => GetFieldData<ulong>(offset),
            DuckDBType.Float => GetFieldData<float>(offset),
            DuckDBType.Double => GetFieldData<double>(offset),
            DuckDBType.HugeInt => GetBigInteger(offset, false),
            DuckDBType.UnsignedHugeInt => GetBigInteger(offset, true),
            DuckDBType.VarInt => GetBigInteger<BigInteger>(offset),
            _ => base.GetValue(offset, targetType)
        };

        if (targetType.IsNumeric())
        {
            try
            {
                return Convert.ChangeType(value, targetType);
            }
            catch (OverflowException)
            {
                throw new InvalidCastException($"Cannot cast from {value.GetType().Name} to {targetType.Name} in column {ColumnName}");
            }
        }

        throw new InvalidCastException($"Cannot cast from {value.GetType().Name} to {targetType.Name} in column {ColumnName}");
    }

    internal unsafe BigInteger GetBigInteger(ulong offset, bool unsigned)
    {
        if (unsigned)
        {
            var unsignedHugeInt = ((DuckDBUHugeInt*)DataPointer + offset);
            return unsignedHugeInt->ToBigInteger();
        }
        else
        {
            var hugeInt = (DuckDBHugeInt*)DataPointer + offset;
            return hugeInt->ToBigInteger();
        }
    }

    private unsafe T GetBigInteger<T>(ulong offset)
    {
        var data = (DuckDBString*)DataPointer + offset;

        if (data->Length < VarIntHeaderSize + 1)
        {
            throw new DuckDBException("Invalid blob size for Varint.");
        }

        var buffer = new ReadOnlySpan<byte>(data->Data, data->Length);

        var isNegative = (buffer[0] & 0x80) == 0;

        var bytes = new List<byte>(data->Length - VarIntHeaderSize);

        for (var index = VarIntHeaderSize; index < buffer.Length; index++)
        {
            if (isNegative)
            {
                bytes.Add((byte)~buffer[index]);
            }
            else
            {
                bytes.Add(buffer[index]);
            }
        }

        var bigIntegerDigits = new Stack<char>();

        while (bytes.Count > 0)
        {
            var quotient = new List<char>();

            byte remainder = 0;

            foreach (var @byte in bytes)
            {
                var newValue = remainder * 256 + @byte;
                quotient.Add(DigitToChar(newValue / 10));

                remainder = (byte)(newValue % 10);
            }

            bigIntegerDigits.Push(DigitToChar(remainder));

            // Remove leading zeros from the quotient
            bytes.Clear();

            foreach (var digit in quotient)
            {
                if (digit != '0' || bytes.Count > 0)
                {
                    bytes.Add(CharToDigit(digit));
                }
            }
        }

        if (isNegative)
        {
            bigIntegerDigits.Push('-');
        }
        
        var integer = BigInteger.Parse(new string(bigIntegerDigits.ToArray()));
        
        try
        {
            return CastTo<T>(integer);
        }
        catch (OverflowException)
        {
            throw new InvalidCastException($"Cannot cast from {nameof(BigInteger)} to {typeof(T).Name} in column {ColumnName}");
        }

        char DigitToChar(int c) => (char)(c + '0');

        byte CharToDigit(char digit) => (byte)(digit-'0');
    }

    private T GetBigInteger<T>(ulong offset, bool unsigned)
    {
        var bigInteger = GetBigInteger(offset, unsigned);

        try
        {
            return CastTo<T>(bigInteger);
        }
        catch (OverflowException)
        {
            throw new InvalidCastException($"Cannot cast from {nameof(BigInteger)} to {typeof(T).Name} in column {ColumnName}");
        }
    }

    private static T CastTo<T>(BigInteger bigInteger)
    {
        if (typeof(T) == typeof(byte))
        {
            return (T)(object)(byte)bigInteger;
        }

        if (typeof(T) == typeof(sbyte))
        {
            return (T)(object)(sbyte)bigInteger;
        }

        if (typeof(T) == typeof(short))
        {
            return (T)(object)(short)bigInteger;
        }

        if (typeof(T) == typeof(ushort))
        {
            return (T)(object)(ushort)bigInteger;
        }

        if (typeof(T) == typeof(int))
        {
            return (T)(object)(int)bigInteger;
        }

        if (typeof(T) == typeof(uint))
        {
            return (T)(object)(uint)bigInteger;
        }

        if (typeof(T) == typeof(long))
        {
            return (T)(object)(long)bigInteger;
        }

        if (typeof(T) == typeof(ulong))
        {
            return (T)(object)(ulong)bigInteger;
        }

        return (T)(object)bigInteger;
    }

    private TResult GetUnmanagedTypeValue<TQuery, TResult>(ulong offset) where TQuery : unmanaged
    {
        var value = GetFieldData<TQuery>(offset);

        if (typeof(TQuery) == typeof(TResult))
        {
            return Unsafe.As<TQuery, TResult>(ref value);
        }

        try
        {
            return (TResult)Convert.ChangeType(value, typeof(TResult));
        }
        catch (OverflowException)
        {
            throw new InvalidCastException($"Cannot cast from {value.GetType().Name} to {typeof(TResult).Name} in column {ColumnName}");
        }
    }
}

1	using System;
2	using System.Collections.Generic;
3	using System.Numerics;
4	using System.Runtime.CompilerServices;
5	using DuckDB.NET.Data.Extensions;
6	using DuckDB.NET.Native;
7
8	namespace DuckDB.NET.Data.Internal.Reader;
9
10	internal sealed class NumericVectorDataReader : VectorDataReaderBase
11	{
12	private const int VarIntHeaderSize = 3;
13
14	internal unsafe NumericVectorDataReader(void* dataPointer, ulong* validityMaskPointer, DuckDBType columnType, string columnName) : base(dataPointer, validityMaskPointer, columnType, columnName)	69,040✔
15	{
16	}	69,040✔
17
18	protected override T GetValidValue<T>(ulong offset, Type targetType)
19	{
20	var isFloatingNumericType = TypeExtensions.IsFloatingNumericType<T>();	2,915,363✔
21	var isIntegralNumericType = TypeExtensions.IsIntegralNumericType<T>();	2,915,363✔
22
23	if (!(isIntegralNumericType \|\| isFloatingNumericType))	2,915,363✔
24	{
25	return base.GetValidValue<T>(offset, targetType);	31,269✔
26	}
27
28	//If T is integral type and column is also integral read the data and use Unsafe.As<> or Convert.ChangeType to change type
29	//If T is floating and column is floating too, read data and cast to T
30	//Otherwise use the non-generic path
31	if (isIntegralNumericType)	2,884,094✔
32	{
33	return DuckDBType switch	1,392,923!
34	{	1,392,923✔
35	DuckDBType.TinyInt => GetUnmanagedTypeValue<sbyte, T>(offset),	96✔
36	DuckDBType.SmallInt => GetUnmanagedTypeValue<short, T>(offset),	96✔
37	DuckDBType.Integer => GetUnmanagedTypeValue<int, T>(offset),	961,163✔
38	DuckDBType.BigInt => GetUnmanagedTypeValue<long, T>(offset),	150,474✔
39	DuckDBType.UnsignedTinyInt => GetUnmanagedTypeValue<byte, T>(offset),	96✔
40	DuckDBType.UnsignedSmallInt => GetUnmanagedTypeValue<ushort, T>(offset),	96✔
41	DuckDBType.UnsignedInteger => GetUnmanagedTypeValue<uint, T>(offset),	96✔
42	DuckDBType.UnsignedBigInt => GetUnmanagedTypeValue<ulong, T>(offset),	168✔
43	DuckDBType.HugeInt => GetBigInteger<T>(offset, false),	66✔
44	DuckDBType.UnsignedHugeInt => GetBigInteger<T>(offset, true),	36✔
45	DuckDBType.VarInt => GetBigInteger<T>(offset),	280,536✔
UNCOV 46	_ => base.GetValidValue<T>(offset, targetType)	×
47	};	1,392,923✔
48	}
49
50	return DuckDBType switch	1,491,171!
51	{	1,491,171✔
52	DuckDBType.Float => (T)(object)GetFieldData<float>(offset),	748,128✔
53	DuckDBType.Double => (T)(object)GetFieldData<double>(offset),	743,043✔
54	_ => base.GetValidValue<T>(offset, targetType)	×
55	};	1,491,171✔
56	}
57
58	internal override object GetValue(ulong offset, Type targetType)
59	{
60	var value = DuckDBType switch	10,644,099!
61	{	10,644,099✔
62	DuckDBType.TinyInt => GetFieldData<sbyte>(offset),	1,125,550✔
63	DuckDBType.SmallInt => GetFieldData<short>(offset),	1,118,072✔
64	DuckDBType.Integer => GetFieldData<int>(offset),	390,031✔
65	DuckDBType.BigInt => GetFieldData<long>(offset),	1,207,906✔
66	DuckDBType.UnsignedTinyInt => GetFieldData<byte>(offset),	1,120,366✔
67	DuckDBType.UnsignedSmallInt => GetFieldData<ushort>(offset),	1,135,364✔
68	DuckDBType.UnsignedInteger => GetFieldData<uint>(offset),	1,121,252✔
69	DuckDBType.UnsignedBigInt => GetFieldData<ulong>(offset),	1,119,730✔
70	DuckDBType.Float => GetFieldData<float>(offset),	374,486✔
71	DuckDBType.Double => GetFieldData<double>(offset),	372,782✔
72	DuckDBType.HugeInt => GetBigInteger(offset, false),	748,716✔
73	DuckDBType.UnsignedHugeInt => GetBigInteger(offset, true),	747,495✔
74	DuckDBType.VarInt => GetBigInteger<BigInteger>(offset),	62,349✔
UNCOV 75	_ => base.GetValue(offset, targetType)	×
76	};	10,644,099✔
77
78	if (targetType.IsNumeric())	10,644,099✔
79	{
80	try
81	{
82	return Convert.ChangeType(value, targetType);	10,612,830✔
83	}
84	catch (OverflowException)	15✔
85	{
86	throw new InvalidCastException($"Cannot cast from {value.GetType().Name} to {targetType.Name} in column {ColumnName}");	15✔
87	}
88	}
89
90	throw new InvalidCastException($"Cannot cast from {value.GetType().Name} to {targetType.Name} in column {ColumnName}");	31,269✔
91	}	10,612,815✔
92
93	internal unsafe BigInteger GetBigInteger(ulong offset, bool unsigned)
94	{
95	if (unsigned)	2,620,813✔
96	{
97	var unsignedHugeInt = ((DuckDBUHugeInt*)DataPointer + offset);	747,531✔
98	return unsignedHugeInt->ToBigInteger();	747,531✔
99	}
100	else
101	{
102	var hugeInt = (DuckDBHugeInt*)DataPointer + offset;	1,873,282✔
103	return hugeInt->ToBigInteger();	1,873,282✔
104	}
105	}
106
107	private unsafe T GetBigInteger<T>(ulong offset)
108	{
109	var data = (DuckDBString*)DataPointer + offset;	342,885✔
110
111	if (data->Length < VarIntHeaderSize + 1)	342,885!
112	{
NEW 113	throw new DuckDBException("Invalid blob size for Varint.");	×
114	}
115
116	var buffer = new ReadOnlySpan<byte>(data->Data, data->Length);	342,885✔
117
118	var isNegative = (buffer[0] & 0x80) == 0;	342,885✔
119
120	var bytes = new List<byte>(data->Length - VarIntHeaderSize);	342,885✔
121
122	for (var index = VarIntHeaderSize; index < buffer.Length; index++)	4,256,550✔
123	{
124	if (isNegative)	1,785,390✔
125	{
126	bytes.Add((byte)~buffer[index]);	879,465✔
127	}
128	else
129	{
130	bytes.Add(buffer[index]);	905,925✔
131	}
132	}
133
134	var bigIntegerDigits = new Stack<char>();	342,885✔
135
136	while (bytes.Count > 0)	4,567,173✔
137	{
138	var quotient = new List<char>();	4,224,288✔
139
140	byte remainder = 0;	4,224,288✔
141
142	foreach (var @byte in bytes)	41,351,550✔
143	{
144	var newValue = remainder * 256 + @byte;	16,451,487✔
145	quotient.Add(DigitToChar(newValue / 10));	16,451,487✔
146
147	remainder = (byte)(newValue % 10);	16,451,487✔
148	}
149
150	bigIntegerDigits.Push(DigitToChar(remainder));	4,224,288✔
151
152	// Remove leading zeros from the quotient
153	bytes.Clear();	4,224,288✔
154
155	foreach (var digit in quotient)	41,351,550✔
156	{
157	if (digit != '0' \|\| bytes.Count > 0)	16,451,487✔
158	{
159	bytes.Add(CharToDigit(digit));	14,666,097✔
160	}
161	}
162	}
163
164	if (isNegative)	342,885✔
165	{
166	bigIntegerDigits.Push('-');	170,586✔
167	}
168
169	var integer = BigInteger.Parse(new string(bigIntegerDigits.ToArray()));	342,885✔
170
171	try
172	{
173	return CastTo<T>(integer);	342,885✔
174	}
175	catch (OverflowException)	164,055✔
176	{
177	throw new InvalidCastException($"Cannot cast from {nameof(BigInteger)} to {typeof(T).Name} in column {ColumnName}");	164,055✔
178	}
179
180	char DigitToChar(int c) => (char)(c + '0');	20,675,775✔
181
182	byte CharToDigit(char digit) => (byte)(digit-'0');	14,666,097✔
183	}	178,830✔
184
185	private T GetBigInteger<T>(ulong offset, bool unsigned)
186	{
187	var bigInteger = GetBigInteger(offset, unsigned);	102✔
188
189	try
190	{
191	return CastTo<T>(bigInteger);	102✔
192	}
NEW 193	catch (OverflowException)	×
194	{
NEW 195	throw new InvalidCastException($"Cannot cast from {nameof(BigInteger)} to {typeof(T).Name} in column {ColumnName}");	×
196	}
197	}	102✔
198
199	private static T CastTo<T>(BigInteger bigInteger)
200	{
201	if (typeof(T) == typeof(byte))	342,987✔
202	{
203	return (T)(object)(byte)bigInteger;	31,170✔
204	}
205
206	if (typeof(T) == typeof(sbyte))	311,817✔
207	{
208	return (T)(object)(sbyte)bigInteger;	31,173✔
209	}
210
211	if (typeof(T) == typeof(short))	280,644✔
212	{
213	return (T)(object)(short)bigInteger;	31,173✔
214	}
215
216	if (typeof(T) == typeof(ushort))	249,471✔
217	{
218	return (T)(object)(ushort)bigInteger;	31,170✔
219	}
220
221	if (typeof(T) == typeof(int))	218,301✔
222	{
223	return (T)(object)(int)bigInteger;	31,173✔
224	}
225
226	if (typeof(T) == typeof(uint))	187,128✔
227	{
228	return (T)(object)(uint)bigInteger;	31,173✔
229	}
230
231	if (typeof(T) == typeof(long))	155,955✔
232	{
233	return (T)(object)(long)bigInteger;	31,173✔
234	}
235
236	if (typeof(T) == typeof(ulong))	124,782✔
237	{
238	return (T)(object)(ulong)bigInteger;	31,173✔
239	}
240
241	return (T)(object)bigInteger;	93,609✔
242	}
243
244	private TResult GetUnmanagedTypeValue<TQuery, TResult>(ulong offset) where TQuery : unmanaged
245	{
246	var value = GetFieldData<TQuery>(offset);	1,112,285✔
247
248	if (typeof(TQuery) == typeof(TResult))	1,112,285✔
249	{
250	return Unsafe.As<TQuery, TResult>(ref value);	1,111,586✔
251	}
252
253	try
254	{
255	return (TResult)Convert.ChangeType(value, typeof(TResult));	699✔
256	}
257	catch (OverflowException)	243✔
258	{
259	throw new InvalidCastException($"Cannot cast from {value.GetType().Name} to {typeof(TResult).Name} in column {ColumnName}");	243✔
260	}
261	}	456✔
262	}

Giorgi / DuckDB.NET / 10762260192

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous