40296324877

Committed 26 Oct 2026 09:00AM UTC coverage: 92.811% (+0.3%) from 92.531%

Build # 40296324877

Build Type

push

github

Committed by

Giorgi

Commit Message

Test the allocator grouping on whichever option the running DuckDB has

DuckDB's main branch replaced the block_allocator_memory setting with
enable_block_allocator (duckdb/duckdb#27520), so the nightly build failed to
find it. The grouping rule covers both, as both names have an allocator and a
block segment. The test now takes whichever option the running DuckDB has, so
it runs against DuckDB 2.0 and against the nightly builds.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01F9b3CGWkCMzmhoxge1QiZ8

Coverage Stats

2599 of 2865 branches covered (90.72%)

Branch coverage included in aggregate %.

4205 of 4466 relevant lines covered (94.16%)

1056578.13 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.11

/DuckDB.NET.Data/DataChunk/Reader/StringVectorDataReader.cs

using System.IO;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Runtime.Intrinsics;
using System.Text;

namespace DuckDB.NET.Data.DataChunk.Reader;

internal sealed class StringVectorDataReader : VectorDataReaderBase
{
    internal unsafe StringVectorDataReader(void* dataPointer, ulong* validityMaskPointer, DuckDBType columnType, string columnName) : base(dataPointer, validityMaskPointer, columnType, columnName)
    {
    }

    internal override string ReadString(ulong offset) => GetValidValue<string>(offset);
    internal override Stream ReadStream(ulong offset) => GetValidValue<Stream>(offset);

    protected override T GetValidValue<T>(ulong offset)
    {
        return DuckDBType switch
        {
            DuckDBType.Bit => GetBitString<T>(offset),
            DuckDBType.Blob => typeof(T) == typeof(byte[])
                ? (T)(object)GetBytes(offset)
                : (T)(object)GetStream(offset),
            DuckDBType.Varchar => (T)(object)GetString(offset),
            _ => base.GetValidValue<T>(offset)
        };
    }

    internal override object GetValueCore(ulong offset, Type targetType)
    {
        return DuckDBType switch
        {
            DuckDBType.Bit => GetBitString(offset),
            DuckDBType.Blob => targetType == typeof(byte[]) ? GetBytes(offset) : GetStream(offset),
            DuckDBType.Varchar => GetString(offset),
            _ => base.GetValueCore(offset, targetType)
        };
    }

    private T GetBitString<T>(ulong offset)
    {
        if (typeof(T) == typeof(string))
        {
            return (T)(object)GetBitString(offset);
        }

        if (typeof(T) == typeof(BitArray))
        {
            return (T)(object)GetBitStringAsBitArray(offset);
        }

        //Not GetValueStrict<T>: that comes back to this method
        return base.GetValidValue<T>(offset);
    }

    private string GetBitString(ulong offset)
    {
        var bitArray = GetBitStringAsBitArray(offset);

        return string.Create(bitArray.Length, bitArray, (chars, array) =>
        {
            for (int index = 0; index < array.Length; index++)
            {
                chars[index] = array[index] ? '1' : '0';
            }
        });
    }

    //Copied from https://github.com/duckdb/duckdb/blob/8a17511028d306561d88da9425f9e0e88dedd70c/src/common/types/bit.cpp#L63
    private unsafe BitArray GetBitStringAsBitArray(ulong offset)
    {
        var bits = (DuckDBString*)DataPointer + offset;
        var data = bits->Data;

        var bitLength = (bits->Length - 1) * 8 - *data;

        var output = new BitArray(bitLength);
        var outputIndex = 0;

        for (var bitIndex = *data; bitIndex < 8; bitIndex++)
        {
            output[outputIndex++] = (*(data + 1) & (1 << (7 - bitIndex))) > 0;
        }

        for (var byteIndex = 2; byteIndex < bits->Length; byteIndex++)
        {
            for (var bitIndex = 0; bitIndex < 8; bitIndex++)
            {
                output[outputIndex++] = (*(data + byteIndex) & (1 << (7 - bitIndex))) > 0;
            }
        }

        return output;
    }

    private unsafe string GetString(ulong offset)
    {
        var data = (DuckDBString*)DataPointer + offset;

        // The string constructor that takes an encoding validates more, wraps the pointer in a span and ends in the
        // same decoder, so this is the shorter way to it.
        return Encoding.UTF8.GetString((byte*)data->Data, data->Length);
    }

    // The value of the last GetChars call and its row in the chunk, so that reading a value in parts decodes it once.
    // Cleared when the reader moves to another chunk.
    private string? charsValue;
    private ulong charsRow;

    internal override void Reset(IntPtr vector)
    {
        charsValue = null;
        base.Reset(vector);
    }

    internal unsafe long GetChars(ulong offset, long dataOffset, char[]? buffer, int bufferOffset, int length)
    {
        if (DuckDBType != DuckDBType.Varchar)
        {
            throw GetCharsNotSupported();
        }

        if (!IsValid(offset))
        {
            throw ValueIsNull();
        }

        if (buffer is null)
        {
            if (charsValue is not null && charsRow == offset)
            {
                return charsValue.Length;
            }

            // The length alone is counted from the UTF-8 bytes, without decoding them.
            var data = (DuckDBString*)DataPointer + offset;
            return CountUtf16Chars(new ReadOnlySpan<byte>((byte*)data->Data, data->Length));
        }

        if (charsValue is null || charsRow != offset)
        {
            charsValue = GetString(offset);
            charsRow = offset;
        }

        if (dataOffset >= charsValue.Length)
        {
            return 0;
        }

        var count = (int)Math.Min(length, charsValue.Length - dataOffset);
        charsValue.CopyTo((int)dataOffset, buffer, bufferOffset, count);
        return count;
    }

    // The UTF-16 length of valid UTF-8, which DuckDB guarantees for VARCHAR. ASCII bytes are one char each; otherwise
    // see CountNonAsciiUtf16Chars.
    internal static long CountUtf16Chars(ReadOnlySpan<byte> bytes)
    {
        // Ascii.IsValid stops at the first block with a non-ASCII byte.
        if (Ascii.IsValid(bytes))
        {
            return bytes.Length;
        }

        return CountNonAsciiUtf16Chars(bytes);
    }

    // One char for every byte that is not a continuation byte (0x80-0xBF), and a second one for every lead byte of a
    // four-byte sequence (0xF0-0xF4), which becomes a surrogate pair. As signed bytes, continuation bytes are -128..-65
    // and four-byte lead bytes -16..-12; -16..-1 is tested, because 0xF5-0xFF never occur in valid UTF-8.
    private static long CountNonAsciiUtf16Chars(ReadOnlySpan<byte> bytes)
    {
        long count = bytes.Length;
        var index = 0;

        if (Vector128.IsHardwareAccelerated)
        {
            ref var start = ref MemoryMarshal.GetReference(bytes);
            var continuationMax = Vector128.Create((sbyte)-65);
            var fourByteLeadMin = Vector128.Create((sbyte)-16);

            for (; index <= bytes.Length - Vector128<byte>.Count; index += Vector128<byte>.Count)
            {
                var block = Vector128.LoadUnsafe(ref start, (nuint)index).AsSByte();
                var continuation = Vector128.LessThanOrEqual(block, continuationMax);
                var fourByteLead = Vector128.GreaterThanOrEqual(block, fourByteLeadMin) & Vector128.LessThan(block, Vector128<sbyte>.Zero);

                count -= BitOperations.PopCount(continuation.ExtractMostSignificantBits());
                count += BitOperations.PopCount(fourByteLead.ExtractMostSignificantBits());
            }
        }

        for (; index < bytes.Length; index++)
        {
            var value = bytes[index];
            if ((value & 0xC0) == 0x80)
            {
                count--;
            }
            else if (value >= 0xF0)
            {
                count++;
            }
        }

        return count;
    }

    [MethodImpl(MethodImplOptions.NoInlining)]
    private NotSupportedException GetCharsNotSupported() => new($"GetChars is not supported for {DuckDBType} columns");

    [MethodImpl(MethodImplOptions.NoInlining)]
    private InvalidCastException ValueIsNull() => new($"Column '{ColumnName}' value is null");

    private unsafe Stream GetStream(ulong offset)
    {
        var data = (DuckDBString*)DataPointer + offset;

        return new UnmanagedMemoryStream((byte*)data->Data, data->Length, data->Length, FileAccess.Read);
    }

    private unsafe byte[] GetBytes(ulong offset)
    {
        var data = (DuckDBString*)DataPointer + offset;
        return new ReadOnlySpan<byte>((void*)data->Data, checked((int)data->Length)).ToArray();
    }
}

1	using System.IO;
2	using System.Runtime.CompilerServices;
3	using System.Runtime.InteropServices;
4	using System.Runtime.Intrinsics;
5	using System.Text;
6
7	namespace DuckDB.NET.Data.DataChunk.Reader;
8
9	internal sealed class StringVectorDataReader : VectorDataReaderBase
10	{
11	internal unsafe StringVectorDataReader(void* dataPointer, ulong* validityMaskPointer, DuckDBType columnType, string columnName) : base(dataPointer, validityMaskPointer, columnType, columnName)	4,116 ✔
12	{
13	}	4,116 ✔
14
15	internal override string ReadString(ulong offset) => GetValidValue<string>(offset);	1,500,306 ✔
16	internal override Stream ReadStream(ulong offset) => GetValidValue<Stream>(offset);	99,051 ✔
17
18	protected override T GetValidValue<T>(ulong offset)
19	{
20	return DuckDBType switch	1,698,414 !
21	{	1,698,414 ✔
22	DuckDBType.Bit => GetBitString<T>(offset),	198,042 ✔
23	DuckDBType.Blob => typeof(T) == typeof(byte[])	99,069 ✔
24	? (T)(object)GetBytes(offset)	99,069 ✔
25	: (T)(object)GetStream(offset),	99,069 ✔
26	DuckDBType.Varchar => (T)(object)GetString(offset),	1,401,303 ✔
27	_ => base.GetValidValue<T>(offset)	×
28	};	1,698,414 ✔
29	}
30
31	internal override object GetValueCore(ulong offset, Type targetType)
32	{
33	return DuckDBType switch	189,775 !
34	{	189,775 ✔
35	DuckDBType.Bit => GetBitString(offset),	66 ✔
36	DuckDBType.Blob => targetType == typeof(byte[]) ? GetBytes(offset) : GetStream(offset),	36 !
37	DuckDBType.Varchar => GetString(offset),	189,673 ✔
38	_ => base.GetValueCore(offset, targetType)	×
39	};	189,775 ✔
40	}
41
42	private T GetBitString<T>(ulong offset)
43	{
44	if (typeof(T) == typeof(string))	198,042 ✔
45	{
46	return (T)(object)GetBitString(offset);	99,012 ✔
47	}
48
49	if (typeof(T) == typeof(BitArray))	99,030 ✔
50	{
51	return (T)(object)GetBitStringAsBitArray(offset);	99,003 ✔
52	}
53
54	//Not GetValueStrict<T>: that comes back to this method
55	return base.GetValidValue<T>(offset);	27 ✔
56	}
57
58	private string GetBitString(ulong offset)
59	{
60	var bitArray = GetBitStringAsBitArray(offset);	99,078 ✔
61
62	return string.Create(bitArray.Length, bitArray, (chars, array) =>	99,078 ✔
63	{	99,078 ✔
64	for (int index = 0; index < array.Length; index++)	991,092 ✔
65	{	99,078 ✔
66	chars[index] = array[index] ? '1' : '0';	396,468 ✔
67	}	99,078 ✔
68	});	198,156 ✔
69	}
70
71	//Copied from https://github.com/duckdb/duckdb/blob/8a17511028d306561d88da9425f9e0e88dedd70c/src/common/types/bit.cpp#L63
72	private unsafe BitArray GetBitStringAsBitArray(ulong offset)
73	{
74	var bits = (DuckDBString*)DataPointer + offset;	198,081 ✔
75	var data = bits->Data;	198,081 ✔
76
77	var bitLength = (bits->Length - 1) * 8 - *data;	198,081 ✔
78
79	var output = new BitArray(bitLength);	198,081 ✔
80	var outputIndex = 0;	198,081 ✔
81
82	for (var bitIndex = *data; bitIndex < 8; bitIndex++)	1,980,738 ✔
83	{
84	output[outputIndex++] = (*(data + 1) & (1 << (7 - bitIndex))) > 0;	792,288 ✔
85	}
86
87	for (var byteIndex = 2; byteIndex < bits->Length; byteIndex++)	396,216 ✔
88	{
89	for (var bitIndex = 0; bitIndex < 8; bitIndex++)	486 ✔
90	{
91	output[outputIndex++] = (*(data + byteIndex) & (1 << (7 - bitIndex))) > 0;	216 ✔
92	}
93	}
94
95	return output;	198,081 ✔
96	}
97
98	private unsafe string GetString(ulong offset)
99	{
100	var data = (DuckDBString*)DataPointer + offset;	1,605,982 ✔
101
102	// The string constructor that takes an encoding validates more, wraps the pointer in a span and ends in the
103	// same decoder, so this is the shorter way to it.
104	return Encoding.UTF8.GetString((byte*)data->Data, data->Length);	1,605,982 ✔
105	}
106
107	// The value of the last GetChars call and its row in the chunk, so that reading a value in parts decodes it once.
108	// Cleared when the reader moves to another chunk.
109	private string? charsValue;
110	private ulong charsRow;
111
112	internal override void Reset(IntPtr vector)
113	{
114	charsValue = null;	1,119 ✔
115	base.Reset(vector);	1,119 ✔
116	}	1,119 ✔
117
118	internal unsafe long GetChars(ulong offset, long dataOffset, char[]? buffer, int bufferOffset, int length)
119	{
120	if (DuckDBType != DuckDBType.Varchar)	118,818 !
121	{
122	throw GetCharsNotSupported();	×
123	}
124
125	if (!IsValid(offset))	118,818 !
126	{
127	throw ValueIsNull();	×
128	}
129
130	if (buffer is null)	118,818 ✔
131	{
132	if (charsValue is not null && charsRow == offset)	30,006 ✔
133	{
134	return charsValue.Length;	15,000 ✔
135	}
136
137	// The length alone is counted from the UTF-8 bytes, without decoding them.
138	var data = (DuckDBString*)DataPointer + offset;	15,006 ✔
139	return CountUtf16Chars(new ReadOnlySpan<byte>((byte*)data->Data, data->Length));	15,006 ✔
140	}
141
142	if (charsValue is null \|\| charsRow != offset)	88,812 ✔
143	{
144	charsValue = GetString(offset);	15,006 ✔
145	charsRow = offset;	15,006 ✔
146	}
147
148	if (dataOffset >= charsValue.Length)	88,812 ✔
149	{
150	return 0;	15,003 ✔
151	}
152
153	var count = (int)Math.Min(length, charsValue.Length - dataOffset);	73,809 ✔
154	charsValue.CopyTo((int)dataOffset, buffer, bufferOffset, count);	73,809 ✔
155	return count;	73,809 ✔
156	}
157
158	// The UTF-16 length of valid UTF-8, which DuckDB guarantees for VARCHAR. ASCII bytes are one char each; otherwise
159	// see CountNonAsciiUtf16Chars.
160	internal static long CountUtf16Chars(ReadOnlySpan<byte> bytes)
161	{
162	// Ascii.IsValid stops at the first block with a non-ASCII byte.
163	if (Ascii.IsValid(bytes))	3,504,201 ✔
164	{
165	return bytes.Length;	9,618 ✔
166	}
167
168	return CountNonAsciiUtf16Chars(bytes);	3,494,583 ✔
169	}
170
171	// One char for every byte that is not a continuation byte (0x80-0xBF), and a second one for every lead byte of a
172	// four-byte sequence (0xF0-0xF4), which becomes a surrogate pair. As signed bytes, continuation bytes are -128..-65
173	// and four-byte lead bytes -16..-12; -16..-1 is tested, because 0xF5-0xFF never occur in valid UTF-8.
174	private static long CountNonAsciiUtf16Chars(ReadOnlySpan<byte> bytes)
175	{
176	long count = bytes.Length;	3,494,583 ✔
177	var index = 0;	3,494,583 ✔
178
179	if (Vector128.IsHardwareAccelerated)	3,494,583 !
180	{
181	ref var start = ref MemoryMarshal.GetReference(bytes);	3,494,583 ✔
182	var continuationMax = Vector128.Create((sbyte)-65);	3,494,583 ✔
183	var fourByteLeadMin = Vector128.Create((sbyte)-16);	3,494,583 ✔
184
185	for (; index <= bytes.Length - Vector128<byte>.Count; index += Vector128<byte>.Count)	8,079,729 ✔
186	{
187	var block = Vector128.LoadUnsafe(ref start, (nuint)index).AsSByte();	2,292,573 ✔
188	var continuation = Vector128.LessThanOrEqual(block, continuationMax);	2,292,573 ✔
189	var fourByteLead = Vector128.GreaterThanOrEqual(block, fourByteLeadMin) & Vector128.LessThan(block, Vector128<sbyte>.Zero);	2,292,573 ✔
190
191	count -= BitOperations.PopCount(continuation.ExtractMostSignificantBits());	2,292,573 ✔
192	count += BitOperations.PopCount(fourByteLead.ExtractMostSignificantBits());	2,292,573 ✔
193	}
194	}
195
196	for (; index < bytes.Length; index++)	32,272,101 ✔
197	{
198	var value = bytes[index];	14,388,759 ✔
199	if ((value & 0xC0) == 0x80)	14,388,759 ✔
200	{
201	count--;	10,613,715 ✔
202	}
203	else if (value >= 0xF0)	3,775,044 ✔
204	{
205	count++;	3,248,979 ✔
206	}
207	}
208
209	return count;	3,494,583 ✔
210	}
211
212	[MethodImpl(MethodImplOptions.NoInlining)]
213	private NotSupportedException GetCharsNotSupported() => new($"GetChars is not supported for {DuckDBType} columns");	×
214
215	[MethodImpl(MethodImplOptions.NoInlining)]
216	private InvalidCastException ValueIsNull() => new($"Column '{ColumnName}' value is null");	×
217
218	private unsafe Stream GetStream(ulong offset)
219	{
220	var data = (DuckDBString*)DataPointer + offset;	99,087 ✔
221
222	return new UnmanagedMemoryStream((byte*)data->Data, data->Length, data->Length, FileAccess.Read);	99,087 ✔
223	}
224
225	private unsafe byte[] GetBytes(ulong offset)
226	{
227	var data = (DuckDBString*)DataPointer + offset;	18 ✔
228	return new ReadOnlySpan<byte>((void*)data->Data, checked((int)data->Length)).ToArray();	18 ✔
229	}
230	}

Giorgi / DuckDB.NET / 40296324877

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous