39428820408

Committed 20 Oct 2026 01:15PM UTC coverage: 92.527%. First build

Build # 39428820408

Build Type

Pull #377

github

Committed by

web-flow

Commit Message

Merge 6d85eb0ef into e4ec50f04

Pull Request Pull Request #377: Implement GetBytes and GetChars

Coverage Stats

2102 of 2343 branches covered (89.71%)

Branch coverage included in aggregate %.

57 of 62 new or added lines in 2 files covered. (91.94%)

3841 of 4080 relevant lines covered (94.14%)

990063.41 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.0

/DuckDB.NET.Data/DataChunk/Reader/StringVectorDataReader.cs

using System.Buffers;
using System.IO;
using System.Text;

namespace DuckDB.NET.Data.DataChunk.Reader;

internal sealed class StringVectorDataReader : VectorDataReaderBase
{
    internal unsafe StringVectorDataReader(void* dataPointer, ulong* validityMaskPointer, DuckDBType columnType, string columnName) : base(dataPointer, validityMaskPointer, columnType, columnName)
    {
    }

    internal override string ReadString(ulong offset) => GetValidValue<string>(offset);
    internal override Stream ReadStream(ulong offset) => GetValidValue<Stream>(offset);

    protected override T GetValidValue<T>(ulong offset)
    {
        return DuckDBType switch
        {
            DuckDBType.Bit => GetBitString<T>(offset),
            DuckDBType.Blob => (T)(object)GetStream(offset),
            DuckDBType.Varchar => (T)(object)GetString(offset),
            _ => base.GetValidValue<T>(offset)
        };
    }

    internal override object GetValue(ulong offset, Type targetType)
    {
        return DuckDBType switch
        {
            DuckDBType.Bit => GetBitString(offset),
            DuckDBType.Blob => GetStream(offset),
            DuckDBType.Varchar => GetString(offset),
            _ => base.GetValue(offset, targetType)
        };
    }

    private T GetBitString<T>(ulong offset)
    {
        if (typeof(T) == typeof(string))
        {
            return (T)(object)GetBitString(offset);
        }

        if (typeof(T) == typeof(BitArray))
        {
            return (T)(object)GetBitStringAsBitArray(offset);
        }

        //Not GetValueStrict<T>: that comes back to this method
        return base.GetValidValue<T>(offset);
    }

    private string GetBitString(ulong offset)
    {
        var bitArray = GetBitStringAsBitArray(offset);

        return string.Create(bitArray.Length, bitArray, (chars, array) =>
        {
            for (int index = 0; index < array.Length; index++)
            {
                chars[index] = array[index] ? '1' : '0';
            }
        });
    }

    //Copied from https://github.com/duckdb/duckdb/blob/8a17511028d306561d88da9425f9e0e88dedd70c/src/common/types/bit.cpp#L63
    private unsafe BitArray GetBitStringAsBitArray(ulong offset)
    {
        var bits = (DuckDBString*)DataPointer + offset;
        var data = bits->Data;

        var bitLength = (bits->Length - 1) * 8 - *data;

        var output = new BitArray(bitLength);
        var outputIndex = 0;

        for (var bitIndex = *data; bitIndex < 8; bitIndex++)
        {
            output[outputIndex++] = (*(data + 1) & (1 << (7 - bitIndex))) > 0;
        }

        for (var byteIndex = 2; byteIndex < bits->Length; byteIndex++)
        {
            for (var bitIndex = 0; bitIndex < 8; bitIndex++)
            {
                output[outputIndex++] = (*(data + byteIndex) & (1 << (7 - bitIndex))) > 0;
            }
        }

        return output;
    }

    private unsafe string GetString(ulong offset)
    {
        var data = (DuckDBString*)DataPointer + offset;

        // The string constructor that takes an encoding validates more, wraps the pointer in a span and ends in the
        // same decoder, so this is the shorter way to it.
        return Encoding.UTF8.GetString((byte*)data->Data, data->Length);
    }

    internal unsafe long GetChars(ulong offset, long dataOffset, char[]? buffer, int bufferOffset, int length)
    {
        if (DuckDBType != DuckDBType.Varchar)
        {
            throw new NotSupportedException($"GetChars is not supported for {DuckDBType} columns");
        }

        if (!IsValid(offset))
        {
            throw new InvalidCastException($"Column '{ColumnName}' value is null");
        }

        var data = (DuckDBString*)DataPointer + offset;
        var bytes = new ReadOnlySpan<byte>((byte*)data->Data, data->Length);
        var byteOffset = 0;
        var characterOffset = 0L;
        var copied = 0;

        while (byteOffset < bytes.Length)
        {
            var status = Rune.DecodeFromUtf8(bytes[byteOffset..], out var rune, out var bytesConsumed);
            if (status != OperationStatus.Done)
            {
                rune = Rune.ReplacementChar;
                bytesConsumed = 1;
            }

            var runeLength = rune.Utf16SequenceLength;
            var charactersToSkip = dataOffset - characterOffset;
            if (charactersToSkip > 0)
            {
                if (charactersToSkip < runeLength && buffer != null && copied < length)
                {
                    buffer[bufferOffset + copied++] = GetSurrogate(rune, charactersToSkip);
                }
            }
            else if (buffer != null && copied < length)
            {
                var charactersToCopy = Math.Min(runeLength, length - copied);
                WriteRune(rune, buffer, bufferOffset + copied, charactersToCopy);
                copied += charactersToCopy;
            }

            byteOffset += bytesConsumed;
            characterOffset += runeLength;

            if (buffer != null && copied == length)
            {
                return copied;
            }
        }

        return buffer == null ? characterOffset : copied;
    }

    private static char GetSurrogate(Rune rune, long offset)
    {
        var value = rune.Value - 0x10000;
        return (char)(offset == 0 ? (value >> 10) + 0xD800 : (value & 0x3FF) + 0xDC00);
    }

    private static void WriteRune(Rune rune, char[] buffer, int offset, int length)
    {
        buffer[offset] = rune.Utf16SequenceLength == 1
            ? (char)rune.Value
            : GetSurrogate(rune, 0);

        if (length > 1)
        {
            buffer[offset + 1] = GetSurrogate(rune, 1);
        }
    }

    private unsafe Stream GetStream(ulong offset)
    {
        var data = (DuckDBString*)DataPointer + offset;

        return new UnmanagedMemoryStream((byte*)data->Data, data->Length, data->Length, FileAccess.Read);
    }
}

1	using System.Buffers;
2	using System.IO;
3	using System.Text;
4
5	namespace DuckDB.NET.Data.DataChunk.Reader;
6
7	internal sealed class StringVectorDataReader : VectorDataReaderBase
8	{
9	internal unsafe StringVectorDataReader(void* dataPointer, ulong* validityMaskPointer, DuckDBType columnType, string columnName) : base(dataPointer, validityMaskPointer, columnType, columnName)	3,540 ✔
10	{
11	}	3,540 ✔
12
13	internal override string ReadString(ulong offset) => GetValidValue<string>(offset);	160,887 ✔
14	internal override Stream ReadStream(ulong offset) => GetValidValue<Stream>(offset);	51 ✔
15
16	protected override T GetValidValue<T>(ulong offset)
17	{
18	return DuckDBType switch	160,968 !
19	{	160,968 ✔
20	DuckDBType.Bit => GetBitString<T>(offset),	42 ✔
21	DuckDBType.Blob => (T)(object)GetStream(offset),	51 ✔
22	DuckDBType.Varchar => (T)(object)GetString(offset),	160,875 ✔
23	_ => base.GetValidValue<T>(offset)	×
24	};	160,968 ✔
25	}
26
27	internal override object GetValue(ulong offset, Type targetType)
28	{
29	return DuckDBType switch	991,878 !
30	{	991,878 ✔
31	DuckDBType.Bit => GetBitString(offset),	39 ✔
32	DuckDBType.Blob => GetStream(offset),	6 ✔
33	DuckDBType.Varchar => GetString(offset),	991,833 ✔
34	_ => base.GetValue(offset, targetType)	×
35	};	991,878 ✔
36	}
37
38	private T GetBitString<T>(ulong offset)
39	{
40	if (typeof(T) == typeof(string))	42 ✔
41	{
42	return (T)(object)GetBitString(offset);	12 ✔
43	}
44
45	if (typeof(T) == typeof(BitArray))	30 ✔
46	{
47	return (T)(object)GetBitStringAsBitArray(offset);	3 ✔
48	}
49
50	//Not GetValueStrict<T>: that comes back to this method
51	return base.GetValidValue<T>(offset);	27 ✔
52	}
53
54	private string GetBitString(ulong offset)
55	{
56	var bitArray = GetBitStringAsBitArray(offset);	51 ✔
57
58	return string.Create(bitArray.Length, bitArray, (chars, array) =>	51 ✔
59	{	51 ✔
60	for (int index = 0; index < array.Length; index++)	876 ✔
61	{	51 ✔
62	chars[index] = array[index] ? '1' : '0';	387 ✔
63	}	51 ✔
64	});	102 ✔
65	}
66
67	//Copied from https://github.com/duckdb/duckdb/blob/8a17511028d306561d88da9425f9e0e88dedd70c/src/common/types/bit.cpp#L63
68	private unsafe BitArray GetBitStringAsBitArray(ulong offset)
69	{
70	var bits = (DuckDBString*)DataPointer + offset;	54 ✔
71	var data = bits->Data;	54 ✔
72
73	var bitLength = (bits->Length - 1) * 8 - *data;	54 ✔
74
75	var output = new BitArray(bitLength);	54 ✔
76	var outputIndex = 0;	54 ✔
77
78	for (var bitIndex = *data; bitIndex < 8; bitIndex++)	522 ✔
79	{
80	output[outputIndex++] = (*(data + 1) & (1 << (7 - bitIndex))) > 0;	207 ✔
81	}
82
83	for (var byteIndex = 2; byteIndex < bits->Length; byteIndex++)	162 ✔
84	{
85	for (var bitIndex = 0; bitIndex < 8; bitIndex++)	486 ✔
86	{
87	output[outputIndex++] = (*(data + byteIndex) & (1 << (7 - bitIndex))) > 0;	216 ✔
88	}
89	}
90
91	return output;	54 ✔
92	}
93
94	private unsafe string GetString(ulong offset)
95	{
96	var data = (DuckDBString*)DataPointer + offset;	1,152,708 ✔
97
98	// The string constructor that takes an encoding validates more, wraps the pointer in a span and ends in the
99	// same decoder, so this is the shorter way to it.
100	return Encoding.UTF8.GetString((byte*)data->Data, data->Length);	1,152,708 ✔
101	}
102
103	internal unsafe long GetChars(ulong offset, long dataOffset, char[]? buffer, int bufferOffset, int length)
104	{
105	if (DuckDBType != DuckDBType.Varchar)	99 !
106	{
NEW 107	throw new NotSupportedException($"GetChars is not supported for {DuckDBType} columns");	×
108	}
109
110	if (!IsValid(offset))	99 !
111	{
NEW 112	throw new InvalidCastException($"Column '{ColumnName}' value is null");	×
113	}
114
115	var data = (DuckDBString*)DataPointer + offset;	99 ✔
116	var bytes = new ReadOnlySpan<byte>((byte*)data->Data, data->Length);	99 ✔
117	var byteOffset = 0;	99 ✔
118	var characterOffset = 0L;	99 ✔
119	var copied = 0;	99 ✔
120
121	while (byteOffset < bytes.Length)	976,983 ✔
122	{
123	var status = Rune.DecodeFromUtf8(bytes[byteOffset..], out var rune, out var bytesConsumed);	976,968 ✔
124	if (status != OperationStatus.Done)	976,968 !
125	{
NEW 126	rune = Rune.ReplacementChar;	×
NEW 127	bytesConsumed = 1;	×
128	}
129
130	var runeLength = rune.Utf16SequenceLength;	976,968 ✔
131	var charactersToSkip = dataOffset - characterOffset;	976,968 ✔
132	if (charactersToSkip > 0)	976,968 ✔
133	{
134	if (charactersToSkip < runeLength && buffer != null && copied < length)	844,935 !
135	{
136	buffer[bufferOffset + copied++] = GetSurrogate(rune, charactersToSkip);	15 ✔
137	}
138	}
139	else if (buffer != null && copied < length)	132,033 !
140	{
141	var charactersToCopy = Math.Min(runeLength, length - copied);	66,021 ✔
142	WriteRune(rune, buffer, bufferOffset + copied, charactersToCopy);	66,021 ✔
143	copied += charactersToCopy;	66,021 ✔
144	}
145
146	byteOffset += bytesConsumed;	976,968 ✔
147	characterOffset += runeLength;	976,968 ✔
148
149	if (buffer != null && copied == length)	976,968 ✔
150	{
151	return copied;	84 ✔
152	}
153	}
154
155	return buffer == null ? characterOffset : copied;	15 ✔
156	}
157
158	private static char GetSurrogate(Rune rune, long offset)
159	{
160	var value = rune.Value - 0x10000;	24,006 ✔
161	return (char)(offset == 0 ? (value >> 10) + 0xD800 : (value & 0x3FF) + 0xDC00);	24,006 ✔
162	}
163
164	private static void WriteRune(Rune rune, char[] buffer, int offset, int length)
165	{
166	buffer[offset] = rune.Utf16SequenceLength == 1	66,021 ✔
167	? (char)rune.Value	66,021 ✔
168	: GetSurrogate(rune, 0);	66,021 ✔
169
170	if (length > 1)	66,021 ✔
171	{
172	buffer[offset + 1] = GetSurrogate(rune, 1);	11,988 ✔
173	}
174	}	66,021 ✔
175
176	private unsafe Stream GetStream(ulong offset)
177	{
178	var data = (DuckDBString*)DataPointer + offset;	57 ✔
179
180	return new UnmanagedMemoryStream((byte*)data->Data, data->Length, data->Length, FileAccess.Read);	57 ✔
181	}
182	}

Giorgi / DuckDB.NET / 39428820408

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous