28103508122

Committed 24 Jun 2026 01:51PM UTC coverage: 89.358% (-0.1%) from 89.483%

Build # 28103508122

Build Type

push

github

Committed by

web-flow

Commit Message

Add Apache Arrow result streaming to DuckDBCommand (#334)

* Add Apache Arrow result streaming to DuckDBCommand

Adds a managed Apache Arrow read surface built on DuckDB's current Arrow C Data
Interface (duckdb_to_arrow_schema / duckdb_data_chunk_to_arrow), addressing #26.

- Bindings: duckdb_result_get_arrow_options, duckdb_to_arrow_schema,
  duckdb_data_chunk_to_arrow, error-data helpers, and a DuckDBArrowOptions safe handle.
- Data: DuckDBArrowArrayStream (IArrowArrayStream) that builds the schema once and
  converts each data chunk into an Arrow RecordBatch via Apache.Arrow's C importers.
- DuckDBCommand.ExecuteArrowStream() and ExecuteArrowBatchesAsync() public APIs.
- Apache.Arrow dependency added to DuckDB.NET.Data only.
- Tests covering schema, scalar values, nulls, multi-chunk streaming, and the stream API.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Honor UseStreamingMode in Arrow APIs and fix result stream cleanup

Pass UseStreamingMode through ExecuteArrowStream and fetch chunks via the
streaming or materialized path accordingly. Dispose Arrow options and close
the result if schema building fails in the stream constructor. Add tests for
streaming mode, cancellation, and dispose semantics.

* Refactor Arrow error handling to typed DuckDBErrorData handle

- Add DuckDBErrorData SafeHandle wrapping duckdb_error_data
- Move error-data P/Invokes to NativeMethods.ErrorData
- Retype duckdb_to_arrow_schema/data_chunk_to_arrow returns to DuckDBErrorData
- Add reusable ThrowOnError extension throwing DuckDBException
- Make DuckDBArrowArrayStream internal

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Coverage Stats

1359 of 1583 branches covered (85.85%)

Branch coverage included in aggregate %.

84 of 94 new or added lines in 4 files covered. (89.36%)

2856 of 3134 relevant lines covered (91.13%)

421574.19 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.67

/DuckDB.NET.Bindings/DuckDBWrapperObjects.cs

﻿namespace DuckDB.NET.Native;

public class DuckDBDatabase() : SafeHandleZeroOrMinusOneIsInvalid(true)
{
    protected override bool ReleaseHandle()
    {
        NativeMethods.Startup.DuckDBClose(ref handle);
        return true;
    }
}

public class DuckDBNativeConnection() : SafeHandleZeroOrMinusOneIsInvalid(true)
{
    protected override bool ReleaseHandle()
    {
        NativeMethods.Startup.DuckDBDisconnect(ref handle);
        return true;
    }

    public void Interrupt()
    {
        NativeMethods.Startup.DuckDBInterrupt(this);
    }
}

public class DuckDBPreparedStatement() : SafeHandleZeroOrMinusOneIsInvalid(true)
{
    protected override bool ReleaseHandle()
    {
        NativeMethods.PreparedStatements.DuckDBDestroyPrepare(ref handle);
        return true;
    }
}

public class DuckDBConfig() : SafeHandleZeroOrMinusOneIsInvalid(true)
{
    protected override bool ReleaseHandle()
    {
        NativeMethods.Configuration.DuckDBDestroyConfig(ref handle);
        return true;
    }
}

public class DuckDBAppender() : SafeHandleZeroOrMinusOneIsInvalid(true)
{
    protected override bool ReleaseHandle()
    {
        return NativeMethods.Appender.DuckDBDestroyAppender(ref handle).IsSuccess();
    }
}

public class DuckDBExtractedStatements() : SafeHandleZeroOrMinusOneIsInvalid(true)
{
    protected override bool ReleaseHandle()
    {
        NativeMethods.ExtractStatements.DuckDBDestroyExtracted(ref handle);

        return true;
    }
}

public class DuckDBLogicalType() : SafeHandleZeroOrMinusOneIsInvalid(true)
{
    protected override bool ReleaseHandle()
    {
        NativeMethods.LogicalType.DuckDBDestroyLogicalType(ref handle);
        return true;
    }
}

public class DuckDBArrowOptions() : SafeHandleZeroOrMinusOneIsInvalid(true)
{
    protected override bool ReleaseHandle()
    {
        NativeMethods.Arrow.DuckDBDestroyArrowOptions(ref handle);
        return true;
    }
}

public class DuckDBErrorData() : SafeHandleZeroOrMinusOneIsInvalid(true)
{
    public bool HasError => !IsInvalid && NativeMethods.ErrorData.DuckDBErrorDataHasError(this);

    public string? Message => IsInvalid ? null : NativeMethods.ErrorData.DuckDBErrorDataMessage(this);

    protected override bool ReleaseHandle()
    {
        NativeMethods.ErrorData.DuckDBDestroyErrorData(ref handle);
        return true;
    }
}

public class DuckDBDataChunk : SafeHandleZeroOrMinusOneIsInvalid
{
    public DuckDBDataChunk() : base(true)
    {
    }

    public DuckDBDataChunk(IntPtr chunk) : base(false)
    {
        SetHandle(chunk);
    }

    protected override bool ReleaseHandle()
    {
        NativeMethods.DataChunks.DuckDBDestroyDataChunk(ref handle);
        return true;
    }
}

public class DuckDBValue() : SafeHandleZeroOrMinusOneIsInvalid(true), IDuckDBValueReader
{
    private DuckDBValue[] childValues = [];

    protected override bool ReleaseHandle()
    {
        foreach (var value in childValues)
        {
            value.Dispose();
        }

        NativeMethods.Value.DuckDBDestroyValue(ref handle);
        return true;
    }

    internal void SetChildValues(DuckDBValue[] values)
    {
        childValues = values;
    }

    public bool IsNull() => NativeMethods.Value.DuckDBIsNullValue(this);

    public T GetValue<T>()
    {
        var logicalType = NativeMethods.Value.DuckDBGetValueType(this);

        //Logical type is part of the duckdb_value object and it shouldn't be released separately
        //It will get released when the duckdb_value object is destroyed below.
        var add = false;
        logicalType.DangerousAddRef(ref add);

        var duckDBType = NativeMethods.LogicalType.DuckDBGetTypeId(logicalType);

        return duckDBType switch
        {
            DuckDBType.Boolean => Cast(NativeMethods.Value.DuckDBGetBool(this)),

            DuckDBType.TinyInt => Cast(NativeMethods.Value.DuckDBGetInt8(this)),
            DuckDBType.SmallInt => Cast(NativeMethods.Value.DuckDBGetInt16(this)),
            DuckDBType.Integer => Cast(NativeMethods.Value.DuckDBGetInt32(this)),
            DuckDBType.BigInt => Cast(NativeMethods.Value.DuckDBGetInt64(this)),

            DuckDBType.UnsignedTinyInt => Cast(NativeMethods.Value.DuckDBGetUInt8(this)),
            DuckDBType.UnsignedSmallInt => Cast(NativeMethods.Value.DuckDBGetUInt16(this)),
            DuckDBType.UnsignedInteger => Cast(NativeMethods.Value.DuckDBGetUInt32(this)),
            DuckDBType.UnsignedBigInt => Cast(NativeMethods.Value.DuckDBGetUInt64(this)),

            DuckDBType.Float => Cast(NativeMethods.Value.DuckDBGetFloat(this)),
            DuckDBType.Double => Cast(NativeMethods.Value.DuckDBGetDouble(this)),

            DuckDBType.Decimal => Cast(decimal.Parse(NativeMethods.Value.DuckDBGetVarchar(this), NumberStyles.Any, CultureInfo.InvariantCulture)),

            DuckDBType.Uuid => Cast(new Guid(NativeMethods.Value.DuckDBGetVarchar(this))),

            DuckDBType.HugeInt => Cast(NativeMethods.Value.DuckDBGetHugeInt(this).ToBigInteger()),
            DuckDBType.UnsignedHugeInt => Cast(NativeMethods.Value.DuckDBGetUHugeInt(this).ToBigInteger()),

            DuckDBType.Varchar => Cast(NativeMethods.Value.DuckDBGetVarchar(this)),

            DuckDBType.Date => Cast((DateOnly)DuckDBDateOnly.FromDuckDBDate(NativeMethods.Value.DuckDBGetDate(this))),
            DuckDBType.Time => Cast((TimeOnly)NativeMethods.DateTimeHelpers.DuckDBFromTime(NativeMethods.Value.DuckDBGetTime(this))),
            DuckDBType.TimeTz => Cast(GetTimeTzValue()),
            DuckDBType.Interval => Cast((TimeSpan)NativeMethods.Value.DuckDBGetInterval(this)),
            DuckDBType.Timestamp => Cast(GetTimestampValue(NativeMethods.Value.DuckDBGetTimestamp(this),DuckDBType.Timestamp)),
            DuckDBType.TimestampS => Cast(GetTimestampValue(NativeMethods.Value.DuckDBGetTimestampS(this), DuckDBType.TimestampS)),
            DuckDBType.TimestampMs => Cast(GetTimestampValue(NativeMethods.Value.DuckDBGetTimestampMs(this), DuckDBType.TimestampMs)),
            DuckDBType.TimestampNs => Cast(GetTimestampValue(NativeMethods.Value.DuckDBGetTimestampNs(this), DuckDBType.TimestampNs)),
            DuckDBType.TimestampTz => Cast(GetTimestampValue(NativeMethods.Value.DuckDBGetTimestamp(this), DuckDBType.TimestampTz)),
            _ => throw new NotImplementedException($"Cannot read value of type {typeof(T).FullName}")
        };

        [MethodImpl(MethodImplOptions.AggressiveInlining)]
        static T Cast<TSource>(TSource value) => Unsafe.As<TSource, T>(ref value);
    }

    private DateTime GetTimestampValue(DuckDBTimestampStruct timestampStruct, DuckDBType duckDBType)
    {
        var additionalTicks = 0;

        // The type-specific getters return values in their native units:
        // - TimestampS: seconds
        // - TimestampMs: milliseconds
        // - TimestampNs: nanoseconds
        // We need to convert to microseconds for DuckDBTimestamp.FromDuckDBTimestampStruct()
        if (duckDBType == DuckDBType.TimestampNs)
        {
            additionalTicks = (int)(timestampStruct.Micros % 1000 / 100);
            timestampStruct.Micros /= 1000;
        }
        if (duckDBType == DuckDBType.TimestampMs)
        {
            timestampStruct.Micros *= 1000;
        }
        if (duckDBType == DuckDBType.TimestampS)
        {
            timestampStruct.Micros *= 1000000;
        }

        var timestamp = DuckDBTimestamp.FromDuckDBTimestampStruct(timestampStruct);
        return timestamp.ToDateTime().AddTicks(additionalTicks);
    }

    private DateTimeOffset GetTimeTzValue()
    {
        var timeTzStruct = NativeMethods.Value.DuckDBGetTimeTz(this);
        var timeTz = NativeMethods.DateTimeHelpers.DuckDBFromTimeTz(timeTzStruct);
        return new DateTimeOffset(timeTz.Time.ToDateTime(), TimeSpan.FromSeconds(timeTz.Offset));
    }
}

public class DuckDBClientContext() : SafeHandleZeroOrMinusOneIsInvalid(true)
{
    protected override bool ReleaseHandle()
    {
        NativeMethods.Startup.DuckDBDestroyClientContext(ref handle);
        return true;
    }

    public ulong ConnectionId => NativeMethods.Startup.DuckDBClientContextGetConnectionId(this);
}

1	namespace DuckDB.NET.Native;
2
3	public class DuckDBDatabase() : SafeHandleZeroOrMinusOneIsInvalid(true)	50,204✔
4	{
5	protected override bool ReleaseHandle()
6	{
7	NativeMethods.Startup.DuckDBClose(ref handle);	50,200✔
8	return true;	50,200✔
9	}
10	}
11
12	public class DuckDBNativeConnection() : SafeHandleZeroOrMinusOneIsInvalid(true)	64,956✔
13	{
14	protected override bool ReleaseHandle()
15	{
16	NativeMethods.Startup.DuckDBDisconnect(ref handle);	64,952✔
17	return true;	64,952✔
18	}
19
20	public void Interrupt()
21	{
22	NativeMethods.Startup.DuckDBInterrupt(this);	12✔
23	}	12✔
24	}
25
26	public class DuckDBPreparedStatement() : SafeHandleZeroOrMinusOneIsInvalid(true)	158,355✔
27	{
28	protected override bool ReleaseHandle()
29	{
30	NativeMethods.PreparedStatements.DuckDBDestroyPrepare(ref handle);	158,351✔
31	return true;	158,351✔
32	}
33	}
34
35	public class DuckDBConfig() : SafeHandleZeroOrMinusOneIsInvalid(true)	50,201✔
36	{
37	protected override bool ReleaseHandle()
38	{
39	NativeMethods.Configuration.DuckDBDestroyConfig(ref handle);	50,201✔
40	return true;	50,201✔
41	}
42	}
43
44	public class DuckDBAppender() : SafeHandleZeroOrMinusOneIsInvalid(true)	222✔
45	{
46	protected override bool ReleaseHandle()
47	{
48	return NativeMethods.Appender.DuckDBDestroyAppender(ref handle).IsSuccess();	222✔
49	}
50	}
51
52	public class DuckDBExtractedStatements() : SafeHandleZeroOrMinusOneIsInvalid(true)	158,247✔
53	{
54	protected override bool ReleaseHandle()
55	{
56	NativeMethods.ExtractStatements.DuckDBDestroyExtracted(ref handle);	158,243✔
57
58	return true;	158,243✔
59	}
60	}
61
62	public class DuckDBLogicalType() : SafeHandleZeroOrMinusOneIsInvalid(true)	63,396✔
63	{
64	protected override bool ReleaseHandle()
65	{
66	NativeMethods.LogicalType.DuckDBDestroyLogicalType(ref handle);	63,036✔
67	return true;	63,036✔
68	}
69	}
70
71	public class DuckDBArrowOptions() : SafeHandleZeroOrMinusOneIsInvalid(true)	33✔
72	{
73	protected override bool ReleaseHandle()
74	{
75	NativeMethods.Arrow.DuckDBDestroyArrowOptions(ref handle);	33✔
76	return true;	33✔
77	}
78	}
79
80	public class DuckDBErrorData() : SafeHandleZeroOrMinusOneIsInvalid(true)	66✔
81	{
82	public bool HasError => !IsInvalid && NativeMethods.ErrorData.DuckDBErrorDataHasError(this);	66!
83
NEW 84	public string? Message => IsInvalid ? null : NativeMethods.ErrorData.DuckDBErrorDataMessage(this);	×
85
86	protected override bool ReleaseHandle()
87	{
NEW 88	NativeMethods.ErrorData.DuckDBDestroyErrorData(ref handle);	×
NEW 89	return true;	×
90	}
91	}
92
93	public class DuckDBDataChunk : SafeHandleZeroOrMinusOneIsInvalid
94	{
95	public DuckDBDataChunk() : base(true)	40,998✔
96	{
97	}	40,998✔
98
99	public DuckDBDataChunk(IntPtr chunk) : base(false)	621✔
100	{
101	SetHandle(chunk);	621✔
102	}	621✔
103
104	protected override bool ReleaseHandle()
105	{
106	NativeMethods.DataChunks.DuckDBDestroyDataChunk(ref handle);	40,254✔
107	return true;	40,254✔
108	}
109	}
110
111	public class DuckDBValue() : SafeHandleZeroOrMinusOneIsInvalid(true), IDuckDBValueReader	11,579✔
112	{
113	private DuckDBValue[] childValues = [];	11,579✔
114
115	protected override bool ReleaseHandle()
116	{
117	foreach (var value in childValues)	43,670✔
118	{
119	value.Dispose();	10,265✔
120	}
121
122	NativeMethods.Value.DuckDBDestroyValue(ref handle);	11,570✔
123	return true;	11,570✔
124	}
125
126	internal void SetChildValues(DuckDBValue[] values)
127	{
128	childValues = values;	423✔
129	}	423✔
130
131	public bool IsNull() => NativeMethods.Value.DuckDBIsNullValue(this);	120✔
132
133	public T GetValue<T>()
134	{
135	var logicalType = NativeMethods.Value.DuckDBGetValueType(this);	177✔
136
137	//Logical type is part of the duckdb_value object and it shouldn't be released separately
138	//It will get released when the duckdb_value object is destroyed below.
139	var add = false;	177✔
140	logicalType.DangerousAddRef(ref add);	177✔
141
142	var duckDBType = NativeMethods.LogicalType.DuckDBGetTypeId(logicalType);	177✔
143
144	return duckDBType switch	177!
145	{	177✔
146	DuckDBType.Boolean => Cast(NativeMethods.Value.DuckDBGetBool(this)),	3✔
147		177✔
148	DuckDBType.TinyInt => Cast(NativeMethods.Value.DuckDBGetInt8(this)),	3✔
149	DuckDBType.SmallInt => Cast(NativeMethods.Value.DuckDBGetInt16(this)),	3✔
150	DuckDBType.Integer => Cast(NativeMethods.Value.DuckDBGetInt32(this)),	84✔
151	DuckDBType.BigInt => Cast(NativeMethods.Value.DuckDBGetInt64(this)),	3✔
152		177✔
153	DuckDBType.UnsignedTinyInt => Cast(NativeMethods.Value.DuckDBGetUInt8(this)),	3✔
154	DuckDBType.UnsignedSmallInt => Cast(NativeMethods.Value.DuckDBGetUInt16(this)),	3✔
155	DuckDBType.UnsignedInteger => Cast(NativeMethods.Value.DuckDBGetUInt32(this)),	3✔
156	DuckDBType.UnsignedBigInt => Cast(NativeMethods.Value.DuckDBGetUInt64(this)),	3✔
157		177✔
158	DuckDBType.Float => Cast(NativeMethods.Value.DuckDBGetFloat(this)),	3✔
159	DuckDBType.Double => Cast(NativeMethods.Value.DuckDBGetDouble(this)),	9✔
160		177✔
161	DuckDBType.Decimal => Cast(decimal.Parse(NativeMethods.Value.DuckDBGetVarchar(this), NumberStyles.Any, CultureInfo.InvariantCulture)),	6✔
162		177✔
163	DuckDBType.Uuid => Cast(new Guid(NativeMethods.Value.DuckDBGetVarchar(this))),	3✔
164		177✔
165	DuckDBType.HugeInt => Cast(NativeMethods.Value.DuckDBGetHugeInt(this).ToBigInteger()),	3✔
166	DuckDBType.UnsignedHugeInt => Cast(NativeMethods.Value.DuckDBGetUHugeInt(this).ToBigInteger()),	×
167		177✔
168	DuckDBType.Varchar => Cast(NativeMethods.Value.DuckDBGetVarchar(this)),	18✔
169		177✔
170	DuckDBType.Date => Cast((DateOnly)DuckDBDateOnly.FromDuckDBDate(NativeMethods.Value.DuckDBGetDate(this))),	3✔
171	DuckDBType.Time => Cast((TimeOnly)NativeMethods.DateTimeHelpers.DuckDBFromTime(NativeMethods.Value.DuckDBGetTime(this))),	3✔
172	DuckDBType.TimeTz => Cast(GetTimeTzValue()),	3✔
173	DuckDBType.Interval => Cast((TimeSpan)NativeMethods.Value.DuckDBGetInterval(this)),	3✔
174	DuckDBType.Timestamp => Cast(GetTimestampValue(NativeMethods.Value.DuckDBGetTimestamp(this),DuckDBType.Timestamp)),	3✔
175	DuckDBType.TimestampS => Cast(GetTimestampValue(NativeMethods.Value.DuckDBGetTimestampS(this), DuckDBType.TimestampS)),	3✔
176	DuckDBType.TimestampMs => Cast(GetTimestampValue(NativeMethods.Value.DuckDBGetTimestampMs(this), DuckDBType.TimestampMs)),	3✔
177	DuckDBType.TimestampNs => Cast(GetTimestampValue(NativeMethods.Value.DuckDBGetTimestampNs(this), DuckDBType.TimestampNs)),	3✔
178	DuckDBType.TimestampTz => Cast(GetTimestampValue(NativeMethods.Value.DuckDBGetTimestamp(this), DuckDBType.TimestampTz)),	3✔
179	_ => throw new NotImplementedException($"Cannot read value of type {typeof(T).FullName}")	×
180	};	177✔
181
182	[MethodImpl(MethodImplOptions.AggressiveInlining)]
183	static T Cast<TSource>(TSource value) => Unsafe.As<TSource, T>(ref value);	177✔
184	}
185
186	private DateTime GetTimestampValue(DuckDBTimestampStruct timestampStruct, DuckDBType duckDBType)
187	{
188	var additionalTicks = 0;	15✔
189
190	// The type-specific getters return values in their native units:
191	// - TimestampS: seconds
192	// - TimestampMs: milliseconds
193	// - TimestampNs: nanoseconds
194	// We need to convert to microseconds for DuckDBTimestamp.FromDuckDBTimestampStruct()
195	if (duckDBType == DuckDBType.TimestampNs)	15✔
196	{
197	additionalTicks = (int)(timestampStruct.Micros % 1000 / 100);	3✔
198	timestampStruct.Micros /= 1000;	3✔
199	}
200	if (duckDBType == DuckDBType.TimestampMs)	15✔
201	{
202	timestampStruct.Micros *= 1000;	3✔
203	}
204	if (duckDBType == DuckDBType.TimestampS)	15✔
205	{
206	timestampStruct.Micros *= 1000000;	3✔
207	}
208
209	var timestamp = DuckDBTimestamp.FromDuckDBTimestampStruct(timestampStruct);	15✔
210	return timestamp.ToDateTime().AddTicks(additionalTicks);	15✔
211	}
212
213	private DateTimeOffset GetTimeTzValue()
214	{
215	var timeTzStruct = NativeMethods.Value.DuckDBGetTimeTz(this);	3✔
216	var timeTz = NativeMethods.DateTimeHelpers.DuckDBFromTimeTz(timeTzStruct);	3✔
217	return new DateTimeOffset(timeTz.Time.ToDateTime(), TimeSpan.FromSeconds(timeTz.Offset));	3✔
218	}
219	}
220
221	public class DuckDBClientContext() : SafeHandleZeroOrMinusOneIsInvalid(true)	432✔
222	{
223	protected override bool ReleaseHandle()
224	{
225	NativeMethods.Startup.DuckDBDestroyClientContext(ref handle);	432✔
226	return true;	432✔
227	}
228
229	public ulong ConnectionId => NativeMethods.Startup.DuckDBClientContextGetConnectionId(this);	432✔
230	}

Giorgi / DuckDB.NET / 28103508122

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous