10762504699

Committed 08 Sep 2024 07:21PM UTC coverage: 89.795% (-0.2%) from 90.033%

Build # 10762504699

Build Type

push

github

Committed by

Giorgi

Commit Message

Merge branch 'develop' into nightly-builds

Run Details

920 of 1056 branches covered (87.12%)

Branch coverage included in aggregate %.

58 of 68 new or added lines in 8 files covered. (85.29%)

1 existing line in 1 file now uncovered.

1887 of 2070 relevant lines covered (91.16%)

896537.08 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.04

/DuckDB.NET.Data/Internal/Writer/ListVectorDataWriter.cs

﻿using DuckDB.NET.Native;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Numerics;

namespace DuckDB.NET.Data.Internal.Writer;

internal sealed unsafe class ListVectorDataWriter : VectorDataWriterBase
{
    private ulong offset = 0;
    private readonly ulong arraySize;
    private readonly DuckDBLogicalType childType;
    private readonly VectorDataWriterBase listItemWriter;

    public bool IsList => ColumnType == DuckDBType.List;
    private ulong vectorReservedSize = DuckDBGlobalData.VectorSize;

    public ListVectorDataWriter(IntPtr vector, void* vectorData, DuckDBType columnType, DuckDBLogicalType logicalType) : base(vector, vectorData, columnType)
    {
        childType = IsList ? NativeMethods.LogicalType.DuckDBListTypeChildType(logicalType) : NativeMethods.LogicalType.DuckDBArrayTypeChildType(logicalType);
        var childVector = IsList ? NativeMethods.Vectors.DuckDBListVectorGetChild(vector) : NativeMethods.Vectors.DuckDBArrayVectorGetChild(vector);

        arraySize = IsList ? 0 : (ulong)NativeMethods.LogicalType.DuckDBArrayVectorGetSize(logicalType);
        listItemWriter = VectorDataWriterFactory.CreateWriter(childVector, childType);
    }

    internal override bool AppendCollection(ICollection value, ulong rowIndex)
    {
        var count = (ulong)value.Count;

        ResizeVector(rowIndex % DuckDBGlobalData.VectorSize, count);

        _ = value switch
        {
            IEnumerable<bool> items => WriteItems(items),
            IEnumerable<bool?> items => WriteItems(items),

            IEnumerable<sbyte> items => WriteItems(items),
            IEnumerable<sbyte?> items => WriteItems(items),
            IEnumerable<short> items => WriteItems(items),
            IEnumerable<short?> items => WriteItems(items),
            IEnumerable<int> items => WriteItems(items),
            IEnumerable<int?> items => WriteItems(items),
            IEnumerable<long> items => WriteItems(items),
            IEnumerable<long?> items => WriteItems(items),
            IEnumerable<byte> items => WriteItems(items),
            IEnumerable<byte?> items => WriteItems(items),
            IEnumerable<ushort> items => WriteItems(items),
            IEnumerable<ushort?> items => WriteItems(items),
            IEnumerable<uint> items => WriteItems(items),
            IEnumerable<uint?> items => WriteItems(items),
            IEnumerable<ulong> items => WriteItems(items),
            IEnumerable<ulong?> items => WriteItems(items),

            IEnumerable<float> items => WriteItems(items),
            IEnumerable<float?> items => WriteItems(items),
            IEnumerable<double> items => WriteItems(items),
            IEnumerable<double?> items => WriteItems(items),

            IEnumerable<decimal> items => WriteItems(items),
            IEnumerable<decimal?> items => WriteItems(items),
            IEnumerable<BigInteger> items => WriteItems(items),
            IEnumerable<BigInteger?> items => WriteItems(items),

            IEnumerable<string> items => WriteItems(items),
            IEnumerable<Guid> items => WriteItems(items),
            IEnumerable<Guid?> items => WriteItems(items),
            IEnumerable<DateTime> items => WriteItems(items),
            IEnumerable<DateTime?> items => WriteItems(items),
            IEnumerable<TimeSpan> items => WriteItems(items),
            IEnumerable<TimeSpan?> items => WriteItems(items),
            IEnumerable<DuckDBDateOnly> items => WriteItems(items),
            IEnumerable<DuckDBDateOnly?> items => WriteItems(items),
            IEnumerable<DuckDBTimeOnly> items => WriteItems(items),
            IEnumerable<DuckDBTimeOnly?> items => WriteItems(items),
#if NET6_0_OR_GREATER
            IEnumerable<DateOnly> items => WriteItems(items),
            IEnumerable<DateOnly?> items => WriteItems(items),
            IEnumerable<TimeOnly> items => WriteItems(items),
            IEnumerable<TimeOnly?> items => WriteItems(items),
#endif
            IEnumerable<DateTimeOffset> items => WriteItems(items),
            IEnumerable<DateTimeOffset?> items => WriteItems(items),
            IEnumerable<object> items => WriteItems(items),

            _ => WriteItemsFallback(value),
        };

        var duckDBListEntry = new DuckDBListEntry(offset, count);
        var result = !IsList || AppendValueInternal(duckDBListEntry, rowIndex);

        offset += count;

        return result;

        int WriteItems<T>(IEnumerable<T> items)
        {
            if (IsList == false && count != arraySize)
            {
                throw new InvalidOperationException($"Column has Array size of {arraySize} but the specified value has size of {count}");
            }

            var index = 0ul;

            foreach (var item in items)
            {
                listItemWriter.WriteValue(item, offset + (index++));
            }

            return 0;
        }

        int WriteItemsFallback(IEnumerable items)
        {
            if (IsList == false && count != arraySize)
            {
                throw new InvalidOperationException($"Column has Array size of {arraySize} but the specified value has size of {count}");
            }

            var index = 0ul;

            foreach (var item in items)
            {
                listItemWriter.WriteValue(item, offset + (index++));
            }

            return 0;
        }
    }

    private void ResizeVector(ulong rowIndex, ulong count)
    {
        //If writing to a list column we need to make sure that enough space is allocated. Not needed for Arrays as DuckDB does it for us.
        if (!IsList || offset + count <= vectorReservedSize) return;

        var factor = 2d;

        if (rowIndex > DuckDBGlobalData.VectorSize * 0.25 && rowIndex < DuckDBGlobalData.VectorSize * 0.5)
        {
            factor = 1.75;
        }

        if (rowIndex > DuckDBGlobalData.VectorSize * 0.5 && rowIndex < DuckDBGlobalData.VectorSize * 0.75)
        {
            factor = 1.5;
        }

        if (rowIndex > DuckDBGlobalData.VectorSize * 0.75)
        {
            factor = 1.25;
        }

        vectorReservedSize = (ulong)Math.Max(vectorReservedSize * factor, offset + count);
        var state = NativeMethods.Vectors.DuckDBListVectorReserve(Vector, vectorReservedSize);

        if (!state.IsSuccess())
        {
            throw new DuckDBException($"Failed to reserve {vectorReservedSize} for the list vector");
        }

        listItemWriter.InitializerWriter();
    }

    public override void Dispose()
    {
        listItemWriter.Dispose();
        childType.Dispose();
    }
}

1	using DuckDB.NET.Native;
2	using System;
3	using System.Collections;
4	using System.Collections.Generic;
5	using System.Numerics;
6
7	namespace DuckDB.NET.Data.Internal.Writer;
8
9	internal sealed unsafe class ListVectorDataWriter : VectorDataWriterBase
10	{
11	private ulong offset = 0;
12	private readonly ulong arraySize;
13	private readonly DuckDBLogicalType childType;
14	private readonly VectorDataWriterBase listItemWriter;
15
16	public bool IsList => ColumnType == DuckDBType.List;	2,851,818✔
17	private ulong vectorReservedSize = DuckDBGlobalData.VectorSize;	315✔
18
19	public ListVectorDataWriter(IntPtr vector, void* vectorData, DuckDBType columnType, DuckDBLogicalType logicalType) : base(vector, vectorData, columnType)	315✔
20	{
21	childType = IsList ? NativeMethods.LogicalType.DuckDBListTypeChildType(logicalType) : NativeMethods.LogicalType.DuckDBArrayTypeChildType(logicalType);	315✔
22	var childVector = IsList ? NativeMethods.Vectors.DuckDBListVectorGetChild(vector) : NativeMethods.Vectors.DuckDBArrayVectorGetChild(vector);	315✔
23
24	arraySize = IsList ? 0 : (ulong)NativeMethods.LogicalType.DuckDBArrayVectorGetSize(logicalType);	315✔
25	listItemWriter = VectorDataWriterFactory.CreateWriter(childVector, childType);	315✔
26	}	315✔
27
28	internal override bool AppendCollection(ICollection value, ulong rowIndex)
29	{
30	var count = (ulong)value.Count;	950,293✔
31
32	ResizeVector(rowIndex % DuckDBGlobalData.VectorSize, count);	950,293✔
33
34	_ = value switch	950,293!
35	{	950,293✔
36	IEnumerable<bool> items => WriteItems(items),	22,027✔
37	IEnumerable<bool?> items => WriteItems(items),	22,085✔
38		950,293✔
39	IEnumerable<sbyte> items => WriteItems(items),	21,957✔
40	IEnumerable<sbyte?> items => WriteItems(items),	22,104✔
41	IEnumerable<short> items => WriteItems(items),	22,037✔
42	IEnumerable<short?> items => WriteItems(items),	21,882✔
43	IEnumerable<int> items => WriteItems(items),	44,201✔
44	IEnumerable<int?> items => WriteItems(items),	22,053✔
45	IEnumerable<long> items => WriteItems(items),	21,832✔
46	IEnumerable<long?> items => WriteItems(items),	21,936✔
47	IEnumerable<byte> items => WriteItems(items),	21,713✔
48	IEnumerable<byte?> items => WriteItems(items),	21,960✔
49	IEnumerable<ushort> items => WriteItems(items),	21,804✔
50	IEnumerable<ushort?> items => WriteItems(items),	22,091✔
51	IEnumerable<uint> items => WriteItems(items),	21,765✔
52	IEnumerable<uint?> items => WriteItems(items),	22,011✔
53	IEnumerable<ulong> items => WriteItems(items),	21,909✔
54	IEnumerable<ulong?> items => WriteItems(items),	21,675✔
55		950,293✔
56	IEnumerable<float> items => WriteItems(items),	22,028✔
57	IEnumerable<float?> items => WriteItems(items),	22,041✔
58	IEnumerable<double> items => WriteItems(items),	21,996✔
59	IEnumerable<double?> items => WriteItems(items),	21,908✔
60		950,293✔
61	IEnumerable<decimal> items => WriteItems(items),	21,881✔
62	IEnumerable<decimal?> items => WriteItems(items),	21,962✔
63	IEnumerable<BigInteger> items => WriteItems(items),	43,968✔
64	IEnumerable<BigInteger?> items => WriteItems(items),	×
65		950,293✔
66	IEnumerable<string> items => WriteItems(items),	44,043✔
67	IEnumerable<Guid> items => WriteItems(items),	21,888✔
68	IEnumerable<Guid?> items => WriteItems(items),	21,806✔
69	IEnumerable<DateTime> items => WriteItems(items),	21,949✔
70	IEnumerable<DateTime?> items => WriteItems(items),	×
71	IEnumerable<TimeSpan> items => WriteItems(items),	21,816✔
72	IEnumerable<TimeSpan?> items => WriteItems(items),	×
73	IEnumerable<DuckDBDateOnly> items => WriteItems(items),	×
74	IEnumerable<DuckDBDateOnly?> items => WriteItems(items),	×
75	IEnumerable<DuckDBTimeOnly> items => WriteItems(items),	×
76	IEnumerable<DuckDBTimeOnly?> items => WriteItems(items),	×
77	#if NET6_0_OR_GREATER	950,293✔
78	IEnumerable<DateOnly> items => WriteItems(items),	×
79	IEnumerable<DateOnly?> items => WriteItems(items),	×
80	IEnumerable<TimeOnly> items => WriteItems(items),	×
81	IEnumerable<TimeOnly?> items => WriteItems(items),	×
82	#endif	950,293✔
83	IEnumerable<DateTimeOffset> items => WriteItems(items),	×
84	IEnumerable<DateTimeOffset?> items => WriteItems(items),	×
85	IEnumerable<object> items => WriteItems(items),	204,000✔
86		950,293✔
87	_ => WriteItemsFallback(value),	21,965✔
88	};	950,293✔
89
90	var duckDBListEntry = new DuckDBListEntry(offset, count);	950,287✔
91	var result = !IsList \|\| AppendValueInternal(duckDBListEntry, rowIndex);	950,287✔
92
93	offset += count;	950,287✔
94
95	return result;	950,287✔
96
97	int WriteItems<T>(IEnumerable<T> items)
98	{
99	if (IsList == false && count != arraySize)	928,328✔
100	{
101	throw new InvalidOperationException($"Column has Array size of {arraySize} but the specified value has size of {count}");	6✔
102	}
103
104	var index = 0ul;	928,322✔
105
106	foreach (var item in items)	51,219,770✔
107	{
108	listItemWriter.WriteValue(item, offset + (index++));	24,681,563✔
109	}
110
111	return 0;	928,322✔
112	}
113
114	int WriteItemsFallback(IEnumerable items)
115	{
116	if (IsList == false && count != arraySize)	21,965!
117	{
NEW 118	throw new InvalidOperationException($"Column has Array size of {arraySize} but the specified value has size of {count}");	×
119	}
120
121	var index = 0ul;	21,965✔
122
123	foreach (var item in items)	1,562,818✔
124	{
125	listItemWriter.WriteValue(item, offset + (index++));	759,444✔
126	}
127
128	return 0;	21,965✔
129	}
130	}
131
132	private void ResizeVector(ulong rowIndex, ulong count)
133	{
134	//If writing to a list column we need to make sure that enough space is allocated. Not needed for Arrays as DuckDB does it for us.
135	if (!IsList \|\| offset + count <= vectorReservedSize) return;	1,898,944✔
136
137	var factor = 2d;	1,642✔
138
139	if (rowIndex > DuckDBGlobalData.VectorSize * 0.25 && rowIndex < DuckDBGlobalData.VectorSize * 0.5)	1,642✔
140	{
141	factor = 1.75;	319✔
142	}
143
144	if (rowIndex > DuckDBGlobalData.VectorSize * 0.5 && rowIndex < DuckDBGlobalData.VectorSize * 0.75)	1,642✔
145	{
146	factor = 1.5;	289✔
147	}
148
149	if (rowIndex > DuckDBGlobalData.VectorSize * 0.75)	1,642✔
150	{
151	factor = 1.25;	147✔
152	}
153
154	vectorReservedSize = (ulong)Math.Max(vectorReservedSize * factor, offset + count);	1,642✔
155	var state = NativeMethods.Vectors.DuckDBListVectorReserve(Vector, vectorReservedSize);	1,642✔
156
157	if (!state.IsSuccess())	1,642!
158	{
159	throw new DuckDBException($"Failed to reserve {vectorReservedSize} for the list vector");	×
160	}
161
162	listItemWriter.InitializerWriter();	1,642✔
163	}	1,642✔
164
165	public override void Dispose()
166	{
167	listItemWriter.Dispose();	306✔
168	childType.Dispose();	306✔
169	}	306✔
170	}

Giorgi / DuckDB.NET / 10762504699

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous