21843210681

Committed 09 Feb 2026 10:25PM UTC coverage: 85.34% (+0.04%) from 85.297%

Build # 21843210681

Build Type

push

github

Committed by

moconnell

Commit Message

revert: Microsoft.ApplicationInsights.WorkerService -> 2.23.0

Run Details

266 of 281 branches covered (94.66%)

Branch coverage included in aggregate %.

1818 of 2161 relevant lines covered (84.13%)

37.11 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.56

/src/YoloAbstractions/Extensions/DataFrameExtensions.cs

using System.Numerics;
using MathNet.Numerics.LinearAlgebra;
using Microsoft.Data.Analysis;

using static YoloAbstractions.NormalizationMethod;

namespace YoloAbstractions.Extensions;

public static class DataFrameExtensions
{
    public static DataFrame Normalize(
        this DataFrame df,
        NormalizationMethod method = None,
        int? quantiles = null)
    {
        if (method == None)
            return df;

        var result = df.Clone();

        foreach (var col in df.Columns)
        {
            if (col is not DoubleDataFrameColumn numeric)
                continue;

            result.Columns[col.Name] = numeric.Normalize(method, quantiles);
        }

        return result;
    }

    public static DoubleDataFrameColumn Normalize(this DoubleDataFrameColumn col, NormalizationMethod method, int? quantiles = null)
    {
        if (method == None)
            return col;

        if (method == CrossSectionalBins && (!quantiles.HasValue || quantiles <= 0))
        {
            throw new ArgumentOutOfRangeException(nameof(quantiles), quantiles, $"Quantiles must be a positive integer when using {CrossSectionalBins} normalization.");
        }

        var normalizedValues = method switch
        {
            CrossSectionalBins => col.NormalizeBins(quantiles!.Value),
            CrossSectionalZScore => col.NormalizeZScore(),
            MinMax => col.NormalizeMinMax(),
            Rank => col.NormalizeRank(),
            _ => throw new ArgumentOutOfRangeException(nameof(method), method, $"Unknown normalization method: {method}")
        };

        return new DoubleDataFrameColumn(col.Name, normalizedValues);
    }

    public static DoubleDataFrameColumn PointwiseDivide(this DoubleDataFrameColumn col, MathNet.Numerics.LinearAlgebra.Vector<double> divisor)
    {
        if (col.Length != divisor.Count)
            throw new ArgumentException($"Column length ({col.Length}) must match divisor length ({divisor.Count}).", nameof(divisor));

        var vec = MathNet.Numerics.LinearAlgebra.Vector<double>.Build.DenseOfArray([.. col.Select(x => x ?? double.NaN)]);
        var resultVec = vec.PointwiseDivide(divisor);

        return new DoubleDataFrameColumn(col.Name, resultVec);
    }

    // -------------------------------
    // Internal helpers (column-level)
    // -------------------------------
    internal static double[] NormalizeBins(
        this DoubleDataFrameColumn col,
        int quantiles)
    {
        var items = new List<(double Value, int Index)>();

        for (int i = 0; i < col.Length; i++)
        {
            var v = col[i];
            if (v.HasValue && !double.IsNaN(v.Value))
                items.Add((v.Value, i));
        }

        if (items.Count == 0)
            return [.. Enumerable.Repeat(double.NaN, (int)col.Length)];

        // qcut behaviour
        items.Sort((a, b) => a.Value.CompareTo(b.Value));

        int n = items.Count;
        var bins = new int[n];

        for (int k = 0; k < n; k++)
        {
            var b = (int)Math.Floor((k + 1) / (double)n * quantiles) - 1;
            if (b < 0) b = 0;
            if (b > quantiles - 1) b = quantiles - 1;
            bins[k] = b;
        }

        int maxBin = bins.Max();
        if (maxBin <= 0)
        {
            return EmptyArray(col, items, n);
        }

        var weights = new double[n];
        for (int k = 0; k < n; k++)
            weights[k] = 2.0 * (bins[k] / (double)maxBin) - 1.0;

        // L1 normalisation
        var denom = weights.Sum(w => Math.Abs(w));
        if (denom <= 0)
        {
            return EmptyArray(col, items, n);
        }

        for (int k = 0; k < n; k++)
            weights[k] /= denom;

        double[] result = [.. Enumerable.Repeat(double.NaN, (int)col.Length)];
        for (int k = 0; k < n; k++)
            result[items[k].Index] = weights[k];

        return result;

        static double[] EmptyArray(DoubleDataFrameColumn col, List<(double Value, int Index)> items, int n)
        {
            double[] result = [.. Enumerable.Repeat(double.NaN, (int)col.Length)];
            for (int k = 0; k < n; k++)
                result[items[k].Index] = 0.0;
            return result;
        }
    }

    internal static IEnumerable<double> NormalizeZScore(this DoubleDataFrameColumn col)
    {
        var values = col.Where(v => v.HasValue && !double.IsNaN(v.Value)).Select(v => v!.Value).ToArray();

        if (values.Length == 0)
            return col.Select(v => double.NaN);

        var mean = values.Average();
        var variance = values.Sum(v => Math.Pow(v - mean, 2)) / values.Length;
        var stdDev = Math.Sqrt(variance);

        if (stdDev < 1e-10) // Avoid division by zero for constant columns
            return col.Select(v => 0.0);

        return col.Select(v => v.HasValue && !double.IsNaN(v.Value)
            ? (v.Value - mean) / stdDev
            : double.NaN);
    }

    internal static IEnumerable<double> NormalizeMinMax(this DoubleDataFrameColumn col)
    {
        var values = col.Where(v => v.HasValue && !double.IsNaN(v.Value)).Select(v => v!.Value).ToArray();

        if (values.Length == 0)
            return col.Select(v => double.NaN);

        var min = values.Min();
        var max = values.Max();
        var range = max - min;

        if (range < 1e-10) // Avoid division by zero
            return col.Select(v => 0.0);

        return col.Select(v => v.HasValue && !double.IsNaN(v.Value)
            ? 2 * ((v.Value - min) / range) - 1 // Scale to [-1, 1]
            : double.NaN);
    }

    internal static IEnumerable<double> NormalizeRank(this DoubleDataFrameColumn col)
    {
        var values = col.Select((v, i) => (Value: v, Index: i)).ToArray();
        var validValues = values
            .Where(x => x.Value.HasValue && !double.IsNaN(x.Value.Value))
            .OrderBy(x => x.Value!.Value)
            .ToArray();

        if (validValues.Length == 0)
            return col.Select(v => double.NaN);

        var ranks = new Dictionary<int, double>();
        for (var i = 0; i < validValues.Length; i++)
        {
            // Scale to [-1, 1] range
            ranks[validValues[i].Index] = validValues.Length > 1
                ? 2.0 * i / (validValues.Length - 1) - 1
                : 0.0;
        }

        return values.Select(x => ranks.TryGetValue(x.Index, out var rank) ? rank : double.NaN);
    }
}

1	using System.Numerics;
2	using MathNet.Numerics.LinearAlgebra;
3	using Microsoft.Data.Analysis;
4
5	using static YoloAbstractions.NormalizationMethod;
6
7	namespace YoloAbstractions.Extensions;
8
9	public static class DataFrameExtensions
10	{
11	public static DataFrame Normalize(
12	this DataFrame df,
13	NormalizationMethod method = None,
14	int? quantiles = null)
15	{	4✔
16	if (method == None)	4✔
17	return df;	1✔
18
19	var result = df.Clone();	3✔
20
21	foreach (var col in df.Columns)	14✔
22	{	3✔
23	if (col is not DoubleDataFrameColumn numeric)	3✔
24	continue;	×
25
26	result.Columns[col.Name] = numeric.Normalize(method, quantiles);	3✔
27	}	2✔
28
29	return result;	2✔
30	}	3✔
31
32	public static DoubleDataFrameColumn Normalize(this DoubleDataFrameColumn col, NormalizationMethod method, int? quantiles = null)
33	{	21✔
34	if (method == None)	21✔
35	return col;	17✔
36
37	if (method == CrossSectionalBins && (!quantiles.HasValue \|\| quantiles <= 0))	4✔
38	{	1✔
39	throw new ArgumentOutOfRangeException(nameof(quantiles), quantiles, $"Quantiles must be a positive integer when using {CrossSectionalBins} normalization.");	1✔
40	}
41
42	var normalizedValues = method switch	3✔
43	{	3✔
44	CrossSectionalBins => col.NormalizeBins(quantiles!.Value),	2✔
45	CrossSectionalZScore => col.NormalizeZScore(),	×
46	MinMax => col.NormalizeMinMax(),	×
47	Rank => col.NormalizeRank(),	1✔
48	_ => throw new ArgumentOutOfRangeException(nameof(method), method, $"Unknown normalization method: {method}")	×
49	};	3✔
50
51	return new DoubleDataFrameColumn(col.Name, normalizedValues);	3✔
52	}	20✔
53
54	public static DoubleDataFrameColumn PointwiseDivide(this DoubleDataFrameColumn col, MathNet.Numerics.LinearAlgebra.Vector<double> divisor)
55	{	5✔
56	if (col.Length != divisor.Count)	5✔
57	throw new ArgumentException($"Column length ({col.Length}) must match divisor length ({divisor.Count}).", nameof(divisor));	1✔
58
59	var vec = MathNet.Numerics.LinearAlgebra.Vector<double>.Build.DenseOfArray([.. col.Select(x => x ?? double.NaN)]);	15✔
60	var resultVec = vec.PointwiseDivide(divisor);	4✔
61
62	return new DoubleDataFrameColumn(col.Name, resultVec);	4✔
63	}	4✔
64
65	// -------------------------------
66	// Internal helpers (column-level)
67	// -------------------------------
68	internal static double[] NormalizeBins(
69	this DoubleDataFrameColumn col,
70	int quantiles)
71	{	6✔
72	var items = new List<(double Value, int Index)>();	6✔
73
74	for (int i = 0; i < col.Length; i++)	46✔
75	{	17✔
76	var v = col[i];	17✔
77	if (v.HasValue && !double.IsNaN(v.Value))	17✔
78	items.Add((v.Value, i));	16✔
79	}	17✔
80
81	if (items.Count == 0)	6✔
82	return [.. Enumerable.Repeat(double.NaN, (int)col.Length)];	×
83
84	// qcut behaviour
85	items.Sort((a, b) => a.Value.CompareTo(b.Value));	22✔
86
87	int n = items.Count;	6✔
88	var bins = new int[n];	6✔
89
90	for (int k = 0; k < n; k++)	44✔
91	{	16✔
92	var b = (int)Math.Floor((k + 1) / (double)n * quantiles) - 1;	16✔
93	if (b < 0) b = 0;	17✔
94	if (b > quantiles - 1) b = quantiles - 1;	16✔
95	bins[k] = b;	16✔
96	}	16✔
97
98	int maxBin = bins.Max();	6✔
99	if (maxBin <= 0)	6✔
100	{	1✔
101	return EmptyArray(col, items, n);	1✔
102	}
103
104	var weights = new double[n];	5✔
105	for (int k = 0; k < n; k++)	38✔
106	weights[k] = 2.0 * (bins[k] / (double)maxBin) - 1.0;	14✔
107
108	// L1 normalisation
109	var denom = weights.Sum(w => Math.Abs(w));	19✔
110	if (denom <= 0)	5✔
111	{	×
112	return EmptyArray(col, items, n);	×
113	}
114
115	for (int k = 0; k < n; k++)	38✔
116	weights[k] /= denom;	14✔
117
118	double[] result = [.. Enumerable.Repeat(double.NaN, (int)col.Length)];	5✔
119	for (int k = 0; k < n; k++)	38✔
120	result[items[k].Index] = weights[k];	14✔
121
122	return result;	5✔
123
124	static double[] EmptyArray(DoubleDataFrameColumn col, List<(double Value, int Index)> items, int n)
125	{	1✔
126	double[] result = [.. Enumerable.Repeat(double.NaN, (int)col.Length)];	1✔
127	for (int k = 0; k < n; k++)	6✔
128	result[items[k].Index] = 0.0;	2✔
129	return result;	1✔
130	}	1✔
131	}	6✔
132
133	internal static IEnumerable<double> NormalizeZScore(this DoubleDataFrameColumn col)
134	{	2✔
135	var values = col.Where(v => v.HasValue && !double.IsNaN(v.Value)).Select(v => v!.Value).ToArray();	11✔
136
137	if (values.Length == 0)	2✔
138	return col.Select(v => double.NaN);	4✔
139
140	var mean = values.Average();	1✔
141	var variance = values.Sum(v => Math.Pow(v - mean, 2)) / values.Length;	4✔
142	var stdDev = Math.Sqrt(variance);	1✔
143
144	if (stdDev < 1e-10) // Avoid division by zero for constant columns	1✔
145	return col.Select(v => 0.0);	4✔
146
147	return col.Select(v => v.HasValue && !double.IsNaN(v.Value)	×
148	? (v.Value - mean) / stdDev	×
149	: double.NaN);	×
150	}	2✔
151
152	internal static IEnumerable<double> NormalizeMinMax(this DoubleDataFrameColumn col)
153	{	2✔
154	var values = col.Where(v => v.HasValue && !double.IsNaN(v.Value)).Select(v => v!.Value).ToArray();	16✔
155
156	if (values.Length == 0)	2✔
157	return col.Select(v => double.NaN);	×
158
159	var min = values.Min();	2✔
160	var max = values.Max();	2✔
161	var range = max - min;	2✔
162
163	if (range < 1e-10) // Avoid division by zero	2✔
164	return col.Select(v => 0.0);	4✔
165
166	return col.Select(v => v.HasValue && !double.IsNaN(v.Value)	5✔
167	? 2 * ((v.Value - min) / range) - 1 // Scale to [-1, 1]	5✔
168	: double.NaN);	5✔
169	}	2✔
170
171	internal static IEnumerable<double> NormalizeRank(this DoubleDataFrameColumn col)
172	{	3✔
173	var values = col.Select((v, i) => (Value: v, Index: i)).ToArray();	11✔
174	var validValues = values	3✔
175	.Where(x => x.Value.HasValue && !double.IsNaN(x.Value.Value))	8✔
176	.OrderBy(x => x.Value!.Value)	6✔
177	.ToArray();	3✔
178
179	if (validValues.Length == 0)	3✔
180	return col.Select(v => double.NaN);	×
181
182	var ranks = new Dictionary<int, double>();	3✔
183	for (var i = 0; i < validValues.Length; i++)	20✔
184	{	7✔
185	// Scale to [-1, 1] range
186	ranks[validValues[i].Index] = validValues.Length > 1	7✔
187	? 2.0 * i / (validValues.Length - 1) - 1	7✔
188	: 0.0;	7✔
189	}	7✔
190
191	return values.Select(x => ranks.TryGetValue(x.Index, out var rank) ? rank : double.NaN);	11✔
192	}	3✔
193	}

moconnell / yolo / 21843210681

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous