alan-turing-institute / MLJTuning.jl / 384

Committed 13 Sep 2020 - 20:07 coverage: 88.706% (+1.0%) from 87.744%

Build # 384

Build Type

Pull #75

travis-ci-com

Committed by

web-flow

Commit Message

Merge b3f688ca4 into 9ea80763d

Pull Request Pull Request #75: Allow user to specify a selection heuristic

Run Details

80 of 85 new or added lines in 7 files covered. (94.12%)

377 of 425 relevant lines covered (88.71%)

809.44 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.74

/src/selection_heuristics.jl

abstract type SelectionHeuristic end

## HELPERS

signature(measure) =
    if orientation(measure) == :loss
        1
    elseif orientation(measure) == :score
        -1
    else
        0
    end
measure_adjusted_weights(weights, measures) =
    if weights isa Nothing
        vcat([signature(measures[1]), ], zeros(length(measures) - 1))
    else
        length(weights) == length(measures) ||
            throw(DimensionMismatch(
                "`OptimizeAggregatedMeasurement` heuristic "*
                "is being applied to a list of measures whose length "*
                "differs from that of the specified `weights`. "))
        signature.(measures) .* weights
    end


## OPTIMIZE AGGREGATED MEASURE

"""
    NaiveSelection(; weights=nothing)

Construct a common selection heuristic for use with `TunedModel` instances
which only considers measurements aggregated over all samples (folds)
in resampling.

For each entry in the tuning history, one defines a penalty equal to
the evaluations of the `measure` specified in the `TunedModel`
instance, aggregated over all samples, and multiplied by `-1` if `measure`
is a `:score`, and `+`` if it is a loss. The heuristic declares as
"best" (optimal) the model whose corresponding entry has the lowest
penalty.

If `measure` is a vector, then the first element is used, unless
per-measure `weights` are explicitly specified. Weights associated
with measures that are neither `:loss` nor `:score` are reset to zero.

"""
struct NaiveSelection <: SelectionHeuristic
    weights::Union{Nothing, Vector{Real}}
end
NaiveSelection(; weights=nothing) =
    NaiveSelection(weights)


function best(heuristic::NaiveSelection, history)
    first_entry = history[1]
    measures = first_entry.measure
    weights = measure_adjusted_weights(heuristic.weights, measures)
    measurements = [weights'*(h.measurement) for h in history]
    measure = first(history).measure[1]
    if orientation(measure) == :score
                               measurements = -measurements

    end
    best_index = argmin(measurements)
    return history[best_index]
end

MLJTuning.supports_heuristic(::Any, ::NaiveSelection) =
    true

1	abstract type SelectionHeuristic end
2
3	## HELPERS
4
5	signature(measure) =	102×
6	if orientation(measure) == :loss
7	1	69×
8	elseif orientation(measure) == :score	9×
9	-1	6×
10	else
11	0	4×
12	end
13	measure_adjusted_weights(weights, measures) =	194×
14	if weights isa Nothing
15	vcat([signature(measures[1]), ], zeros(length(measures) - 1))	237×
16	else
17	length(weights) == length(measures) \|\|	6×
18	throw(DimensionMismatch(
19	"`OptimizeAggregatedMeasurement` heuristic "*
20	"is being applied to a list of measures whose length "*
21	"differs from that of the specified `weights`. "))
22	signature.(measures) .* weights	2×
23	end
24
25
26	## OPTIMIZE AGGREGATED MEASURE
27
28	"""
29	NaiveSelection(; weights=nothing)
30
31	Construct a common selection heuristic for use with `TunedModel` instances
32	which only considers measurements aggregated over all samples (folds)
33	in resampling.
34
35	For each entry in the tuning history, one defines a penalty equal to
36	the evaluations of the `measure` specified in the `TunedModel`
37	instance, aggregated over all samples, and multiplied by `-1` if `measure`
38	is a `:score`, and `+`` if it is a loss. The heuristic declares as
39	"best" (optimal) the model whose corresponding entry has the lowest
40	penalty.
41
42	If `measure` is a vector, then the first element is used, unless
43	per-measure `weights` are explicitly specified. Weights associated
44	with measures that are neither `:loss` nor `:score` are reset to zero.
45
46	"""
47	struct NaiveSelection <: SelectionHeuristic
48	weights::Union{Nothing, Vector{Real}}	118×
49	end
50	NaiveSelection(; weights=nothing) =	197×
51	NaiveSelection(weights)
52
53
54	function best(heuristic::NaiveSelection, history)
55	first_entry = history[1]	332×
56	measures = first_entry.measure
57	weights = measure_adjusted_weights(heuristic.weights, measures)	190×
58	measurements = [weights'*(h.measurement) for h in history]	189×
59	measure = first(history).measure[1]
60	if orientation(measure) == :score	87×
61	measurements = -measurements	2×
62
63	end
64	best_index = argmin(measurements)	186×
65	return history[best_index]	189×
66	end
67
NEW 68	MLJTuning.supports_heuristic(::Any, ::NaiveSelection) =	!
69	true

alan-turing-institute / MLJTuning.jl / 384

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous