alan-turing-institute / MLJTuning.jl / 388

## TYPES AND CONSTRUCTOR

mutable struct DeterministicTunedModel{T,M<:Deterministic} <: MLJBase.Deterministic

    tuning::T  # tuning strategy

    resampling # resampling strategy

    measure

    weights::Union{Nothing,Vector{<:Real}}

    operation

    range

    selection_heuristic

    train_best::Bool

    repeats::Int

    n::Union{Int,Nothing}

    acceleration::AbstractResource

    acceleration_resampling::AbstractResource

    check_measure::Bool

end

mutable struct ProbabilisticTunedModel{T,M<:Probabilistic} <: MLJBase.Probabilistic

    tuning::T  # tuning strategy

    resampling # resampling strategy

    measure

    weights::Union{Nothing,AbstractVector{<:Real}}

    operation

    range

    selection_heuristic

    train_best::Bool

    repeats::Int

    n::Union{Int,Nothing}

    acceleration::AbstractResource

    acceleration_resampling::AbstractResource

    check_measure::Bool

end

const EitherTunedModel{T,M} =

    Union{DeterministicTunedModel{T,M},ProbabilisticTunedModel{T,M}}

#todo update:

"""

    tuned_model = TunedModel(; model=nothing,

                             tuning=Grid(),

                             resampling=Holdout(),

                             measure=nothing,

                             weights=nothing,

                             repeats=1,

                             operation=predict,

                             range=nothing,

                             selection_heuristic=NaiveSelection(),

                             n=default_n(tuning, range),

                             train_best=true,

                             acceleration=default_resource(),

                             acceleration_resampling=CPU1(),

                             check_measure=true)

Construct a model wrapper for hyperparameter optimization of a

supervised learner.

Calling `fit!(mach)` on a machine `mach=machine(tuned_model, X, y)` or

`mach=machine(tuned_model, X, y, w)` will:

- Instigate a search, over clones of `model`, with the hyperparameter

  mutations specified by `range`, for a model optimizing the specified

  `measure`, using performance evaluations carried out using the

  specified `tuning` strategy and `resampling` strategy.

- Fit an internal machine, based on the optimal model

  `fitted_params(mach).best_model`, wrapping the optimal `model`

  object in *all* the provided data `X`, `y`(, `w`). Calling

  `predict(mach, Xnew)` then returns predictions on `Xnew` of this

  internal machine. The final train can be supressed by setting

  `train_best=false`.

The `range` objects supported depend on the `tuning` strategy

specified. Query the `strategy` docstring for details. To optimize

over an explicit list `v` of models of the same type, use

`strategy=Explicit()` and specify `model=v[1]` and `range=v`.

The number of models searched is specified by `n`. If unspecified,

then `MLJTuning.default_n(tuning, range)` is used. When `n` is

increased and `fit!(mach)` called again, the old search history is

re-instated and the search continues where it left off.

If `measure` supports weights (`supports_weights(measure) == true`)

then any `weights` specified will be passed to the measure. If more

than one `measure` is specified, then only the first is optimized

(unless `strategy` is multi-objective) but the performance against

every measure specified will be computed and reported in

`report(mach).best_performance` and other relevant attributes of the

generated report.

Specify `repeats > 1` for repeated resampling per model

evaluation. See [`evaluate!`](@ref) options for details.

*Important.* If a custom `measure` is used, and the measure is

a score, rather than a loss, be sure to check that

`MLJ.orientation(measure) == :score` to ensure maximization of the

measure, rather than minimization. Override an incorrect value with

`MLJ.orientation(::typeof(measure)) = :score`.

In the case of two-parameter tuning, a Plots.jl plot of performance

estimates is returned by `plot(mach)` or `heatmap(mach)`.

Once a tuning machine `mach` has bee trained as above, then

`fitted_params(mach)` has these keys/values:

key                 | value

--------------------|--------------------------------------------------

`best_model`        | optimal model instance

`best_fitted_params`| learned parameters of the optimal model

The named tuple `report(mach)` includes these keys/values:

key                 | value

--------------------|--------------------------------------------------

`best_model`        | optimal model instance

`best_history_entry`| corresponding entry in the history, including performance estimate

`best_report`       | report generated by fitting the optimal model to all data

`history`           | tuning strategy-specific history of all evaluations

plus other key/value pairs specific to the `tuning` strategy.

### Summary of key-word arguments

- `model`: `Supervised` model prototype that is cloned and mutated to

  generate models for evaluation

- `tuning=Grid()`: tuning strategy to be applied (eg, `RandomSearch()`)

- `resampling=Holdout()`: resampling strategy (eg, `Holdout()`, `CV()`),

  `StratifiedCV()`) to be applied in performance evaluations

- `measure`: measure or measures to be applied in performance

  evaluations; only the first used in optimization (unless the

  strategy is multi-objective) but all reported to the history

- `weights`: sample weights to be passed the measure(s) in performance

  evaluations, if supported.

- `repeats=1`: for generating train/test sets multiple times in

  resampling; see [`evaluate!`](@ref) for details

- `operation=predict`: operation to be applied to each fitted model;

  usually `predict` but `predict_mean`, `predict_median` or

  `predict_mode` can be used for `Probabilistic` models, if

  the specified measures are `Deterministic`

- `range`: range object; tuning strategy documentation describes

  supported types

- `selection_heuristic`: the rule determining how the best model is

  decided. According to the default heuristic,

  `NaiveSelection()`, `measure` (or the first

  element of `measure`) is evaluated for each resample and these

  per-fold measurements are aggregrated. The model with the lowest

  (resp. highest) aggregate is chosen if the measure is a `:loss`

  (resp. a `:score`).

- `n`: number of iterations (ie, models to be evaluated); set by

  tuning strategy if left unspecified

- `train_best=true`: whether to train the optimal model

- `acceleration=default_resource()`: mode of parallelization for

  tuning strategies that support this

- `acceleration_resampling=CPU1()`: mode of parallelization for

  resampling

- `check_measure`: whether to check `measure` is compatible with the

  specified `model` and `operation`)

"""

function TunedModel(; model=nothing,

                    tuning=Grid(),

                    resampling=MLJBase.Holdout(),

                    measures=nothing,

                    measure=measures,

                    weights=nothing,

                    operation=predict,

                    ranges=nothing,

                    range=ranges,

                    selection_heuristic=NaiveSelection(),

                    train_best=true,

                    repeats=1,

                    n=nothing,

                    acceleration=default_resource(),

                    acceleration_resampling=CPU1(),

                    check_measure=true)

                              "If `tuning=Explicit()`, any model in the "*

                              "range will do. ")

    if model isa Deterministic

        tuned_model = DeterministicTunedModel(model, tuning, resampling,

                                              measure, weights, operation,

                                              range, selection_heuristic,

                                              train_best, repeats, n,

                                              acceleration,

                                              acceleration_resampling,

                                              check_measure)

    elseif model isa Probabilistic

        tuned_model = ProbabilisticTunedModel(model, tuning, resampling,

                                              measure, weights, operation,

                                              range, selection_heuristic,

                                              train_best, repeats, n,

                                              acceleration,

                                              acceleration_resampling,

                                              check_measure)

    else

              "model types supported.")

    end

end

function MLJBase.clean!(tuned_model::EitherTunedModel)

                  "You must specify `measure=...`. ")

        else

            "Setting measure=$(tuned_model.measure). "

        end

    end

        "is not supported by $(tuned_model.tuning). Resetting to "*

        "`NaiveSelectionment()`."

        tuned_model.selection_heuristic = NaiveSelection()

    end

        tuned_model.acceleration_resampling isa CPUProcesses)

        message *=

        "The combination acceleration=$(tuned_model.acceleration) and"*

        " acceleration_resampling=$(tuned_model.acceleration_resampling) is"*

        "  not generally optimal. You may want to consider setting"*

        " `acceleration = CPUProcesses()` and"*

        " `acceleration_resampling = CPUThreads()`."

    end

        tuned_model.acceleration_resampling isa CPUProcesses)

        message *=

        "The combination acceleration=$(tuned_model.acceleration) and"*

        " acceleration_resampling=$(tuned_model.acceleration_resampling) isn't"*

        " supported. \n Resetting to"*

        " `acceleration = CPUProcesses()` and"*

        " `acceleration_resampling = CPUThreads()`."

        tuned_model.acceleration = CPUProcesses()

        tuned_model.acceleration_resampling = CPUThreads()

    end

        _process_accel_settings(tuned_model.acceleration)

end

## FIT AND UPDATE METHODS

# A *metamodel* is either a `Model` instance, `model`, or a tuple

# `(model, s)`, where `s` is extra data associated with `model` that

# the tuning strategy implementation wants available to the `result`

# method for recording in the history.

# returns a (model, result) pair for the history:

function event(metamodel,

               resampling_machine,

               verbosity,

               tuning,

               history,

               state)

    metadata = _last(metamodel)

    resampling_machine.model.model = model

    E = evaluate(resampling_machine)

              measure     = E.measure,

              measurement = E.measurement,

              per_fold    = E.per_fold,

              metadata    = metadata)

    end

    end

end

function assemble_events(metamodels,

                         resampling_machine,

                         verbosity,

                         tuning,

                         history,

                         state,

                         acceleration::CPU1)

         dt = 0,

         desc = "Evaluating over $(n_metamodels) metamodels: ",

         barglyphs = BarGlyphs("[=> ]"),

         barlen = 25,

         color = :yellow)

        r = event(m, resampling_machine, verbosity, tuning, history, state)

        verbosity < 1 || begin

                  p.counter += 1

                  ProgressMeter.updateProgress!(p)

                end

        r

      end

end

function assemble_events(metamodels,

                         resampling_machine,

                         verbosity,

                         tuning,

                         history,

                         state,

                         acceleration::CPUProcesses)

    entries = @sync begin

                     dt = 0,

                     desc = "Evaluating over $n_metamodels metamodels: ",

                     barglyphs = BarGlyphs("[=> ]"),

                     barlen = 25,

                     color = :yellow)

        # printing the progress bar

            update!(p,0)

            @async while take!(channel)

            end

        end

            end

            r

        end

        ret

    end

end

@static if VERSION >= v"1.3.0-DEV.573"

# one machine for each thread; cycle through available threads:

function assemble_events(metamodels,

                         resampling_machine,

                         verbosity,

                         tuning,

                         history,

                         state,

                         acceleration::CPUThreads)

                         resampling_machine,

                         verbosity,

                         tuning,

                         history,

                         state,

                         CPU1())

   end

    #tasks = Vector{Task}(undef, length(partitions))

         dt = 0,

         desc = "Evaluating over $(n_metamodels) metamodels: ",

         barglyphs = BarGlyphs("[=> ]"),

         barlen = 25,

         color = :yellow)

        # printing the progress bar

            end

        end

        # One tresampling_machine per task

                 [machine(Resampler(

                     model= resampling_machine.model.model,

                     resampling    = resampling_machine.model.resampling,

                     measure       = resampling_machine.model.measure,

                     weights       = resampling_machine.model.weights,

                     operation     = resampling_machine.model.operation,

                     check_measure = resampling_machine.model.check_measure,

                     repeats       = resampling_machine.model.repeats,

                     acceleration  = resampling_machine.model.acceleration),

                          resampling_machine.args...) for _ in 2:length(partitions)]...]

                    r = event(m, machs[i],

                              verbosity, tuning, history, state)

                    verbosity < 1 || put!(ch, true)

                    r

                end

            end

        end

    end

end

end # of if VERSION ...

# history is intialized to `nothing` because it's type is not known.

# builds on an existing `history` until the length is `n` or the model

# supply is exhausted (method shared by `fit` and `update`). Returns

# the bigger history:

function build(history,

               n,

               tuning,

               model,

               state,

               verbosity,

               acceleration,

               resampling_machine)

    models_exhausted = false

                                    model,

                                    history,

                                    state,

                                    n - j,

                                    verbosity)

            @info "Only $j (of $n) models evaluated.\n"*

            "Model supply exhausted. "

        end

                                   resampling_machine,

                                   verbosity,

                                   tuning,

                                   history,

                                   state,

                                   acceleration)

    end

end

# given complete history, pick out best model, fit it on all data and

# generate report and cache (meta_state):

function finalize(tuned_model, history, state, verbosity, rm, data...)

    tuning = tuned_model.tuning

        delete(entry, :metadata)

    end

               best_history_entry = best_history_entry,

               history            = user_history)

    else

    end

end

function MLJBase.fit(tuned_model::EitherTunedModel{T,M},

                     verbosity::Integer, data...) where {T,M}

    model = tuned_model.model

    range = tuned_model.range

        default_n(tuning, range) : tuned_model.n

    # instantiate resampler (`model` to be replaced with mutated

    # clones during iteration below):

                          resampling    = tuned_model.resampling,

                          measure       = tuned_model.measure,

                          weights       = tuned_model.weights,

                          operation     = tuned_model.operation,

                          check_measure = tuned_model.check_measure,

                          repeats       = tuned_model.repeats,

                          acceleration  = tuned_model.acceleration_resampling)

                           verbosity, acceleration, resampling_machine)

    rm = resampling_machine

end

function MLJBase.update(tuned_model::EitherTunedModel,

                        verbosity::Integer,

                        old_fitresult, old_meta_state, data...)

    acceleration = tuned_model.acceleration

    tuning = tuned_model.tuning

    range = tuned_model.range

    model = tuned_model.model

    # exclamation points are for values actually used rather than

    # stored:

        default_n(tuning, range) : tuned_model.n

        default_n(tuning, range) : old_tuned_model.n

        n! >= old_n!

        "to search, bringing total to $n!. "

                               verbosity, acceleration, resampling_machine)

        rm = resampling_machine

    else

    end

end

    predict(fitresult, Xnew)

function MLJBase.fitted_params(tuned_model::EitherTunedModel, fitresult)

                best_fitted_params=fitted_params(fitresult))

    else

                best_fitted_params=missing)

    end

end

## METADATA

    MLJBase.supports_weights(M)

    "MLJTuning.DeterministicTunedModel"

    "https://github.com/alan-turing-institute/MLJTuning.jl"

    MLJBase.is_pure_julia(M)

    MLJBase.input_scitype(M)

    MLJBase.target_scitype(M)