9704727222

Committed 27 Jun 2024 11:01PM UTC coverage: 95.922% (+1.3%) from 94.617%

Build # 9704727222

Build Type

Pull #326

github

Committed by

web-flow

Commit Message

Merge 1f104aaf8 into ceddaa424

Pull Request Pull Request #326: BREAKING: Change expression types to `DynamicExpressions.Expression` (from `DynamicExpressions.Node`)

Run Details

301 of 307 new or added lines in 17 files covered. (98.05%)

1 existing line in 1 file now uncovered.

2611 of 2722 relevant lines covered (95.92%)

35611300.15 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

98.46

/src/SingleIteration.jl

module SingleIterationModule

using DynamicExpressions:
    AbstractExpression,
    Node,
    constructorof,
    string_tree,
    simplify_tree!,
    combine_operators,
    parse_expression
using ..UtilsModule: @threads_if
using ..CoreModule: Options, Dataset, RecordType, DATA_TYPE, LOSS_TYPE, create_expression
using ..ComplexityModule: compute_complexity
using ..PopMemberModule: PopMember, generate_reference
using ..PopulationModule: Population, finalize_scores, best_sub_pop
using ..HallOfFameModule: HallOfFame
using ..AdaptiveParsimonyModule: RunningSearchStatistics
using ..RegularizedEvolutionModule: reg_evol_cycle
using ..LossFunctionsModule: score_func_batched, batch_sample
using ..ConstantOptimizationModule: optimize_constants
using ..RecorderModule: @recorder

# Cycle through regularized evolution many times,
# printing the fittest equation every 10% through
function s_r_cycle(
    dataset::D,
    pop::P,
    ncycles::Int,
    curmaxsize::Int,
    running_search_statistics::RunningSearchStatistics;
    verbosity::Int=0,
    options::Options,
    record::RecordType,
)::Tuple{
    P,HallOfFame{T,L,N},Float64
} where {T,L,D<:Dataset{T,L},N<:AbstractExpression{T},P<:Population{T,L,N}}
    max_temp = 1.0
    min_temp = 0.0
    if !options.annealing
        min_temp = max_temp
    end
    all_temperatures = LinRange(max_temp, min_temp, ncycles)
    best_examples_seen = HallOfFame(options, dataset)
    num_evals = 0.0

    # For evaluating on a fixed batch (for batching)
    idx = options.batching ? batch_sample(dataset, options) : Int[]
    example_tree = create_expression(zero(T), options, dataset)
    loss_cache = [(oid=example_tree, score=zero(L)) for member in pop.members]
    first_loop = true

    for temperature in all_temperatures
        pop, tmp_num_evals = reg_evol_cycle(
            dataset,
            pop,
            temperature,
            curmaxsize,
            running_search_statistics,
            options,
            record,
        )
        num_evals += tmp_num_evals
        for (i, member) in enumerate(pop.members)
            size = compute_complexity(member, options)
            score = if options.batching
                oid = member.tree
                if loss_cache[i].oid != oid || first_loop
                    # Evaluate on fixed batch so that we can more accurately
                    # compare expressions with a batched loss (though the batch
                    # changes each iteration, and we evaluate on full-batch outside,
                    # so this is not biased).
                    _score, _ = score_func_batched(
                        dataset, member, options; complexity=size, idx=idx
                    )
                    loss_cache[i] = (oid=copy(oid), score=_score)
                    _score
                else
                    # Already evaluated this particular expression, so just use
                    # the cached score
                    loss_cache[i].score
                end
            else
                member.score
            end
            # TODO: Note that this per-population hall of fame only uses the batched
            #       loss, and is therefore innaccurate. Therefore, some expressions
            #       may be loss if a very small batch size is used.
            # - Could have different batch size for different things (smaller for constant opt)
            # - Could just recompute losses here (expensive)
            # - Average over a few batches
            # - Store multiple expressions in hall of fame
            if 0 < size <= options.maxsize && (
                !best_examples_seen.exists[size] ||
                score < best_examples_seen.members[size].score
            )
                best_examples_seen.exists[size] = true
                best_examples_seen.members[size] = copy(member)
            end
        end
        first_loop = false
    end

    return (pop, best_examples_seen, num_evals)
end

function optimize_and_simplify_population(
    dataset::D, pop::P, options::Options, curmaxsize::Int, record::RecordType
)::Tuple{P,Float64} where {T,L,D<:Dataset{T,L},P<:Population{T,L}}
    array_num_evals = zeros(Float64, pop.n)
    do_optimization = rand(pop.n) .< options.optimizer_probability
    @threads_if !(options.deterministic) for j in 1:(pop.n)
        if options.should_simplify
            tree = pop.members[j].tree
            tree = simplify_tree!(tree, options.operators)
            if tree isa Node
                tree = combine_operators(tree, options.operators)
            end
            pop.members[j].tree = tree
        end
        if options.should_optimize_constants && do_optimization[j]
            # TODO: Might want to do full batch optimization here?
            pop.members[j], array_num_evals[j] = optimize_constants(
                dataset, pop.members[j], options
            )
        end
    end
    num_evals = sum(array_num_evals)
    pop, tmp_num_evals = finalize_scores(dataset, pop, options)
    num_evals += tmp_num_evals

    # Now, we create new references for every member,
    # and optionally record which operations occurred.
    for j in 1:(pop.n)
        old_ref = pop.members[j].ref
        new_ref = generate_reference()
        pop.members[j].parent = old_ref
        pop.members[j].ref = new_ref

        @recorder begin
            # Same structure as in RegularizedEvolution.jl,
            # except we assume that the record already exists.
            @assert haskey(record, "mutations")
            member = pop.members[j]
            if !haskey(record["mutations"], "$(member.ref)")
                record["mutations"]["$(member.ref)"] = RecordType(
                    "events" => Vector{RecordType}(),
                    "tree" => string_tree(member.tree, options),
                    "score" => member.score,
                    "loss" => member.loss,
                    "parent" => member.parent,
                )
            end
            optimize_and_simplify_event = RecordType(
                "type" => "tuning",
                "time" => time(),
                "child" => new_ref,
                "mutation" => RecordType(
                    "type" =>
                        if (do_optimization[j] && options.should_optimize_constants)
                            "simplification_and_optimization"
                        else
                            "simplification"
                        end,
                ),
            )
            death_event = RecordType("type" => "death", "time" => time())

            push!(record["mutations"]["$(old_ref)"]["events"], optimize_and_simplify_event)
            push!(record["mutations"]["$(old_ref)"]["events"], death_event)
        end
    end
    return (pop, num_evals)
end

end

1	module SingleIterationModule
2
3	using DynamicExpressions:
4	AbstractExpression,
5	Node,
6	constructorof,
7	string_tree,
8	simplify_tree!,
9	combine_operators,
10	parse_expression
11	using ..UtilsModule: @threads_if
12	using ..CoreModule: Options, Dataset, RecordType, DATA_TYPE, LOSS_TYPE, create_expression
13	using ..ComplexityModule: compute_complexity
14	using ..PopMemberModule: PopMember, generate_reference
15	using ..PopulationModule: Population, finalize_scores, best_sub_pop
16	using ..HallOfFameModule: HallOfFame
17	using ..AdaptiveParsimonyModule: RunningSearchStatistics
18	using ..RegularizedEvolutionModule: reg_evol_cycle
19	using ..LossFunctionsModule: score_func_batched, batch_sample
20	using ..ConstantOptimizationModule: optimize_constants
21	using ..RecorderModule: @recorder
22
23	# Cycle through regularized evolution many times,
24	# printing the fittest equation every 10% through
25	function s_r_cycle(	61,761✔
26	dataset::D,
27	pop::P,
28	ncycles::Int,
29	curmaxsize::Int,
30	running_search_statistics::RunningSearchStatistics;
31	verbosity::Int=0,
32	options::Options,
33	record::RecordType,
34	)::Tuple{
35	P,HallOfFame{T,L,N},Float64
36	} where {T,L,D<:Dataset{T,L},N<:AbstractExpression{T},P<:Population{T,L,N}}
37	max_temp = 1.0	39,192✔
38	min_temp = 0.0	33,413✔
39	if !options.annealing	33,768✔
40	min_temp = max_temp	33,200✔
41	end
42	all_temperatures = LinRange(max_temp, min_temp, ncycles)	33,768✔
43	best_examples_seen = HallOfFame(options, dataset)	50,811✔
44	num_evals = 0.0	33,416✔
45
46	# For evaluating on a fixed batch (for batching)
47	idx = options.batching ? batch_sample(dataset, options) : Int[]	39,758✔
48	example_tree = create_expression(zero(T), options, dataset)	33,767✔
49	loss_cache = [(oid=example_tree, score=zero(L)) for member in pop.members]	39,546✔
50	first_loop = true	33,415✔
51
52	for temperature in all_temperatures	67,535✔
53	pop, tmp_num_evals = reg_evol_cycle(	26,575,683✔
54	dataset,
55	pop,
56	temperature,
57	curmaxsize,
58	running_search_statistics,
59	options,
60	record,
61	)
62	num_evals += tmp_num_evals	17,655,997✔
63	for (i, member) in enumerate(pop.members)	23,643,201✔
64	size = compute_complexity(member, options)	1,001,317,357✔
65	score = if options.batching	747,412,472✔
66	oid = member.tree	25,317,777✔
67	if loss_cache[i].oid != oid \|\| first_loop	60,722,942✔
68	# Evaluate on fixed batch so that we can more accurately
69	# compare expressions with a batched loss (though the batch
70	# changes each iteration, and we evaluate on full-batch outside,
71	# so this is not biased).
72	_score, _ = score_func_batched(	786,270✔
73	dataset, member, options; complexity=size, idx=idx
74	)
75	loss_cache[i] = (oid=copy(oid), score=_score)	1,106,281✔
76	_score	786,270✔
77	else
78	# Already evaluated this particular expression, so just use
79	# the cached score
80	loss_cache[i].score	49,849,282✔
81	end
82	else
83	member.score	1,469,285,672✔
84	end
85	# TODO: Note that this per-population hall of fame only uses the batched
86	# loss, and is therefore innaccurate. Therefore, some expressions
87	# may be loss if a very small batch size is used.
88	# - Could have different batch size for different things (smaller for constant opt)
89	# - Could just recompute losses here (expensive)
90	# - Average over a few batches
91	# - Store multiple expressions in hall of fame
92	if 0 < size <= options.maxsize && (	1,494,298,635✔
93	!best_examples_seen.exists[size] \|\|
94	score < best_examples_seen.members[size].score
95	)
96	best_examples_seen.exists[size] = true	1,705,470✔
97	best_examples_seen.members[size] = copy(member)	127,601,183✔
98	end
99	end	1,224,079,157✔
100	first_loop = false	20,671,655✔
101	end	29,236,998✔
102
103	return (pop, best_examples_seen, num_evals)	33,767✔
104	end
105
106	function optimize_and_simplify_population(	33,767✔
107	dataset::D, pop::P, options::Options, curmaxsize::Int, record::RecordType
108	)::Tuple{P,Float64} where {T,L,D<:Dataset{T,L},P<:Population{T,L}}
109	array_num_evals = zeros(Float64, pop.n)	33,767✔
110	do_optimization = rand(pop.n) .< options.optimizer_probability	40,505✔
111	@threads_if !(options.deterministic) for j in 1:(pop.n)	34,321✔
112	if options.should_simplify	1,643,948✔
113	tree = pop.members[j].tree	1,510,611✔
114	tree = simplify_tree!(tree, options.operators)	1,511,106✔
115	if tree isa Node	1,511,228✔
UNCOV 116	tree = combine_operators(tree, options.operators)	×
117	end
118	pop.members[j].tree = tree	1,511,075✔
119	end
120	if options.should_optimize_constants && do_optimization[j]	1,643,812✔
121	# TODO: Might want to do full batch optimization here?
122	pop.members[j], array_num_evals[j] = optimize_constants(	203,622✔
123	dataset, pop.members[j], options
124	)
125	end
126	end
127	num_evals = sum(array_num_evals)	33,690✔
128	pop, tmp_num_evals = finalize_scores(dataset, pop, options)	33,690✔
129	num_evals += tmp_num_evals	33,690✔
130
131	# Now, we create new references for every member,
132	# and optionally record which operations occurred.
133	for j in 1:(pop.n)	33,690✔
134	old_ref = pop.members[j].ref	1,642,384✔
135	new_ref = generate_reference()	1,642,331✔
136	pop.members[j].parent = old_ref	1,642,022✔
137	pop.members[j].ref = new_ref	1,642,379✔
138
139	@recorder begin	1,648,036✔
140	# Same structure as in RegularizedEvolution.jl,
141	# except we assume that the record already exists.
142	@assert haskey(record, "mutations")	6,600✔
143	member = pop.members[j]	6,600✔
144	if !haskey(record["mutations"], "$(member.ref)")	6,600✔
145	record["mutations"]["$(member.ref)"] = RecordType(	6,600✔
146	"events" => Vector{RecordType}(),
147	"tree" => string_tree(member.tree, options),
148	"score" => member.score,
149	"loss" => member.loss,
150	"parent" => member.parent,
151	)
152	end
153	optimize_and_simplify_event = RecordType(	6,599✔
154	"type" => "tuning",
155	"time" => time(),
156	"child" => new_ref,
157	"mutation" => RecordType(
158	"type" =>
159	if (do_optimization[j] && options.should_optimize_constants)
160	"simplification_and_optimization"	909✔
161	else
162	"simplification"	12,291✔
163	end,
164	),
165	)
166	death_event = RecordType("type" => "death", "time" => time())	6,600✔
167
168	push!(record["mutations"]["$(old_ref)"]["events"], optimize_and_simplify_event)	6,600✔
169	push!(record["mutations"]["$(old_ref)"]["events"], death_event)	6,600✔
170	end
171	end	1,360,675✔
172	return (pop, num_evals)	33,690✔
173	end
174
175	end

MilesCranmer / SymbolicRegression.jl / 9704727222

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous