• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

JuliaLang / julia / #38002

06 Feb 2025 06:14AM UTC coverage: 20.322% (-2.4%) from 22.722%
#38002

push

local

web-flow
bpart: Fully switch to partitioned semantics (#57253)

This is the final PR in the binding partitions series (modulo bugs and
tweaks), i.e. it closes #54654 and thus closes #40399, which was the
original design sketch.

This thus activates the full designed semantics for binding partitions,
in particular allowing safe replacement of const bindings. It in
particular allows struct redefinitions. This thus closes
timholy/Revise.jl#18 and also closes #38584.

The biggest semantic change here is probably that this gets rid of the
notion of "resolvedness" of a binding. Previously, a lot of the behavior
of our implementation depended on when bindings were "resolved", which
could happen at basically an arbitrary point (in the compiler, in REPL
completion, in a different thread), making a lot of the semantics around
bindings ill- or at least implementation-defined. There are several
related issues in the bugtracker, so this closes #14055 closes #44604
closes #46354 closes #30277

It is also the last step to close #24569.
It also supports bindings for undef->defined transitions and thus closes
#53958 closes #54733 - however, this is not activated yet for
performance reasons and may need some further optimization.

Since resolvedness no longer exists, we need to replace it with some
hopefully more well-defined semantics. I will describe the semantics
below, but before I do I will make two notes:

1. There are a number of cases where these semantics will behave
slightly differently than the old semantics absent some other task going
around resolving random bindings.
2. The new behavior (except for the replacement stuff) was generally
permissible under the old semantics if the bindings happened to be
resolved at the right time.

With all that said, there are essentially three "strengths" of bindings:

1. Implicit Bindings: Anything implicitly obtained from `using Mod`, "no
binding", plus slightly more exotic corner cases around conflicts

2. Weakly declared bindin... (continued)

11 of 111 new or added lines in 7 files covered. (9.91%)

1273 existing lines in 68 files now uncovered.

9908 of 48755 relevant lines covered (20.32%)

105126.48 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

3.08
/base/staticdata.jl
1
# This file is a part of Julia. License is MIT: https://julialang.org/license
2

3
module StaticData
4

5
using Core: CodeInstance, MethodInstance
6
using Base: get_world_counter
7

8
const WORLD_AGE_REVALIDATION_SENTINEL::UInt = 1
9
const _jl_debug_method_invalidation = Ref{Union{Nothing,Vector{Any}}}(nothing)
10
debug_method_invalidation(onoff::Bool) =
×
11
    _jl_debug_method_invalidation[] = onoff ? Any[] : nothing
12

13
function get_ci_mi(codeinst::CodeInstance)
UNCOV
14
    def = codeinst.def
×
UNCOV
15
    if def isa Core.ABIOverride
×
16
        return def.def
×
17
    else
UNCOV
18
        return def::MethodInstance
×
19
    end
20
end
21

22
# Restore backedges to external targets
23
# `edges` = [caller1, ...], the list of worklist-owned code instances internally
24
# `ext_ci_list` = [caller1, ...], the list of worklist-owned code instances externally
25
function insert_backedges(edges::Vector{Any}, ext_ci_list::Union{Nothing,Vector{Any}}, extext_methods::Vector{Any}, internal_methods::Vector{Any})
26
    # determine which CodeInstance objects are still valid in our image
27
    # to enable any applicable new codes
28
    methods_with_invalidated_source = Base.scan_new_methods(extext_methods, internal_methods)
20✔
29
    stack = CodeInstance[]
20✔
30
    visiting = IdDict{CodeInstance,Int}()
20✔
31
    _insert_backedges(edges, stack, visiting, methods_with_invalidated_source)
20✔
32
    if ext_ci_list !== nothing
×
33
        _insert_backedges(ext_ci_list, stack, visiting, methods_with_invalidated_source, #=external=#true)
20✔
34
    end
35
end
36

UNCOV
37
function _insert_backedges(edges::Vector{Any}, stack::Vector{CodeInstance}, visiting::IdDict{CodeInstance,Int}, mwis::IdSet{Method}, external::Bool=false)
×
38
    for i = 1:length(edges)
20✔
UNCOV
39
        codeinst = edges[i]::CodeInstance
×
UNCOV
40
        verify_method_graph(codeinst, stack, visiting, mwis)
×
UNCOV
41
        minvalid = codeinst.min_world
×
UNCOV
42
        maxvalid = codeinst.max_world
×
UNCOV
43
        if maxvalid ≥ minvalid && external
×
UNCOV
44
            caller = get_ci_mi(codeinst)
×
UNCOV
45
            @assert isdefined(codeinst, :inferred) # See #53586, #53109
×
UNCOV
46
            inferred = @ccall jl_rettype_inferred(
×
47
                codeinst.owner::Any, caller::Any, minvalid::UInt, maxvalid::UInt)::Any
UNCOV
48
            if inferred !== nothing
×
49
                # We already got a code instance for this world age range from
50
                # somewhere else - we don't need this one.
51
            else
UNCOV
52
                @ccall jl_mi_cache_insert(caller::Any, codeinst::Any)::Cvoid
×
53
            end
54
        end
UNCOV
55
    end
×
56
end
57

UNCOV
58
function verify_method_graph(codeinst::CodeInstance, stack::Vector{CodeInstance}, visiting::IdDict{CodeInstance,Int}, mwis::IdSet{Method})
×
UNCOV
59
    @assert isempty(stack); @assert isempty(visiting);
×
UNCOV
60
    child_cycle, minworld, maxworld = verify_method(codeinst, stack, visiting, mwis)
×
UNCOV
61
    @assert child_cycle == 0
×
UNCOV
62
    @assert isempty(stack); @assert isempty(visiting);
×
63
    nothing
×
64
end
65

66
# Test all edges relevant to a method:
67
# - Visit the entire call graph, starting from edges[idx] to determine if that method is valid
68
# - Implements Tarjan's SCC (strongly connected components) algorithm, simplified to remove the count variable
69
#   and slightly modified with an early termination option once the computation reaches its minimum
UNCOV
70
function verify_method(codeinst::CodeInstance, stack::Vector{CodeInstance}, visiting::IdDict{CodeInstance,Int}, mwis::IdSet{Method})
×
UNCOV
71
    world = codeinst.min_world
×
UNCOV
72
    let max_valid2 = codeinst.max_world
×
UNCOV
73
        if max_valid2 ≠ WORLD_AGE_REVALIDATION_SENTINEL
×
UNCOV
74
            return 0, world, max_valid2
×
75
        end
76
    end
UNCOV
77
    current_world = get_world_counter()
×
78
    local minworld::UInt, maxworld::UInt = 1, current_world
×
UNCOV
79
    def = get_ci_mi(codeinst).def
×
UNCOV
80
    @assert def isa Method
×
UNCOV
81
    if haskey(visiting, codeinst)
×
UNCOV
82
        return visiting[codeinst], minworld, maxworld
×
83
    end
UNCOV
84
    push!(stack, codeinst)
×
UNCOV
85
    depth = length(stack)
×
UNCOV
86
    visiting[codeinst] = depth
×
87
    # TODO JL_TIMING(VERIFY_IMAGE, VERIFY_Methods)
UNCOV
88
    callees = codeinst.edges
×
89
    # Check for invalidation of the implicit edges from GlobalRef in the Method source
UNCOV
90
    if def in mwis
×
91
        maxworld = 0
×
92
        invalidations = _jl_debug_method_invalidation[]
×
93
        if invalidations !== nothing
×
94
            push!(invalidations, def, "method_globalref", codeinst, nothing)
×
95
        end
96
    end
97
    # verify current edges
UNCOV
98
    if isempty(callees)
×
99
        # quick return: no edges to verify (though we probably shouldn't have gotten here from WORLD_AGE_REVALIDATION_SENTINEL)
UNCOV
100
    elseif maxworld == unsafe_load(cglobal(:jl_require_world, UInt))
×
101
        # if no new worlds were allocated since serializing the base module, then no new validation is worth doing right now either
102
        minworld = maxworld
×
103
    else
104
        j = 1
×
UNCOV
105
        while j ≤ length(callees)
×
106
            local min_valid2::UInt, max_valid2::UInt
×
UNCOV
107
            edge = callees[j]
×
UNCOV
108
            @assert !(edge isa Method) # `Method`-edge isn't allowed for the optimized one-edge format
×
UNCOV
109
            if edge isa CodeInstance
×
UNCOV
110
                edge = get_ci_mi(edge)
×
111
            end
UNCOV
112
            if edge isa MethodInstance
×
UNCOV
113
                sig = typeintersect((edge.def::Method).sig, edge.specTypes) # TODO??
×
UNCOV
114
                min_valid2, max_valid2, matches = verify_call(sig, callees, j, 1, world)
×
UNCOV
115
                j += 1
×
UNCOV
116
            elseif edge isa Int
×
UNCOV
117
                sig = callees[j+1]
×
UNCOV
118
                min_valid2, max_valid2, matches = verify_call(sig, callees, j+2, edge, world)
×
UNCOV
119
                j += 2 + edge
×
120
                edge = sig
×
UNCOV
121
            elseif edge isa Core.Binding
×
UNCOV
122
                j += 1
×
123
                min_valid2 = minworld
×
UNCOV
124
                max_valid2 = maxworld
×
UNCOV
125
                if !Base.binding_was_invalidated(edge)
×
UNCOV
126
                    if isdefined(edge, :partitions)
×
UNCOV
127
                        min_valid2 = edge.partitions.min_world
×
UNCOV
128
                        max_valid2 = edge.partitions.max_world
×
129
                    end
130
                else
131
                    # Binding was previously invalidated
132
                    min_valid2 = 1
×
133
                    max_valid2 = 0
×
134
                end
UNCOV
135
                matches = nothing
×
136
            else
UNCOV
137
                callee = callees[j+1]
×
UNCOV
138
                if callee isa Core.MethodTable # skip the legacy edge (missing backedge)
×
UNCOV
139
                    j += 2
×
UNCOV
140
                    continue
×
141
                end
UNCOV
142
                if callee isa CodeInstance
×
UNCOV
143
                    callee = get_ci_mi(callee)
×
144
                end
UNCOV
145
                if callee isa MethodInstance
×
UNCOV
146
                    meth = callee.def::Method
×
147
                else
148
                    meth = callee::Method
×
149
                end
UNCOV
150
                min_valid2, max_valid2 = verify_invokesig(edge, meth, world)
×
151
                matches = nothing
×
UNCOV
152
                j += 2
×
153
            end
UNCOV
154
            if minworld < min_valid2
×
155
                minworld = min_valid2
×
156
            end
UNCOV
157
            if maxworld > max_valid2
×
158
                maxworld = max_valid2
×
159
            end
UNCOV
160
            invalidations = _jl_debug_method_invalidation[]
×
UNCOV
161
            if max_valid2 ≠ typemax(UInt) && invalidations !== nothing
×
162
                push!(invalidations, edge, "insert_backedges_callee", codeinst, matches)
×
163
            end
UNCOV
164
            if max_valid2 == 0 && invalidations === nothing
×
165
                break
×
166
            end
UNCOV
167
        end
×
168
    end
169
    # verify recursive edges (if valid, or debugging)
UNCOV
170
    cycle = depth
×
171
    cause = codeinst
×
UNCOV
172
    if maxworld ≠ 0 || _jl_debug_method_invalidation[] !== nothing
×
UNCOV
173
        for j = 1:length(callees)
×
UNCOV
174
            edge = callees[j]
×
UNCOV
175
            if !(edge isa CodeInstance)
×
UNCOV
176
                continue
×
177
            end
UNCOV
178
            callee = edge
×
179
            local min_valid2::UInt, max_valid2::UInt
UNCOV
180
            child_cycle, min_valid2, max_valid2 = verify_method(callee, stack, visiting, mwis)
×
UNCOV
181
            if minworld < min_valid2
×
182
                minworld = min_valid2
×
183
            end
UNCOV
184
            if minworld > max_valid2
×
185
                max_valid2 = 0
×
186
            end
UNCOV
187
            if maxworld > max_valid2
×
188
                cause = callee
×
189
                maxworld = max_valid2
×
190
            end
UNCOV
191
            if max_valid2 == 0
×
192
                # found what we were looking for, so terminate early
193
                break
×
UNCOV
194
            elseif child_cycle ≠ 0 && child_cycle < cycle
×
195
                # record the cycle will resolve at depth "cycle"
196
                cycle = child_cycle
×
197
            end
UNCOV
198
        end
×
199
    end
UNCOV
200
    if maxworld ≠ 0 && cycle ≠ depth
×
UNCOV
201
        return cycle, minworld, maxworld
×
202
    end
203
    # If we are the top of the current cycle, now mark all other parts of
204
    # our cycle with what we found.
205
    # Or if we found a failed edge, also mark all of the other parts of the
206
    # cycle as also having a failed edge.
UNCOV
207
    while length(stack) ≥ depth
×
UNCOV
208
        child = pop!(stack)
×
UNCOV
209
        if maxworld ≠ 0
×
UNCOV
210
            @atomic :monotonic child.min_world = minworld
×
211
        end
UNCOV
212
        if maxworld == current_world
×
UNCOV
213
            Base.Compiler.store_backedges(child, child.edges)
×
UNCOV
214
            @atomic :monotonic child.max_world = typemax(UInt)
×
215
        else
216
            @atomic :monotonic child.max_world = maxworld
×
217
        end
UNCOV
218
        @assert visiting[child] == length(stack) + 1
×
UNCOV
219
        delete!(visiting, child)
×
UNCOV
220
        invalidations = _jl_debug_method_invalidation[]
×
UNCOV
221
        if invalidations !== nothing && maxworld < current_world
×
222
            push!(invalidations, child, "verify_methods", cause)
×
223
        end
UNCOV
224
    end
×
UNCOV
225
    return 0, minworld, maxworld
×
226
end
227

228
function verify_call(@nospecialize(sig), expecteds::Core.SimpleVector, i::Int, n::Int, world::UInt)
229
    # verify that these edges intersect with the same methods as before
UNCOV
230
    lim = _jl_debug_method_invalidation[] !== nothing ? Int(typemax(Int32)) : n
×
UNCOV
231
    minworld = Ref{UInt}(1)
×
UNCOV
232
    maxworld = Ref{UInt}(typemax(UInt))
×
UNCOV
233
    has_ambig = Ref{Int32}(0)
×
UNCOV
234
    result = Base._methods_by_ftype(sig, nothing, lim, world, #=ambig=#false, minworld, maxworld, has_ambig)
×
UNCOV
235
    if result === nothing
×
236
        maxworld[] = 0
×
237
    else
238
        # setdiff!(result, expected)
UNCOV
239
        if length(result) ≠ n
×
240
            maxworld[] = 0
×
241
        end
UNCOV
242
        ins = 0
×
UNCOV
243
        for k = 1:length(result)
×
UNCOV
244
            match = result[k]::Core.MethodMatch
×
245
            local found = false
×
UNCOV
246
            for j = 1:n
×
UNCOV
247
                t = expecteds[i+j-1]
×
UNCOV
248
                if t isa Method
×
UNCOV
249
                    meth = t
×
250
                else
UNCOV
251
                    if t isa CodeInstance
×
UNCOV
252
                        t = get_ci_mi(t)
×
253
                    else
UNCOV
254
                        t = t::MethodInstance
×
255
                    end
UNCOV
256
                    meth = t.def::Method
×
257
                end
UNCOV
258
                if match.method == meth
×
259
                    found = true
×
UNCOV
260
                    break
×
261
                end
UNCOV
262
            end
×
UNCOV
263
            if !found
×
264
                # intersection has a new method or a method was
265
                # deleted--this is now probably no good, just invalidate
266
                # everything about it now
267
                maxworld[] = 0
×
268
                if _jl_debug_method_invalidation[] === nothing
×
269
                    break
×
270
                end
271
                ins += 1
×
272
                result[ins] = match.method
×
273
            end
UNCOV
274
        end
×
UNCOV
275
        if maxworld[] ≠ typemax(UInt) && _jl_debug_method_invalidation[] !== nothing
×
276
            resize!(result, ins)
×
277
        end
278
    end
UNCOV
279
    return minworld[], maxworld[], result
×
280
end
281

UNCOV
282
function verify_invokesig(@nospecialize(invokesig), expected::Method, world::UInt)
×
UNCOV
283
    @assert invokesig isa Type
×
284
    local minworld::UInt, maxworld::UInt
UNCOV
285
    if invokesig === expected.sig
×
286
        # the invoke match is `expected` for `expected->sig`, unless `expected` is invalid
UNCOV
287
        minworld = expected.primary_world
×
UNCOV
288
        maxworld = expected.deleted_world
×
UNCOV
289
        @assert minworld ≤ world
×
UNCOV
290
        if maxworld < world
×
291
            maxworld = 0
×
292
        end
293
    else
294
        minworld = 1
×
295
        maxworld = typemax(UInt)
×
UNCOV
296
        mt = Base.get_methodtable(expected)
×
UNCOV
297
        if mt === nothing
×
298
            maxworld = 0
×
299
        else
UNCOV
300
            matched, valid_worlds = Base.Compiler._findsup(invokesig, mt, world)
×
301
            minworld, maxworld = valid_worlds.min_world, valid_worlds.max_world
×
UNCOV
302
            if matched === nothing
×
303
                maxworld = 0
×
UNCOV
304
            elseif matched.method != expected
×
305
                maxworld = 0
×
306
            end
307
        end
308
    end
UNCOV
309
    return minworld, maxworld
×
310
end
311

312
end # module StaticData
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc