• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

JuliaLang / julia / 1608

16 Nov 2026 09:30PM UTC coverage: 93.209% (+0.2%) from 92.989%
1608

push

buildkite

web-flow
[JuliaLowering] Call `jl_lower` from fl-lowered generated functions (#64931)

23 of 26 new or added lines in 4 files covered. (88.46%)

385 existing lines in 15 files now uncovered.

100361 of 107673 relevant lines covered (93.21%)

397.51 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

96.39
/stdlib/Serialization/src/Serialization.jl
1
# This file is a part of Julia. License is MIT: https://julialang.org/license
2

3
"""
4
    Serialization
5

6
Writing Julia objects to a stream with [`serialize`](@ref), and reading them back with
7
[`deserialize`](@ref). Newer Julia releases can read what older 1.x releases wrote, but not
8
the other way round.
9
"""
10
module Serialization
11

12
import Base: Bottom, unsafe_convert
13
import Base.ScopedValues: ScopedValue, with
14
import Core: svec, SimpleVector
15
using Base: @assume_effects, unaliascopy, unwrap_unionall, require_one_based_indexing, ntupleany
16
using Core.IR
17

18
export serialize, deserialize, AbstractSerializer, Serializer
19

20
abstract type AbstractSerializer end
21

22
# Dict is generally a much better dictionary than IdDict, but we want objectid comparison
23
struct IdKey val::Any end
274 ✔
24
Base.hash(k::IdKey, h::UInt) = hash(objectid(k.val), h)
274 ✔
25
Base.isequal(a::IdKey, b::IdKey) = a.val === b.val
174 ✔
26

27
mutable struct Serializer{I<:IO} <: AbstractSerializer
28
    const io::I
29
    counter::Int
30
    const cycle_table::Dict{IdKey,Int}
31
    const backref_table::Vector{Any}
32
    const pending_refs::Vector{Int}
33
    const known_object_data::Dict{UInt64,Any}
34
    version::Int
35
    Serializer{I}(io::I) where I<:IO = new(io, 0, Dict{IdKey,Int}(), Any[], Int[], Dict{UInt64,Any}(), ser_version)
274 ✔
36
end
37

38
Serializer(io::IO) = Serializer{typeof(io)}(io)
274 ✔
39

40
const current_module = ScopedValue{Union{Nothing,Module}}(nothing)
41

42
## serializing values ##
43

44
const n_int_literals = 33
45
const n_reserved_slots = 24
46
const n_reserved_tags = 8
47

48
const TAGS = Any[
49
    Symbol, Int8, UInt8, Int16, UInt16, Int32, UInt32, Int64, UInt64, Int128, UInt128,
50
    Float16, Float32, Float64, Char, DataType, Union, UnionAll, Core.TypeName, Tuple,
51
    Array, Expr, LineNumberNode, :__LabelNode__, GotoNode, QuoteNode, CodeInfo, TypeVar,
52
    Core.Box, Core.MethodInstance, Module, Task, String, SimpleVector, Method,
53
    GlobalRef, SlotNumber, Const, NewvarNode, SSAValue,
54

55
    # dummy entries for tags that don't correspond directly to types
56
    Symbol, # UNDEFREF_TAG
57
    Symbol, # BACKREF_TAG
58
    Symbol, # LONGBACKREF_TAG
59
    Symbol, # SHORTBACKREF_TAG
60
    Symbol, # LONGTUPLE_TAG
61
    Symbol, # LONGSYMBOL_TAG
62
    Symbol, # LONGEXPR_TAG
63
    Symbol, # LONGSTRING_TAG
64
    Symbol, # SHORTINT64_TAG
65
    Symbol, # FULL_DATATYPE_TAG
66
    Symbol, # WRAPPER_DATATYPE_TAG
67
    Symbol, # OBJECT_TAG
68
    Symbol, # REF_OBJECT_TAG
69
    Symbol, # FULL_GLOBALREF_TAG
70
    Symbol, # HEADER_TAG
71
    Symbol, # IDDICT_TAG
72
    Symbol, # SHARED_REF_TAG
73
    ReturnNode, GotoIfNot,
74
    fill(Symbol, n_reserved_tags)...,
75

76
    (), Bool, Any, Bottom, Core.TypeofBottom, Type, svec(), Tuple{}, false, true, nothing,
77
    :Any, :Array, :TypeVar, :Box, :Tuple, :Ptr, :return, :call, Symbol("::"), :Function,
78
    :(=), :(==), :(===), :gotoifnot, :A, :B, :C, :M, :N, :T, :S, :X, :Y, :a, :b, :c, :d, :e, :f,
79
    :g, :h, :i, :j, :k, :l, :m, :n, :o, :p, :q, :r, :s, :t, :u, :v, :w, :x, :y, :z, :add_int,
80
    :sub_int, :mul_int, :add_float, :sub_float, :new, :mul_float, :bitcast, :start, :done, :next,
81
    :indexed_iterate, :getfield, :meta, :eq_int, :slt_int, :sle_int, :ne_int, :push_loc, :pop_loc,
82
    :pop, :arrayset, :arrayref, :apply_type, :inbounds, :getindex, :setindex!, :Core, :!, :+,
83
    :Base, :static_parameter, :convert, :colon, Symbol("#self#"), Symbol("#temp#"), :tuple, Symbol(""),
84

85
    fill(:_reserved_, n_reserved_slots)...,
86

87
    (Int32(0):Int32(n_int_literals-1))...,
88
    (Int64(0):Int64(n_int_literals-1))...
89
]
90

91
const NTAGS = length(TAGS)
92
@assert NTAGS == 255
93

94
const ser_version = 31 # do not make changes without bumping the version #!
95

96
format_version(::AbstractSerializer) = ser_version
120 ✔
97
format_version(s::Serializer) = s.version
3 ✔
98

99
# static lookup table of serializee value --> TAG index via objectid + linear probe
100
# omit constant ints & reserved slots from the table as sertag doesn't return these
101
const NSERTAG_KEYS = NTAGS - n_reserved_slots - 2*n_int_literals
102

103
# keeps >50% sparse so linear probes hit in 1-2 steps. also allows :terminates
104
const SERTAG_TABLE_SIZE = nextpow(2, 2 * NSERTAG_KEYS)
105

UNCOV
106
struct SertagEmpty end
×
107
const sertag_empty = SertagEmpty()
108

109
struct SertagTable
UNCOV
110
    keys::Memory{Any}
×
111
    vals::Memory{Int32}
112
end
113

114
const sertag_table = let
115
    keys = Memory{Any}(undef, SERTAG_TABLE_SIZE)
116
    vals = Memory{Int32}(undef, SERTAG_TABLE_SIZE)
117
    fill!(keys, sertag_empty)
118
    @assume_effects :terminates_locally :noub @inbounds for i in Iterators.reverse(1:NSERTAG_KEYS)
119
        key = TAGS[i]
120
        loc = mod1(objectid(key), SERTAG_TABLE_SIZE)
121
        while true
122
            k = keys[loc]
123
            if k === sertag_empty || k === key
124
                keys[loc] = key
125
                vals[loc] = Int32(i)
126
                break
127
            end
128
            loc = mod1(loc + 1, SERTAG_TABLE_SIZE)
129
        end
130
    end
131
    SertagTable(keys, vals)
132
end
133

134
@inline function sertag(@nospecialize(v))
462 ✔
135
    (; keys, vals) = sertag_table
462 ✔
136
    loc = mod1(objectid(v), SERTAG_TABLE_SIZE)
462 ✔
137
    @assume_effects :terminates_locally :noub @inbounds while true
462 ✔
138
        @inbounds k = keys[loc]
462 ✔
139
        if k === v
462 ✔
140
            return vals[loc]
420 ✔
141
        elseif k === sertag_empty
458 ✔
142
            return Int32(-1)
458 ✔
143
        else
144
            loc = mod1(loc + 1, SERTAG_TABLE_SIZE)
385 ✔
145
        end
146
    end
385 ✔
147
end
148
desertag(i::Int32) = @inbounds(TAGS[i])
521 ✔
149

150
# tags >= this just represent themselves, their whole representation is 1 byte
151
const VALUE_TAGS = sertag(())
152
const ZERO32_TAG = Int32(NTAGS-(2*n_int_literals-1))
153
const ZERO64_TAG = Int64(NTAGS-(n_int_literals-1))
154
const TRUE_TAG = sertag(true)
155
const FALSE_TAG = sertag(false)
156
const EMPTYTUPLE_TAG = sertag(())
157
const TUPLE_TAG = sertag(Tuple)
158
const SIMPLEVECTOR_TAG = sertag(SimpleVector)
159
const SYMBOL_TAG = sertag(Symbol)
160
const INT8_TAG = sertag(Int8)
161
const ARRAY_TAG = sertag(Array)
162
const EXPR_TAG = sertag(Expr)
163
const MODULE_TAG = sertag(Module)
164
const METHODINSTANCE_TAG = sertag(Core.MethodInstance)
165
const METHOD_TAG = sertag(Method)
166
const TASK_TAG = sertag(Task)
167
const DATATYPE_TAG = sertag(DataType)
168
const TYPENAME_TAG = sertag(Core.TypeName)
169
const INT32_TAG = sertag(Int32)
170
const INT64_TAG = sertag(Int64)
171
const GLOBALREF_TAG = sertag(GlobalRef)
172
const BOTTOM_TAG = sertag(Bottom)
173
const UNIONALL_TAG = sertag(UnionAll)
174
const STRING_TAG = sertag(String)
175
const o0 = sertag(SSAValue)
176
const UNDEFREF_TAG         = Int32(o0+1)
177
const BACKREF_TAG          = Int32(o0+2)
178
const LONGBACKREF_TAG      = Int32(o0+3)
179
const SHORTBACKREF_TAG     = Int32(o0+4)
180
const LONGTUPLE_TAG        = Int32(o0+5)
181
const LONGSYMBOL_TAG       = Int32(o0+6)
182
const LONGEXPR_TAG         = Int32(o0+7)
183
const LONGSTRING_TAG       = Int32(o0+8)
184
const SHORTINT64_TAG       = Int32(o0+9)
185
const FULL_DATATYPE_TAG    = Int32(o0+10)
186
const WRAPPER_DATATYPE_TAG = Int32(o0+11)
187
const OBJECT_TAG           = Int32(o0+12)
188
const REF_OBJECT_TAG       = Int32(o0+13)
189
const FULL_GLOBALREF_TAG   = Int32(o0+14)
190
const HEADER_TAG           = Int32(o0+15)
191
const IDDICT_TAG           = Int32(o0+16)
192
const SHARED_REF_TAG       = Int32(o0+17)
193

194
writetag(s::IO, tag) = (write(s, UInt8(tag)); nothing)
523 ✔
195

196
function write_as_tag(s::IO, tag)
420 ✔
197
    tag < VALUE_TAGS && write(s, UInt8(0))
420 ✔
198
    write(s, UInt8(tag))
420 ✔
199
    nothing
420 ✔
200
end
201

202
# cycle handling
203
_getcycle(s::AbstractSerializer, @nospecialize(x)) = get(s.table, x, -1)::Int
311 ✔
204
_getcycle(s::Serializer, @nospecialize(x)) = get(s.cycle_table, IdKey(x), -1)
274 ✔
205
_setcycle!(s::AbstractSerializer, @nospecialize(x), v::Int) = (s.table[x] = v; nothing)
311 ✔
206
_setcycle!(s::Serializer, @nospecialize(x), v::Int) = (s.cycle_table[IdKey(x)] = v; nothing)
274 ✔
207

208
_setbackref!(s::AbstractSerializer, slot::Int, @nospecialize(x)) = (s.table[slot] = x; nothing)
413 ✔
209
function _setbackref!(s::Serializer, slot::Int, @nospecialize(x))
135 ✔
210
    bt = s.backref_table
135 ✔
211
    i = slot + 1
135 ✔
212
    i > length(bt) && resize!(bt, max(i, 2 * length(bt) + 1))
135 ✔
213
    @inbounds bt[i] = x
135 ✔
214
    nothing
135 ✔
215
end
216

UNCOV
217
@noinline function __getbackref_error(id::Int)
×
UNCOV
218
    error("""Inconsistent Serializer state when deserializing.
×
219
            Attempt to access internal table with key $id failed.
220

221
            This might occur if the Serializer contexts when serializing and deserializing are inconsistent.
222
            In particular, if multiple serialize calls use the same Serializer object then
223
            the corresponding deserialize calls should also use the same Serializer object.
224
        """)
225
end
226

227
_getbackref(s::AbstractSerializer, id::Int) = get(() -> __getbackref_error(id), s.table, id)
389 ✔
228
function _getbackref(s::Serializer, id::Int)
117 ✔
229
    bt = s.backref_table
117 ✔
230
    i = id + 1
117 ✔
231
    (id < 0 || i > length(bt) || !isassigned(bt, i)) && __getbackref_error(id)
117 ✔
232
    @inbounds return bt[i]
117 ✔
233
end
234

235
_sizehint_cycle!(s::AbstractSerializer, n::Integer) = (sizehint!(s.table, n); nothing)
277 ✔
236
_sizehint_cycle!(s::Serializer, n::Integer) = (sizehint!(s.cycle_table, n; shrink=false); nothing)
171 ✔
237

238
_sizehint_backref!(s::AbstractSerializer, n::Integer) = (sizehint!(s.table, n); nothing)
413 ✔
239
_sizehint_backref!(s::Serializer, n::Integer) = (sizehint!(s.backref_table, n; shrink=false); nothing)
14 ✔
240

241
function _emit_backref(io::IO, offs::Int)
285 ✔
242
    if offs <= typemax(UInt16)
285 ✔
243
        writetag(io, SHORTBACKREF_TAG)
285 ✔
244
        write(io, UInt16(offs))
285 ✔
245
    elseif offs <= typemax(Int32)
6 ✔
246
        writetag(io, BACKREF_TAG)
6 ✔
247
        write(io, Int32(offs))
6 ✔
248
    else
UNCOV
249
        writetag(io, LONGBACKREF_TAG)
×
UNCOV
250
        write(io, Int64(offs))
×
251
    end
252
    nothing
285 ✔
253
end
254

255
function serialize_cycle(s::AbstractSerializer, @nospecialize(x))
411 ✔
256
    offs = _getcycle(s, x)
411 ✔
257
    if offs != -1
411 ✔
258
        _emit_backref(s.io, offs)
285 ✔
259
        return true
285 ✔
260
    end
261
    _setcycle!(s, x, s.counter)
411 ✔
262
    s.counter += 1
411 ✔
263
    return false
411 ✔
264
end
265

266
function serialize_cycle_header(s::AbstractSerializer, @nospecialize(x))
238 ✔
267
    serialize_cycle(s, x) && return true
238 ✔
268
    serialize_type(s, typeof(x), true)
238 ✔
269
    return false
238 ✔
270
end
271

272
function reset_state(s::AbstractSerializer)
423 ✔
273
    s.counter = 0
423 ✔
274
    empty!(s.table)
423 ✔
275
    empty!(s.pending_refs)
423 ✔
276
    s
423 ✔
277
end
278
function reset_state(s::Serializer)
3 ✔
279
    s.counter = 0
3 ✔
280
    empty!(s.cycle_table)
3 ✔
281
    empty!(s.backref_table)
3 ✔
282
    empty!(s.pending_refs)
3 ✔
283
    s
3 ✔
284
end
285

286
serialize(s::AbstractSerializer, x::Bool) = x ? writetag(s.io, TRUE_TAG) :
373 ✔
287
                                                writetag(s.io, FALSE_TAG)
288

289
serialize(s::AbstractSerializer, p::Ptr) = serialize_any(s, oftype(p, C_NULL))
101 ✔
290

291
serialize(s::AbstractSerializer, ::Tuple{}) = writetag(s.io, EMPTYTUPLE_TAG)
269 ✔
292

293
function serialize(s::AbstractSerializer, t::Tuple)
382 ✔
294
    l = length(t)
382 ✔
295
    if l <= NTAGS
382 ✔
296
        writetag(s.io, TUPLE_TAG)
382 ✔
297
        write(s.io, UInt8(l))
382 ✔
298
    else
299
        writetag(s.io, LONGTUPLE_TAG)
3 ✔
300
        write(s.io, Int32(l))
3 ✔
301
    end
302
    for x in t
382 ✔
303
        serialize(s, x)
382 ✔
304
    end
382 ✔
305
end
306

307
function serialize(s::AbstractSerializer, v::SimpleVector)
189 ✔
308
    writetag(s.io, SIMPLEVECTOR_TAG)
189 ✔
309
    write(s.io, Int32(length(v)))
189 ✔
310
    for x in v
189 ✔
311
        serialize(s, x)
180 ✔
312
    end
180 ✔
313
end
314

315
function serialize(s::AbstractSerializer, x::Symbol)
458 ✔
316
    tag = sertag(x)
458 ✔
317
    if tag > 0
458 ✔
318
        return write_as_tag(s.io, tag)
382 ✔
319
    end
320
    pname = unsafe_convert(Ptr{UInt8}, x)
458 ✔
321
    len = Int(ccall(:strlen, Csize_t, (Cstring,), pname))
458 ✔
322
    if len > 7
458 ✔
323
        serialize_cycle(s, x) && return
384 ✔
324
    end
325
    if len <= NTAGS
458 ✔
326
        writetag(s.io, SYMBOL_TAG)
458 ✔
327
        write(s.io, UInt8(len))
458 ✔
328
    else
329
        writetag(s.io, LONGSYMBOL_TAG)
83 ✔
330
        write(s.io, Int32(len))
83 ✔
331
    end
332
    unsafe_write(s.io, pname, len)
458 ✔
333
    nothing
458 ✔
334
end
335

336
function serialize_array_data(s::IO, a)
298 ✔
337
    require_one_based_indexing(a)
298 ✔
338
    isempty(a) && return 0
298 ✔
339
    if eltype(a) === Bool
282 ✔
340
        last = a[1]::Bool
3 ✔
341
        count = 1
3 ✔
342
        for i = 2:length(a)
3 ✔
343
            if a[i]::Bool != last || count == 127
3 ✔
344
                write(s, UInt8((UInt8(last) << 7) | count))
3 ✔
345
                last = a[i]::Bool
3 ✔
346
                count = 1
3 ✔
347
            else
348
                count += 1
3 ✔
349
            end
350
        end
3 ✔
351
        write(s, UInt8((UInt8(last) << 7) | count))
3 ✔
352
    else
353
        write(s, a)
282 ✔
354
    end
355
end
356

357
function _serialize_non_bits_elements!(s::AbstractSerializer, a)
277 ✔
358
    _sizehint_cycle!(s, div(length(a), 4))  # prepare for lots of pointers
277 ✔
359
    @inbounds for i in eachindex(a)
277 ✔
360
        if isassigned(a, i)
264 ✔
361
            serialize(s, a[i])
264 ✔
362
        else
363
            writetag(s.io, UNDEFREF_TAG)
3 ✔
364
        end
365
    end
264 ✔
366
end
367

368
function serialize(s::AbstractSerializer, a::Array)
295 ✔
369
    serialize_cycle(s, a) && return
295 ✔
370
    elty = eltype(a)
295 ✔
371
    writetag(s.io, ARRAY_TAG)
295 ✔
372
    if elty !== UInt8
295 ✔
373
        serialize(s, elty)
289 ✔
374
    end
375
    if ndims(a) != 1
295 ✔
376
        serialize(s, size(a))
33 ✔
377
    else
378
        serialize(s, length(a))
295 ✔
379
    end
380
    if isbitstype(elty)
295 ✔
381
        serialize_array_data(s.io, a)
244 ✔
382
    else
383
        _serialize_non_bits_elements!(s, a)
277 ✔
384
    end
385
end
386

387
function serialize(s::AbstractSerializer, a::SubArray{T,N,A}) where {T,N,A<:Array}
6 ✔
388
    # SubArray's copy only selects the relevant data (and reduces the size) but does not
389
    # preserve the type of the argument. This internal function does both:
390
    b = unaliascopy(a)
6 ✔
391
    serialize_any(s, b)
6 ✔
392
end
393

394
serialize(s::AbstractSerializer, m::GenericMemory) = error("GenericMemory{:atomic} currently cannot be serialized")
3 ✔
395
function serialize(s::AbstractSerializer, m::Memory)
68 ✔
396
    serialize_cycle_header(s, m) && return
68 ✔
397
    serialize(s, length(m))
68 ✔
398
    elty = eltype(m)
68 ✔
399
    if isbitstype(elty)
68 ✔
400
        serialize_array_data(s.io, m)
68 ✔
401
    else
402
        _serialize_non_bits_elements!(s, m)
3 ✔
403
    end
404
end
405

406
function serialize(s::AbstractSerializer, x::GenericMemoryRef)
3 ✔
407
    serialize_type(s, typeof(x))
3 ✔
408
    serialize(s, getfield(x, :mem))
3 ✔
409
    serialize(s, Base.memoryrefoffset(x))
3 ✔
410
end
411

412
function serialize(s::AbstractSerializer, ss::String)
368 ✔
413
    len = sizeof(ss)
368 ✔
414
    if len > 7
368 ✔
415
        serialize_cycle(s, ss) && return
350 ✔
416
        writetag(s.io, SHARED_REF_TAG)
350 ✔
417
    end
418
    if len <= NTAGS
368 ✔
419
        writetag(s.io, STRING_TAG)
368 ✔
420
        write(s.io, UInt8(len))
368 ✔
421
    else
422
        writetag(s.io, LONGSTRING_TAG)
178 ✔
423
        write(s.io, Int64(len))
178 ✔
424
    end
425
    write(s.io, ss)
368 ✔
426
    nothing
368 ✔
427
end
428

429
function serialize(s::AbstractSerializer, ss::SubString{String})
118 ✔
430
    # avoid saving a copy of the parent string, keeping the type of ss
431
    serialize_any(s, SubString(String(ss)))
118 ✔
432
end
433

434
# Don't serialize the pointers
435
function serialize(s::AbstractSerializer, r::Regex)
3 ✔
436
    serialize_type(s, typeof(r))
3 ✔
437
    serialize(s, r.pattern)
3 ✔
438
    serialize(s, r.compile_options)
3 ✔
439
    serialize(s, r.match_options)
3 ✔
440
end
441

442
function serialize(s::AbstractSerializer, n::BigInt)
3 ✔
443
    serialize_type(s, BigInt)
3 ✔
444
    serialize(s, string(n, base = 62))
3 ✔
445
end
446

447
function serialize(s::AbstractSerializer, ex::Expr)
216 ✔
448
    serialize_cycle(s, ex) && return
216 ✔
449
    l = length(ex.args)
216 ✔
450
    if l <= NTAGS
216 ✔
451
        writetag(s.io, EXPR_TAG)
216 ✔
452
        write(s.io, UInt8(l))
216 ✔
453
    else
454
        writetag(s.io, LONGEXPR_TAG)
3 ✔
455
        write(s.io, Int32(l))
3 ✔
456
    end
457
    serialize(s, ex.head)
216 ✔
458
    for a in ex.args
216 ✔
459
        serialize(s, a)
216 ✔
460
    end
216 ✔
461
end
462

463
function serialize_dict_data(s::AbstractSerializer, d::AbstractDict)
182 ✔
464
    write(s.io, Int32(length(d)))
182 ✔
465
    for (k,v) in d
182 ✔
466
        serialize(s, k)
180 ✔
467
        serialize(s, v)
180 ✔
468
    end
180 ✔
469
end
470

471
function serialize(s::AbstractSerializer, d::Dict)
182 ✔
472
    serialize_cycle_header(s, d) && return
182 ✔
473
    serialize_dict_data(s, d)
182 ✔
474
end
475

476
function serialize(s::AbstractSerializer, d::IdDict)
3 ✔
477
    serialize_cycle(s, d) && return
3 ✔
478
    writetag(s.io, IDDICT_TAG)
3 ✔
479
    serialize_type_data(s, typeof(d))
3 ✔
480
    serialize_dict_data(s, d)
3 ✔
481
end
482

483
function serialize_mod_names(s::AbstractSerializer, m::Module)
384 ✔
484
    p = parentmodule(m)
384 ✔
485
    if p === m || m === Base
384 ✔
486
        key = Base.root_module_key(m)
384 ✔
487
        uuid = key.uuid
384 ✔
488
        serialize(s, uuid === nothing ? nothing : uuid.value)
384 ✔
489
        serialize(s, Symbol(key.name))
384 ✔
490
    else
491
        serialize_mod_names(s, p)
165 ✔
492
        serialize(s, nameof(m))
165 ✔
493
    end
494
end
495

496
function serialize(s::AbstractSerializer, m::Module)
384 ✔
497
    writetag(s.io, MODULE_TAG)
384 ✔
498
    serialize_mod_names(s, m)
384 ✔
499
    writetag(s.io, EMPTYTUPLE_TAG)
384 ✔
500
end
501

502
# TODO: make this bidirectional, so objects can be sent back via the same key
503
const object_numbers = WeakKeyDict()
504
const obj_number_salt = Ref{UInt64}(0)
505
function object_number(s::AbstractSerializer, @nospecialize(l))
3 ✔
506
    global obj_number_salt, object_numbers
3 ✔
507
    if haskey(object_numbers, l)
3 ✔
508
        return object_numbers[l]
3 ✔
509
    end
510
    ln = obj_number_salt[]
3 ✔
511
    object_numbers[l] = ln
3 ✔
512
    obj_number_salt[] += 1
3 ✔
513
    return ln::UInt64
3 ✔
514
end
515

516
lookup_object_number(s::AbstractSerializer, n::UInt64) = nothing
×
517

UNCOV
518
remember_object(s::AbstractSerializer, @nospecialize(o), n::UInt64) = nothing
×
519

520
function lookup_object_number(s::Serializer, n::UInt64)
3 ✔
521
    return get(s.known_object_data, n, nothing)
3 ✔
522
end
523

524
function remember_object(s::Serializer, @nospecialize(o), n::UInt64)
3 ✔
525
    s.known_object_data[n] = o
3 ✔
526
    return nothing
3 ✔
527
end
528

529
function serialize(s::AbstractSerializer, meth::Method)
189 ✔
530
    serialize_cycle(s, meth) && return
189 ✔
531
    writetag(s.io, METHOD_TAG)
189 ✔
532
    write(s.io, object_number(s, meth))
189 ✔
533
    serialize(s, meth.module)
189 ✔
534
    serialize(s, meth.name)
189 ✔
535
    serialize(s, meth.file)
189 ✔
536
    serialize(s, meth.line)
189 ✔
537
    serialize(s, meth.sig)
189 ✔
538
    serialize(s, meth.slot_syms)
189 ✔
539
    serialize(s, meth.nargs)
189 ✔
540
    serialize(s, meth.isva)
189 ✔
541
    serialize(s, meth.is_for_opaque_closure)
189 ✔
542
    serialize(s, meth.nospecializeinfer)
189 ✔
543
    serialize(s, meth.constprop)
189 ✔
544
    serialize(s, meth.purity)
189 ✔
545
    if isdefined(meth, :source)
189 ✔
546
        serialize(s, Base._uncompressed_ast(meth))
189 ✔
547
    else
548
        serialize(s, nothing)
3 ✔
549
    end
550
    if isdefined(meth, :generator)
189 ✔
551
        serialize(s, meth.generator)
3 ✔
552
    else
553
        serialize(s, nothing)
189 ✔
554
    end
555
    if isdefined(meth, :recursion_relation)
189 ✔
556
        serialize(s, meth.recursion_relation)
3 ✔
557
    else
558
        serialize(s, nothing)
189 ✔
559
    end
560
    if isdefined(meth, :external_mt)
189 ✔
561
        error("cannot serialize Method objects with external method tables")
3 ✔
562
    end
563
    nothing
189 ✔
564
end
565

566
function serialize(s::AbstractSerializer, mt::Core.MethodTable)
3 ✔
567
    serialize_type(s, typeof(mt))
3 ✔
568
    serialize(s, mt.name)
3 ✔
569
    serialize(s, mt.module)
3 ✔
570
    nothing
3 ✔
571
end
572

573
function serialize(s::AbstractSerializer, mc::Core.MethodCache)
3 ✔
574
    error("cannot serialize MethodCache objects")
3 ✔
575
end
576

577
# Only the (strong) parent links and the state bytes are serialized; the
578
# weak intrusive child lists are rebuilt on deserialization by relinking
579
# the source under its parents as if it were newly constructed (which also
580
# re-inherits the parents' current cancellation state).
581
function serialize(s::AbstractSerializer, src::Core.CancellationTokenSource)
3 ✔
582
    serialize_cycle_header(s, src) && return
3 ✔
583
    np = Int(src.nparents)
3 ✔
584
    serialize(s, np % Int64)  # fixed width: streams are word-size independent
3 ✔
585
    for i in 1:np
3 ✔
586
        serialize(s, Base._cancel_parent(src, i))
3 ✔
587
    end
3 ✔
588
    serialize(s, @atomic src.state)
3 ✔
589
    nothing
3 ✔
590
end
591

592
function deserialize(s::AbstractSerializer, ::Type{Core.CancellationTokenSource})
3 ✔
593
    np = Int(deserialize(s)::Int64)
3 ✔
594
    parents = Vector{Any}(undef, np)
3 ✔
595
    for i in 1:np
3 ✔
596
        parents[i] = deserialize(s)::Core.CancellationTokenSource
3 ✔
597
    end
3 ✔
598
    src = Core._new_cancel_source(parents...)::Core.CancellationTokenSource
3 ✔
599
    deserialize_cycle(s, src)
3 ✔
600
    state = deserialize(s)::UInt8
3 ✔
601
    if state != 0x00
3 ✔
602
        # reject severities the API cannot produce rather than letting a
603
        # corrupt stream inject an unescalatable bogus level
604
        if !(state == 0x1 || state == 0x3 || state == 0x4)
3 ✔
605
            throw(ArgumentError("invalid cancellation severity $(repr(state)) in serialized CancellationTokenSource"))
3 ✔
606
        end
607
        # CAS-max with whatever the relinking inherited from the parents;
608
        # severities only ever escalate
609
        Base._raise_state!(src, state)
3 ✔
610
    end
611
    return src
3 ✔
612
end
613

614
# Core.WaitEntryN has a hidden variable-length slot tail that the generic
615
# deserializer - which allocates only the fixed datatype size - cannot
616
# reproduce (the GC would then scan a nonexistent tail). Its contents are
617
# transient wait-registration state that does not round-trip: only the slot
618
# count is written (plus a `nothing` task), and deserialization
619
# reconstructs through the runtime allocator with fresh, free slots.
620
function serialize(s::AbstractSerializer, w::Core.WaitEntryN)
3 ✔
621
    serialize_cycle_header(s, w) && return
3 ✔
622
    serialize(s, nothing)  # task: transient, never round-trips
3 ✔
623
    serialize(s, Int64(w.nslots))
3 ✔
624
    nothing
3 ✔
625
end
626

627
function deserialize(s::AbstractSerializer, ::Type{Core.WaitEntryN})
3 ✔
628
    task = deserialize(s)::Union{Task, Nothing}
3 ✔
629
    ns = Int(deserialize(s)::Int64)
3 ✔
630
    0 <= ns <= typemax(UInt32) ||
3 ✔
631
        throw(ArgumentError("invalid slot count $ns in serialized WaitEntryN"))
632
    w = Base.WaitEntryN(task, ns)
3 ✔
633
    deserialize_cycle(s, w)
3 ✔
634
    return w
3 ✔
635
end
636

637

638
function serialize(s::AbstractSerializer, linfo::Core.MethodInstance)
3 ✔
639
    serialize_cycle(s, linfo) && return
3 ✔
640
    writetag(s.io, METHODINSTANCE_TAG)
3 ✔
641
    serialize(s, nothing)  # for backwards compat
3 ✔
642
    serialize(s, linfo.sparam_vals)
3 ✔
643
    serialize(s, Any)  # for backwards compat
3 ✔
644
    serialize(s, linfo.specTypes)
3 ✔
645
    serialize(s, linfo.def)
3 ✔
646
    nothing
3 ✔
647
end
648

649
function serialize(s::AbstractSerializer, @nospecialize(u::Union))
168 ✔
650
    serialize_type(s, Union, false)
168 ✔
651
    serialize(s, u.a)
168 ✔
652
    serialize(s, u.b)
168 ✔
653
end
654

655
function serialize(s::AbstractSerializer, t::Task)
6 ✔
656
    serialize_cycle(s, t) && return
6 ✔
657
    if istaskstarted(t) && !istaskdone(t)
6 ✔
658
        error("cannot serialize a running Task")
3 ✔
659
    end
660
    # Compiler-injected CodeInstances are process-local optimization metadata and are
661
    # intentionally omitted. Other targets carry explicit Core.invoke semantics.
662
    has_invoked = isdefined(t, :invoked) && !(getfield(t, :invoked) isa Core.CodeInstance)
6 ✔
663
    writetag(s.io, TASK_TAG)
6 ✔
664
    serialize(s, t.code)
6 ✔
665
    serialize(s, t.storage)
6 ✔
666
    serialize(s, t.state)
6 ✔
667
    if t._isexception && (stk = Base.current_exceptions(t); !isempty(stk))
6 ✔
668
        # the exception stack field is hidden inside the task, so if there
669
        # is any information there make a CapturedException from it instead.
670
        # TODO: Handle full exception chain, not just the first one.
671
        serialize(s, CapturedException(stk[1].exception, stk[1].backtrace))
6 ✔
672
    else
673
        serialize(s, t.result)
3 ✔
674
    end
675
    serialize(s, t._isexception)
6 ✔
676
    serialize(s, has_invoked)
6 ✔
677
    if has_invoked
6 ✔
678
        serialize(s, getfield(t, :invoked))
3 ✔
679
    end
680
end
681

682
function serialize(s::AbstractSerializer, g::GlobalRef)
189 ✔
683
    if (g.mod === __deserialized_types__ ) ||
189 ✔
684
        (g.mod === Main && isdefined(g.mod, g.name) && isconst(g.mod, g.name))
685

686
        v = getglobal(g.mod, g.name)
28 ✔
687
        unw = unwrap_unionall(v)
28 ✔
688
        if isa(unw,DataType) && v === unw.name.wrapper && should_send_whole_type(s, unw)
28 ✔
689
            # handle references to types in Main by sending the whole type.
690
            # needed to be able to send nested functions (#15451).
691
            writetag(s.io, FULL_GLOBALREF_TAG)
15 ✔
692
            serialize(s, v)
15 ✔
693
            return
15 ✔
694
        end
695
    end
696
    writetag(s.io, GLOBALREF_TAG)
189 ✔
697
    serialize(s, g.mod)
189 ✔
698
    serialize(s, g.name)
189 ✔
699
end
700

701
function serialize(s::AbstractSerializer, t::Core.TypeName)
3 ✔
702
    serialize_cycle(s, t) && return
3 ✔
703
    writetag(s.io, TYPENAME_TAG)
3 ✔
704
    write(s.io, object_number(s, t))
3 ✔
705
    serialize_typename(s, t)
3 ✔
706
end
707

708
function serialize_typename(s::AbstractSerializer, t::Core.TypeName)
189 ✔
709
    serialize(s, t.name)
189 ✔
710
    serialize(s, t.names)
189 ✔
711
    primary = unwrap_unionall(t.wrapper)
189 ✔
712
    serialize(s, primary.super)
189 ✔
713
    serialize(s, primary.parameters)
189 ✔
714
    serialize(s, primary.types)
189 ✔
715
    serialize(s, Base.issingletontype(primary))
189 ✔
716
    serialize(s, t.flags & 0x1 == 0x1) # .abstract
189 ✔
717
    serialize(s, t.flags & 0x2 == 0x2) # .mutable
189 ✔
718
    serialize(s, Int32(length(primary.types) - t.n_uninitialized))
189 ✔
719
    serialize(s, t.max_methods)
189 ✔
720
    ms = Base.matches_to_methods(Base._methods_by_ftype(Tuple{t.wrapper, Vararg}, -1, Base.get_world_counter()), t, nothing).ms
189 ✔
721
    if t.singletonname !== t.name || !isempty(ms)
189 ✔
722
        serialize(s, t.singletonname)
189 ✔
723
        serialize(s, ms)
189 ✔
724
        serialize(s, t.max_args)
189 ✔
725
        kws = Base.matches_to_methods(Base._methods_by_ftype(Tuple{typeof(Core.kwcall), Any, t.wrapper, Vararg}, -1, Base.get_world_counter()), t, nothing).ms
189 ✔
726
        if isempty(kws)
189 ✔
727
            writetag(s.io, UNDEFREF_TAG)
189 ✔
728
        else
729
            serialize(s, kws)
3 ✔
730
        end
731
    else
UNCOV
732
        writetag(s.io, UNDEFREF_TAG)
×
733
    end
734
    nothing
189 ✔
735
end
736

737
# decide whether to send all data for a type (instead of just its name)
738
function should_send_whole_type(s, t::DataType)
384 ✔
739
    tn = t.name
384 ✔
740
    # TODO improve somehow?
741
    # send whole type for anonymous functions in Main
742
    name = tn.singletonname
384 ✔
743
    mod = tn.module
384 ✔
744
    mod === __deserialized_types__ && return true
384 ✔
745
    isanonfunction = mod === Main && # only Main
384 ✔
746
        t.super === Function && # only Functions
747
        unsafe_load(unsafe_convert(Ptr{UInt8}, tn.name)) == UInt8('#') && # hidden type
748
        (!isdefined(mod, name) || t != typeof(getglobal(mod, name))) # XXX: 95% accurate test for this being an inner function
749
        # TODO: more accurate test? (tn.name !== "#" name)
750
    return isanonfunction
384 ✔
751
end
752

753
function serialize_type_data(s, @nospecialize(t::DataType))
384 ✔
754
    whole = should_send_whole_type(s, t)
384 ✔
755
    iswrapper = (t === unwrap_unionall(t.name.wrapper))
384 ✔
756
    if whole && iswrapper
384 ✔
757
        writetag(s.io, WRAPPER_DATATYPE_TAG)
189 ✔
758
        serialize(s, t.name)
189 ✔
759
        return
189 ✔
760
    end
761
    serialize_cycle(s, t) && return
384 ✔
762
    if whole
384 ✔
763
        writetag(s.io, FULL_DATATYPE_TAG)
180 ✔
764
        serialize(s, t.name)
180 ✔
765
    else
766
        writetag(s.io, DATATYPE_TAG)
384 ✔
767
        serialize(s, nameof(t))
384 ✔
768
        serialize(s, parentmodule(t))
384 ✔
769
    end
770
    if !isempty(t.parameters)
384 ✔
771
        if iswrapper
379 ✔
772
            write(s.io, Int32(0))
3 ✔
773
        else
774
            write(s.io, Int32(length(t.parameters)))
379 ✔
775
            for p in t.parameters
379 ✔
776
                serialize(s, p)
379 ✔
777
            end
379 ✔
778
        end
779
    end
780
    nothing
384 ✔
781
end
782

783
function serialize(s::AbstractSerializer, t::DataType)
409 ✔
784
    tag = sertag(t)
409 ✔
785
    tag > 0 && return write_as_tag(s.io, tag)
409 ✔
786
    if t === Tuple
364 ✔
787
        # `sertag` is not able to find types === to `Tuple` because they
788
        # will not have been hash-consed. Plus `serialize_type_data` does not
789
        # handle this case correctly, since Tuple{} != Tuple. `Tuple` is the
790
        # only type with this property. issue #15849
UNCOV
791
        return write_as_tag(s.io, TUPLE_TAG)
×
792
    end
793
    serialize_type_data(s, t)
364 ✔
794
end
795

796
function serialize_type(s::AbstractSerializer, @nospecialize(t::DataType), ref::Bool = false)
384 ✔
797
    tag = sertag(t)
384 ✔
798
    tag > 0 && return writetag(s.io, tag)
384 ✔
799
    writetag(s.io, ref ? REF_OBJECT_TAG : OBJECT_TAG)
384 ✔
800
    serialize_type_data(s, t)
384 ✔
801
end
802

803
function serialize(s::AbstractSerializer, n::Int32)
337 ✔
804
    if 0 <= n <= (n_int_literals-1)
337 ✔
805
        write(s.io, UInt8(ZERO32_TAG+n))
198 ✔
806
    else
807
        writetag(s.io, INT32_TAG)
337 ✔
808
        write(s.io, n)
337 ✔
809
    end
810
    nothing
337 ✔
811
end
812

813
function serialize(s::AbstractSerializer, n::Int64)
521 ✔
814
    if 0 <= n <= (n_int_literals-1)
521 ✔
815
        write(s.io, UInt8(ZERO64_TAG+n))
521 ✔
816
    elseif typemin(Int32) <= n <= typemax(Int32)
425 ✔
817
        writetag(s.io, SHORTINT64_TAG)
425 ✔
818
        write(s.io, Int32(n))
425 ✔
819
    else
820
        writetag(s.io, INT64_TAG)
85 ✔
821
        write(s.io, n)
85 ✔
822
    end
823
    nothing
521 ✔
824
end
825

826
for i in 0:13
827
    tag = Int32(INT8_TAG + i)
828
    ty = TAGS[tag]
829
    (ty === Int32 || ty === Int64) && continue
830
    @eval serialize(s::AbstractSerializer, n::$ty) = (writetag(s.io, $tag); write(s.io, n); nothing)
393 ✔
831
end
832

833
serialize(s::AbstractSerializer, ::Type{Bottom}) = write_as_tag(s.io, BOTTOM_TAG)
269 ✔
834

835
function serialize(s::AbstractSerializer, u::UnionAll)
180 ✔
836
    writetag(s.io, UNIONALL_TAG)
180 ✔
837
    n = 0; t = u
180 ✔
838
    while isa(t, UnionAll)
180 ✔
839
        t = t.body
180 ✔
840
        n += 1
180 ✔
841
    end
180 ✔
842
    if isa(t, DataType) && t === unwrap_unionall(t.name.wrapper)
180 ✔
843
        write(s.io, UInt8(1))
180 ✔
844
        write(s.io, Int16(n))
180 ✔
845
        serialize(s, t)
180 ✔
846
    else
847
        write(s.io, UInt8(0))
3 ✔
848
        serialize(s, u.var)
3 ✔
849
        serialize(s, u.body)
3 ✔
850
    end
851
end
852

853
serialize(s::AbstractSerializer, @nospecialize(x)) = serialize_any(s, x)
401 ✔
854

855
function serialize(s::AbstractSerializer, x::Core.AddrSpace)
68 ✔
856
    serialize_type(s, typeof(x))
68 ✔
857
    write(s.io, Core.bitcast(UInt8, x))
68 ✔
858
end
859

860
function serialize(s::AbstractSerializer, x::Core.IntrinsicFunction)
3 ✔
861
    serialize_type(s, typeof(x))
3 ✔
862
    serialize(s, nameof(x))
3 ✔
863
end
864

865
function serialize_any(s::AbstractSerializer, @nospecialize(x))
401 ✔
866
    tag = sertag(x)
401 ✔
867
    if tag > 0
401 ✔
868
        return write_as_tag(s.io, tag)
399 ✔
869
    end
870
    t = typeof(x)::DataType
384 ✔
871
    if isprimitivetype(t)
384 ✔
872
        serialize_type(s, t)
109 ✔
873
        write(s.io, x)
109 ✔
874
    else
875
        if ismutable(x)
384 ✔
876
            serialize_cycle(s, x) && return
290 ✔
877
            serialize_type(s, t, true)
290 ✔
878
        else
879
            serialize_type(s, t, false)
378 ✔
880
        end
881
        nf = nfields(x)
384 ✔
882
        for i in 1:nf
384 ✔
883
            if isdefined(x, i)
384 ✔
884
                serialize(s, getfield(x, i))
384 ✔
885
            else
886
                writetag(s.io, UNDEFREF_TAG)
6 ✔
887
            end
888
        end
384 ✔
889
    end
890
    nothing
384 ✔
891
end
892

893
"""
894
    Serialization.writeheader(s::AbstractSerializer)
895

896
Write an identifying header to the specified serializer. The header consists of
897
8 bytes as follows:
898

899
| Offset | Description                                     |
900
|:-------|:------------------------------------------------|
901
|   0    | tag byte (0x37)                                 |
902
|   1-2  | signature bytes "JL"                            |
903
|   3    | protocol version                                |
904
|   4    | bits 0-1: endianness: 0 = little, 1 = big       |
905
|   4    | bits 2-3: platform: 0 = 32-bit, 1 = 64-bit      |
906
|   5-7  | reserved                                        |
907
"""
908
function writeheader(s::AbstractSerializer)
274 ✔
909
    io = s.io
274 ✔
910
    writetag(io, HEADER_TAG)
274 ✔
911
    write(io, "JL")  # magic bytes
274 ✔
912
    write(io, UInt8(ser_version))
274 ✔
913
    endianness = (ENDIAN_BOM == 0x04030201 ? 0 :
274 ✔
914
                  ENDIAN_BOM == 0x01020304 ? 1 :
915
                  error("unsupported endianness in serializer"))
916
    machine = (sizeof(Int) == 4 ? 0 :
274 ✔
917
               sizeof(Int) == 8 ? 1 :
918
               error("unsupported word size in serializer"))
919
    write(io, UInt8(endianness) | (UInt8(machine) << 2))
274 ✔
920
    write(io, [0x00,0x00,0x00]) # 3 reserved bytes
274 ✔
921
    nothing
274 ✔
922
end
923

924
function readheader(s::AbstractSerializer)
135 ✔
925
    # Tag already read
926
    io = s.io
135 ✔
927
    m1 = read(io, UInt8)
135 ✔
928
    m2 = read(io, UInt8)
135 ✔
929
    if m1 != UInt8('J') || m2 != UInt8('L')
135 ✔
930
        error("Unsupported serialization format (got header magic bytes $m1 $m2)")
3 ✔
931
    end
932
    version    = read(io, UInt8)
135 ✔
933
    flags      = read(io, UInt8)
135 ✔
934
    reserved1  = read(io, UInt8)
135 ✔
935
    reserved2  = read(io, UInt8)
135 ✔
936
    reserved3  = read(io, UInt8)
135 ✔
937
    endianflag = flags & 0x3
135 ✔
938
    wordflag   = (flags >> 2) & 0x3
135 ✔
939
    wordsize = wordflag == 0 ? 4 :
135 ✔
940
               wordflag == 1 ? 8 :
941
               error("Unknown word size flag in header")
942
    endian_bom = endianflag == 0 ? 0x04030201 :
135 ✔
943
                 endianflag == 1 ? 0x01020304 :
944
                 error("Unknown endianness flag in header")
945
    # Check protocol compatibility.
946
    endian_bom == ENDIAN_BOM  || error("Serialized byte order mismatch ($(repr(endian_bom)))")
135 ✔
947
    # We don't check wordsize == sizeof(Int) here, as Int is encoded concretely
948
    # as Int32 or Int64, which should be enough to correctly deserialize a range
949
    # of data structures between Julia versions.
950
    if version > ser_version
135 ✔
951
        error("""Cannot read stream serialized with a newer version of Julia.
3 ✔
952
                 Got data version $version > current version $ser_version""")
953
    end
954
    s.version = version
135 ✔
955
    return
135 ✔
956
end
957

958
"""
959
    serialize(stream::IO, value)
960

961
Write an arbitrary value to a stream in an opaque format, such that it can be read back by
962
[`deserialize`](@ref). The read-back value will be as identical as possible to the original,
963
but note that `Ptr` values are serialized as all-zero bit patterns (`NULL`).
964

965
An 8-byte identifying header is written to the stream first. To avoid writing the header,
966
construct a `Serializer` and use it as the first argument to `serialize` instead.
967
See also [`Serialization.writeheader`](@ref).
968

969
The data format can change in minor (1.x) Julia releases, but files written by prior 1.x
970
versions will remain readable. The main exception to this is when the definition of a
971
type in an external package changes. If that occurs, it may be necessary to specify
972
an explicit compatible version of the affected package in your environment.
973
Renaming functions, even private functions, inside packages can also put existing files
974
out of sync. Anonymous functions require special care: because their names are automatically
975
generated, minor code changes can cause them to be renamed.
976
Serializing anonymous functions should be avoided in files intended for long-term storage.
977

978
In some cases, the word size (32- or 64-bit) of the reading and writing machines must match.
979
In rarer cases the OS or architecture must also match, for example when using packages
980
that contain platform-dependent code.
981
"""
982
function serialize(s::IO, x)
274 ✔
983
    ss = Serializer(s)
274 ✔
984
    writeheader(ss)
274 ✔
985
    serialize(ss, x)
274 ✔
986
end
987

988
"""
989
    serialize(filename::AbstractString, value)
990

991
Open a file and serialize the given value to it.
992

993
!!! compat "Julia 1.1"
994
    This method is available as of Julia 1.1.
995
"""
996
serialize(filename::AbstractString, x) = open(io->serialize(io, x), filename, "w")
171 ✔
997

998
## deserializing values ##
999

1000
"""
1001
    deserialize(stream)
1002

1003
Read a value written by [`serialize`](@ref). `deserialize` assumes the binary data read from
1004
`stream` is correct and has been serialized by a compatible implementation of [`serialize`](@ref).
1005
`deserialize` is designed for simplicity and performance, and so does not validate
1006
the data read. Malformed data can result in process termination. The caller must ensure
1007
the integrity and correctness of data read from `stream`.
1008

1009
!!! warning
1010
    `deserialize` is not secure against maliciously constructed data, which can execute
1011
    arbitrary code when deserialized. Never deserialize data received from an untrusted or
1012
    unauthenticated source.
1013
"""
1014
deserialize(s::IO) = deserialize(Serializer(s))
135 ✔
1015

1016
"""
1017
    deserialize(filename::AbstractString)
1018

1019
Open a file and deserialize its contents.
1020

1021
!!! compat "Julia 1.1"
1022
    This method is available as of Julia 1.1.
1023
"""
1024
deserialize(filename::AbstractString) = open(deserialize, filename)
6 ✔
1025

1026
function deserialize(s::AbstractSerializer)
521 ✔
1027
    handle_deserialize(s, Int32(read(s.io, UInt8)::UInt8))
521 ✔
1028
end
1029

1030
function deserialize_cycle(s::AbstractSerializer, @nospecialize(x))
241 ✔
1031
    slot = pop!(s.pending_refs)
241 ✔
1032
    _setbackref!(s, slot, x)
241 ✔
1033
    nothing
241 ✔
1034
end
1035

1036
# optimized version of:
1037
#     slot = s.counter; s.counter += 1
1038
#     push!(s.pending_refs, slot)
1039
#     slot = pop!(s.pending_refs)
1040
#     _setbackref!(s, slot, x)
1041
function resolve_ref_immediately(s::AbstractSerializer, @nospecialize(x))
490 ✔
1042
    _setbackref!(s, s.counter, x)
490 ✔
1043
    s.counter += 1
490 ✔
1044
    nothing
490 ✔
1045
end
1046

1047
# deserialize_ is an internal function to dispatch on the tag
1048
# describing the serialized representation. the number of
1049
# representations is fixed, so deserialize_ does not get extended.
1050
function handle_deserialize(s::AbstractSerializer, b::Int32)
521 ✔
1051
    if b == 0
521 ✔
1052
        return desertag(Int32(read(s.io, UInt8)::UInt8))
502 ✔
1053
    end
1054
    if b >= VALUE_TAGS
521 ✔
1055
        return desertag(b)
521 ✔
1056
    elseif b == TUPLE_TAG
521 ✔
1057
        return deserialize_tuple(s, Int(read(s.io, UInt8)::UInt8))
389 ✔
1058
    elseif b == SHORTBACKREF_TAG
521 ✔
1059
        id = read(s.io, UInt16)::UInt16
487 ✔
1060
        return _getbackref(s, Int(id))
487 ✔
1061
    elseif b == BACKREF_TAG
521 ✔
1062
        id = read(s.io, Int32)::Int32
6 ✔
1063
        return _getbackref(s, Int(id))
6 ✔
1064
    elseif b == ARRAY_TAG
521 ✔
1065
        return deserialize_array(s)
413 ✔
1066
    elseif b == DATATYPE_TAG
521 ✔
1067
        return deserialize_datatype(s, false)
481 ✔
1068
    elseif b == FULL_DATATYPE_TAG
521 ✔
1069
        return deserialize_datatype(s, true)
33 ✔
1070
    elseif b == WRAPPER_DATATYPE_TAG
521 ✔
1071
        tname = deserialize(s)::Core.TypeName
123 ✔
1072
        return unwrap_unionall(tname.wrapper)
123 ✔
1073
    elseif b == OBJECT_TAG
521 ✔
1074
        t = deserialize(s)
481 ✔
1075
        if t === Missing
481 ✔
1076
            return missing
3 ✔
1077
        end
1078
        return deserialize(s, t)
481 ✔
1079
    elseif b == REF_OBJECT_TAG
521 ✔
1080
        slot = s.counter; s.counter += 1
166 ✔
1081
        push!(s.pending_refs, slot)
166 ✔
1082
        t = deserialize(s)
166 ✔
1083
        return deserialize(s, t)
166 ✔
1084
    elseif b == SHARED_REF_TAG
521 ✔
1085
        slot = s.counter; s.counter += 1
466 ✔
1086
        obj = deserialize(s)
466 ✔
1087
        _setbackref!(s, slot, obj)
466 ✔
1088
        return obj
466 ✔
1089
    elseif b == SYMBOL_TAG
521 ✔
1090
        return deserialize_symbol(s, Int(read(s.io, UInt8)::UInt8))
517 ✔
1091
    elseif b == SHORTINT64_TAG
518 ✔
1092
        return Int64(read(s.io, Int32)::Int32)
454 ✔
1093
    elseif b == EXPR_TAG
508 ✔
1094
        return deserialize_expr(s, Int(read(s.io, UInt8)::UInt8))
311 ✔
1095
    elseif b == MODULE_TAG
508 ✔
1096
        return deserialize_module(s)
481 ✔
1097
    elseif b == STRING_TAG
508 ✔
1098
        return deserialize_string(s, Int(read(s.io, UInt8)::UInt8))
487 ✔
1099
    elseif b == LONGSTRING_TAG
487 ✔
1100
        return deserialize_string(s, Int(read(s.io, Int64)::Int64))
122 ✔
1101
    elseif b == SIMPLEVECTOR_TAG
487 ✔
1102
        return deserialize_svec(s)
123 ✔
1103
    elseif b == GLOBALREF_TAG
487 ✔
1104
        return GlobalRef(deserialize(s)::Module, deserialize(s)::Symbol)
123 ✔
1105
    elseif b == FULL_GLOBALREF_TAG
487 ✔
1106
        ty = deserialize(s)
18 ✔
1107
        tn = unwrap_unionall(ty).name
18 ✔
1108
        return GlobalRef(tn.module, tn.name)
18 ✔
1109
    elseif b == LONGTUPLE_TAG
487 ✔
1110
        return deserialize_tuple(s, Int(read(s.io, Int32)::Int32))
3 ✔
1111
    elseif b == LONGEXPR_TAG
487 ✔
1112
        return deserialize_expr(s, Int(read(s.io, Int32)::Int32))
3 ✔
1113
    elseif b == LONGBACKREF_TAG
487 ✔
UNCOV
1114
        id = read(s.io, Int64)::Int64
×
UNCOV
1115
        return _getbackref(s, Int(id))
×
1116
    elseif b == LONGSYMBOL_TAG
487 ✔
1117
        return deserialize_symbol(s, Int(read(s.io, Int32)::Int32))
103 ✔
1118
    elseif b == HEADER_TAG
487 ✔
1119
        readheader(s)
135 ✔
1120
        return deserialize(s)
135 ✔
1121
    elseif b == INT8_TAG
487 ✔
1122
        return read(s.io, Int8)
21 ✔
1123
    elseif b == INT8_TAG+1
487 ✔
1124
        return read(s.io, UInt8)
126 ✔
1125
    elseif b == INT8_TAG+2
487 ✔
1126
        return read(s.io, Int16)
21 ✔
1127
    elseif b == INT8_TAG+3
487 ✔
1128
        return read(s.io, UInt16)
129 ✔
1129
    elseif b == INT32_TAG
487 ✔
1130
        return read(s.io, Int32)
109 ✔
1131
    elseif b == INT8_TAG+5
487 ✔
1132
        return read(s.io, UInt32)
39 ✔
1133
    elseif b == INT64_TAG
487 ✔
1134
        return read(s.io, Int64)
6 ✔
1135
    elseif b == INT8_TAG+7
487 ✔
1136
        return read(s.io, UInt64)
128 ✔
1137
    elseif b == INT8_TAG+8
487 ✔
1138
        return read(s.io, Int128)
6 ✔
1139
    elseif b == INT8_TAG+9
487 ✔
1140
        return read(s.io, UInt128)
481 ✔
1141
    elseif b == INT8_TAG+10
318 ✔
1142
        return read(s.io, Float16)
6 ✔
1143
    elseif b == INT8_TAG+11
318 ✔
1144
        return read(s.io, Float32)
3 ✔
1145
    elseif b == INT8_TAG+12
318 ✔
1146
        return read(s.io, Float64)
42 ✔
1147
    elseif b == INT8_TAG+13
317 ✔
1148
        return read(s.io, Char)
21 ✔
1149
    elseif b == IDDICT_TAG
317 ✔
1150
        slot = s.counter; s.counter += 1
3 ✔
1151
        push!(s.pending_refs, slot)
3 ✔
1152
        t = deserialize(s)
3 ✔
1153
        return deserialize_dict(s, t)
3 ✔
1154
    end
1155
    t = desertag(b)::DataType
317 ✔
1156
    if ismutabletype(t) && fieldcount(t) > 0
317 ✔
1157
        slot = s.counter; s.counter += 1
123 ✔
1158
        push!(s.pending_refs, slot)
123 ✔
1159
    end
1160
    return deserialize(s, t)
317 ✔
1161
end
1162

1163
function deserialize_symbol(s::AbstractSerializer, len::Int)
517 ✔
1164
    str = Base._string_n(len)
517 ✔
1165
    unsafe_read(s.io, pointer(str), len)
517 ✔
1166
    sym = Symbol(str)
517 ✔
1167
    if len > 7
517 ✔
1168
        resolve_ref_immediately(s, sym)
490 ✔
1169
    end
1170
    return sym
517 ✔
1171
end
1172

1173
function deserialize_tuple(s::AbstractSerializer, len)
389 ✔
1174
    len == 0 && return ()
389 ✔
1175
    Base.Cartesian.@nexprs 10 i -> begin
389 ✔
1176
        len == i && return (Base.Cartesian.@ntuple i _ -> deserialize(s))
1177
    end
1178
    return ntupleany(i -> deserialize(s), len)
3 ✔
1179
end
1180

1181
function deserialize_svec(s::AbstractSerializer)
123 ✔
1182
    n = read(s.io, Int32)
123 ✔
1183
    svec(Any[ deserialize(s) for i=1:n ]...)
123 ✔
1184
end
1185

1186
function deserialize_module(s::AbstractSerializer)
481 ✔
1187
    mkey = deserialize(s)
481 ✔
1188
    if isa(mkey, Tuple)
481 ✔
1189
        # old version, TODO: remove
UNCOV
1190
        if mkey === ()
×
1191
            return Main
×
1192
        end
1193
        m = Base.root_module(mkey[1])
×
1194
        for i = 2:length(mkey)
×
UNCOV
1195
            m = getglobal(m, mkey[i])::Module
×
UNCOV
1196
        end
×
1197
    else
1198
        name = String(deserialize(s)::Symbol)
481 ✔
1199
        pkg = (mkey === nothing) ? Base.PkgId(name) : Base.PkgId(Base.UUID(mkey), name)
481 ✔
1200
        m = Base.root_module(pkg)
481 ✔
1201
        mname = deserialize(s)
481 ✔
1202
        while mname !== ()
481 ✔
1203
            m = getglobal(m, mname)::Module
338 ✔
1204
            mname = deserialize(s)
338 ✔
1205
        end
338 ✔
1206
    end
1207
    return m
481 ✔
1208
end
1209

1210
function deserialize(s::AbstractSerializer, ::Type{Method})
123 ✔
1211
    lnumber = read(s.io, UInt64)
123 ✔
1212
    meth = lookup_object_number(s, lnumber)
123 ✔
1213
    if meth !== nothing
123 ✔
1214
        meth = meth::Method
3 ✔
1215
        makenew = false
3 ✔
1216
    else
1217
        meth = ccall(:jl_new_method_uninit, Ref{Method}, (Any,), Main)
123 ✔
1218
        makenew = true
123 ✔
1219
    end
1220
    deserialize_cycle(s, meth)
123 ✔
1221
    mod = deserialize(s)::Module
123 ✔
1222
    name = deserialize(s)::Symbol
123 ✔
1223
    file = deserialize(s)::Symbol
123 ✔
1224
    line = deserialize(s)::Int32
123 ✔
1225
    sig = deserialize(s)::Type
123 ✔
1226
    syms = deserialize(s)
123 ✔
1227
    if syms isa SimpleVector
123 ✔
1228
        # < v1.2
1229
        _ambig = deserialize(s)
3 ✔
1230
    else
1231
        slot_syms = syms::String
123 ✔
1232
    end
1233
    nargs = deserialize(s)::Int32
123 ✔
1234
    isva = deserialize(s)::Bool
123 ✔
1235
    is_for_opaque_closure = false
123 ✔
1236
    nospecializeinfer = false
123 ✔
1237
    constprop = 0x00
123 ✔
1238
    purity = 0x0000
123 ✔
1239
    template_or_is_opaque = with(current_module => mod) do
123 ✔
1240
        deserialize(s)
123 ✔
1241
    end
1242
    template = if isa(template_or_is_opaque, Bool)
123 ✔
1243
        is_for_opaque_closure = template_or_is_opaque
123 ✔
1244
        if format_version(s) >= 24
123 ✔
1245
            nospecializeinfer = deserialize(s)::Bool
123 ✔
1246
        end
1247
        if format_version(s) >= 14
123 ✔
1248
            constprop = deserialize(s)::UInt8
123 ✔
1249
        end
1250
        if format_version(s) >= 26
123 ✔
1251
            purity = deserialize(s)::UInt16
123 ✔
UNCOV
1252
        elseif format_version(s) >= 17
×
UNCOV
1253
            purity = UInt16(deserialize(s)::UInt8)
×
1254
        end
1255
        with(current_module => mod) do
123 ✔
1256
            deserialize(s)
123 ✔
1257
        end
1258
    else
1259
        template_or_is_opaque
123 ✔
1260
    end
1261
    generator = deserialize(s)
123 ✔
1262
    recursion_relation = nothing
123 ✔
1263
    if format_version(s) >= 15
123 ✔
1264
        recursion_relation = deserialize(s)
123 ✔
1265
    end
1266
    if makenew
123 ✔
1267
        meth.module = mod
123 ✔
1268
        meth.debuginfo = NullDebugInfo
123 ✔
1269
        meth.name = name
123 ✔
1270
        meth.file = file
123 ✔
1271
        meth.line = line
123 ✔
1272
        meth.sig = sig
123 ✔
1273
        meth.nargs = nargs
123 ✔
1274
        meth.isva = isva
123 ✔
1275
        meth.is_for_opaque_closure = is_for_opaque_closure
123 ✔
1276
        meth.nospecializeinfer = nospecializeinfer
123 ✔
1277
        meth.constprop = constprop
123 ✔
1278
        meth.purity = purity
123 ✔
1279
        if template !== nothing
123 ✔
1280
            # TODO: compress template
1281
            template = template::CodeInfo
123 ✔
1282
            if format_version(s) < 29
123 ✔
1283
                template.nargs = nargs
3 ✔
1284
                template.isva = isva
3 ✔
1285
            end
1286
            meth.source = template
123 ✔
1287
            meth.debuginfo = template.debuginfo
123 ✔
1288
            if !@isdefined(slot_syms)
123 ✔
1289
                slot_syms = ccall(:jl_compress_argnames, Ref{String}, (Any,), meth.source.slotnames)
3 ✔
1290
            end
1291
        end
1292
        meth.slot_syms = slot_syms
123 ✔
1293
        if generator !== nothing
123 ✔
1294
            meth.generator = generator
3 ✔
1295
        end
1296
        if recursion_relation !== nothing
123 ✔
1297
            meth.recursion_relation = recursion_relation
3 ✔
1298
        end
1299
        if !is_for_opaque_closure
123 ✔
1300
            mt = Core.methodtable
123 ✔
1301
            if nothing === ccall(:jl_methtable_lookup, Any, (Any, UInt), sig, Base.get_world_counter()) # XXX: quite sketchy?
123 ✔
1302
                ccall(:jl_method_table_insert, Cvoid, (Any, Any, Ptr{Cvoid}), mt, meth, C_NULL)
123 ✔
1303
            end
1304
        end
1305
        remember_object(s, meth, lnumber)
123 ✔
1306
    end
1307
    return meth
123 ✔
1308
end
1309

1310
function deserialize(s::AbstractSerializer, ::Type{Core.MethodTable})
3 ✔
1311
    name = deserialize(s)::Symbol
3 ✔
1312
    mod = deserialize(s)::Module
3 ✔
1313
    return getglobal(mod, name)::Core.MethodTable
3 ✔
1314
end
1315

1316
function deserialize(s::AbstractSerializer, ::Type{Core.MethodInstance})
3 ✔
1317
    linfo = ccall(:jl_new_method_instance_uninit, Ref{Core.MethodInstance}, (Ptr{Cvoid},), C_NULL)
3 ✔
1318
    deserialize_cycle(s, linfo)
3 ✔
1319
    if format_version(s) < 28
3 ✔
1320
        tag = Int32(read(s.io, UInt8)::UInt8)
×
1321
        if tag != UNDEFREF_TAG
×
1322
            code = handle_deserialize(s, tag)::CodeInfo
×
UNCOV
1323
            ci = ccall(:jl_new_codeinst_for_uninferred, Ref{CodeInstance}, (Any, Any), linfo, code)
×
UNCOV
1324
            @atomic linfo.cache = ci
×
1325
        end
1326
    end
1327
    tag = Int32(read(s.io, UInt8)::UInt8)
3 ✔
1328
    if tag != UNDEFREF_TAG
3 ✔
1329
        # for reading files prior to v1.2
1330
        handle_deserialize(s, tag)
3 ✔
1331
    end
1332
    linfo.sparam_vals = deserialize(s)::SimpleVector
3 ✔
1333
    _rettype = deserialize(s)  # for backwards compat
3 ✔
1334
    linfo.specTypes = deserialize(s)
3 ✔
1335
    linfo.def = deserialize(s)
3 ✔
1336
    return linfo
3 ✔
1337
end
1338

1339
function deserialize(s::AbstractSerializer, ::Type{Core.LineInfoNode})
3 ✔
1340
    mod = deserialize(s)
3 ✔
1341
    if mod isa Module
3 ✔
1342
        method = deserialize(s)
3 ✔
1343
    else
1344
        # files post v1.2 and pre v1.6 are broken
UNCOV
1345
        method = mod
×
UNCOV
1346
        mod = Main
×
1347
    end
1348
    return Core.LineInfoNode(mod, method, deserialize(s)::Symbol, Int32(deserialize(s)::Union{Int32, Int}), Int32(deserialize(s)::Union{Int32, Int}))
3 ✔
1349
end
1350

1351

1352
function deserialize(s::AbstractSerializer, ::Type{PhiNode})
3 ✔
1353
    edges = deserialize(s)
3 ✔
1354
    if edges isa Vector{Any}
3 ✔
UNCOV
1355
        edges = Vector{Int32}(edges)
×
1356
    end
1357
    values = deserialize(s)::Vector{Any}
3 ✔
1358
    return PhiNode(edges, values)
3 ✔
1359
end
1360

1361
# v1.12 disallows bare symbols in IR, but older CodeInfos might still have them
1362
function symbol_to_globalref(@nospecialize(x), m::Module)
123 ✔
1363
    mapper(@nospecialize(x)) = symbol_to_globalref(x, m)
123 ✔
1364
    if x isa Symbol
123 ✔
1365
        return GlobalRef(m, x)
3 ✔
1366
    elseif x isa Expr
123 ✔
1367
        return Expr(x.head, map(mapper, x.args)...)
123 ✔
1368
    elseif x isa ReturnNode
123 ✔
1369
        return ReturnNode(mapper(x.val))
123 ✔
1370
    elseif x isa GotoIfNot
123 ✔
1371
        return GotoIfNot(mapper(x.cond), x.dest)
57 ✔
1372
    else
1373
        return x
123 ✔
1374
    end
1375
end
1376

1377
function deserialize(s::AbstractSerializer, ::Type{CodeInfo})
123 ✔
1378
    ci = ccall(:jl_new_code_info_uninit, Ref{CodeInfo}, ())
123 ✔
1379
    deserialize_cycle(s, ci)
123 ✔
1380
    code = deserialize(s)::Vector{Any}
123 ✔
1381
    ci.code = code
123 ✔
1382
    ci.debuginfo = NullDebugInfo
123 ✔
1383
    # allow older-style IR with return and gotoifnot Exprs
1384
    for i in 1:length(code)
123 ✔
1385
        stmt = code[i]
123 ✔
1386
        if isa(stmt, Expr)
123 ✔
1387
            ex = stmt::Expr
123 ✔
1388
            if ex.head === :return
123 ✔
1389
                code[i] = ReturnNode(isempty(ex.args) ? nothing : ex.args[1])
3 ✔
1390
            elseif ex.head === :gotoifnot
123 ✔
UNCOV
1391
                code[i] = GotoIfNot(ex.args[1], ex.args[2])
×
1392
            end
1393
        end
1394
    end
123 ✔
1395
    if current_module[] !== nothing
123 ✔
1396
        map!(x->symbol_to_globalref(x, current_module[]), code)
123 ✔
1397
    end
1398
    _x = deserialize(s)
123 ✔
1399
    have_debuginfo = _x isa Core.DebugInfo
123 ✔
1400
    if have_debuginfo
123 ✔
1401
        ci.debuginfo = _x
123 ✔
1402
    else
1403
        codelocs = _x::Vector{Int32}
3 ✔
1404
        # TODO: convert codelocs to debuginfo format?
1405
    end
1406
    _x = deserialize(s)
123 ✔
1407
    if _x isa Array || _x isa Int
123 ✔
1408
        pre_12 = false
123 ✔
1409
    else
1410
        pre_12 = true
3 ✔
1411
        # < v1.2
1412
        ci.method_for_inference_limit_heuristics = _x
3 ✔
1413
        _x = deserialize(s)
3 ✔
1414
    end
1415
    ci.ssavaluetypes = _x
123 ✔
1416
    if pre_12
123 ✔
1417
        linetable = deserialize(s)
3 ✔
1418
        # TODO: convert linetable to debuginfo format?
1419
    end
1420
    ssaflags = deserialize(s)
123 ✔
1421
    if length(ssaflags) ≠ length(code)
123 ✔
1422
        # make sure the length of `ssaflags` matches that of `code`
1423
        # so that the latest inference doesn't throw on IRs serialized from old versions
1424
        ssaflags = UInt32[0x00 for _ in 1:length(code)]
3 ✔
1425
    elseif eltype(ssaflags) != UInt32
123 ✔
UNCOV
1426
        ssaflags = map(UInt32, ssaflags)
×
1427
    end
1428
    ci.ssaflags = ssaflags
123 ✔
1429
    if pre_12
123 ✔
1430
        ci.slotflags = deserialize(s)
3 ✔
1431
    else
1432
        if format_version(s) <= 26
123 ✔
1433
            ci.method_for_inference_limit_heuristics = deserialize(s)
3 ✔
1434
        end
1435
        if !have_debuginfo # pre v1.11 format
123 ✔
1436
            linetable = deserialize(s)
3 ✔
1437
            # TODO: convert linetable to debuginfo format?
1438
        end
1439
    end
1440
    ci.slotnames = deserialize(s)
123 ✔
1441
    if !pre_12
123 ✔
1442
        ci.slotflags = deserialize(s)
123 ✔
1443
        ci.slottypes = deserialize(s)
123 ✔
1444
        ci.rettype = deserialize(s)
123 ✔
1445
        ci.parent = deserialize(s)
123 ✔
1446
        if format_version(s) < 29 && ci.parent isa MethodInstance && ci.parent.def isa Method
123 ✔
UNCOV
1447
            ci.nargs = ci.parent.def.nargs
×
1448
        end
1449
        world_or_edges = deserialize(s)
123 ✔
1450
        pre_13 = isa(world_or_edges, Union{UInt, Int})
123 ✔
1451
        if pre_13
123 ✔
UNCOV
1452
            ci.min_world = reinterpret(UInt, world_or_edges)
×
UNCOV
1453
            ci.max_world = reinterpret(UInt, deserialize(s))
×
1454
        else
1455
            ci.edges = world_or_edges
123 ✔
1456
            ci.min_world = deserialize(s)::UInt
123 ✔
1457
            ci.max_world = deserialize(s)::UInt
123 ✔
1458
        end
1459
        if format_version(s) >= 29
123 ✔
1460
            ci.method_for_inference_limit_heuristics = deserialize(s)
123 ✔
1461
        end
1462
    end
1463
    if format_version(s) <= 26
123 ✔
1464
        deserialize(s)::Bool # inferred
3 ✔
1465
    end
1466
    if format_version(s) < 22
123 ✔
1467
        inlining_cost = deserialize(s)
3 ✔
1468
        if isa(inlining_cost, Bool)
3 ✔
1469
            Core.Compiler.set_inlineable!(ci, inlining_cost)
3 ✔
1470
        else
UNCOV
1471
            ci.inlining_cost = inlining_cost
×
1472
        end
1473
    end
1474
    if format_version(s) >= 29
123 ✔
1475
        ci.nargs = deserialize(s)
123 ✔
1476
    end
1477
    ci.propagate_inbounds = deserialize(s)
123 ✔
1478
    if format_version(s) < 23
123 ✔
1479
        deserialize(s) # `pure` field has been removed
3 ✔
1480
    end
1481
    if format_version(s) >= 20
123 ✔
1482
        ci.has_fcall = deserialize(s)
123 ✔
1483
    end
1484
    if format_version(s) >= 30
123 ✔
1485
        ci.has_image_globalref = deserialize(s)::Bool
123 ✔
1486
    end
1487
    if format_version(s) >= 24
123 ✔
1488
        ci.nospecializeinfer = deserialize(s)::Bool
123 ✔
1489
    end
1490
    if format_version(s) >= 29
123 ✔
1491
        ci.isva = deserialize(s)::Bool
123 ✔
1492
    end
1493
    if format_version(s) >= 21
123 ✔
1494
        ci.inlining = deserialize(s)::UInt8
123 ✔
1495
    end
1496
    if format_version(s) >= 14
123 ✔
1497
        ci.constprop = deserialize(s)::UInt8
123 ✔
1498
    end
1499
    if format_version(s) >= 26
123 ✔
1500
        ci.purity = deserialize(s)::UInt16
123 ✔
1501
    elseif format_version(s) >= 17
3 ✔
UNCOV
1502
        ci.purity = deserialize(s)::UInt8
×
1503
    end
1504
    if format_version(s) >= 22
123 ✔
1505
        ci.inlining_cost = deserialize(s)::UInt16
123 ✔
1506
    end
1507
    ci.debuginfo = NullDebugInfo
123 ✔
1508
    return ci
123 ✔
1509
end
1510

1511
import Core: NullDebugInfo
1512

1513
if Int === Int64
1514
const OtherInt = Int32
1515
else
1516
const OtherInt = Int64
1517
end
1518

1519
function deserialize_array(s::AbstractSerializer)
413 ✔
1520
    slot = s.counter; s.counter += 1
413 ✔
1521
    d1 = deserialize(s)
413 ✔
1522
    if isa(d1, Type)
413 ✔
1523
        elty = d1
413 ✔
1524
        d1 = deserialize(s)
413 ✔
1525
    else
1526
        elty = UInt8
125 ✔
1527
    end
1528
    if isa(d1, Int32) || isa(d1, Int64)
413 ✔
1529
        if elty !== Bool && isbitstype(elty)
413 ✔
1530
            a = Vector{elty}(undef, d1)
389 ✔
1531
            _setbackref!(s, slot, a)
389 ✔
1532
            return read!(s.io, a)
389 ✔
1533
        end
1534
        dims = (Int(d1),)
413 ✔
1535
    elseif d1 isa Dims
24 ✔
1536
        dims = d1::Dims
24 ✔
1537
    else
UNCOV
1538
        dims = convert(Dims, d1::Tuple{Vararg{OtherInt}})::Dims
×
1539
    end
1540
    if isbitstype(elty)
413 ✔
1541
        n = prod(dims)::Int
21 ✔
1542
        if elty === Bool && n > 0
21 ✔
1543
            A = Array{Bool, length(dims)}(undef, dims)
3 ✔
1544
            i = 1
3 ✔
1545
            while i <= n
3 ✔
1546
                b = read(s.io, UInt8)::UInt8
3 ✔
1547
                v = (b >> 7) != 0
3 ✔
1548
                count = b & 0x7f
3 ✔
1549
                nxt = i + count
3 ✔
1550
                while i < nxt
3 ✔
1551
                    A[i] = v
3 ✔
1552
                    i += 1
3 ✔
1553
                end
3 ✔
1554
            end
3 ✔
1555
        else
1556
            A = read!(s.io, Array{elty}(undef, dims))
21 ✔
1557
        end
1558
        _setbackref!(s, slot, A)
21 ✔
1559
        return A
21 ✔
1560
    end
1561
    A = Array{elty, length(dims)}(undef, dims)
413 ✔
1562
    _setbackref!(s, slot, A)
413 ✔
1563
    _sizehint_backref!(s, s.counter + div(length(A)::Int,4))
413 ✔
1564
    deserialize_fillarray!(A, s)
413 ✔
1565
    return A
413 ✔
1566
end
1567

1568
function deserialize_fillarray!(A::Union{Array{T},Memory{T}}, s::AbstractSerializer) where {T}
413 ✔
1569
    for i = eachindex(A)
413 ✔
1570
        tag = Int32(read(s.io, UInt8)::UInt8)
372 ✔
1571
        if tag != UNDEFREF_TAG
372 ✔
1572
            @inbounds A[i] = handle_deserialize(s, tag)
372 ✔
1573
        end
1574
    end
372 ✔
1575
    return A
413 ✔
1576
end
1577

1578
function deserialize(s::AbstractSerializer, X::Type{Memory{T}} where T)
102 ✔
1579
    slot = pop!(s.pending_refs) # e.g. deserialize_cycle
102 ✔
1580
    n = deserialize(s)::Int
102 ✔
1581
    elty = eltype(X)
102 ✔
1582
    if isbitstype(elty)
102 ✔
1583
        A = X(undef, n)
102 ✔
1584
        if X === Memory{Bool}
102 ✔
1585
            i = 1
3 ✔
1586
            while i <= n
3 ✔
1587
                b = read(s.io, UInt8)::UInt8
3 ✔
1588
                v = (b >> 7) != 0
3 ✔
1589
                count = b & 0x7f
3 ✔
1590
                nxt = i + count
3 ✔
1591
                while i < nxt
3 ✔
1592
                    A[i] = v
3 ✔
1593
                    i += 1
3 ✔
1594
                end
3 ✔
1595
            end
3 ✔
1596
        else
1597
            A = read!(s.io, A)::X
102 ✔
1598
        end
1599
        _setbackref!(s, slot, A)
102 ✔
1600
        return A
102 ✔
1601
    end
1602
    A = X(undef, n)
3 ✔
1603
    _setbackref!(s, slot, A)
3 ✔
1604
    _sizehint_backref!(s, s.counter + div(n, 4))
3 ✔
1605
    deserialize_fillarray!(A, s)
3 ✔
1606
    return A
3 ✔
1607
end
1608

1609
function deserialize(s::AbstractSerializer, X::Type{MemoryRef{T}} where T)
3 ✔
1610
    x = Core.memoryref(deserialize(s))::X
3 ✔
1611
    i = deserialize(s)::Int
3 ✔
1612
    i == 1 || (x = Core.memoryrefnew(x, i, true))
3 ✔
1613
    return x::X
3 ✔
1614
end
1615

1616
function deserialize(s::AbstractSerializer, X::Type{Core.AddrSpace{M}} where M)
102 ✔
1617
    Core.bitcast(X, read(s.io, UInt8))
102 ✔
1618
end
1619

1620
function deserialize(s::AbstractSerializer, ::Type{Core.IntrinsicFunction})
3 ✔
1621
    name = deserialize(s)::Symbol
3 ✔
1622
    return getfield(Core.Intrinsics, name)::Core.IntrinsicFunction
3 ✔
1623
end
1624

1625
function deserialize_expr(s::AbstractSerializer, len)
311 ✔
1626
    e = Expr(:temp)
311 ✔
1627
    resolve_ref_immediately(s, e)
311 ✔
1628
    e.head = deserialize(s)::Symbol
311 ✔
1629
    e.args = Any[ deserialize(s) for i = 1:len ]
311 ✔
1630

1631
    # Rewrite old :method expressions to define_method calls
1632
    if e.head === :method
311 ✔
1633
        mod = current_module[]
3 ✔
1634
        if mod === nothing
3 ✔
1635
            # No current module context, keep the :method expression and hope for the best
1636
            # This shouldn't happen in practice for deserialization of top-level code
1637
            return e
3 ✔
1638
        end
1639

1640
        if len == 1
3 ✔
1641
            # Short form: (:method name) → (call Core.define_method module (inert name))
1642
            name = e.args[1]
3 ✔
1643
            if name isa GlobalRef
3 ✔
1644
                # Extract module and name from GlobalRef
1645
                mod_ref = name.mod
3 ✔
1646
                sym = name.name
3 ✔
1647
                e = Expr(:call, GlobalRef(Core, :define_method), mod_ref, QuoteNode(sym))
3 ✔
1648
            else
1649
                # Simple symbol - use the current module
1650
                e = Expr(:call, GlobalRef(Core, :define_method), mod, QuoteNode(name))
3 ✔
1651
            end
1652
        elseif len == 3
3 ✔
1653
            # Long form: (:method name_or_mt sigtype code) → (call Core.define_method module name_or_mt sigtype code)
1654
            name_or_mt = e.args[1]
3 ✔
1655
            sigtype = e.args[2]
3 ✔
1656
            code = e.args[3]
3 ✔
1657
            if name_or_mt isa Symbol
3 ✔
1658
                name_or_mt = QuoteNode(name_or_mt)
3 ✔
1659
                e = Expr(:call, GlobalRef(Core, :define_method), mod, name_or_mt, sigtype, code)
3 ✔
1660
            elseif name_or_mt isa GlobalRef
3 ✔
1661
                mod_ref = name_or_mt.mod
3 ✔
1662
                sym = name_or_mt.name
3 ✔
1663
                e = Expr(:call, GlobalRef(Core, :define_method), mod_ref, QuoteNode(sym), sigtype, code)
3 ✔
1664
            else
1665
                # name_or_mt is already something else (like false or a method table)
1666
                e = Expr(:call, GlobalRef(Core, :define_method), mod, name_or_mt, sigtype, code)
3 ✔
1667
            end
1668
        end
1669
    end
1670
    e
311 ✔
1671
end
1672

1673
module __deserialized_types__ end
1674

1675
function deserialize(s::AbstractSerializer, ::Type{Core.TypeName})
3 ✔
1676
    number = read(s.io, UInt64)
3 ✔
1677
    return deserialize_typename(s, number)
3 ✔
1678
end
1679

1680
function deserialize_typename(s::AbstractSerializer, number)
123 ✔
1681
    name = deserialize(s)::Symbol
123 ✔
1682
    tn = lookup_object_number(s, number)
123 ✔
1683
    if tn !== nothing
123 ✔
1684
        makenew = false
6 ✔
1685
    else
1686
        # reuse the same name for the type, if possible, for nicer debugging
1687
        tn_name = isdefined(__deserialized_types__, name) ? gensym() : name
123 ✔
1688
        tn = ccall(:jl_new_typename_in, Any, (Any, Any, Cint, Cint),
123 ✔
1689
                   tn_name, __deserialized_types__, false, false)
1690
        makenew = true
123 ✔
1691
    end
1692
    tn = tn::Core.TypeName
123 ✔
1693
    remember_object(s, tn, number)
123 ✔
1694
    deserialize_cycle(s, tn)
123 ✔
1695

1696
    names = deserialize(s)::SimpleVector
123 ✔
1697
    super = deserialize(s)::Type
123 ✔
1698
    parameters = deserialize(s)::SimpleVector
123 ✔
1699
    types = deserialize(s)::SimpleVector
123 ✔
1700
    attrs = Core.svec()
123 ✔
1701
    has_instance = deserialize(s)::Bool
123 ✔
1702
    abstr = deserialize(s)::Bool
123 ✔
1703
    mutabl = deserialize(s)::Bool
123 ✔
1704
    ninitialized = deserialize(s)::Int32
123 ✔
1705
    maxm = format_version(s) >= 18 ? deserialize(s)::UInt8 : UInt8(0)
123 ✔
1706

1707
    if makenew
123 ✔
1708
        # TODO: there's an unhanded cycle in the dependency graph at this point:
1709
        # while deserializing super and/or types, we may have encountered
1710
        # tn.wrapper and throw UndefRefException before we get to this point
1711
        ndt = ccall(:jl_new_datatype, Any, (Any, Any, Any, Any, Any, Any, Any, Cint, Cint, Cint),
123 ✔
1712
                    tn, tn.module, super, parameters, names, types, attrs,
1713
                    abstr, mutabl, ninitialized)
1714
        @assert tn == ndt.name
123 ✔
1715
        ccall(:jl_set_const, Cvoid, (Any, Any, Any), tn.module, tn.name, tn.wrapper)
123 ✔
1716
        ty = tn.wrapper
123 ✔
1717
        tn.max_methods = maxm
123 ✔
1718
        if has_instance
123 ✔
1719
            ty = ty::DataType
114 ✔
1720
            if !isdefined(ty, :instance)
114 ✔
1721
                singleton = ccall(:jl_new_struct, Any, (Any, Any...), ty)
×
1722
                # use setfield! directly to avoid `fieldtype` lowering expecting to see a Singleton object already on ty
UNCOV
1723
                ccall(:jl_set_nth_field, Cvoid, (Any, Csize_t, Any), ty, Base.fieldindex(DataType, :instance)-1, singleton)
×
1724
            end
1725
        end
1726
    end
1727

1728
    tag = Int32(read(s.io, UInt8)::UInt8)
123 ✔
1729
    if tag != UNDEFREF_TAG
123 ✔
1730
        mtname = handle_deserialize(s, tag)
123 ✔
1731
        defs = deserialize(s)
123 ✔
1732
        maxa = deserialize(s)::Union{Int,Int32}
123 ✔
1733
        if makenew
123 ✔
1734
            tn.singletonname = mtname
123 ✔
1735
            setfield!(tn, :max_args, Int32(maxa), :monotonic)
123 ✔
1736
        end
1737
        tag = Int32(read(s.io, UInt8)::UInt8)
123 ✔
1738
        if tag != UNDEFREF_TAG
123 ✔
1739
            kws = handle_deserialize(s, tag)
3 ✔
1740
            if makenew && !(kws isa Vector{Method})
3 ✔
1741
                # old object format -- try to forward from old to new
UNCOV
1742
                @eval Core.kwcall(kwargs::NamedTuple, f::$ty, args...) = $kws(kwargs, f, args...)
×
1743
            end
1744
        end
1745
    end
1746
    return tn
123 ✔
1747
end
1748

1749
function deserialize_datatype(s::AbstractSerializer, full::Bool)
481 ✔
1750
    slot = s.counter; s.counter += 1
481 ✔
1751
    if full
481 ✔
1752
        tname = deserialize(s)::Core.TypeName
33 ✔
1753
        ty = tname.wrapper
33 ✔
1754
    else
1755
        name = deserialize(s)::Symbol
481 ✔
1756
        mod = deserialize(s)::Module
481 ✔
1757
        ty = getglobal(mod, name)
481 ✔
1758
    end
1759
    if isa(ty,DataType) && isempty(ty.parameters)
481 ✔
1760
        t = ty
481 ✔
1761
    else
1762
        np = Int(read(s.io, Int32)::Int32)
481 ✔
1763
        if np == 0
481 ✔
1764
            t = unwrap_unionall(ty)
3 ✔
1765
        elseif ty === Tuple
481 ✔
1766
            # note np==0 has its own tag
1767
            if np == 1
288 ✔
1768
                t = Tuple{deserialize(s)}
233 ✔
1769
            elseif np == 2
100 ✔
1770
                t = Tuple{deserialize(s), deserialize(s)}
82 ✔
1771
            elseif np == 3
69 ✔
1772
                t = Tuple{deserialize(s), deserialize(s), deserialize(s)}
27 ✔
1773
            elseif np == 4
69 ✔
1774
                t = Tuple{deserialize(s), deserialize(s), deserialize(s), deserialize(s)}
63 ✔
1775
            else
1776
                t = Tuple{Any[ deserialize(s) for i=1:np ]...}
27 ✔
1777
            end
1778
        else
1779
            t = ty
481 ✔
1780
            for i = 1:np
481 ✔
1781
                t = t{deserialize(s)}
481 ✔
1782
            end
481 ✔
1783
        end
1784
    end
1785
    _setbackref!(s, slot, t)
481 ✔
1786
    return t
481 ✔
1787
end
1788

1789
function deserialize(s::AbstractSerializer, ::Type{UnionAll})
33 ✔
1790
    form = read(s.io, UInt8)
33 ✔
1791
    if form == 0
33 ✔
1792
        var = deserialize(s)
3 ✔
1793
        body = deserialize(s)
3 ✔
1794
        return UnionAll(var, body)
3 ✔
1795
    else
1796
        n = read(s.io, Int16)
33 ✔
1797
        t = deserialize(s)::DataType
33 ✔
1798
        w = t.name.wrapper
33 ✔
1799
        k = 0
33 ✔
1800
        while isa(w, UnionAll)
33 ✔
1801
            w = w.body
33 ✔
1802
            k += 1
33 ✔
1803
        end
33 ✔
1804
        w = t.name.wrapper
33 ✔
1805
        k -= n
33 ✔
1806
        while k > 0
33 ✔
1807
            w = w.body
3 ✔
1808
            k -= 1
3 ✔
1809
        end
3 ✔
1810
        return w
33 ✔
1811
    end
1812
end
1813

1814
function deserialize(s::AbstractSerializer, ::Type{Task})
6 ✔
1815
    # The task code is replaced below, so prevent attaching invoke metadata for the dummy closure.
1816
    t = Task(Base.inferencebarrier(()->nothing))
6 ✔
1817
    deserialize_cycle(s, t)
6 ✔
1818
    t.code = deserialize(s)
6 ✔
1819
    t.storage = deserialize(s)
6 ✔
1820
    state = deserialize(s)
6 ✔
1821
    if state === :runnable
6 ✔
1822
        @atomic :release t._state = Base.task_state_runnable
3 ✔
1823
    elseif state === :done
6 ✔
1824
        @atomic :release t._state = Base.task_state_done
3 ✔
1825
    elseif state === :failed
6 ✔
1826
        @atomic :release t._state = Base.task_state_failed
6 ✔
1827
    else
UNCOV
1828
        @assert false
×
1829
    end
1830
    setfield!(t, :result, deserialize(s))
6 ✔
1831
    exc = deserialize(s)
6 ✔
1832
    if exc === nothing
6 ✔
UNCOV
1833
        t._isexception = false
×
1834
    elseif exc isa Bool
6 ✔
1835
        t._isexception = exc
6 ✔
1836
    else
UNCOV
1837
        t._isexception = true
×
UNCOV
1838
        setfield!(t, :result, exc)
×
1839
    end
1840
    if format_version(s) >= 31 && (deserialize(s)::Bool)
6 ✔
1841
        setfield!(t, :invoked, deserialize(s))
3 ✔
1842
    end
1843
    t
6 ✔
1844
end
1845

1846
function deserialize_string(s::AbstractSerializer, len::Int)
489 ✔
1847
    out = ccall(:jl_alloc_string, Ref{String}, (Csize_t,), len)
489 ✔
1848
    unsafe_read(s.io, pointer(out), len)
489 ✔
1849
    return out
489 ✔
1850
end
1851

1852
# default DataType deserializer
1853
function deserialize(s::AbstractSerializer, t::DataType)
487 ✔
1854
    nf = fieldcount(t)
487 ✔
1855
    if isprimitivetype(t)
487 ✔
1856
        return read(s.io, t)
17 ✔
1857
    elseif ismutabletype(t)
487 ✔
1858
        x = ccall(:jl_new_struct_uninit, Any, (Any,), t)
165 ✔
1859
        deserialize_cycle(s, x)
165 ✔
1860
        for i in 1:nf
165 ✔
1861
            tag = Int32(read(s.io, UInt8)::UInt8)
165 ✔
1862
            if tag != UNDEFREF_TAG
165 ✔
1863
                ccall(:jl_set_nth_field, Cvoid, (Any, Csize_t, Any), x, i-1, handle_deserialize(s, tag))
165 ✔
1864
            end
1865
        end
165 ✔
1866
        return x
165 ✔
1867
    elseif nf == 0
487 ✔
1868
        return ccall(:jl_new_struct_uninit, Any, (Any,), t)
389 ✔
1869
    else
1870
        na = nf
487 ✔
1871
        vflds = Vector{Any}(undef, nf)
487 ✔
1872
        for i in 1:nf
487 ✔
1873
            tag = Int32(read(s.io, UInt8)::UInt8)
487 ✔
1874
            if tag != UNDEFREF_TAG
487 ✔
1875
                f = handle_deserialize(s, tag)
487 ✔
1876
                na >= i && (vflds[i] = f)
487 ✔
1877
            else
1878
                na >= i && (na = i - 1) # rest of tail must be undefined values
12 ✔
1879
            end
1880
        end
487 ✔
1881
        return ccall(:jl_new_structv, Any, (Any, Ptr{Any}, UInt32), t, vflds, na)
487 ✔
1882
    end
1883
end
1884

1885
function deserialize_dict(s::AbstractSerializer, T::Type{<:AbstractDict})
13 ✔
1886
    n = read(s.io, Int32)
13 ✔
1887
    t = T(); sizehint!(t, n)
13 ✔
1888
    deserialize_cycle(s, t)
13 ✔
1889
    for i = 1:n
13 ✔
1890
        k = deserialize(s)
13 ✔
1891
        v = deserialize(s)
13 ✔
1892
        t[k] = v
13 ✔
1893
    end
13 ✔
1894
    return t
13 ✔
1895
end
1896

1897
function deserialize(s::AbstractSerializer, T::Type{Dict{K,V}}) where {K,V}
13 ✔
1898
    return deserialize_dict(s, T)
13 ✔
1899
end
1900

1901
deserialize(s::AbstractSerializer, ::Type{BigInt}) = parse(BigInt, deserialize(s), base = 62)
3 ✔
1902

1903
function deserialize(s::AbstractSerializer, t::Type{Regex})
3 ✔
1904
    pattern = deserialize(s)
3 ✔
1905
    compile_options = deserialize(s)
3 ✔
1906
    match_options = deserialize(s)
3 ✔
1907
    return Regex(pattern, compile_options, match_options)
3 ✔
1908
end
1909

1910
## StackTraces
1911

1912
# provide a custom serializer that skips attempting to serialize the `outer_linfo`
1913
# which is likely to contain complex references, types, and module references
1914
# that may not exist on the receiver end
1915
function serialize(s::AbstractSerializer, frame::Base.StackTraces.StackFrame)
50 ✔
1916
    serialize_type(s, typeof(frame))
50 ✔
1917
    serialize(s, frame.func)
50 ✔
1918
    serialize(s, frame.file)
50 ✔
1919
    write(s.io, frame.line)
50 ✔
1920
    write(s.io, frame.from_c)
50 ✔
1921
    write(s.io, frame.inlined)
50 ✔
1922
    write(s.io, frame.pointer)
50 ✔
1923
    nothing
50 ✔
1924
end
1925

1926
function deserialize(s::AbstractSerializer, ::Type{Base.StackTraces.StackFrame})
14 ✔
1927
    func = deserialize(s)
14 ✔
1928
    file = deserialize(s)
14 ✔
1929
    line = read(s.io, Int)
14 ✔
1930
    from_c = read(s.io, Bool)
14 ✔
1931
    inlined = read(s.io, Bool)
14 ✔
1932
    pointer = read(s.io, UInt64)
14 ✔
1933
    return Base.StackTraces.StackFrame(func, file, line, nothing, from_c, inlined, pointer)
14 ✔
1934
end
1935

1936
function serialize(s::AbstractSerializer, lock::Base.AbstractLock)
12 ✔
1937
    # assert_havelock(lock)
1938
    serialize_cycle_header(s, lock)
12 ✔
1939
    nothing
12 ✔
1940
end
1941

1942
function deserialize(s::AbstractSerializer, ::Type{T}) where T<:Base.AbstractLock
32 ✔
1943
    lock = T()
32 ✔
1944
    deserialize_cycle(s, lock)
32 ✔
1945
    return lock
32 ✔
1946
end
1947

1948
function serialize(s::AbstractSerializer, cond::Base.GenericCondition)
3 ✔
1949
    serialize_cycle_header(s, cond) && return
3 ✔
1950
    serialize(s, cond.lock)
3 ✔
1951
    nothing
3 ✔
1952
end
1953

1954
function deserialize(s::AbstractSerializer, ::Type{T}) where T<:Base.GenericCondition
3 ✔
1955
    lock = deserialize(s)
3 ✔
1956
    cond = T(lock)
3 ✔
1957
    deserialize_cycle(s, cond)
3 ✔
1958
    return cond
3 ✔
1959
end
1960

1961
serialize(s::AbstractSerializer, l::LazyString) =
6 ✔
1962
    invoke(serialize, Tuple{AbstractSerializer,Any}, s, Base._LazyString((), string(l)))
6 ✔
1963

1964
end
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE TRIAL · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc