#37822

Committed 28 Jun 2024 01:08AM UTC coverage: 86.381% (-1.1%) from 87.488%

Build # #37822

Build Type

push

local

Committed by

web-flow

Commit Message

inference: implement an opt-in interface to cache generated sources (#54916)

In Cassette-like systems, where inference has to infer many calls of
`@generated` function and the generated function involves complex code
transformations, the overhead from code generation itself can become
significant. This is because the results of code generation are not
cached, leading to duplicated code generation in the following contexts:
- `method_for_inference_heuristics` for regular inference on cached
`@generated` function calls (since
`method_for_inference_limit_heuristics` isn't stored in cached optimized
sources, but is attached to generated unoptimized sources).
- `retrieval_code_info` for constant propagation on cached `@generated`
function calls.

Having said that, caching unoptimized sources generated by `@generated`
functions is not a good tradeoff in general cases, considering the
memory space consumed (and the image bloat). The code generation for
generators like `GeneratedFunctionStub` produced by the front end is
generally very simple, and the first duplicated code generation
mentioned above does not occur for `GeneratedFunctionStub`.

So this unoptimized source caching should be enabled in an opt-in
manner.

Based on this idea, this commit defines the trait `abstract type
Core.CachedGenerator` as an interface for the external systems to
opt-in. If the generator is a subtype of this trait, inference caches
the generated unoptimized code, sacrificing memory space to improve the
performance of subsequent inferences. Specifically, the mechanism for
caching the unoptimized source uses the infrastructure already
implemented in JuliaLang/julia#54362. Thanks to JuliaLang/julia#54362,
the cache for generated functions is now partitioned by world age, so
even if the unoptimized source is cached, the existing invalidation
system will invalidate it as expected.

In JuliaDebug/CassetteOverlay.jl#56, the following benchmark result... (continued)

Run Details

7 of 8 new or added lines in 1 file covered. (87.5%)

1385 existing lines in 56 files now uncovered.

76243 of 88264 relevant lines covered (86.38%)

15480525.49 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

25.17

/base/strings/cstring.jl

# This file is a part of Julia. License is MIT: https://julialang.org/license

import Core.Intrinsics: bitcast

"""
    Cwstring

A C-style string composed of the native wide character type
[`Cwchar_t`](@ref)s. `Cwstring`s are NUL-terminated. For
C-style strings composed of the native character
type, see [`Cstring`](@ref). For more information
about string interoperability with C, see the
[manual](@ref man-bits-types).

"""
Cwstring

"""
    Cstring

A C-style string composed of the native character type
[`Cchar`](@ref)s. `Cstring`s are NUL-terminated. For
C-style strings composed of the native wide character
type, see [`Cwstring`](@ref). For more information
about string interoperability with C, see the
[manual](@ref man-bits-types).
"""
Cstring

# construction from pointers
Cstring(p::Union{Ptr{Int8},Ptr{UInt8},Ptr{Cvoid}}) = bitcast(Cstring, p)
Cwstring(p::Union{Ptr{Cwchar_t},Ptr{Cvoid}})       = bitcast(Cwstring, p)
Ptr{T}(p::Cstring) where {T<:Union{Int8,UInt8,Cvoid}} = bitcast(Ptr{T}, p)
Ptr{T}(p::Cwstring) where {T<:Union{Cwchar_t,Cvoid}}  = bitcast(Ptr{Cwchar_t}, p)

convert(::Type{Cstring}, p::Union{Ptr{Int8},Ptr{UInt8},Ptr{Cvoid}}) = Cstring(p)
convert(::Type{Cwstring}, p::Union{Ptr{Cwchar_t},Ptr{Cvoid}}) = Cwstring(p)
convert(::Type{Ptr{T}}, p::Cstring) where {T<:Union{Int8,UInt8,Cvoid}} = Ptr{T}(p)
convert(::Type{Ptr{T}}, p::Cwstring) where {T<:Union{Cwchar_t,Cvoid}} = Ptr{T}(p)

"""
    pointer(array [, index])

Get the native address of an array or string, optionally at a given location `index`.

This function is "unsafe". Be careful to ensure that a Julia reference to
`array` exists as long as this pointer will be used. The [`GC.@preserve`](@ref)
macro should be used to protect the `array` argument from garbage collection
within a given block of code.

Calling [`Ref(array[, index])`](@ref Ref) is generally preferable to this function as it guarantees validity.
"""
function pointer end

pointer(p::Cstring) = convert(Ptr{Cchar}, p)
pointer(p::Cwstring) = convert(Ptr{Cwchar_t}, p)

# comparisons against pointers (mainly to support `cstr==C_NULL`)
==(x::Union{Cstring,Cwstring}, y::Ptr) = pointer(x) == y
==(x::Ptr, y::Union{Cstring,Cwstring}) = x == pointer(y)

unsafe_string(s::Cstring) = unsafe_string(convert(Ptr{UInt8}, s))

# convert strings to String etc. to pass as pointers
cconvert(::Type{Cstring}, s::String) = s
cconvert(::Type{Cstring}, s::AbstractString) =
    cconvert(Cstring, String(s)::String)

function cconvert(::Type{Cwstring}, s::AbstractString)
    v = transcode(Cwchar_t, String(s))
    push!(v, 0)
    return cconvert(Cwstring, v)
end

eltype(::Type{Cstring}) = Cchar
eltype(::Type{Cwstring}) = Cwchar_t

containsnul(p::Ptr, len) =
    C_NULL != ccall(:memchr, Ptr{Cchar}, (Ptr{Cchar}, Cint, Csize_t), p, 0, len)
containsnul(s::String) = containsnul(unsafe_convert(Ptr{Cchar}, s), sizeof(s))
containsnul(s::AbstractString) = '\0' in s

function unsafe_convert(::Type{Cstring}, s::String)
    p = unsafe_convert(Ptr{Cchar}, s)
    containsnul(p, sizeof(s)) &&
        throw(ArgumentError("embedded NULs are not allowed in C strings: $(repr(s))"))
    return Cstring(p)
end

unsafe_convert(::Type{Cstring}, s::Union{Memory{UInt8},Memory{Int8}}) = Cstring(unsafe_convert(Ptr{Cvoid}, s))

function cconvert(::Type{Cwstring}, v::Vector{Cwchar_t})
    for i = 1:length(v)-1
        v[i] == 0 &&
            throw(ArgumentError("embedded NULs are not allowed in C strings: $(repr(v))"))
    end
    v[end] == 0 ||
        throw(ArgumentError("C string data must be NUL terminated: $(repr(v))"))
    return cconvert(Ptr{Cwchar_t}, v)
end
unsafe_convert(::Type{Cwstring}, s) = Cwstring(unsafe_convert(Ptr{Cwchar_t}, s))
unsafe_convert(::Type{Cwstring}, s::Cwstring) = s

# symbols are guaranteed not to contain embedded NUL
cconvert(::Type{Cstring}, s::Symbol) = s
unsafe_convert(::Type{Cstring}, s::Symbol) = Cstring(unsafe_convert(Ptr{Cchar}, s))

if ccall(:jl_get_UNAME, Any, ()) === :NT
"""
    Base.cwstring(s)

Converts a string `s` to a NUL-terminated `Vector{Cwchar_t}`, suitable for passing to C
functions expecting a `Ptr{Cwchar_t}`. The main advantage of using this over the implicit
conversion provided by [`Cwstring`](@ref) is if the function is called multiple times with the
same argument.

This is only available on Windows.
"""
function cwstring(s::AbstractString)
    bytes = codeunits(String(s))
    0 in bytes && throw(ArgumentError("embedded NULs are not allowed in C strings: $(repr(s))"))
    return push!(transcode(UInt16, bytes), 0)
end
end

# transcoding between data in UTF-8 and UTF-16 for Windows APIs,
# and also UTF-32 for APIs using Cwchar_t on other platforms.

"""
    transcode(T, src)

Convert string data between Unicode encodings. `src` is either a
`String` or a `Vector{UIntXX}` of UTF-XX code units, where
`XX` is 8, 16, or 32. `T` indicates the encoding of the return value:
`String` to return a (UTF-8 encoded) `String` or `UIntXX`
to return a `Vector{UIntXX}` of UTF-`XX` data. (The alias [`Cwchar_t`](@ref)
can also be used as the integer type, for converting `wchar_t*` strings
used by external C libraries.)

The `transcode` function succeeds as long as the input data can be
reasonably represented in the target encoding; it always succeeds for
conversions between UTF-XX encodings, even for invalid Unicode data.

Only conversion to/from UTF-8 is currently supported.

# Examples
```jldoctest
julia> str = "αβγ"
"αβγ"

julia> transcode(UInt16, str)
3-element Vector{UInt16}:
 0x03b1
 0x03b2
 0x03b3

julia> transcode(String, transcode(UInt16, str))
"αβγ"
```
"""
function transcode end

transcode(::Type{T}, src::AbstractVector{T}) where {T<:Union{UInt8,UInt16,UInt32,Int32}} = src
transcode(::Type{T}, src::String) where {T<:Union{Int32,UInt32}} = T[T(c) for c in src]
transcode(::Type{T}, src::AbstractVector{UInt8}) where {T<:Union{Int32,UInt32}} =
    transcode(T, String(Vector(src)))
transcode(::Type{T}, src::CodeUnits{UInt8,String}) where {T<:Union{Int32,UInt32}} =
    transcode(T, String(src))

function transcode(::Type{UInt8}, src::Vector{<:Union{Int32,UInt32}})
    buf = IOBuffer()
    for c in src
        print(buf, Char(c))
    end
    take!(buf)
end
transcode(::Type{String}, src::String) = src
transcode(T, src::String) = transcode(T, codeunits(src))
transcode(::Type{String}, src) = String(transcode(UInt8, src))

function transcode(::Type{UInt16}, src::AbstractVector{UInt8})
    require_one_based_indexing(src)
    dst = UInt16[]
    i, n = 1, length(src)
    n > 0 || return dst
    sizehint!(dst, 2n)
    a = src[1]
    while true
        if i < n && -64 <= a % Int8 <= -12 # multi-byte character
            b = src[i += 1]
            if -64 <= (b % Int8) || a == 0xf4 && 0x8f < b
                # invalid UTF-8 (non-continuation or too-high code point)
                push!(dst, a)
                a = b; continue
            elseif a < 0xe0 # 2-byte UTF-8
                push!(dst, xor(0x3080, UInt16(a) << 6, b))
            elseif i < n # 3/4-byte character
                c = src[i += 1]
                if -64 <= (c % Int8) # invalid UTF-8 (non-continuation)
                    push!(dst, a, b)
                    a = c; continue
                elseif a < 0xf0 # 3-byte UTF-8
                    push!(dst, xor(0x2080, UInt16(a) << 12, UInt16(b) << 6, c))
                elseif i < n
                    d = src[i += 1]
                    if -64 <= (d % Int8) # invalid UTF-8 (non-continuation)
                        push!(dst, a, b, c)
                        a = d; continue
                    elseif a == 0xf0 && b < 0x90 # overlong encoding
                        push!(dst, xor(0x2080, UInt16(b) << 12, UInt16(c) << 6, d))
                    else # 4-byte UTF-8
                        push!(dst, 0xe5b8 + (UInt16(a) << 8) + (UInt16(b) << 2) + (c >> 4),
                                   xor(0xdc80, UInt16(c & 0xf) << 6, d))
                    end
                else # too short
                    push!(dst, a, b, c)
                    break
                end
            else # too short
                push!(dst, a, b)
                break
            end
        else # ASCII or invalid UTF-8 (continuation byte or too-high code point)
            push!(dst, a)
        end
        i < n || break
        a = src[i += 1]
    end
    return dst
end

function transcode(::Type{UInt8}, src::AbstractVector{UInt16})
    require_one_based_indexing(src)
    n = length(src)
    n == 0 && return UInt8[]

    # Precompute m = sizeof(dst).   This involves annoying duplication
    # of the loop over the src array.   However, this is not just an
    # optimization: it is problematic for security reasons to grow
    # dst dynamically, because Base.winprompt uses this function to
    # convert passwords to UTF-8 and we don't want to make unintentional
    # copies of the password data.
    a = src[1]
    i, m = 1, 0
    while true
        if a < 0x80
            m += 1
        elseif a < 0x800 # 2-byte UTF-8
            m += 2
        elseif a & 0xfc00 == 0xd800 && i < length(src)
            b = src[i += 1]
            if (b & 0xfc00) == 0xdc00 # 2-unit UTF-16 sequence => 4-byte UTF-8
                m += 4
            else
                m += 3
                a = b; continue
            end
        else
            # 1-unit high UTF-16 or unpaired high surrogate
            # either way, encode as 3-byte UTF-8 code point
            m += 3
        end
        i < n || break
        a = src[i += 1]
    end

    dst = StringVector(m)
    a = src[1]
    i, j = 1, 0
    while true
        if a < 0x80 # ASCII
            dst[j += 1] = a % UInt8
        elseif a < 0x800 # 2-byte UTF-8
            dst[j += 1] = 0xc0 | ((a >> 6) % UInt8)
            dst[j += 1] = 0x80 | ((a % UInt8) & 0x3f)
        elseif a & 0xfc00 == 0xd800 && i < n
            b = src[i += 1]
            if (b & 0xfc00) == 0xdc00
                # 2-unit UTF-16 sequence => 4-byte UTF-8
                a += 0x2840
                dst[j += 1] = 0xf0 | ((a >> 8) % UInt8)
                dst[j += 1] = 0x80 | ((a % UInt8) >> 2)
                dst[j += 1] = xor(0xf0, ((a % UInt8) << 4) & 0x3f, (b >> 6) % UInt8)
                dst[j += 1] = 0x80 | ((b % UInt8) & 0x3f)
            else
                dst[j += 1] = 0xe0 | ((a >> 12) % UInt8)
                dst[j += 1] = 0x80 | (((a >> 6) % UInt8) & 0x3f)
                dst[j += 1] = 0x80 | ((a % UInt8) & 0x3f)
                a = b; continue
            end
        else
            # 1-unit high UTF-16 or unpaired high surrogate
            # either way, encode as 3-byte UTF-8 code point
            dst[j += 1] = 0xe0 | ((a >> 12) % UInt8)
            dst[j += 1] = 0x80 | (((a >> 6) % UInt8) & 0x3f)
            dst[j += 1] = 0x80 | ((a % UInt8) & 0x3f)
        end
        i < n || break
        a = src[i += 1]
    end
    return dst
end

function unsafe_string(p::Ptr{T}, length::Integer) where {T<:Union{UInt16,UInt32,Cwchar_t}}
    transcode(String, unsafe_wrap(Array, p, length; own=false))
end
function unsafe_string(cw::Cwstring)
    p = convert(Ptr{Cwchar_t}, cw)
    n = 1
    while unsafe_load(p, n) != 0
        n += 1
    end
    return unsafe_string(p, n - 1)
end

1	# This file is a part of Julia. License is MIT: https://julialang.org/license
2
3	import Core.Intrinsics: bitcast
4
5	"""
6	Cwstring
7
8	A C-style string composed of the native wide character type
9	[`Cwchar_t`](@ref)s. `Cwstring`s are NUL-terminated. For
10	C-style strings composed of the native character
11	type, see [`Cstring`](@ref). For more information
12	about string interoperability with C, see the
13	[manual](@ref man-bits-types).
14
15	"""
16	Cwstring
17
18	"""
19	Cstring
20
21	A C-style string composed of the native character type
22	[`Cchar`](@ref)s. `Cstring`s are NUL-terminated. For
23	C-style strings composed of the native wide character
24	type, see [`Cwstring`](@ref). For more information
25	about string interoperability with C, see the
26	[manual](@ref man-bits-types).
27	"""
28	Cstring
29
30	# construction from pointers
31	Cstring(p::Union{Ptr{Int8},Ptr{UInt8},Ptr{Cvoid}}) = bitcast(Cstring, p)	78,761,157✔
32	Cwstring(p::Union{Ptr{Cwchar_t},Ptr{Cvoid}}) = bitcast(Cwstring, p)	32✔
33	Ptr{T}(p::Cstring) where {T<:Union{Int8,UInt8,Cvoid}} = bitcast(Ptr{T}, p)	2,666,790✔
34	Ptr{T}(p::Cwstring) where {T<:Union{Cwchar_t,Cvoid}} = bitcast(Ptr{Cwchar_t}, p)	×
35
36	convert(::Type{Cstring}, p::Union{Ptr{Int8},Ptr{UInt8},Ptr{Cvoid}}) = Cstring(p)	734,658✔
37	convert(::Type{Cwstring}, p::Union{Ptr{Cwchar_t},Ptr{Cvoid}}) = Cwstring(p)	×
38	convert(::Type{Ptr{T}}, p::Cstring) where {T<:Union{Int8,UInt8,Cvoid}} = Ptr{T}(p)	2,666,790✔
39	convert(::Type{Ptr{T}}, p::Cwstring) where {T<:Union{Cwchar_t,Cvoid}} = Ptr{T}(p)	×
40
41	"""
42	pointer(array [, index])
43
44	Get the native address of an array or string, optionally at a given location `index`.
45
46	This function is "unsafe". Be careful to ensure that a Julia reference to
47	`array` exists as long as this pointer will be used. The [`GC.@preserve`](@ref)
48	macro should be used to protect the `array` argument from garbage collection
49	within a given block of code.
50
51	Calling [`Ref(array[, index])`](@ref Ref) is generally preferable to this function as it guarantees validity.
52	"""
53	function pointer end
54
55	pointer(p::Cstring) = convert(Ptr{Cchar}, p)	1,367,557✔
56	pointer(p::Cwstring) = convert(Ptr{Cwchar_t}, p)	×
57
58	# comparisons against pointers (mainly to support `cstr==C_NULL`)
59	==(x::Union{Cstring,Cwstring}, y::Ptr) = pointer(x) == y	1,367,552✔
60	==(x::Ptr, y::Union{Cstring,Cwstring}) = x == pointer(y)	4✔
61
62	unsafe_string(s::Cstring) = unsafe_string(convert(Ptr{UInt8}, s))	1,299,217✔
63
64	# convert strings to String etc. to pass as pointers
65	cconvert(::Type{Cstring}, s::String) = s	144✔
66	cconvert(::Type{Cstring}, s::AbstractString) =	5,348✔
67	cconvert(Cstring, String(s)::String)
68
69	function cconvert(::Type{Cwstring}, s::AbstractString)
70	v = transcode(Cwchar_t, String(s))	32✔
71	push!(v, 0)	32✔
72	return cconvert(Cwstring, v)	32✔
73	end
74
75	eltype(::Type{Cstring}) = Cchar	×
76	eltype(::Type{Cwstring}) = Cwchar_t	×
77
78	containsnul(p::Ptr, len) =	3,145,277✔
79	C_NULL != ccall(:memchr, Ptr{Cchar}, (Ptr{Cchar}, Cint, Csize_t), p, 0, len)
80	containsnul(s::String) = containsnul(unsafe_convert(Ptr{Cchar}, s), sizeof(s))	38,210✔
81	containsnul(s::AbstractString) = '\0' in s	3,548✔
82
83	function unsafe_convert(::Type{Cstring}, s::String)	4✔
84	p = unsafe_convert(Ptr{Cchar}, s)	3,105,786✔
85	containsnul(p, sizeof(s)) &&	3,105,786✔
86	throw(ArgumentError("embedded NULs are not allowed in C strings: $(repr(s))"))
87	return Cstring(p)	3,105,735✔
88	end
89
UNCOV 90	unsafe_convert(::Type{Cstring}, s::Union{Memory{UInt8},Memory{Int8}}) = Cstring(unsafe_convert(Ptr{Cvoid}, s))	×
91
92	function cconvert(::Type{Cwstring}, v::Vector{Cwchar_t})
93	for i = 1:length(v)-1	32✔
94	v[i] == 0 &&	432✔
95	throw(ArgumentError("embedded NULs are not allowed in C strings: $(repr(v))"))
96	end	832✔
97	v[end] == 0 \|\|	32✔
98	throw(ArgumentError("C string data must be NUL terminated: $(repr(v))"))
99	return cconvert(Ptr{Cwchar_t}, v)	32✔
100	end
101	unsafe_convert(::Type{Cwstring}, s) = Cwstring(unsafe_convert(Ptr{Cwchar_t}, s))	32✔
102	unsafe_convert(::Type{Cwstring}, s::Cwstring) = s	×
103
104	# symbols are guaranteed not to contain embedded NUL
105	cconvert(::Type{Cstring}, s::Symbol) = s	76✔
106	unsafe_convert(::Type{Cstring}, s::Symbol) = Cstring(unsafe_convert(Ptr{Cchar}, s))	37,515,306✔
107
108	if ccall(:jl_get_UNAME, Any, ()) === :NT
109	"""
110	Base.cwstring(s)
111
112	Converts a string `s` to a NUL-terminated `Vector{Cwchar_t}`, suitable for passing to C
113	functions expecting a `Ptr{Cwchar_t}`. The main advantage of using this over the implicit
114	conversion provided by [`Cwstring`](@ref) is if the function is called multiple times with the
115	same argument.
116
117	This is only available on Windows.
118	"""
119	function cwstring(s::AbstractString)	×
120	bytes = codeunits(String(s))	×
121	0 in bytes && throw(ArgumentError("embedded NULs are not allowed in C strings: $(repr(s))"))	×
122	return push!(transcode(UInt16, bytes), 0)	×
123	end
124	end
125
126	# transcoding between data in UTF-8 and UTF-16 for Windows APIs,
127	# and also UTF-32 for APIs using Cwchar_t on other platforms.
128
129	"""
130	transcode(T, src)
131
132	Convert string data between Unicode encodings. `src` is either a
133	`String` or a `Vector{UIntXX}` of UTF-XX code units, where
134	`XX` is 8, 16, or 32. `T` indicates the encoding of the return value:
135	`String` to return a (UTF-8 encoded) `String` or `UIntXX`
136	to return a `Vector{UIntXX}` of UTF-`XX` data. (The alias [`Cwchar_t`](@ref)
137	can also be used as the integer type, for converting `wchar_t*` strings
138	used by external C libraries.)
139
140	The `transcode` function succeeds as long as the input data can be
141	reasonably represented in the target encoding; it always succeeds for
142	conversions between UTF-XX encodings, even for invalid Unicode data.
143
144	Only conversion to/from UTF-8 is currently supported.
145
146	# Examples
147	```jldoctest
148	julia> str = "αβγ"
149	"αβγ"
150
151	julia> transcode(UInt16, str)
152	3-element Vector{UInt16}:
153	0x03b1
154	0x03b2
155	0x03b3
156
157	julia> transcode(String, transcode(UInt16, str))
158	"αβγ"
159	```
160	"""
161	function transcode end
162
163	transcode(::Type{T}, src::AbstractVector{T}) where {T<:Union{UInt8,UInt16,UInt32,Int32}} = src	×
164	transcode(::Type{T}, src::String) where {T<:Union{Int32,UInt32}} = T[T(c) for c in src]	32✔
165	transcode(::Type{T}, src::AbstractVector{UInt8}) where {T<:Union{Int32,UInt32}} =	×
166	transcode(T, String(Vector(src)))
167	transcode(::Type{T}, src::CodeUnits{UInt8,String}) where {T<:Union{Int32,UInt32}} =	×
168	transcode(T, String(src))
169
170	function transcode(::Type{UInt8}, src::Vector{<:Union{Int32,UInt32}})	32✔
171	buf = IOBuffer()	32✔
172	for c in src	32✔
173	print(buf, Char(c))	650✔
174	end	650✔
175	take!(buf)	32✔
176	end
177	transcode(::Type{String}, src::String) = src	×
178	transcode(T, src::String) = transcode(T, codeunits(src))	75,169✔
179	transcode(::Type{String}, src) = String(transcode(UInt8, src))	64✔
180
181	function transcode(::Type{UInt16}, src::AbstractVector{UInt8})	×
182	require_one_based_indexing(src)	×
183	dst = UInt16[]	×
184	i, n = 1, length(src)	×
185	n > 0 \|\| return dst	×
186	sizehint!(dst, 2n)	×
187	a = src[1]	×
188	while true	×
189	if i < n && -64 <= a % Int8 <= -12 # multi-byte character	×
190	b = src[i += 1]	×
191	if -64 <= (b % Int8) \|\| a == 0xf4 && 0x8f < b	×
192	# invalid UTF-8 (non-continuation or too-high code point)
193	push!(dst, a)	×
194	a = b; continue	×
195	elseif a < 0xe0 # 2-byte UTF-8	×
196	push!(dst, xor(0x3080, UInt16(a) << 6, b))	×
197	elseif i < n # 3/4-byte character	×
198	c = src[i += 1]	×
199	if -64 <= (c % Int8) # invalid UTF-8 (non-continuation)	×
200	push!(dst, a, b)	×
201	a = c; continue	×
202	elseif a < 0xf0 # 3-byte UTF-8	×
203	push!(dst, xor(0x2080, UInt16(a) << 12, UInt16(b) << 6, c))	×
204	elseif i < n	×
205	d = src[i += 1]	×
206	if -64 <= (d % Int8) # invalid UTF-8 (non-continuation)	×
207	push!(dst, a, b, c)	×
208	a = d; continue	×
209	elseif a == 0xf0 && b < 0x90 # overlong encoding	×
210	push!(dst, xor(0x2080, UInt16(b) << 12, UInt16(c) << 6, d))	×
211	else # 4-byte UTF-8
212	push!(dst, 0xe5b8 + (UInt16(a) << 8) + (UInt16(b) << 2) + (c >> 4),	×
213	xor(0xdc80, UInt16(c & 0xf) << 6, d))
214	end
215	else # too short
216	push!(dst, a, b, c)	×
217	break	×
218	end
219	else # too short
220	push!(dst, a, b)	×
221	break	×
222	end
223	else # ASCII or invalid UTF-8 (continuation byte or too-high code point)
224	push!(dst, a)	×
225	end
226	i < n \|\| break	×
227	a = src[i += 1]	×
228	end	×
229	return dst	×
230	end
231
232	function transcode(::Type{UInt8}, src::AbstractVector{UInt16})	×
233	require_one_based_indexing(src)	×
234	n = length(src)	×
235	n == 0 && return UInt8[]	×
236
237	# Precompute m = sizeof(dst). This involves annoying duplication
238	# of the loop over the src array. However, this is not just an
239	# optimization: it is problematic for security reasons to grow
240	# dst dynamically, because Base.winprompt uses this function to
241	# convert passwords to UTF-8 and we don't want to make unintentional
242	# copies of the password data.
243	a = src[1]	×
244	i, m = 1, 0	×
245	while true	×
246	if a < 0x80	×
247	m += 1	×
248	elseif a < 0x800 # 2-byte UTF-8	×
249	m += 2	×
250	elseif a & 0xfc00 == 0xd800 && i < length(src)	×
251	b = src[i += 1]	×
252	if (b & 0xfc00) == 0xdc00 # 2-unit UTF-16 sequence => 4-byte UTF-8	×
253	m += 4	×
254	else
255	m += 3	×
256	a = b; continue	×
257	end
258	else
259	# 1-unit high UTF-16 or unpaired high surrogate
260	# either way, encode as 3-byte UTF-8 code point
261	m += 3	×
262	end
263	i < n \|\| break	×
264	a = src[i += 1]	×
265	end	×
266
267	dst = StringVector(m)	×
268	a = src[1]	×
269	i, j = 1, 0	×
270	while true	×
271	if a < 0x80 # ASCII	×
272	dst[j += 1] = a % UInt8	×
273	elseif a < 0x800 # 2-byte UTF-8	×
274	dst[j += 1] = 0xc0 \| ((a >> 6) % UInt8)	×
275	dst[j += 1] = 0x80 \| ((a % UInt8) & 0x3f)	×
276	elseif a & 0xfc00 == 0xd800 && i < n	×
277	b = src[i += 1]	×
278	if (b & 0xfc00) == 0xdc00	×
279	# 2-unit UTF-16 sequence => 4-byte UTF-8
280	a += 0x2840	×
281	dst[j += 1] = 0xf0 \| ((a >> 8) % UInt8)	×
282	dst[j += 1] = 0x80 \| ((a % UInt8) >> 2)	×
283	dst[j += 1] = xor(0xf0, ((a % UInt8) << 4) & 0x3f, (b >> 6) % UInt8)	×
284	dst[j += 1] = 0x80 \| ((b % UInt8) & 0x3f)	×
285	else
286	dst[j += 1] = 0xe0 \| ((a >> 12) % UInt8)	×
287	dst[j += 1] = 0x80 \| (((a >> 6) % UInt8) & 0x3f)	×
288	dst[j += 1] = 0x80 \| ((a % UInt8) & 0x3f)	×
289	a = b; continue	×
290	end
291	else
292	# 1-unit high UTF-16 or unpaired high surrogate
293	# either way, encode as 3-byte UTF-8 code point
294	dst[j += 1] = 0xe0 \| ((a >> 12) % UInt8)	×
295	dst[j += 1] = 0x80 \| (((a >> 6) % UInt8) & 0x3f)	×
296	dst[j += 1] = 0x80 \| ((a % UInt8) & 0x3f)	×
297	end
298	i < n \|\| break	×
299	a = src[i += 1]	×
300	end	×
301	return dst	×
302	end
303
304	function unsafe_string(p::Ptr{T}, length::Integer) where {T<:Union{UInt16,UInt32,Cwchar_t}}	×
305	transcode(String, unsafe_wrap(Array, p, length; own=false))	×
306	end
307	function unsafe_string(cw::Cwstring)	×
308	p = convert(Ptr{Cwchar_t}, cw)	×
309	n = 1	×
310	while unsafe_load(p, n) != 0	×
311	n += 1	×
312	end	×
313	return unsafe_string(p, n - 1)	×
314	end

JuliaLang / julia / #37822

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous