#37776

Committed 11 May 2024 04:45AM UTC coverage: 86.279% (-1.1%) from 87.427%

Build # #37776

Build Type

push

local

Committed by

web-flow

Commit Message

Move nargs/isva to CodeInfo (#54341)

This changes the canonical source of truth for va handling from `Method`
to `CodeInfo`. There are multiple goals for this change:

1. This addresses a longstanding complaint about the way that
CodeInfo-returning generated functions work. Previously, the va-ness or
not of the returned CodeInfo always had to match that of the generator.
For Cassette-like transforms that generally have one big generator
function that is varargs (while then looking up lowered code that is not
varargs), this could become quite annoying. It's possible to workaround,
but there is really no good reason to tie the two together. As we
observed when we implemented OpaqueClosures, the vararg-ness of the
signature and the `vararg arguments`->`tuple` transformation are mostly
independent concepts. With this PR, generated functions can return
CodeInfos with whatever combination of nargs/isva is convenient.

2. This change requires clarifying where the va processing boundary is
in inference. #54076 was already moving in that direction for irinterp,
and this essentially does much of the same for regular inference. As a
consequence the constprop cache is now using non-va-cooked signatures,
which I think is preferable.

3. This further decouples codegen from the presence of a `Method` (which
is already not assumed, since the code being generated could be a
toplevel thunk, but some codegen features are only available to things
that come from Methods). There are a number of upcoming features that
will require codegen of things that are not quite method specializations
(See design doc linked in #52797 and things like #50641). This helps
pave the road for that.

4. I've previously considered expanding the kinds of vararg signatures
that can be described (see e.g. #53851), which also requires a
decoupling of the signature and ast notions of vararg. This again lays
the groundwork for that, although I have no immediate plans... (continued)

Run Details

91 of 100 new or added lines in 9 files covered. (91.0%)

1301 existing lines in 49 files now uncovered.

75382 of 87370 relevant lines covered (86.28%)

15956665.03 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.33

/stdlib/Base64/src/encode.jl

# This file is a part of Julia. License is MIT: https://julialang.org/license

# Generate encode table.
const BASE64_ENCODE = [UInt8(x) for x in append!(['A':'Z'; 'a':'z'; '0':'9'], ['+', '/'])]
encode(x::UInt8) = @inbounds return BASE64_ENCODE[(x & 0x3f) + 1]
encodepadding()  = UInt8('=')

"""
    Base64EncodePipe(ostream)

Return a new write-only I/O stream, which converts any bytes written to it into
base64-encoded ASCII bytes written to `ostream`.  Calling [`close`](@ref) on the
`Base64EncodePipe` stream is necessary to complete the encoding (but does not
close `ostream`).

# Examples
```jldoctest
julia> io = IOBuffer();

julia> iob64_encode = Base64EncodePipe(io);

julia> write(iob64_encode, "Hello!")
6

julia> close(iob64_encode);

julia> str = String(take!(io))
"SGVsbG8h"

julia> String(base64decode(str))
"Hello!"
```
"""
struct Base64EncodePipe <: IO
    io::IO
    buffer::Buffer

    function Base64EncodePipe(io::IO)
        # The buffer size must be at least 3.
        buffer = Buffer(512)
        pipe = new(io, buffer)
        finalizer(_ -> close(pipe), buffer)
        return pipe
    end
end

Base.isreadable(::Base64EncodePipe) = false
Base.iswritable(pipe::Base64EncodePipe) = iswritable(pipe.io)

function Base.unsafe_write(pipe::Base64EncodePipe, ptr::Ptr{UInt8}, n::UInt)::Int
    buffer = pipe.buffer
    m = buffer.size
    b1, b2, b3, k = loadtriplet!(buffer, ptr, n)
    @assert k ≥ m
    p = ptr + k - m
    if k < 3
        if k == 1
            buffer[1] = b1
            buffer.size = 1
        elseif k == 2
            buffer[1] = b1
            buffer[2] = b2
            buffer.size = 2
        end
        return p - ptr
    end
    @assert buffer.size == 0

    i = 0
    p_end = ptr + n
    while true
        buffer[i+1] = encode(b1 >> 2          )
        buffer[i+2] = encode(b1 << 4 | b2 >> 4)
        buffer[i+3] = encode(b2 << 2 | b3 >> 6)
        buffer[i+4] = encode(          b3     )
        i += 4
        if p + 2 < p_end
            b1 = unsafe_load(p, 1)
            b2 = unsafe_load(p, 2)
            b3 = unsafe_load(p, 3)
            p += 3
        else
            break
        end
        if i + 4 > capacity(buffer)
            unsafe_write(pipe.io, pointer(buffer), i)
            i = 0
        end
    end
    if i > 0
        unsafe_write(pipe.io, pointer(buffer), i)
    end

    while p < p_end
        buffer[buffer.size+=1] = unsafe_load(p)
        p += 1
    end
    return p - ptr
end

function Base.write(pipe::Base64EncodePipe, x::UInt8)
    buffer = pipe.buffer
    buffer[buffer.size+=1] = x
    if buffer.size == 3
        unsafe_write(pipe, C_NULL, 0)
    end
    return 1
end

function Base.close(pipe::Base64EncodePipe)
    b1, b2, b3, k = loadtriplet!(pipe.buffer, Ptr{UInt8}(C_NULL), UInt(0))
    if k == 0
        # no leftover and padding
    elseif k == 1
        write(pipe.io,
              encode(b1 >> 2),
              encode(b1 << 4),
              encodepadding(),
              encodepadding())
    elseif k == 2
        write(pipe.io,
              encode(          b1 >> 2),
              encode(b1 << 4 | b2 >> 4),
              encode(b2 << 2          ),
              encodepadding())
    else
        @assert k == 3
        write(pipe.io,
              encode(b1 >> 2          ),
              encode(b1 << 4 | b2 >> 4),
              encode(b2 << 2 | b3 >> 6),
              encode(          b3     ))
    end
    return nothing
end

# Load three bytes from buffer and ptr.
function loadtriplet!(buffer::Buffer, ptr::Ptr{UInt8}, n::UInt)
    b1 = b2 = b3 = 0x00
    if buffer.size == 0
        if n == 0
            k = 0
        elseif n == 1
            b1 = unsafe_load(ptr, 1)
            k = 1
        elseif n == 2
            b1 = unsafe_load(ptr, 1)
            b2 = unsafe_load(ptr, 2)
            k = 2
        else
            b1 = unsafe_load(ptr, 1)
            b2 = unsafe_load(ptr, 2)
            b3 = unsafe_load(ptr, 3)
            k = 3
        end
    elseif buffer.size == 1
        b1 = buffer[1]
        if n == 0
            k = 1
        elseif n == 1
            b2 = unsafe_load(ptr, 1)
            k = 2
        else
            b2 = unsafe_load(ptr, 1)
            b3 = unsafe_load(ptr, 2)
            k = 3
        end
    elseif buffer.size == 2
        b1 = buffer[1]
        b2 = buffer[2]
        if n == 0
            k = 2
        else
            b3 = unsafe_load(ptr, 1)
            k = 3
        end
    else
        @assert buffer.size == 3
        b1 = buffer[1]
        b2 = buffer[2]
        b3 = buffer[3]
        k = 3
    end
    empty!(buffer)
    return b1, b2, b3, k
end

"""
    base64encode(writefunc, args...; context=nothing)
    base64encode(args...; context=nothing)

Given a [`write`](@ref)-like function `writefunc`, which takes an I/O stream as
its first argument, `base64encode(writefunc, args...)` calls `writefunc` to
write `args...` to a base64-encoded string, and returns the string.
`base64encode(args...)` is equivalent to `base64encode(write, args...)`: it
converts its arguments into bytes using the standard [`write`](@ref) functions
and returns the base64-encoded string.

The optional keyword argument `context` can be set to `:key=>value` pair
or an `IO` or [`IOContext`](@ref) object whose attributes are used for the I/O
stream passed to `writefunc` or `write`.

See also [`base64decode`](@ref).
"""
function base64encode(f::Function, args...; context=nothing)
    s = IOBuffer()
    b = Base64EncodePipe(s)
    if context === nothing
        f(b, args...)
    else
        f(IOContext(b, context), args...)
    end
    close(b)
    return String(take!(s))
end
base64encode(args...; context=nothing) = base64encode(write, args...; context=context)

1	# This file is a part of Julia. License is MIT: https://julialang.org/license
2
3	# Generate encode table.
4	const BASE64_ENCODE = [UInt8(x) for x in append!(['A':'Z'; 'a':'z'; '0':'9'], ['+', '/'])]
5	encode(x::UInt8) = @inbounds return BASE64_ENCODE[(x & 0x3f) + 1]	198,844✔
6	encodepadding() = UInt8('=')	×
7
8	"""
9	Base64EncodePipe(ostream)
10
11	Return a new write-only I/O stream, which converts any bytes written to it into
12	base64-encoded ASCII bytes written to `ostream`. Calling [`close`](@ref) on the
13	`Base64EncodePipe` stream is necessary to complete the encoding (but does not
14	close `ostream`).
15
16	# Examples
17	```jldoctest
18	julia> io = IOBuffer();
19
20	julia> iob64_encode = Base64EncodePipe(io);
21
22	julia> write(iob64_encode, "Hello!")
23	6
24
25	julia> close(iob64_encode);
26
27	julia> str = String(take!(io))
28	"SGVsbG8h"
29
30	julia> String(base64decode(str))
31	"Hello!"
32	```
33	"""
34	struct Base64EncodePipe <: IO
35	io::IO
36	buffer::Buffer
37
38	function Base64EncodePipe(io::IO)	2✔
39	# The buffer size must be at least 3.
40	buffer = Buffer(512)	1,007✔
41	pipe = new(io, buffer)	1,007✔
42	finalizer(_ -> close(pipe), buffer)	2,014✔
43	return pipe	1,007✔
44	end
45	end
46
47	Base.isreadable(::Base64EncodePipe) = false	×
48	Base.iswritable(pipe::Base64EncodePipe) = iswritable(pipe.io)	2✔
49
50	function Base.unsafe_write(pipe::Base64EncodePipe, ptr::Ptr{UInt8}, n::UInt)::Int	1,095✔
51	buffer = pipe.buffer	1,095✔
52	m = buffer.size	1,095✔
53	b1, b2, b3, k = loadtriplet!(buffer, ptr, n)	2,094✔
54	@assert k ≥ m	1,095✔
55	p = ptr + k - m	1,095✔
56	if k < 3	1,095✔
57	if k == 1	8✔
UNCOV 58	buffer[1] = b1	×
UNCOV 59	buffer.size = 1	×
60	elseif k == 2	8✔
61	buffer[1] = b1	1✔
62	buffer[2] = b2	1✔
63	buffer.size = 2	1✔
64	end
65	return p - ptr	8✔
66	end
67	@assert buffer.size == 0	1,087✔
68
69	i = 0	1,087✔
70	p_end = ptr + n	1,087✔
71	while true	49,324✔
72	buffer[i+1] = encode(b1 >> 2 )	49,324✔
73	buffer[i+2] = encode(b1 << 4 \| b2 >> 4)	49,324✔
74	buffer[i+3] = encode(b2 << 2 \| b3 >> 6)	49,324✔
75	buffer[i+4] = encode( b3 )	49,324✔
76	i += 4	49,324✔
77	if p + 2 < p_end	49,324✔
78	b1 = unsafe_load(p, 1)	48,237✔
79	b2 = unsafe_load(p, 2)	48,237✔
80	b3 = unsafe_load(p, 3)	48,237✔
81	p += 3	48,237✔
82	else
83	break	1,087✔
84	end
85	if i + 4 > capacity(buffer)	48,237✔
86	unsafe_write(pipe.io, pointer(buffer), i)	×
87	i = 0	×
88	end
89	end	48,237✔
90	if i > 0	1,087✔
91	unsafe_write(pipe.io, pointer(buffer), i)	1,087✔
92	end
93
94	while p < p_end	2,007✔
95	buffer[buffer.size+=1] = unsafe_load(p)	921✔
96	p += 1	921✔
97	end	921✔
98	return p - ptr	1,086✔
99	end
100
101	function Base.write(pipe::Base64EncodePipe, x::UInt8)	269✔
102	buffer = pipe.buffer	269✔
103	buffer[buffer.size+=1] = x	269✔
104	if buffer.size == 3	269✔
105	unsafe_write(pipe, C_NULL, 0)	89✔
106	end
107	return 1	269✔
108	end
109
110	function Base.close(pipe::Base64EncodePipe)	2,014✔
111	b1, b2, b3, k = loadtriplet!(pipe.buffer, Ptr{UInt8}(C_NULL), UInt(0))	2,637✔
112	if k == 0	2,014✔
113	# no leftover and padding
114	elseif k == 1	623✔
115	write(pipe.io,	321✔
116	encode(b1 >> 2),
117	encode(b1 << 4),
118	encodepadding(),
119	encodepadding())
120	elseif k == 2	302✔
121	write(pipe.io,	302✔
122	encode( b1 >> 2),
123	encode(b1 << 4 \| b2 >> 4),
124	encode(b2 << 2 ),
125	encodepadding())
126	else
127	@assert k == 3	×
128	write(pipe.io,	×
129	encode(b1 >> 2 ),
130	encode(b1 << 4 \| b2 >> 4),
131	encode(b2 << 2 \| b3 >> 6),
132	encode( b3 ))
133	end
134	return nothing	2,014✔
135	end
136
137	# Load three bytes from buffer and ptr.
138	function loadtriplet!(buffer::Buffer, ptr::Ptr{UInt8}, n::UInt)
139	b1 = b2 = b3 = 0x00	3,109✔
140	if buffer.size == 0	3,109✔
141	if n == 0	2,397✔
142	k = 0	1,398✔
143	elseif n == 1	999✔
UNCOV 144	b1 = unsafe_load(ptr, 1)	×
UNCOV 145	k = 1	×
146	elseif n == 2	999✔
147	b1 = unsafe_load(ptr, 1)	1✔
148	b2 = unsafe_load(ptr, 2)	1✔
149	k = 2	1✔
150	else
151	b1 = unsafe_load(ptr, 1)	998✔
152	b2 = unsafe_load(ptr, 2)	998✔
153	b3 = unsafe_load(ptr, 3)	998✔
154	k = 3	998✔
155	end
156	elseif buffer.size == 1	712✔
157	b1 = buffer[1]	321✔
158	if n == 0	321✔
159	k = 1	321✔
160	elseif n == 1	×
161	b2 = unsafe_load(ptr, 1)	×
162	k = 2	×
163	else
164	b2 = unsafe_load(ptr, 1)	×
165	b3 = unsafe_load(ptr, 2)	×
166	k = 3	×
167	end
168	elseif buffer.size == 2	391✔
169	b1 = buffer[1]	302✔
170	b2 = buffer[2]	302✔
171	if n == 0	302✔
172	k = 2	302✔
173	else
174	b3 = unsafe_load(ptr, 1)	×
175	k = 3	×
176	end
177	else
178	@assert buffer.size == 3	89✔
179	b1 = buffer[1]	89✔
180	b2 = buffer[2]	89✔
181	b3 = buffer[3]	89✔
182	k = 3	89✔
183	end
184	empty!(buffer)	3,109✔
185	return b1, b2, b3, k	3,109✔
186	end
187
188	"""
189	base64encode(writefunc, args...; context=nothing)
190	base64encode(args...; context=nothing)
191
192	Given a [`write`](@ref)-like function `writefunc`, which takes an I/O stream as
193	its first argument, `base64encode(writefunc, args...)` calls `writefunc` to
194	write `args...` to a base64-encoded string, and returns the string.
195	`base64encode(args...)` is equivalent to `base64encode(write, args...)`: it
196	converts its arguments into bytes using the standard [`write`](@ref) functions
197	and returns the base64-encoded string.
198
199	The optional keyword argument `context` can be set to `:key=>value` pair
200	or an `IO` or [`IOContext`](@ref) object whose attributes are used for the I/O
201	stream passed to `writefunc` or `write`.
202
203	See also [`base64decode`](@ref).
204	"""
205	function base64encode(f::Function, args...; context=nothing)	2,008✔
206	s = IOBuffer()	1,004✔
207	b = Base64EncodePipe(s)	1,004✔
208	if context === nothing	1,004✔
209	f(b, args...)	1,004✔
210	else
211	f(IOContext(b, context), args...)	×
212	end
213	close(b)	1,004✔
214	return String(take!(s))	1,004✔
215	end
216	base64encode(args...; context=nothing) = base64encode(write, args...; context=context)	2,002✔

JuliaLang / julia / #37776

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous