#38002

Committed 06 Feb 2025 06:14AM UTC coverage: 20.322% (-2.4%) from 22.722%

Build # #38002

Build Type

push

local

Committed by

web-flow

Commit Message

bpart: Fully switch to partitioned semantics (#57253)

This is the final PR in the binding partitions series (modulo bugs and
tweaks), i.e. it closes #54654 and thus closes #40399, which was the
original design sketch.

This thus activates the full designed semantics for binding partitions,
in particular allowing safe replacement of const bindings. It in
particular allows struct redefinitions. This thus closes
timholy/Revise.jl#18 and also closes #38584.

The biggest semantic change here is probably that this gets rid of the
notion of "resolvedness" of a binding. Previously, a lot of the behavior
of our implementation depended on when bindings were "resolved", which
could happen at basically an arbitrary point (in the compiler, in REPL
completion, in a different thread), making a lot of the semantics around
bindings ill- or at least implementation-defined. There are several
related issues in the bugtracker, so this closes #14055 closes #44604
closes #46354 closes #30277

It is also the last step to close #24569.
It also supports bindings for undef->defined transitions and thus closes
#53958 closes #54733 - however, this is not activated yet for
performance reasons and may need some further optimization.

Since resolvedness no longer exists, we need to replace it with some
hopefully more well-defined semantics. I will describe the semantics
below, but before I do I will make two notes:

1. There are a number of cases where these semantics will behave
slightly differently than the old semantics absent some other task going
around resolving random bindings.
2. The new behavior (except for the replacement stuff) was generally
permissible under the old semantics if the bindings happened to be
resolved at the right time.

With all that said, there are essentially three "strengths" of bindings:

1. Implicit Bindings: Anything implicitly obtained from `using Mod`, "no
binding", plus slightly more exotic corner cases around conflicts

2. Weakly declared bindin... (continued)

Run Details

11 of 111 new or added lines in 7 files covered. (9.91%)

1273 existing lines in 68 files now uncovered.

9908 of 48755 relevant lines covered (20.32%)

105126.48 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

25.42

/base/env.jl

# This file is a part of Julia. License is MIT: https://julialang.org/license

if Sys.iswindows()
    const ERROR_ENVVAR_NOT_FOUND = UInt32(203)

    const env_dict = Lockable(Dict{String, Vector{Cwchar_t}}())

    function memoized_env_lookup(str::AbstractString)
        # Windows environment variables have a different format from Linux / MacOS, and previously
        # incurred allocations because we had to convert a String to a Vector{Cwchar_t} each time
        # an environment variable was looked up. This function memoizes that lookup process, storing
        # the String => Vector{Cwchar_t} pairs in env_dict
        @lock env_dict begin
            var = get(env_dict[], str, nothing)
            if isnothing(var)
                var = cwstring(str)
                env_dict[][str] = var
            end
            return var
        end
    end

    _getenvlen(var::Vector{UInt16}) = ccall(:GetEnvironmentVariableW,stdcall,UInt32,(Ptr{UInt16},Ptr{UInt16},UInt32),var,C_NULL,0)
    _hasenv(s::Vector{UInt16}) = _getenvlen(s) != 0 || Libc.GetLastError() != ERROR_ENVVAR_NOT_FOUND
    _hasenv(s::AbstractString) = _hasenv(memoized_env_lookup(s))

    function access_env(onError::Function, str::AbstractString)
        var = memoized_env_lookup(str)
        len = _getenvlen(var)
        if len == 0
            return Libc.GetLastError() != ERROR_ENVVAR_NOT_FOUND ? "" : onError(str)
        end
        val = zeros(UInt16,len)
        ret = ccall(:GetEnvironmentVariableW,stdcall,UInt32,(Ptr{UInt16},Ptr{UInt16},UInt32),var,val,len)
        windowserror(:getenv, (ret == 0 && len != 1) || ret != len-1 || val[end] != 0)
        pop!(val) # NUL
        return transcode(String, val)
    end

    function _setenv(svar::AbstractString, sval::AbstractString, overwrite::Bool=true)
        var = memoized_env_lookup(svar)
        val = cwstring(sval)
        if overwrite || !_hasenv(var)
            ret = ccall(:SetEnvironmentVariableW,stdcall,Int32,(Ptr{UInt16},Ptr{UInt16}),var,val)
            windowserror(:setenv, ret == 0)
        end
    end

    function _unsetenv(svar::AbstractString)
        var = memoized_env_lookup(svar)
        ret = ccall(:SetEnvironmentVariableW,stdcall,Int32,(Ptr{UInt16},Ptr{UInt16}),var,C_NULL)
        windowserror(:setenv, ret == 0 && Libc.GetLastError() != ERROR_ENVVAR_NOT_FOUND)
    end
else # !windows
    _getenv(var::AbstractString) = ccall(:getenv, Cstring, (Cstring,), var)
    _hasenv(s::AbstractString) = _getenv(s) != C_NULL

    function access_env(onError::Function, var::AbstractString)
        val = _getenv(var)
        val == C_NULL ? onError(var) : unsafe_string(val)
    end

    function _setenv(var::AbstractString, val::AbstractString, overwrite::Bool=true)
        ret = ccall(:setenv, Int32, (Cstring,Cstring,Int32), var, val, overwrite)
        systemerror(:setenv, ret != 0)
    end

    function _unsetenv(var::AbstractString)
        ret = ccall(:unsetenv, Int32, (Cstring,), var)
        systemerror(:unsetenv, ret != 0)
    end
end # os test

## ENV: hash interface ##

"""
    EnvDict() -> EnvDict

A singleton of this type provides a hash table interface to environment variables.
"""
struct EnvDict <: AbstractDict{String,String}; end

"""
    ENV

Reference to the singleton `EnvDict`, providing a dictionary interface to system environment
variables.

(On Windows, system environment variables are case-insensitive, and `ENV` correspondingly converts
all keys to uppercase for display, iteration, and copying. Portable code should not rely on the
ability to distinguish variables by case, and should beware that setting an ostensibly lowercase
variable may result in an uppercase `ENV` key.)

!!! warning
    Mutating the environment is not thread-safe.

# Examples
```julia-repl
julia> ENV
Base.EnvDict with "50" entries:
  "SECURITYSESSIONID"            => "123"
  "USER"                         => "username"
  "MallocNanoZone"               => "0"
  ⋮                              => ⋮

julia> ENV["JULIA_EDITOR"] = "vim"
"vim"

julia> ENV["JULIA_EDITOR"]
"vim"
```

See also: [`withenv`](@ref), [`addenv`](@ref).
"""
const ENV = EnvDict()

const get_bool_env_truthy = (
    "t", "T",
    "true", "True", "TRUE",
    "y", "Y",
    "yes", "Yes", "YES",
    "1")
const get_bool_env_falsy = (
    "f", "F",
    "false", "False", "FALSE",
    "n", "N",
    "no", "No", "NO",
    "0")

"""
    Base.get_bool_env(name::String, default::Bool; throw=false)::Union{Bool,Nothing}
    Base.get_bool_env(f_default::Callable, name::String; throw=false)::Union{Bool,Nothing}

Evaluate whether the value of environment variable `name` is a truthy or falsy string,
and return `nothing` (or throw if `throw=true`) if it is not recognized as either. If
the variable is not set, or is set to "", return `default` or the result of executing `f_default()`.

Recognized values are the following, and their Capitalized and UPPERCASE forms:
    truthy: "t", "true", "y", "yes", "1"
    falsy:  "f", "false", "n", "no", "0"
"""
get_bool_env(name::String, default::Bool; kwargs...) = get_bool_env(Returns(default), name; kwargs...)
function get_bool_env(f_default::Callable, name::String; kwargs...)
    if haskey(ENV, name)
        val = ENV[name]
        if !isempty(val)
            return parse_bool_env(name, val; kwargs...)
        end
    end
    return f_default()
end
function parse_bool_env(name::String, val::String = ENV[name]; throw::Bool=false)
    if val in get_bool_env_truthy
        return true
    elseif val in get_bool_env_falsy
        return false
    elseif throw
        Base.throw(ArgumentError("Value for environment variable `$name` could not be parsed as Boolean: $(repr(val))"))
    else
        return nothing
    end
end

getindex(::EnvDict, k::AbstractString) = access_env(k->throw(KeyError(k)), k)
get(::EnvDict, k::AbstractString, def) = access_env(Returns(def), k)
get(f::Callable, ::EnvDict, k::AbstractString) = access_env(k->f(), k)
function get!(default::Callable, ::EnvDict, k::AbstractString)
    haskey(ENV, k) && return ENV[k]
    ENV[k] = default()
end
in(k::AbstractString, ::KeySet{String, EnvDict}) = _hasenv(k)
pop!(::EnvDict, k::AbstractString) = (v = ENV[k]; _unsetenv(k); v)
pop!(::EnvDict, k::AbstractString, def) = haskey(ENV,k) ? pop!(ENV,k) : def
delete!(::EnvDict, k::AbstractString) = (_unsetenv(k); ENV)
setindex!(::EnvDict, v, k::AbstractString) = _setenv(k,string(v))
push!(::EnvDict, kv::Pair{<:AbstractString}) = setindex!(ENV, kv.second, kv.first)

if Sys.iswindows()
    GESW() = (pos = ccall(:GetEnvironmentStringsW, stdcall, Ptr{UInt16}, ()); (pos, pos))
    function winuppercase(s::AbstractString)
        isempty(s) && return s
        LOCALE_INVARIANT = 0x0000007f
        LCMAP_UPPERCASE  = 0x00000200
        ws = transcode(UInt16, String(s))
        result = ccall(:LCMapStringW, stdcall, Cint, (UInt32, UInt32, Ptr{UInt16}, Cint, Ptr{UInt16}, Cint),
                       LOCALE_INVARIANT, LCMAP_UPPERCASE, ws, length(ws), ws, length(ws))
        systemerror(:LCMapStringW, iszero(result))
        return transcode(String, ws)
    end
    function iterate(hash::EnvDict, block::Tuple{Ptr{UInt16},Ptr{UInt16}} = GESW())
        while true
            if unsafe_load(block[1]) == 0
                ccall(:FreeEnvironmentStringsW, stdcall, Int32, (Ptr{UInt16},), block[2])
                return nothing
            end
            pos = block[1]
            blk = block[2]
            len = ccall(:wcslen, UInt, (Ptr{UInt16},), pos)
            buf = Vector{UInt16}(undef, len)
            GC.@preserve buf unsafe_copyto!(pointer(buf), pos, len)
            env = transcode(String, buf)
            pos += (len + 1) * 2
            if !isempty(env)
                m = findnext('=', env, nextind(env, firstindex(env)))
            else
                m = nothing
            end
            if m === nothing
                @warn "malformed environment entry" env
                continue
            end
            return (Pair{String,String}(winuppercase(env[1:prevind(env, m)]), env[nextind(env, m):end]), (pos, blk))
        end
    end
else # !windows
    function iterate(::EnvDict, i=0)
        while true
            env = ccall(:jl_environ, Any, (Int32,), i)
            env === nothing && return nothing
            env = env::String
            m = findfirst('=', env)
            if m === nothing
                @warn "malformed environment entry" env
                continue
            end
            return (Pair{String,String}(env[1:prevind(env, m)], env[nextind(env, m):end]), i+1)
        end
    end
end # os-test

#TODO: Make these more efficient
function length(::EnvDict)
    i = 0
    for (k,v) in ENV
        i += 1
    end
    return i
end

function show(io::IO, ::EnvDict)
    for (k,v) = ENV
        println(io, "$k=$v")
    end
end

"""
    withenv(f, kv::Pair...)

Execute `f` in an environment that is temporarily modified (not replaced as in `setenv`)
by zero or more `"var"=>val` arguments `kv`. `withenv` is generally used via the
`withenv(kv...) do ... end` syntax. A value of `nothing` can be used to temporarily unset an
environment variable (if it is set). When `withenv` returns, the original environment has
been restored.

!!! warning
    Changing the environment is not thread-safe. For running external commands with a different
    environment from the parent process, prefer using [`addenv`](@ref) over `withenv`.
"""
function withenv(f, keyvals::Pair{T}...) where T<:AbstractString
    old = Dict{T,Any}()
    for (key,val) in keyvals
        old[key] = get(ENV,key,nothing)
        val !== nothing ? (ENV[key]=val) : delete!(ENV, key)
    end
    try f()
    finally
        for (key,val) in old
            val !== nothing ? (ENV[key]=val) : delete!(ENV, key)
        end
    end
end
withenv(f) = f() # handle empty keyvals case; see #10853

1	# This file is a part of Julia. License is MIT: https://julialang.org/license
2
3	if Sys.iswindows()
4	const ERROR_ENVVAR_NOT_FOUND = UInt32(203)
5
6	const env_dict = Lockable(Dict{String, Vector{Cwchar_t}}())
7
8	function memoized_env_lookup(str::AbstractString)	×
9	# Windows environment variables have a different format from Linux / MacOS, and previously
10	# incurred allocations because we had to convert a String to a Vector{Cwchar_t} each time
11	# an environment variable was looked up. This function memoizes that lookup process, storing
12	# the String => Vector{Cwchar_t} pairs in env_dict
13	@lock env_dict begin	×
14	var = get(env_dict[], str, nothing)	×
15	if isnothing(var)	×
16	var = cwstring(str)	×
17	env_dict[][str] = var	×
18	end
19	return var	×
20	end
21	end
22
23	_getenvlen(var::Vector{UInt16}) = ccall(:GetEnvironmentVariableW,stdcall,UInt32,(Ptr{UInt16},Ptr{UInt16},UInt32),var,C_NULL,0)	×
24	_hasenv(s::Vector{UInt16}) = _getenvlen(s) != 0 \|\| Libc.GetLastError() != ERROR_ENVVAR_NOT_FOUND	×
25	_hasenv(s::AbstractString) = _hasenv(memoized_env_lookup(s))	×
26
27	function access_env(onError::Function, str::AbstractString)	×
28	var = memoized_env_lookup(str)	×
29	len = _getenvlen(var)	×
30	if len == 0	×
31	return Libc.GetLastError() != ERROR_ENVVAR_NOT_FOUND ? "" : onError(str)	×
32	end
33	val = zeros(UInt16,len)	×
34	ret = ccall(:GetEnvironmentVariableW,stdcall,UInt32,(Ptr{UInt16},Ptr{UInt16},UInt32),var,val,len)	×
35	windowserror(:getenv, (ret == 0 && len != 1) \|\| ret != len-1 \|\| val[end] != 0)	×
36	pop!(val) # NUL	×
37	return transcode(String, val)	×
38	end
39
40	function _setenv(svar::AbstractString, sval::AbstractString, overwrite::Bool=true)	×
41	var = memoized_env_lookup(svar)	×
42	val = cwstring(sval)	×
43	if overwrite \|\| !_hasenv(var)	×
44	ret = ccall(:SetEnvironmentVariableW,stdcall,Int32,(Ptr{UInt16},Ptr{UInt16}),var,val)	×
45	windowserror(:setenv, ret == 0)	×
46	end
47	end
48
49	function _unsetenv(svar::AbstractString)	×
50	var = memoized_env_lookup(svar)	×
51	ret = ccall(:SetEnvironmentVariableW,stdcall,Int32,(Ptr{UInt16},Ptr{UInt16}),var,C_NULL)	×
52	windowserror(:setenv, ret == 0 && Libc.GetLastError() != ERROR_ENVVAR_NOT_FOUND)	×
53	end
54	else # !windows
55	_getenv(var::AbstractString) = ccall(:getenv, Cstring, (Cstring,), var)	67✔
56	_hasenv(s::AbstractString) = _getenv(s) != C_NULL	45✔
57
58	function access_env(onError::Function, var::AbstractString)	22✔
59	val = _getenv(var)	22✔
60	val == C_NULL ? onError(var) : unsafe_string(val)	22✔
61	end
62
63	function _setenv(var::AbstractString, val::AbstractString, overwrite::Bool=true)	1✔
64	ret = ccall(:setenv, Int32, (Cstring,Cstring,Int32), var, val, overwrite)	29✔
65	systemerror(:setenv, ret != 0)	1✔
66	end
67
68	function _unsetenv(var::AbstractString)
69	ret = ccall(:unsetenv, Int32, (Cstring,), var)	13✔
70	systemerror(:unsetenv, ret != 0)	13✔
71	end
72	end # os test
73
74	## ENV: hash interface ##
75
76	"""
77	EnvDict() -> EnvDict
78
79	A singleton of this type provides a hash table interface to environment variables.
80	"""
81	struct EnvDict <: AbstractDict{String,String}; end
82
83	"""
84	ENV
85
86	Reference to the singleton `EnvDict`, providing a dictionary interface to system environment
87	variables.
88
89	(On Windows, system environment variables are case-insensitive, and `ENV` correspondingly converts
90	all keys to uppercase for display, iteration, and copying. Portable code should not rely on the
91	ability to distinguish variables by case, and should beware that setting an ostensibly lowercase
92	variable may result in an uppercase `ENV` key.)
93
94	!!! warning
95	Mutating the environment is not thread-safe.
96
97	# Examples
98	```julia-repl
99	julia> ENV
100	Base.EnvDict with "50" entries:
101	"SECURITYSESSIONID" => "123"
102	"USER" => "username"
103	"MallocNanoZone" => "0"
104	⋮ => ⋮
105
106	julia> ENV["JULIA_EDITOR"] = "vim"
107	"vim"
108
109	julia> ENV["JULIA_EDITOR"]
110	"vim"
111	```
112
113	See also: [`withenv`](@ref), [`addenv`](@ref).
114	"""
115	const ENV = EnvDict()
116
117	const get_bool_env_truthy = (
118	"t", "T",
119	"true", "True", "TRUE",
120	"y", "Y",
121	"yes", "Yes", "YES",
122	"1")
123	const get_bool_env_falsy = (
124	"f", "F",
125	"false", "False", "FALSE",
126	"n", "N",
127	"no", "No", "NO",
128	"0")
129
130	"""
131	Base.get_bool_env(name::String, default::Bool; throw=false)::Union{Bool,Nothing}
132	Base.get_bool_env(f_default::Callable, name::String; throw=false)::Union{Bool,Nothing}
133
134	Evaluate whether the value of environment variable `name` is a truthy or falsy string,
135	and return `nothing` (or throw if `throw=true`) if it is not recognized as either. If
136	the variable is not set, or is set to "", return `default` or the result of executing `f_default()`.
137
138	Recognized values are the following, and their Capitalized and UPPERCASE forms:
139	truthy: "t", "true", "y", "yes", "1"
140	falsy: "f", "false", "n", "no", "0"
141	"""
142	get_bool_env(name::String, default::Bool; kwargs...) = get_bool_env(Returns(default), name; kwargs...)	314✔
143	function get_bool_env(f_default::Callable, name::String; kwargs...)	174✔
144	if haskey(ENV, name)	17✔
145	val = ENV[name]	×
146	if !isempty(val)	×
147	return parse_bool_env(name, val; kwargs...)	×
148	end
149	end
150	return f_default()	17✔
151	end
152	function parse_bool_env(name::String, val::String = ENV[name]; throw::Bool=false)	×
153	if val in get_bool_env_truthy	×
154	return true	×
155	elseif val in get_bool_env_falsy	×
156	return false	×
157	elseif throw	×
158	Base.throw(ArgumentError("Value for environment variable `$name` could not be parsed as Boolean: $(repr(val))"))	×
159	else
160	return nothing	×
161	end
162	end
163
164	getindex(::EnvDict, k::AbstractString) = access_env(k->throw(KeyError(k)), k)	9✔
165	get(::EnvDict, k::AbstractString, def) = access_env(Returns(def), k)	1,983✔
166	get(f::Callable, ::EnvDict, k::AbstractString) = access_env(k->f(), k)	×
167	function get!(default::Callable, ::EnvDict, k::AbstractString)	×
168	haskey(ENV, k) && return ENV[k]	×
169	ENV[k] = default()	×
170	end
171	in(k::AbstractString, ::KeySet{String, EnvDict}) = _hasenv(k)	45✔
172	pop!(::EnvDict, k::AbstractString) = (v = ENV[k]; _unsetenv(k); v)	×
173	pop!(::EnvDict, k::AbstractString, def) = haskey(ENV,k) ? pop!(ENV,k) : def	×
174	delete!(::EnvDict, k::AbstractString) = (_unsetenv(k); ENV)	13✔
175	setindex!(::EnvDict, v, k::AbstractString) = _setenv(k,string(v))	28✔
176	push!(::EnvDict, kv::Pair{<:AbstractString}) = setindex!(ENV, kv.second, kv.first)	×
177
178	if Sys.iswindows()
179	GESW() = (pos = ccall(:GetEnvironmentStringsW, stdcall, Ptr{UInt16}, ()); (pos, pos))	×
180	function winuppercase(s::AbstractString)	×
181	isempty(s) && return s	×
182	LOCALE_INVARIANT = 0x0000007f	×
183	LCMAP_UPPERCASE = 0x00000200	×
184	ws = transcode(UInt16, String(s))	×
185	result = ccall(:LCMapStringW, stdcall, Cint, (UInt32, UInt32, Ptr{UInt16}, Cint, Ptr{UInt16}, Cint),	×
186	LOCALE_INVARIANT, LCMAP_UPPERCASE, ws, length(ws), ws, length(ws))
187	systemerror(:LCMapStringW, iszero(result))	×
188	return transcode(String, ws)	×
189	end
190	function iterate(hash::EnvDict, block::Tuple{Ptr{UInt16},Ptr{UInt16}} = GESW())	×
191	while true	×
192	if unsafe_load(block[1]) == 0	×
193	ccall(:FreeEnvironmentStringsW, stdcall, Int32, (Ptr{UInt16},), block[2])	×
194	return nothing	×
195	end
196	pos = block[1]	×
197	blk = block[2]	×
198	len = ccall(:wcslen, UInt, (Ptr{UInt16},), pos)	×
199	buf = Vector{UInt16}(undef, len)	×
200	GC.@preserve buf unsafe_copyto!(pointer(buf), pos, len)	×
201	env = transcode(String, buf)	×
202	pos += (len + 1) * 2	×
203	if !isempty(env)	×
204	m = findnext('=', env, nextind(env, firstindex(env)))	×
205	else
206	m = nothing	×
207	end
208	if m === nothing	×
209	@warn "malformed environment entry" env	×
210	continue	×
211	end
212	return (Pair{String,String}(winuppercase(env[1:prevind(env, m)]), env[nextind(env, m):end]), (pos, blk))	×
213	end	×
214	end
215	else # !windows
216	function iterate(::EnvDict, i=0)
217	while true	62✔
218	env = ccall(:jl_environ, Any, (Int32,), i)
219	env === nothing && return nothing
220	env = env::String
221	m = findfirst('=', env)
222	if m === nothing
223	@warn "malformed environment entry" env
224	continue
225	end
226	return (Pair{String,String}(env[1:prevind(env, m)], env[nextind(env, m):end]), i+1)
227	end
228	end
229	end # os-test
230
231	#TODO: Make these more efficient
UNCOV 232	function length(::EnvDict)	×
233	i = 0	×
UNCOV 234	for (k,v) in ENV	×
UNCOV 235	i += 1	×
UNCOV 236	end	×
UNCOV 237	return i	×
238	end
239
240	function show(io::IO, ::EnvDict)	×
241	for (k,v) = ENV	×
242	println(io, "$k=$v")	×
243	end	×
244	end
245
246	"""
247	withenv(f, kv::Pair...)
248
249	Execute `f` in an environment that is temporarily modified (not replaced as in `setenv`)
250	by zero or more `"var"=>val` arguments `kv`. `withenv` is generally used via the
251	`withenv(kv...) do ... end` syntax. A value of `nothing` can be used to temporarily unset an
252	environment variable (if it is set). When `withenv` returns, the original environment has
253	been restored.
254
255	!!! warning
256	Changing the environment is not thread-safe. For running external commands with a different
257	environment from the parent process, prefer using [`addenv`](@ref) over `withenv`.
258	"""
259	function withenv(f, keyvals::Pair{T}...) where T<:AbstractString	19✔
260	old = Dict{T,Any}()	19✔
261	for (key,val) in keyvals	19✔
262	old[key] = get(ENV,key,nothing)	31✔
263	val !== nothing ? (ENV[key]=val) : delete!(ENV, key)	19✔
264	end	19✔
265	try f()	19✔
266	finally
267	for (key,val) in old	38✔
268	val !== nothing ? (ENV[key]=val) : delete!(ENV, key)	26✔
269	end	19✔
270	end
271	end
272	withenv(f) = f() # handle empty keyvals case; see #10853	×

JuliaLang / julia / #38002

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous