#37616

Committed 10 Sep 2023 01:51AM UTC coverage: 86.489% (+0.3%) from 86.196%

Build # #37616

Build Type

push

local

Committed by

web-flow

Commit Message

Make _global_logstate a typed global (#51257)

Run Details

73902 of 85447 relevant lines covered (86.49%)

13068259.04 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

37.5

/stdlib/FileWatching/src/pidfile.jl

module Pidfile


export mkpidlock, trymkpidlock

using Base:
    IOError, UV_EEXIST, UV_ESRCH,
    Process

using Base.Libc: rand

using Base.Filesystem:
    File, open, JL_O_CREAT, JL_O_RDWR, JL_O_RDONLY, JL_O_EXCL,
    rename, samefile, path_separator

using ..FileWatching: watch_file
using Base.Sys: iswindows

"""
    mkpidlock([f::Function], at::String, [pid::Cint]; kwopts...)
    mkpidlock(at::String, proc::Process; kwopts...)

Create a pidfile lock for the path "at" for the current process
or the process identified by pid or proc. Can take a function to execute once locked,
for usage in `do` blocks, after which the lock will be automatically closed. If the lock fails
and `wait` is false, then an error is thrown.

The lock will be released by either `close`, a `finalizer`, or shortly after `proc` exits.
Make sure the return value is live through the end of the critical section of
your program, so the `finalizer` does not reclaim it early.

Optional keyword arguments:
 - `mode`: file access mode (modified by the process umask). Defaults to world-readable.
 - `poll_interval`: Specify the maximum time to between attempts (if `watch_file` doesn't work)
 - `stale_age`: Delete an existing pidfile (ignoring the lock) if it is older than this many seconds, based on its mtime.
     The file won't be deleted until 25x longer than this if the pid in the file appears that it may be valid.
     By default this is disabled (`stale_age` = 0), but a typical recommended value would be about 3-5x an
     estimated normal completion time.
 - `refresh`: Keeps a lock from becoming stale by updating the mtime every interval of time that passes.
     By default, this is set to `stale_age/2`, which is the recommended value.
 - `wait`: If true, block until we get the lock, if false, raise error if lock fails.
"""
function mkpidlock end

"""
    trymkpidlock([f::Function], at::String, [pid::Cint]; kwopts...)
    trymkpidlock(at::String, proc::Process; kwopts...)

Like `mkpidlock` except returns `false` instead of waiting if the file is already locked.

!!! compat "Julia 1.10"
    This function requires at least Julia 1.10.
"""
function trymkpidlock end

# mutable only because we want to add a finalizer
mutable struct LockMonitor
    const path::String
    const fd::File
    const update::Union{Nothing,Timer}

    global function mkpidlock(at::String, pid::Cint; stale_age::Real=0, refresh::Real=stale_age/2, kwopts...)
        local lock
        atdir, atname = splitdir(at)
        isempty(atdir) && (atdir = pwd())
        at = realpath(atdir) * path_separator * atname
        fd = open_exclusive(at; stale_age=stale_age, kwopts...)
        update = nothing
        try
            write_pidfile(fd, pid)
            if refresh > 0
                # N.b.: to ensure our finalizer works we are careful to capture
                # `fd` here instead of `lock`.
                update = Timer(t -> isopen(t) && touch(fd), refresh; interval=refresh)
            end
            lock = new(at, fd, update)
            finalizer(close, lock)
        catch ex
            tryrmopenfile(at)
            close(fd)
            rethrow(ex)
        end
        return lock
    end
end

mkpidlock(at::String; kwopts...) = mkpidlock(at, getpid(); kwopts...)
mkpidlock(f::Function, at::String; kwopts...) = mkpidlock(f, at, getpid(); kwopts...)

function mkpidlock(f::Function, at::String, pid::Cint; kwopts...)
    lock = mkpidlock(at, pid; kwopts...)
    try
        return f()
    finally
        close(lock)
    end
end

function mkpidlock(at::String, proc::Process; kwopts...)
    lock = mkpidlock(at, getpid(proc); kwopts...)
    closer = @async begin
        wait(proc)
        close(lock)
    end
    isdefined(Base, :errormonitor) && Base.errormonitor(closer)
    return lock
end

function trymkpidlock(args...; kwargs...)
    try
        mkpidlock(args...; kwargs..., wait=false)
    catch ex
        if ex isa PidlockedError
            return false
        else
            rethrow()
        end
    end
end

"""
    Base.touch(::Pidfile.LockMonitor)

Update the `mtime` on the lock, to indicate it is still fresh.

See also the `refresh` keyword in the [`mkpidlock`](@ref) constructor.
"""
Base.touch(lock::LockMonitor) = (touch(lock.fd); lock)

"""
    write_pidfile(io, pid)

Write our pidfile format to an open IO descriptor.
"""
function write_pidfile(io::IO, pid::Cint)
    print(io, "$pid $(gethostname())")
end

"""
    parse_pidfile(file::Union{IO, String}) => (pid, hostname, age)

Attempt to parse our pidfile format,
replaced an element with (0, "", 0.0), respectively, for any read that failed.
"""
function parse_pidfile(io::IO)
    fields = split(read(io, String), ' ', limit = 2)
    pid = tryparse(Cuint, fields[1])
    pid === nothing && (pid = Cuint(0))
    hostname = (length(fields) == 2) ? fields[2] : ""
    when = mtime(io)
    age = time() - when
    return (pid, hostname, age)
end

function parse_pidfile(path::String)
    try
        existing = open(path, JL_O_RDONLY)
        try
            return parse_pidfile(existing)
        finally
            close(existing)
        end
    catch ex
        isa(ex, EOFError) || isa(ex, IOError) || rethrow(ex)
        return (Cuint(0), "", 0.0)
    end
end

"""
    isvalidpid(hostname::String, pid::Cuint) :: Bool

Attempt to conservatively estimate whether pid is a valid process id.
"""
function isvalidpid(hostname::AbstractString, pid::Cuint)
    # can't inspect remote hosts
    (hostname == "" || hostname == gethostname()) || return true
    # pid < 0 is never valid (must be a parser error or different OS),
    # and would have a completely different meaning when passed to kill
    !iswindows() && pid > typemax(Cint) && return false
    # (similarly for pid 0)
    pid == 0 && return false
    # see if the process id exists by querying kill without sending a signal
    # and checking if it returned ESRCH (no such process)
    return ccall(:uv_kill, Cint, (Cuint, Cint), pid, 0) != UV_ESRCH
end

"""
    stale_pidfile(path::String, stale_age::Real) :: Bool

Helper function for `open_exclusive` for deciding if a pidfile is stale.
"""
function stale_pidfile(path::String, stale_age::Real)
    pid, hostname, age = parse_pidfile(path)
    age < -stale_age && @warn "filesystem time skew detected" path=path
    if age > stale_age
        if (age > stale_age * 25) || !isvalidpid(hostname, pid)
            return true
        end
    end
    return false
end

"""
    tryopen_exclusive(path::String, mode::Integer = 0o444) :: Union{Void, File}

Try to create a new file for read-write advisory-exclusive access,
return nothing if it already exists.
"""
function tryopen_exclusive(path::String, mode::Integer = 0o444)
    try
        return open(path, JL_O_RDWR | JL_O_CREAT | JL_O_EXCL, mode)
    catch ex
        (isa(ex, IOError) && ex.code == UV_EEXIST) || rethrow(ex)
    end
    return nothing
end

struct PidlockedError <: Exception
    msg::AbstractString
end

"""
    open_exclusive(path::String; mode, poll_interval, wait, stale_age) :: File

Create a new a file for read-write advisory-exclusive access.
If `wait` is `false` then error out if the lock files exist
otherwise block until we get the lock.

For a description of the keyword arguments, see [`mkpidlock`](@ref).
"""
function open_exclusive(path::String;
                        mode::Integer = 0o444 #= read-only =#,
                        poll_interval::Real = 10 #= seconds =#,
                        wait::Bool = true #= return on failure if false =#,
                        stale_age::Real = 0 #= disabled =#)
    # fast-path: just try to open it
    file = tryopen_exclusive(path, mode)
    file === nothing || return file
    if !wait
        if file === nothing && stale_age > 0
            if stale_age > 0 && stale_pidfile(path, stale_age)
                @warn "attempting to remove probably stale pidfile" path=path
                tryrmopenfile(path)
            end
            file = tryopen_exclusive(path, mode)
        end
        if file === nothing
            throw(PidlockedError("Failed to get pidfile lock for $(repr(path))."))
        else
            return file
        end
    end
    # fall-back: wait for the lock

    while true
        # start the file-watcher prior to checking for the pidfile existence
        t = @async try
            watch_file(path, poll_interval)
        catch ex
            isa(ex, IOError) || rethrow(ex)
            sleep(poll_interval) # if the watch failed, convert to just doing a sleep
        end
        # now try again to create it
        file = tryopen_exclusive(path, mode)
        file === nothing || return file
        Base.wait(t) # sleep for a bit before trying again
        if stale_age > 0 && stale_pidfile(path, stale_age)
            # if the file seems stale, try to remove it before attempting again
            # set stale_age to zero so we won't attempt again, even if the attempt fails
            stale_age -= stale_age
            @warn "attempting to remove probably stale pidfile" path=path
            tryrmopenfile(path)
        end
    end
end

function _rand_filename(len::Int=4) # modified from Base.Libc
    slug = Base.StringVector(len)
    chars = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
    for i = 1:len
        slug[i] = chars[(Libc.rand() % length(chars)) + 1]
    end
    return String(slug)
end

function tryrmopenfile(path::String)
    # Deleting open file on Windows is a bit hard
    # if we want to reuse the name immediately after:
    # we need to first rename it, then delete it.
    if Sys.iswindows()
        try
            local rmpath
            rmdir, rmname = splitdir(path)
            while true
                rmpath = string(rmdir, isempty(rmdir) ? "" : path_separator,
                    "\$", _rand_filename(), rmname, ".deleted")
                ispath(rmpath) || break
            end
            rename(path, rmpath)
            path = rmpath
        catch ex
            isa(ex, IOError) || rethrow(ex)
        end
    end
    return try
        rm(path)
        true
    catch ex
        isa(ex, IOError) || rethrow(ex)
        ex
    end
end

"""
    close(lock::LockMonitor)

Release a pidfile lock.
"""
function Base.close(lock::LockMonitor)
    update = lock.update
    update === nothing || close(update)
    isopen(lock.fd) || return false
    removed = false
    path = lock.path
    pathstat = try
            # Windows sometimes likes to return EACCES here,
            # if the path is in the process of being deleted
            stat(path)
        catch ex
            ex isa IOError || rethrow()
            removed = ex
            nothing
        end
    if pathstat !== nothing && samefile(stat(lock.fd), pathstat)
        # try not to delete someone else's lock
        removed = tryrmopenfile(path)
    end
    close(lock.fd)
    havelock = removed === true
    havelock || @warn "failed to remove pidfile on close" path=path removed=removed
    return havelock
end

end # module

1	module Pidfile
2
3
4	export mkpidlock, trymkpidlock
5
6	using Base:
7	IOError, UV_EEXIST, UV_ESRCH,
8	Process
9
10	using Base.Libc: rand
11
12	using Base.Filesystem:
13	File, open, JL_O_CREAT, JL_O_RDWR, JL_O_RDONLY, JL_O_EXCL,
14	rename, samefile, path_separator
15
16	using ..FileWatching: watch_file
17	using Base.Sys: iswindows
18
19	"""
20	mkpidlock([f::Function], at::String, [pid::Cint]; kwopts...)
21	mkpidlock(at::String, proc::Process; kwopts...)
22
23	Create a pidfile lock for the path "at" for the current process
24	or the process identified by pid or proc. Can take a function to execute once locked,
25	for usage in `do` blocks, after which the lock will be automatically closed. If the lock fails
26	and `wait` is false, then an error is thrown.
27
28	The lock will be released by either `close`, a `finalizer`, or shortly after `proc` exits.
29	Make sure the return value is live through the end of the critical section of
30	your program, so the `finalizer` does not reclaim it early.
31
32	Optional keyword arguments:
33	- `mode`: file access mode (modified by the process umask). Defaults to world-readable.
34	- `poll_interval`: Specify the maximum time to between attempts (if `watch_file` doesn't work)
35	- `stale_age`: Delete an existing pidfile (ignoring the lock) if it is older than this many seconds, based on its mtime.
36	The file won't be deleted until 25x longer than this if the pid in the file appears that it may be valid.
37	By default this is disabled (`stale_age` = 0), but a typical recommended value would be about 3-5x an
38	estimated normal completion time.
39	- `refresh`: Keeps a lock from becoming stale by updating the mtime every interval of time that passes.
40	By default, this is set to `stale_age/2`, which is the recommended value.
41	- `wait`: If true, block until we get the lock, if false, raise error if lock fails.
42	"""
43	function mkpidlock end
44
45	"""
46	trymkpidlock([f::Function], at::String, [pid::Cint]; kwopts...)
47	trymkpidlock(at::String, proc::Process; kwopts...)
48
49	Like `mkpidlock` except returns `false` instead of waiting if the file is already locked.
50
51	!!! compat "Julia 1.10"
52	This function requires at least Julia 1.10.
53	"""
54	function trymkpidlock end
55
56	# mutable only because we want to add a finalizer
57	mutable struct LockMonitor
58	const path::String
59	const fd::File
60	const update::Union{Nothing,Timer}
61
62	global function mkpidlock(at::String, pid::Cint; stale_age::Real=0, refresh::Real=stale_age/2, kwopts...)	142✔
63	local lock	47✔
64	atdir, atname = splitdir(at)	71✔
65	isempty(atdir) && (atdir = pwd())	71✔
66	at = realpath(atdir) * path_separator * atname	71✔
67	fd = open_exclusive(at; stale_age=stale_age, kwopts...)	71✔
68	update = nothing	47✔
69	try	71✔
70	write_pidfile(fd, pid)	71✔
71	if refresh > 0	71✔
72	# N.b.: to ensure our finalizer works we are careful to capture
73	# `fd` here instead of `lock`.
74	update = Timer(t -> isopen(t) && touch(fd), refresh; interval=refresh)	72✔
75	end
76	lock = new(at, fd, update)	71✔
77	finalizer(close, lock)	71✔
78	catch ex
79	tryrmopenfile(at)	×
80	close(fd)	×
81	rethrow(ex)	×
82	end
83	return lock	71✔
84	end
85	end
86
87	mkpidlock(at::String; kwopts...) = mkpidlock(at, getpid(); kwopts...)	×
88	mkpidlock(f::Function, at::String; kwopts...) = mkpidlock(f, at, getpid(); kwopts...)	142✔
89
90	function mkpidlock(f::Function, at::String, pid::Cint; kwopts...)	142✔
91	lock = mkpidlock(at, pid; kwopts...)	71✔
92	try	71✔
93	return f()	74✔
94	finally
95	close(lock)	71✔
96	end
97	end
98
99	function mkpidlock(at::String, proc::Process; kwopts...)	×
100	lock = mkpidlock(at, getpid(proc); kwopts...)	×
101	closer = @async begin	×
102	wait(proc)	×
103	close(lock)	×
104	end
105	isdefined(Base, :errormonitor) && Base.errormonitor(closer)	×
106	return lock	×
107	end
108
109	function trymkpidlock(args...; kwargs...)	136✔
110	try	68✔
111	mkpidlock(args...; kwargs..., wait=false)	68✔
112	catch ex
113	if ex isa PidlockedError	3✔
114	return false	×
115	else
116	rethrow()	3✔
117	end
118	end
119	end
120
121	"""
122	Base.touch(::Pidfile.LockMonitor)
123
124	Update the `mtime` on the lock, to indicate it is still fresh.
125
126	See also the `refresh` keyword in the [`mkpidlock`](@ref) constructor.
127	"""
128	Base.touch(lock::LockMonitor) = (touch(lock.fd); lock)	×
129
130	"""
131	write_pidfile(io, pid)
132
133	Write our pidfile format to an open IO descriptor.
134	"""
135	function write_pidfile(io::IO, pid::Cint)	71✔
136	print(io, "$pid $(gethostname())")	71✔
137	end
138
139	"""
140	parse_pidfile(file::Union{IO, String}) => (pid, hostname, age)
141
142	Attempt to parse our pidfile format,
143	replaced an element with (0, "", 0.0), respectively, for any read that failed.
144	"""
145	function parse_pidfile(io::IO)	×
146	fields = split(read(io, String), ' ', limit = 2)	×
147	pid = tryparse(Cuint, fields[1])	×
148	pid === nothing && (pid = Cuint(0))	×
149	hostname = (length(fields) == 2) ? fields[2] : ""	×
150	when = mtime(io)	×
151	age = time() - when	×
152	return (pid, hostname, age)	×
153	end
154
155	function parse_pidfile(path::String)	×
156	try	×
157	existing = open(path, JL_O_RDONLY)	×
158	try	×
159	return parse_pidfile(existing)	×
160	finally
161	close(existing)	×
162	end
163	catch ex
164	isa(ex, EOFError) \|\| isa(ex, IOError) \|\| rethrow(ex)	×
165	return (Cuint(0), "", 0.0)	×
166	end
167	end
168
169	"""
170	isvalidpid(hostname::String, pid::Cuint) :: Bool
171
172	Attempt to conservatively estimate whether pid is a valid process id.
173	"""
174	function isvalidpid(hostname::AbstractString, pid::Cuint)	×
175	# can't inspect remote hosts
176	(hostname == "" \|\| hostname == gethostname()) \|\| return true	×
177	# pid < 0 is never valid (must be a parser error or different OS),
178	# and would have a completely different meaning when passed to kill
179	!iswindows() && pid > typemax(Cint) && return false	×
180	# (similarly for pid 0)
181	pid == 0 && return false	×
182	# see if the process id exists by querying kill without sending a signal
183	# and checking if it returned ESRCH (no such process)
184	return ccall(:uv_kill, Cint, (Cuint, Cint), pid, 0) != UV_ESRCH	×
185	end
186
187	"""
188	stale_pidfile(path::String, stale_age::Real) :: Bool
189
190	Helper function for `open_exclusive` for deciding if a pidfile is stale.
191	"""
192	function stale_pidfile(path::String, stale_age::Real)	×
193	pid, hostname, age = parse_pidfile(path)	×
194	age < -stale_age && @warn "filesystem time skew detected" path=path	×
195	if age > stale_age	×
196	if (age > stale_age * 25) \|\| !isvalidpid(hostname, pid)	×
197	return true	×
198	end
199	end
200	return false	×
201	end
202
203	"""
204	tryopen_exclusive(path::String, mode::Integer = 0o444) :: Union{Void, File}
205
206	Try to create a new file for read-write advisory-exclusive access,
207	return nothing if it already exists.
208	"""
209	function tryopen_exclusive(path::String, mode::Integer = 0o444)	71✔
210	try	71✔
211	return open(path, JL_O_RDWR \| JL_O_CREAT \| JL_O_EXCL, mode)	71✔
212	catch ex
213	(isa(ex, IOError) && ex.code == UV_EEXIST) \|\| rethrow(ex)	×
214	end
215	return nothing	×
216	end
217
218	struct PidlockedError <: Exception
219	msg::AbstractString
220	end
221
222	"""
223	open_exclusive(path::String; mode, poll_interval, wait, stale_age) :: File
224
225	Create a new a file for read-write advisory-exclusive access.
226	If `wait` is `false` then error out if the lock files exist
227	otherwise block until we get the lock.
228
229	For a description of the keyword arguments, see [`mkpidlock`](@ref).
230	"""
231	function open_exclusive(path::String;	142✔
232	mode::Integer = 0o444 #= read-only =#,
233	poll_interval::Real = 10 #= seconds =#,
234	wait::Bool = true #= return on failure if false =#,
235	stale_age::Real = 0 #= disabled =#)
236	# fast-path: just try to open it
237	file = tryopen_exclusive(path, mode)	71✔
238	file === nothing \|\| return file	142✔
239	if !wait	×
240	if file === nothing && stale_age > 0	×
241	if stale_age > 0 && stale_pidfile(path, stale_age)	×
242	@warn "attempting to remove probably stale pidfile" path=path	×
243	tryrmopenfile(path)	×
244	end
245	file = tryopen_exclusive(path, mode)	×
246	end
247	if file === nothing	×
248	throw(PidlockedError("Failed to get pidfile lock for $(repr(path))."))	×
249	else
250	return file	×
251	end
252	end
253	# fall-back: wait for the lock
254
255	while true	×
256	# start the file-watcher prior to checking for the pidfile existence
257	t = @async try	×
258	watch_file(path, poll_interval)
259	catch ex
260	isa(ex, IOError) \|\| rethrow(ex)
261	sleep(poll_interval) # if the watch failed, convert to just doing a sleep
262	end
263	# now try again to create it
264	file = tryopen_exclusive(path, mode)	×
265	file === nothing \|\| return file	×
266	Base.wait(t) # sleep for a bit before trying again	×
267	if stale_age > 0 && stale_pidfile(path, stale_age)	×
268	# if the file seems stale, try to remove it before attempting again
269	# set stale_age to zero so we won't attempt again, even if the attempt fails
270	stale_age -= stale_age	×
271	@warn "attempting to remove probably stale pidfile" path=path	×
272	tryrmopenfile(path)	×
273	end
274	end	×
275	end
276
277	function _rand_filename(len::Int=4) # modified from Base.Libc	×
278	slug = Base.StringVector(len)	×
279	chars = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"	×
280	for i = 1:len	×
281	slug[i] = chars[(Libc.rand() % length(chars)) + 1]	×
282	end	×
283	return String(slug)	×
284	end
285
286	function tryrmopenfile(path::String)	71✔
287	# Deleting open file on Windows is a bit hard
288	# if we want to reuse the name immediately after:
289	# we need to first rename it, then delete it.
290	if Sys.iswindows()	×
291	try	×
292	local rmpath	×
293	rmdir, rmname = splitdir(path)	×
294	while true	×
295	rmpath = string(rmdir, isempty(rmdir) ? "" : path_separator,	×
296	"\$", _rand_filename(), rmname, ".deleted")
297	ispath(rmpath) \|\| break	×
298	end	×
299	rename(path, rmpath)	×
300	path = rmpath	×
301	catch ex
302	isa(ex, IOError) \|\| rethrow(ex)	×
303	end
304	end
305	return try	71✔
306	rm(path)	71✔
307	true	71✔
308	catch ex
309	isa(ex, IOError) \|\| rethrow(ex)	×
310	ex	×
311	end
312	end
313
314	"""
315	close(lock::LockMonitor)
316
317	Release a pidfile lock.
318	"""
319	function Base.close(lock::LockMonitor)	126✔
320	update = lock.update	126✔
321	update === nothing \|\| close(update)	252✔
322	isopen(lock.fd) \|\| return false	181✔
323	removed = false	×
324	path = lock.path	71✔
325	pathstat = try	71✔
326	# Windows sometimes likes to return EACCES here,
327	# if the path is in the process of being deleted
328	stat(path)	71✔
329	catch ex
330	ex isa IOError \|\| rethrow()	×
331	removed = ex	×
332	nothing	71✔
333	end
334	if pathstat !== nothing && samefile(stat(lock.fd), pathstat)	71✔
335	# try not to delete someone else's lock
336	removed = tryrmopenfile(path)	71✔
337	end
338	close(lock.fd)	71✔
339	havelock = removed === true	71✔
340	havelock \|\| @warn "failed to remove pidfile on close" path=path removed=removed	71✔
341	return havelock	71✔
342	end
343
344	end # module

JuliaLang / julia / #37616

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous