#37728

Committed 26 Mar 2024 03:46AM UTC coverage: 80.612% (-0.8%) from 81.423%

Build # #37728

Build Type

push

local

Committed by

web-flow

Commit Message

Update zlib to 1.3.1 (#53841)

Released January 22, 2024

Run Details

69920 of 86737 relevant lines covered (80.61%)

14456248.65 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

34.06

/stdlib/FileWatching/src/pidfile.jl

module Pidfile


export mkpidlock, trymkpidlock

using Base:
    IOError, UV_EEXIST, UV_ESRCH,
    Process

using Base.Filesystem:
    File, open, JL_O_CREAT, JL_O_RDWR, JL_O_RDONLY, JL_O_EXCL,
    rename, samefile, path_separator

using ..FileWatching: watch_file
using Base.Sys: iswindows

"""
    mkpidlock([f::Function], at::String, [pid::Cint]; kwopts...)
    mkpidlock(at::String, proc::Process; kwopts...)

Create a pidfile lock for the path "at" for the current process
or the process identified by pid or proc. Can take a function to execute once locked,
for usage in `do` blocks, after which the lock will be automatically closed. If the lock fails
and `wait` is false, then an error is thrown.

The lock will be released by either `close`, a `finalizer`, or shortly after `proc` exits.
Make sure the return value is live through the end of the critical section of
your program, so the `finalizer` does not reclaim it early.

Optional keyword arguments:
 - `mode`: file access mode (modified by the process umask). Defaults to world-readable.
 - `poll_interval`: Specify the maximum time to between attempts (if `watch_file` doesn't work)
 - `stale_age`: Delete an existing pidfile (ignoring the lock) if it is older than this many seconds, based on its mtime.
     The file won't be deleted until 5x longer than this if the pid in the file appears that it may be valid.
     Or 25x longer if `refresh` is overridden to 0 to disable lock refreshing.
     By default this is disabled (`stale_age` = 0), but a typical recommended value would be about 3-5x an
     estimated normal completion time.
 - `refresh`: Keeps a lock from becoming stale by updating the mtime every interval of time that passes.
     By default, this is set to `stale_age/2`, which is the recommended value.
 - `wait`: If true, block until we get the lock, if false, raise error if lock fails.
"""
function mkpidlock end

"""
    trymkpidlock([f::Function], at::String, [pid::Cint]; kwopts...)
    trymkpidlock(at::String, proc::Process; kwopts...)

Like `mkpidlock` except returns `false` instead of waiting if the file is already locked.

!!! compat "Julia 1.10"
    This function requires at least Julia 1.10.
"""
function trymkpidlock end

# mutable only because we want to add a finalizer
mutable struct LockMonitor
    const path::String
    const fd::File
    const update::Union{Nothing,Timer}

    global function mkpidlock(at::String, pid::Cint; stale_age::Real=0, refresh::Real=stale_age/2, kwopts...)
        local lock
        atdir, atname = splitdir(at)
        isempty(atdir) && (atdir = pwd())
        at = realpath(atdir) * path_separator * atname
        fd = open_exclusive(at; stale_age, refresh, kwopts...)
        update = nothing
        try
            write_pidfile(fd, pid)
            if refresh > 0
                # N.b.: to ensure our finalizer works we are careful to capture
                # `fd` here instead of `lock`.
                update = Timer(t -> isopen(t) && touch(fd), refresh; interval=refresh)
            end
            lock = new(at, fd, update)
            finalizer(close, lock)
        catch ex
            update === nothing || close(update)
            tryrmopenfile(at)
            close(fd)
            rethrow(ex)
        end
        return lock
    end
end

mkpidlock(at::String; kwopts...) = mkpidlock(at, getpid(); kwopts...)
mkpidlock(f::Function, at::String; kwopts...) = mkpidlock(f, at, getpid(); kwopts...)

function mkpidlock(f::Function, at::String, pid::Cint; kwopts...)
    lock = mkpidlock(at, pid; kwopts...)
    try
        return f()
    finally
        close(lock)
    end
end

function mkpidlock(at::String, proc::Process; kwopts...)
    lock = mkpidlock(at, getpid(proc); kwopts...)
    closer = @async begin
        try
            wait(proc)
        finally
            close(lock)
        end
    end
    Base.errormonitor(closer)
    return lock
end

function trymkpidlock(args...; kwargs...)
    try
        mkpidlock(args...; kwargs..., wait=false)
    catch ex
        if ex isa PidlockedError
            return false
        else
            rethrow()
        end
    end
end

"""
    Base.touch(::Pidfile.LockMonitor)

Update the `mtime` on the lock, to indicate it is still fresh.

See also the `refresh` keyword in the [`mkpidlock`](@ref) constructor.
"""
Base.touch(lock::LockMonitor) = (touch(lock.fd); lock)

"""
    write_pidfile(io, pid)

Write our pidfile format to an open IO descriptor.
"""
function write_pidfile(io::IO, pid::Cint)
    print(io, "$pid $(gethostname())")
end

"""
    parse_pidfile(file::Union{IO, String}) => (pid, hostname, age)

Attempt to parse our pidfile format,
replaced an element with (0, "", 0.0), respectively, for any read that failed.
"""
function parse_pidfile(io::IO)
    fields = split(read(io, String), ' ', limit = 2)
    pid = tryparse(Cuint, fields[1])
    pid === nothing && (pid = Cuint(0))
    hostname = (length(fields) == 2) ? fields[2] : ""
    when = mtime(io)
    age = time() - when
    return (pid, hostname, age)
end

function parse_pidfile(path::String)
    try
        existing = open(path, JL_O_RDONLY)
        try
            return parse_pidfile(existing)
        finally
            close(existing)
        end
    catch ex
        isa(ex, EOFError) || isa(ex, IOError) || rethrow(ex)
        return (Cuint(0), "", 0.0)
    end
end

"""
    isvalidpid(hostname::String, pid::Cuint) :: Bool

Attempt to conservatively estimate whether pid is a valid process id.
"""
function isvalidpid(hostname::AbstractString, pid::Cuint)
    # can't inspect remote hosts
    (hostname == "" || hostname == gethostname()) || return true
    # pid < 0 is never valid (must be a parser error or different OS),
    # and would have a completely different meaning when passed to kill
    !iswindows() && pid > typemax(Cint) && return false
    # (similarly for pid 0)
    pid == 0 && return false
    # see if the process id exists by querying kill without sending a signal
    # and checking if it returned ESRCH (no such process)
    return ccall(:uv_kill, Cint, (Cuint, Cint), pid, 0) != UV_ESRCH
end

"""
    stale_pidfile(path::String, stale_age::Real, refresh::Real) :: Bool

Helper function for `open_exclusive` for deciding if a pidfile is stale.
"""
function stale_pidfile(path::String, stale_age::Real, refresh::Real)
    pid, hostname, age = parse_pidfile(path)
    age < -stale_age && @warn "filesystem time skew detected" path=path
    longer_factor = refresh == 0 ? 25 : 5
    if age > stale_age
        if (age > stale_age * longer_factor) || !isvalidpid(hostname, pid)
            return true
        end
    end
    return false
end

"""
    tryopen_exclusive(path::String, mode::Integer = 0o444) :: Union{Void, File}

Try to create a new file for read-write advisory-exclusive access,
return nothing if it already exists.
"""
function tryopen_exclusive(path::String, mode::Integer = 0o444)
    try
        return open(path, JL_O_RDWR | JL_O_CREAT | JL_O_EXCL, mode)
    catch ex
        (isa(ex, IOError) && ex.code == UV_EEXIST) || rethrow(ex)
    end
    return nothing
end

struct PidlockedError <: Exception
    msg::AbstractString
end

"""
    open_exclusive(path::String; mode, poll_interval, wait, stale_age, refresh) :: File

Create a new a file for read-write advisory-exclusive access.
If `wait` is `false` then error out if the lock files exist
otherwise block until we get the lock.

For a description of the keyword arguments, see [`mkpidlock`](@ref).
"""
function open_exclusive(path::String;
                        mode::Integer = 0o444 #= read-only =#,
                        poll_interval::Real = 10 #= seconds =#,
                        wait::Bool = true #= return on failure if false =#,
                        stale_age::Real = 0 #= disabled =#,
                        refresh::Real = stale_age/2)
    # fast-path: just try to open it
    file = tryopen_exclusive(path, mode)
    file === nothing || return file
    if !wait
        if file === nothing && stale_age > 0
            if stale_age > 0 && stale_pidfile(path, stale_age, refresh)
                @warn "attempting to remove probably stale pidfile" path=path
                tryrmopenfile(path)
            end
            file = tryopen_exclusive(path, mode)
        end
        if file === nothing
            throw(PidlockedError("Failed to get pidfile lock for $(repr(path))."))
        else
            return file
        end
    end
    # fall-back: wait for the lock

    while true
        # start the file-watcher prior to checking for the pidfile existence
        t = @async try
            watch_file(path, poll_interval)
        catch ex
            isa(ex, IOError) || rethrow(ex)
            sleep(poll_interval) # if the watch failed, convert to just doing a sleep
        end
        # now try again to create it
        file = tryopen_exclusive(path, mode)
        file === nothing || return file
        Base.wait(t) # sleep for a bit before trying again
        if stale_age > 0 && stale_pidfile(path, stale_age, refresh)
            # if the file seems stale, try to remove it before attempting again
            # set stale_age to zero so we won't attempt again, even if the attempt fails
            stale_age -= stale_age
            @warn "attempting to remove probably stale pidfile" path=path
            tryrmopenfile(path)
        end
    end
end

function _rand_filename(len::Int=4) # modified from Base.Libc
    slug = Base.StringVector(len)
    chars = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
    for i = 1:len
        slug[i] = chars[(Libc.rand() % length(chars)) + 1]
    end
    return String(slug)
end

function tryrmopenfile(path::String)
    # Deleting open file on Windows is a bit hard
    # if we want to reuse the name immediately after:
    # we need to first rename it, then delete it.
    if Sys.iswindows()
        try
            local rmpath
            rmdir, rmname = splitdir(path)
            while true
                rmpath = string(rmdir, isempty(rmdir) ? "" : path_separator,
                    "\$", _rand_filename(), rmname, ".deleted")
                ispath(rmpath) || break
            end
            rename(path, rmpath)
            path = rmpath
        catch ex
            isa(ex, IOError) || rethrow(ex)
        end
    end
    return try
        rm(path)
        true
    catch ex
        isa(ex, IOError) || rethrow(ex)
        ex
    end
end

"""
    close(lock::LockMonitor)

Release a pidfile lock.
"""
function Base.close(lock::LockMonitor)
    update = lock.update
    update === nothing || close(update)
    isopen(lock.fd) || return false
    removed = false
    path = lock.path
    pathstat = try
            # Windows sometimes likes to return EACCES here,
            # if the path is in the process of being deleted
            stat(path)
        catch ex
            ex isa IOError || rethrow()
            removed = ex
            nothing
        end
    if pathstat !== nothing && samefile(stat(lock.fd), pathstat)
        # try not to delete someone else's lock
        removed = tryrmopenfile(path)
    end
    close(lock.fd)
    havelock = removed === true
    havelock || @warn "failed to remove pidfile on close" path=path removed=removed
    return havelock
end

end # module

1	module Pidfile
2
3
4	export mkpidlock, trymkpidlock
5
6	using Base:
7	IOError, UV_EEXIST, UV_ESRCH,
8	Process
9
10	using Base.Filesystem:
11	File, open, JL_O_CREAT, JL_O_RDWR, JL_O_RDONLY, JL_O_EXCL,
12	rename, samefile, path_separator
13
14	using ..FileWatching: watch_file
15	using Base.Sys: iswindows
16
17	"""
18	mkpidlock([f::Function], at::String, [pid::Cint]; kwopts...)
19	mkpidlock(at::String, proc::Process; kwopts...)
20
21	Create a pidfile lock for the path "at" for the current process
22	or the process identified by pid or proc. Can take a function to execute once locked,
23	for usage in `do` blocks, after which the lock will be automatically closed. If the lock fails
24	and `wait` is false, then an error is thrown.
25
26	The lock will be released by either `close`, a `finalizer`, or shortly after `proc` exits.
27	Make sure the return value is live through the end of the critical section of
28	your program, so the `finalizer` does not reclaim it early.
29
30	Optional keyword arguments:
31	- `mode`: file access mode (modified by the process umask). Defaults to world-readable.
32	- `poll_interval`: Specify the maximum time to between attempts (if `watch_file` doesn't work)
33	- `stale_age`: Delete an existing pidfile (ignoring the lock) if it is older than this many seconds, based on its mtime.
34	The file won't be deleted until 5x longer than this if the pid in the file appears that it may be valid.
35	Or 25x longer if `refresh` is overridden to 0 to disable lock refreshing.
36	By default this is disabled (`stale_age` = 0), but a typical recommended value would be about 3-5x an
37	estimated normal completion time.
38	- `refresh`: Keeps a lock from becoming stale by updating the mtime every interval of time that passes.
39	By default, this is set to `stale_age/2`, which is the recommended value.
40	- `wait`: If true, block until we get the lock, if false, raise error if lock fails.
41	"""
42	function mkpidlock end
43
44	"""
45	trymkpidlock([f::Function], at::String, [pid::Cint]; kwopts...)
46	trymkpidlock(at::String, proc::Process; kwopts...)
47
48	Like `mkpidlock` except returns `false` instead of waiting if the file is already locked.
49
50	!!! compat "Julia 1.10"
51	This function requires at least Julia 1.10.
52	"""
53	function trymkpidlock end
54
55	# mutable only because we want to add a finalizer
56	mutable struct LockMonitor
57	const path::String
58	const fd::File
59	const update::Union{Nothing,Timer}
60
61	global function mkpidlock(at::String, pid::Cint; stale_age::Real=0, refresh::Real=stale_age/2, kwopts...)	128✔
62	local lock	×
63	atdir, atname = splitdir(at)	64✔
64	isempty(atdir) && (atdir = pwd())	64✔
65	at = realpath(atdir) * path_separator * atname	128✔
66	fd = open_exclusive(at; stale_age, refresh, kwopts...)	64✔
67	update = nothing	×
68	try	64✔
69	write_pidfile(fd, pid)	64✔
70	if refresh > 0	64✔
71	# N.b.: to ensure our finalizer works we are careful to capture
72	# `fd` here instead of `lock`.
73	update = Timer(t -> isopen(t) && touch(fd), refresh; interval=refresh)	666✔
74	end
75	lock = new(at, fd, update)	64✔
76	finalizer(close, lock)	64✔
77	catch ex
78	update === nothing \|\| close(update)	×
79	tryrmopenfile(at)	×
80	close(fd)	×
81	rethrow(ex)	×
82	end
83	return lock	64✔
84	end
85	end
86
87	mkpidlock(at::String; kwopts...) = mkpidlock(at, getpid(); kwopts...)	×
88	mkpidlock(f::Function, at::String; kwopts...) = mkpidlock(f, at, getpid(); kwopts...)	128✔
89
90	function mkpidlock(f::Function, at::String, pid::Cint; kwopts...)	128✔
91	lock = mkpidlock(at, pid; kwopts...)	64✔
92	try	64✔
93	return f()	64✔
94	finally
95	close(lock)	64✔
96	end
97	end
98
99	function mkpidlock(at::String, proc::Process; kwopts...)	×
100	lock = mkpidlock(at, getpid(proc); kwopts...)	×
101	closer = @async begin	×
102	try	×
103	wait(proc)	×
104	finally
105	close(lock)	×
106	end
107	end
108	Base.errormonitor(closer)	×
109	return lock	×
110	end
111
112	function trymkpidlock(args...; kwargs...)	84✔
113	try	42✔
114	mkpidlock(args...; kwargs..., wait=false)	42✔
115	catch ex
116	if ex isa PidlockedError	×
117	return false	×
118	else
119	rethrow()	×
120	end
121	end
122	end
123
124	"""
125	Base.touch(::Pidfile.LockMonitor)
126
127	Update the `mtime` on the lock, to indicate it is still fresh.
128
129	See also the `refresh` keyword in the [`mkpidlock`](@ref) constructor.
130	"""
131	Base.touch(lock::LockMonitor) = (touch(lock.fd); lock)	×
132
133	"""
134	write_pidfile(io, pid)
135
136	Write our pidfile format to an open IO descriptor.
137	"""
138	function write_pidfile(io::IO, pid::Cint)	45✔
139	print(io, "$pid $(gethostname())")	45✔
140	end
141
142	"""
143	parse_pidfile(file::Union{IO, String}) => (pid, hostname, age)
144
145	Attempt to parse our pidfile format,
146	replaced an element with (0, "", 0.0), respectively, for any read that failed.
147	"""
148	function parse_pidfile(io::IO)
149	fields = split(read(io, String), ' ', limit = 2)	×
150	pid = tryparse(Cuint, fields[1])	×
151	pid === nothing && (pid = Cuint(0))	×
152	hostname = (length(fields) == 2) ? fields[2] : ""	×
153	when = mtime(io)	×
154	age = time() - when	×
155	return (pid, hostname, age)	×
156	end
157
158	function parse_pidfile(path::String)	×
159	try	×
160	existing = open(path, JL_O_RDONLY)	×
161	try	×
162	return parse_pidfile(existing)	×
163	finally
164	close(existing)	×
165	end
166	catch ex
167	isa(ex, EOFError) \|\| isa(ex, IOError) \|\| rethrow(ex)	×
168	return (Cuint(0), "", 0.0)	×
169	end
170	end
171
172	"""
173	isvalidpid(hostname::String, pid::Cuint) :: Bool
174
175	Attempt to conservatively estimate whether pid is a valid process id.
176	"""
177	function isvalidpid(hostname::AbstractString, pid::Cuint)	×
178	# can't inspect remote hosts
179	(hostname == "" \|\| hostname == gethostname()) \|\| return true	×
180	# pid < 0 is never valid (must be a parser error or different OS),
181	# and would have a completely different meaning when passed to kill
182	!iswindows() && pid > typemax(Cint) && return false	×
183	# (similarly for pid 0)
184	pid == 0 && return false	×
185	# see if the process id exists by querying kill without sending a signal
186	# and checking if it returned ESRCH (no such process)
187	return ccall(:uv_kill, Cint, (Cuint, Cint), pid, 0) != UV_ESRCH	×
188	end
189
190	"""
191	stale_pidfile(path::String, stale_age::Real, refresh::Real) :: Bool
192
193	Helper function for `open_exclusive` for deciding if a pidfile is stale.
194	"""
195	function stale_pidfile(path::String, stale_age::Real, refresh::Real)	×
196	pid, hostname, age = parse_pidfile(path)	×
197	age < -stale_age && @warn "filesystem time skew detected" path=path	×
198	longer_factor = refresh == 0 ? 25 : 5	×
199	if age > stale_age	×
200	if (age > stale_age * longer_factor) \|\| !isvalidpid(hostname, pid)	×
201	return true	×
202	end
203	end
204	return false	×
205	end
206
207	"""
208	tryopen_exclusive(path::String, mode::Integer = 0o444) :: Union{Void, File}
209
210	Try to create a new file for read-write advisory-exclusive access,
211	return nothing if it already exists.
212	"""
213	function tryopen_exclusive(path::String, mode::Integer = 0o444)	45✔
214	try	45✔
215	return open(path, JL_O_RDWR \| JL_O_CREAT \| JL_O_EXCL, mode)	45✔
216	catch ex
217	(isa(ex, IOError) && ex.code == UV_EEXIST) \|\| rethrow(ex)	×
218	end
219	return nothing	×
220	end
221
222	struct PidlockedError <: Exception
223	msg::AbstractString
224	end
225
226	"""
227	open_exclusive(path::String; mode, poll_interval, wait, stale_age, refresh) :: File
228
229	Create a new a file for read-write advisory-exclusive access.
230	If `wait` is `false` then error out if the lock files exist
231	otherwise block until we get the lock.
232
233	For a description of the keyword arguments, see [`mkpidlock`](@ref).
234	"""
235	function open_exclusive(path::String;	109✔
236	mode::Integer = 0o444 #= read-only =#,
237	poll_interval::Real = 10 #= seconds =#,
238	wait::Bool = true #= return on failure if false =#,
239	stale_age::Real = 0 #= disabled =#,
240	refresh::Real = stale_age/2)
241	# fast-path: just try to open it
242	file = tryopen_exclusive(path, mode)	45✔
243	file === nothing \|\| return file	90✔
244	if !wait	×
245	if file === nothing && stale_age > 0	×
246	if stale_age > 0 && stale_pidfile(path, stale_age, refresh)	×
247	@warn "attempting to remove probably stale pidfile" path=path	×
248	tryrmopenfile(path)	×
249	end
250	file = tryopen_exclusive(path, mode)	×
251	end
252	if file === nothing	×
253	throw(PidlockedError("Failed to get pidfile lock for $(repr(path))."))	×
254	else
255	return file	×
256	end
257	end
258	# fall-back: wait for the lock
259
260	while true	×
261	# start the file-watcher prior to checking for the pidfile existence
262	t = @async try	×
263	watch_file(path, poll_interval)
264	catch ex
265	isa(ex, IOError) \|\| rethrow(ex)
266	sleep(poll_interval) # if the watch failed, convert to just doing a sleep
267	end
268	# now try again to create it
269	file = tryopen_exclusive(path, mode)	×
270	file === nothing \|\| return file	×
271	Base.wait(t) # sleep for a bit before trying again	×
272	if stale_age > 0 && stale_pidfile(path, stale_age, refresh)	×
273	# if the file seems stale, try to remove it before attempting again
274	# set stale_age to zero so we won't attempt again, even if the attempt fails
275	stale_age -= stale_age	×
276	@warn "attempting to remove probably stale pidfile" path=path	×
277	tryrmopenfile(path)	×
278	end
279	end	×
280	end
281
282	function _rand_filename(len::Int=4) # modified from Base.Libc	×
283	slug = Base.StringVector(len)	×
284	chars = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"	×
285	for i = 1:len	×
286	slug[i] = chars[(Libc.rand() % length(chars)) + 1]	×
287	end	×
288	return String(slug)	×
289	end
290
291	function tryrmopenfile(path::String)	45✔
292	# Deleting open file on Windows is a bit hard
293	# if we want to reuse the name immediately after:
294	# we need to first rename it, then delete it.
295	if Sys.iswindows()	×
296	try	×
297	local rmpath	×
298	rmdir, rmname = splitdir(path)	×
299	while true	×
300	rmpath = string(rmdir, isempty(rmdir) ? "" : path_separator,	×
301	"\$", _rand_filename(), rmname, ".deleted")
302	ispath(rmpath) \|\| break	×
303	end	×
304	rename(path, rmpath)	×
305	path = rmpath	×
306	catch ex
307	isa(ex, IOError) \|\| rethrow(ex)	×
308	end
309	end
310	return try	45✔
311	rm(path)	45✔
312	true	45✔
313	catch ex
314	isa(ex, IOError) \|\| rethrow(ex)	×
315	ex	×
316	end
317	end
318
319	"""
320	close(lock::LockMonitor)
321
322	Release a pidfile lock.
323	"""
324	function Base.close(lock::LockMonitor)	67✔
325	update = lock.update	67✔
326	update === nothing \|\| close(update)	134✔
327	isopen(lock.fd) \|\| return false	89✔
328	removed = false	×
329	path = lock.path	45✔
330	pathstat = try	45✔
331	# Windows sometimes likes to return EACCES here,
332	# if the path is in the process of being deleted
333	stat(path)	45✔
334	catch ex
335	ex isa IOError \|\| rethrow()	×
336	removed = ex	×
337	nothing	45✔
338	end
339	if pathstat !== nothing && samefile(stat(lock.fd), pathstat)	45✔
340	# try not to delete someone else's lock
341	removed = tryrmopenfile(path)	45✔
342	end
343	close(lock.fd)	45✔
344	havelock = removed === true	45✔
345	havelock \|\| @warn "failed to remove pidfile on close" path=path removed=removed	45✔
346	return havelock	45✔
347	end
348
349	end # module

JuliaLang / julia / #37728

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous