#38002

Committed 06 Feb 2025 06:14AM UTC coverage: 20.322% (-2.4%) from 22.722%

Build # #38002

Build Type

push

local

Committed by

web-flow

Commit Message

bpart: Fully switch to partitioned semantics (#57253)

This is the final PR in the binding partitions series (modulo bugs and
tweaks), i.e. it closes #54654 and thus closes #40399, which was the
original design sketch.

This thus activates the full designed semantics for binding partitions,
in particular allowing safe replacement of const bindings. It in
particular allows struct redefinitions. This thus closes
timholy/Revise.jl#18 and also closes #38584.

The biggest semantic change here is probably that this gets rid of the
notion of "resolvedness" of a binding. Previously, a lot of the behavior
of our implementation depended on when bindings were "resolved", which
could happen at basically an arbitrary point (in the compiler, in REPL
completion, in a different thread), making a lot of the semantics around
bindings ill- or at least implementation-defined. There are several
related issues in the bugtracker, so this closes #14055 closes #44604
closes #46354 closes #30277

It is also the last step to close #24569.
It also supports bindings for undef->defined transitions and thus closes
#53958 closes #54733 - however, this is not activated yet for
performance reasons and may need some further optimization.

Since resolvedness no longer exists, we need to replace it with some
hopefully more well-defined semantics. I will describe the semantics
below, but before I do I will make two notes:

1. There are a number of cases where these semantics will behave
slightly differently than the old semantics absent some other task going
around resolving random bindings.
2. The new behavior (except for the replacement stuff) was generally
permissible under the old semantics if the bindings happened to be
resolved at the right time.

With all that said, there are essentially three "strengths" of bindings:

1. Implicit Bindings: Anything implicitly obtained from `using Mod`, "no
binding", plus slightly more exotic corner cases around conflicts

2. Weakly declared bindin... (continued)

Run Details

11 of 111 new or added lines in 7 files covered. (9.91%)

1273 existing lines in 68 files now uncovered.

9908 of 48755 relevant lines covered (20.32%)

105126.48 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/base/shell.jl

# This file is a part of Julia. License is MIT: https://julialang.org/license

## shell-like command parsing ##

const shell_special = "#{}()[]<>|&*?~;"

(@doc raw"""
    rstrip_shell(s::AbstractString)

Strip trailing whitespace from a shell command string, while respecting a trailing backslash followed by a space ("\\ ").

```jldoctest
julia> Base.rstrip_shell("echo 'Hello World' \\ ")
"echo 'Hello World' \\ "

julia> Base.rstrip_shell("echo 'Hello World'    ")
"echo 'Hello World'"
```
"""
function rstrip_shell(s::AbstractString)
    c_old = nothing
    for (i, c) in Iterators.reverse(pairs(s))
        i::Int; c::AbstractChar
        ((c == '\\') && c_old == ' ') && return SubString(s, 1, i+1)
        isspace(c) || return SubString(s, 1, i)
        c_old = c
    end
    SubString(s, 1, 0)
end)

function shell_parse(str::AbstractString, interpolate::Bool=true;
                     special::AbstractString="", filename="none")
    last_arg = firstindex(str) # N.B.: This is used by REPLCompletions
    s = SubString(str, last_arg)
    s = rstrip_shell(lstrip(s))

    isempty(s) && return interpolate ? (Expr(:tuple,:()), last_arg) : ([], last_arg)

    in_single_quotes = false
    in_double_quotes = false

    args = []
    arg = []
    i = firstindex(s)
    st = Iterators.Stateful(pairs(s))
    update_last_arg = false # true after spaces or interpolate

    function push_nonempty!(list, x)
        if !isa(x,AbstractString) || !isempty(x)
            push!(list, x)
        end
        return nothing
    end
    function consume_upto!(list, s, i, j)
        push_nonempty!(list, s[i:prevind(s, j)::Int])
        something(peek(st), lastindex(s)::Int+1 => '\0').first::Int
    end
    function append_2to1!(list, innerlist)
        if isempty(innerlist); push!(innerlist, ""); end
        push!(list, copy(innerlist))
        empty!(innerlist)
    end

    C = eltype(str)
    P = Pair{Int,C}
    for (j, c) in st
        j, c = j::Int, c::C
        if !in_single_quotes && !in_double_quotes && isspace(c)
            update_last_arg = true
            i = consume_upto!(arg, s, i, j)
            append_2to1!(args, arg)
            while !isempty(st)
                # We've made sure above that we don't end in whitespace,
                # so updating `i` here is ok
                (i, c) = peek(st)::P
                isspace(c) || break
                popfirst!(st)
            end
        elseif interpolate && !in_single_quotes && c == '$'
            i = consume_upto!(arg, s, i, j)
            isempty(st) && error("\$ right before end of command")
            stpos, c = popfirst!(st)::P
            isspace(c) && error("space not allowed right after \$")
            if startswith(SubString(s, stpos), "var\"")
                # Disallow var"#" syntax in cmd interpolations.
                # TODO: Allow only identifiers after the $ for consistency with
                # string interpolation syntax (see #3150)
                ex, j = :var, stpos+3
            else
                # use parseatom instead of parse to respect filename (#28188)
                ex, j = Meta.parseatom(s, stpos, filename=filename)
            end
            last_arg = stpos + s.offset
            update_last_arg = true
            push!(arg, ex)
            s = SubString(s, j)
            Iterators.reset!(st, pairs(s))
            i = firstindex(s)
        else
            if update_last_arg
                last_arg = i + s.offset
                update_last_arg = false
            end
            if !in_double_quotes && c == '\''
                in_single_quotes = !in_single_quotes
                i = consume_upto!(arg, s, i, j)
            elseif !in_single_quotes && c == '"'
                in_double_quotes = !in_double_quotes
                i = consume_upto!(arg, s, i, j)
            elseif !in_single_quotes && c == '\\'
                if !isempty(st) && (peek(st)::P)[2] in ('\n', '\r')
                    i = consume_upto!(arg, s, i, j) + 1
                    if popfirst!(st)[2] == '\r' && (peek(st)::P)[2] == '\n'
                        i += 1
                        popfirst!(st)
                    end
                    while !isempty(st) && (peek(st)::P)[2] in (' ', '\t')
                        i = nextind(str, i)
                        _ = popfirst!(st)
                    end
                elseif in_double_quotes
                    isempty(st) && error("unterminated double quote")
                    k, c′ = peek(st)::P
                    if c′ == '"' || c′ == '$' || c′ == '\\'
                        i = consume_upto!(arg, s, i, j)
                        _ = popfirst!(st)
                    end
                else
                    isempty(st) && error("dangling backslash")
                    i = consume_upto!(arg, s, i, j)
                    _ = popfirst!(st)
                end
            elseif !in_single_quotes && !in_double_quotes && c in special
                error("parsing command `$str`: special characters \"$special\" must be quoted in commands")
            end
        end
    end

    if in_single_quotes; error("unterminated single quote"); end
    if in_double_quotes; error("unterminated double quote"); end

    push_nonempty!(arg, s[i:end])
    append_2to1!(args, arg)

    interpolate || return args, last_arg

    # construct an expression
    ex = Expr(:tuple)
    for arg in args
        push!(ex.args, Expr(:tuple, arg...))
    end
    return ex, last_arg
end

"""
    shell_split(command::AbstractString)

Split a shell command string into its individual components.

# Examples
```jldoctest
julia> Base.shell_split("git commit -m 'Initial commit'")
4-element Vector{String}:
 "git"
 "commit"
 "-m"
 "Initial commit"
```
"""
function shell_split(s::AbstractString)
    parsed = shell_parse(s, false)[1]
    args = String[]
    for arg in parsed
        push!(args, string(arg...))
    end
    args
end

function print_shell_word(io::IO, word::AbstractString, special::AbstractString = "")
    has_single = false
    has_special = false
    for c in word
        if isspace(c) || c=='\\' || c=='\'' || c=='"' || c=='$' || c in special
            has_special = true
            if c == '\''
                has_single = true
            end
        end
    end
    if isempty(word)
        print(io, "''")
    elseif !has_special
        print(io, word)
    elseif !has_single
        print(io, '\'', word, '\'')
    else
        print(io, '"')
        for c in word
            if c == '"' || c == '$'
                print(io, '\\')
            end
            print(io, c)
        end
        print(io, '"')
    end
    nothing
end

function print_shell_escaped(io::IO, cmd::AbstractString, args::AbstractString...;
                             special::AbstractString="")
    print_shell_word(io, cmd, special)
    for arg in args
        print(io, ' ')
        print_shell_word(io, arg, special)
    end
end
print_shell_escaped(io::IO; special::String="") = nothing

"""
    shell_escape(args::Union{Cmd,AbstractString...}; special::AbstractString="")

The unexported `shell_escape` function is the inverse of the unexported [`Base.shell_split()`](@ref) function:
it takes a string or command object and escapes any special characters in such a way that calling
[`Base.shell_split()`](@ref) on it would give back the array of words in the original command. The `special`
keyword argument controls what characters in addition to whitespace, backslashes, quotes and
dollar signs are considered to be special (default: none).

# Examples
```jldoctest
julia> Base.shell_escape("cat", "/foo/bar baz", "&&", "echo", "done")
"cat '/foo/bar baz' && echo done"

julia> Base.shell_escape("echo", "this", "&&", "that")
"echo this && that"
```
"""
shell_escape(args::AbstractString...; special::AbstractString="") =
    sprint((io, args...) -> print_shell_escaped(io, args..., special=special), args...)


function print_shell_escaped_posixly(io::IO, args::AbstractString...)
    first = true
    for arg in args
        first || print(io, ' ')
        # avoid printing quotes around simple enough strings
        # that any (reasonable) shell will definitely never consider them to be special
        have_single::Bool = false
        have_double::Bool = false
        function isword(c::AbstractChar)
            if '0' <= c <= '9' || 'a' <= c <= 'z' || 'A' <= c <= 'Z'
                # word characters
            elseif c == '_' || c == '/' || c == '+' || c == '-' || c == '.'
                # other common characters
            elseif c == '\''
                have_single = true
            elseif c == '"'
                have_double && return false # switch to single quoting
                have_double = true
            elseif !first && c == '='
                # equals is special if it is first (e.g. `env=val ./cmd`)
            else
                # anything else
                return false
            end
            return true
        end
        if isempty(arg)
            print(io, "''")
        elseif all(isword, arg)
            have_single && (arg = replace(arg, '\'' => "\\'"))
            have_double && (arg = replace(arg, '"' => "\\\""))
            print(io, arg)
        else
            print(io, '\'', replace(arg, '\'' => "'\\''"), '\'')
        end
        first = false
    end
end

"""
    shell_escape_posixly(args::Union{Cmd,AbstractString...})

The unexported `shell_escape_posixly` function
takes a string or command object and escapes any special characters in such a way that
it is safe to pass it as an argument to a posix shell.

See also: [`Base.shell_escape()`](@ref)

# Examples
```jldoctest
julia> Base.shell_escape_posixly("cat", "/foo/bar baz", "&&", "echo", "done")
"cat '/foo/bar baz' '&&' echo done"

julia> Base.shell_escape_posixly("echo", "this", "&&", "that")
"echo this '&&' that"
```
"""
shell_escape_posixly(args::AbstractString...) =
    sprint(print_shell_escaped_posixly, args...)

"""
    shell_escape_csh(args::Union{Cmd,AbstractString...})
    shell_escape_csh(io::IO, args::Union{Cmd,AbstractString...})

This function quotes any metacharacters in the string arguments such
that the string returned can be inserted into a command-line for
interpretation by the Unix C shell (csh, tcsh), where each string
argument will form one word.

In contrast to a POSIX shell, csh does not support the use of the
backslash as a general escape character in double-quoted strings.
Therefore, this function wraps strings that might contain
metacharacters in single quotes, except for parts that contain single
quotes, which it wraps in double quotes instead. It switches between
these types of quotes as needed. Linefeed characters are escaped with
a backslash.

This function should also work for a POSIX shell, except if the input
string contains a linefeed (`"\\n"`) character.

See also: [`Base.shell_escape_posixly()`](@ref)
"""
function shell_escape_csh(io::IO, args::AbstractString...)
    first = true
    for arg in args
        first || write(io, ' ')
        first = false
        i = 1
        while true
            for (r,e) = (r"^[A-Za-z0-9/\._-]+\z"sa => "",
                         r"^[^']*\z"sa => "'", r"^[^\$\`\"]*\z"sa => "\"",
                         r"^[^']+"sa  => "'", r"^[^\$\`\"]+"sa  => "\"")
                if ((m = match(r, SubString(arg, i))) !== nothing)
                    write(io, e)
                    write(io, replace(m.match, '\n' => "\\\n"))
                    write(io, e)
                    i += ncodeunits(m.match)
                    break
                end
            end
            i <= lastindex(arg) || break
        end
    end
end
shell_escape_csh(args::AbstractString...) =
    sprint(shell_escape_csh, args...;
           sizehint = sum(sizeof.(args)) + length(args) * 3)

"""
    shell_escape_wincmd(s::AbstractString)
    shell_escape_wincmd(io::IO, s::AbstractString)

The unexported `shell_escape_wincmd` function escapes Windows `cmd.exe` shell
meta characters. It escapes `()!^<>&|` by placing a `^` in front. An `@` is
only escaped at the start of the string. Pairs of `"` characters and the
strings they enclose are passed through unescaped. Any remaining `"` is escaped
with `^` to ensure that the number of unescaped `"` characters in the result
remains even.

Since `cmd.exe` substitutes variable references (like `%USER%`) _before_
processing the escape characters `^` and `"`, this function makes no attempt to
escape the percent sign (`%`), the presence of `%` in the input may cause
severe breakage, depending on where the result is used.

Input strings with ASCII control characters that cannot be escaped (NUL, CR,
LF) will cause an `ArgumentError` exception.

The result is safe to pass as an argument to a command call being processed by
`CMD.exe /S /C " ... "` (with surrounding double-quote pair) and will be
received verbatim by the target application if the input does not contain `%`
(else this function will fail with an ArgumentError). The presence of `%` in
the input string may result in command injection vulnerabilities and may
invalidate any claim of suitability of the output of this function for use as
an argument to cmd (due to the ordering described above), so use caution when
assembling a string from various sources.

This function may be useful in concert with the `windows_verbatim` flag to
[`Cmd`](@ref) when constructing process pipelines.

```julia
wincmd(c::String) =
   run(Cmd(Cmd(["cmd.exe", "/s /c \\" \$c \\""]);
           windows_verbatim=true))
wincmd_echo(s::String) =
   wincmd("echo " * Base.shell_escape_wincmd(s))
wincmd_echo("hello \$(ENV["USERNAME"]) & the \\"whole\\" world! (=^I^=)")
```

But take note that if the input string `s` contains a `%`, the argument list
and echo'ed text may get corrupted, resulting in arbitrary command execution.
The argument can alternatively be passed as an environment variable, which
avoids the problem with `%` and the need for the `windows_verbatim` flag:

```julia
cmdargs = Base.shell_escape_wincmd("Passing args with %cmdargs% works 100%!")
run(setenv(`cmd /C echo %cmdargs%`, "cmdargs" => cmdargs))
```

!!! warning
    The argument parsing done by CMD when calling batch files (either inside
    `.bat` files or as arguments to them) is not fully compatible with the
    output of this function. In particular, the processing of `%` is different.

!!! important
    Due to a peculiar behavior of the CMD parser/interpreter, each command
    after a literal `|` character (indicating a command pipeline) must have
    `shell_escape_wincmd` applied twice since it will be parsed twice by CMD.
    This implies ENV variables would also be expanded twice!
    For example:
    ```julia
    to_print = "All for 1 & 1 for all!"
    to_print_esc = Base.shell_escape_wincmd(Base.shell_escape_wincmd(to_print))
    run(Cmd(Cmd(["cmd", "/S /C \\" break | echo \$(to_print_esc) \\""]), windows_verbatim=true))
    ```

With an I/O stream parameter `io`, the result will be written there,
rather than returned as a string.

See also [`Base.escape_microsoft_c_args()`](@ref), [`Base.shell_escape_posixly()`](@ref).

# Examples
```jldoctest
julia> Base.shell_escape_wincmd("a^\\"^o\\"^u\\"")
"a^^\\"^o\\"^^u^\\""
```
"""
function shell_escape_wincmd(io::IO, s::AbstractString)
    # https://stackoverflow.com/a/4095133/1990689
    occursin(r"[\r\n\0]"sa, s) &&
        throw(ArgumentError("control character unsupported by CMD.EXE"))
    i = 1
    len = ncodeunits(s)
    if len > 0 && s[1] == '@'
        write(io, '^')
    end
    while i <= len
        c = s[i]
        if c == '"' && (j = findnext('"', s, nextind(s,i))) !== nothing
            write(io, SubString(s,i,j))
            i = j
        else
            if c in ('"', '(', ')', '!', '^', '<', '>', '&', '|')
                write(io, '^', c)
            else
                write(io, c)
            end
        end
        i = nextind(s,i)
    end
end
shell_escape_wincmd(s::AbstractString) = sprint(shell_escape_wincmd, s;
                                                sizehint = 2*sizeof(s))

"""
    escape_microsoft_c_args(args::Union{Cmd,AbstractString...})
    escape_microsoft_c_args(io::IO, args::Union{Cmd,AbstractString...})

Convert a collection of string arguments into a string that can be
passed to many Windows command-line applications.

Microsoft Windows passes the entire command line as a single string to
the application (unlike POSIX systems, where the shell splits the
command line into a list of arguments). Many Windows API applications
(including julia.exe), use the conventions of the [Microsoft C/C++
runtime](https://docs.microsoft.com/en-us/cpp/c-language/parsing-c-command-line-arguments)
to split that command line into a list of strings.

This function implements an inverse for a parser compatible with these rules.
It joins command-line arguments to be passed to a Windows
C/C++/Julia application into a command line, escaping or quoting the
meta characters space, TAB, double quote and backslash where needed.

See also [`Base.shell_escape_wincmd()`](@ref), [`Base.escape_raw_string()`](@ref).
"""
function escape_microsoft_c_args(io::IO, args::AbstractString...)
    # http://daviddeley.com/autohotkey/parameters/parameters.htm#WINCRULES
    first = true
    for arg in args
        if first
            first = false
        else
            write(io, ' ')  # separator
        end
        if isempty(arg) || occursin(r"[ \t\"]"sa, arg)
            # Julia raw strings happen to use the same escaping convention
            # as the argv[] parser in Microsoft's C runtime library.
            write(io, '"')
            escape_raw_string(io, arg)
            write(io, '"')
        else
            write(io, arg)
        end
    end
end
escape_microsoft_c_args(args::AbstractString...) =
    sprint(escape_microsoft_c_args, args...;
           sizehint = (sum(sizeof.(args)) + 3*length(args)))

1	# This file is a part of Julia. License is MIT: https://julialang.org/license
2
3	## shell-like command parsing ##
4
5	const shell_special = "#{}()[]<>\|&*?~;"
6
7	(@doc raw"""
8	rstrip_shell(s::AbstractString)
9
10	Strip trailing whitespace from a shell command string, while respecting a trailing backslash followed by a space ("\\ ").
11
12	```jldoctest
13	julia> Base.rstrip_shell("echo 'Hello World' \\ ")
14	"echo 'Hello World' \\ "
15
16	julia> Base.rstrip_shell("echo 'Hello World' ")
17	"echo 'Hello World'"
18	```
19	"""
20	function rstrip_shell(s::AbstractString)	×
21	c_old = nothing	×
22	for (i, c) in Iterators.reverse(pairs(s))	×
23	i::Int; c::AbstractChar	×
24	((c == '\\') && c_old == ' ') && return SubString(s, 1, i+1)	×
25	isspace(c) \|\| return SubString(s, 1, i)	×
26	c_old = c	×
27	end	×
28	SubString(s, 1, 0)	×
29	end)
30
31	function shell_parse(str::AbstractString, interpolate::Bool=true;	×
32	special::AbstractString="", filename="none")
33	last_arg = firstindex(str) # N.B.: This is used by REPLCompletions
34	s = SubString(str, last_arg)
35	s = rstrip_shell(lstrip(s))
36
37	isempty(s) && return interpolate ? (Expr(:tuple,:()), last_arg) : ([], last_arg)
38
39	in_single_quotes = false
40	in_double_quotes = false
41
42	args = []
43	arg = []
44	i = firstindex(s)
45	st = Iterators.Stateful(pairs(s))
46	update_last_arg = false # true after spaces or interpolate
47
48	function push_nonempty!(list, x)
49	if !isa(x,AbstractString) \|\| !isempty(x)
50	push!(list, x)
51	end
52	return nothing
53	end
54	function consume_upto!(list, s, i, j)
55	push_nonempty!(list, s[i:prevind(s, j)::Int])
56	something(peek(st), lastindex(s)::Int+1 => '\0').first::Int
57	end
58	function append_2to1!(list, innerlist)
59	if isempty(innerlist); push!(innerlist, ""); end
60	push!(list, copy(innerlist))
61	empty!(innerlist)
62	end
63
64	C = eltype(str)
65	P = Pair{Int,C}
66	for (j, c) in st
67	j, c = j::Int, c::C
68	if !in_single_quotes && !in_double_quotes && isspace(c)
69	update_last_arg = true
70	i = consume_upto!(arg, s, i, j)
71	append_2to1!(args, arg)
72	while !isempty(st)
73	# We've made sure above that we don't end in whitespace,
74	# so updating `i` here is ok
75	(i, c) = peek(st)::P
76	isspace(c) \|\| break
77	popfirst!(st)
78	end
79	elseif interpolate && !in_single_quotes && c == '$'
80	i = consume_upto!(arg, s, i, j)
81	isempty(st) && error("\$ right before end of command")
82	stpos, c = popfirst!(st)::P
83	isspace(c) && error("space not allowed right after \$")
84	if startswith(SubString(s, stpos), "var\"")
85	# Disallow var"#" syntax in cmd interpolations.
86	# TODO: Allow only identifiers after the $ for consistency with
87	# string interpolation syntax (see #3150)
88	ex, j = :var, stpos+3
89	else
90	# use parseatom instead of parse to respect filename (#28188)
91	ex, j = Meta.parseatom(s, stpos, filename=filename)
92	end
93	last_arg = stpos + s.offset
94	update_last_arg = true
95	push!(arg, ex)
96	s = SubString(s, j)
97	Iterators.reset!(st, pairs(s))
98	i = firstindex(s)
99	else
100	if update_last_arg
101	last_arg = i + s.offset
102	update_last_arg = false
103	end
104	if !in_double_quotes && c == '\''
105	in_single_quotes = !in_single_quotes
106	i = consume_upto!(arg, s, i, j)
107	elseif !in_single_quotes && c == '"'
108	in_double_quotes = !in_double_quotes
109	i = consume_upto!(arg, s, i, j)
110	elseif !in_single_quotes && c == '\\'
111	if !isempty(st) && (peek(st)::P)[2] in ('\n', '\r')
112	i = consume_upto!(arg, s, i, j) + 1
113	if popfirst!(st)[2] == '\r' && (peek(st)::P)[2] == '\n'
114	i += 1
115	popfirst!(st)
116	end
117	while !isempty(st) && (peek(st)::P)[2] in (' ', '\t')
118	i = nextind(str, i)
119	_ = popfirst!(st)
120	end
121	elseif in_double_quotes
122	isempty(st) && error("unterminated double quote")
123	k, c′ = peek(st)::P
124	if c′ == '"' \|\| c′ == '$' \|\| c′ == '\\'
125	i = consume_upto!(arg, s, i, j)
126	_ = popfirst!(st)
127	end
128	else
129	isempty(st) && error("dangling backslash")
130	i = consume_upto!(arg, s, i, j)
131	_ = popfirst!(st)
132	end
133	elseif !in_single_quotes && !in_double_quotes && c in special
134	error("parsing command `$str`: special characters \"$special\" must be quoted in commands")
135	end
136	end
137	end
138
139	if in_single_quotes; error("unterminated single quote"); end
140	if in_double_quotes; error("unterminated double quote"); end
141
142	push_nonempty!(arg, s[i:end])
143	append_2to1!(args, arg)
144
145	interpolate \|\| return args, last_arg
146
147	# construct an expression
148	ex = Expr(:tuple)
149	for arg in args
150	push!(ex.args, Expr(:tuple, arg...))
151	end
152	return ex, last_arg
153	end
154
155	"""
156	shell_split(command::AbstractString)
157
158	Split a shell command string into its individual components.
159
160	# Examples
161	```jldoctest
162	julia> Base.shell_split("git commit -m 'Initial commit'")
163	4-element Vector{String}:
164	"git"
165	"commit"
166	"-m"
167	"Initial commit"
168	```
169	"""
170	function shell_split(s::AbstractString)	×
171	parsed = shell_parse(s, false)[1]	×
172	args = String[]	×
173	for arg in parsed	×
174	push!(args, string(arg...))	×
175	end	×
176	args	×
177	end
178
UNCOV 179	function print_shell_word(io::IO, word::AbstractString, special::AbstractString = "")	×
UNCOV 180	has_single = false	×
UNCOV 181	has_special = false	×
UNCOV 182	for c in word	×
UNCOV 183	if isspace(c) \|\| c=='\\' \|\| c=='\'' \|\| c=='"' \|\| c=='$' \|\| c in special	×
184	has_special = true	×
185	if c == '\''	×
186	has_single = true	×
187	end
188	end
UNCOV 189	end	×
UNCOV 190	if isempty(word)	×
191	print(io, "''")	×
UNCOV 192	elseif !has_special	×
UNCOV 193	print(io, word)	×
194	elseif !has_single	×
195	print(io, '\'', word, '\'')	×
196	else
197	print(io, '"')	×
198	for c in word	×
199	if c == '"' \|\| c == '$'	×
200	print(io, '\\')	×
201	end
202	print(io, c)	×
203	end	×
204	print(io, '"')	×
205	end
UNCOV 206	nothing	×
207	end
208
209	function print_shell_escaped(io::IO, cmd::AbstractString, args::AbstractString...;	×
210	special::AbstractString="")
211	print_shell_word(io, cmd, special)	×
212	for arg in args	×
213	print(io, ' ')	×
214	print_shell_word(io, arg, special)	×
215	end	×
216	end
217	print_shell_escaped(io::IO; special::String="") = nothing	×
218
219	"""
220	shell_escape(args::Union{Cmd,AbstractString...}; special::AbstractString="")
221
222	The unexported `shell_escape` function is the inverse of the unexported [`Base.shell_split()`](@ref) function:
223	it takes a string or command object and escapes any special characters in such a way that calling
224	[`Base.shell_split()`](@ref) on it would give back the array of words in the original command. The `special`
225	keyword argument controls what characters in addition to whitespace, backslashes, quotes and
226	dollar signs are considered to be special (default: none).
227
228	# Examples
229	```jldoctest
230	julia> Base.shell_escape("cat", "/foo/bar baz", "&&", "echo", "done")
231	"cat '/foo/bar baz' && echo done"
232
233	julia> Base.shell_escape("echo", "this", "&&", "that")
234	"echo this && that"
235	```
236	"""
237	shell_escape(args::AbstractString...; special::AbstractString="") =	×
238	sprint((io, args...) -> print_shell_escaped(io, args..., special=special), args...)	×
239
240
241	function print_shell_escaped_posixly(io::IO, args::AbstractString...)	×
242	first = true	×
243	for arg in args	×
244	first \|\| print(io, ' ')	×
245	# avoid printing quotes around simple enough strings
246	# that any (reasonable) shell will definitely never consider them to be special
247	have_single::Bool = false	×
248	have_double::Bool = false	×
249	function isword(c::AbstractChar)	×
250	if '0' <= c <= '9' \|\| 'a' <= c <= 'z' \|\| 'A' <= c <= 'Z'	×
251	# word characters
252	elseif c == '_' \|\| c == '/' \|\| c == '+' \|\| c == '-' \|\| c == '.'	×
253	# other common characters
254	elseif c == '\''	×
255	have_single = true	×
256	elseif c == '"'	×
257	have_double && return false # switch to single quoting	×
258	have_double = true	×
259	elseif !first && c == '='	×
260	# equals is special if it is first (e.g. `env=val ./cmd`)
261	else
262	# anything else
263	return false	×
264	end
265	return true	×
266	end
267	if isempty(arg)	×
268	print(io, "''")	×
269	elseif all(isword, arg)	×
270	have_single && (arg = replace(arg, '\'' => "\\'"))	×
271	have_double && (arg = replace(arg, '"' => "\\\""))	×
272	print(io, arg)	×
273	else
274	print(io, '\'', replace(arg, '\'' => "'\\''"), '\'')	×
275	end
276	first = false	×
277	end	×
278	end
279
280	"""
281	shell_escape_posixly(args::Union{Cmd,AbstractString...})
282
283	The unexported `shell_escape_posixly` function
284	takes a string or command object and escapes any special characters in such a way that
285	it is safe to pass it as an argument to a posix shell.
286
287	See also: [`Base.shell_escape()`](@ref)
288
289	# Examples
290	```jldoctest
291	julia> Base.shell_escape_posixly("cat", "/foo/bar baz", "&&", "echo", "done")
292	"cat '/foo/bar baz' '&&' echo done"
293
294	julia> Base.shell_escape_posixly("echo", "this", "&&", "that")
295	"echo this '&&' that"
296	```
297	"""
298	shell_escape_posixly(args::AbstractString...) =	×
299	sprint(print_shell_escaped_posixly, args...)
300
301	"""
302	shell_escape_csh(args::Union{Cmd,AbstractString...})
303	shell_escape_csh(io::IO, args::Union{Cmd,AbstractString...})
304
305	This function quotes any metacharacters in the string arguments such
306	that the string returned can be inserted into a command-line for
307	interpretation by the Unix C shell (csh, tcsh), where each string
308	argument will form one word.
309
310	In contrast to a POSIX shell, csh does not support the use of the
311	backslash as a general escape character in double-quoted strings.
312	Therefore, this function wraps strings that might contain
313	metacharacters in single quotes, except for parts that contain single
314	quotes, which it wraps in double quotes instead. It switches between
315	these types of quotes as needed. Linefeed characters are escaped with
316	a backslash.
317
318	This function should also work for a POSIX shell, except if the input
319	string contains a linefeed (`"\\n"`) character.
320
321	See also: [`Base.shell_escape_posixly()`](@ref)
322	"""
323	function shell_escape_csh(io::IO, args::AbstractString...)	×
324	first = true	×
325	for arg in args	×
326	first \|\| write(io, ' ')	×
327	first = false	×
328	i = 1	×
329	while true	×
330	for (r,e) = (r"^[A-Za-z0-9/\._-]+\z"sa => "",	×
331	r"^[^']\z"sa => "'", r"^[^\$\`\"]\z"sa => "\"",	×
332	r"^[^']+"sa => "'", r"^[^\$\`\"]+"sa => "\"")	×
333	if ((m = match(r, SubString(arg, i))) !== nothing)	×
334	write(io, e)	×
335	write(io, replace(m.match, '\n' => "\\\n"))	×
336	write(io, e)	×
337	i += ncodeunits(m.match)	×
338	break	×
339	end
340	end	×
341	i <= lastindex(arg) \|\| break	×
342	end	×
343	end	×
344	end
345	shell_escape_csh(args::AbstractString...) =	×
346	sprint(shell_escape_csh, args...;
347	sizehint = sum(sizeof.(args)) + length(args) * 3)
348
349	"""
350	shell_escape_wincmd(s::AbstractString)
351	shell_escape_wincmd(io::IO, s::AbstractString)
352
353	The unexported `shell_escape_wincmd` function escapes Windows `cmd.exe` shell
354	meta characters. It escapes `()!^<>&\|` by placing a `^` in front. An `@` is
355	only escaped at the start of the string. Pairs of `"` characters and the
356	strings they enclose are passed through unescaped. Any remaining `"` is escaped
357	with `^` to ensure that the number of unescaped `"` characters in the result
358	remains even.
359
360	Since `cmd.exe` substitutes variable references (like `%USER%`) _before_
361	processing the escape characters `^` and `"`, this function makes no attempt to
362	escape the percent sign (`%`), the presence of `%` in the input may cause
363	severe breakage, depending on where the result is used.
364
365	Input strings with ASCII control characters that cannot be escaped (NUL, CR,
366	LF) will cause an `ArgumentError` exception.
367
368	The result is safe to pass as an argument to a command call being processed by
369	`CMD.exe /S /C " ... "` (with surrounding double-quote pair) and will be
370	received verbatim by the target application if the input does not contain `%`
371	(else this function will fail with an ArgumentError). The presence of `%` in
372	the input string may result in command injection vulnerabilities and may
373	invalidate any claim of suitability of the output of this function for use as
374	an argument to cmd (due to the ordering described above), so use caution when
375	assembling a string from various sources.
376
377	This function may be useful in concert with the `windows_verbatim` flag to
378	[`Cmd`](@ref) when constructing process pipelines.
379
380	```julia
381	wincmd(c::String) =
382	run(Cmd(Cmd(["cmd.exe", "/s /c \\" \$c \\""]);
383	windows_verbatim=true))
384	wincmd_echo(s::String) =
385	wincmd("echo " * Base.shell_escape_wincmd(s))
386	wincmd_echo("hello \$(ENV["USERNAME"]) & the \\"whole\\" world! (=^I^=)")
387	```
388
389	But take note that if the input string `s` contains a `%`, the argument list
390	and echo'ed text may get corrupted, resulting in arbitrary command execution.
391	The argument can alternatively be passed as an environment variable, which
392	avoids the problem with `%` and the need for the `windows_verbatim` flag:
393
394	```julia
395	cmdargs = Base.shell_escape_wincmd("Passing args with %cmdargs% works 100%!")
396	run(setenv(`cmd /C echo %cmdargs%`, "cmdargs" => cmdargs))
397	```
398
399	!!! warning
400	The argument parsing done by CMD when calling batch files (either inside
401	`.bat` files or as arguments to them) is not fully compatible with the
402	output of this function. In particular, the processing of `%` is different.
403
404	!!! important
405	Due to a peculiar behavior of the CMD parser/interpreter, each command
406	after a literal `\|` character (indicating a command pipeline) must have
407	`shell_escape_wincmd` applied twice since it will be parsed twice by CMD.
408	This implies ENV variables would also be expanded twice!
409	For example:
410	```julia
411	to_print = "All for 1 & 1 for all!"
412	to_print_esc = Base.shell_escape_wincmd(Base.shell_escape_wincmd(to_print))
413	run(Cmd(Cmd(["cmd", "/S /C \\" break \| echo \$(to_print_esc) \\""]), windows_verbatim=true))
414	```
415
416	With an I/O stream parameter `io`, the result will be written there,
417	rather than returned as a string.
418
419	See also [`Base.escape_microsoft_c_args()`](@ref), [`Base.shell_escape_posixly()`](@ref).
420
421	# Examples
422	```jldoctest
423	julia> Base.shell_escape_wincmd("a^\\"^o\\"^u\\"")
424	"a^^\\"^o\\"^^u^\\""
425	```
426	"""
427	function shell_escape_wincmd(io::IO, s::AbstractString)	×
428	# https://stackoverflow.com/a/4095133/1990689
429	occursin(r"[\r\n\0]"sa, s) &&	×
430	throw(ArgumentError("control character unsupported by CMD.EXE"))
431	i = 1	×
432	len = ncodeunits(s)	×
433	if len > 0 && s[1] == '@'	×
434	write(io, '^')	×
435	end
436	while i <= len	×
437	c = s[i]	×
438	if c == '"' && (j = findnext('"', s, nextind(s,i))) !== nothing	×
439	write(io, SubString(s,i,j))	×
440	i = j	×
441	else
442	if c in ('"', '(', ')', '!', '^', '<', '>', '&', '\|')	×
443	write(io, '^', c)	×
444	else
445	write(io, c)	×
446	end
447	end
448	i = nextind(s,i)	×
449	end	×
450	end
451	shell_escape_wincmd(s::AbstractString) = sprint(shell_escape_wincmd, s;	×
452	sizehint = 2*sizeof(s))
453
454	"""
455	escape_microsoft_c_args(args::Union{Cmd,AbstractString...})
456	escape_microsoft_c_args(io::IO, args::Union{Cmd,AbstractString...})
457
458	Convert a collection of string arguments into a string that can be
459	passed to many Windows command-line applications.
460
461	Microsoft Windows passes the entire command line as a single string to
462	the application (unlike POSIX systems, where the shell splits the
463	command line into a list of arguments). Many Windows API applications
464	(including julia.exe), use the conventions of the [Microsoft C/C++
465	runtime](https://docs.microsoft.com/en-us/cpp/c-language/parsing-c-command-line-arguments)
466	to split that command line into a list of strings.
467
468	This function implements an inverse for a parser compatible with these rules.
469	It joins command-line arguments to be passed to a Windows
470	C/C++/Julia application into a command line, escaping or quoting the
471	meta characters space, TAB, double quote and backslash where needed.
472
473	See also [`Base.shell_escape_wincmd()`](@ref), [`Base.escape_raw_string()`](@ref).
474	"""
475	function escape_microsoft_c_args(io::IO, args::AbstractString...)	×
476	# http://daviddeley.com/autohotkey/parameters/parameters.htm#WINCRULES
477	first = true	×
478	for arg in args	×
479	if first	×
480	first = false	×
481	else
482	write(io, ' ') # separator	×
483	end
484	if isempty(arg) \|\| occursin(r"[ \t\"]"sa, arg)	×
485	# Julia raw strings happen to use the same escaping convention
486	# as the argv[] parser in Microsoft's C runtime library.
487	write(io, '"')	×
488	escape_raw_string(io, arg)	×
489	write(io, '"')	×
490	else
491	write(io, arg)	×
492	end
493	end	×
494	end
495	escape_microsoft_c_args(args::AbstractString...) =	×
496	sprint(escape_microsoft_c_args, args...;
497	sizehint = (sum(sizeof.(args)) + 3*length(args)))

JuliaLang / julia / #38002

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous