#37658

Committed 20 Oct 2023 08:24PM UTC coverage: 87.459% (-0.5%) from 87.929%

Build # #37658

Build Type

push

local

Committed by

web-flow

Commit Message

fix unicode indexing in parse(Complex, string) (#51758)

This fixes a string-indexing bug introduced in #24713 (Julia 0.7).
Sometimes, this would cause `parse(Complex{T}, string)` to throw a
`StringIndexError` rather than an `ArgumentError`, e.g. for
`parse(ComplexF64, "3 β+ 4im")` or `parse(ComplexF64, "3 + 4αm")`. (As
far as I can tell, it can never cause parsing to fail for valid
strings.)

The source of the error is that if `i` is the index of an ASCII
character in a string `s`, then `s[i+1]` is valid (even if the next
character is non-ASCII) but `s[i-1]` is invalid if the previous
character is non-ASCII.

Run Details

5 of 5 new or added lines in 1 file covered. (100.0%)

73572 of 84122 relevant lines covered (87.46%)

11577017.06 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.2

/stdlib/Markdown/src/parse/parse.jl

# This file is a part of Julia. License is MIT: https://julialang.org/license

mutable struct MD
    content::Vector{Any}
    meta::Dict{Symbol, Any}

    MD(content::AbstractVector, meta::Dict = Dict()) =
        new(content, meta)
end

MD(xs...) = MD(vcat(xs...))

function MD(cfg::Config, xs...)
    md = MD(xs...)
    md.meta[:config] = cfg
    return md
end

config(md::MD) = md.meta[:config]::Config

# Forward some array methods

Base.push!(md::MD, x) = push!(md.content, x)
Base.getindex(md::MD, args...) = md.content[args...]
Base.setindex!(md::MD, args...) = setindex!(md.content, args...)
Base.lastindex(md::MD) = lastindex(md.content)
Base.firstindex(md::MD) = firstindex(md.content)
Base.length(md::MD) = length(md.content)
Base.isempty(md::MD) = isempty(md.content)
Base.copy(md::MD) = MD(copy(md.content), copy(md.meta))

==(a::MD, b::MD) = (html(a) == html(b))

# Parser functions:
#   md – should be modified appropriately
#   return – basically, true if parse was successful
#     false uses the next parser in the queue, true
#     goes back to the beginning
#
# Inner parsers:
#   return – element to use or nothing

# Inner parsing

function parseinline(stream::IO, md::MD, parsers::Vector{Function})
    for parser in parsers
        inner = parser(stream, md)
        inner ≡ nothing || return inner
    end
end

function parseinline(stream::IO, md::MD, config::Config)
    content = []
    buffer = IOBuffer()
    while !eof(stream)
        char = peek(stream, Char)
        if haskey(config.inner, char) &&
                (inner = parseinline(stream, md, config.inner[char])) !== nothing
            c = String(take!(buffer))
            !isempty(c) && push!(content, c)
            buffer = IOBuffer()
            push!(content, inner)
        else
            write(buffer, read(stream, Char))
        end
    end
    c = String(take!(buffer))
    !isempty(c) && push!(content, c)
    return content
end

parseinline(s::AbstractString, md::MD, c::Config) =
    parseinline(IOBuffer(s), md, c)

parseinline(s, md::MD) = parseinline(s, md, config(md))

# Block parsing

function parse(stream::IO, block::MD, config::Config; breaking = false)
    skipblank(stream)
    eof(stream) && return false
    for parser in (breaking ? config.breaking : [config.breaking; config.regular])
        parser(stream, block) && return true
    end
    return false
end

parse(stream::IO, block::MD; breaking = false) =
  parse(stream, block, config(block), breaking = breaking)

function parse(stream::IO; flavor = julia)
    isa(flavor, Symbol) && (flavor = flavors[flavor])
    markdown = MD(flavor)
    while parse(stream, markdown, flavor) end
    return markdown
end

1	# This file is a part of Julia. License is MIT: https://julialang.org/license
2
3	mutable struct MD
4	content::Vector{Any}
5	meta::Dict{Symbol, Any}
6
7	MD(content::AbstractVector, meta::Dict = Dict()) =	30,682✔
8	new(content, meta)
9	end
10
11	MD(xs...) = MD(vcat(xs...))	9,300✔
12
13	function MD(cfg::Config, xs...)	×
14	md = MD(xs...)	9,038✔
15	md.meta[:config] = cfg	9,038✔
16	return md	9,038✔
17	end
18
19	config(md::MD) = md.meta[:config]::Config	38,181✔
20
21	# Forward some array methods
22
23	Base.push!(md::MD, x) = push!(md.content, x)	23,465✔
24	Base.getindex(md::MD, args...) = md.content[args...]	×
25	Base.setindex!(md::MD, args...) = setindex!(md.content, args...)	×
26	Base.lastindex(md::MD) = lastindex(md.content)	×
27	Base.firstindex(md::MD) = firstindex(md.content)	×
28	Base.length(md::MD) = length(md.content)	4✔
29	Base.isempty(md::MD) = isempty(md.content)	1✔
30	Base.copy(md::MD) = MD(copy(md.content), copy(md.meta))	1✔
31
32	==(a::MD, b::MD) = (html(a) == html(b))	50✔
33
34	# Parser functions:
35	# md – should be modified appropriately
36	# return – basically, true if parse was successful
37	# false uses the next parser in the queue, true
38	# goes back to the beginning
39	#
40	# Inner parsers:
41	# return – element to use or nothing
42
43	# Inner parsing
44
45	function parseinline(stream::IO, md::MD, parsers::Vector{Function})	42,829✔
46	for parser in parsers	42,829✔
47	inner = parser(stream, md)	49,097✔
48	inner ≡ nothing \|\| return inner	88,545✔
49	end	9,649✔
50	end
51
52	function parseinline(stream::IO, md::MD, config::Config)	22,489✔
53	content = []	22,489✔
54	buffer = IOBuffer()	22,489✔
55	while !eof(stream)	1,532,270✔
56	char = peek(stream, Char)	1,509,781✔
57	if haskey(config.inner, char) &&	3,019,562✔
58	(inner = parseinline(stream, md, config.inner[char])) !== nothing
59	c = String(take!(buffer))	39,448✔
60	!isempty(c) && push!(content, c)	39,448✔
61	buffer = IOBuffer()	39,448✔
62	push!(content, inner)	39,448✔
63	else
64	write(buffer, read(stream, Char))	1,470,333✔
65	end
66	end	1,509,781✔
67	c = String(take!(buffer))	22,489✔
68	!isempty(c) && push!(content, c)	22,489✔
69	return content	22,489✔
70	end
71
72	parseinline(s::AbstractString, md::MD, c::Config) =	6,604✔
73	parseinline(IOBuffer(s), md, c)
74
75	parseinline(s, md::MD) = parseinline(s, md, config(md))	22,489✔
76
77	# Block parsing
78
79	function parse(stream::IO, block::MD, config::Config; breaking = false)	98,596✔
80	skipblank(stream)	49,298✔
81	eof(stream) && return false	49,298✔
82	for parser in (breaking ? config.breaking : [config.breaking; config.regular])	40,260✔
83	parser(stream, block) && return true	309,110✔
84	end	297,338✔
85	return false	14,244✔
86	end
87
88	parse(stream::IO, block::MD; breaking = false) =	29,202✔
89	parse(stream, block, config(block), breaking = breaking)
90
91	function parse(stream::IO; flavor = julia)	18,076✔
92	isa(flavor, Symbol) && (flavor = flavors[flavor])	196✔
93	markdown = MD(flavor)	9,038✔
94	while parse(stream, markdown, flavor) end	60,356✔
95	return markdown	9,038✔
96	end

JuliaLang / julia / #37658

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous