mmcdole / gofeed / 399

Committed 07 Mar 2020 02:02AM UTC coverage: 60.637% (-0.2%) from 60.839%

Build # 399

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

<a href="https://github.com/mmcdole/gofeed/commit/<a class=hub.com/mmcdole/gofeed/commit/1ed4d84b36ec028b02a00b3651a7ca9918082e13">1ed4d84b3<a href="https://github.com/mmcdole/gofeed/commit/1ed4d84b36ec028b02a00b3651a7ca9918082e13">&quot;&gt;Patch godexsoft@ commit &lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/mmcdole/gofeed/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/mmcdole/gofeed/commit/<a class=hub.com/mmcdole/gofeed/commit/ed86b2f106e14101bd44e2f6e4b62d061293ffe6">ed86b2f10">ed86b2f10</a><a href="https://github.com/mmcdole/gofeed/commit/1ed4d84b36ec028b02a00b3651a7ca9918082e13"> (#137)

* Patch godexsoft@ commit ed86b2f106e14101bd44e2f6e4b62d061293ffe6 to a
new branch as they are out of town.

Original cl description:
Fix truncated entity error for ampersand literal

* Fix TestDecodeEntitiesInvalid to allow for the more permissive decoder

* Add back unknown entity test

Co-authored-by: Daniel Hobe &lt;hobe@google.com&gt;</a>

Run Details

47 of 47 new or added lines in 1 file covered. (100.0%)

1465 of 2416 relevant lines covered (60.64%)

149.01 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

31.34

/internal/shared/parseutils.go

package shared

import (
        "bytes"
        "errors"
        "fmt"
        "regexp"
        "strconv"
        "strings"

        xpp "github.com/mmcdole/goxpp"
)

var (
        emailNameRgx = regexp.MustCompile(`^([^@]+@[^\s]+)\s+\(([^@]+)\)$`)
        nameEmailRgx = regexp.MustCompile(`^([^@]+)\s+\(([^@]+@[^)]+)\)$`)
        nameOnlyRgx  = regexp.MustCompile(`^([^@()]+)$`)
        emailOnlyRgx = regexp.MustCompile(`^([^@()]+@[^@()]+)$`)

        TruncatedEntity         = errors.New("truncated entity")
        InvalidNumericReference = errors.New("invalid numeric reference")
)

const CDATA_START = "<![CDATA["
const CDATA_END = "]]>"

// ParseText is a helper function for parsing the text
// from the current element of the XMLPullParser.
// This function can handle parsing naked XML text from
// an element.
func ParseText(p *xpp.XMLPullParser) (string, error) {
        var text struct {
                Type     string `xml:"type,attr"`
                InnerXML string `xml:",innerxml"`
        }

        err := p.DecodeElement(&text)
        if err != nil {
                return "", err
        }

        result := text.InnerXML
        result = strings.TrimSpace(result)

        if strings.Contains(result, CDATA_START) {
                return StripCDATA(result), nil
        }

        return DecodeEntities(result)
}

// StripCDATA removes CDATA tags from the string
// content outside of CDATA tags is passed via DecodeEntities
func StripCDATA(str string) string {
        buf := bytes.NewBuffer([]byte{})

        curr := 0

        for curr < len(str) {

                start := indexAt(str, CDATA_START, curr)

                if start == -1 {
                        dec, _ := DecodeEntities(str[curr:])
                        buf.Write([]byte(dec))
                        return buf.String()
                }

                end := indexAt(str, CDATA_END, start)

                if end == -1 {
                        dec, _ := DecodeEntities(str[curr:])
                        buf.Write([]byte(dec))
                        return buf.String()
                }

                buf.Write([]byte(str[start+len(CDATA_START) : end]))

                curr = curr + end + len(CDATA_END)
        }

        return buf.String()
}

// DecodeEntities decodes escaped XML entities
// in a string and returns the unescaped string
func DecodeEntities(str string) (string, error) {
        data := []byte(str)
        buf := bytes.NewBuffer([]byte{})

        for len(data) > 0 {
                // Find the next entity
                idx := bytes.IndexByte(data, '&')
                if idx == -1 {
                        buf.Write(data)
                        break
                }

                buf.Write(data[:idx])
                data = data[idx:]

                // If there is only the '&' left here
                if len(data) == 1 {
                        buf.Write(data)
                        return buf.String(), nil
                }

                // Find the end of the entity
                end := bytes.IndexByte(data, ';')
                if end == -1 {
                        // it's not an entitiy. just a plain old '&' possibly with extra bytes
                        buf.Write(data)
                        return buf.String(), nil
                }

                // Check if there is a space somewhere within the 'entitiy'.
                // If there is then skip the whole thing since it's not a real entity.
                if strings.Contains(string(data[1:end]), " ") {
                        buf.Write(data)
                        return buf.String(), nil
                } else {
                        if data[1] == '#' {
                                // Numerical character reference
                                var str string
                                base := 10

                                if len(data) > 2 && data[2] == 'x' {
                                        str = string(data[3:end])
                                        base = 16
                                } else {
                                        str = string(data[2:end])
                                }

                                i, err := strconv.ParseUint(str, base, 32)
                                if err != nil {
                                        return "", InvalidNumericReference
                                }

                                buf.WriteRune(rune(i))
                        } else {
                                // Predefined entity
                                name := string(data[1:end])

                                var c byte
                                switch name {
                                case "lt":
                                        c = '<'
                                case "gt":
                                        c = '>'
                                case "quot":
                                        c = '"'
                                case "apos":
                                        c = '\''
                                case "amp":
                                        c = '&'
                                default:
                                        return "", fmt.Errorf("unknown predefined "+
                                                "entity &%s;", name)
                                }

                                buf.WriteByte(c)
                        }
                }

                // Skip the entity
                data = data[end+1:]
        }

        return buf.String(), nil
}

// ParseNameAddress parses name/email strings commonly
// found in RSS feeds of the format "Example Name (example@site.com)"
// and other variations of this format.
func ParseNameAddress(nameAddressText string) (name string, address string) {
        if nameAddressText == "" {
                return
        }

        if emailNameRgx.MatchString(nameAddressText) {
                result := emailNameRgx.FindStringSubmatch(nameAddressText)
                address = result[1]
                name = result[2]
        } else if nameEmailRgx.MatchString(nameAddressText) {
                result := nameEmailRgx.FindStringSubmatch(nameAddressText)
                name = result[1]
                address = result[2]
        } else if nameOnlyRgx.MatchString(nameAddressText) {
                result := nameOnlyRgx.FindStringSubmatch(nameAddressText)
                name = result[1]
        } else if emailOnlyRgx.MatchString(nameAddressText) {
                result := emailOnlyRgx.FindStringSubmatch(nameAddressText)
                address = result[1]
        }
        return
}

func indexAt(str, substr string, start int) int {
        idx := strings.Index(str[start:], substr)
        if idx > -1 {
                idx += start
        }
        return idx
}

1	package shared
2
3	import (
4	"bytes"
5	"errors"
6	"fmt"
7	"regexp"
8	"strconv"
9	"strings"
10
11	xpp "github.com/mmcdole/goxpp"
12	)
13
14	var (
15	emailNameRgx = regexp.MustCompile(`^([^@]+@[^\s]+)\s+\(([^@]+)\)$`)
16	nameEmailRgx = regexp.MustCompile(`^([^@]+)\s+\(([^@]+@[^)]+)\)$`)
17	nameOnlyRgx = regexp.MustCompile(`^([^@()]+)$`)
18	emailOnlyRgx = regexp.MustCompile(`^([^@()]+@[^@()]+)$`)
19
20	TruncatedEntity = errors.New("truncated entity")
21	InvalidNumericReference = errors.New("invalid numeric reference")
22	)
23
24	const CDATA_START = "<![CDATA["
25	const CDATA_END = "]]>"
26
27	// ParseText is a helper function for parsing the text
28	// from the current element of the XMLPullParser.
29	// This function can handle parsing naked XML text from
30	// an element.
31	func ParseText(p *xpp.XMLPullParser) (string, error) {	200✔
32	var text struct {	200✔
33	Type string `xml:"type,attr"`	200✔
34	InnerXML string `xml:",innerxml"`	200✔
35	}	200✔
36		200✔
37	err := p.DecodeElement(&text)	200✔
38	if err != nil {	200✔
39	return "", err	×
40	}	×
41
42	result := text.InnerXML	200✔
43	result = strings.TrimSpace(result)	200✔
44		200✔
45	if strings.Contains(result, CDATA_START) {	200✔
46	return StripCDATA(result), nil	×
47	}	×
48
49	return DecodeEntities(result)	200✔
50	}
51
52	// StripCDATA removes CDATA tags from the string
53	// content outside of CDATA tags is passed via DecodeEntities
54	func StripCDATA(str string) string {	×
55	buf := bytes.NewBuffer([]byte{})	×
56		×
57	curr := 0	×
58		×
59	for curr < len(str) {	×
60		×
61	start := indexAt(str, CDATA_START, curr)	×
62		×
63	if start == -1 {	×
64	dec, _ := DecodeEntities(str[curr:])	×
65	buf.Write([]byte(dec))	×
66	return buf.String()	×
67	}	×
68
69	end := indexAt(str, CDATA_END, start)	×
70		×
71	if end == -1 {	×
72	dec, _ := DecodeEntities(str[curr:])	×
73	buf.Write([]byte(dec))	×
74	return buf.String()	×
75	}	×
76
77	buf.Write([]byte(str[start+len(CDATA_START) : end]))	×
78		×
79	curr = curr + end + len(CDATA_END)	×
80	}
81
82	return buf.String()	×
83	}
84
85	// DecodeEntities decodes escaped XML entities
86	// in a string and returns the unescaped string
87	func DecodeEntities(str string) (string, error) {	336✔
88	data := []byte(str)	336✔
89	buf := bytes.NewBuffer([]byte{})	336✔
90		336✔
91	for len(data) > 0 {	672✔
92	// Find the next entity	336✔
93	idx := bytes.IndexByte(data, '&')	336✔
94	if idx == -1 {	672✔
95	buf.Write(data)	336✔
96	break	336✔
97	}
98
99	buf.Write(data[:idx])	×
100	data = data[idx:]	×
101		×
102	// If there is only the '&' left here	×
103	if len(data) == 1 {	×
104	buf.Write(data)	×
105	return buf.String(), nil	×
106	}	×
107
108	// Find the end of the entity
109	end := bytes.IndexByte(data, ';')	×
110	if end == -1 {	×
111	// it's not an entitiy. just a plain old '&' possibly with extra bytes	×
112	buf.Write(data)	×
113	return buf.String(), nil	×
114	}	×
115
116	// Check if there is a space somewhere within the 'entitiy'.
117	// If there is then skip the whole thing since it's not a real entity.
118	if strings.Contains(string(data[1:end]), " ") {	×
119	buf.Write(data)	×
120	return buf.String(), nil	×
121	} else {	×
122	if data[1] == '#' {	×
123	// Numerical character reference	×
124	var str string	×
125	base := 10	×
126		×
127	if len(data) > 2 && data[2] == 'x' {	×
128	str = string(data[3:end])	×
129	base = 16	×
130	} else {	×
131	str = string(data[2:end])	×
132	}	×
133
134	i, err := strconv.ParseUint(str, base, 32)	×
135	if err != nil {	×
136	return "", InvalidNumericReference	×
137	}	×
138
139	buf.WriteRune(rune(i))	×
140	} else {	×
141	// Predefined entity	×
142	name := string(data[1:end])	×
143		×
144	var c byte	×
145	switch name {	×
146	case "lt":	×
147	c = '<'	×
148	case "gt":	×
149	c = '>'	×
150	case "quot":	×
151	c = '"'	×
152	case "apos":	×
153	c = '\''	×
154	case "amp":	×
155	c = '&'	×
156	default:	×
157	return "", fmt.Errorf("unknown predefined "+	×
158	"entity &%s;", name)	×
159	}
160
161	buf.WriteByte(c)	×
162	}
163	}
164
165	// Skip the entity
166	data = data[end+1:]	×
167	}
168
169	return buf.String(), nil	336✔
170	}
171
172	// ParseNameAddress parses name/email strings commonly
173	// found in RSS feeds of the format "Example Name (example@site.com)"
174	// and other variations of this format.
175	func ParseNameAddress(nameAddressText string) (name string, address string) {	48✔
176	if nameAddressText == "" {	48✔
177	return	×
178	}	×
179
180	if emailNameRgx.MatchString(nameAddressText) {	60✔
181	result := emailNameRgx.FindStringSubmatch(nameAddressText)	12✔
182	address = result[1]	12✔
183	name = result[2]	12✔
184	} else if nameEmailRgx.MatchString(nameAddressText) {	60✔
185	result := nameEmailRgx.FindStringSubmatch(nameAddressText)	12✔
186	name = result[1]	12✔
187	address = result[2]	12✔
188	} else if nameOnlyRgx.MatchString(nameAddressText) {	48✔
189	result := nameOnlyRgx.FindStringSubmatch(nameAddressText)	12✔
190	name = result[1]	12✔
191	} else if emailOnlyRgx.MatchString(nameAddressText) {	36✔
192	result := emailOnlyRgx.FindStringSubmatch(nameAddressText)	12✔
193	address = result[1]	12✔
194	}	12✔
195	return	48✔
196	}
197
198	func indexAt(str, substr string, start int) int {	×
199	idx := strings.Index(str[start:], substr)	×
200	if idx > -1 {	×
201	idx += start	×
202	}	×
203	return idx	×
204	}

mmcdole / gofeed / 399

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous