10559899724

Committed 26 Aug 2024 12:38PM UTC coverage: 71.125% (+0.3%) from 70.871%

Build # 10559899724

Build Type

push

github

Committed by

HDT3213

Commit Message

fix wrong state of WriteSetObject

Run Details

3 of 5 new or added lines in 1 file covered. (60.0%)

82 existing lines in 4 files now uncovered.

1606 of 2258 relevant lines covered (71.12%)

0.78 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

71.6

/core/string.go

package core

import (
        "encoding/binary"
        "errors"
        "fmt"
        "github.com/hdt3213/rdb/lzf"
        "math"
        "strconv"
        "unicode"
)

const (
        len6Bit      = 0
        len14Bit     = 1
        len32or64Bit = 2
        lenSpecial   = 3
        len32Bit     = 0x80
        len64Bit     = 0x81

        encodeInt8  = 0
        encodeInt16 = 1
        encodeInt32 = 2
        encodeLZF   = 3

        maxUint6  = 1<<6 - 1
        maxUint14 = 1<<14 - 1
        minInt24  = -1 << 23
        maxInt24  = 1<<23 - 1

        len14BitMask      byte = 0b01000000
        encodeInt8Prefix       = lenSpecial<<6 | encodeInt8
        encodeInt16Prefix      = lenSpecial<<6 | encodeInt16
        encodeInt32Prefix      = lenSpecial<<6 | encodeInt32
        encodeLZFPrefix        = lenSpecial<<6 | encodeLZF
)

// readLength parse Length Encoding
// see: https://github.com/sripathikrishnan/redis-rdb-tools/wiki/Redis-RDB-Dump-File-Format#length-encoding
func (dec *Decoder) readLength() (uint64, bool, error) {
        firstByte, err := dec.readByte()
        if err != nil {
                return 0, false, fmt.Errorf("read length failed: %v", err)
        }
        lenType := (firstByte & 0xc0) >> 6 // get first 2 bits
        var length uint64
        special := false
        switch lenType {
        case len6Bit:
                length = uint64(firstByte) & 0x3f
        case len14Bit:
                nextByte, err := dec.readByte()
                if err != nil {
                        return 0, false, fmt.Errorf("read len14Bit failed: %v", err)
                }
                length = (uint64(firstByte)&0x3f)<<8 | uint64(nextByte)
        case len32or64Bit:
                if firstByte == len32Bit {
                        err = dec.readFull(dec.buffer[0:4])
                        if err != nil {
                                return 0, false, fmt.Errorf("read len32Bit failed: %v", err)
                        }
                        length = uint64(binary.BigEndian.Uint32(dec.buffer))
                } else if firstByte == len64Bit {
                        err = dec.readFull(dec.buffer)
                        if err != nil {
                                return 0, false, fmt.Errorf("read len64Bit failed: %v", err)
                        }
                        length = binary.BigEndian.Uint64(dec.buffer)
                } else {
                        return 0, false, fmt.Errorf("illegal length encoding: %x", firstByte)
                }
        case lenSpecial:
                special = true
                length = uint64(firstByte) & 0x3f
        }
        return length, special, nil
}

func (dec *Decoder) readString() ([]byte, error) {
        length, special, err := dec.readLength()
        if err != nil {
                return nil, err
        }

        if special {
                switch length {
                case encodeInt8:
                        b, err := dec.readByte()
                        return []byte(strconv.Itoa(int(int8(b)))), err
                case encodeInt16:
                        b, err := dec.readUint16()
                        return []byte(strconv.Itoa(int(int16(b)))), err
                case encodeInt32:
                        b, err := dec.readUint32()
                        return []byte(strconv.Itoa(int(int32(b)))), err
                case encodeLZF:
                        return dec.readLZF()
                default:
                        return []byte{}, errors.New("Unknown string encode type ")
                }
        }

        res := make([]byte, length)
        err = dec.readFull(res)
        return res, err
}

func (dec *Decoder) readUint16() (uint16, error) {
        err := dec.readFull(dec.buffer[:2])
        if err != nil {
                return 0, fmt.Errorf("read uint16 error: %v", err)
        }

        i := binary.LittleEndian.Uint16(dec.buffer[:2])
        return i, nil
}

func (dec *Decoder) readUint32() (uint32, error) {
        err := dec.readFull(dec.buffer[:4])
        if err != nil {
                return 0, fmt.Errorf("read uint16 error: %v", err)
        }

        i := binary.LittleEndian.Uint32(dec.buffer[:4])
        return i, nil
}

func (dec *Decoder) readLiteralFloat() (float64, error) {
        first, err := dec.readByte()
        if err != nil {
                return 0, err
        }
        if first == 0xff {
                return math.Inf(-1), nil
        } else if first == 0xfe {
                return math.Inf(1), nil
        } else if first == 0xfd {
                return math.NaN(), nil
        }
        buf := make([]byte, first)
        err = dec.readFull(buf)
        if err != nil {
                return 0, err
        }
        str := unsafeBytes2Str(buf)
        val, err := strconv.ParseFloat(str, 64)
        if err != nil {
                return 0, fmt.Errorf("")
        }
        return val, err
}

func (dec *Decoder) readFloat() (float64, error) {
        err := dec.readFull(dec.buffer)
        if err != nil {
                return 0, err
        }
        bits := binary.LittleEndian.Uint64(dec.buffer)
        return math.Float64frombits(bits), nil
}
func (dec *Decoder) readFloat32() (f float32, err error) {
        err = dec.readFull(dec.buffer[:4])
        if err != nil {
                return 0, err
        }
        bits := binary.LittleEndian.Uint32(dec.buffer[:4])
        return math.Float32frombits(bits), nil
}

func (dec *Decoder) readLZF() ([]byte, error) {
        inLen, _, err := dec.readLength()
        if err != nil {
                return nil, err
        }
        outLen, _, err := dec.readLength()
        if err != nil {
                return nil, err
        }
        val := make([]byte, inLen)
        err = dec.readFull(val)
        if err != nil {
                return nil, err
        }
        return lzf.Decompress(val, int(inLen), int(outLen))
}

func (enc *Encoder) writeLength(value uint64) error {
        var buf []byte
        if value <= maxUint6 {
                // 00 + 6 bits of data
                enc.buffer[0] = byte(value)
                buf = enc.buffer[0:1]
        } else if value <= maxUint14 {
                enc.buffer[0] = byte(value>>8) | len14BitMask // high 6 bit and mask(0x40)
                enc.buffer[1] = byte(value)                   // low 8 bit
                buf = enc.buffer[0:2]
        } else if value <= math.MaxUint32 {
                buf = make([]byte, 5)
                buf[0] = len32Bit
                binary.BigEndian.PutUint32(buf[1:], uint32(value))
        } else {
                buf = make([]byte, 9)
                buf[0] = len64Bit
                binary.BigEndian.PutUint64(buf[1:], value)
        }
        return enc.write(buf)
}

func (enc *Encoder) writeSimpleString(s string) error {
        err := enc.writeLength(uint64(len(s)))
        if err != nil {
                return err
        }
        return enc.write([]byte(s))
}

func (enc *Encoder) tryWriteIntString(s string) (bool, error) {
        intVal, ok := isEncodableUint32(s)
        if !ok {
                return false, nil
        }
        var err error
        if intVal >= math.MinInt8 && intVal <= math.MaxInt8 {
                err = enc.write([]byte{encodeInt8Prefix, byte(int8(intVal))})
        } else if intVal >= math.MinInt16 && intVal <= math.MaxInt16 {
                buf := enc.buffer[0:3]
                buf[0] = encodeInt16Prefix
                binary.LittleEndian.PutUint16(buf[1:], uint16(int16(intVal)))
                err = enc.write(buf)
        } else if intVal >= math.MinInt32 && intVal <= math.MaxInt32 {
                buf := enc.buffer[0:5]
                buf[0] = encodeInt32Prefix
                binary.LittleEndian.PutUint32(buf[1:], uint32(int32(intVal)))
                err = enc.write(buf)
        } else {
                // beyond int32 range, but within int64 range
                return false, nil
        }
        if err != nil {
                return true, err
        }
        return true, nil
}

func (enc *Encoder) writeLZFString(s string) error {
        out, err := lzf.Compress([]byte(s))
        if err != nil {
                return err
        }
        err = enc.write([]byte{encodeLZFPrefix})
        if err != nil {
                return err
        }
        // write compressed length
        err = enc.writeLength(uint64(len(out)))
        if err != nil {
                return err
        }
        // write uncompressed length
        err = enc.writeLength(uint64(len(s)))
        if err != nil {
                return err
        }
        return enc.write(out)
}

func (enc *Encoder) writeString(s string) error {
        isInt, err := enc.tryWriteIntString(s)
        if err != nil {
                return err
        }
        if isInt {
                return nil
        }
        // Try LZF compression - under 20 bytes it's unable to compress even so skip it
        // see rdbSaveRawString at [rdb.c](https://github.com/redis/redis/blob/unstable/src/rdb.c#L449)
        if enc.compress && len(s) > 20 {
                err = enc.writeLZFString(s)
                if err == nil { // lzf may failed, while out > in
                        return nil
                }
        }
        return enc.writeSimpleString(s)
}

// write string without try int string. for tryWriteIntSetEncoding, writeZipList
func (enc *Encoder) writeNanString(s string) error {
        if enc.compress && len(s) > 20 {
                err := enc.writeLZFString(s)
                if err == nil { // lzf may failed, while out > in
                        return nil
                }
        }
        return enc.writeSimpleString(s)
}

func (enc *Encoder) WriteStringObject(key string, value []byte, options ...interface{}) error {
        err := enc.beforeWriteObject(options...)
        if err != nil {
                return err
        }
        err = enc.write([]byte{typeString})
        if err != nil {
                return err
        }
        err = enc.writeString(key)
        if err != nil {
                return err
        }
        err = enc.writeString(unsafeBytes2Str(value))
        if err != nil {
                return err
        }
        enc.state = writtenObjectState
        return nil
}

func (enc *Encoder) writeFloat64(f float64) error {
        bin := math.Float64bits(f)
        binary.LittleEndian.PutUint64(enc.buffer, bin)
        return enc.write(enc.buffer)
}

// a string might be encoded as an integer, but only subset (aka uint32) of the number set
// can be encoded and then decoded back.
// e.g. the following strings can not be encoded as an integer:
// "007", "-0", "-1", "+0", "+1", "0x11"
func isEncodableUint32(s string) (int64, bool) {
        if s == "" {
                return 0, false
        }
        if s[0] == '0' && len(s) > 1 {
                return 0, false
        }
        for _, r := range s {
                if !unicode.IsDigit(r) {
                        return 0, false
                }
        }
        intVal, err := strconv.ParseInt(s, 10, 32)
        if err != nil {
                return 0, false
        }
        return intVal, true
}

1	package core
2
3	import (
4	"encoding/binary"
5	"errors"
6	"fmt"
7	"github.com/hdt3213/rdb/lzf"
8	"math"
9	"strconv"
10	"unicode"
11	)
12
13	const (
14	len6Bit = 0
15	len14Bit = 1
16	len32or64Bit = 2
17	lenSpecial = 3
18	len32Bit = 0x80
19	len64Bit = 0x81
20
21	encodeInt8 = 0
22	encodeInt16 = 1
23	encodeInt32 = 2
24	encodeLZF = 3
25
26	maxUint6 = 1<<6 - 1
27	maxUint14 = 1<<14 - 1
28	minInt24 = -1 << 23
29	maxInt24 = 1<<23 - 1
30
31	len14BitMask byte = 0b01000000
32	encodeInt8Prefix = lenSpecial<<6 \| encodeInt8
33	encodeInt16Prefix = lenSpecial<<6 \| encodeInt16
34	encodeInt32Prefix = lenSpecial<<6 \| encodeInt32
35	encodeLZFPrefix = lenSpecial<<6 \| encodeLZF
36	)
37
38	// readLength parse Length Encoding
39	// see: https://github.com/sripathikrishnan/redis-rdb-tools/wiki/Redis-RDB-Dump-File-Format#length-encoding
40	func (dec *Decoder) readLength() (uint64, bool, error) {	1✔
41	firstByte, err := dec.readByte()	1✔
42	if err != nil {	1✔
43	return 0, false, fmt.Errorf("read length failed: %v", err)	×
UNCOV 44	}	×
45	lenType := (firstByte & 0xc0) >> 6 // get first 2 bits	1✔
46	var length uint64	1✔
47	special := false	1✔
48	switch lenType {	1✔
49	case len6Bit:	1✔
50	length = uint64(firstByte) & 0x3f	1✔
51	case len14Bit:	1✔
52	nextByte, err := dec.readByte()	1✔
53	if err != nil {	1✔
54	return 0, false, fmt.Errorf("read len14Bit failed: %v", err)	×
UNCOV 55	}	×
56	length = (uint64(firstByte)&0x3f)<<8 \| uint64(nextByte)	1✔
57	case len32or64Bit:	1✔
58	if firstByte == len32Bit {	2✔
59	err = dec.readFull(dec.buffer[0:4])	1✔
60	if err != nil {	1✔
61	return 0, false, fmt.Errorf("read len32Bit failed: %v", err)	×
UNCOV 62	}	×
63	length = uint64(binary.BigEndian.Uint32(dec.buffer))	1✔
64	} else if firstByte == len64Bit {	2✔
65	err = dec.readFull(dec.buffer)	1✔
66	if err != nil {	1✔
67	return 0, false, fmt.Errorf("read len64Bit failed: %v", err)	×
UNCOV 68	}	×
69	length = binary.BigEndian.Uint64(dec.buffer)	1✔
70	} else {	×
71	return 0, false, fmt.Errorf("illegal length encoding: %x", firstByte)	×
UNCOV 72	}	×
73	case lenSpecial:	1✔
74	special = true	1✔
75	length = uint64(firstByte) & 0x3f	1✔
76	}
77	return length, special, nil	1✔
78	}
79
80	func (dec *Decoder) readString() ([]byte, error) {	1✔
81	length, special, err := dec.readLength()	1✔
82	if err != nil {	1✔
83	return nil, err	×
UNCOV 84	}	×
85
86	if special {	2✔
87	switch length {	1✔
88	case encodeInt8:	1✔
89	b, err := dec.readByte()	1✔
90	return []byte(strconv.Itoa(int(int8(b)))), err	1✔
91	case encodeInt16:	1✔
92	b, err := dec.readUint16()	1✔
93	return []byte(strconv.Itoa(int(int16(b)))), err	1✔
94	case encodeInt32:	1✔
95	b, err := dec.readUint32()	1✔
96	return []byte(strconv.Itoa(int(int32(b)))), err	1✔
97	case encodeLZF:	1✔
98	return dec.readLZF()	1✔
99	default:	×
UNCOV 100	return []byte{}, errors.New("Unknown string encode type ")	×
101	}
102	}
103
104	res := make([]byte, length)	1✔
105	err = dec.readFull(res)	1✔
106	return res, err	1✔
107	}
108
109	func (dec *Decoder) readUint16() (uint16, error) {	1✔
110	err := dec.readFull(dec.buffer[:2])	1✔
111	if err != nil {	1✔
112	return 0, fmt.Errorf("read uint16 error: %v", err)	×
UNCOV 113	}	×
114
115	i := binary.LittleEndian.Uint16(dec.buffer[:2])	1✔
116	return i, nil	1✔
117	}
118
119	func (dec *Decoder) readUint32() (uint32, error) {	1✔
120	err := dec.readFull(dec.buffer[:4])	1✔
121	if err != nil {	1✔
122	return 0, fmt.Errorf("read uint16 error: %v", err)	×
UNCOV 123	}	×
124
125	i := binary.LittleEndian.Uint32(dec.buffer[:4])	1✔
126	return i, nil	1✔
127	}
128
129	func (dec *Decoder) readLiteralFloat() (float64, error) {	1✔
130	first, err := dec.readByte()	1✔
131	if err != nil {	1✔
132	return 0, err	×
UNCOV 133	}	×
134	if first == 0xff {	1✔
UNCOV 135	return math.Inf(-1), nil	×
136	} else if first == 0xfe {	1✔
UNCOV 137	return math.Inf(1), nil	×
138	} else if first == 0xfd {	1✔
139	return math.NaN(), nil	×
UNCOV 140	}	×
141	buf := make([]byte, first)	1✔
142	err = dec.readFull(buf)	1✔
143	if err != nil {	1✔
144	return 0, err	×
UNCOV 145	}	×
146	str := unsafeBytes2Str(buf)	1✔
147	val, err := strconv.ParseFloat(str, 64)	1✔
148	if err != nil {	1✔
149	return 0, fmt.Errorf("")	×
UNCOV 150	}	×
151	return val, err	1✔
152	}
153
154	func (dec *Decoder) readFloat() (float64, error) {	1✔
155	err := dec.readFull(dec.buffer)	1✔
156	if err != nil {	1✔
157	return 0, err	×
UNCOV 158	}	×
159	bits := binary.LittleEndian.Uint64(dec.buffer)	1✔
160	return math.Float64frombits(bits), nil	1✔
161	}
162	func (dec *Decoder) readFloat32() (f float32, err error) {	×
163	err = dec.readFull(dec.buffer[:4])	×
164	if err != nil {	×
165	return 0, err	×
166	}	×
167	bits := binary.LittleEndian.Uint32(dec.buffer[:4])	×
UNCOV 168	return math.Float32frombits(bits), nil	×
169	}
170
171	func (dec *Decoder) readLZF() ([]byte, error) {	1✔
172	inLen, _, err := dec.readLength()	1✔
173	if err != nil {	1✔
174	return nil, err	×
UNCOV 175	}	×
176	outLen, _, err := dec.readLength()	1✔
177	if err != nil {	1✔
178	return nil, err	×
UNCOV 179	}	×
180	val := make([]byte, inLen)	1✔
181	err = dec.readFull(val)	1✔
182	if err != nil {	1✔
183	return nil, err	×
UNCOV 184	}	×
185	return lzf.Decompress(val, int(inLen), int(outLen))	1✔
186	}
187
188	func (enc *Encoder) writeLength(value uint64) error {	1✔
189	var buf []byte	1✔
190	if value <= maxUint6 {	2✔
191	// 00 + 6 bits of data	1✔
192	enc.buffer[0] = byte(value)	1✔
193	buf = enc.buffer[0:1]	1✔
194	} else if value <= maxUint14 {	3✔
195	enc.buffer[0] = byte(value>>8) \| len14BitMask // high 6 bit and mask(0x40)	1✔
196	enc.buffer[1] = byte(value) // low 8 bit	1✔
197	buf = enc.buffer[0:2]	1✔
198	} else if value <= math.MaxUint32 {	3✔
199	buf = make([]byte, 5)	1✔
200	buf[0] = len32Bit	1✔
201	binary.BigEndian.PutUint32(buf[1:], uint32(value))	1✔
202	} else {	2✔
203	buf = make([]byte, 9)	1✔
204	buf[0] = len64Bit	1✔
205	binary.BigEndian.PutUint64(buf[1:], value)	1✔
206	}	1✔
207	return enc.write(buf)	1✔
208	}
209
210	func (enc *Encoder) writeSimpleString(s string) error {	1✔
211	err := enc.writeLength(uint64(len(s)))	1✔
212	if err != nil {	1✔
213	return err	×
UNCOV 214	}	×
215	return enc.write([]byte(s))	1✔
216	}
217
218	func (enc *Encoder) tryWriteIntString(s string) (bool, error) {	1✔
219	intVal, ok := isEncodableUint32(s)	1✔
220	if !ok {	2✔
221	return false, nil	1✔
222	}	1✔
223	var err error	1✔
224	if intVal >= math.MinInt8 && intVal <= math.MaxInt8 {	2✔
225	err = enc.write([]byte{encodeInt8Prefix, byte(int8(intVal))})	1✔
226	} else if intVal >= math.MinInt16 && intVal <= math.MaxInt16 {	3✔
227	buf := enc.buffer[0:3]	1✔
228	buf[0] = encodeInt16Prefix	1✔
229	binary.LittleEndian.PutUint16(buf[1:], uint16(int16(intVal)))	1✔
230	err = enc.write(buf)	1✔
231	} else if intVal >= math.MinInt32 && intVal <= math.MaxInt32 {	3✔
232	buf := enc.buffer[0:5]	1✔
233	buf[0] = encodeInt32Prefix	1✔
234	binary.LittleEndian.PutUint32(buf[1:], uint32(int32(intVal)))	1✔
235	err = enc.write(buf)	1✔
236	} else {	1✔
237	// beyond int32 range, but within int64 range	×
238	return false, nil	×
UNCOV 239	}	×
240	if err != nil {	1✔
241	return true, err	×
UNCOV 242	}	×
243	return true, nil	1✔
244	}
245
246	func (enc *Encoder) writeLZFString(s string) error {	1✔
247	out, err := lzf.Compress([]byte(s))	1✔
248	if err != nil {	1✔
249	return err	×
UNCOV 250	}	×
251	err = enc.write([]byte{encodeLZFPrefix})	1✔
252	if err != nil {	1✔
253	return err	×
UNCOV 254	}	×
255	// write compressed length
256	err = enc.writeLength(uint64(len(out)))	1✔
257	if err != nil {	1✔
258	return err	×
UNCOV 259	}	×
260	// write uncompressed length
261	err = enc.writeLength(uint64(len(s)))	1✔
262	if err != nil {	1✔
263	return err	×
UNCOV 264	}	×
265	return enc.write(out)	1✔
266	}
267
268	func (enc *Encoder) writeString(s string) error {	1✔
269	isInt, err := enc.tryWriteIntString(s)	1✔
270	if err != nil {	1✔
271	return err	×
UNCOV 272	}	×
273	if isInt {	2✔
274	return nil	1✔
275	}	1✔
276	// Try LZF compression - under 20 bytes it's unable to compress even so skip it
277	// see rdbSaveRawString at [rdb.c](https://github.com/redis/redis/blob/unstable/src/rdb.c#L449)
278	if enc.compress && len(s) > 20 {	2✔
279	err = enc.writeLZFString(s)	1✔
280	if err == nil { // lzf may failed, while out > in	2✔
281	return nil	1✔
282	}	1✔
283	}
284	return enc.writeSimpleString(s)	1✔
285	}
286
287	// write string without try int string. for tryWriteIntSetEncoding, writeZipList
288	func (enc *Encoder) writeNanString(s string) error {	1✔
289	if enc.compress && len(s) > 20 {	1✔
290	err := enc.writeLZFString(s)	×
291	if err == nil { // lzf may failed, while out > in	×
292	return nil	×
UNCOV 293	}	×
294	}
295	return enc.writeSimpleString(s)	1✔
296	}
297
298	func (enc *Encoder) WriteStringObject(key string, value []byte, options ...interface{}) error {	1✔
299	err := enc.beforeWriteObject(options...)	1✔
300	if err != nil {	1✔
301	return err	×
UNCOV 302	}	×
303	err = enc.write([]byte{typeString})	1✔
304	if err != nil {	1✔
305	return err	×
UNCOV 306	}	×
307	err = enc.writeString(key)	1✔
308	if err != nil {	1✔
309	return err	×
UNCOV 310	}	×
311	err = enc.writeString(unsafeBytes2Str(value))	1✔
312	if err != nil {	1✔
313	return err	×
UNCOV 314	}	×
315	enc.state = writtenObjectState	1✔
316	return nil	1✔
317	}
318
319	func (enc *Encoder) writeFloat64(f float64) error {	1✔
320	bin := math.Float64bits(f)	1✔
321	binary.LittleEndian.PutUint64(enc.buffer, bin)	1✔
322	return enc.write(enc.buffer)	1✔
323	}	1✔
324
325	// a string might be encoded as an integer, but only subset (aka uint32) of the number set
326	// can be encoded and then decoded back.
327	// e.g. the following strings can not be encoded as an integer:
328	// "007", "-0", "-1", "+0", "+1", "0x11"
329	func isEncodableUint32(s string) (int64, bool) {	1✔
330	if s == "" {	2✔
331	return 0, false	1✔
332	}	1✔
333	if s[0] == '0' && len(s) > 1 {	2✔
334	return 0, false	1✔
335	}	1✔
336	for _, r := range s {	2✔
337	if !unicode.IsDigit(r) {	2✔
338	return 0, false	1✔
339	}	1✔
340	}
341	intVal, err := strconv.ParseInt(s, 10, 32)	1✔
342	if err != nil {	2✔
343	return 0, false	1✔
344	}	1✔
345	return intVal, true	1✔
346	}

HDT3213 / rdb / 10559899724

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous