• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

leonchen83 / redis-replicator / #2076

09 Dec 2023 02:37AM UTC coverage: 71.561%. Remained the same
#2076

push

Baoyi Chen
test

3 of 3 new or added lines in 2 files covered. (100.0%)

140 existing lines in 4 files now uncovered.

6628 of 9262 relevant lines covered (71.56%)

0.72 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.63
/src/main/java/com/moilioncircle/redis/replicator/rdb/RdbParser.java
1
/*
2
 * Copyright 2016-2018 Leon Chen
3
 *
4
 * Licensed under the Apache License, Version 2.0 (the "License");
5
 * you may not use this file except in compliance with the License.
6
 * You may obtain a copy of the License at
7
 *
8
 *     http://www.apache.org/licenses/LICENSE-2.0
9
 *
10
 * Unless required by applicable law or agreed to in writing, software
11
 * distributed under the License is distributed on an "AS IS" BASIS,
12
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
 * See the License for the specific language governing permissions and
14
 * limitations under the License.
15
 */
16

17
package com.moilioncircle.redis.replicator.rdb;
18

19
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_AUX;
20
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_EOF;
21
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_EXPIRETIME;
22
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_EXPIRETIME_MS;
23
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_FREQ;
24
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_FUNCTION;
25
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_FUNCTION2;
26
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_IDLE;
27
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_MODULE_AUX;
28
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_RESIZEDB;
29
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_SELECTDB;
30
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_HASH;
31
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_HASH_LISTPACK;
32
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_HASH_ZIPLIST;
33
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_HASH_ZIPMAP;
34
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_LIST;
35
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_LIST_QUICKLIST;
36
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_LIST_QUICKLIST_2;
37
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_LIST_ZIPLIST;
38
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_MODULE;
39
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_MODULE_2;
40
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_SET;
41
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_SET_INTSET;
42
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_SET_LISTPACK;
43
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_STREAM_LISTPACKS;
44
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_STREAM_LISTPACKS_2;
45
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_STREAM_LISTPACKS_3;
46
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_STRING;
47
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_ZSET;
48
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_ZSET_2;
49
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_ZSET_LISTPACK;
50
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_ZSET_ZIPLIST;
51
import static com.moilioncircle.redis.replicator.Status.CONNECTED;
52
import static com.moilioncircle.redis.replicator.util.Tuples.of;
53

54
import java.io.IOException;
55

56
import org.slf4j.Logger;
57
import org.slf4j.LoggerFactory;
58

59
import com.moilioncircle.redis.replicator.AbstractReplicator;
60
import com.moilioncircle.redis.replicator.event.Event;
61
import com.moilioncircle.redis.replicator.event.PostRdbSyncEvent;
62
import com.moilioncircle.redis.replicator.event.PreRdbSyncEvent;
63
import com.moilioncircle.redis.replicator.io.RedisInputStream;
64
import com.moilioncircle.redis.replicator.rdb.datatype.ContextKeyValuePair;
65
import com.moilioncircle.redis.replicator.rdb.datatype.DB;
66

67
/**
68
 * Redis RDB format
69
 * <p>
70
 *
71
 * @author Leon Chen
72
 * @see <a href="https://github.com/antirez/redis/blob/3.0/src/rdb.c">rdb.c</a>
73
 * @see <a href="https://github.com/leonchen83/redis-replicator/wiki/RDB-dump-data-format">Redis rdb dump data format</a>
74
 * @since 2.1.0
75
 */
76
public class RdbParser {
77

78
    protected final RedisInputStream in;
79
    protected final RdbVisitor rdbVisitor;
80
    protected final AbstractReplicator replicator;
81
    protected static final Logger logger = LoggerFactory.getLogger(RdbParser.class);
1✔
82

83
    public RdbParser(RedisInputStream in, AbstractReplicator replicator) {
1✔
84
        this.in = in;
1✔
85
        this.replicator = replicator;
1✔
86
        this.rdbVisitor = this.replicator.getRdbVisitor();
1✔
87
    }
1✔
88

89
    /**
90
     * The RDB E-BNF
91
     * <p>
92
     * RDB        =    'REDIS', $version, [AUX], [MODULE_AUX], [FUNCTION], {SELECTDB, [RESIZEDB], {RECORD}}, '0xFF', [$checksum];
93
     * <p>
94
     * RECORD     =    [EXPIRED], [IDLE | FREQ], KEY, VALUE;
95
     * <p>
96
     * SELECTDB   =    '0xFE', $length;
97
     * <p>
98
     * AUX        =    {'0xFA', $string, $string};            (*Introduced in rdb version 7*)
99
     * <p>
100
     * MODULE_AUX =    {'0xF7', $module2};                    (*Introduced in rdb version 9*)
101
     * <p>
102
     * FUNCTION   =    {'0xF6', $function};                   (*Introduced in rdb version 10*)
103
     * <p>
104
     * RESIZEDB   =    '0xFB', $length, $length;              (*Introduced in rdb version 7*)
105
     * <p>
106
     * EXPIRED    =    ('0xFD', $second) | ('0xFC', $millisecond);
107
     * <p>
108
     * IDLE       =    {'0xF8', $value-type};                 (*Introduced in rdb version 9*)
109
     * <p>
110
     * FREQ       =    {'0xF9', $length};                     (*Introduced in rdb version 9*)
111
     * <p>
112
     * KEY        =    $string;
113
     * <p>
114
     * VALUE      =    $value-type, ( $string
115
     * <p>
116
     * | $list
117
     * <p>
118
     * | $set
119
     * <p>
120
     * | $setlistpack            (*Introduced in rdb version 11*)
121
     * <p>
122
     * | $zset
123
     * <p>
124
     * | $hash
125
     * <p>
126
     * | $zset2                  (*Introduced in rdb version 8*)
127
     * <p>
128
     * | $module                 (*Introduced in rdb version 8*)
129
     * <p>
130
     * | $module2                (*Introduced in rdb version 8*)
131
     * <p>
132
     * | $hashzipmap
133
     * <p>
134
     * | $listziplist
135
     * <p>
136
     * | $setintset
137
     * <p>
138
     * | $zsetziplist
139
     * <p>
140
     * | $hashziplist
141
     * <p>
142
     * | $listquicklist          (*Introduced in rdb version 7*)
143
     * <p>
144
     * | $streamlistpacks);      (*Introduced in rdb version 9*)
145
     * <p>
146
     * | $zsetlistpack);         (*Introduced in rdb version 10*)
147
     * <p>
148
     * | $hashlistpack);         (*Introduced in rdb version 10*)
149
     * <p>
150
     * | $listquicklist2);       (*Introduced in rdb version 10*)
151
     * <p>
152
     * | $streamlistpacks2);      (*Introduced in rdb version 10*)
153
     * <p>
154
     * | $streamlistpacks3);      (*Introduced in rdb version 11*)
155
     * <p>
156
     * @return read bytes
157
     * @throws IOException when read timeout
158
     */
159
    public long parse() throws IOException {
160
        /*
161
         * ----------------------------
162
         * 52 45 44 49 53              # Magic String "REDIS"
163
         * 30 30 30 33                 # RDB Version Number in big endian. In this case, version = 0003 = 3
164
         * ----------------------------
165
         */
166
        long offset = 0L;
1✔
167
        boolean discard = this.replicator.getConfiguration().isDiscardRdbEvent();
1✔
168
        if (!discard) this.replicator.submitEvent(new PreRdbSyncEvent(), of(0L, 0L));
1✔
169
        in.mark();
1✔
170
        rdbVisitor.applyMagic(in);
1✔
171
        int version = rdbVisitor.applyVersion(in);
1✔
172
        offset += in.unmark();
1✔
173
        DB db = null;
1✔
174
        /*
175
         * rdb
176
         */
177
        loop:
178
        while (this.replicator.getStatus() == CONNECTED) {
1✔
179
            Event event = null;
1✔
180
            in.mark();
1✔
181
            int type = rdbVisitor.applyType(in);
1✔
182
            ContextKeyValuePair kv = new ContextKeyValuePair();
1✔
183
            kv.setDb(db);
1✔
184
            switch (type) {
1✔
185
                case RDB_OPCODE_EXPIRETIME:
UNCOV
186
                    event = rdbVisitor.applyExpireTime(in, version, kv);
×
UNCOV
187
                    break;
×
188
                case RDB_OPCODE_EXPIRETIME_MS:
189
                    event = rdbVisitor.applyExpireTimeMs(in, version, kv);
1✔
190
                    break;
1✔
191
                case RDB_OPCODE_FREQ:
192
                    event = rdbVisitor.applyFreq(in, version, kv);
1✔
193
                    break;
1✔
194
                case RDB_OPCODE_IDLE:
195
                    event = rdbVisitor.applyIdle(in, version, kv);
1✔
196
                    break;
1✔
197
                case RDB_OPCODE_AUX:
198
                    event = rdbVisitor.applyAux(in, version);
1✔
199
                    break;
1✔
200
                case RDB_OPCODE_MODULE_AUX:
UNCOV
201
                    event = rdbVisitor.applyModuleAux(in, version);
×
UNCOV
202
                    break;
×
203
                case RDB_OPCODE_FUNCTION:
204
                    event = rdbVisitor.applyFunction(in, version);
1✔
205
                    break;
1✔
206
                case RDB_OPCODE_FUNCTION2:
207
                    event = rdbVisitor.applyFunction2(in, version);
1✔
208
                    break;
1✔
209
                case RDB_OPCODE_RESIZEDB:
210
                    rdbVisitor.applyResizeDB(in, version, kv);
1✔
211
                    break;
1✔
212
                case RDB_OPCODE_SELECTDB:
213
                    db = rdbVisitor.applySelectDB(in, version);
1✔
214
                    break;
1✔
215
                case RDB_OPCODE_EOF:
216
                    long checksum = rdbVisitor.applyEof(in, version);
1✔
217
                    long start = offset;
1✔
218
                    offset += in.unmark();
1✔
219
                    if (!discard) this.replicator.submitEvent(new PostRdbSyncEvent(checksum), of(start, offset));
1✔
220
                    break loop;
221
                case RDB_TYPE_STRING:
222
                    event = rdbVisitor.applyString(in, version, kv);
1✔
223
                    break;
1✔
224
                case RDB_TYPE_LIST:
225
                    event = rdbVisitor.applyList(in, version, kv);
1✔
226
                    break;
1✔
227
                case RDB_TYPE_SET:
228
                    event = rdbVisitor.applySet(in, version, kv);
1✔
229
                    break;
1✔
230
                case RDB_TYPE_SET_LISTPACK:
231
                    event = rdbVisitor.applySetListPack(in, version, kv);
1✔
232
                    break;
1✔
233
                case RDB_TYPE_ZSET:
234
                    event = rdbVisitor.applyZSet(in, version, kv);
1✔
235
                    break;
1✔
236
                case RDB_TYPE_ZSET_2:
237
                    event = rdbVisitor.applyZSet2(in, version, kv);
1✔
238
                    break;
1✔
239
                case RDB_TYPE_HASH:
240
                    event = rdbVisitor.applyHash(in, version, kv);
1✔
241
                    break;
1✔
242
                case RDB_TYPE_HASH_ZIPMAP:
243
                    event = rdbVisitor.applyHashZipMap(in, version, kv);
1✔
244
                    break;
1✔
245
                case RDB_TYPE_LIST_ZIPLIST:
246
                    event = rdbVisitor.applyListZipList(in, version, kv);
1✔
247
                    break;
1✔
248
                case RDB_TYPE_SET_INTSET:
249
                    event = rdbVisitor.applySetIntSet(in, version, kv);
1✔
250
                    break;
1✔
251
                case RDB_TYPE_ZSET_ZIPLIST:
252
                    event = rdbVisitor.applyZSetZipList(in, version, kv);
1✔
253
                    break;
1✔
254
                case RDB_TYPE_ZSET_LISTPACK:
255
                    event = rdbVisitor.applyZSetListPack(in, version, kv);
1✔
256
                    break;
1✔
257
                case RDB_TYPE_HASH_ZIPLIST:
258
                    event = rdbVisitor.applyHashZipList(in, version, kv);
1✔
259
                    break;
1✔
260
                case RDB_TYPE_HASH_LISTPACK:
261
                    event = rdbVisitor.applyHashListPack(in, version, kv);
1✔
262
                    break;
1✔
263
                case RDB_TYPE_LIST_QUICKLIST:
264
                    event = rdbVisitor.applyListQuickList(in, version, kv);
1✔
265
                    break;
1✔
266
                case RDB_TYPE_LIST_QUICKLIST_2:
267
                    event = rdbVisitor.applyListQuickList2(in, version, kv);
1✔
268
                    break;
1✔
269
                case RDB_TYPE_MODULE:
270
                    event = rdbVisitor.applyModule(in, version, kv);
1✔
271
                    break;
1✔
272
                case RDB_TYPE_MODULE_2:
UNCOV
273
                    event = rdbVisitor.applyModule2(in, version, kv);
×
UNCOV
274
                    break;
×
275
                case RDB_TYPE_STREAM_LISTPACKS:
276
                    event = rdbVisitor.applyStreamListPacks(in, version, kv);
1✔
277
                    break;
1✔
278
                case RDB_TYPE_STREAM_LISTPACKS_2:
279
                    event = rdbVisitor.applyStreamListPacks2(in, version, kv);
1✔
280
                    break;
1✔
281
                case RDB_TYPE_STREAM_LISTPACKS_3:
282
                    event = rdbVisitor.applyStreamListPacks3(in, version, kv);
1✔
283
                    break;
1✔
284
                default:
UNCOV
285
                    throw new AssertionError("unexpected value type:" + type + ", check your ModuleParser or ValueIterableRdbVisitor.");
×
286
            }
287
            long start = offset;
1✔
288
            offset += in.unmark();
1✔
289
            if (event == null) continue;
1✔
290
            if (replicator.verbose() && logger.isDebugEnabled()) logger.debug("{}", event);
1✔
291
            if (!discard) this.replicator.submitEvent(event, of(start, offset));
1✔
292
        }
1✔
293
        return offset;
1✔
294
    }
295
}
296

STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc