• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

leonchen83 / redis-replicator / #2171

06 Jun 2025 05:39AM UTC coverage: 71.185% (-0.2%) from 71.406%
#2171

push

chenby
redis 8.0

8 of 33 new or added lines in 7 files covered. (24.24%)

5 existing lines in 2 files now uncovered.

6638 of 9325 relevant lines covered (71.18%)

0.71 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.38
/src/main/java/com/moilioncircle/redis/replicator/rdb/RdbParser.java
1
/*
2
 * Copyright 2016-2018 Leon Chen
3
 *
4
 * Licensed under the Apache License, Version 2.0 (the "License");
5
 * you may not use this file except in compliance with the License.
6
 * You may obtain a copy of the License at
7
 *
8
 *     http://www.apache.org/licenses/LICENSE-2.0
9
 *
10
 * Unless required by applicable law or agreed to in writing, software
11
 * distributed under the License is distributed on an "AS IS" BASIS,
12
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
 * See the License for the specific language governing permissions and
14
 * limitations under the License.
15
 */
16

17
package com.moilioncircle.redis.replicator.rdb;
18

19
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_AUX;
20
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_EOF;
21
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_EXPIRETIME;
22
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_EXPIRETIME_MS;
23
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_FREQ;
24
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_FUNCTION;
25
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_FUNCTION2;
26
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_IDLE;
27
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_MODULE_AUX;
28
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_RESIZEDB;
29
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_SELECTDB;
30
import static com.moilioncircle.redis.replicator.Constants.RDB_OPCODE_SLOT_INFO;
31
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_HASH;
32
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_HASH_LISTPACK;
33
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_HASH_LISTPACK_EX;
34
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_HASH_METADATA;
35
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_HASH_ZIPLIST;
36
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_HASH_ZIPMAP;
37
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_LIST;
38
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_LIST_QUICKLIST;
39
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_LIST_QUICKLIST_2;
40
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_LIST_ZIPLIST;
41
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_MODULE;
42
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_MODULE_2;
43
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_SET;
44
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_SET_INTSET;
45
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_SET_LISTPACK;
46
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_STREAM_LISTPACKS;
47
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_STREAM_LISTPACKS_2;
48
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_STREAM_LISTPACKS_3;
49
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_STRING;
50
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_ZSET;
51
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_ZSET_2;
52
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_ZSET_LISTPACK;
53
import static com.moilioncircle.redis.replicator.Constants.RDB_TYPE_ZSET_ZIPLIST;
54
import static com.moilioncircle.redis.replicator.Status.CONNECTED;
55
import static com.moilioncircle.redis.replicator.util.Tuples.of;
56

57
import java.io.IOException;
58

59
import org.slf4j.Logger;
60
import org.slf4j.LoggerFactory;
61

62
import com.moilioncircle.redis.replicator.AbstractReplicator;
63
import com.moilioncircle.redis.replicator.event.Event;
64
import com.moilioncircle.redis.replicator.event.PostRdbSyncEvent;
65
import com.moilioncircle.redis.replicator.event.PreRdbSyncEvent;
66
import com.moilioncircle.redis.replicator.io.RedisInputStream;
67
import com.moilioncircle.redis.replicator.rdb.datatype.ContextKeyValuePair;
68
import com.moilioncircle.redis.replicator.rdb.datatype.DB;
69
import com.moilioncircle.redis.replicator.rdb.datatype.Slot;
70

71
/**
72
 * Redis RDB format
73
 * <p>
74
 *
75
 * @author Leon Chen
76
 * @see <a href="https://github.com/antirez/redis/blob/3.0/src/rdb.c">rdb.c</a>
77
 * @see <a href="https://github.com/leonchen83/redis-replicator/wiki/RDB-dump-data-format">Redis rdb dump data format</a>
78
 * @since 2.1.0
79
 */
80
public class RdbParser {
81

82
    protected final RedisInputStream in;
83
    protected final RdbVisitor rdbVisitor;
84
    protected final AbstractReplicator replicator;
85
    protected static final Logger logger = LoggerFactory.getLogger(RdbParser.class);
1✔
86

87
    public RdbParser(RedisInputStream in, AbstractReplicator replicator) {
1✔
88
        this.in = in;
1✔
89
        this.replicator = replicator;
1✔
90
        this.rdbVisitor = this.replicator.getRdbVisitor();
1✔
91
    }
1✔
92

93
    /**
94
     * The RDB E-BNF
95
     * <p>
96
     * RDB        =    'REDIS', $version, [AUX], [MODULE_AUX], [FUNCTION], {SELECTDB, [RESIZEDB], {RECORD}}, '0xFF', [$checksum];
97
     * <p>
98
     * RECORD     =    [EXPIRED], [IDLE | FREQ], KEY, VALUE;
99
     * <p>
100
     * SELECTDB   =    '0xFE', $length;
101
     * <p>
102
     * AUX        =    {'0xFA', $string, $string};            (*Introduced in rdb version 7*)
103
     * <p>
104
     * MODULE_AUX =    {'0xF7', $module2};                    (*Introduced in rdb version 9*)
105
     * <p>
106
     * FUNCTION   =    {'0xF6', $function};                   (*Introduced in rdb version 10*)
107
     * <p>
108
     * RESIZEDB   =    '0xFB', $length, $length;              (*Introduced in rdb version 7*)
109
     * <p>
110
     * EXPIRED    =    ('0xFD', $second) | ('0xFC', $millisecond);
111
     * <p>
112
     * IDLE       =    {'0xF8', $value-type};                 (*Introduced in rdb version 9*)
113
     * <p>
114
     * FREQ       =    {'0xF9', $length};                     (*Introduced in rdb version 9*)
115
     * <p>
116
     * KEY        =    $string;
117
     * <p>
118
     * VALUE      =    $value-type, ( $string
119
     * <p>
120
     * | $list
121
     * <p>
122
     * | $set
123
     * <p>
124
     * | $setlistpack            (*Introduced in rdb version 11*)
125
     * <p>
126
     * | $zset
127
     * <p>
128
     * | $hash
129
     * <p>
130
     * | $zset2                  (*Introduced in rdb version 8*)
131
     * <p>
132
     * | $module                 (*Introduced in rdb version 8*)
133
     * <p>
134
     * | $module2                (*Introduced in rdb version 8*)
135
     * <p>
136
     * | $hashzipmap
137
     * <p>
138
     * | $listziplist
139
     * <p>
140
     * | $setintset
141
     * <p>
142
     * | $zsetziplist
143
     * <p>
144
     * | $hashziplist
145
     * <p>
146
     * | $listquicklist          (*Introduced in rdb version 7*)
147
     * <p>
148
     * | $streamlistpacks);      (*Introduced in rdb version 9*)
149
     * <p>
150
     * | $zsetlistpack);         (*Introduced in rdb version 10*)
151
     * <p>
152
     * | $hashlistpack);         (*Introduced in rdb version 10*)
153
     * <p>
154
     * | $listquicklist2);       (*Introduced in rdb version 10*)
155
     * <p>
156
     * | $streamlistpacks2);      (*Introduced in rdb version 10*)
157
     * <p>
158
     * | $streamlistpacks3);      (*Introduced in rdb version 11*)
159
     * <p>
160
     * | $hashlistpackex);      (*Introduced in rdb version 12*)
161
     * <p>
162
     * | $hashmetadata);      (*Introduced in rdb version 12*)
163
     * <p>
164
     * @return read bytes
165
     * @throws IOException when read timeout
166
     */
167
    public long parse() throws IOException {
168
        /*
169
         * ----------------------------
170
         * 52 45 44 49 53              # Magic String "REDIS"
171
         * 30 30 30 33                 # RDB Version Number in big endian. In this case, version = 0003 = 3
172
         * ----------------------------
173
         */
174
        long offset = 0L;
1✔
175
        boolean discard = this.replicator.getConfiguration().isDiscardRdbEvent();
1✔
176
        if (!discard) this.replicator.submitEvent(new PreRdbSyncEvent(), of(0L, 0L));
1✔
177
        in.mark();
1✔
178
        rdbVisitor.applyMagic(in);
1✔
179
        int version = rdbVisitor.applyVersion(in);
1✔
180
        offset += in.unmark();
1✔
181
        DB db = null;
1✔
182
        Slot slot = null;
1✔
183
        /*
184
         * rdb
185
         */
186
        loop:
187
        while (this.replicator.getStatus() == CONNECTED) {
1✔
188
            Event event = null;
1✔
189
            in.mark();
1✔
190
            int type = rdbVisitor.applyType(in);
1✔
191
            ContextKeyValuePair kv = new ContextKeyValuePair();
1✔
192
            kv.setDb(db);
1✔
193
            kv.setSlot(slot);
1✔
194
            switch (type) {
1✔
195
                case RDB_OPCODE_EXPIRETIME:
196
                    event = rdbVisitor.applyExpireTime(in, version, kv);
×
197
                    break;
×
198
                case RDB_OPCODE_EXPIRETIME_MS:
199
                    event = rdbVisitor.applyExpireTimeMs(in, version, kv);
1✔
200
                    break;
1✔
201
                case RDB_OPCODE_FREQ:
202
                    event = rdbVisitor.applyFreq(in, version, kv);
1✔
203
                    break;
1✔
204
                case RDB_OPCODE_IDLE:
205
                    event = rdbVisitor.applyIdle(in, version, kv);
1✔
206
                    break;
1✔
207
                case RDB_OPCODE_AUX:
208
                    event = rdbVisitor.applyAux(in, version);
1✔
209
                    break;
1✔
210
                case RDB_OPCODE_MODULE_AUX:
211
                    event = rdbVisitor.applyModuleAux(in, version);
×
212
                    break;
×
213
                case RDB_OPCODE_FUNCTION:
214
                    event = rdbVisitor.applyFunction(in, version);
1✔
215
                    break;
1✔
216
                case RDB_OPCODE_FUNCTION2:
217
                    event = rdbVisitor.applyFunction2(in, version);
1✔
218
                    break;
1✔
219
                case RDB_OPCODE_RESIZEDB:
220
                    rdbVisitor.applyResizeDB(in, version, kv);
1✔
221
                    break;
1✔
222
                case RDB_OPCODE_SELECTDB:
223
                    db = rdbVisitor.applySelectDB(in, version);
1✔
224
                    break;
1✔
225
                case RDB_OPCODE_SLOT_INFO:
NEW
226
                    slot = rdbVisitor.applySlotInfo(in, version);
×
NEW
227
                    break;
×
228
                case RDB_OPCODE_EOF:
229
                    long checksum = rdbVisitor.applyEof(in, version);
1✔
230
                    long start = offset;
1✔
231
                    offset += in.unmark();
1✔
232
                    if (!discard) this.replicator.submitEvent(new PostRdbSyncEvent(checksum), of(start, offset));
1✔
233
                    break loop;
234
                case RDB_TYPE_STRING:
235
                    event = rdbVisitor.applyString(in, version, kv);
1✔
236
                    break;
1✔
237
                case RDB_TYPE_LIST:
238
                    event = rdbVisitor.applyList(in, version, kv);
1✔
239
                    break;
1✔
240
                case RDB_TYPE_SET:
241
                    event = rdbVisitor.applySet(in, version, kv);
1✔
242
                    break;
1✔
243
                case RDB_TYPE_SET_LISTPACK:
244
                    event = rdbVisitor.applySetListPack(in, version, kv);
1✔
245
                    break;
1✔
246
                case RDB_TYPE_ZSET:
247
                    event = rdbVisitor.applyZSet(in, version, kv);
1✔
248
                    break;
1✔
249
                case RDB_TYPE_ZSET_2:
250
                    event = rdbVisitor.applyZSet2(in, version, kv);
1✔
251
                    break;
1✔
252
                case RDB_TYPE_HASH:
253
                    event = rdbVisitor.applyHash(in, version, kv);
1✔
254
                    break;
1✔
255
                case RDB_TYPE_HASH_ZIPMAP:
256
                    event = rdbVisitor.applyHashZipMap(in, version, kv);
1✔
257
                    break;
1✔
258
                case RDB_TYPE_LIST_ZIPLIST:
259
                    event = rdbVisitor.applyListZipList(in, version, kv);
1✔
260
                    break;
1✔
261
                case RDB_TYPE_SET_INTSET:
262
                    event = rdbVisitor.applySetIntSet(in, version, kv);
1✔
263
                    break;
1✔
264
                case RDB_TYPE_ZSET_ZIPLIST:
265
                    event = rdbVisitor.applyZSetZipList(in, version, kv);
1✔
266
                    break;
1✔
267
                case RDB_TYPE_ZSET_LISTPACK:
268
                    event = rdbVisitor.applyZSetListPack(in, version, kv);
1✔
269
                    break;
1✔
270
                case RDB_TYPE_HASH_ZIPLIST:
271
                    event = rdbVisitor.applyHashZipList(in, version, kv);
1✔
272
                    break;
1✔
273
                case RDB_TYPE_HASH_LISTPACK:
274
                    event = rdbVisitor.applyHashListPack(in, version, kv);
1✔
275
                    break;
1✔
276
                case RDB_TYPE_LIST_QUICKLIST:
277
                    event = rdbVisitor.applyListQuickList(in, version, kv);
1✔
278
                    break;
1✔
279
                case RDB_TYPE_LIST_QUICKLIST_2:
280
                    event = rdbVisitor.applyListQuickList2(in, version, kv);
1✔
281
                    break;
1✔
282
                case RDB_TYPE_MODULE:
283
                    event = rdbVisitor.applyModule(in, version, kv);
1✔
284
                    break;
1✔
285
                case RDB_TYPE_MODULE_2:
286
                    event = rdbVisitor.applyModule2(in, version, kv);
×
287
                    break;
×
288
                case RDB_TYPE_STREAM_LISTPACKS:
289
                    event = rdbVisitor.applyStreamListPacks(in, version, kv);
1✔
290
                    break;
1✔
291
                case RDB_TYPE_STREAM_LISTPACKS_2:
292
                    event = rdbVisitor.applyStreamListPacks2(in, version, kv);
1✔
293
                    break;
1✔
294
                case RDB_TYPE_STREAM_LISTPACKS_3:
295
                    event = rdbVisitor.applyStreamListPacks3(in, version, kv);
1✔
296
                    break;
1✔
297
                case RDB_TYPE_HASH_METADATA:
298
                    event = rdbVisitor.applyHashMetadata(in, version, kv);
×
299
                    break;
×
300
                case RDB_TYPE_HASH_LISTPACK_EX:
301
                    event = rdbVisitor.applyHashListPackEx(in, version, kv);
×
302
                    break;
×
303
                default:
304
                    throw new AssertionError("unexpected value type:" + type + ", check your ModuleParser or ValueIterableRdbVisitor.");
×
305
            }
306
            long start = offset;
1✔
307
            offset += in.unmark();
1✔
308
            if (event == null) continue;
1✔
309
            if (replicator.verbose() && logger.isDebugEnabled()) logger.debug("{}", event);
1✔
310
            if (!discard) this.replicator.submitEvent(event, of(start, offset));
1✔
311
        }
1✔
312
        return offset;
1✔
313
    }
314
}
315

STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc