• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

taosdata / TDengine / #4830

31 Oct 2025 03:37AM UTC coverage: 58.792% (+9.1%) from 49.734%
#4830

push

travis-ci

SallyHuo-TAOS
Merge remote-tracking branch 'origin/cover/3.0' into cover/3.0

# Conflicts:
#	test/ci/run.sh

149913 of 324176 branches covered (46.24%)

Branch coverage included in aggregate %.

199119 of 269498 relevant lines covered (73.89%)

239518383.75 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

75.58
/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c
1
/*
2
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3
 *
4
 * This program is free software: you can use, redistribute, and/or modify
5
 * it under the terms of the GNU Affero General Public License, version 3
6
 * or later ("AGPL"), as published by the Free Software Foundation.
7
 *
8
 * This program is distributed in the hope that it will be useful, but WITHOUT
9
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10
 * FITNESS FOR A PARTICULAR PURPOSE.
11
 *
12
 * You should have received a copy of the GNU Affero General Public License
13
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
14
 */
15

16
#include "tsdbSttFileRW.h"
17
#include "meta.h"
18
#include "tsdbDataFileRW.h"
19

20
// SSttFReader ============================================================
21
struct SSttFileReader {
22
  SSttFileReaderConfig config[1];
23
  STsdbFD             *fd;
24
  SSttFooter           footer[1];
25
  struct {
26
    bool sttBlkLoaded;
27
    bool statisBlkLoaded;
28
    bool tombBlkLoaded;
29
  } ctx[1];
30
  TSttBlkArray    sttBlkArray[1];
31
  TStatisBlkArray statisBlkArray[1];
32
  TTombBlkArray   tombBlkArray[1];
33
  SBuffer         local[10];
34
  SBuffer        *buffers;
35
};
36

37
// SSttFileReader
38
int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *config, SSttFileReader **reader) {
1,367,996,450✔
39
  int32_t code = 0;
1,367,996,450✔
40
  int32_t lino = 0;
1,367,996,450✔
41

42
  reader[0] = taosMemoryCalloc(1, sizeof(*reader[0]));
1,368,107,444✔
43
  if (reader[0] == NULL) {
1,367,073,284!
44
    TAOS_CHECK_GOTO(terrno, &lino, _exit);
×
45
  }
46

47
  reader[0]->config[0] = config[0];
1,367,090,364✔
48
  reader[0]->buffers = config->buffers;
1,367,244,187✔
49
  if (reader[0]->buffers == NULL) {
1,367,940,502✔
50
    reader[0]->buffers = reader[0]->local;
1,367,883,361✔
51
  }
52

53
  // open file
54
  if (fname) {
1,367,968,339!
55
    TAOS_CHECK_GOTO(tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0), &lino, _exit);
1,367,968,339!
56
  } else {
57
    char fname1[TSDB_FILENAME_LEN];
×
58
    tsdbTFileName(config->tsdb, config->file, fname1);
×
59
    TAOS_CHECK_GOTO(tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0), &lino, _exit);
×
60
  }
61

62
  // // open each segment reader
63
  int64_t offset = config->file->size - sizeof(SSttFooter);
1,367,686,355✔
64
  if (offset < TSDB_FHDR_SIZE) {
1,367,701,088!
65
    tsdbError("vgId:%d %s failed at %s:%d since file size is too small: %" PRId64 " fname:%s",
×
66
              TD_VID(config->tsdb->pVnode), __func__, __FILE__, __LINE__, config->file->size, reader[0]->fd->path);
67
    TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
×
68
  }
69

70
  int32_t encryptAlgoirthm = config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
1,367,701,088✔
71
  char   *encryptKey = config->tsdb->pVnode->config.tsdbCfg.encryptKey;
1,368,000,869✔
72
#if 1
73
  TAOS_CHECK_GOTO(tsdbReadFile(reader[0]->fd, offset, (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0,
1,367,952,117!
74
                               encryptAlgoirthm, encryptKey),
75
                  &lino, _exit);
76
#else
77
  int64_t size = config->file->size;
78

79
  for (; size > TSDB_FHDR_SIZE; size--) {
80
    code = tsdbReadFile(reader[0]->fd, size - sizeof(SSttFooter), (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0,
81
                        encryptAlgoirthm, encryptKey);
82
    if (code) continue;
83
    if ((*reader)->footer->sttBlkPtr->offset + (*reader)->footer->sttBlkPtr->size + sizeof(SSttFooter) == size ||
84
        (*reader)->footer->statisBlkPtr->offset + (*reader)->footer->statisBlkPtr->size + sizeof(SSttFooter) == size ||
85
        (*reader)->footer->tombBlkPtr->offset + (*reader)->footer->tombBlkPtr->size + sizeof(SSttFooter) == size) {
86
      break;
87
    }
88
  }
89
  if (size <= TSDB_FHDR_SIZE) {
90
    TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
91
  }
92
#endif
93

94
_exit:
1,367,099,642✔
95
  if (code) {
1,366,972,111!
96
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(config->tsdb->pVnode), __func__, __FILE__, lino,
×
97
              tstrerror(code));
98
    tsdbSttFileReaderClose(reader);
×
99
  }
100
  return code;
1,366,972,111✔
101
}
102

103
void tsdbSttFileReaderClose(SSttFileReader **reader) {
1,367,873,595✔
104
  if (reader[0]) {
1,367,873,595!
105
    for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->local); ++i) {
2,147,483,647✔
106
      tBufferDestroy(reader[0]->local + i);
2,147,483,647✔
107
    }
108
    tsdbCloseFile(&reader[0]->fd);
1,367,774,149✔
109
    TARRAY2_DESTROY(reader[0]->tombBlkArray, NULL);
1,367,875,916!
110
    TARRAY2_DESTROY(reader[0]->statisBlkArray, NULL);
1,367,811,561✔
111
    TARRAY2_DESTROY(reader[0]->sttBlkArray, NULL);
1,368,016,160✔
112
    taosMemoryFree(reader[0]);
1,368,046,141✔
113
    reader[0] = NULL;
1,367,923,471✔
114
  }
115
}
1,367,820,043✔
116

117
// SSttFSegReader
118
int32_t tsdbSttFileReadStatisBlk(SSttFileReader *reader, const TStatisBlkArray **statisBlkArray) {
1,358,545,307✔
119
  if (!reader->ctx->statisBlkLoaded) {
1,358,545,307✔
120
    if (reader->footer->statisBlkPtr->size > 0) {
1,359,038,369✔
121
      if (reader->footer->statisBlkPtr->size % sizeof(SStatisBlk) != 0) {
1,356,073,747!
122
        tsdbError("vgId:%d %s failed at %s:%d since stt file statis block size is not valid, fname:%s",
×
123
                  TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, __LINE__, reader->fd->path);
124
        return TSDB_CODE_FILE_CORRUPTED;
×
125
      }
126

127
      int32_t size = reader->footer->statisBlkPtr->size / sizeof(SStatisBlk);
1,355,620,263✔
128
      void   *data = taosMemoryMalloc(reader->footer->statisBlkPtr->size);
1,355,921,650✔
129
      if (!data) {
1,355,629,877!
130
        return terrno;
×
131
      }
132

133
      int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
1,355,629,877✔
134
      char   *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
1,356,195,171✔
135
      int32_t code = tsdbReadFile(reader->fd, reader->footer->statisBlkPtr->offset, data,
1,356,279,200✔
136
                                  reader->footer->statisBlkPtr->size, 0, encryptAlgorithm, encryptKey);
137
      if (code) {
1,356,091,269!
138
        taosMemoryFree(data);
×
139
        return code;
×
140
      }
141

142
      TARRAY2_INIT_EX(reader->statisBlkArray, size, size, data);
1,356,091,269✔
143
    } else {
144
      TARRAY2_INIT(reader->statisBlkArray);
2,973,856✔
145
    }
146

147
    reader->ctx->statisBlkLoaded = true;
1,359,153,448✔
148
  }
149

150
  statisBlkArray[0] = reader->statisBlkArray;
1,358,971,574✔
151
  return 0;
1,359,310,992✔
152
}
153

154
int32_t tsdbSttFileReadTombBlk(SSttFileReader *reader, const TTombBlkArray **tombBlkArray) {
1,367,347,935✔
155
  if (!reader->ctx->tombBlkLoaded) {
1,367,347,935✔
156
    if (reader->footer->tombBlkPtr->size > 0) {
1,367,706,843✔
157
      if (reader->footer->tombBlkPtr->size % sizeof(STombBlk) != 0) {
36,649,295!
158
        tsdbError("vgId:%d %s failed at %s:%d since stt file tomb block size is not valid, fname:%s",
×
159
                  TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, __LINE__, reader->fd->path);
160
        return TSDB_CODE_FILE_CORRUPTED;
×
161
      }
162

163
      int32_t size = reader->footer->tombBlkPtr->size / sizeof(STombBlk);
36,646,247✔
164
      void   *data = taosMemoryMalloc(reader->footer->tombBlkPtr->size);
36,649,295!
165
      if (!data) {
36,647,771!
166
        return terrno;
×
167
      }
168

169
      int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
36,647,771✔
170
      char   *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
36,650,819✔
171
      int32_t code = tsdbReadFile(reader->fd, reader->footer->tombBlkPtr->offset, data,
36,646,247✔
172
                                  reader->footer->tombBlkPtr->size, 0, encryptAlgorithm, encryptKey);
173
      if (code) {
36,649,724!
174
        taosMemoryFree(data);
×
175
        return code;
×
176
      }
177

178
      TARRAY2_INIT_EX(reader->tombBlkArray, size, size, data);
36,649,724✔
179
    } else {
180
      TARRAY2_INIT(reader->tombBlkArray);
1,330,506,402✔
181
    }
182

183
    reader->ctx->tombBlkLoaded = true;
1,367,804,093✔
184
  }
185

186
  tombBlkArray[0] = reader->tombBlkArray;
1,367,501,282✔
187
  return 0;
1,367,932,353✔
188
}
189

190
int32_t tsdbSttFileReadSttBlk(SSttFileReader *reader, const TSttBlkArray **sttBlkArray) {
1,367,343,105✔
191
  if (!reader->ctx->sttBlkLoaded) {
1,367,343,105✔
192
    if (reader->footer->sttBlkPtr->size > 0) {
1,367,567,322✔
193
      if (reader->footer->sttBlkPtr->size % sizeof(SSttBlk) != 0) {
1,364,477,123!
194
        tsdbError("vgId:%d %s failed at %s:%d since stt file stt block size is not valid, fname:%s",
×
195
                  TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, __LINE__, reader->fd->path);
196
        return TSDB_CODE_FILE_CORRUPTED;
×
197
      }
198

199
      int32_t size = reader->footer->sttBlkPtr->size / sizeof(SSttBlk);
1,363,648,859✔
200
      void   *data = taosMemoryMalloc(reader->footer->sttBlkPtr->size);
1,364,185,414✔
201
      if (!data) {
1,363,793,275!
202
        return terrno;
×
203
      }
204

205
      int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
1,363,793,275✔
206
      char   *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
1,364,510,488✔
207
      int32_t code = tsdbReadFile(reader->fd, reader->footer->sttBlkPtr->offset, data, reader->footer->sttBlkPtr->size,
1,364,647,979✔
208
                                  0, encryptAlgorithm, encryptKey);
209
      if (code) {
1,364,517,507!
210
        taosMemoryFree(data);
×
211
        return code;
×
212
      }
213

214
      TARRAY2_INIT_EX(reader->sttBlkArray, size, size, data);
1,364,517,507✔
215
    } else {
216
      TARRAY2_INIT(reader->sttBlkArray);
3,381,564✔
217
    }
218

219
    reader->ctx->sttBlkLoaded = true;
1,367,739,479✔
220
  }
221

222
  sttBlkArray[0] = reader->sttBlkArray;
1,367,988,010✔
223
  return 0;
1,367,976,622✔
224
}
225

226
int32_t tsdbSttFileReadBlockData(SSttFileReader *reader, const SSttBlk *sttBlk, SBlockData *bData) {
29,001,088✔
227
  int32_t code = 0;
29,001,088✔
228
  int32_t lino = 0;
29,001,088✔
229

230
  SBuffer *buffer0 = reader->buffers + 0;
29,001,088✔
231
  SBuffer *assist = reader->buffers + 1;
29,002,610✔
232

233
  int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
29,000,264✔
234
  char   *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
29,000,261✔
235
  // load data
236
  tBufferClear(buffer0);
237
  TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szBlock, buffer0, 0,
29,003,306!
238
                                       encryptAlgorithm, encryptKey),
239
                  &lino, _exit);
240

241
  SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0);
29,003,306✔
242
  TAOS_CHECK_GOTO(tBlockDataDecompress(&br, bData, assist), &lino, _exit);
29,003,306!
243

244
_exit:
29,000,263✔
245
  if (code) {
29,000,263!
246
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino,
×
247
              tstrerror(code));
248
  }
249
  return code;
29,000,263✔
250
}
251

252
int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *sttBlk, SBlockData *bData,
690,329,942✔
253
                                         STSchema *pTSchema, int16_t cids[], int32_t ncid) {
254
  int32_t code = 0;
690,329,942✔
255
  int32_t lino = 0;
690,329,942✔
256

257
  SDiskDataHdr hdr;
690,318,306✔
258
  SBuffer     *buffer0 = reader->buffers + 0;
690,450,723✔
259
  SBuffer     *buffer1 = reader->buffers + 1;
690,610,455✔
260
  SBuffer     *assist = reader->buffers + 2;
690,613,946✔
261

262
  int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
690,077,974✔
263
  char   *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
690,730,436✔
264
  // load key part
265
  tBufferClear(buffer0);
266
  TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szKey, buffer0, 0,
690,492,754!
267
                                       encryptAlgorithm, encryptKey),
268
                  &lino, _exit);
269

270
  // decode header
271
  SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0);
690,947,205✔
272
  TAOS_CHECK_GOTO(tGetDiskDataHdr(&br, &hdr), &lino, _exit);
690,972,209!
273

274
  if (hdr.delimiter != TSDB_FILE_DLMT) {
690,194,136!
275
    tsdbError("vgId:%d %s failed at %s:%d since disk data header delimiter is invalid, fname:%s",
×
276
              TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, __LINE__, reader->fd->path);
277
    TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
×
278
  }
279

280
  // set data container
281
  tBlockDataReset(bData);
690,194,136✔
282
  bData->suid = hdr.suid;
690,392,562✔
283
  bData->uid = (sttBlk->suid == 0) ? sttBlk->minUid : 0;
690,801,775✔
284
  bData->nRow = hdr.nRow;
690,543,788✔
285

286
  // key part
287
  TAOS_CHECK_GOTO(tBlockDataDecompressKeyPart(&hdr, &br, bData, assist), &lino, _exit);
690,261,963!
288
  if (br.offset != buffer0->size) {
690,626,190!
289
    tsdbError("vgId:%d %s failed at %s:%d since key part size mismatch, expected:%u, actual:%u, fname:%s",
×
290
              TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, __LINE__, buffer0->size, br.offset,
291
              reader->fd->path);
292
    TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
×
293
  }
294

295
  bool loadExtra = false;
690,498,679✔
296
  for (int i = 0; i < ncid; i++) {
805,643,499✔
297
    if (tBlockDataGetColData(bData, cids[i]) == NULL) {
591,325,559✔
298
      loadExtra = true;
476,206,023✔
299
      break;
476,206,023✔
300
    }
301
  }
302

303
  if (!loadExtra) {
690,523,963✔
304
    goto _exit;
214,342,669✔
305
  }
306

307
  // load SBlockCol part
308
  tBufferClear(buffer0);
309
  TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey, hdr.szBlkCol, buffer0, 0,
476,198,566!
310
                                       encryptAlgorithm, encryptKey),
311
                  &lino, _exit);
312

313
  // load each column
314
  SBlockCol blockCol = {
476,444,025✔
315
      .cid = 0,
316
  };
317
  br = BUFFER_READER_INITIALIZER(0, buffer0);
476,468,808✔
318
  for (int32_t i = 0; i < ncid; i++) {
2,147,483,647✔
319
    int16_t cid = cids[i];
1,923,385,120✔
320

321
    if (tBlockDataGetColData(bData, cid)) {  // already loaded
1,923,464,703✔
322
      continue;
73,735,736✔
323
    }
324

325
    while (cid > blockCol.cid) {
2,147,483,647✔
326
      if (br.offset >= buffer0->size) {
2,147,483,647✔
327
        blockCol.cid = INT16_MAX;
49,424,401✔
328
        break;
49,424,401✔
329
      }
330

331
      TAOS_CHECK_GOTO(tGetBlockCol(&br, &blockCol, hdr.fmtVer, hdr.cmprAlg), &lino, _exit);
2,147,483,647!
332
    }
333

334
    if (cid < blockCol.cid) {
1,850,279,998✔
335
      const STColumn *tcol = tTSchemaSearchColumn(pTSchema, cid);
320,047,997✔
336
      TSDB_CHECK_NULL(tcol, code, lino, _exit, TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER);
320,044,042!
337
      SBlockCol none = {
320,044,042✔
338
          .cid = cid,
339
          .type = tcol->type,
320,023,728✔
340
          .cflag = tcol->flags,
320,037,166✔
341
          .flag = HAS_NONE,
342
          .szOrigin = 0,
343
          .szBitmap = 0,
344
          .szOffset = 0,
345
          .szValue = 0,
346
          .offset = 0,
347
      };
348
      TAOS_CHECK_GOTO(tBlockDataDecompressColData(&hdr, &none, &br, bData, assist), &lino, _exit);
320,033,609!
349
    } else if (cid == blockCol.cid) {
1,530,232,001✔
350
      // load from file
351
      tBufferClear(buffer1);
352
      TAOS_CHECK_GOTO(
1,530,103,880!
353
          tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey + hdr.szBlkCol + blockCol.offset,
354
                               blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, 0, encryptAlgorithm,
355
                               encryptKey),
356
          &lino, _exit);
357

358
      // decode the buffer
359
      SBufferReader br1 = BUFFER_READER_INITIALIZER(0, buffer1);
1,530,295,207✔
360
      TAOS_CHECK_GOTO(tBlockDataDecompressColData(&hdr, &blockCol, &br1, bData, assist), &lino, _exit);
1,530,305,385!
361
    }
362
  }
363

364
_exit:
691,589,072✔
365
  if (code) {
690,868,357!
366
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino,
×
367
              tstrerror(code));
368
  }
369
  return code;
690,868,357✔
370
}
371

372
int32_t tsdbSttFileReadTombBlock(SSttFileReader *reader, const STombBlk *tombBlk, STombBlock *tombBlock) {
17,567,042✔
373
  int32_t code = 0;
17,567,042✔
374
  int32_t lino = 0;
17,567,042✔
375

376
  SBuffer *buffer0 = reader->buffers + 0;
17,567,042✔
377
  SBuffer *assist = reader->buffers + 1;
17,568,137✔
378

379
  int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
17,568,137✔
380
  char   *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
17,567,042✔
381
  // load
382
  tBufferClear(buffer0);
383
  TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, tombBlk->dp->offset, tombBlk->dp->size, buffer0, 0, encryptAlgorithm,
17,567,042!
384
                                       encryptKey),
385
                  &lino, _exit);
386

387
  // decode
388
  int32_t       size = 0;
17,568,137✔
389
  SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0);
17,568,137✔
390
  tTombBlockClear(tombBlock);
17,568,137✔
391
  tombBlock->numOfRecords = tombBlk->numRec;
17,567,042✔
392
  for (int32_t i = 0; i < ARRAY_SIZE(tombBlock->buffers); ++i) {
105,408,822✔
393
    SCompressInfo cinfo = {
87,840,685✔
394
        .cmprAlg = tombBlk->cmprAlg,
87,840,685✔
395
        .dataType = TSDB_DATA_TYPE_BIGINT,
396
        .originalSize = tombBlk->numRec * sizeof(int64_t),
87,840,685✔
397
        .compressedSize = tombBlk->size[i],
87,840,685✔
398
    };
399
    TAOS_CHECK_GOTO(tDecompressDataToBuffer(BR_PTR(&br), &cinfo, tombBlock->buffers + i, assist), &lino, _exit);
87,840,685!
400
    br.offset += tombBlk->size[i];
87,840,685✔
401
  }
402

403
  if (br.offset != tombBlk->dp->size) {
17,568,137!
404
    tsdbError("vgId:%d %s failed at %s:%d since tomb block size mismatch, fname:%s",
×
405
              TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, __LINE__, reader->fd->path);
406
    TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
×
407
  }
408

409
_exit:
17,568,137✔
410
  if (code) {
17,568,137!
411
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino,
×
412
              tstrerror(code));
413
  }
414
  return code;
17,568,137✔
415
}
416

417
int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *statisBlk, STbStatisBlock *statisBlock) {
851,282,771✔
418
  int32_t code = 0;
851,282,771✔
419
  int32_t lino = 0;
851,282,771✔
420

421
  SBuffer *buffer0 = reader->buffers + 0;
851,376,669✔
422
  SBuffer *assist = reader->buffers + 1;
851,281,021✔
423

424
  int32_t encryptAlgorithm = reader->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
851,491,898✔
425
  char   *encryptKey = reader->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
851,402,907✔
426
  // load data
427
  tBufferClear(buffer0);
428
  TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, statisBlk->dp->offset, statisBlk->dp->size, buffer0, 0,
851,524,446!
429
                                       encryptAlgorithm, encryptKey),
430
                  &lino, _exit);
431

432
  // decode data
433
  tStatisBlockClear(statisBlock);
851,645,096✔
434
  statisBlock->numOfPKs = statisBlk->numOfPKs;
851,515,532✔
435
  statisBlock->numOfRecords = statisBlk->numRec;
851,372,682✔
436
  SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0);
851,624,539✔
437
  for (int32_t i = 0; i < ARRAY_SIZE(statisBlock->buffers); ++i) {
2,147,483,647✔
438
    SCompressInfo info = {
2,147,483,647✔
439
        .dataType = TSDB_DATA_TYPE_BIGINT,
440
        .cmprAlg = statisBlk->cmprAlg,
2,147,483,647✔
441
        .compressedSize = statisBlk->size[i],
2,147,483,647✔
442
        .originalSize = statisBlk->numRec * sizeof(int64_t),
2,147,483,647✔
443
    };
444

445
    TAOS_CHECK_GOTO(tDecompressDataToBuffer(BR_PTR(&br), &info, &statisBlock->buffers[i], assist), &lino, _exit);
2,147,483,647!
446
    br.offset += statisBlk->size[i];
2,147,483,647✔
447
  }
448

449
  if (statisBlk->numOfPKs > 0) {
851,912,107✔
450
    SValueColumnCompressInfo firstKeyInfos[TD_MAX_PK_COLS];
179,834,342✔
451
    SValueColumnCompressInfo lastKeyInfos[TD_MAX_PK_COLS];
179,831,252✔
452

453
    // decode compress info
454
    for (int32_t i = 0; i < statisBlk->numOfPKs; i++) {
359,652,930✔
455
      TAOS_CHECK_GOTO(tValueColumnCompressInfoDecode(&br, &firstKeyInfos[i]), &lino, _exit);
179,820,834!
456
    }
457

458
    for (int32_t i = 0; i < statisBlk->numOfPKs; i++) {
359,661,696✔
459
      TAOS_CHECK_GOTO(tValueColumnCompressInfoDecode(&br, &lastKeyInfos[i]), &lino, _exit);
179,797,163!
460
    }
461

462
    // decode value columns
463
    for (int32_t i = 0; i < statisBlk->numOfPKs; i++) {
359,611,948✔
464
      TAOS_CHECK_GOTO(tValueColumnDecompress(BR_PTR(&br), firstKeyInfos + i, &statisBlock->firstKeyPKs[i], assist),
179,813,873!
465
                      &lino, _exit);
466
      br.offset += (firstKeyInfos[i].dataCompressedSize + firstKeyInfos[i].offsetCompressedSize);
179,801,012✔
467
    }
468

469
    for (int32_t i = 0; i < statisBlk->numOfPKs; i++) {
359,569,375✔
470
      TAOS_CHECK_GOTO(tValueColumnDecompress(BR_PTR(&br), &lastKeyInfos[i], &statisBlock->lastKeyPKs[i], assist), &lino,
179,784,969!
471
                      _exit);
472
      br.offset += (lastKeyInfos[i].dataCompressedSize + lastKeyInfos[i].offsetCompressedSize);
179,824,865✔
473
    }
474
  }
475

476
  if (br.offset != buffer0->size) {
851,890,119!
477
    tsdbError("vgId:%d %s failed at %s:%d since statis block size mismatch, expected: %u, actual: %u, fname:%s",
×
478
              TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, __LINE__, buffer0->size, br.offset,
479
              reader->fd->path);
480
    TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
×
481
  }
482

483
_exit:
851,714,165✔
484
  if (code) {
851,753,057!
485
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino,
×
486
              tstrerror(code));
487
  }
488
  return code;
851,753,057✔
489
}
490

491
// SSttFWriter ============================================================
492
struct SSttFileWriter {
493
  SSttFileWriterConfig config[1];
494
  struct {
495
    bool    opened;
496
    TABLEID tbid[1];
497
    // range
498
    SVersionRange range;
499
  } ctx[1];
500
  // file
501
  STsdbFD *fd;
502
  STFile   file[1];
503
  // data
504
  SSttFooter      footer[1];
505
  TTombBlkArray   tombBlkArray[1];
506
  TSttBlkArray    sttBlkArray[1];
507
  TStatisBlkArray statisBlkArray[1];
508
  STombBlock      tombBlock[1];
509
  STbStatisBlock  staticBlock[1];
510
  SBlockData      blockData[1];
511
  // helper data
512
  SSkmInfo skmTb[1];
513
  SSkmInfo skmRow[1];
514
  SBuffer  local[10];
515
  SBuffer *buffers;
516
  // SColCompressInfo2 pInfo;
517
};
518

519
static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, SColCompressInfo *info,
169,005,042✔
520
                                           int64_t *fileSize, TSttBlkArray *sttBlkArray, SBuffer *buffers,
521
                                           SVersionRange *range, int32_t encryptAlgorithm, char *encryptKey) {
522
  if (blockData->nRow == 0) return 0;
169,005,042!
523

524
  int32_t code = 0;
169,003,757✔
525

526
  SSttBlk sttBlk[1] = {{
169,038,103✔
527
      .suid = blockData->suid,
169,020,696✔
528
      .minUid = blockData->uid ? blockData->uid : blockData->aUid[0],
169,025,346✔
529
      .maxUid = blockData->uid ? blockData->uid : blockData->aUid[blockData->nRow - 1],
169,054,769✔
530
      .minKey = blockData->aTSKEY[0],
169,000,527✔
531
      .maxKey = blockData->aTSKEY[0],
169,018,394✔
532
      .minVer = blockData->aVersion[0],
169,055,271✔
533
      .maxVer = blockData->aVersion[0],
169,041,422✔
534
      .nRow = blockData->nRow,
169,049,055✔
535
  }};
536

537
  for (int32_t iRow = 1; iRow < blockData->nRow; iRow++) {
2,147,483,647✔
538
    if (sttBlk->minKey > blockData->aTSKEY[iRow]) sttBlk->minKey = blockData->aTSKEY[iRow];
2,147,483,647✔
539
    if (sttBlk->maxKey < blockData->aTSKEY[iRow]) sttBlk->maxKey = blockData->aTSKEY[iRow];
2,147,483,647✔
540
    if (sttBlk->minVer > blockData->aVersion[iRow]) sttBlk->minVer = blockData->aVersion[iRow];
2,147,483,647✔
541
    if (sttBlk->maxVer < blockData->aVersion[iRow]) sttBlk->maxVer = blockData->aVersion[iRow];
2,147,483,647✔
542
  }
543

544
  tsdbWriterUpdVerRange(range, sttBlk->minVer, sttBlk->maxVer);
169,067,706✔
545
  TAOS_CHECK_RETURN(tBlockDataCompress(blockData, info, buffers, buffers + 4));
169,032,275!
546

547
  sttBlk->bInfo.offset = *fileSize;
168,962,386✔
548
  sttBlk->bInfo.szKey = buffers[0].size + buffers[1].size;
168,980,393✔
549
  sttBlk->bInfo.szBlock = buffers[2].size + buffers[3].size + sttBlk->bInfo.szKey;
169,013,115✔
550
  for (int i = 0; i < 4; i++) {
845,198,378✔
551
    if (buffers[i].size) {
676,117,239✔
552
      TAOS_CHECK_RETURN(tsdbWriteFile(fd, *fileSize, buffers[i].data, buffers[i].size, encryptAlgorithm, encryptKey));
675,474,020!
553
      *fileSize += buffers[i].size;
675,498,778✔
554
    }
555
  }
556

557
  TAOS_CHECK_RETURN(TARRAY2_APPEND_PTR(sttBlkArray, sttBlk));
338,133,306!
558

559
  tBlockDataClear(blockData);
169,052,167✔
560
  return 0;
169,065,863✔
561
}
562

563
static int32_t tsdbSttFileDoWriteBlockData(SSttFileWriter *writer) {
302,998,179✔
564
  if (writer->blockData->nRow == 0) return 0;
302,998,179✔
565

566
  int32_t code = 0;
169,047,199✔
567
  int32_t lino = 0;
169,047,199✔
568

569
  tb_uid_t         uid = writer->blockData->suid == 0 ? writer->blockData->uid : writer->blockData->suid;
169,052,408✔
570
  SColCompressInfo info = {.defaultCmprAlg = writer->config->cmprAlg, .pColCmpr = NULL};
169,054,797✔
571
  code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, uid, &(info.pColCmpr));
169,044,835✔
572

573
  int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
169,052,045✔
574
  char   *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
169,089,417✔
575
  TAOS_CHECK_GOTO(
169,088,345!
576
      tsdbFileDoWriteSttBlockData(writer->fd, writer->blockData, &info, &writer->file->size, writer->sttBlkArray,
577
                                  writer->buffers, &writer->ctx->range, encryptAlgorithm, encryptKey),
578
      &lino, _exit);
579

580
_exit:
169,020,631✔
581
  if (code) {
169,058,809!
582
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
583
              tstrerror(code));
584
  }
585
  taosHashCleanup(info.pColCmpr);
169,058,809✔
586
  return code;
168,994,292✔
587
}
588

589
static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) {
135,451,217✔
590
  if (writer->staticBlock->numOfRecords == 0) {
135,451,217✔
591
    return 0;
4,257,028✔
592
  }
593

594
  int32_t code = 0;
131,216,750✔
595
  int32_t lino = 0;
131,216,750✔
596

597
  SBuffer *buffer0 = writer->buffers + 0;
131,228,930✔
598
  SBuffer *buffer1 = writer->buffers + 1;
131,230,318✔
599
  SBuffer *assist = writer->buffers + 2;
131,215,788✔
600

601
  STbStatisRecord record;
131,214,785✔
602
  STbStatisBlock *statisBlock = writer->staticBlock;
131,226,568✔
603
  SStatisBlk      statisBlk = {0};
131,217,628✔
604

605
  statisBlk.dp->offset = writer->file->size;
131,198,410✔
606
  statisBlk.dp->size = 0;
131,206,409✔
607
  statisBlk.numRec = statisBlock->numOfRecords;
131,206,409✔
608
  statisBlk.cmprAlg = writer->config->cmprAlg;
131,215,206✔
609
  statisBlk.numOfPKs = statisBlock->numOfPKs;
131,204,336✔
610

611
  code = tStatisBlockGet(statisBlock, 0, &record);
131,216,092✔
612
  TSDB_CHECK_CODE(code, lino, _exit);
131,210,092!
613
  statisBlk.minTbid.suid = record.suid;
131,210,092✔
614
  statisBlk.minTbid.uid = record.uid;
131,210,092✔
615

616
  code = tStatisBlockGet(statisBlock, statisBlock->numOfRecords - 1, &record);
131,210,092✔
617
  TSDB_CHECK_CODE(code, lino, _exit);
131,245,555!
618
  statisBlk.maxTbid.suid = record.suid;
131,245,555✔
619
  statisBlk.maxTbid.uid = record.uid;
131,245,555✔
620

621
  int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
131,245,555✔
622
  char   *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
131,248,886✔
623

624
  // compress each column
625
  for (int32_t i = 0; i < ARRAY_SIZE(statisBlk.size); i++) {
787,426,354✔
626
    SCompressInfo info = {
656,170,074✔
627
        .dataType = TSDB_DATA_TYPE_BIGINT,
628
        .cmprAlg = statisBlk.cmprAlg,
656,145,504✔
629
        .originalSize = statisBlock->buffers[i].size,
656,145,504✔
630
    };
631

632
    tBufferClear(buffer0);
633
    TAOS_CHECK_GOTO(tCompressDataToBuffer(statisBlock->buffers[i].data, &info, buffer0, assist), &lino, _exit);
656,140,292!
634
    TAOS_CHECK_GOTO(
656,166,694!
635
        tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, info.compressedSize, encryptAlgorithm, encryptKey),
636
        &lino, _exit);
637

638
    statisBlk.size[i] = info.compressedSize;
656,256,851✔
639
    statisBlk.dp->size += info.compressedSize;
656,240,679✔
640
    writer->file->size += info.compressedSize;
656,240,679✔
641
  }
642

643
  // compress primary keys
644
  if (statisBlk.numOfPKs > 0) {
131,256,280✔
645
    SValueColumnCompressInfo compressInfo = {.cmprAlg = statisBlk.cmprAlg};
2,986,546✔
646

647
    tBufferClear(buffer0);
648
    tBufferClear(buffer1);
649

650
    for (int32_t i = 0; i < statisBlk.numOfPKs; i++) {
5,971,500✔
651
      TAOS_CHECK_GOTO(tValueColumnCompress(&statisBlock->firstKeyPKs[i], &compressInfo, buffer1, assist), &lino, _exit);
2,986,546!
652
      TAOS_CHECK_GOTO(tValueColumnCompressInfoEncode(&compressInfo, buffer0), &lino, _exit);
2,985,998!
653
    }
654

655
    for (int32_t i = 0; i < statisBlk.numOfPKs; i++) {
5,971,500✔
656
      TAOS_CHECK_GOTO(tValueColumnCompress(&statisBlock->lastKeyPKs[i], &compressInfo, buffer1, assist), &lino, _exit);
2,982,782!
657
      TAOS_CHECK_GOTO(tValueColumnCompressInfoEncode(&compressInfo, buffer0), &lino, _exit);
2,986,546!
658
    }
659

660
    TAOS_CHECK_GOTO(
2,988,718!
661
        tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, buffer0->size, encryptAlgorithm, encryptKey),
662
        &lino, _exit);
663
    writer->file->size += buffer0->size;
2,985,985✔
664
    statisBlk.dp->size += buffer0->size;
2,985,437✔
665

666
    TAOS_CHECK_GOTO(
2,985,998!
667
        tsdbWriteFile(writer->fd, writer->file->size, buffer1->data, buffer1->size, encryptAlgorithm, encryptKey),
668
        &lino, _exit);
669
    writer->file->size += buffer1->size;
2,985,989✔
670
    statisBlk.dp->size += buffer1->size;
2,984,925✔
671
  }
672

673
  TAOS_CHECK_GOTO(TARRAY2_APPEND_PTR(writer->statisBlkArray, &statisBlk), &lino, _exit);
262,479,048!
674

675
  tStatisBlockClear(writer->staticBlock);
131,224,389✔
676

677
_exit:
131,198,334✔
678
  if (code) {
131,247,048!
679
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
680
              tstrerror(code));
681
  }
682
  return code;
131,211,362✔
683
}
684

685
static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) {
130,222,332✔
686
  if (TOMB_BLOCK_SIZE(writer->tombBlock) == 0) {
130,222,332✔
687
    return 0;
125,440,642✔
688
  }
689

690
  int32_t code = 0;
4,815,026✔
691
  int32_t lino = 0;
4,815,026✔
692

693
  int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
4,815,026✔
694
  char   *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
4,815,026✔
695

696
  TAOS_CHECK_GOTO(
4,815,026!
697
      tsdbFileWriteTombBlock(writer->fd, writer->tombBlock, writer->config->cmprAlg, &writer->file->size,
698
                             writer->tombBlkArray, writer->buffers, &writer->ctx->range, encryptAlgorithm, encryptKey),
699
      &lino, _exit);
700

701
_exit:
4,815,026✔
702
  if (code) {
4,815,026!
703
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
704
              tstrerror(code));
705
  }
706
  return code;
4,815,026✔
707
}
708

709
int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize,
130,227,489✔
710
                            int32_t encryptAlgorithm, char *encryptKey) {
711
  ptr->size = TARRAY2_DATA_LEN(sttBlkArray);
130,227,489✔
712
  if (ptr->size > 0) {
130,235,312✔
713
    ptr->offset = *fileSize;
129,826,149✔
714

715
    TAOS_CHECK_RETURN(tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(sttBlkArray), ptr->size,
129,829,471!
716
                                    encryptAlgorithm, encryptKey));
717

718
    *fileSize += ptr->size;
129,839,913✔
719
  }
720
  return 0;
130,254,568✔
721
}
722

723
static int32_t tsdbSttFileDoWriteSttBlk(SSttFileWriter *writer) {
130,224,331✔
724
  int32_t code = 0;
130,224,331✔
725
  int32_t lino;
130,222,644✔
726

727
  int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
130,246,884✔
728
  char   *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
130,263,081✔
729

730
  TAOS_CHECK_GOTO(tsdbFileWriteSttBlk(writer->fd, writer->sttBlkArray, writer->footer->sttBlkPtr, &writer->file->size,
130,255,657!
731
                                      encryptAlgorithm, encryptKey),
732
                  &lino, _exit);
733

734
_exit:
130,240,380✔
735
  if (code) {
130,239,898!
736
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
737
              tstrerror(code));
738
  }
739
  return code;
130,239,898✔
740
}
741

742
static int32_t tsdbSttFileDoWriteStatisBlk(SSttFileWriter *writer) {
130,207,205✔
743
  int32_t code = 0;
130,207,205✔
744
  int32_t lino;
130,205,518✔
745
  int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
130,225,434✔
746
  char   *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
130,243,386✔
747

748
  writer->footer->statisBlkPtr->size = TARRAY2_DATA_LEN(writer->statisBlkArray);
130,210,217✔
749
  if (writer->footer->statisBlkPtr->size) {
130,196,986✔
750
    writer->footer->statisBlkPtr->offset = writer->file->size;
129,800,001✔
751
    TAOS_CHECK_GOTO(tsdbWriteFile(writer->fd, writer->file->size, (const uint8_t *)TARRAY2_DATA(writer->statisBlkArray),
129,815,807!
752
                                  writer->footer->statisBlkPtr->size, encryptAlgorithm, encryptKey),
753
                    &lino, _exit);
754
    writer->file->size += writer->footer->statisBlkPtr->size;
129,840,640✔
755
  }
756

757
_exit:
130,249,017✔
758
  if (code) {
130,251,512!
759
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
760
              tstrerror(code));
761
  }
762
  return code;
130,251,512✔
763
}
764

765
static int32_t tsdbSttFileDoWriteTombBlk(SSttFileWriter *writer) {
130,201,233✔
766
  int32_t code = 0;
130,201,233✔
767
  int32_t lino = 0;
130,201,233✔
768

769
  int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
130,217,623✔
770
  char   *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
130,239,361✔
771

772
  TAOS_CHECK_GOTO(tsdbFileWriteTombBlk(writer->fd, writer->tombBlkArray, writer->footer->tombBlkPtr,
130,220,393!
773
                                       &writer->file->size, encryptAlgorithm, encryptKey),
774
                  &lino, _exit);
775

776
_exit:
130,236,803✔
777
  if (code) {
130,243,262!
778
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
779
              tstrerror(code));
780
  }
781
  return code;
130,243,262✔
782
}
783

784
int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize, int32_t encryptAlgorithm,
130,190,416✔
785
                               char *encryptKey) {
786
  TAOS_CHECK_RETURN(
130,190,416!
787
      tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptAlgorithm, encryptKey));
788
  *fileSize += sizeof(*footer);
130,260,235✔
789
  return 0;
130,243,256✔
790
}
791

792
static int32_t tsdbSttFileDoWriteFooter(SSttFileWriter *writer) {
130,199,137✔
793
  int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
130,199,137✔
794
  char   *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
130,211,700✔
795

796
  return tsdbFileWriteSttFooter(writer->fd, writer->footer, &writer->file->size, encryptAlgorithm, encryptKey);
130,224,983✔
797
}
798

799
static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) {
129,965,487✔
800
  int32_t code = 0;
129,965,487✔
801
  int32_t lino = 0;
129,965,487✔
802
  STsdb  *tsdb = writer->config->tsdb;
130,033,394✔
803

804
  // set
805
  if (!writer->config->skmTb) writer->config->skmTb = writer->skmTb;
130,108,428!
806
  if (!writer->config->skmRow) writer->config->skmRow = writer->skmRow;
130,128,004!
807
  writer->buffers = writer->config->buffers;
130,112,003✔
808
  if (writer->buffers == NULL) {
130,114,115!
809
    writer->buffers = writer->local;
×
810
  }
811

812
  // alloc disk id
813
  SDiskID diskId = {0};
130,179,852✔
814
  code = tsdbAllocateDisk(tsdb, tsdbFTypeLabel(TSDB_FTYPE_STT), writer->config->expLevel, &diskId);
130,075,161✔
815
  TSDB_CHECK_CODE(code, lino, _exit);
130,146,673!
816

817
  writer->file[0] = (STFile){
130,169,297✔
818
      .type = TSDB_FTYPE_STT,
819
      .did = diskId,
820
      .fid = writer->config->fid,
130,146,673✔
821
      .cid = writer->config->cid,
130,154,127✔
822
      .size = 0,
823
      .minVer = VERSION_MAX,
824
      .maxVer = VERSION_MIN,
825
      .stt[0] =
826
          {
827
              .level = writer->config->level,
130,165,526✔
828
          },
829
  };
830

831
  // open file
832
  int32_t flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
130,092,646✔
833
  char    fname[TSDB_FILENAME_LEN];
130,090,959✔
834

835
  tsdbTFileName(writer->config->tsdb, writer->file, fname);
130,166,280✔
836
  TAOS_CHECK_GOTO(tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, 0), &lino, _exit);
130,070,113!
837

838
  uint8_t hdr[TSDB_FHDR_SIZE] = {0};
130,081,950✔
839
  int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
130,117,474✔
840
  char   *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
130,131,466✔
841

842
  TAOS_CHECK_GOTO(tsdbWriteFile(writer->fd, 0, hdr, sizeof(hdr), encryptAlgorithm, encryptKey), &lino, _exit);
130,211,383!
843
  writer->file->size += sizeof(hdr);
130,114,520✔
844

845
  // range
846
  writer->ctx->range = (SVersionRange){.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
130,230,595✔
847

848
  writer->ctx->opened = true;
130,217,278✔
849

850
_exit:
130,218,220✔
851
  if (code) {
130,188,883!
852
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
853
              tstrerror(code));
854
  }
855
  return code;
130,188,883✔
856
}
857

858
static void tsdbSttFWriterDoClose(SSttFileWriter *writer) {
130,205,456✔
859
  for (int32_t i = 0; i < ARRAY_SIZE(writer->local); ++i) {
1,430,398,054✔
860
    tBufferDestroy(writer->local + i);
1,300,332,494!
861
  }
862
  tDestroyTSchema(writer->skmRow->pTSchema);
130,065,560!
863
  tDestroyTSchema(writer->skmTb->pTSchema);
130,204,826!
864
  tTombBlockDestroy(writer->tombBlock);
130,191,621✔
865
  tStatisBlockDestroy(writer->staticBlock);
130,145,487✔
866
  tBlockDataDestroy(writer->blockData);
130,211,519✔
867
  TARRAY2_DESTROY(writer->tombBlkArray, NULL);
130,268,443!
868
  TARRAY2_DESTROY(writer->statisBlkArray, NULL);
130,236,120!
869
  TARRAY2_DESTROY(writer->sttBlkArray, NULL);
130,230,522!
870
}
130,247,521✔
871

872
static int32_t tsdbSttFileDoUpdateHeader(SSttFileWriter *writer) {
130,216,138✔
873
  // TODO
874
  return 0;
130,216,138✔
875
}
876

877
static int32_t tsdbSttFWriterCloseCommit(SSttFileWriter *writer, TFileOpArray *opArray) {
130,226,516✔
878
  int32_t lino;
130,224,829✔
879
  int32_t code;
880

881
  TAOS_CHECK_GOTO(tsdbSttFileDoWriteBlockData(writer), &lino, _exit);
130,222,835!
882
  TAOS_CHECK_GOTO(tsdbSttFileDoWriteStatisBlock(writer), &lino, _exit);
130,228,884!
883
  TAOS_CHECK_GOTO(tsdbSttFileDoWriteTombBlock(writer), &lino, _exit);
130,255,577!
884
  TAOS_CHECK_GOTO(tsdbSttFileDoWriteSttBlk(writer), &lino, _exit);
130,233,392!
885
  TAOS_CHECK_GOTO(tsdbSttFileDoWriteStatisBlk(writer), &lino, _exit);
130,190,529!
886
  TAOS_CHECK_GOTO(tsdbSttFileDoWriteTombBlk(writer), &lino, _exit);
130,204,879!
887
  TAOS_CHECK_GOTO(tsdbSttFileDoWriteFooter(writer), &lino, _exit);
130,206,604!
888
  TAOS_CHECK_GOTO(tsdbSttFileDoUpdateHeader(writer), &lino, _exit);
130,246,381!
889

890
  int32_t encryptAlgorithm = writer->config->tsdb->pVnode->config.tsdbCfg.encryptAlgorithm;
130,215,131✔
891
  char   *encryptKey = writer->config->tsdb->pVnode->config.tsdbCfg.encryptKey;
130,217,174✔
892

893
  TAOS_CHECK_GOTO(tsdbFsyncFile(writer->fd, encryptAlgorithm, encryptKey), &lino, _exit);
130,208,204!
894

895
  tsdbCloseFile(&writer->fd);
130,258,844✔
896

897
  STFileOp op = (STFileOp){
260,390,947✔
898
      .optype = TSDB_FOP_CREATE,
899
      .fid = writer->config->fid,
130,181,175✔
900
      .nf = writer->file[0],
901
  };
902
  tsdbTFileUpdVerRange(&op.nf, writer->ctx->range);
130,243,624✔
903

904
  TAOS_CHECK_GOTO(TARRAY2_APPEND(opArray, op), &lino, _exit);
260,301,956!
905

906
_exit:
130,096,665✔
907
  if (code) {
130,105,278!
908
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
909
              tstrerror(code));
910
  }
911
  return code;
130,105,278✔
912
}
913

914
static int32_t tsdbSttFWriterCloseAbort(SSttFileWriter *writer) {
×
915
  char fname[TSDB_FILENAME_LEN];
×
916
  tsdbTFileName(writer->config->tsdb, writer->file, fname);
×
917
  tsdbCloseFile(&writer->fd);
×
918
  tsdbRemoveFile(fname);
×
919
  return 0;
×
920
}
921

922
int32_t tsdbSttFileWriterOpen(const SSttFileWriterConfig *config, SSttFileWriter **writer) {
133,495,902✔
923
  writer[0] = taosMemoryCalloc(1, sizeof(*writer[0]));
133,495,902!
924
  if (writer[0] == NULL) {
133,412,258!
925
    return terrno;
×
926
  }
927

928
  writer[0]->config[0] = config[0];
133,445,955✔
929
  writer[0]->ctx->opened = false;
133,470,906✔
930
  return 0;
133,549,430✔
931
}
932

933
int32_t tsdbSttFileWriterClose(SSttFileWriter **writer, int8_t abort, TFileOpArray *opArray) {
133,592,391✔
934
  int32_t code = 0;
133,592,391✔
935
  int32_t lino = 0;
133,592,391✔
936

937
  if (writer[0]->ctx->opened) {
133,606,572✔
938
    if (abort) {
130,232,248!
939
      TAOS_CHECK_GOTO(tsdbSttFWriterCloseAbort(writer[0]), &lino, _exit);
×
940
    } else {
941
      TAOS_CHECK_GOTO(tsdbSttFWriterCloseCommit(writer[0], opArray), &lino, _exit);
130,232,248!
942
    }
943
    tsdbSttFWriterDoClose(writer[0]);
130,131,654✔
944
  }
945
  taosMemoryFree(writer[0]);
133,594,330!
946
  writer[0] = NULL;
133,582,949✔
947

948
_exit:
133,584,688✔
949
  if (code) {
133,598,042!
950
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer[0]->config->tsdb->pVnode), __func__, __FILE__, lino,
×
951
              tstrerror(code));
952
  }
953
  return code;
133,598,042✔
954
}
955

956
int32_t tsdbSttFileWriteRow(SSttFileWriter *writer, SRowInfo *row) {
2,147,483,647✔
957
  int32_t code = 0;
2,147,483,647✔
958
  int32_t lino = 0;
2,147,483,647✔
959

960
  if (!writer->ctx->opened) {
2,147,483,647✔
961
    TAOS_CHECK_GOTO(tsdbSttFWriterDoOpen(writer), &lino, _exit);
129,736,019!
962
  }
963

964
  if (!TABLE_SAME_SCHEMA(row->suid, row->uid, writer->ctx->tbid->suid, writer->ctx->tbid->uid)) {
2,147,483,647✔
965
    TAOS_CHECK_GOTO(tsdbSttFileDoWriteBlockData(writer), &lino, _exit);
140,595,210!
966

967
    TAOS_CHECK_GOTO(tsdbUpdateSkmTb(writer->config->tsdb, (TABLEID *)row, writer->config->skmTb), &lino, _exit);
140,546,110!
968

969
    TABLEID id = {
140,484,464✔
970
        .suid = row->suid,
140,438,674✔
971
        .uid = row->suid ? 0 : row->uid,
140,472,070✔
972
    };
973
    TAOS_CHECK_GOTO(tBlockDataInit(writer->blockData, &id, writer->config->skmTb->pTSchema, NULL, 0), &lino, _exit);
140,417,230!
974
  }
975

976
  if (writer->ctx->tbid->uid != row->uid) {
2,147,483,647✔
977
    writer->ctx->tbid->suid = row->suid;
417,222,467✔
978
    writer->ctx->tbid->uid = row->uid;
417,172,875✔
979
  }
980

981
  STsdbRowKey key;
2,147,483,647✔
982
  tsdbRowGetKey(&row->row, &key);
2,147,483,647✔
983

984
  for (;;) {
985
    code = tStatisBlockPut(writer->staticBlock, row, writer->config->maxRow);
2,147,483,647✔
986
    if (code == TSDB_CODE_INVALID_PARA) {
2,147,483,647✔
987
      TAOS_CHECK_GOTO(tsdbSttFileDoWriteStatisBlock(writer), &lino, _exit);
815,707!
988
      continue;
816,223✔
989
    } else {
990
      TSDB_CHECK_CODE(code, lino, _exit);
2,147,483,647!
991
    }
992
    break;
2,147,483,647✔
993
  }
994

995
  if (row->row.type == TSDBROW_ROW_FMT) {
2,147,483,647✔
996
    TAOS_CHECK_GOTO(tsdbUpdateSkmRow(writer->config->tsdb, writer->ctx->tbid,  //
2,147,483,647!
997
                                     TSDBROW_SVERSION(&row->row), writer->config->skmRow),
998
                    &lino, _exit);
999
  }
1000

1001
  // row to col conversion
1002
  if (key.version <= writer->config->compactVersion                                //
2,147,483,647✔
1003
      && writer->blockData->nRow > 0                                               //
2,147,483,647✔
1004
      && (writer->blockData->uid                                                   //
2,147,483,647✔
1005
              ? writer->blockData->uid                                             //
1006
              : writer->blockData->aUid[writer->blockData->nRow - 1]) == row->uid  //
2,147,483,647✔
1007
      && tsdbRowCompareWithoutVersion(&row->row,
2,147,483,647✔
1008
                                      &tsdbRowFromBlockData(writer->blockData, writer->blockData->nRow - 1)) == 0  //
2,147,483,647✔
1009
  ) {
1010
    TAOS_CHECK_GOTO(tBlockDataUpdateRow(writer->blockData, &row->row, writer->config->skmRow->pTSchema), &lino, _exit);
391,797,797!
1011
  } else {
1012
    if (writer->blockData->nRow >= writer->config->maxRow) {
2,147,483,647✔
1013
      TAOS_CHECK_GOTO(tsdbSttFileDoWriteBlockData(writer), &lino, _exit);
27,883,181!
1014
    }
1015

1016
    TAOS_CHECK_GOTO(tBlockDataAppendRow(writer->blockData, &row->row, writer->config->skmRow->pTSchema, row->uid),
2,147,483,647!
1017
                    &lino, _exit);
1018
  }
1019

1020
_exit:
2,147,483,647✔
1021
  if (code) {
2,147,483,647!
1022
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
1023
              tstrerror(code));
1024
  }
1025
  return code;
2,147,483,647✔
1026
}
1027

1028
int32_t tsdbSttFileWriteBlockData(SSttFileWriter *writer, SBlockData *bdata) {
9,274,231✔
1029
  int32_t code = 0;
9,274,231✔
1030
  int32_t lino = 0;
9,274,231✔
1031

1032
  SRowInfo row[1];
9,272,354✔
1033
  row->suid = bdata->suid;
9,274,231✔
1034
  for (int32_t i = 0; i < bdata->nRow; i++) {
211,757,134✔
1035
    row->uid = bdata->uid ? bdata->uid : bdata->aUid[i];
202,482,552!
1036
    row->row = tsdbRowFromBlockData(bdata, i);
202,481,056✔
1037

1038
    TAOS_CHECK_GOTO(tsdbSttFileWriteRow(writer, row), &lino, _exit);
202,481,056!
1039
  }
1040

1041
_exit:
9,274,231✔
1042
  if (code) {
9,274,231!
1043
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
1044
              tstrerror(code));
1045
  }
1046
  return code;
9,274,231✔
1047
}
1048

1049
int32_t tsdbSttFileWriteTombRecord(SSttFileWriter *writer, const STombRecord *record) {
17,567,103✔
1050
  int32_t code;
1051
  int32_t lino;
17,567,103✔
1052

1053
  if (!writer->ctx->opened) {
17,567,103!
1054
    TAOS_CHECK_GOTO(tsdbSttFWriterDoOpen(writer), &lino, _exit);
407,708!
1055
  } else {
1056
    if (writer->blockData->nRow > 0) {
17,159,395✔
1057
      TAOS_CHECK_GOTO(tsdbSttFileDoWriteBlockData(writer), &lino, _exit);
4,407,318!
1058
    }
1059

1060
    if (STATIS_BLOCK_SIZE(writer->staticBlock) > 0) {
17,159,395✔
1061
      TAOS_CHECK_GOTO(tsdbSttFileDoWriteStatisBlock(writer), &lino, _exit);
4,407,318!
1062
    }
1063
  }
1064

1065
  TAOS_CHECK_GOTO(tTombBlockPut(writer->tombBlock, record), &lino, _exit);
17,567,103!
1066

1067
  if (TOMB_BLOCK_SIZE(writer->tombBlock) >= writer->config->maxRow) {
17,567,103!
1068
    TAOS_CHECK_GOTO(tsdbSttFileDoWriteTombBlock(writer), &lino, _exit);
×
1069
  }
1070

1071
_exit:
17,567,103✔
1072
  if (code) {
17,567,103!
1073
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
1074
              tstrerror(code));
1075
  } else {
1076
    tsdbTrace("vgId:%d write tomb record to stt file:%s, cid:%" PRId64 ", suid:%" PRId64 ", uid:%" PRId64
17,567,103✔
1077
              ", version:%" PRId64,
1078
              TD_VID(writer->config->tsdb->pVnode), writer->fd->path, writer->config->cid, record->suid, record->uid,
1079
              record->version);
1080
  }
1081
  return code;
17,567,103✔
1082
}
1083

1084
bool tsdbSttFileWriterIsOpened(SSttFileWriter *writer) { return writer->ctx->opened; }
11,884,928!
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc