• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

taosdata / TDengine / #5084

17 May 2026 01:15AM UTC coverage: 73.366% (-0.01%) from 73.377%
#5084

push

travis-ci

web-flow
feat (TDgpt): Dynamic Model Synchronization Enhancements (#35344)

* refactor: do some internal refactor.

* fix: fix multiprocess sync issue.

* feat: add dynamic anomaly detection and forecasting services

* fix: log error message for undeploying model in exception handling

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* fix: handle undeploy when model exists only on disk

Agent-Logs-Url: https://github.com/taosdata/TDengine/sessions/286aafa0-c3ce-4c27-b803-2707571e9dc1

Co-authored-by: hjxilinx <8252296+hjxilinx@users.noreply.github.com>

* fix: guard dynamic registry concurrent access

Agent-Logs-Url: https://github.com/taosdata/TDengine/sessions/5e4db858-6458-40f4-ac28-d1b1b7f97c18

Co-authored-by: hjxilinx <8252296+hjxilinx@users.noreply.github.com>

* fix: tighten service list locking scope

Agent-Logs-Url: https://github.com/taosdata/TDengine/sessions/5e4db858-6458-40f4-ac28-d1b1b7f97c18

Co-authored-by: hjxilinx <8252296+hjxilinx@users.noreply.github.com>

* fix: restore prophet support and update tests per review feedback

Agent-Logs-Url: https://github.com/taosdata/TDengine/sessions/92298ae1-7da6-4d07-b20e-101c7cd0b26b

Co-authored-by: hjxilinx <8252296+hjxilinx@users.noreply.github.com>

* fix: improve test name and move copy inside lock scope

Agent-Logs-Url: https://github.com/taosdata/TDengine/sessions/92298ae1-7da6-4d07-b20e-101c7cd0b26b

Co-authored-by: hjxilinx <8252296+hjxilinx@users.noreply.github.com>

* Potential fix for pull request finding

Co-au... (continued)

281574 of 383795 relevant lines covered (73.37%)

136040085.98 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.21
/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c
1
/*
2
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3
 *
4
 * This program is free software: you can use, redistribute, and/or modify
5
 * it under the terms of the GNU Affero General Public License, version 3
6
 * or later ("AGPL"), as published by the Free Software Foundation.
7
 *
8
 * This program is distributed in the hope that it will be useful, but WITHOUT
9
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10
 * FITNESS FOR A PARTICULAR PURPOSE.
11
 *
12
 * You should have received a copy of the GNU Affero General Public License
13
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
14
 */
15

16
#include "tsdbSttFileRW.h"
17
#include "meta.h"
18
#include "tsdbDataFileRW.h"
19

20
// SSttFReader ============================================================
21
struct SSttFileReader {
22
  SSttFileReaderConfig config[1];
23
  STsdbFD             *fd;
24
  SSttFooter           footer[1];
25
  struct {
26
    bool sttBlkLoaded;
27
    bool statisBlkLoaded;
28
    bool tombBlkLoaded;
29
  } ctx[1];
30
  TSttBlkArray    sttBlkArray[1];
31
  TStatisBlkArray statisBlkArray[1];
32
  TTombBlkArray   tombBlkArray[1];
33
  SBuffer         local[10];
34
  SBuffer        *buffers;
35
};
36

37
// SSttFileReader
38
int32_t tsdbSttFileReaderOpen(const char *fname, const SSttFileReaderConfig *config, SSttFileReader **reader) {
285,988,491✔
39
  int32_t code = 0;
285,988,491✔
40
  int32_t lino = 0;
285,988,491✔
41

42
  reader[0] = taosMemoryCalloc(1, sizeof(*reader[0]));
286,006,824✔
43
  if (reader[0] == NULL) {
285,851,777✔
44
    TAOS_CHECK_GOTO(terrno, &lino, _exit);
×
45
  }
46

47
  reader[0]->config[0] = config[0];
285,856,041✔
48
  reader[0]->buffers = config->buffers;
285,889,807✔
49
  if (reader[0]->buffers == NULL) {
285,971,917✔
50
    reader[0]->buffers = reader[0]->local;
285,993,060✔
51
  }
52

53
  // open file
54
  if (fname) {
285,959,555✔
55
    TAOS_CHECK_GOTO(tsdbOpenFile(fname, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0), &lino, _exit);
285,959,555✔
56
  } else {
57
    char fname1[TSDB_FILENAME_LEN];
×
58
    tsdbTFileName(config->tsdb, config->file, fname1);
×
59
    TAOS_CHECK_GOTO(tsdbOpenFile(fname1, config->tsdb, TD_FILE_READ, &reader[0]->fd, 0), &lino, _exit);
×
60
  }
61

62
  // // open each segment reader
63
  int64_t offset = config->file->size - sizeof(SSttFooter);
285,992,118✔
64
  if (offset < TSDB_FHDR_SIZE) {
285,893,589✔
65
    tsdbError("vgId:%d %s failed at %s:%d since file size is too small: %" PRId64 " fname:%s",
×
66
              TD_VID(config->tsdb->pVnode), __func__, __FILE__, __LINE__, config->file->size, reader[0]->fd->path);
67
    TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
×
68
  }
69

70
  SEncryptData *pEncryptData = &(config->tsdb->pVnode->config.tsdbCfg.encryptData);
285,893,589✔
71

72
#if 1
73
  TAOS_CHECK_GOTO(
285,975,455✔
74
      tsdbReadFile(reader[0]->fd, offset, (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0, pEncryptData), &lino,
75
      _exit);
76
#else
77
  int64_t size = config->file->size;
78

79
  for (; size > TSDB_FHDR_SIZE; size--) {
80
    code = tsdbReadFile(reader[0]->fd, size - sizeof(SSttFooter), (uint8_t *)(reader[0]->footer), sizeof(SSttFooter), 0,
81
                        encryptAlgoirthm, encryptKey);
82
    if (code) continue;
83
    if ((*reader)->footer->sttBlkPtr->offset + (*reader)->footer->sttBlkPtr->size + sizeof(SSttFooter) == size ||
84
        (*reader)->footer->statisBlkPtr->offset + (*reader)->footer->statisBlkPtr->size + sizeof(SSttFooter) == size ||
85
        (*reader)->footer->tombBlkPtr->offset + (*reader)->footer->tombBlkPtr->size + sizeof(SSttFooter) == size) {
86
      break;
87
    }
88
  }
89
  if (size <= TSDB_FHDR_SIZE) {
90
    TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
91
  }
92
#endif
93

94
_exit:
285,986,747✔
95
  if (code) {
285,983,484✔
96
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(config->tsdb->pVnode), __func__, __FILE__, lino,
×
97
              tstrerror(code));
98
    tsdbSttFileReaderClose(reader);
×
99
  }
100
  return code;
285,983,484✔
101
}
102

103
void tsdbSttFileReaderClose(SSttFileReader **reader) {
286,010,557✔
104
  if (reader[0]) {
286,010,557✔
105
    for (int32_t i = 0; i < ARRAY_SIZE(reader[0]->local); ++i) {
2,147,483,647✔
106
      tBufferDestroy(reader[0]->local + i);
2,147,483,647✔
107
    }
108
    tsdbCloseFile(&reader[0]->fd);
286,048,602✔
109
    TARRAY2_DESTROY(reader[0]->tombBlkArray, NULL);
286,003,874✔
110
    TARRAY2_DESTROY(reader[0]->statisBlkArray, NULL);
285,986,056✔
111
    TARRAY2_DESTROY(reader[0]->sttBlkArray, NULL);
286,008,223✔
112
    taosMemoryFree(reader[0]);
286,016,112✔
113
    reader[0] = NULL;
286,000,893✔
114
  }
115
}
285,990,650✔
116

117
// SSttFSegReader
118
int32_t tsdbSttFileReadStatisBlk(SSttFileReader *reader, const TStatisBlkArray **statisBlkArray) {
282,772,607✔
119
  if (!reader->ctx->statisBlkLoaded) {
282,772,607✔
120
    if (reader->footer->statisBlkPtr->size > 0) {
282,776,296✔
121
      if (reader->footer->statisBlkPtr->size % sizeof(SStatisBlk) != 0) {
282,599,310✔
122
        tsdbError("vgId:%d %s failed at %s:%d since stt file statis block size is not valid, fname:%s",
×
123
                  TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, __LINE__, reader->fd->path);
124
        return TSDB_CODE_FILE_CORRUPTED;
×
125
      }
126

127
      int32_t size = reader->footer->statisBlkPtr->size / sizeof(SStatisBlk);
282,567,526✔
128
      void   *data = taosMemoryMalloc(reader->footer->statisBlkPtr->size);
282,623,542✔
129
      if (!data) {
282,583,747✔
130
        return terrno;
×
131
      }
132

133
      SEncryptData *pEncryptData = &(reader->config->tsdb->pVnode->config.tsdbCfg.encryptData);
282,583,747✔
134

135
      int32_t code = tsdbReadFile(reader->fd, reader->footer->statisBlkPtr->offset, data,
282,614,417✔
136
                                  reader->footer->statisBlkPtr->size, 0, pEncryptData);
137
      if (code) {
282,611,624✔
138
        taosMemoryFree(data);
×
139
        return code;
×
140
      }
141

142
      TARRAY2_INIT_EX(reader->statisBlkArray, size, size, data);
282,611,624✔
143
    } else {
144
      TARRAY2_INIT(reader->statisBlkArray);
163,345✔
145
    }
146

147
    reader->ctx->statisBlkLoaded = true;
282,781,244✔
148
  }
149

150
  statisBlkArray[0] = reader->statisBlkArray;
282,784,349✔
151
  return 0;
282,773,885✔
152
}
153

154
int32_t tsdbSttFileReadTombBlk(SSttFileReader *reader, const TTombBlkArray **tombBlkArray) {
285,984,124✔
155
  if (!reader->ctx->tombBlkLoaded) {
285,984,124✔
156
    if (reader->footer->tombBlkPtr->size > 0) {
285,977,614✔
157
      if (reader->footer->tombBlkPtr->size % sizeof(STombBlk) != 0) {
13,050,596✔
158
        tsdbError("vgId:%d %s failed at %s:%d since stt file tomb block size is not valid, fname:%s",
×
159
                  TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, __LINE__, reader->fd->path);
160
        return TSDB_CODE_FILE_CORRUPTED;
×
161
      }
162

163
      int32_t size = reader->footer->tombBlkPtr->size / sizeof(STombBlk);
13,050,596✔
164
      void   *data = taosMemoryMalloc(reader->footer->tombBlkPtr->size);
13,050,596✔
165
      if (!data) {
13,050,596✔
166
        return terrno;
×
167
      }
168

169
      SEncryptData *pEncryptData = &(reader->config->tsdb->pVnode->config.tsdbCfg.encryptData);
13,050,596✔
170

171
      int32_t code = tsdbReadFile(reader->fd, reader->footer->tombBlkPtr->offset, data,
13,050,596✔
172
                                  reader->footer->tombBlkPtr->size, 0, pEncryptData);
173
      if (code) {
13,050,596✔
174
        taosMemoryFree(data);
×
175
        return code;
×
176
      }
177

178
      TARRAY2_INIT_EX(reader->tombBlkArray, size, size, data);
13,050,596✔
179
    } else {
180
      TARRAY2_INIT(reader->tombBlkArray);
272,880,493✔
181
    }
182

183
    reader->ctx->tombBlkLoaded = true;
285,960,844✔
184
  }
185

186
  tombBlkArray[0] = reader->tombBlkArray;
285,960,694✔
187
  return 0;
285,978,427✔
188
}
189

190
int32_t tsdbSttFileReadSttBlk(SSttFileReader *reader, const TSttBlkArray **sttBlkArray) {
285,957,760✔
191
  if (!reader->ctx->sttBlkLoaded) {
285,957,760✔
192
    if (reader->footer->sttBlkPtr->size > 0) {
285,981,183✔
193
      if (reader->footer->sttBlkPtr->size % sizeof(SSttBlk) != 0) {
285,754,400✔
194
        tsdbError("vgId:%d %s failed at %s:%d since stt file stt block size is not valid, fname:%s",
×
195
                  TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, __LINE__, reader->fd->path);
196
        return TSDB_CODE_FILE_CORRUPTED;
×
197
      }
198

199
      int32_t size = reader->footer->sttBlkPtr->size / sizeof(SSttBlk);
285,702,095✔
200
      void   *data = taosMemoryMalloc(reader->footer->sttBlkPtr->size);
285,748,277✔
201
      if (!data) {
285,705,004✔
202
        return terrno;
×
203
      }
204

205
      SEncryptData *pEncryptData = &(reader->config->tsdb->pVnode->config.tsdbCfg.encryptData);
285,705,004✔
206

207
      int32_t code = tsdbReadFile(reader->fd, reader->footer->sttBlkPtr->offset, data, reader->footer->sttBlkPtr->size,
285,757,830✔
208
                                  0, pEncryptData);
209
      if (code) {
285,773,488✔
210
        taosMemoryFree(data);
×
211
        return code;
×
212
      }
213

214
      TARRAY2_INIT_EX(reader->sttBlkArray, size, size, data);
285,773,488✔
215
    } else {
216
      TARRAY2_INIT(reader->sttBlkArray);
216,879✔
217
    }
218

219
    reader->ctx->sttBlkLoaded = true;
285,977,293✔
220
  }
221

222
  sttBlkArray[0] = reader->sttBlkArray;
286,009,942✔
223
  return 0;
285,987,335✔
224
}
225

226
int32_t tsdbSttFileReadBlockData(SSttFileReader *reader, const SSttBlk *sttBlk, SBlockData *bData) {
18,838,577✔
227
  int32_t code = 0;
18,838,577✔
228
  int32_t lino = 0;
18,838,577✔
229

230
  SBuffer *buffer0 = reader->buffers + 0;
18,839,200✔
231
  SBuffer *assist = reader->buffers + 1;
18,839,924✔
232

233
  SEncryptData *pEncryptData = &(reader->config->tsdb->pVnode->config.tsdbCfg.encryptData);
18,839,195✔
234

235
  // load data
236
  tBufferClear(buffer0);
237
  TAOS_CHECK_GOTO(
18,839,199✔
238
      tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szBlock, buffer0, 0, pEncryptData), &lino,
239
      _exit);
240

241
  SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0);
18,841,377✔
242
  TAOS_CHECK_GOTO(tBlockDataDecompress(&br, bData, assist), &lino, _exit);
18,840,651✔
243

244
_exit:
18,840,651✔
245
  if (code) {
18,841,377✔
246
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino,
×
247
              tstrerror(code));
248
  }
249
  return code;
18,841,377✔
250
}
251

252
int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *sttBlk, SBlockData *bData,
210,637,921✔
253
                                         STSchema *pTSchema, int16_t cids[], int32_t ncid) {
254
  int32_t code = 0;
210,637,921✔
255
  int32_t lino = 0;
210,637,921✔
256

257
  SDiskDataHdr hdr;
210,640,847✔
258
  SBuffer     *buffer0 = reader->buffers + 0;
210,623,479✔
259
  SBuffer     *buffer1 = reader->buffers + 1;
210,598,567✔
260
  SBuffer     *assist = reader->buffers + 2;
210,631,289✔
261

262
  SEncryptData *pEncryptData = &(reader->config->tsdb->pVnode->config.tsdbCfg.encryptData);
210,655,161✔
263

264
  // load key part
265
  tBufferClear(buffer0);
266
  TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset, sttBlk->bInfo.szKey, buffer0, 0, pEncryptData),
210,652,424✔
267
                  &lino, _exit);
268

269
  // decode header
270
  SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0);
210,668,543✔
271
  TAOS_CHECK_GOTO(tGetDiskDataHdr(&br, &hdr), &lino, _exit);
210,656,391✔
272

273
  if (hdr.delimiter != TSDB_FILE_DLMT) {
210,586,275✔
274
    tsdbError("vgId:%d %s failed at %s:%d since disk data header delimiter is invalid, fname:%s",
×
275
              TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, __LINE__, reader->fd->path);
276
    TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
×
277
  }
278

279
  // set data container
280
  tBlockDataReset(bData);
210,586,275✔
281
  bData->suid = hdr.suid;
210,631,282✔
282
  bData->uid = (sttBlk->suid == 0) ? sttBlk->minUid : 0;
210,631,284✔
283
  bData->nRow = hdr.nRow;
210,613,369✔
284

285
  // key part
286
  TAOS_CHECK_GOTO(tBlockDataDecompressKeyPart(&hdr, &br, bData, assist), &lino, _exit);
210,622,457✔
287
  if (br.offset != buffer0->size) {
210,642,160✔
288
    tsdbError("vgId:%d %s failed at %s:%d since key part size mismatch, expected:%u, actual:%u, fname:%s",
×
289
              TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, __LINE__, buffer0->size, br.offset,
290
              reader->fd->path);
291
    TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
×
292
  }
293

294
  bool loadExtra = false;
210,651,163✔
295
  for (int i = 0; i < ncid; i++) {
213,164,542✔
296
    if (tBlockDataGetColData(bData, cids[i]) == NULL) {
180,183,354✔
297
      loadExtra = true;
177,674,657✔
298
      break;
177,674,657✔
299
    }
300
  }
301

302
  if (!loadExtra) {
210,655,845✔
303
    goto _exit;
32,981,532✔
304
  }
305

306
  // load SBlockCol part
307
  tBufferClear(buffer0);
308
  TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey, hdr.szBlkCol, buffer0, 0,
177,676,631✔
309
                                       pEncryptData),
310
                  &lino, _exit);
311

312
  // load each column
313
  SBlockCol blockCol = {
177,689,552✔
314
      .cid = 0,
315
  };
316
  br = BUFFER_READER_INITIALIZER(0, buffer0);
177,691,619✔
317
  for (int32_t i = 0; i < ncid; i++) {
667,928,240✔
318
    int16_t cid = cids[i];
490,221,720✔
319

320
    if (tBlockDataGetColData(bData, cid)) {  // already loaded
490,231,892✔
321
      continue;
2,178,780✔
322
    }
323

324
    while (cid > blockCol.cid) {
1,720,208,946✔
325
      if (br.offset >= buffer0->size) {
1,233,107,211✔
326
        blockCol.cid = INT16_MAX;
991,331✔
327
        break;
991,331✔
328
      }
329

330
      TAOS_CHECK_GOTO(tGetBlockCol(&br, &blockCol, hdr.fmtVer, hdr.cmprAlg), &lino, _exit);
1,232,125,204✔
331
    }
332

333
    if (cid < blockCol.cid) {
488,093,066✔
334
      const STColumn *tcol = tTSchemaSearchColumn(pTSchema, cid);
4,573,672✔
335
      TSDB_CHECK_NULL(tcol, code, lino, _exit, TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER);
4,574,327✔
336
      SBlockCol none = {
4,574,327✔
337
          .cid = cid,
338
          .type = tcol->type,
4,574,327✔
339
          .cflag = tcol->flags,
4,574,327✔
340
          .flag = HAS_NONE,
341
          .szOrigin = 0,
342
          .szBitmap = 0,
343
          .szOffset = 0,
344
          .szValue = 0,
345
          .offset = 0,
346
      };
347
      TAOS_CHECK_GOTO(tBlockDataDecompressColData(&hdr, &none, &br, bData, assist), &lino, _exit);
4,574,327✔
348
    } else if (cid == blockCol.cid) {
483,519,394✔
349
      // load from file
350
      tBufferClear(buffer1);
351
      TAOS_CHECK_GOTO(
483,490,296✔
352
          tsdbReadFileToBuffer(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey + hdr.szBlkCol + blockCol.offset,
353
                               blockCol.szBitmap + blockCol.szOffset + blockCol.szValue, buffer1, 0, pEncryptData),
354
          &lino, _exit);
355

356
      // decode the buffer
357
      SBufferReader br1 = BUFFER_READER_INITIALIZER(0, buffer1);
483,496,245✔
358
      TAOS_CHECK_GOTO(tBlockDataDecompressColData(&hdr, &blockCol, &br1, bData, assist), &lino, _exit);
483,498,592✔
359
    }
360
  }
361

362
_exit:
210,721,405✔
363
  if (code) {
210,653,603✔
364
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino,
×
365
              tstrerror(code));
366
  }
367
  return code;
210,653,603✔
368
}
369

370
int32_t tsdbSttFileReadTombBlock(SSttFileReader *reader, const STombBlk *tombBlk, STombBlock *tombBlock) {
6,632,668✔
371
  int32_t code = 0;
6,632,668✔
372
  int32_t lino = 0;
6,632,668✔
373

374
  SBuffer *buffer0 = reader->buffers + 0;
6,632,668✔
375
  SBuffer *assist = reader->buffers + 1;
6,632,668✔
376

377
  SEncryptData *pEncryptData = &(reader->config->tsdb->pVnode->config.tsdbCfg.encryptData);
6,632,668✔
378

379
  // load
380
  tBufferClear(buffer0);
381
  TAOS_CHECK_GOTO(tsdbReadFileToBuffer(reader->fd, tombBlk->dp->offset, tombBlk->dp->size, buffer0, 0, pEncryptData),
6,632,668✔
382
                  &lino, _exit);
383

384
  // decode
385
  int32_t       size = 0;
6,632,668✔
386
  SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0);
6,632,668✔
387
  tTombBlockClear(tombBlock);
6,632,668✔
388
  tombBlock->numOfRecords = tombBlk->numRec;
6,632,668✔
389
  for (int32_t i = 0; i < ARRAY_SIZE(tombBlock->buffers); ++i) {
39,795,371✔
390
    SCompressInfo cinfo = {
33,162,703✔
391
        .cmprAlg = tombBlk->cmprAlg,
33,162,703✔
392
        .dataType = TSDB_DATA_TYPE_BIGINT,
393
        .originalSize = tombBlk->numRec * sizeof(int64_t),
33,162,703✔
394
        .compressedSize = tombBlk->size[i],
33,162,703✔
395
    };
396
    TAOS_CHECK_GOTO(tDecompressDataToBuffer(BR_PTR(&br), &cinfo, tombBlock->buffers + i, assist), &lino, _exit);
33,162,703✔
397
    br.offset += tombBlk->size[i];
33,162,703✔
398
  }
399

400
  if (br.offset != tombBlk->dp->size) {
6,632,668✔
401
    tsdbError("vgId:%d %s failed at %s:%d since tomb block size mismatch, fname:%s",
×
402
              TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, __LINE__, reader->fd->path);
403
    TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
×
404
  }
405

406
_exit:
6,632,668✔
407
  if (code) {
6,632,668✔
408
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino,
×
409
              tstrerror(code));
410
  }
411
  return code;
6,632,668✔
412
}
413

414
int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *statisBlk, STbStatisBlock *statisBlock) {
229,761,143✔
415
  int32_t code = 0;
229,761,143✔
416
  int32_t lino = 0;
229,761,143✔
417

418
  if (statisBlk->numOfPKs > TD_MAX_PK_COLS) {
229,763,865✔
419
    tsdbError("vgId:%d %s failed at %s:%d since statis block numOfPKs %d exceeds max %d (file corrupted)",
×
420
              TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, __LINE__, (int)statisBlk->numOfPKs,
421
              (int)TD_MAX_PK_COLS);
422
    TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
×
423
  }
424

425
  SBuffer *buffer0 = reader->buffers + 0;
229,694,348✔
426
  SBuffer *assist = reader->buffers + 1;
229,762,017✔
427

428
  SEncryptData *pEncryptData = &(reader->config->tsdb->pVnode->config.tsdbCfg.encryptData);
229,763,564✔
429

430
  // load data
431
  tBufferClear(buffer0);
432
  TAOS_CHECK_GOTO(
229,759,850✔
433
      tsdbReadFileToBuffer(reader->fd, statisBlk->dp->offset, statisBlk->dp->size, buffer0, 0, pEncryptData), &lino,
434
      _exit);
435

436
  // decode data
437
  tStatisBlockClear(statisBlock);
229,774,390✔
438
  statisBlock->numOfPKs = statisBlk->numOfPKs;
229,739,614✔
439
  statisBlock->numOfRecords = statisBlk->numRec;
229,751,632✔
440
  SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0);
229,738,029✔
441
  for (int32_t i = 0; i < ARRAY_SIZE(statisBlock->buffers); ++i) {
1,378,690,721✔
442
    SCompressInfo info = {
1,148,889,774✔
443
        .dataType = TSDB_DATA_TYPE_BIGINT,
444
        .cmprAlg = statisBlk->cmprAlg,
1,148,871,745✔
445
        .compressedSize = statisBlk->size[i],
1,148,894,991✔
446
        .originalSize = statisBlk->numRec * sizeof(int64_t),
1,148,850,135✔
447
    };
448

449
    TAOS_CHECK_GOTO(tDecompressDataToBuffer(BR_PTR(&br), &info, &statisBlock->buffers[i], assist), &lino, _exit);
1,148,898,151✔
450
    br.offset += statisBlk->size[i];
1,148,920,585✔
451
  }
452

453
  if (statisBlk->numOfPKs > 0) {
229,800,947✔
454
    SValueColumnCompressInfo firstKeyInfos[TD_MAX_PK_COLS];
12,979,289✔
455
    SValueColumnCompressInfo lastKeyInfos[TD_MAX_PK_COLS];
12,979,289✔
456

457
    // decode compress info
458
    for (int32_t i = 0; i < statisBlk->numOfPKs; i++) {
25,957,850✔
459
      TAOS_CHECK_GOTO(tValueColumnCompressInfoDecode(&br, &firstKeyInfos[i]), &lino, _exit);
12,979,289✔
460
    }
461

462
    for (int32_t i = 0; i < statisBlk->numOfPKs; i++) {
25,957,850✔
463
      TAOS_CHECK_GOTO(tValueColumnCompressInfoDecode(&br, &lastKeyInfos[i]), &lino, _exit);
12,977,833✔
464
    }
465

466
    // decode value columns
467
    for (int32_t i = 0; i < statisBlk->numOfPKs; i++) {
25,956,394✔
468
      TAOS_CHECK_GOTO(tValueColumnDecompress(BR_PTR(&br), firstKeyInfos + i, &statisBlock->firstKeyPKs[i], assist),
12,979,289✔
469
                      &lino, _exit);
470
      br.offset += (firstKeyInfos[i].dataCompressedSize + firstKeyInfos[i].offsetCompressedSize);
12,978,561✔
471
    }
472

473
    for (int32_t i = 0; i < statisBlk->numOfPKs; i++) {
25,957,122✔
474
      TAOS_CHECK_GOTO(tValueColumnDecompress(BR_PTR(&br), &lastKeyInfos[i], &statisBlock->lastKeyPKs[i], assist), &lino,
12,979,289✔
475
                      _exit);
476
      br.offset += (lastKeyInfos[i].dataCompressedSize + lastKeyInfos[i].offsetCompressedSize);
12,979,289✔
477
    }
478
  }
479

480
  if (br.offset != buffer0->size) {
229,793,538✔
481
    tsdbError("vgId:%d %s failed at %s:%d since statis block size mismatch, expected: %u, actual: %u, fname:%s",
×
482
              TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, __LINE__, buffer0->size, br.offset,
483
              reader->fd->path);
484
    TSDB_CHECK_CODE(code = TSDB_CODE_FILE_CORRUPTED, lino, _exit);
×
485
  }
486

487
_exit:
229,786,755✔
488
  if (code) {
229,787,370✔
489
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(reader->config->tsdb->pVnode), __func__, __FILE__, lino,
×
490
              tstrerror(code));
491
  }
492
  return code;
229,787,370✔
493
}
494

495
// SSttFWriter ============================================================
496
struct SSttFileWriter {
497
  SSttFileWriterConfig config[1];
498
  struct {
499
    bool    opened;
500
    TABLEID tbid[1];
501
    // range
502
    SVersionRange range;
503
  } ctx[1];
504
  // file
505
  STsdbFD *fd;
506
  STFile   file[1];
507
  // data
508
  SSttFooter      footer[1];
509
  TTombBlkArray   tombBlkArray[1];
510
  TSttBlkArray    sttBlkArray[1];
511
  TStatisBlkArray statisBlkArray[1];
512
  STombBlock      tombBlock[1];
513
  STbStatisBlock  staticBlock[1];
514
  SBlockData      blockData[1];
515
  // helper data
516
  SSkmInfo skmTb[1];
517
  SSkmInfo skmRow[1];
518
  SBuffer  local[10];
519
  SBuffer *buffers;
520
  // SColCompressInfo2 pInfo;
521
};
522

523
static int32_t tsdbFileDoWriteSttBlockData(STsdbFD *fd, SBlockData *blockData, SColCompressInfo *info,
48,398,393✔
524
                                           int64_t *fileSize, TSttBlkArray *sttBlkArray, SBuffer *buffers,
525
                                           SVersionRange *range, SEncryptData *encryptData) {
526
  if (blockData->nRow == 0) return 0;
48,398,393✔
527

528
  int32_t code = 0;
48,400,231✔
529

530
  SSttBlk sttBlk[1] = {{
48,403,233✔
531
      .suid = blockData->suid,
48,399,690✔
532
      .minUid = blockData->uid ? blockData->uid : blockData->aUid[0],
48,398,117✔
533
      .maxUid = blockData->uid ? blockData->uid : blockData->aUid[blockData->nRow - 1],
48,404,000✔
534
      .minKey = blockData->aTSKEY[0],
48,401,309✔
535
      .maxKey = blockData->aTSKEY[0],
48,401,975✔
536
      .minVer = blockData->aVersion[0],
48,403,438✔
537
      .maxVer = blockData->aVersion[0],
48,401,558✔
538
      .nRow = blockData->nRow,
48,400,966✔
539
  }};
540

541
  for (int32_t iRow = 1; iRow < blockData->nRow; iRow++) {
2,147,483,647✔
542
    if (sttBlk->minKey > blockData->aTSKEY[iRow]) sttBlk->minKey = blockData->aTSKEY[iRow];
2,147,483,647✔
543
    if (sttBlk->maxKey < blockData->aTSKEY[iRow]) sttBlk->maxKey = blockData->aTSKEY[iRow];
2,147,483,647✔
544
    if (sttBlk->minVer > blockData->aVersion[iRow]) sttBlk->minVer = blockData->aVersion[iRow];
2,147,483,647✔
545
    if (sttBlk->maxVer < blockData->aVersion[iRow]) sttBlk->maxVer = blockData->aVersion[iRow];
2,147,483,647✔
546
  }
547

548
  tsdbWriterUpdVerRange(range, sttBlk->minVer, sttBlk->maxVer);
48,405,311✔
549
  TAOS_CHECK_RETURN(tBlockDataCompress(blockData, info, buffers, buffers + 4));
48,401,705✔
550

551
  sttBlk->bInfo.offset = *fileSize;
48,399,268✔
552
  sttBlk->bInfo.szKey = buffers[0].size + buffers[1].size;
48,398,758✔
553
  sttBlk->bInfo.szBlock = buffers[2].size + buffers[3].size + sttBlk->bInfo.szKey;
48,402,504✔
554
  for (int i = 0; i < 4; i++) {
242,017,276✔
555
    if (buffers[i].size) {
193,611,093✔
556
      TAOS_CHECK_RETURN(tsdbWriteFile(fd, *fileSize, buffers[i].data, buffers[i].size, encryptData));
193,350,902✔
557
      *fileSize += buffers[i].size;
193,352,416✔
558
    }
559
  }
560

561
  TAOS_CHECK_RETURN(TARRAY2_APPEND_PTR(sttBlkArray, sttBlk));
96,810,860✔
562

563
  tBlockDataClear(blockData);
48,404,677✔
564
  return 0;
48,405,629✔
565
}
566

567
static int32_t tsdbSttFileDoWriteBlockData(SSttFileWriter *writer) {
70,392,052✔
568
  if (writer->blockData->nRow == 0) return 0;
70,392,052✔
569

570
  int32_t code = 0;
48,406,681✔
571
  int32_t lino = 0;
48,406,681✔
572

573
  tb_uid_t         uid = writer->blockData->suid == 0 ? writer->blockData->uid : writer->blockData->suid;
48,405,571✔
574
  SColCompressInfo info = {.defaultCmprAlg = writer->config->cmprAlg, .pColCmpr = NULL};
48,407,098✔
575
  code = metaGetColCmpr(writer->config->tsdb->pVnode->pMeta, uid, &(info.pColCmpr));
48,404,516✔
576

577
  SEncryptData *pEncryptData = &(writer->config->tsdb->pVnode->config.tsdbCfg.encryptData);
48,405,026✔
578

579
  TAOS_CHECK_GOTO(tsdbFileDoWriteSttBlockData(writer->fd, writer->blockData, &info, &writer->file->size,
48,404,678✔
580
                                              writer->sttBlkArray, writer->buffers, &writer->ctx->range, pEncryptData),
581
                  &lino, _exit);
582

583
_exit:
48,403,783✔
584
  if (code) {
48,403,357✔
585
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
586
              tstrerror(code));
587
  }
588
  taosHashCleanup(info.pColCmpr);
48,403,357✔
589
  return code;
48,399,270✔
590
}
591

592
static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) {
22,341,179✔
593
  if (writer->staticBlock->numOfRecords == 0) {
22,341,179✔
594
    return 0;
1,500,152✔
595
  }
596

597
  int32_t code = 0;
20,841,145✔
598
  int32_t lino = 0;
20,841,145✔
599

600
  SBuffer *buffer0 = writer->buffers + 0;
20,841,145✔
601
  SBuffer *buffer1 = writer->buffers + 1;
20,843,464✔
602
  SBuffer *assist = writer->buffers + 2;
20,845,149✔
603

604
  STbStatisRecord record;
20,844,104✔
605
  STbStatisBlock *statisBlock = writer->staticBlock;
20,846,109✔
606
  SStatisBlk      statisBlk = {0};
20,846,564✔
607

608
  statisBlk.dp->offset = writer->file->size;
20,846,564✔
609
  statisBlk.dp->size = 0;
20,846,372✔
610
  statisBlk.numRec = statisBlock->numOfRecords;
20,846,372✔
611
  statisBlk.cmprAlg = writer->config->cmprAlg;
20,841,871✔
612
  statisBlk.numOfPKs = statisBlock->numOfPKs;
20,844,910✔
613

614
  code = tStatisBlockGet(statisBlock, 0, &record);
20,844,758✔
615
  TSDB_CHECK_CODE(code, lino, _exit);
20,844,826✔
616
  statisBlk.minTbid.suid = record.suid;
20,844,826✔
617
  statisBlk.minTbid.uid = record.uid;
20,844,826✔
618

619
  code = tStatisBlockGet(statisBlock, statisBlock->numOfRecords - 1, &record);
20,844,826✔
620
  TSDB_CHECK_CODE(code, lino, _exit);
20,845,078✔
621
  statisBlk.maxTbid.suid = record.suid;
20,845,078✔
622
  statisBlk.maxTbid.uid = record.uid;
20,845,078✔
623

624
  SEncryptData *pEncryptData = &(writer->config->tsdb->pVnode->config.tsdbCfg.encryptData);
20,845,078✔
625

626
  // compress each column
627
  for (int32_t i = 0; i < ARRAY_SIZE(statisBlk.size); i++) {
125,071,340✔
628
    SCompressInfo info = {
104,223,767✔
629
        .dataType = TSDB_DATA_TYPE_BIGINT,
630
        .cmprAlg = statisBlk.cmprAlg,
104,220,325✔
631
        .originalSize = statisBlock->buffers[i].size,
104,220,325✔
632
    };
633

634
    tBufferClear(buffer0);
635
    TAOS_CHECK_GOTO(tCompressDataToBuffer(statisBlock->buffers[i].data, &info, buffer0, assist), &lino, _exit);
104,225,831✔
636
    TAOS_CHECK_GOTO(tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, info.compressedSize, pEncryptData),
104,230,452✔
637
                    &lino, _exit);
638

639
    statisBlk.size[i] = info.compressedSize;
104,236,918✔
640
    statisBlk.dp->size += info.compressedSize;
104,233,470✔
641
    writer->file->size += info.compressedSize;
104,233,470✔
642
  }
643

644
  // compress primary keys
645
  if (statisBlk.numOfPKs > 0) {
20,847,573✔
646
    SValueColumnCompressInfo compressInfo = {.cmprAlg = statisBlk.cmprAlg};
551,478✔
647

648
    tBufferClear(buffer0);
649
    tBufferClear(buffer1);
650

651
    for (int32_t i = 0; i < statisBlk.numOfPKs; i++) {
1,102,956✔
652
      TAOS_CHECK_GOTO(tValueColumnCompress(&statisBlock->firstKeyPKs[i], &compressInfo, buffer1, assist), &lino, _exit);
550,753✔
653
      TAOS_CHECK_GOTO(tValueColumnCompressInfoEncode(&compressInfo, buffer0), &lino, _exit);
550,753✔
654
    }
655

656
    for (int32_t i = 0; i < statisBlk.numOfPKs; i++) {
1,103,681✔
657
      TAOS_CHECK_GOTO(tValueColumnCompress(&statisBlock->lastKeyPKs[i], &compressInfo, buffer1, assist), &lino, _exit);
550,753✔
658
      TAOS_CHECK_GOTO(tValueColumnCompressInfoEncode(&compressInfo, buffer0), &lino, _exit);
551,478✔
659
    }
660

661
    TAOS_CHECK_GOTO(tsdbWriteFile(writer->fd, writer->file->size, buffer0->data, buffer0->size, pEncryptData), &lino,
552,928✔
662
                    _exit);
663
    writer->file->size += buffer0->size;
551,478✔
664
    statisBlk.dp->size += buffer0->size;
550,753✔
665

666
    TAOS_CHECK_GOTO(tsdbWriteFile(writer->fd, writer->file->size, buffer1->data, buffer1->size, pEncryptData), &lino,
550,753✔
667
                    _exit);
668
    writer->file->size += buffer1->size;
551,478✔
669
    statisBlk.dp->size += buffer1->size;
550,753✔
670
  }
671

672
  TAOS_CHECK_GOTO(TARRAY2_APPEND_PTR(writer->statisBlkArray, &statisBlk), &lino, _exit);
41,693,432✔
673

674
  tStatisBlockClear(writer->staticBlock);
20,845,859✔
675

676
_exit:
20,847,412✔
677
  if (code) {
20,846,692✔
678
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
679
              tstrerror(code));
680
  }
681
  return code;
20,840,949✔
682
}
683

684
static int32_t tsdbSttFileDoWriteTombBlock(SSttFileWriter *writer) {
20,543,404✔
685
  if (TOMB_BLOCK_SIZE(writer->tombBlock) == 0) {
20,543,404✔
686
    return 0;
18,827,982✔
687
  }
688

689
  int32_t code = 0;
1,717,292✔
690
  int32_t lino = 0;
1,717,292✔
691

692
  SEncryptData *pEncryptData = &(writer->config->tsdb->pVnode->config.tsdbCfg.encryptData);
1,717,292✔
693

694
  TAOS_CHECK_GOTO(tsdbFileWriteTombBlock(writer->fd, writer->tombBlock, writer->config->cmprAlg, &writer->file->size,
1,717,292✔
695
                                         writer->tombBlkArray, writer->buffers, &writer->ctx->range, pEncryptData),
696
                  &lino, _exit);
697

698
_exit:
1,717,292✔
699
  if (code) {
1,717,292✔
700
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
701
              tstrerror(code));
702
  }
703
  return code;
1,717,292✔
704
}
705

706
int32_t tsdbFileWriteSttBlk(STsdbFD *fd, const TSttBlkArray *sttBlkArray, SFDataPtr *ptr, int64_t *fileSize,
20,544,013✔
707
                            SEncryptData *encryptData) {
708
  ptr->size = TARRAY2_DATA_LEN(sttBlkArray);
20,544,013✔
709
  if (ptr->size > 0) {
20,545,788✔
710
    ptr->offset = *fileSize;
20,490,269✔
711

712
    TAOS_CHECK_RETURN(tsdbWriteFile(fd, *fileSize, (const uint8_t *)TARRAY2_DATA(sttBlkArray), ptr->size, encryptData));
20,490,215✔
713

714
    *fileSize += ptr->size;
20,489,920✔
715
  }
716
  return 0;
20,545,968✔
717
}
718

719
static int32_t tsdbSttFileDoWriteSttBlk(SSttFileWriter *writer) {
20,542,172✔
720
  int32_t code = 0;
20,542,172✔
721
  int32_t lino;
20,541,731✔
722

723
  SEncryptData *pEncryptData = &(writer->config->tsdb->pVnode->config.tsdbCfg.encryptData);
20,544,497✔
724

725
  TAOS_CHECK_GOTO(tsdbFileWriteSttBlk(writer->fd, writer->sttBlkArray, writer->footer->sttBlkPtr, &writer->file->size,
20,546,669✔
726
                                      pEncryptData),
727
                  &lino, _exit);
728

729
_exit:
20,545,067✔
730
  if (code) {
20,545,067✔
731
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
732
              tstrerror(code));
733
  }
734
  return code;
20,545,067✔
735
}
736

737
static int32_t tsdbSttFileDoWriteStatisBlk(SSttFileWriter *writer) {
20,541,804✔
738
  int32_t code = 0;
20,541,804✔
739
  int32_t lino;
20,541,363✔
740
  SEncryptData *pEncryptData = &(writer->config->tsdb->pVnode->config.tsdbCfg.encryptData);
20,543,753✔
741

742
  writer->footer->statisBlkPtr->size = TARRAY2_DATA_LEN(writer->statisBlkArray);
20,544,181✔
743
  if (writer->footer->statisBlkPtr->size) {
20,544,984✔
744
    writer->footer->statisBlkPtr->offset = writer->file->size;
20,490,002✔
745
    TAOS_CHECK_GOTO(tsdbWriteFile(writer->fd, writer->file->size, (const uint8_t *)TARRAY2_DATA(writer->statisBlkArray),
20,490,097✔
746
                                  writer->footer->statisBlkPtr->size, pEncryptData),
747
                    &lino, _exit);
748
    writer->file->size += writer->footer->statisBlkPtr->size;
20,491,499✔
749
  }
750

751
_exit:
20,544,656✔
752
  if (code) {
20,544,023✔
753
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
754
              tstrerror(code));
755
  }
756
  return code;
20,544,023✔
757
}
758

759
static int32_t tsdbSttFileDoWriteTombBlk(SSttFileWriter *writer) {
20,541,505✔
760
  int32_t code = 0;
20,541,505✔
761
  int32_t lino = 0;
20,541,505✔
762

763
  SEncryptData *pEncryptData = &(writer->config->tsdb->pVnode->config.tsdbCfg.encryptData);
20,543,028✔
764

765
  TAOS_CHECK_GOTO(tsdbFileWriteTombBlk(writer->fd, writer->tombBlkArray, writer->footer->tombBlkPtr,
20,542,210✔
766
                                       &writer->file->size, pEncryptData),
767
                  &lino, _exit);
768

769
_exit:
20,540,802✔
770
  if (code) {
20,540,458✔
771
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
772
              tstrerror(code));
773
  }
774
  return code;
20,540,458✔
775
}
776

777
int32_t tsdbFileWriteSttFooter(STsdbFD *fd, const SSttFooter *footer, int64_t *fileSize, SEncryptData *encryptData) {
20,541,684✔
778
  TAOS_CHECK_RETURN(tsdbWriteFile(fd, *fileSize, (const uint8_t *)footer, sizeof(*footer), encryptData));
20,541,684✔
779
  *fileSize += sizeof(*footer);
20,544,658✔
780
  return 0;
20,541,129✔
781
}
782

783
static int32_t tsdbSttFileDoWriteFooter(SSttFileWriter *writer) {
20,541,365✔
784
  SEncryptData *pEncryptData = &(writer->config->tsdb->pVnode->config.tsdbCfg.encryptData);
20,541,365✔
785

786
  return tsdbFileWriteSttFooter(writer->fd, writer->footer, &writer->file->size, pEncryptData);
20,543,816✔
787
}
788

789
static int32_t tsdbSttFWriterDoOpen(SSttFileWriter *writer) {
20,528,644✔
790
  int32_t code = 0;
20,528,644✔
791
  int32_t lino = 0;
20,528,644✔
792
  STsdb  *tsdb = writer->config->tsdb;
20,532,756✔
793

794
  // set
795
  if (!writer->config->skmTb) writer->config->skmTb = writer->skmTb;
20,533,044✔
796
  if (!writer->config->skmRow) writer->config->skmRow = writer->skmRow;
20,534,222✔
797
  writer->buffers = writer->config->buffers;
20,539,701✔
798
  if (writer->buffers == NULL) {
20,539,295✔
799
    writer->buffers = writer->local;
×
800
  }
801

802
  // alloc disk id
803
  SDiskID diskId = {0};
20,537,669✔
804
  code = tsdbAllocateDisk(tsdb, tsdbFTypeLabel(TSDB_FTYPE_STT), writer->config->expLevel, &diskId);
20,537,664✔
805
  TSDB_CHECK_CODE(code, lino, _exit);
20,533,540✔
806

807
  writer->file[0] = (STFile){
20,533,166✔
808
      .type = TSDB_FTYPE_STT,
809
      .did = diskId,
810
      .fid = writer->config->fid,
20,533,540✔
811
      .cid = writer->config->cid,
20,540,026✔
812
      .size = 0,
813
      .minVer = VERSION_MAX,
814
      .maxVer = VERSION_MIN,
815
      .stt[0] =
816
          {
817
              .level = writer->config->level,
20,540,226✔
818
          },
819
  };
820

821
  // open file
822
  int32_t flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
20,536,956✔
823
  char    fname[TSDB_FILENAME_LEN];
20,536,515✔
824

825
  tsdbTFileName(writer->config->tsdb, writer->file, fname);
20,533,420✔
826
  TAOS_CHECK_GOTO(tsdbOpenFile(fname, writer->config->tsdb, flag, &writer->fd, 0), &lino, _exit);
20,532,030✔
827

828
  uint8_t hdr[TSDB_FHDR_SIZE] = {0};
20,534,156✔
829
  SEncryptData *pEncryptData = &(writer->config->tsdb->pVnode->config.tsdbCfg.encryptData);
20,533,733✔
830

831
  TAOS_CHECK_GOTO(tsdbWriteFile(writer->fd, 0, hdr, sizeof(hdr), pEncryptData), &lino, _exit);
20,535,175✔
832
  writer->file->size += sizeof(hdr);
20,529,734✔
833

834
  // range
835
  writer->ctx->range = (SVersionRange){.minVer = VERSION_MAX, .maxVer = VERSION_MIN};
20,543,201✔
836

837
  writer->ctx->opened = true;
20,537,234✔
838

839
_exit:
20,536,468✔
840
  if (code) {
20,537,124✔
841
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
842
              tstrerror(code));
843
  }
844
  return code;
20,537,124✔
845
}
846

847
static void tsdbSttFWriterDoClose(SSttFileWriter *writer) {
20,541,710✔
848
  for (int32_t i = 0; i < ARRAY_SIZE(writer->local); ++i) {
225,841,070✔
849
    tBufferDestroy(writer->local + i);
205,304,718✔
850
  }
851
  tDestroyTSchema(writer->skmRow->pTSchema);
20,536,352✔
852
  tDestroyTSchema(writer->skmTb->pTSchema);
20,541,774✔
853
  tTombBlockDestroy(writer->tombBlock);
20,540,867✔
854
  tStatisBlockDestroy(writer->staticBlock);
20,539,756✔
855
  tBlockDataDestroy(writer->blockData);
20,546,087✔
856
  TARRAY2_DESTROY(writer->tombBlkArray, NULL);
20,544,950✔
857
  TARRAY2_DESTROY(writer->statisBlkArray, NULL);
20,545,352✔
858
  TARRAY2_DESTROY(writer->sttBlkArray, NULL);
20,540,511✔
859
}
20,543,457✔
860

861
static int32_t tsdbSttFileDoUpdateHeader(SSttFileWriter *writer) {
20,541,319✔
862
  // TODO
863
  return 0;
20,541,319✔
864
}
865

866
static int32_t tsdbSttFWriterCloseCommit(SSttFileWriter *writer, TFileOpArray *opArray) {
20,543,704✔
867
  int32_t lino;
20,543,263✔
868
  int32_t code;
869

870
  TAOS_CHECK_GOTO(tsdbSttFileDoWriteBlockData(writer), &lino, _exit);
20,543,113✔
871
  TAOS_CHECK_GOTO(tsdbSttFileDoWriteStatisBlock(writer), &lino, _exit);
20,542,319✔
872
  TAOS_CHECK_GOTO(tsdbSttFileDoWriteTombBlock(writer), &lino, _exit);
20,545,274✔
873
  TAOS_CHECK_GOTO(tsdbSttFileDoWriteSttBlk(writer), &lino, _exit);
20,543,835✔
874
  TAOS_CHECK_GOTO(tsdbSttFileDoWriteStatisBlk(writer), &lino, _exit);
20,541,804✔
875
  TAOS_CHECK_GOTO(tsdbSttFileDoWriteTombBlk(writer), &lino, _exit);
20,543,224✔
876
  TAOS_CHECK_GOTO(tsdbSttFileDoWriteFooter(writer), &lino, _exit);
20,539,253✔
877
  TAOS_CHECK_GOTO(tsdbSttFileDoUpdateHeader(writer), &lino, _exit);
20,542,308✔
878

879
  SEncryptData *pEncryptData = &(writer->config->tsdb->pVnode->config.tsdbCfg.encryptData);
20,541,319✔
880

881
  TAOS_CHECK_GOTO(tsdbFsyncFile(writer->fd, pEncryptData), &lino, _exit);
20,543,290✔
882

883
  tsdbCloseFile(&writer->fd);
20,544,995✔
884

885
  STFileOp op = (STFileOp){
41,083,907✔
886
      .optype = TSDB_FOP_CREATE,
887
      .fid = writer->config->fid,
20,541,064✔
888
      .nf = writer->file[0],
889
  };
890
  tsdbTFileUpdVerRange(&op.nf, writer->ctx->range);
20,546,155✔
891

892
  TAOS_CHECK_GOTO(TARRAY2_APPEND(opArray, op), &lino, _exit);
41,077,339✔
893

894
_exit:
20,537,080✔
895
  if (code) {
20,536,600✔
896
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
897
              tstrerror(code));
898
  }
899
  return code;
20,536,600✔
900
}
901

902
static int32_t tsdbSttFWriterCloseAbort(SSttFileWriter *writer) {
×
903
  char fname[TSDB_FILENAME_LEN];
×
904
  tsdbTFileName(writer->config->tsdb, writer->file, fname);
×
905
  tsdbCloseFile(&writer->fd);
×
906
  tsdbRemoveFile(fname);
×
907
  return 0;
×
908
}
909

910
int32_t tsdbSttFileWriterOpen(const SSttFileWriterConfig *config, SSttFileWriter **writer) {
21,350,853✔
911
  writer[0] = taosMemoryCalloc(1, sizeof(*writer[0]));
21,350,853✔
912
  if (writer[0] == NULL) {
21,337,964✔
913
    return terrno;
×
914
  }
915

916
  writer[0]->config[0] = config[0];
21,339,458✔
917
  writer[0]->ctx->opened = false;
21,345,264✔
918
  return 0;
21,358,763✔
919
}
920

921
int32_t tsdbSttFileWriterClose(SSttFileWriter **writer, int8_t abort, TFileOpArray *opArray) {
21,362,057✔
922
  if (writer == NULL || writer[0] == NULL) return 0;
21,362,057✔
923

924
  int32_t code = 0;
21,363,501✔
925
  int32_t lino = 0;
21,363,501✔
926

927
  if (writer[0]->ctx->opened) {
21,362,967✔
928
    if (abort) {
20,543,823✔
929
      TAOS_CHECK_GOTO(tsdbSttFWriterCloseAbort(writer[0]), &lino, _exit);
×
930
    } else {
931
      TAOS_CHECK_GOTO(tsdbSttFWriterCloseCommit(writer[0], opArray), &lino, _exit);
20,543,823✔
932
    }
933
    tsdbSttFWriterDoClose(writer[0]);
20,536,279✔
934
  }
935
  taosMemoryFree(writer[0]);
21,359,747✔
936
  writer[0] = NULL;
21,358,573✔
937

938
_exit:
21,359,988✔
939
  if (code) {
21,359,894✔
940
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer[0]->config->tsdb->pVnode), __func__, __FILE__, lino,
×
941
              tstrerror(code));
942
  }
943
  return code;
21,359,803✔
944
}
945

946
int32_t tsdbSttFileWriteRow(SSttFileWriter *writer, SRowInfo *row) {
2,147,483,647✔
947
  int32_t code = 0;
2,147,483,647✔
948
  int32_t lino = 0;
2,147,483,647✔
949

950
  if (!writer->ctx->opened) {
2,147,483,647✔
951
    TAOS_CHECK_GOTO(tsdbSttFWriterDoOpen(writer), &lino, _exit);
20,483,945✔
952
  }
953

954
  if (!TABLE_SAME_SCHEMA(row->suid, row->uid, writer->ctx->tbid->suid, writer->ctx->tbid->uid)) {
2,147,483,647✔
955
    TAOS_CHECK_GOTO(tsdbSttFileDoWriteBlockData(writer), &lino, _exit);
27,169,311✔
956

957
    TAOS_CHECK_GOTO(tsdbUpdateSkmTb(writer->config->tsdb, (TABLEID *)row, writer->config->skmTb), &lino, _exit);
27,167,918✔
958

959
    TABLEID id = {
27,164,855✔
960
        .suid = row->suid,
27,165,107✔
961
        .uid = row->suid ? 0 : row->uid,
27,169,029✔
962
    };
963
    TAOS_CHECK_GOTO(tBlockDataInit(writer->blockData, &id, writer->config->skmTb->pTSchema, NULL, 0), &lino, _exit);
27,163,266✔
964
  }
965

966
  if (writer->ctx->tbid->uid != row->uid) {
2,147,483,647✔
967
    writer->ctx->tbid->suid = row->suid;
106,191,672✔
968
    writer->ctx->tbid->uid = row->uid;
106,194,271✔
969
  }
970

971
  STsdbRowKey key;
2,147,483,647✔
972
  tsdbRowGetKey(&row->row, &key);
2,147,483,647✔
973

974
  for (;;) {
975
    code = tStatisBlockPut(writer->staticBlock, row, writer->config->maxRow);
2,147,483,647✔
976
    if (code == TSDB_CODE_INVALID_PARA) {
2,147,483,647✔
977
      TAOS_CHECK_GOTO(tsdbSttFileDoWriteStatisBlock(writer), &lino, _exit);
135,287✔
978
      continue;
135,217✔
979
    } else {
980
      TSDB_CHECK_CODE(code, lino, _exit);
2,147,483,647✔
981
    }
982
    break;
2,147,483,647✔
983
  }
984

985
  if (row->row.type == TSDBROW_ROW_FMT) {
2,147,483,647✔
986
    TAOS_CHECK_GOTO(tsdbUpdateSkmRow(writer->config->tsdb, writer->ctx->tbid,  //
2,147,483,647✔
987
                                     TSDBROW_SVERSION(&row->row), writer->config->skmRow),
988
                    &lino, _exit);
989
  }
990

991
  // row to col conversion
992
  if (key.version <= writer->config->compactVersion                                //
2,147,483,647✔
993
      && writer->blockData->nRow > 0                                               //
2,147,483,647✔
994
      && (writer->blockData->uid                                                   //
2,147,483,647✔
995
              ? writer->blockData->uid                                             //
996
              : writer->blockData->aUid[writer->blockData->nRow - 1]) == row->uid  //
2,147,483,647✔
997
      && tsdbRowCompareWithoutVersion(&row->row,
2,147,483,647✔
998
                                      &tsdbRowFromBlockData(writer->blockData, writer->blockData->nRow - 1)) == 0  //
2,147,483,647✔
999
  ) {
1000
    TAOS_CHECK_GOTO(tBlockDataUpdateRow(writer->blockData, &row->row, writer->config->skmRow->pTSchema), &lino, _exit);
201,505,796✔
1001
  } else {
1002
    if (writer->blockData->nRow >= writer->config->maxRow) {
2,147,483,647✔
1003
      TAOS_CHECK_GOTO(tsdbSttFileDoWriteBlockData(writer), &lino, _exit);
21,019,620✔
1004
    }
1005

1006
    TAOS_CHECK_GOTO(tBlockDataAppendRow(writer->blockData, &row->row, writer->config->skmRow->pTSchema, row->uid),
2,147,483,647✔
1007
                    &lino, _exit);
1008
  }
1009

1010
_exit:
2,147,483,647✔
1011
  if (code) {
2,147,483,647✔
1012
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
1013
              tstrerror(code));
1014
  }
1015
  return code;
2,147,483,647✔
1016
}
1017

1018
int32_t tsdbSttFileWriteBlockData(SSttFileWriter *writer, SBlockData *bdata) {
20,916,299✔
1019
  int32_t code = 0;
20,916,299✔
1020
  int32_t lino = 0;
20,916,299✔
1021

1022
  SRowInfo row[1];
20,915,909✔
1023
  row->suid = bdata->suid;
20,916,299✔
1024
  for (int32_t i = 0; i < bdata->nRow; i++) {
167,745,879✔
1025
    row->uid = bdata->uid ? bdata->uid : bdata->aUid[i];
146,829,312✔
1026
    row->row = tsdbRowFromBlockData(bdata, i);
146,827,710✔
1027

1028
    TAOS_CHECK_GOTO(tsdbSttFileWriteRow(writer, row), &lino, _exit);
146,827,710✔
1029
  }
1030

1031
_exit:
20,916,299✔
1032
  if (code) {
20,916,299✔
1033
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
1034
              tstrerror(code));
1035
  }
1036
  return code;
20,916,299✔
1037
}
1038

1039
int32_t tsdbSttFileWriteTombRecord(SSttFileWriter *writer, const STombRecord *record) {
6,894,562✔
1040
  int32_t code;
1041
  int32_t lino;
6,894,562✔
1042

1043
  if (!writer->ctx->opened) {
6,894,562✔
1044
    TAOS_CHECK_GOTO(tsdbSttFWriterDoOpen(writer), &lino, _exit);
53,534✔
1045
  } else {
1046
    if (writer->blockData->nRow > 0) {
6,841,028✔
1047
      TAOS_CHECK_GOTO(tsdbSttFileDoWriteBlockData(writer), &lino, _exit);
1,663,758✔
1048
    }
1049

1050
    if (STATIS_BLOCK_SIZE(writer->staticBlock) > 0) {
6,841,028✔
1051
      TAOS_CHECK_GOTO(tsdbSttFileDoWriteStatisBlock(writer), &lino, _exit);
1,663,758✔
1052
    }
1053
  }
1054

1055
  TAOS_CHECK_GOTO(tTombBlockPut(writer->tombBlock, record), &lino, _exit);
6,894,562✔
1056

1057
  if (TOMB_BLOCK_SIZE(writer->tombBlock) >= writer->config->maxRow) {
6,894,562✔
1058
    TAOS_CHECK_GOTO(tsdbSttFileDoWriteTombBlock(writer), &lino, _exit);
×
1059
  }
1060

1061
_exit:
6,894,562✔
1062
  if (code) {
6,894,562✔
1063
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(writer->config->tsdb->pVnode), __func__, __FILE__, lino,
×
1064
              tstrerror(code));
1065
  } else {
1066
    tsdbTrace("vgId:%d write tomb record to stt file:%s, cid:%" PRId64 ", suid:%" PRId64 ", uid:%" PRId64
6,894,562✔
1067
              ", version:%" PRId64,
1068
              TD_VID(writer->config->tsdb->pVnode), writer->fd->path, writer->config->cid, record->suid, record->uid,
1069
              record->version);
1070
  }
1071
  return code;
6,894,562✔
1072
}
1073

1074
bool tsdbSttFileWriterIsOpened(SSttFileWriter *writer) { return writer->ctx->opened; }
4,406,427✔
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc