• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

taosdata / TDengine / #4069

12 May 2025 05:35AM UTC coverage: 63.048% (+0.5%) from 62.547%
#4069

push

travis-ci

web-flow
Merge pull request #31053 from taosdata/merge/mainto3.0

merge: from main to 3.0 branch

157521 of 317858 branches covered (49.56%)

Branch coverage included in aggregate %.

374 of 573 new or added lines in 31 files covered. (65.27%)

4949 existing lines in 87 files now uncovered.

242707 of 316936 relevant lines covered (76.58%)

18229906.31 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

71.5
/source/dnode/vnode/src/tsdb/tsdbReadUtil.c
1
/*
2
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3
 *
4
 * This program is free software: you can use, redistribute, and/or modify
5
 * it under the terms of the GNU Affero General Public License, version 3
6
 * or later ("AGPL"), as published by the Free Software Foundation.
7
 *
8
 * This program is distributed in the hope that it will be useful, but WITHOUT
9
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10
 * FITNESS FOR A PARTICULAR PURPOSE.
11
 *
12
 * You should have received a copy of the GNU Affero General Public License
13
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
14
 */
15

16
#include "tsdbReadUtil.h"
17
#include "tsdb.h"
18
#include "tsdbDataFileRW.h"
19
#include "tsdbFS2.h"
20
#include "tsdbMerge.h"
21
#include "tsdbUtil2.h"
22
#include "tsimplehash.h"
23

24
static bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord,
25
                                            int32_t order);
26

27
static int32_t initBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) {
4,952,151✔
28
  int32_t              code = TSDB_CODE_SUCCESS;
4,952,151✔
29
  int32_t              lino = 0;
4,952,151✔
30
  int32_t              num = 0;
4,952,151✔
31
  int32_t              remainder = 0;
4,952,151✔
32
  STableBlockScanInfo* p = NULL;
4,952,151✔
33
  const void*          px = NULL;
4,952,151✔
34

35
  TSDB_CHECK_CONDITION(pBuf && pBuf->numPerBucket > 0, code, lino, _end, TSDB_CODE_INVALID_PARA);
4,952,151!
36
  TSDB_CHECK_CONDITION(numOfTables >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);
4,952,151!
37

38
  num = numOfTables / pBuf->numPerBucket;
4,952,151✔
39
  remainder = numOfTables % pBuf->numPerBucket;
4,952,151✔
40

41
  if (pBuf->pData == NULL) {
4,952,151!
42
    pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES);
4,965,388✔
43
    TSDB_CHECK_NULL(pBuf->pData, code, lino, _end, terrno);
4,965,345!
44
  }
45

46
  for (int32_t i = 0; i < num; ++i) {
4,965,166✔
47
    p = taosMemoryCalloc(pBuf->numPerBucket, sizeof(STableBlockScanInfo));
46!
48
    TSDB_CHECK_NULL(p, code, lino, _end, terrno);
46!
49

50
    px = taosArrayPush(pBuf->pData, &p);
46✔
51
    TSDB_CHECK_NULL(px, code, lino, _end, terrno);
46!
52
    p = NULL;
46✔
53
  }
54

55
  if (remainder > 0) {
4,965,120✔
56
    p = taosMemoryCalloc(remainder, sizeof(STableBlockScanInfo));
4,811,950!
57
    TSDB_CHECK_NULL(p, code, lino, _end, terrno);
4,816,834!
58

59
    px = taosArrayPush(pBuf->pData, &p);
4,816,834✔
60
    TSDB_CHECK_NULL(px, code, lino, _end, terrno);
4,814,935!
61
    p = NULL;
4,814,935✔
62
  }
63

64
  pBuf->numOfTables = numOfTables;
4,968,105✔
65

66
_end:
4,968,105✔
67
  if (code != TSDB_CODE_SUCCESS) {
4,968,105!
68
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
69
  }
70
  if (p) {
4,958,019!
71
    taosMemoryFreeClear(p);
×
72
  }
73
  return code;
4,958,019✔
74
}
75

76
int32_t uidComparFunc(const void* p1, const void* p2) {
10,386,247✔
77
  uint64_t pu1 = *(const uint64_t*)p1;
10,386,247✔
78
  uint64_t pu2 = *(const uint64_t*)p2;
10,386,247✔
79
  if (pu1 == pu2) {
10,386,247!
80
    return 0;
×
81
  } else {
82
    return (pu1 < pu2) ? -1 : 1;
10,386,247!
83
  }
84
}
85

86
int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) {
9✔
87
  int32_t              code = TSDB_CODE_SUCCESS;
9✔
88
  int32_t              lino = 0;
9✔
89
  int32_t              num = 0;
9✔
90
  int32_t              remainder = 0;
9✔
91
  STableBlockScanInfo* p = NULL;
9✔
92
  const void*          px = NULL;
9✔
93

94
  TSDB_CHECK_CONDITION(pBuf && pBuf->numPerBucket > 0 && pBuf->numOfTables >= 0, code, lino, _end,
9!
95
                       TSDB_CODE_INVALID_PARA);
96
  TSDB_CHECK_CONDITION(numOfTables >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);
9!
97

98
  if (numOfTables <= pBuf->numOfTables) {
9!
99
    goto _end;
×
100
  }
101

102
  remainder = pBuf->numOfTables % pBuf->numPerBucket;
9✔
103
  if (remainder > 0) {
9!
104
    TSDB_CHECK_CONDITION(taosArrayGetSize(pBuf->pData) > 0, code, lino, _end, TSDB_CODE_INVALID_PARA);
9!
105
    px = taosArrayPop(pBuf->pData);
9✔
106
    TSDB_CHECK_NULL(px, code, lino, _end, TSDB_CODE_INVALID_PARA);
9!
107
    p = *(STableBlockScanInfo**)px;
9✔
108
    taosMemoryFreeClear(p);
9!
109
    pBuf->numOfTables -= remainder;
9✔
110
  }
111

112
  num = (numOfTables - pBuf->numOfTables) / pBuf->numPerBucket;
9✔
113
  remainder = (numOfTables - pBuf->numOfTables) % pBuf->numPerBucket;
9✔
114

115
  if (pBuf->pData == NULL) {
9!
116
    pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES);
×
117
    TSDB_CHECK_NULL(pBuf->pData, code, lino, _end, terrno);
×
118
  }
119

120
  for (int32_t i = 0; i < num; ++i) {
9!
121
    p = taosMemoryCalloc(pBuf->numPerBucket, sizeof(STableBlockScanInfo));
×
122
    TSDB_CHECK_NULL(p, code, lino, _end, terrno);
×
123

124
    px = taosArrayPush(pBuf->pData, &p);
×
125
    TSDB_CHECK_NULL(px, code, lino, _end, terrno);
×
126
    p = NULL;
×
127
  }
128

129
  if (remainder > 0) {
9!
130
    p = taosMemoryCalloc(remainder, sizeof(STableBlockScanInfo));
9!
131
    TSDB_CHECK_NULL(p, code, lino, _end, terrno);
9!
132

133
    px = taosArrayPush(pBuf->pData, &p);
9✔
134
    TSDB_CHECK_NULL(px, code, lino, _end, terrno);
9!
135
    p = NULL;
9✔
136
  }
137

138
  pBuf->numOfTables = numOfTables;
9✔
139

140
_end:
9✔
141
  if (code != TSDB_CODE_SUCCESS) {
9!
142
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
143
  }
144
  if (p) {
9!
145
    taosMemoryFreeClear(p);
×
146
  }
147
  return code;
9✔
148
}
149

150
void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf) {
5,031,779✔
151
  if (pBuf == NULL) return;
5,031,779!
152
  if (pBuf->pData != NULL) {
5,031,779✔
153
    taosArrayDestroyP(pBuf->pData, NULL);
4,967,358✔
154
    pBuf->pData = NULL;
4,967,637✔
155
  }
156
}
157

158
int32_t getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index, STableBlockScanInfo** pInfo) {
12,383,870✔
159
  int32_t               code = TSDB_CODE_SUCCESS;
12,383,870✔
160
  int32_t               lino = 0;
12,383,870✔
161
  int32_t               bucketIndex = 0;
12,383,870✔
162
  STableBlockScanInfo** pBucket = NULL;
12,383,870✔
163

164
  TSDB_CHECK_CONDITION(pBuf && pBuf->numPerBucket > 0, code, lino, _end, TSDB_CODE_INVALID_PARA);
12,383,870!
165
  TSDB_CHECK_CONDITION(index >= 0 && index < pBuf->numOfTables, code, lino, _end, TSDB_CODE_INVALID_PARA);
12,383,870!
166
  TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
12,383,870!
167

168
  *pInfo = NULL;
12,383,870✔
169

170
  bucketIndex = index / pBuf->numPerBucket;
12,383,870✔
171
  pBucket = taosArrayGet(pBuf->pData, bucketIndex);
12,383,870✔
172
  TSDB_CHECK_NULL(pBucket, code, lino, _end, terrno);
12,355,493!
173

174
  *pInfo = (*pBucket) + (index % pBuf->numPerBucket);
12,358,858✔
175

176
_end:
12,358,858✔
177
  if (code != TSDB_CODE_SUCCESS) {
12,358,858!
178
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
179
  }
180
  return code;
12,358,411✔
181
}
182

183
int32_t getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, STableBlockScanInfo** pInfo, const char* id) {
16,685,095✔
184
  int32_t               code = TSDB_CODE_SUCCESS;
16,685,095✔
185
  int32_t               lino = 0;
16,685,095✔
186
  STableBlockScanInfo** pVal = NULL;
16,685,095✔
187

188
  TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
16,685,095!
189
  TSDB_CHECK_NULL(id, code, lino, _end, TSDB_CODE_INVALID_PARA);
16,685,095!
190

191
  pVal = (STableBlockScanInfo**)tSimpleHashGet(pTableMap, &uid, sizeof(uid));
16,685,095✔
192
  if (pVal == NULL) {
16,685,723!
193
    int32_t size = tSimpleHashGetSize(pTableMap);
×
194
    tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id);
×
195
    code = TSDB_CODE_INVALID_PARA;
×
196
    TSDB_CHECK_CODE(code, lino, _end);
×
197
  }
198
  *pInfo = *pVal;
16,685,507✔
199
  TSDB_CHECK_NULL(*pInfo, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
16,685,507!
200

201
_end:
16,685,507✔
202
  if (code != TSDB_CODE_SUCCESS) {
16,685,507!
203
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
204
  }
205
  return code;
16,685,479✔
206
}
207

208
int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len, bool asc) {
54,309,844✔
209
  int32_t code = TSDB_CODE_SUCCESS;
54,309,844✔
210
  int32_t lino = 0;
54,309,844✔
211

212
  TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA);
54,309,844!
213

214
  pKey->numOfPKs = numOfPks;
54,309,844✔
215
  pKey->ts = ts;
54,309,844✔
216

217
  if (numOfPks > 0) {
54,309,844✔
218
    pKey->pks[0].type = type;
2,068,248✔
219

220
    if (IS_NUMERIC_TYPE(type)) {
2,068,248!
221
      if (asc) {
1,825,108✔
222
        switch (type) {
1,240,486!
223
          case TSDB_DATA_TYPE_BIGINT: {
978,462✔
224
            VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT64_MIN);
978,462✔
225
            break;
978,462✔
226
          }
227
          case TSDB_DATA_TYPE_INT: {
90,247✔
228
            int32_t min = INT32_MIN;
90,247✔
229
            valueSetDatum(pKey->pks, type, &min, tDataTypes[type].bytes);
90,247✔
230
            break;
90,262✔
231
          }
232
          case TSDB_DATA_TYPE_SMALLINT: {
×
233
            int16_t min = INT16_MIN;
×
234
            valueSetDatum(pKey->pks, type, &min, tDataTypes[type].bytes);
×
235
            break;
×
236
          }
237
          case TSDB_DATA_TYPE_TINYINT: {
×
238
            int8_t min = INT8_MIN;
×
239
            valueSetDatum(pKey->pks, type, &min, tDataTypes[type].bytes);
×
240
            break;
×
241
          }
242
          case TSDB_DATA_TYPE_UTINYINT:
171,753✔
243
          case TSDB_DATA_TYPE_USMALLINT:
244
          case TSDB_DATA_TYPE_UINT:
245
          case TSDB_DATA_TYPE_UBIGINT: {
246
            VALUE_SET_TRIVIAL_DATUM(pKey->pks, 0);
171,753✔
247
            break;
171,753✔
248
          }
249
          default:
24✔
250
            code = TSDB_CODE_INVALID_PARA;
24✔
251
            TSDB_CHECK_CODE(code, lino, _end);
24!
252
        }
253
      } else {
254
        switch (type) {
584,622!
255
          case TSDB_DATA_TYPE_BIGINT:
414,500✔
256
            VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT64_MAX);
414,500✔
257
            break;
414,500✔
258
          case TSDB_DATA_TYPE_INT:
57,478✔
259
            VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT32_MAX);
57,478✔
260
            break;
57,478✔
261
          case TSDB_DATA_TYPE_SMALLINT:
×
262
            VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT16_MAX);
×
263
            break;
×
264
          case TSDB_DATA_TYPE_TINYINT:
×
265
            VALUE_SET_TRIVIAL_DATUM(pKey->pks, INT8_MAX);
×
266
            break;
×
267
          case TSDB_DATA_TYPE_UBIGINT:
55,629✔
268
            VALUE_SET_TRIVIAL_DATUM(pKey->pks, UINT64_MAX);
55,629✔
269
            break;
55,629✔
270
          case TSDB_DATA_TYPE_UINT:
57,108✔
271
            VALUE_SET_TRIVIAL_DATUM(pKey->pks, UINT32_MAX);
57,108✔
272
            break;
57,108✔
273
          case TSDB_DATA_TYPE_USMALLINT:
×
274
            VALUE_SET_TRIVIAL_DATUM(pKey->pks, UINT16_MAX);
×
275
            break;
×
276
          case TSDB_DATA_TYPE_UTINYINT:
×
277
            VALUE_SET_TRIVIAL_DATUM(pKey->pks, UINT8_MAX);
×
278
            break;
×
279
          default:
×
280
            code = TSDB_CODE_INVALID_PARA;
×
281
            TSDB_CHECK_CODE(code, lino, _end);
×
282
        }
283
      }
284
    } else {
285
      pKey->pks[0].nData = 0;
243,140✔
286
      pKey->pks[0].pData = taosMemoryCalloc(1, len);
243,140!
287
      TSDB_CHECK_NULL(pKey->pks[0].pData, code, lino, _end, terrno);
243,251!
288

289
      if (!asc) {
243,251✔
290
        pKey->numOfPKs = 2;
110,107✔
291
      }
292
    }
293
  }
294

295
_end:
52,374,740✔
296
  if (code != TSDB_CODE_SUCCESS) {
54,309,970!
297
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
298
  }
299
  return code;
54,323,752✔
300
}
301

302
void clearRowKey(SRowKey* pKey) {
1,024,631,814✔
303
  if (pKey == NULL || pKey->numOfPKs == 0 || (!IS_VAR_DATA_TYPE(pKey->pks[0].type))) {
1,024,631,814!
304
    return;
1,024,676,869✔
305
  }
UNCOV
306
  taosMemoryFreeClear(pKey->pks[0].pData);
×
307
}
308

309
static int32_t initLastProcKey(STableBlockScanInfo* pScanInfo, const STsdbReader* pReader) {
12,363,805✔
310
  int32_t  code = TSDB_CODE_SUCCESS;
12,363,805✔
311
  int32_t  lino = 0;
12,363,805✔
312
  int32_t  numOfPks = 0;
12,363,805✔
313
  int32_t  type = 0;
12,363,805✔
314
  int32_t  bytes = 0;
12,363,805✔
315
  bool     asc = false;
12,363,805✔
316
  SRowKey* pRowKey = NULL;
12,363,805✔
317

318
  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
12,363,805!
319
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
12,363,805!
320

321
  numOfPks = pReader->suppInfo.numOfPks;
12,363,805✔
322
  asc = ASCENDING_TRAVERSE(pReader->info.order);
12,363,805✔
323
  type = pReader->suppInfo.pk.type;
12,363,805✔
324
  bytes = pReader->suppInfo.pk.bytes;
12,363,805✔
325

326
  pRowKey = &pScanInfo->lastProcKey;
12,363,805✔
327
  if (asc) {
12,363,805✔
328
    int64_t skey = pReader->info.window.skey;
9,992,491✔
329
    int64_t ts = (skey > INT64_MIN) ? (skey - 1) : skey;
9,992,491!
330

331
    code = initRowKey(pRowKey, ts, numOfPks, type, bytes, asc);
9,992,491✔
332
    TSDB_CHECK_CODE(code, lino, _end);
9,994,648!
333

334
    code = initRowKey(&pScanInfo->sttKeyInfo.nextProcKey, skey, numOfPks, type, bytes, asc);
9,994,648✔
335
    TSDB_CHECK_CODE(code, lino, _end);
9,996,264!
336
  } else {
337
    int64_t ekey = pReader->info.window.ekey;
2,371,314✔
338
    int64_t ts = (ekey < INT64_MAX) ? (ekey + 1) : ekey;
2,371,314✔
339

340
    code = initRowKey(pRowKey, ts, numOfPks, type, bytes, asc);
2,371,314✔
341
    TSDB_CHECK_CODE(code, lino, _end);
2,385,638!
342

343
    code = initRowKey(&pScanInfo->sttKeyInfo.nextProcKey, ekey, numOfPks, type, bytes, asc);
2,385,638✔
344
    TSDB_CHECK_CODE(code, lino, _end);
2,386,837!
345
  }
346

347
  code = initRowKey(&pScanInfo->sttRange.skey, INT64_MAX, numOfPks, type, bytes, asc);
12,383,101✔
348
  TSDB_CHECK_CODE(code, lino, _end);
12,380,513!
349

350
  code = initRowKey(&pScanInfo->sttRange.ekey, INT64_MIN, numOfPks, type, bytes, asc);
12,380,513✔
351
  TSDB_CHECK_CODE(code, lino, _end);
12,378,198!
352

353
_end:
12,378,198✔
354
  if (code != TSDB_CODE_SUCCESS) {
12,378,198!
355
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
356
  }
357
  return code;
12,378,901✔
358
}
359

360
int32_t initTableBlockScanInfo(STableBlockScanInfo* pScanInfo, uint64_t uid, SSHashObj* pTableMap,
12,354,939✔
361
                               STsdbReader* pReader) {
362
  int32_t code = TSDB_CODE_SUCCESS;
12,354,939✔
363
  int32_t lino = 0;
12,354,939✔
364

365
  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
12,354,939!
366
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
12,354,939!
367

368
  pScanInfo->uid = uid;
12,354,939✔
369
  INIT_KEYRANGE(&pScanInfo->sttRange);
12,354,939✔
370
  INIT_TIMEWINDOW(&pScanInfo->filesetWindow);
12,354,939✔
371

372
  pScanInfo->cleanSttBlocks = false;
12,354,939✔
373
  pScanInfo->sttBlockReturned = false;
12,354,939✔
374

375
  code = initLastProcKey(pScanInfo, pReader);
12,354,939✔
376
  TSDB_CHECK_CODE(code, lino, _end);
12,379,878!
377

378
  pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT;
12,379,878✔
379
  code = tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES);
12,379,878✔
380
  TSDB_CHECK_CODE(code, lino, _end);
12,388,388!
381

382
  tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pReader, pScanInfo->uid,
12,388,388✔
383
            pScanInfo->lastProcKey.ts, pReader->idStr);
384

385
_end:
12,315,220✔
386
  if (code != TSDB_CODE_SUCCESS) {
12,388,391!
387
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
388
  }
389
  return code;
12,386,330✔
390
}
391

392
// NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model
393
int32_t createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList,
4,950,848✔
394
                                STableUidList* pUidList, int32_t numOfTables, SSHashObj** pHashObj) {
395
  int32_t    code = TSDB_CODE_SUCCESS;
4,950,848✔
396
  int32_t    lino = 0;
4,950,848✔
397
  SSHashObj* pTableMap = NULL;
4,950,848✔
398
  int64_t    st = 0;
4,950,848✔
399

400
  TSDB_CHECK_NULL(pUidList, code, lino, _end, TSDB_CODE_INVALID_PARA);
4,950,848!
401
  TSDB_CHECK_NULL(pHashObj, code, lino, _end, TSDB_CODE_INVALID_PARA);
4,950,848!
402

403
  *pHashObj = NULL;
4,950,848✔
404

405
  // allocate buffer in order to load data blocks from file
406
  // todo use simple hash instead, optimize the memory consumption
407
  pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
4,950,848✔
408
  TSDB_CHECK_NULL(pTableMap, code, lino, _end, terrno);
4,963,373!
409

410
  st = taosGetTimestampUs();
4,958,937✔
411
  code = initBlockScanInfoBuf(pBuf, numOfTables);
4,958,937✔
412
  if (code != TSDB_CODE_SUCCESS) {
4,958,575!
413
    tSimpleHashCleanup(pTableMap);
×
414
    TSDB_CHECK_CODE(code, lino, _end);
×
415
  }
416

417
  pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t));
4,958,575✔
418
  if (pUidList->tableUidList == NULL) {
4,964,780!
419
    tSimpleHashCleanup(pTableMap);
×
420
    TSDB_CHECK_NULL(pUidList->tableUidList, code, lino, _end, terrno);
×
421
  }
422

423
  pUidList->currentIndex = 0;
4,964,780✔
424

425
  for (int32_t j = 0; j < numOfTables; ++j) {
17,331,951✔
426
    pUidList->tableUidList[j] = idList[j].uid;
12,366,608✔
427

428
    STableBlockScanInfo* pScanInfo = NULL;
12,366,608✔
429
    code = getPosInBlockInfoBuf(pBuf, j, &pScanInfo);
12,366,608✔
430
    if (code != TSDB_CODE_SUCCESS) {
12,338,011!
431
      lino = __LINE__;
×
432
      break;
×
433
    }
434

435
    code = initTableBlockScanInfo(pScanInfo, idList[j].uid, pTableMap, pTsdbReader);
12,338,011✔
436
    if (code != TSDB_CODE_SUCCESS) {
12,367,171!
437
      lino = __LINE__;
×
438
      break;
×
439
    }
440
  }
441

442
  taosSort(pUidList->tableUidList, numOfTables, sizeof(uint64_t), uidComparFunc);
4,965,343✔
443

444
  pTsdbReader->cost.createScanInfoList = (taosGetTimestampUs() - st) / 1000.0;
4,964,738✔
445
  tsdbDebug("%p create %d tables scan-info, size:%.2f Kb, elapsed time:%.2f ms, %s", pTsdbReader, numOfTables,
4,964,738✔
446
            (sizeof(STableBlockScanInfo) * numOfTables) / 1024.0, pTsdbReader->cost.createScanInfoList,
447
            pTsdbReader->idStr);
448

449
  *pHashObj = pTableMap;
4,965,307✔
450

451
_end:
4,965,307✔
452
  if (code != TSDB_CODE_SUCCESS) {
4,965,307!
453
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
454
  }
455
  return code;
4,965,440✔
456
}
457

458
void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) {
39,003✔
459
  STableBlockScanInfo** p = NULL;
39,003✔
460
  int32_t               iter = 0;
39,003✔
461

462
  while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) {
78,131✔
463
    STableBlockScanInfo* pInfo = *p;
39,129✔
464
    if (pInfo == NULL) {
39,129!
465
      continue;
×
466
    }
467

468
    pInfo->iterInit = false;
39,129✔
469
    pInfo->iter.hasVal = false;
39,129✔
470
    pInfo->iiter.hasVal = false;
39,129✔
471

472
    if (pInfo->iter.iter != NULL) {
39,129✔
473
      pInfo->iter.iter = tsdbTbDataIterDestroy(pInfo->iter.iter);
18,241✔
474
    }
475

476
    if (pInfo->iiter.iter != NULL) {
39,129✔
477
      pInfo->iiter.iter = tsdbTbDataIterDestroy(pInfo->iiter.iter);
21✔
478
    }
479

480
    taosArrayDestroy(pInfo->delSkyline);
39,129✔
481
    pInfo->delSkyline = NULL;
39,128✔
482
    pInfo->lastProcKey.ts = ts;
39,128✔
483
    // todo check the nextProcKey info
484
    pInfo->sttKeyInfo.nextProcKey.ts = ts + step;
39,128✔
485
  }
486
}
39,010✔
487

488
void clearBlockScanInfo(STableBlockScanInfo* p) {
12,400,387✔
489
  if (p == NULL) {
12,400,387!
490
    return;
×
491
  }
492

493
  p->iterInit = false;
12,400,387✔
494
  p->iter.hasVal = false;
12,400,387✔
495
  p->iiter.hasVal = false;
12,400,387✔
496
  p->sttKeyInfo.status = STT_FILE_READER_UNINIT;
12,400,387✔
497

498
  if (p->iter.iter != NULL) {
12,400,387✔
499
    p->iter.iter = tsdbTbDataIterDestroy(p->iter.iter);
4,510,447✔
500
  }
501

502
  if (p->iiter.iter != NULL) {
12,401,417✔
503
    p->iiter.iter = tsdbTbDataIterDestroy(p->iiter.iter);
27,713✔
504
  }
505

506
  taosArrayDestroy(p->delSkyline);
12,401,555✔
507
  p->delSkyline = NULL;
12,401,593✔
508
  taosArrayDestroy(p->pBlockList);
12,401,593✔
509
  p->pBlockList = NULL;
12,401,216✔
510
  taosArrayDestroy(p->pBlockIdxList);
12,401,216✔
511
  p->pBlockIdxList = NULL;
12,400,203✔
512
  taosArrayDestroy(p->pMemDelData);
12,400,203✔
513
  p->pMemDelData = NULL;
12,402,308✔
514
  taosArrayDestroy(p->pFileDelData);
12,402,308✔
515
  p->pFileDelData = NULL;
12,402,137✔
516

517
  clearRowKey(&p->lastProcKey);
12,402,137✔
518
  clearRowKey(&p->sttRange.skey);
12,401,509✔
519
  clearRowKey(&p->sttRange.ekey);
12,400,831✔
520
  clearRowKey(&p->sttKeyInfo.nextProcKey);
12,400,321✔
521
}
522

523
void destroyAllBlockScanInfo(SSHashObj** pTableMap) {
4,965,896✔
524
  STableBlockScanInfo** p = NULL;
4,965,896✔
525
  int32_t               iter = 0;
4,965,896✔
526

527
  if (pTableMap == NULL || *pTableMap == NULL) {
4,965,896!
528
    return;
×
529
  }
530

531
  while ((p = tSimpleHashIterate(*pTableMap, p, &iter)) != NULL) {
17,346,678✔
532
    clearBlockScanInfo(*p);
12,380,635✔
533
  }
534

535
  tSimpleHashCleanup(*pTableMap);
4,965,596✔
536
  *pTableMap = NULL;
4,967,701✔
537
}
538

539
static void doCleanupInfoForNextFileset(STableBlockScanInfo* pScanInfo) {
14,211,437✔
540
  if (pScanInfo == NULL) {
14,211,437!
541
    return;
×
542
  }
543
  // reset the index in last block when handing a new file
544
  taosArrayClear(pScanInfo->pBlockList);
14,211,437✔
545
  taosArrayClear(pScanInfo->pBlockIdxList);
14,213,544✔
546
  taosArrayClear(pScanInfo->pFileDelData);  // del data from each file set
14,213,925✔
547
  pScanInfo->cleanSttBlocks = false;
14,213,056✔
548
  pScanInfo->numOfRowsInStt = 0;
14,213,056✔
549
  pScanInfo->sttBlockReturned = false;
14,213,056✔
550
  INIT_KEYRANGE(&pScanInfo->sttRange);
14,213,056✔
551
  INIT_TIMEWINDOW(&pScanInfo->filesetWindow);
14,213,056✔
552
  pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT;
14,213,056✔
553
}
554

555
void cleanupInfoForNextFileset(SSHashObj* pTableMap) {
6,421,286✔
556
  STableBlockScanInfo** p = NULL;
6,421,286✔
557
  int32_t               iter = 0;
6,421,286✔
558

559
  while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) {
20,634,468✔
560
    doCleanupInfoForNextFileset(*p);
14,215,925✔
561
  }
562
}
6,418,524✔
563

564
// brin records iterator
565
void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList) {
6,425,483✔
566
  if (pIter == NULL) {
6,425,483!
567
    return;
×
568
  }
569

570
  (void)memset(&pIter->block, 0, sizeof(SBrinBlock));
6,425,483✔
571
  (void)memset(&pIter->record, 0, sizeof(SBrinRecord));
6,425,483✔
572
  pIter->blockIndex = -1;
6,425,483✔
573
  pIter->recordIndex = -1;
6,425,483✔
574

575
  pIter->pReader = pReader;
6,425,483✔
576
  pIter->pBrinBlockList = pList;
6,425,483✔
577
}
578

579
int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) {
23,224,130✔
580
  int32_t code = TSDB_CODE_SUCCESS;
23,224,130✔
581
  int32_t lino = 0;
23,224,130✔
582

583
  TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA);
23,224,130!
584
  TSDB_CHECK_NULL(pRecord, code, lino, _end, TSDB_CODE_INVALID_PARA);
23,224,130!
585

586
  *pRecord = NULL;
23,224,130✔
587

588
  if (pIter->blockIndex == -1 || (pIter->recordIndex + 1) >= pIter->block.numOfRecords) {
23,224,130✔
589
    pIter->blockIndex += 1;
6,566,671✔
590
    if (pIter->blockIndex >= taosArrayGetSize(pIter->pBrinBlockList)) {
6,566,671✔
591
      goto _end;
6,328,334✔
592
    }
593

594
    pIter->pCurrentBlk = taosArrayGet(pIter->pBrinBlockList, pIter->blockIndex);
238,294✔
595
    TSDB_CHECK_NULL(pIter->pCurrentBlk, code, lino, _end, terrno);
238,852!
596

597
    tBrinBlockClear(&pIter->block);
238,852✔
598
    TSDB_CHECK_NULL(pIter->pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
238,964!
599
    code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block);
238,964✔
600
    if (code != TSDB_CODE_SUCCESS) {
238,956!
601
      tsdbError("failed to read brinBlock from file, code:%s", tstrerror(code));
×
602
      TSDB_CHECK_CODE(code, lino, _end);
×
603
    }
604

605
    pIter->recordIndex = -1;
238,945✔
606
  }
607

608
  pIter->recordIndex += 1;
16,896,404✔
609
  code = tBrinBlockGet(&pIter->block, pIter->recordIndex, &pIter->record);
16,896,404✔
610
  *pRecord = &pIter->record;
16,893,790✔
611
  TSDB_CHECK_CODE(code, lino, _end);
16,893,790!
612

613
_end:
16,893,790✔
614
  if (code != TSDB_CODE_SUCCESS) {
23,222,124!
615
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
616
  }
617
  return code;
23,221,012✔
618
}
619

620
void clearBrinBlockIter(SBrinRecordIter* pIter) {
6,422,959✔
621
  if (pIter != NULL) {
6,422,959!
622
    tBrinBlockDestroy(&pIter->block);
6,424,058✔
623
  }
624
}
6,422,553✔
625

626
// initialize the file block access order
627
//  sort the file blocks according to the offset of each data block in the files
628
static void cleanupBlockOrderSupporter(SBlockOrderSupporter* pSup) {
207,455✔
629
  if (pSup == NULL) {
207,455!
630
    return;
×
631
  }
632

633
  taosMemoryFreeClear(pSup->numOfBlocksPerTable);
207,455!
634
  taosMemoryFreeClear(pSup->indexPerTable);
207,490!
635

636
  if (pSup->pDataBlockInfo != NULL) {
207,495✔
637
    for (int32_t i = 0; i < pSup->numOfTables; ++i) {
650,565✔
638
      SBlockOrderWrapper* pBlockInfo = pSup->pDataBlockInfo[i];
443,070✔
639
      taosMemoryFreeClear(pBlockInfo);
443,070!
640
    }
641

642
    taosMemoryFreeClear(pSup->pDataBlockInfo);
207,495!
643
  }
644
}
645

646
static int32_t initBlockOrderSupporter(SBlockOrderSupporter* pSup, int32_t numOfTables) {
207,403✔
647
  int32_t code = TSDB_CODE_SUCCESS;
207,403✔
648
  int32_t lino = 0;
207,403✔
649

650
  TSDB_CHECK_NULL(pSup, code, lino, _end, TSDB_CODE_INVALID_PARA);
207,403!
651
  TSDB_CHECK_CONDITION(numOfTables >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);
207,403!
652

653
  pSup->pDataBlockInfo = taosMemoryCalloc(numOfTables, POINTER_BYTES);
207,403!
654
  TSDB_CHECK_NULL(pSup->pDataBlockInfo, code, lino, _end, terrno);
207,527!
655
  pSup->indexPerTable = taosMemoryCalloc(numOfTables, sizeof(int32_t));
207,527✔
656
  TSDB_CHECK_NULL(pSup->indexPerTable, code, lino, _end, terrno);
207,540!
657
  pSup->numOfBlocksPerTable = taosMemoryCalloc(numOfTables, sizeof(int32_t));
207,540!
658
  TSDB_CHECK_NULL(pSup->numOfBlocksPerTable, code, lino, _end, terrno);
207,545!
659
  pSup->numOfTables = 0;
207,545✔
660

661
_end:
207,545✔
662
  if (code != TSDB_CODE_SUCCESS) {
207,545!
663
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
664
  }
665
  return code;
207,441✔
666
}
667

668
static int32_t fileDataBlockOrderCompar(const void* pLeft, const void* pRight, void* param) {
2,582,258✔
669
  int32_t                     leftIndex = 0;
2,582,258✔
670
  int32_t                     rightIndex = 0;
2,582,258✔
671
  int32_t                     leftTableBlockIndex = 0;
2,582,258✔
672
  int32_t                     rightTableBlockIndex = 0;
2,582,258✔
673
  const SBlockOrderSupporter* pSupporter = NULL;
2,582,258✔
674
  const SBlockOrderWrapper*   pLeftBlock = NULL;
2,582,258✔
675
  const SBlockOrderWrapper*   pRightBlock = NULL;
2,582,258✔
676

677
  leftIndex = *(const int32_t*)pLeft;
2,582,258✔
678
  rightIndex = *(const int32_t*)pRight;
2,582,258✔
679
  pSupporter = (const SBlockOrderSupporter*)param;
2,582,258✔
680

681
  leftTableBlockIndex = pSupporter->indexPerTable[leftIndex];
2,582,258✔
682
  rightTableBlockIndex = pSupporter->indexPerTable[rightIndex];
2,582,258✔
683

684
  if (leftTableBlockIndex >= pSupporter->numOfBlocksPerTable[leftIndex]) {
2,582,258✔
685
    /* left block is empty */
686
    return 1;
728,361✔
687
  } else if (rightTableBlockIndex >= pSupporter->numOfBlocksPerTable[rightIndex]) {
1,853,897✔
688
    /* right block is empty */
689
    return -1;
235,604✔
690
  }
691

692
  pLeftBlock = &pSupporter->pDataBlockInfo[leftIndex][leftTableBlockIndex];
1,618,293✔
693
  pRightBlock = &pSupporter->pDataBlockInfo[rightIndex][rightTableBlockIndex];
1,618,293✔
694

695
  return pLeftBlock->offset > pRightBlock->offset ? 1 : -1;
1,618,293✔
696
}
697

698
int32_t recordToBlockInfo(SFileDataBlockInfo* pBlockInfo, SBrinRecord* record) {
6,675,899✔
699
  int32_t        code = TSDB_CODE_SUCCESS;
6,675,899✔
700
  int32_t        lino = 0;
6,675,899✔
701
  const SRowKey* pFirstKey = NULL;
6,675,899✔
702
  const SRowKey* pLastKey = NULL;
6,675,899✔
703

704
  TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
6,675,899!
705
  TSDB_CHECK_NULL(record, code, lino, _end, TSDB_CODE_INVALID_PARA);
6,675,899!
706

707
  pBlockInfo->uid = record->uid;
6,675,899✔
708
  pBlockInfo->firstKey = record->firstKey.key.ts;
6,675,899✔
709
  pBlockInfo->lastKey = record->lastKey.key.ts;
6,675,899✔
710
  pBlockInfo->minVer = record->minVer;
6,675,899✔
711
  pBlockInfo->maxVer = record->maxVer;
6,675,899✔
712
  pBlockInfo->blockOffset = record->blockOffset;
6,675,899✔
713
  pBlockInfo->smaOffset = record->smaOffset;
6,675,899✔
714
  pBlockInfo->blockSize = record->blockSize;
6,675,899✔
715
  pBlockInfo->blockKeySize = record->blockKeySize;
6,675,899✔
716
  pBlockInfo->smaSize = record->smaSize;
6,675,899✔
717
  pBlockInfo->numRow = record->numRow;
6,675,899✔
718
  pBlockInfo->count = record->count;
6,675,899✔
719

720
  pFirstKey = &record->firstKey.key;
6,675,899✔
721
  pLastKey = &record->lastKey.key;
6,675,899✔
722
  TSDB_CHECK_CONDITION((pFirstKey->numOfPKs == pLastKey->numOfPKs), code, lino, _end, TSDB_CODE_INVALID_PARA);
6,675,899!
723
  if (pFirstKey->numOfPKs > 0) {
6,675,899✔
724
    if (IS_NUMERIC_TYPE(pFirstKey->pks[0].type)) {
198,958!
725
      pBlockInfo->firstPk.val = VALUE_GET_TRIVIAL_DATUM(pFirstKey->pks);
198,958✔
726
      pBlockInfo->lastPk.val = VALUE_GET_TRIVIAL_DATUM(pLastKey->pks);
198,958✔
727
    } else {
728
      int32_t keyLen = pFirstKey->pks[0].nData;
×
729
      char*   p = taosMemoryMalloc(keyLen + VARSTR_HEADER_SIZE);
×
730
      TSDB_CHECK_NULL(p, code, lino, _end, terrno);
×
731
      memcpy(varDataVal(p), pFirstKey->pks[0].pData, keyLen);
×
732
      varDataSetLen(p, keyLen);
×
733
      pBlockInfo->firstPk.pData = (uint8_t*)p;
×
734

735
      keyLen = pLastKey->pks[0].nData;
×
736
      p = taosMemoryMalloc(keyLen + VARSTR_HEADER_SIZE);
×
737
      TSDB_CHECK_NULL(p, code, lino, _end, terrno);
×
738
      memcpy(varDataVal(p), pLastKey->pks[0].pData, keyLen);
×
739
      varDataSetLen(p, keyLen);
×
740
      pBlockInfo->lastPk.pData = (uint8_t*)p;
×
741
    }
742
  }
743

744
_end:
6,476,941✔
745
  if (code != TSDB_CODE_SUCCESS) {
6,675,899!
746
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
747
  }
748
  return code;
6,677,746✔
749
}
750

751
static void freePkItem(void* pItem) {
×
752
  SFileDataBlockInfo* p = pItem;
×
753
  if (p != NULL) {
×
754
    taosMemoryFreeClear(p->firstPk.pData);
×
755
    taosMemoryFreeClear(p->lastPk.pData);
×
756
  }
757
}
×
758

759
void clearDataBlockIterator(SDataBlockIter* pIter, bool needFree) {
6,640,624✔
760
  if (pIter == NULL) {
6,640,624!
761
    return;
×
762
  }
763

764
  pIter->index = -1;
6,640,624✔
765
  pIter->numOfBlocks = 0;
6,640,624✔
766

767
  if (needFree) {
6,640,624✔
768
    taosArrayClearEx(pIter->blockList, freePkItem);
224,934✔
769
  } else {
770
    taosArrayClear(pIter->blockList);
6,415,690✔
771
  }
772
}
773

774
void cleanupDataBlockIterator(SDataBlockIter* pIter, bool needFree) {
5,030,192✔
775
  if (pIter == NULL) {
5,030,192!
776
    return;
×
777
  }
778

779
  pIter->index = -1;
5,030,192✔
780
  pIter->numOfBlocks = 0;
5,030,192✔
781
  if (needFree) {
5,030,192✔
782
    taosArrayDestroyEx(pIter->blockList, freePkItem);
28,653✔
783
  } else {
784
    taosArrayDestroy(pIter->blockList);
5,001,539✔
785
  }
786
  pIter->blockList = NULL;
5,032,363✔
787
}
788

789
int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList) {
207,429✔
790
  int32_t                   code = TSDB_CODE_SUCCESS;
207,429✔
791
  int32_t                   lino = 0;
207,429✔
792
  bool                      asc = false;
207,429✔
793
  int32_t                   numOfTables = 0;
207,429✔
794
  int64_t                   st = 0;
207,429✔
795
  int64_t                   et = 0;
207,429✔
796
  int32_t                   cnt = 0;
207,429✔
797
  SBlockOrderSupporter      sup = {0};
207,429✔
798
  SMultiwayMergeTreeInfo*   pTree = NULL;
207,429✔
799
  STableBlockScanInfo*      pTableScanInfo = NULL;
207,429✔
800
  const SFileDataBlockInfo* pBlockInfo = NULL;
207,429✔
801
  const void*               px = NULL;
207,429✔
802

803
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
207,429!
804
  TSDB_CHECK_NULL(pBlockIter, code, lino, _end, TSDB_CODE_INVALID_PARA);
207,429!
805
  TSDB_CHECK_CONDITION(numOfBlocks >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);
207,429!
806

807
  asc = ASCENDING_TRAVERSE(pReader->info.order);
207,429✔
808
  clearDataBlockIterator(pBlockIter, shouldFreePkBuf(&pReader->suppInfo));
207,429✔
809
  pBlockIter->numOfBlocks = numOfBlocks;
207,470✔
810

811
  // access data blocks according to the offset of each block in asc/desc order.
812
  numOfTables = taosArrayGetSize(pTableList);
207,470✔
813

814
  st = taosGetTimestampUs();
207,478✔
815
  code = initBlockOrderSupporter(&sup, numOfTables);
207,478✔
816
  TSDB_CHECK_CODE(code, lino, _end);
207,435!
817

818
  for (int32_t i = 0; i < numOfTables; ++i) {
650,450✔
819
    pTableScanInfo = taosArrayGetP(pTableList, i);
442,858✔
820

821
    size_t              num = taosArrayGetSize(pTableScanInfo->pBlockList);
442,841✔
822
    SBlockOrderWrapper* buf = taosMemoryMalloc(sizeof(SBlockOrderWrapper) * num);
442,874!
823
    TSDB_CHECK_NULL(buf, code, lino, _end, terrno);
443,019!
824
    sup.numOfBlocksPerTable[sup.numOfTables] = num;
443,019✔
825
    sup.pDataBlockInfo[sup.numOfTables] = buf;
443,019✔
826
    sup.numOfTables++;
443,019✔
827

828
    for (int32_t k = 0; k < num; ++k) {
7,121,217✔
829
      pBlockInfo = taosArrayGet(pTableScanInfo->pBlockList, k);
6,678,202✔
830
      TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
6,678,198!
831

832
      sup.pDataBlockInfo[i][k] =
6,678,198✔
833
          (SBlockOrderWrapper){.uid = pTableScanInfo->uid, .offset = pBlockInfo->blockOffset, .pInfo = pTableScanInfo};
6,678,198✔
834
      cnt++;
6,678,198✔
835
    }
836
  }
837

838
  TSDB_CHECK_CONDITION(!(numOfBlocks != cnt && sup.numOfTables != numOfTables), code, lino, _end,
207,592!
839
                       TSDB_CODE_INVALID_PARA);
840

841
  // since there is only one table qualified, blocks are not sorted
842
  if (sup.numOfTables == 1) {
207,592✔
843
    pTableScanInfo = taosArrayGetP(pTableList, 0);
148,166✔
844
    for (int32_t i = 0; i < numOfBlocks; ++i) {
6,369,733✔
845
      STableDataBlockIdx tableDataBlockIdx = {.globalIndex = i};
6,221,539✔
846
      px = taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx);
6,221,539✔
847
      TSDB_CHECK_NULL(px, code, lino, _end, terrno);
6,221,545!
848
    }
849

850
    px = taosArrayAddAll(pBlockIter->blockList, pTableScanInfo->pBlockList);
148,194✔
851
    TSDB_CHECK_NULL(px, code, lino, _end, terrno);
148,167!
852

853
    taosArrayDestroy(pTableScanInfo->pBlockList);
148,167✔
854
    pTableScanInfo->pBlockList = NULL;
148,217✔
855

856
    et = taosGetTimestampUs();
148,208✔
857
    tsdbDebug("%p create blocks info struct completed for one table, %d blocks not sorted, elapsed time:%.2f ms %s",
148,208✔
858
              pReader, numOfBlocks, (et - st) / 1000.0, pReader->idStr);
859

860
    pBlockIter->index = asc ? 0 : (numOfBlocks - 1);
148,220✔
861
    goto _end;
148,220✔
862
  }
863

864
  tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables %s", pReader, cnt, sup.numOfTables,
59,426✔
865
            pReader->idStr);
866

867
  code = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, fileDataBlockOrderCompar);
59,426✔
868
  TSDB_CHECK_CODE(code, lino, _end);
59,301!
869

870
  for (int32_t i = 0; i < cnt; ++i) {
515,371✔
871
    int32_t pos = tMergeTreeGetChosenIndex(pTree);
456,076✔
872
    int32_t index = sup.indexPerTable[pos]++;
456,076✔
873
    pTableScanInfo = sup.pDataBlockInfo[pos][index].pInfo;
456,076✔
874

875
    pBlockInfo = taosArrayGet(pTableScanInfo->pBlockList, index);
456,076✔
876
    TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, terrno);
456,072!
877

878
    px = taosArrayPush(pBlockIter->blockList, pBlockInfo);
456,072✔
879
    TSDB_CHECK_NULL(px, code, lino, _end, terrno);
455,943!
880

881
    STableDataBlockIdx tableDataBlockIdx = {.globalIndex = i};
455,943✔
882
    px = taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx);
455,943✔
883
    TSDB_CHECK_NULL(px, code, lino, _end, terrno);
456,021!
884

885
    // set data block index overflow, in order to disable the offset comparator
886
    if (sup.indexPerTable[pos] >= sup.numOfBlocksPerTable[pos]) {
456,021✔
887
      sup.indexPerTable[pos] = sup.numOfBlocksPerTable[pos] + 1;
294,869✔
888
    }
889

890
    code = tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree));
456,021✔
891
    TSDB_CHECK_CODE(code, lino, _end);
456,070!
892
  }
893

894
  for (int32_t i = 0; i < numOfTables; ++i) {
354,197✔
895
    pTableScanInfo = taosArrayGetP(pTableList, i);
294,895✔
896
    taosArrayDestroy(pTableScanInfo->pBlockList);
294,876✔
897
    pTableScanInfo->pBlockList = NULL;
294,902✔
898
  }
899

900
  et = taosGetTimestampUs();
59,302✔
901
  tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, numOfBlocks,
59,302✔
902
            (et - st) / 1000.0, pReader->idStr);
903

904
  pBlockIter->index = asc ? 0 : (numOfBlocks - 1);
59,302✔
905

906
_end:
207,522✔
907
  if (code != TSDB_CODE_SUCCESS) {
207,522!
908
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
909
  }
910
  cleanupBlockOrderSupporter(&sup);
207,522✔
911
  if (pTree != NULL) {
207,450✔
912
    tMergeTreeDestroy(&pTree);
59,302✔
913
  }
914
  return code;
207,471✔
915
}
916

917
bool blockIteratorNext(SDataBlockIter* pBlockIter) {
6,673,996✔
918
  bool asc = false;
6,673,996✔
919

920
  if (pBlockIter == NULL) {
6,673,996!
921
    return false;
×
922
  }
923

924
  asc = ASCENDING_TRAVERSE(pBlockIter->order);
6,673,996✔
925
  if ((pBlockIter->index >= pBlockIter->numOfBlocks - 1 && asc) || (pBlockIter->index <= 0 && (!asc))) {
6,673,996✔
926
    return false;
206,712✔
927
  }
928
  pBlockIter->index += asc ? 1 : -1;
6,467,284✔
929
  return true;
6,467,284✔
930
}
931

932
typedef enum {
933
  BLK_CHECK_CONTINUE = 0x1,
934
  BLK_CHECK_QUIT = 0x2,
935
} ETombBlkCheckEnum;
936

937
static int32_t loadNextStatisticsBlock(SSttFileReader* pSttFileReader, STbStatisBlock* pStatisBlock,
938
                                       const TStatisBlkArray* pStatisBlkArray, int32_t numOfRows, int32_t* i,
939
                                       int32_t* j);
940
static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_t numOfTables, int32_t* j,
232,503✔
941
                                ETombBlkCheckEnum* pRet) {
942
  int32_t              code = TSDB_CODE_SUCCESS;
232,503✔
943
  int32_t              lino = 0;
232,503✔
944
  STombRecord          record = {0};
232,503✔
945
  uint64_t             uid = 0;
232,503✔
946
  STableBlockScanInfo* pScanInfo = NULL;
232,503✔
947

948
  TSDB_CHECK_NULL(pBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
232,503!
949
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
232,503!
950
  TSDB_CHECK_NULL(pRet, code, lino, _end, TSDB_CODE_INVALID_PARA);
232,503!
951

952
  *pRet = BLK_CHECK_QUIT;
232,503✔
953
  uid = pReader->status.uidList.tableUidList[*j];
232,503✔
954
  code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr);
232,503✔
955
  TSDB_CHECK_CODE(code, lino, _end);
232,504!
956

957
  if (pScanInfo->pFileDelData == NULL) {
232,504✔
958
    pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData));
99,089✔
959
    TSDB_CHECK_NULL(pScanInfo->pFileDelData, code, lino, _end, terrno);
99,088✔
960
  }
961

962
  for (int32_t k = 0; k < pBlock->numOfRecords; ++k) {
168,371,022✔
963
    code = tTombBlockGet(pBlock, k, &record);
168,337,690✔
964
    TSDB_CHECK_CODE(code, lino, _end);
168,322,187!
965

966
    if (record.suid < pReader->info.suid) {
168,322,187✔
967
      continue;
921,856✔
968
    }
969

970
    if (record.suid > pReader->info.suid) {
167,400,331✔
971
      *pRet = BLK_CHECK_QUIT;
3,766✔
972
      goto _end;
3,766✔
973
    }
974

975
    if (uid < record.uid) {
167,396,565✔
976
      while ((*j) < numOfTables && pReader->status.uidList.tableUidList[*j] < record.uid) {
454,999✔
977
        (*j) += 1;
228,424✔
978
      }
979

980
      if ((*j) >= numOfTables) {
226,575✔
981
        *pRet = BLK_CHECK_QUIT;
203,650✔
982
        goto _end;
203,650✔
983
      }
984

985
      uid = pReader->status.uidList.tableUidList[*j];
22,925✔
986
      code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr);
22,925✔
987
      TSDB_CHECK_CODE(code, lino, _end);
22,930!
988

989
      if (pScanInfo->pFileDelData == NULL) {
22,930✔
990
        pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData));
5,729✔
991
        TSDB_CHECK_NULL(pScanInfo->pFileDelData, code, lino, _end, terrno);
5,744!
992
      }
993
    }
994

995
    if (record.uid < uid) {
167,216,588✔
996
      continue;
166,363,615✔
997
    }
998

999
    TSDB_CHECK_CONDITION((record.suid == pReader->info.suid) && (uid == record.uid), code, lino, _end,
852,973!
1000
                         TSDB_CODE_INTERNAL_ERROR);
1001

1002
    if (record.version <= pReader->info.verRange.maxVer) {
852,973!
1003
      SDelData    delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey};
852,976✔
1004
      const void* px = taosArrayPush(pScanInfo->pFileDelData, &delData);
852,976✔
1005
      TSDB_CHECK_NULL(px, code, lino, _end, terrno);
853,052!
1006
    }
1007
  }
1008

1009
  *pRet = BLK_CHECK_CONTINUE;
33,332✔
1010

1011
_end:
240,748✔
1012
  if (code != TSDB_CODE_SUCCESS) {
240,748!
1013
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1014
  }
1015
  return code;
232,503✔
1016
}
1017

1018
// load tomb data API
1019
static int32_t doLoadTombDataFromTombBlk(const TTombBlkArray* pTombBlkArray, STsdbReader* pReader, void* pFileReader,
6,473,458✔
1020
                                         bool isFile) {
1021
  int32_t              code = TSDB_CODE_SUCCESS;
6,473,458✔
1022
  int32_t              lino = 0;
6,473,458✔
1023
  const STableUidList* pList = NULL;
6,473,458✔
1024
  int32_t              numOfTables = 0;
6,473,458✔
1025

1026
  TSDB_CHECK_NULL(pTombBlkArray, code, lino, _end, TSDB_CODE_INVALID_PARA);
6,473,458!
1027
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
6,473,458!
1028

1029
  pList = &pReader->status.uidList;
6,473,458✔
1030
  numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);
6,473,458✔
1031

1032
  int32_t i = 0, j = 0;
6,474,682✔
1033
  while (i < pTombBlkArray->size && j < numOfTables) {
6,546,156!
1034
    STombBlk* pTombBlk = &pTombBlkArray->data[i];
287,187✔
1035
    if (pTombBlk->maxTbid.suid < pReader->info.suid) {
287,187✔
1036
      i += 1;
9,211✔
1037
      continue;
46,386✔
1038
    }
1039

1040
    if (pTombBlk->minTbid.suid > pReader->info.suid) {
277,976✔
1041
      break;
215,716✔
1042
    }
1043

1044
    TSDB_CHECK_CONDITION(
272,091!
1045
        (pTombBlk->minTbid.suid <= pReader->info.suid) && (pTombBlk->maxTbid.suid >= pReader->info.suid), code, lino,
1046
        _end, TSDB_CODE_INTERNAL_ERROR);
1047

1048
    if (pTombBlk->maxTbid.suid == pReader->info.suid && pTombBlk->maxTbid.uid < pList->tableUidList[0]) {
272,091✔
1049
      i += 1;
37,175✔
1050
      continue;
37,175✔
1051
    }
1052

1053
    if (pTombBlk->minTbid.suid == pReader->info.suid && pTombBlk->minTbid.uid > pList->tableUidList[numOfTables - 1]) {
234,916✔
1054
      break;
2,415✔
1055
    }
1056

1057
    STombBlock block = {0};
232,501✔
1058
    code = isFile ? tsdbDataFileReadTombBlock(pFileReader, &pTombBlkArray->data[i], &block)
118,613✔
1059
                  : tsdbSttFileReadTombBlock(pFileReader, &pTombBlkArray->data[i], &block);
232,501✔
1060
    TSDB_CHECK_CODE(code, lino, _end);
232,503!
1061

1062
    ETombBlkCheckEnum ret = 0;
232,503✔
1063
    code = doCheckTombBlock(&block, pReader, numOfTables, &j, &ret);
232,503✔
1064

1065
    tTombBlockDestroy(&block);
232,502✔
1066
    TSDB_CHECK_CODE(code, lino, _end);
232,504!
1067
    if (ret == BLK_CHECK_QUIT) {
232,504✔
1068
      break;
207,416✔
1069
    }
1070

1071
    i += 1;
25,088✔
1072
  }
1073

1074
_end:
6,258,969✔
1075
  if (code != TSDB_CODE_SUCCESS) {
6,474,685!
1076
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1077
  }
1078
  return code;
6,474,497✔
1079
}
1080

1081
int32_t loadDataFileTombDataForAll(STsdbReader* pReader) {
8,743,439✔
1082
  int32_t              code = TSDB_CODE_SUCCESS;
8,743,439✔
1083
  int32_t              lino = 0;
8,743,439✔
1084
  const TTombBlkArray* pBlkArray = NULL;
8,743,439✔
1085

1086
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
8,743,439!
1087

1088
  if ((pReader->status.pCurrentFileset == NULL) || (pReader->status.pCurrentFileset->farr[TSDB_FTYPE_TOMB] == NULL)) {
8,743,439!
1089
    return TSDB_CODE_SUCCESS;
8,616,827✔
1090
  }
1091

1092
  code = tsdbDataFileReadTombBlk(pReader->pFileReader, &pBlkArray);
126,612✔
1093
  TSDB_CHECK_CODE(code, lino, _end);
126,611!
1094

1095
  code = doLoadTombDataFromTombBlk(pBlkArray, pReader, pReader->pFileReader, true);
126,611✔
1096
  TSDB_CHECK_CODE(code, lino, _end);
126,611!
1097

1098
_end:
126,611✔
1099
  if (code != TSDB_CODE_SUCCESS) {
126,611!
1100
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1101
  }
1102
  return code;
126,611✔
1103
}
1104

1105
int32_t loadSttTombDataForAll(STsdbReader* pReader, SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pLoadInfo) {
6,348,772✔
1106
  int32_t              code = TSDB_CODE_SUCCESS;
6,348,772✔
1107
  int32_t              lino = 0;
6,348,772✔
1108
  const TTombBlkArray* pBlkArray = NULL;
6,348,772✔
1109

1110
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
6,348,772!
1111

1112
  code = tsdbSttFileReadTombBlk(pSttFileReader, &pBlkArray);
6,348,772✔
1113
  TSDB_CHECK_CODE(code, lino, _end);
6,348,858!
1114

1115
  code = doLoadTombDataFromTombBlk(pBlkArray, pReader, pSttFileReader, false);
6,348,858✔
1116
  TSDB_CHECK_CODE(code, lino, _end);
6,348,100!
1117

1118
_end:
6,348,100✔
1119
  if (code != TSDB_CODE_SUCCESS) {
6,348,100!
1120
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1121
  }
1122
  return code;
6,349,098✔
1123
}
1124

1125
int32_t loadMemTombData(SArray** ppMemDelData, STbData* pMemTbData, STbData* piMemTbData, int64_t ver) {
12,316,751✔
1126
  int32_t         code = TSDB_CODE_SUCCESS;
12,316,751✔
1127
  int32_t         lino = 0;
12,316,751✔
1128
  SArray*         pMemDelData = NULL;
12,316,751✔
1129
  const SDelData* p = NULL;
12,316,751✔
1130
  const void*     px = NULL;
12,316,751✔
1131

1132
  TSDB_CHECK_NULL(ppMemDelData, code, lino, _end, TSDB_CODE_INVALID_PARA);
12,316,751!
1133

1134
  if (*ppMemDelData == NULL) {
12,316,751✔
1135
    *ppMemDelData = taosArrayInit(4, sizeof(SDelData));
12,299,715✔
1136
    TSDB_CHECK_NULL(*ppMemDelData, code, lino, _end, terrno);
12,302,635!
1137
  }
1138

1139
  pMemDelData = *ppMemDelData;
12,319,671✔
1140

1141
  if (pMemTbData != NULL) {
12,319,671✔
1142
    taosRLockLatch(&pMemTbData->lock);
4,542,184✔
1143
    p = pMemTbData->pHead;
4,542,953✔
1144
    while (p) {
5,228,029✔
1145
      if (p->version <= ver) {
685,078✔
1146
        px = taosArrayPush(pMemDelData, p);
679,676✔
1147
        if (px == NULL) {
679,676!
1148
          taosRUnLockLatch(&pMemTbData->lock);
×
1149
          TSDB_CHECK_NULL(px, code, lino, _end, terrno);
×
1150
        }
1151
      }
1152
      p = p->pNext;
685,076✔
1153
    }
1154
    taosRUnLockLatch(&pMemTbData->lock);
4,542,951✔
1155
  }
1156

1157
  if (piMemTbData != NULL) {
12,321,653✔
1158
    p = piMemTbData->pHead;
42,074✔
1159
    while (p) {
97,565✔
1160
      if (p->version <= ver) {
55,493!
1161
        px = taosArrayPush(pMemDelData, p);
55,491✔
1162
        TSDB_CHECK_NULL(px, code, lino, _end, terrno);
55,491!
1163
      }
1164
      p = p->pNext;
55,491✔
1165
    }
1166
  }
1167

1168
_end:
12,321,651✔
1169
  if (code != TSDB_CODE_SUCCESS) {
12,321,651!
1170
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1171
  }
1172
  return code;
12,321,125✔
1173
}
1174

1175
int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo* pBlockLoadInfo,
4,871✔
1176
                               TStatisBlkArray* pStatisBlkArray, uint64_t suid, const uint64_t* pUidList,
1177
                               int32_t numOfTables, int32_t* pNumOfRows) {
1178
  int32_t           code = TSDB_CODE_SUCCESS;
4,871✔
1179
  int32_t           lino = 0;
4,871✔
1180
  int32_t           num = 0;
4,871✔
1181
  int64_t           st = 0;
4,871✔
1182
  const SStatisBlk* p = NULL;
4,871✔
1183
  STbStatisBlock*   pStatisBlock = NULL;
4,871✔
1184

1185
  TSDB_CHECK_NULL(pSttFileReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
4,871!
1186
  TSDB_CHECK_NULL(pBlockLoadInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
4,871!
1187
  TSDB_CHECK_NULL(pStatisBlkArray, code, lino, _end, TSDB_CODE_INVALID_PARA);
4,871!
1188
  TSDB_CHECK_NULL(pNumOfRows, code, lino, _end, TSDB_CODE_INVALID_PARA);
4,871!
1189

1190
  *pNumOfRows = 0;
4,871✔
1191

1192
  if (TARRAY2_SIZE(pStatisBlkArray) <= 0) {
4,871!
1193
    return code;
×
1194
  }
1195

1196
  int32_t i = 0;
4,871✔
1197
  while ((i < TARRAY2_SIZE(pStatisBlkArray)) && (pStatisBlkArray->data[i].maxTbid.suid < suid)) {
6,815✔
1198
    ++i;
1,944✔
1199
  }
1200

1201
  if (i >= TARRAY2_SIZE(pStatisBlkArray)) {
4,871✔
1202
    goto _end;
1,944✔
1203
  }
1204

1205
  p = &pStatisBlkArray->data[i];
2,927✔
1206
  pStatisBlock = taosMemoryCalloc(1, sizeof(STbStatisBlock));
2,927!
1207
  TSDB_CHECK_NULL(pStatisBlock, code, lino, _end, terrno);
2,933!
1208

1209
  code = tStatisBlockInit(pStatisBlock);
2,933✔
1210
  TSDB_CHECK_CODE(code, lino, _end);
2,937!
1211

1212
  st = taosGetTimestampUs();
2,933✔
1213
  code = tsdbSttFileReadStatisBlock(pSttFileReader, p, pStatisBlock);
2,933✔
1214
  TSDB_CHECK_CODE(code, lino, _end);
2,941!
1215

1216
  double el = (taosGetTimestampUs() - st) / 1000.0;
2,936✔
1217
  pBlockLoadInfo->cost.loadStatisBlocks += 1;
2,936✔
1218
  pBlockLoadInfo->cost.statisElapsedTime += el;
2,936✔
1219

1220
  int32_t index = 0;
2,936✔
1221
  while (index < pStatisBlock->numOfRecords && ((int64_t*)pStatisBlock->suids.data)[index] < suid) {
5,852✔
1222
    ++index;
2,916✔
1223
  }
1224

1225
  if (index >= pStatisBlock->numOfRecords) {
2,936!
1226
    *pNumOfRows = num;
×
1227
    goto _end;
×
1228
  }
1229

1230
  int32_t j = index;
2,936✔
1231
  int32_t uidIndex = 0;
2,936✔
1232
  while (i < TARRAY2_SIZE(pStatisBlkArray) && uidIndex < numOfTables) {
7,820✔
1233
    p = &pStatisBlkArray->data[i];
4,886✔
1234
    if (p->minTbid.suid > suid) {
4,886✔
1235
      *pNumOfRows = num;
1✔
1236
      goto _end;
1✔
1237
    }
1238

1239
    uint64_t uid = pUidList[uidIndex];
4,885✔
1240

1241
    if (((int64_t*)pStatisBlock->uids.data)[j] == uid) {
4,885✔
1242
      num += ((int64_t*)pStatisBlock->counts.data)[j];
2,942✔
1243
      uidIndex += 1;
2,942✔
1244
      j += 1;
2,942✔
1245
      code = loadNextStatisticsBlock(pSttFileReader, pStatisBlock, pStatisBlkArray, pStatisBlock->numOfRecords, &i, &j);
2,942✔
1246
      TSDB_CHECK_CODE(code, lino, _end);
2,941!
1247
    } else if (((int64_t*)pStatisBlock->uids.data)[j] < uid) {
1,943!
1248
      j += 1;
×
1249
      code = loadNextStatisticsBlock(pSttFileReader, pStatisBlock, pStatisBlkArray, pStatisBlock->numOfRecords, &i, &j);
×
1250
      TSDB_CHECK_CODE(code, lino, _end);
×
1251
    } else {
1252
      uidIndex += 1;
1,943✔
1253
    }
1254
  }
1255

1256
  *pNumOfRows = num;
2,934✔
1257

1258
_end:
4,879✔
1259
  if (code != TSDB_CODE_SUCCESS) {
4,879!
1260
    tsdbError("%s with %p failed at line %d since %s", __func__, pSttFileReader, lino, tstrerror(code));
×
1261
  }
1262
  if (pStatisBlock) {
4,883✔
1263
    tStatisBlockDestroy(pStatisBlock);
2,938✔
1264
    taosMemoryFreeClear(pStatisBlock);
2,940!
1265
  }
1266
  return code;
4,884✔
1267
}
1268

1269
// load next stt statistics block
1270
static int32_t loadNextStatisticsBlock(SSttFileReader* pSttFileReader, STbStatisBlock* pStatisBlock,
2,941✔
1271
                                       const TStatisBlkArray* pStatisBlkArray, int32_t numOfRows, int32_t* i,
1272
                                       int32_t* j) {
1273
  int32_t code = TSDB_CODE_SUCCESS;
2,941✔
1274
  int32_t lino = 0;
2,941✔
1275

1276
  if ((*j) >= numOfRows) {
2,941✔
1277
    (*i) += 1;
1,969✔
1278
    (*j) = 0;
1,969✔
1279
    if ((*i) < TARRAY2_SIZE(pStatisBlkArray)) {
1,969!
1280
      code = tsdbSttFileReadStatisBlock(pSttFileReader, &pStatisBlkArray->data[(*i)], pStatisBlock);
×
1281
      if (code != 0) {
×
1282
        tsdbError("%p failed to read statisBlock, code:%s", pSttFileReader, tstrerror(code));
×
1283
        TSDB_CHECK_CODE(code, lino, _end);
×
1284
      }
1285
    }
1286
  }
1287

1288
_end:
2,941✔
1289
  if (code != TSDB_CODE_SUCCESS) {
2,941!
1290
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1291
  }
1292
  return code;
2,942✔
1293
}
1294

1295
int32_t doAdjustValidDataIters(SArray* pLDIterList, int32_t numOfFileObj) {
13,953,322✔
1296
  int32_t     code = TSDB_CODE_SUCCESS;
13,953,322✔
1297
  int32_t     lino = 0;
13,953,322✔
1298
  int32_t     size = 0;
13,953,322✔
1299
  int32_t     inc = 0;
13,953,322✔
1300
  SLDataIter* pIter = NULL;
13,953,322✔
1301

1302
  size = taosArrayGetSize(pLDIterList);
13,953,322✔
1303

1304
  if (size < numOfFileObj) {
13,953,268✔
1305
    inc = numOfFileObj - size;
6,321,296✔
1306
    for (int32_t k = 0; k < inc; ++k) {
12,679,141✔
1307
      pIter = taosMemoryCalloc(1, sizeof(SLDataIter));
6,351,102!
1308
      TSDB_CHECK_NULL(pIter, code, lino, _end, terrno);
6,358,811!
1309
      void* px = taosArrayPush(pLDIterList, &pIter);
6,357,845✔
1310
      if (px == NULL) {
6,357,845!
1311
        TSDB_CHECK_NULL(px, code, lino, _end, terrno);
×
1312
        taosMemoryFreeClear(pIter);
×
1313
      }
1314
    }
1315
  } else if (size > numOfFileObj) {  // remove unused LDataIter
7,631,972!
1316
    inc = size - numOfFileObj;
×
1317

1318
    for (int32_t i = 0; i < inc; ++i) {
×
1319
      pIter = taosArrayPop(pLDIterList);
×
1320
      destroyLDataIter(pIter);
×
1321
    }
1322
  }
1323

1324
_end:
7,631,972✔
1325
  if (code != TSDB_CODE_SUCCESS) {
13,960,011!
1326
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1327
  }
1328
  return code;
13,960,547✔
1329
}
1330

1331
int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet) {
13,802,330✔
1332
  int32_t  code = TSDB_CODE_SUCCESS;
13,802,330✔
1333
  int32_t  lino = 0;
13,802,330✔
1334
  int32_t  numOfLevels = 0;
13,802,330✔
1335
  SSttLvl* pSttLevel = NULL;
13,802,330✔
1336
  SArray*  pList = NULL;
13,802,330✔
1337

1338
  TSDB_CHECK_NULL(pFileSet, code, lino, _end, TSDB_CODE_INVALID_PARA);
13,802,330!
1339

1340
  numOfLevels = pFileSet->lvlArr->size;
13,802,330✔
1341

1342
  // add the list/iter placeholder
1343
  while (taosArrayGetSize(pSttFileBlockIterArray) < numOfLevels) {
20,126,534✔
1344
    pList = taosArrayInit(4, POINTER_BYTES);
6,311,202✔
1345
    TSDB_CHECK_NULL(pList, code, lino, _end, terrno);
6,329,973!
1346
    void* px = taosArrayPush(pSttFileBlockIterArray, &pList);
6,324,204✔
1347
    if (px == NULL) {
6,324,204!
1348
      taosArrayDestroy(pList);
×
1349
      TSDB_CHECK_NULL(px, code, lino, _end, terrno);
×
1350
    }
1351
  }
1352

1353
  for (int32_t j = 0; j < numOfLevels; ++j) {
27,764,529✔
1354
    pSttLevel = pFileSet->lvlArr->data[j];
13,962,355✔
1355
    pList = taosArrayGetP(pSttFileBlockIterArray, j);
13,962,355✔
1356
    code = doAdjustValidDataIters(pList, TARRAY2_SIZE(pSttLevel->fobjArr));
13,954,079✔
1357
    TSDB_CHECK_CODE(code, lino, _end);
13,961,167!
1358
  }
1359

1360
_end:
13,802,174✔
1361
  if (code != TSDB_CODE_SUCCESS) {
13,802,174!
1362
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1363
  }
1364
  return code;
13,800,907✔
1365
}
1366

1367
int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArray, STsdb* pTsdb, SMergeTreeConf* pConf,
4,864✔
1368
                              const char* pstr) {
1369
  int32_t code = TSDB_CODE_SUCCESS;
4,864✔
1370
  int32_t lino = 0;
4,864✔
1371
  int32_t numOfRows = 0;
4,864✔
1372
  int32_t numOfLevels = 0;
4,864✔
1373

1374
  TSDB_CHECK_NULL(pFileSet, code, lino, _end, TSDB_CODE_INVALID_PARA);
4,864!
1375

1376
  // no data exists, go to end
1377
  numOfLevels = pFileSet->lvlArr->size;
4,864✔
1378
  if (numOfLevels == 0) {
4,864!
1379
    goto _end;
×
1380
  }
1381

1382
  // add the list/iter placeholder
1383
  code = adjustSttDataIters(pSttFileBlockIterArray, pFileSet);
4,864✔
1384
  TSDB_CHECK_CODE(code, lino, _end);
4,874!
1385

1386
  for (int32_t j = 0; j < numOfLevels; ++j) {
9,754✔
1387
    SSttLvl* pSttLevel = pFileSet->lvlArr->data[j];
4,873✔
1388
    SArray*  pList = taosArrayGetP(pSttFileBlockIterArray, j);
4,873✔
1389

1390
    for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) {  // open all last file
9,747✔
1391
      SLDataIter* pIter = taosArrayGetP(pList, i);
4,842✔
1392

1393
      // open stt file reader if not opened yet
1394
      // if failed to open this stt file, ignore the error and try next one
1395
      if (pIter->pReader == NULL) {
4,855!
1396
        SSttFileReaderConfig conf = {.tsdb = pTsdb, .szPage = pTsdb->pVnode->config.tsdbPageSize};
4,855✔
1397
        conf.file[0] = *pSttLevel->fobjArr->data[i]->f;
4,855✔
1398

1399
        const char* pName = pSttLevel->fobjArr->data[i]->fname;
4,855✔
1400
        code = tsdbSttFileReaderOpen(pName, &conf, &pIter->pReader);
4,855✔
1401
        if (code != TSDB_CODE_SUCCESS) {
4,878!
1402
          tsdbError("open stt file reader error. file:%s, code %s, %s", pName, tstrerror(code), pstr);
×
1403
          continue;
×
1404
        }
1405
      }
1406

1407
      if (pIter->pBlockLoadInfo == NULL) {
4,879!
1408
        code = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols, &pIter->pBlockLoadInfo);
4,880✔
1409
        if (code != TSDB_CODE_SUCCESS) {
4,871!
1410
          tsdbError("failed to create block load info, code: out of memory, %s", pstr);
×
1411
          continue;
×
1412
        }
1413
      }
1414

1415
      // load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file
1416
      TStatisBlkArray* pStatisBlkArray = NULL;
4,870✔
1417
      code = tsdbSttFileReadStatisBlk(pIter->pReader, (const TStatisBlkArray**)&pStatisBlkArray);
4,870✔
1418
      if (code != TSDB_CODE_SUCCESS) {
4,876!
1419
        tsdbError("failed to load stt block statistics, code:%s, %s", tstrerror(code), pstr);
×
1420
        continue;
×
1421
      }
1422

1423
      // extract rows from each stt file one-by-one
1424
      STsdbReader* pReader = pConf->pReader;
4,876✔
1425
      int32_t      numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);
4,876✔
1426
      uint64_t*    pUidList = pReader->status.uidList.tableUidList;
4,872✔
1427
      int32_t      n = 0;
4,872✔
1428
      code = getNumOfRowsInSttBlock(pIter->pReader, pIter->pBlockLoadInfo, pStatisBlkArray, pConf->suid, pUidList,
4,872✔
1429
                                    numOfTables, &n);
1430
      numOfRows += n;
4,884✔
1431
      if (code) {
4,884!
1432
        tsdbError("%s failed to get rows in stt blocks, code:%s", pstr, tstrerror(code));
×
1433
      }
1434
    }
1435
  }
1436

1437
_end:
4,881✔
1438
  if (code != TSDB_CODE_SUCCESS) {
4,881!
1439
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1440
  }
1441
  return numOfRows;
4,880✔
1442
}
1443

1444
static bool overlapHelper(const STimeWindow* pLeft, TSKEY minKey, TSKEY maxKey) {
36,183✔
1445
  return (pLeft != NULL) && (pLeft->ekey >= minKey) && (pLeft->skey <= maxKey);
36,183!
1446
}
1447

1448
static bool overlapWithTimeWindow(STimeWindow* p1, STimeWindow* pQueryWindow, STableBlockScanInfo* pBlockScanInfo,
93,305✔
1449
                                  int32_t order) {
1450
  SIterInfo*   pMemIter = NULL;
93,305✔
1451
  SIterInfo*   pIMemIter = NULL;
93,305✔
1452
  STbData*     pTbData = NULL;
93,305✔
1453
  STimeWindow* pFileWin = NULL;
93,305✔
1454

1455
  if (p1 == NULL || pQueryWindow == NULL) {
93,305!
1456
    return false;
×
1457
  }
1458

1459
  // overlap with query window
1460
  if (!(p1->skey >= pQueryWindow->skey && p1->ekey <= pQueryWindow->ekey)) {
93,350✔
1461
    return true;
157✔
1462
  }
1463

1464
  if (pBlockScanInfo == NULL) {
93,193!
1465
    return false;
×
1466
  }
1467

1468
  pMemIter = &pBlockScanInfo->iter;
93,193✔
1469
  pIMemIter = &pBlockScanInfo->iiter;
93,193✔
1470

1471
  // overlap with mem data
1472
  if (pMemIter->hasVal) {
93,193✔
1473
    pTbData = pMemIter->iter->pTbData;
32,604✔
1474
    if (overlapHelper(p1, pTbData->minKey, pTbData->maxKey)) {
32,604✔
1475
      return true;
32,423✔
1476
    }
1477
  }
1478

1479
  // overlap with imem data
1480
  if (pIMemIter->hasVal) {
60,770✔
1481
    pTbData = pIMemIter->iter->pTbData;
1,387✔
1482
    if (overlapHelper(p1, pTbData->minKey, pTbData->maxKey)) {
1,387✔
1483
      return true;
1,378✔
1484
    }
1485
  }
1486

1487
  // overlap with data file block
1488
  pFileWin = &pBlockScanInfo->filesetWindow;
59,392✔
1489
  if ((taosArrayGetSize(pBlockScanInfo->pBlockIdxList) > 0) && overlapHelper(p1, pFileWin->skey, pFileWin->ekey)) {
59,392✔
1490
    return true;
2,196✔
1491
  }
1492

1493
  // overlap with deletion skyline
1494
  SBrinRecord record = {.firstKey = p1->skey, .lastKey = p1->ekey};
57,194✔
1495
  if (overlapWithDelSkylineWithoutVer(pBlockScanInfo, &record, order)) {
57,194✔
1496
    return true;
8,038✔
1497
  }
1498

1499
  return false;
49,197✔
1500
}
1501

1502
static int32_t sortUidComparFn(const void* p1, const void* p2) {
66,438✔
1503
  const SSttKeyRange* px1 = p1;
66,438✔
1504
  const SSttKeyRange* px2 = p2;
66,438✔
1505

1506
  int32_t ret = tRowKeyCompare(&px1->skey, &px2->skey);
66,438✔
1507
  return ret;
66,504✔
1508
}
1509

1510
bool isCleanSttBlock(SArray* pKeyRangeList, STimeWindow* pQueryWindow, STableBlockScanInfo* pScanInfo, int32_t order) {
229,991✔
1511
  int32_t       num = 0;
229,991✔
1512
  SSttKeyRange* pRange = NULL;
229,991✔
1513
  STimeWindow   w;
1514

1515
  num = taosArrayGetSize(pKeyRangeList);
229,991✔
1516
  if (num == 0) {
230,004✔
1517
    return false;
136,722✔
1518
  }
1519

1520
  // check if it overlap with del skyline
1521
  taosArraySort(pKeyRangeList, sortUidComparFn);
93,282✔
1522

1523
  pRange = taosArrayGet(pKeyRangeList, 0);
93,358✔
1524
  if (pRange == NULL) {
93,337!
1525
    return false;
×
1526
  }
1527

1528
  w = (STimeWindow){.skey = pRange->skey.ts, .ekey = pRange->ekey.ts};
93,337✔
1529
  if (overlapWithTimeWindow(&w, pQueryWindow, pScanInfo, order)) {
93,337✔
1530
    return false;
44,226✔
1531
  }
1532

1533
  for (int32_t i = 0; i < num - 1; ++i) {
49,196✔
1534
    SSttKeyRange* p1 = taosArrayGet(pKeyRangeList, i);
24,290✔
1535
    SSttKeyRange* p2 = taosArrayGet(pKeyRangeList, i + 1);
24,286✔
1536
    if (p1 == NULL || p2 == NULL) {
24,287!
1537
      return false;
24,284✔
1538
    }
1539

1540
    if (p1->ekey.ts >= p2->skey.ts) {
24,287✔
1541
      return false;
24,284✔
1542
    }
1543

1544
    w = (STimeWindow){.skey = p2->skey.ts, .ekey = p2->ekey.ts};
3✔
1545
    bool overlap = overlapWithTimeWindow(&w, pQueryWindow, pScanInfo, order);
3✔
1546
    if (overlap) {
1!
1547
      return false;
×
1548
    }
1549
  }
1550

1551
  return true;
24,906✔
1552
}
1553

1554
static bool doCheckDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord,
148,152✔
1555
                                    int32_t startIndex) {
1556
  size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline);
148,152✔
1557

1558
  for (int32_t i = startIndex; i < num; i += 1) {
578,142✔
1559
    TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i);
569,542✔
1560
    if (p == NULL) {
569,540!
1561
      return false;
×
1562
    }
1563

1564
    if (p->ts >= pRecord->firstKey.key.ts && p->ts <= pRecord->lastKey.key.ts) {
569,540✔
1565
      if (p->version >= pRecord->minVer) {
187,444✔
1566
        return true;
122,062✔
1567
      }
1568
    } else if (p->ts < pRecord->firstKey.key.ts) {  // p->ts < pBlock->minKey.ts
382,096✔
1569
      if (p->version >= pRecord->minVer) {
376,802✔
1570
        if (i < num - 1) {
136,226!
1571
          TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1);
136,226✔
1572
          if (pnext == NULL) {
136,226!
1573
            return false;
×
1574
          }
1575

1576
          if (pnext->ts >= pRecord->firstKey.key.ts) {
136,226✔
1577
            return true;
12,194✔
1578
          }
1579
        } else {  // it must be the last point
1580
          if (!(p->version == 0)) {
×
1581
            tsdbError("unexpected version:%" PRId64, p->version);
×
1582
          }
1583
        }
1584
      }
1585
    } else {  // (p->ts > pBlock->maxKey.ts) {
1586
      return false;
5,294✔
1587
    }
1588
  }
1589

1590
  return false;
8,600✔
1591
}
1592

1593
static bool doCheckDatablockOverlapWithoutVersion(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord,
8,038✔
1594
                                                  int32_t startIndex) {
1595
  size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline);
8,038✔
1596

1597
  for (int32_t i = startIndex; i < num; i += 1) {
8,038!
1598
    TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, i);
8,038✔
1599
    if (p == NULL) {
8,038!
1600
      return false;
×
1601
    }
1602

1603
    if (p->ts >= pRecord->firstKey.key.ts && p->ts <= pRecord->lastKey.key.ts) {
8,038!
1604
      return true;
8,038✔
1605
    } else if (p->ts < pRecord->firstKey.key.ts) {  // p->ts < pBlock->minKey.ts
×
1606
      if (i < num - 1) {
×
1607
        TSDBKEY* pnext = taosArrayGet(pBlockScanInfo->delSkyline, i + 1);
×
1608
        if (pnext == NULL) {
×
1609
          return false;
×
1610
        }
1611

1612
        if (pnext->ts >= pRecord->firstKey.key.ts) {
×
1613
          return true;
×
1614
        }
1615
      }
1616
    } else {  // (p->ts > pBlock->maxKey.ts) {
1617
      return false;
×
1618
    }
1619
  }
1620

1621
  return false;
×
1622
}
1623

1624
bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order) {
6,672,215✔
1625
  if (pBlockScanInfo->delSkyline == NULL || (taosArrayGetSize(pBlockScanInfo->delSkyline) == 0)) {
6,672,215✔
1626
    return false;
6,508,951✔
1627
  }
1628

1629
  // ts is not overlap
1630
  TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0);
163,264✔
1631
  TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline);
163,326✔
1632
  if (pFirst == NULL || pLast == NULL) {
163,326!
1633
    return false;
×
1634
  }
1635

1636
  if (pRecord->firstKey.key.ts > pLast->ts || pRecord->lastKey.key.ts < pFirst->ts) {
163,326✔
1637
    return false;
15,175✔
1638
  }
1639

1640
  // version is not overlap
1641
  if (ASCENDING_TRAVERSE(order)) {
148,151✔
1642
    return doCheckDatablockOverlap(pBlockScanInfo, pRecord, pBlockScanInfo->fileDelIndex);
100,608✔
1643
  } else {
1644
    int32_t index = pBlockScanInfo->fileDelIndex;
47,543✔
1645
    while (1) {
328,386✔
1646
      TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index);
375,929✔
1647
      if (p == NULL) {
375,930!
1648
        return false;
×
1649
      }
1650

1651
      if (p->ts > pRecord->firstKey.key.ts && index > 0) {
375,930✔
1652
        index -= 1;
328,386✔
1653
      } else {  // find the first point that is smaller than the minKey.ts of dataBlock.
1654
        if (p->ts == pRecord->firstKey.key.ts && p->version < pRecord->maxVer && index > 0) {
47,544✔
1655
          index -= 1;
128✔
1656
        }
1657
        break;
47,544✔
1658
      }
1659
    }
1660

1661
    return doCheckDatablockOverlap(pBlockScanInfo, pRecord, index);
47,544✔
1662
  }
1663
}
1664

1665
bool overlapWithDelSkylineWithoutVer(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order) {
57,219✔
1666
  if (pBlockScanInfo->delSkyline == NULL || (taosArrayGetSize(pBlockScanInfo->delSkyline) == 0)) {
57,219✔
1667
    return false;
49,207✔
1668
  }
1669

1670
  // ts is not overlap
1671
  TSDBKEY* pFirst = taosArrayGet(pBlockScanInfo->delSkyline, 0);
8,012✔
1672
  TSDBKEY* pLast = taosArrayGetLast(pBlockScanInfo->delSkyline);
8,038✔
1673
  if (pFirst == NULL || pLast == NULL) {
8,038!
1674
    return false;
×
1675
  }
1676

1677
  if (pRecord->firstKey.key.ts > pLast->ts || pRecord->lastKey.key.ts < pFirst->ts) {
8,038!
1678
    return false;
×
1679
  }
1680

1681
  // version is not overlap
1682
  if (ASCENDING_TRAVERSE(order)) {
8,038!
1683
    return doCheckDatablockOverlapWithoutVersion(pBlockScanInfo, pRecord, pBlockScanInfo->fileDelIndex);
8,038✔
1684
  } else {
1685
    int32_t index = pBlockScanInfo->fileDelIndex;
×
1686
    while (1) {
×
1687
      TSDBKEY* p = taosArrayGet(pBlockScanInfo->delSkyline, index);
×
1688
      if (p == NULL) {
×
1689
        return false;
×
1690
      }
1691

1692
      if (p->ts > pRecord->firstKey.key.ts && index > 0) {
×
1693
        index -= 1;
×
1694
      } else {  // find the first point that is smaller than the minKey.ts of dataBlock.
1695
        if (p->ts == pRecord->firstKey.key.ts && index > 0) {
×
1696
          index -= 1;
×
1697
        }
1698
        break;
×
1699
      }
1700
    }
1701

1702
    return doCheckDatablockOverlapWithoutVersion(pBlockScanInfo, pRecord, index);
×
1703
  }
1704
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc