• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

taosdata / TDengine / #4790

15 Oct 2025 05:45AM UTC coverage: 60.906% (-0.004%) from 60.91%
#4790

push

travis-ci

web-flow
Merge b5da8883b into 01668ff4e

154696 of 324376 branches covered (47.69%)

Branch coverage included in aggregate %.

186 of 217 new or added lines in 11 files covered. (85.71%)

2655 existing lines in 9 files now uncovered.

206920 of 269356 relevant lines covered (76.82%)

125086749.75 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

70.22
/source/dnode/vnode/src/tsdb/tsdbRead2.c
1
/*
2
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3
 *
4
 * This program is free software: you can use, redistribute, and/or modify
5
 * it under the terms of the GNU Affero General Public License, version 3
6
 * or later ("AGPL"), as published by the Free Software Foundation.
7
 *
8
 * This program is distributed in the hope that it will be useful, but WITHOUT
9
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10
 * FITNESS FOR A PARTICULAR PURPOSE.
11
 *
12
 * You should have received a copy of the GNU Affero General Public License
13
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
14
 */
15

16
#include "osDef.h"
17
#include "tsdb.h"
18
#include "tsdbDataFileRW.h"
19
#include "tsdbFS2.h"
20
#include "tsdbMerge.h"
21
#include "tsdbReadUtil.h"
22
#include "tsdbUtil2.h"
23
#include "tsimplehash.h"
24

25
#define ASCENDING_TRAVERSE(o)       (o == TSDB_ORDER_ASC)
26
#define getCurrentKeyInSttBlock(_r) (&((_r)->currentKey))
27
#define tColRowGetKeyDeepCopy(_pBlock, _irow, _slotId, _pKey)             \
28
  do {                                                                    \
29
    (_pKey)->ts = (_pBlock)->aTSKEY[(_irow)];                             \
30
    (_pKey)->numOfPKs = 0;                                                \
31
    if ((_slotId) != -1) {                                                \
32
      code = tColRowGetPriamyKeyDeepCopy(_pBlock, _irow, _slotId, _pKey); \
33
      TSDB_CHECK_CODE(code, lino, _end);                                  \
34
    }                                                                     \
35
  } while (0)
36

37
#define outOfTimeWindow(_ts, _window) (((_ts) > (_window)->ekey) || ((_ts) < (_window)->skey))
38

39
typedef struct {
40
  bool overlapWithNeighborBlock;
41
  bool hasDupTs;
42
  bool overlapWithDelInfo;
43
  bool overlapWithSttBlock;
44
  bool overlapWithKeyInBuf;
45
  bool partiallyRequired;
46
  bool moreThanCapcity;
47
} SDataBlockToLoadInfo;
48

49
static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInfo** pInfo, const char* idStr);
50
static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity,
51
                                         STsdbReader* pReader);
52
static int32_t getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader, TSDBROW** pRes);
53
static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey,
54
                                       STsdbReader* pReader);
55
static int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo,
56
                                     SRowMerger* pMerger, int32_t pkSrcSlot, SVersionRange* pVerRange, const char* id);
57
static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArray* pDelList,
58
                                STsdbReader* pReader);
59
static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow,
60
                                    STableBlockScanInfo* pScanInfo);
61
static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData,
62
                                        int32_t rowIndex);
63
static void    setComposedBlockFlag(STsdbReader* pReader, bool composed);
64
static int32_t hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order,
65
                              SVersionRange* pVerRange, bool hasPk, bool* dropped);
66

67
static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIterInfo* pIter, SArray* pDelList,
68
                                        TSDBROW* pResRow, STsdbReader* pReader, bool* freeTSRow);
69
static int32_t doMergeMemIMemRows(TSDBROW* pRow, SRowKey* pRowKey, TSDBROW* piRow, SRowKey* piRowKey,
70
                                  STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, SRow** pTSRow);
71
static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, SRowKey* pKey,
72
                                     STsdbReader* pReader);
73
static int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo,
74
                                    STsdbReader* pReader);
75

76
static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SReadCostSummary* pCost);
77
static void getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idstr,
78
                                int8_t* pLevel, STsdb** pTsdb);
79
static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level);
80
static int32_t       doBuildDataBlock(STsdbReader* pReader);
81
static int32_t       getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader, TSDBKEY* key);
82
static bool          hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo);
83
static bool          hasDataInSttBlock(STableBlockScanInfo* pInfo);
84
static int32_t       initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter);
85
static int32_t       getInitialDelIndex(const SArray* pDelSkyline, int32_t order);
86
static int32_t       resetTableListIndex(SReaderStatus* pStatus, const char* id);
87
static void          getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey, int64_t* pMinKey);
88
static void          updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInfo* pBlockScanInfo);
89
static int32_t       buildFromPreFilesetBuffer(STsdbReader* pReader);
90

91
static void resetPreFilesetMemTableListIndex(SReaderStatus* pStatus);
92
int32_t     tsdbReaderSuspend2(STsdbReader* pReader);
93

94
FORCE_INLINE int32_t pkCompEx(SRowKey* p1, SRowKey* p2) {
184,758✔
95
  if (p2 == NULL) {
2,147,483,647!
96
    return 1;
2,147,483,647✔
97
  }
98

99
  if (p1 == NULL) {
2,147,483,647!
100
    return -1;
×
101
  }
102

103
  if (p1->ts < p2->ts) {
2,147,483,647!
104
    return -1;
2,147,483,647✔
105
  } else if (p1->ts > p2->ts) {
2,147,483,647!
106
    return 1;
2,147,483,647✔
107
  }
108

109
  if (p1->numOfPKs == 0) {
2,147,483,647!
110
    return 0;
2,147,483,647✔
111
  } else {
112
    return tRowKeyCompare(p1, p2);
100,223,287✔
113
  }
114
}
115

116
static int32_t tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_t slotId, SRowKey* pKey) {
19,286,172✔
117
  int32_t   code = TSDB_CODE_SUCCESS;
19,286,172✔
118
  int32_t   lino = 0;
19,286,172✔
119
  SColData* pColData = NULL;
19,286,172✔
120
  SColVal   cv;
19,286,172✔
121

122
  TSDB_CHECK_CONDITION((pBlock != NULL) && (pBlock->aColData != NULL), code, lino, _end, TSDB_CODE_INVALID_PARA);
19,286,172!
123
  TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA);
19,285,430!
124

125
  pColData = &pBlock->aColData[slotId];
19,285,430✔
126

127
  code = tColDataGetValue(pColData, irow, &cv);
19,286,172✔
128
  TSDB_CHECK_CODE(code, lino, _end);
19,284,448!
129

130
  pKey->numOfPKs = 1;
19,284,448✔
131
  pKey->pks[0].type = cv.value.type;
19,284,448✔
132

133
  if (IS_NUMERIC_TYPE(cv.value.type)) {
19,284,448!
134
    valueCloneDatum(pKey->pks, &cv.value, cv.value.type);
12,887,556✔
135
  } else {
136
    pKey->pks[0].nData = cv.value.nData;
6,396,892✔
137
    TAOS_MEMCPY(pKey->pks[0].pData, cv.value.pData, cv.value.nData);
6,396,892!
138
  }
139

140
_end:
19,285,932✔
141
  if (code != TSDB_CODE_SUCCESS) {
19,285,932!
142
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
143
  }
144
  return code;
19,285,932✔
145
}
146

147
// for test purpose, todo remove it
148
static int32_t tGetPrimaryKeyIndex(uint8_t* p, SPrimaryKeyIndex* index) {
157,135,017✔
149
  int32_t n = 0;
157,135,017✔
150
  n += tGetI8(p + n, &index->type);
157,135,017✔
151
  n += tGetU32v(p + n, &index->offset);
157,135,291!
152
  return n;
157,137,308✔
153
}
154

155
static int32_t tRowGetPrimaryKeyDeepCopy(SRow* pRow, SRowKey* pKey) {
157,128,134✔
156
  int32_t          code = TSDB_CODE_SUCCESS;
157,128,134✔
157
  int32_t          lino = 0;
157,128,134✔
158
  SPrimaryKeyIndex indices[TD_MAX_PK_COLS];
157,128,134✔
159
  uint8_t*         data = NULL;
157,127,414✔
160
  int32_t          len = 0;
157,127,414✔
161

162
  TSDB_CHECK_NULL(pRow, code, lino, _end, TSDB_CODE_INVALID_PARA);
157,127,414!
163
  if (pRow->numOfPKs > 0) {
157,127,414!
164
    TSDB_CHECK_NULL(pRow->data, code, lino, _end, TSDB_CODE_INVALID_PARA);
157,137,309!
165
    TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA);
157,135,313!
166
  }
167

168
  data = pRow->data;
157,131,026✔
169
  for (int32_t i = 0; i < pRow->numOfPKs; i++) {
314,272,853✔
170
    data += tGetPrimaryKeyIndex(data, &indices[i]);
157,136,565✔
171
  }
172

173
  // primary keys
174
  for (int32_t i = 0; i < pRow->numOfPKs; i++) {
314,267,308✔
175
    pKey->pks[i].type = indices[i].type;
157,133,255✔
176

177
    uint8_t* tdata = data + indices[i].offset;
157,132,723✔
178
    if (pRow->flag >> 4) {
157,133,742✔
179
      tdata += tGetI16v(tdata, NULL);
151,843,868✔
180
    }
181

182
    if (IS_VAR_DATA_TYPE(indices[i].type)) {
157,132,495!
183
      tdata += tGetU32v(tdata, &pKey->pks[i].nData);
104,037,970✔
184
      TAOS_MEMCPY(pKey->pks[i].pData, tdata, pKey->pks[i].nData);
52,024,964!
185
    } else {
186
      valueSetDatum(pKey->pks + i, indices[i].type, data + indices[i].offset, tDataTypes[pKey->pks[i].type].bytes);
105,114,417✔
187
    }
188
  }
189

190
_end:
157,133,288✔
191
  if (code != TSDB_CODE_SUCCESS) {
157,133,798!
192
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
193
  }
194
  return code;
157,133,798✔
195
}
196

197
static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList,
199,339,471✔
198
                                   int32_t numOfCols) {
199
  int32_t code = TSDB_CODE_SUCCESS;
199,339,471✔
200
  int32_t lino = 0;
199,339,471✔
201

202
  TSDB_CHECK_NULL(pSupInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
199,339,471!
203
  if (numOfCols > 0) {
199,339,471!
204
    TSDB_CHECK_NULL(pSlotIdList, code, lino, _end, TSDB_CODE_INVALID_PARA);
199,355,264!
205
    TSDB_CHECK_NULL(pCols, code, lino, _end, TSDB_CODE_INVALID_PARA);
199,355,264!
206
  }
207

208
  pSupInfo->pk.pk = 0;
199,339,471✔
209
  pSupInfo->numOfPks = 0;
199,340,856✔
210
  pSupInfo->pkSrcSlot = -1;
199,326,105✔
211
  pSupInfo->pkDstSlot = -1;
199,302,384✔
212
  pSupInfo->smaValid = true;
199,330,744✔
213
  pSupInfo->numOfCols = numOfCols;
199,358,087✔
214

215
  pSupInfo->colId = taosMemoryCalloc(numOfCols, sizeof(int16_t) * 2 + POINTER_BYTES);
199,340,170✔
216
  TSDB_CHECK_NULL(pSupInfo->colId, code, lino, _end, terrno);
199,340,549!
217

218
  pSupInfo->slotId = (int16_t*)((char*)pSupInfo->colId + (sizeof(int16_t) * numOfCols));
199,260,192✔
219
  pSupInfo->buildBuf = (char**)((char*)pSupInfo->slotId + (sizeof(int16_t) * numOfCols));
199,337,678✔
220
  for (int32_t i = 0; i < numOfCols; ++i) {
1,022,956,958✔
221
    pSupInfo->colId[i] = pCols[i].colId;
823,593,356✔
222
    pSupInfo->slotId[i] = pSlotIdList[i];
823,601,195✔
223

224
    if (pCols[i].pk) {
823,568,186✔
225
      pSupInfo->pk = pCols[i];
7,237,303✔
226
      pSupInfo->pkSrcSlot = i - 1;
7,229,141✔
227
      pSupInfo->pkDstSlot = pSlotIdList[i];
7,225,431✔
228
      pSupInfo->numOfPks += 1;
7,237,303✔
229
    }
230

231
    if (IS_VAR_DATA_TYPE(pCols[i].type)) {
823,594,344✔
232
      pSupInfo->buildBuf[i] = taosMemoryMalloc(pCols[i].bytes);
100,721,892!
233
      if (pSupInfo->buildBuf[i] == NULL) {
100,402,773!
234
        tsdbError("failed to prepare memory for set columnId slot list, size:%d, code:%s", pCols[i].bytes,
×
235
                  tstrerror(terrno));
236
      }
237
      TSDB_CHECK_NULL(pSupInfo->buildBuf[i], code, lino, _end, terrno);
100,399,551!
238
    } else {
239
      pSupInfo->buildBuf[i] = NULL;
723,100,249✔
240
    }
241
  }
242

243
_end:
199,363,602✔
244
  if (code != TSDB_CODE_SUCCESS) {
199,363,602!
245
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
246
  }
247
  return code;
199,335,402✔
248
}
249

250
static int32_t updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo) {
194,144,271✔
251
  int32_t   code = TSDB_CODE_SUCCESS;
194,144,271✔
252
  int32_t   lino = 0;
194,144,271✔
253
  int32_t   i = 0, j = 0;
194,144,271✔
254
  STColumn* pTCol = NULL;
194,144,271✔
255

256
  TSDB_CHECK_NULL(pSchema, code, lino, _end, TSDB_CODE_INVALID_PARA);
194,144,271!
257
  TSDB_CHECK_NULL(pSupInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
194,144,271!
258

259
  while (i < pSchema->numOfCols && j < pSupInfo->numOfCols) {
1,563,508,727✔
260
    pTCol = &pSchema->columns[i];
1,389,162,903✔
261
    if (pTCol->colId == pSupInfo->colId[j]) {
1,389,078,027✔
262
      if (!IS_BSMA_ON(pTCol) && (PRIMARYKEY_TIMESTAMP_COL_ID != pTCol->colId)) {
792,448,970✔
263
        pSupInfo->smaValid = false;
19,980,121✔
264
        goto _end;
19,899,978✔
265
      }
266

267
      i += 1;
772,438,828✔
268
      j += 1;
772,438,828✔
269
    } else if (pTCol->colId < pSupInfo->colId[j]) {  // do nothing
596,994,207✔
270
      i += 1;
596,925,628✔
271
    } else {
272
      code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER;
6,603✔
273
      TSDB_CHECK_CODE(code, lino, _end);
6,603!
274
    }
275
  }
276

277
_end:
174,103,953✔
278
  if (code != TSDB_CODE_SUCCESS) {
194,010,534✔
279
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
6,603!
280
  }
281
  return code;
194,183,173✔
282
}
283

284
static bool isEmptyQueryTimeWindow(STimeWindow* pWindow) {
1,704,977,721✔
285
  return (pWindow == NULL) || (pWindow->skey > pWindow->ekey);
1,704,977,721!
286
}
287

288
// Update the query time window according to the data time to live(TTL) information, in order to avoid to return
289
// the expired data to client, even it is queried already.
290
static int32_t updateQueryTimeWindow(STsdb* pTsdb, STimeWindow* pWindow, STimeWindow* out) {
229,571,790✔
291
  int32_t code = TSDB_CODE_SUCCESS;
229,571,790✔
292
  int32_t lino = 0;
229,571,790✔
293
  int64_t earlyTs = 0;
229,571,790✔
294

295
  TSDB_CHECK_NULL(pTsdb, code, lino, _end, TSDB_CODE_INVALID_PARA);
229,571,790!
296
  TSDB_CHECK_NULL(pWindow, code, lino, _end, TSDB_CODE_INVALID_PARA);
229,571,790!
297
  TSDB_CHECK_NULL(out, code, lino, _end, TSDB_CODE_INVALID_PARA);
229,571,790!
298

299
  earlyTs = tsdbGetEarliestTs(pTsdb);
229,571,790✔
300
  *out = *pWindow;
229,546,527✔
301
  if (out->skey < earlyTs) {
229,575,001✔
302
    out->skey = earlyTs;
155,717,015✔
303
  }
304

305
_end:
73,783,212✔
306
  if (code != TSDB_CODE_SUCCESS) {
229,505,804!
307
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
308
  }
309
  return code;
229,555,531✔
310
}
311

312
// init file iterator
313
static int32_t initFilesetIterator(SFilesetIter* pIter, TFileSetArray* pFileSetArray, STsdbReader* pReader) {
207,274,915✔
314
  int32_t             code = TSDB_CODE_SUCCESS;
207,274,915✔
315
  int32_t             lino = 0;
207,274,915✔
316
  SBlockLoadSuppInfo* pInfo = NULL;
207,274,915✔
317
  SSttBlockReader*    pSttReader = NULL;
207,274,915✔
318
  size_t              numOfFileset = 0;
207,274,915✔
319
  bool                asc = false;
207,274,915✔
320

321
  TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA);
207,274,915!
322
  TSDB_CHECK_NULL(pFileSetArray, code, lino, _end, TSDB_CODE_INVALID_PARA);
207,274,915!
323
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
207,274,915!
324

325
  pInfo = &pReader->suppInfo;
207,274,915✔
326
  numOfFileset = TARRAY2_SIZE(pFileSetArray);
207,294,944✔
327
  asc = ASCENDING_TRAVERSE(pReader->info.order);
207,311,627✔
328

329
  pIter->index = asc ? -1 : numOfFileset;
207,259,230✔
330
  pIter->order = pReader->info.order;
207,261,367✔
331
  pIter->pFilesetList = pFileSetArray;
207,295,052✔
332
  pIter->numOfFiles = numOfFileset;
207,245,394✔
333

334
  if (pIter->pSttBlockReader == NULL) {
207,243,492✔
335
    pIter->pSttBlockReader = taosMemoryCalloc(1, sizeof(struct SSttBlockReader));
177,078,630✔
336
    if (pIter->pSttBlockReader == NULL) {
176,995,804!
337
      tsdbError("failed to prepare the last block iterator, since:%s %s", tstrerror(terrno), pReader->idStr);
×
338
    }
339
    TSDB_CHECK_NULL(pIter->pSttBlockReader, code, lino, _end, terrno);
177,023,708!
340
  }
341

342
  pSttReader = pIter->pSttBlockReader;
207,253,443✔
343
  pSttReader->order = pReader->info.order;
207,280,415✔
344
  pSttReader->window = pReader->info.window;
207,221,882✔
345
  pSttReader->verRange = pReader->info.verRange;
207,264,146✔
346
  pSttReader->numOfPks = pReader->suppInfo.numOfPks;
207,271,695✔
347
  pSttReader->uid = 0;
207,266,678✔
348

349
  tMergeTreeClose(&pSttReader->mergeTree);
207,283,461✔
350
  code = initRowKey(&pSttReader->currentKey, INT64_MIN, pInfo->numOfPks, pInfo->pk.type, pInfo->pk.bytes, asc);
207,279,387✔
351
  if (code != TSDB_CODE_SUCCESS) {
207,175,320!
352
    tsdbError("failed init row key, %s", pReader->idStr);
×
353
  } else {
354
    tsdbDebug("init fileset iterator, total files:%d %s", pIter->numOfFiles, pReader->idStr);
207,175,320✔
355
  }
356
  TSDB_CHECK_CODE(code, lino, _end);
207,241,864!
357

358
_end:
207,241,864✔
359
  if (code != TSDB_CODE_SUCCESS) {
207,241,864!
360
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
361
  }
362
  return code;
207,249,188✔
363
}
364

365
static int32_t filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader, bool* hasNext) {
233,119,790✔
366
  int32_t           code = TSDB_CODE_SUCCESS;
233,119,790✔
367
  int32_t           lino = 0;
233,119,790✔
368
  bool              asc = false;
233,119,790✔
369
  int32_t           step = 0;
233,119,790✔
370
  SReadCostSummary* pCost = NULL;
233,119,790✔
371
  STFileObj**       pFileObj = NULL;
233,119,790✔
372

373
  TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA);
233,119,790!
374
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
233,119,790!
375
  TSDB_CHECK_NULL(hasNext, code, lino, _end, TSDB_CODE_INVALID_PARA);
233,119,790!
376

377
  asc = ASCENDING_TRAVERSE(pIter->order);
233,119,790✔
378
  step = asc ? 1 : -1;
233,106,578✔
379
  *hasNext = false;
233,106,578✔
380

381
  pIter->index += step;
233,109,377✔
382
  if ((asc && pIter->index >= pIter->numOfFiles) || ((!asc) && pIter->index < 0)) {
233,114,734✔
383
    *hasNext = false;
33,978,527✔
384
    goto _end;
33,947,310✔
385
  }
386

387
  pCost = &pReader->cost;
199,133,107✔
388

389
  TSDB_CHECK_NULL(pIter->pSttBlockReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
199,156,352!
390
  pIter->pSttBlockReader->uid = 0;
199,166,702✔
391
  tMergeTreeClose(&pIter->pSttBlockReader->mergeTree);
199,146,799✔
392
  destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost);
199,163,578✔
393

394
  pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
199,106,132✔
395
  TSDB_CHECK_NULL(pReader->status.pLDataIterArray, code, lino, _end, terrno);
199,157,288!
396

397
  // check file the time range of coverage
398
  STimeWindow win = {0};
199,149,921✔
399

400
  while (1) {
4,000,492✔
401
    if (pReader->pFileReader != NULL) {
203,147,502✔
402
      tsdbDataFileReaderClose(&pReader->pFileReader);
8,745,486✔
403
    }
404

405
    TSDB_CHECK_CONDITION(pIter->index < pIter->pFilesetList->size, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
203,165,745!
406
    pReader->status.pCurrentFileset = pIter->pFilesetList->data[pIter->index];
203,152,462✔
407

408
    pFileObj = pReader->status.pCurrentFileset->farr;
203,162,867✔
409
    if (pFileObj[0] != NULL || pFileObj[3] != NULL) {
203,171,070✔
410
      SDataFileReaderConfig conf = {.tsdb = pReader->pTsdb, .szPage = pReader->pTsdb->pVnode->config.tsdbPageSize};
12,772,780✔
411

412
      const char* filesName[4] = {0};
12,770,709✔
413

414
      if (pFileObj[0] != NULL) {
12,770,951✔
415
        conf.files[0].file = *pFileObj[0]->f;
9,356,021✔
416
        conf.files[0].exist = true;
9,356,021✔
417
        filesName[0] = pFileObj[0]->fname;
9,356,021✔
418

419
        conf.files[1].file = *pFileObj[1]->f;
9,355,779✔
420
        conf.files[1].exist = true;
9,355,393✔
421
        filesName[1] = pFileObj[1]->fname;
9,355,393✔
422

423
        conf.files[2].file = *pFileObj[2]->f;
9,355,958✔
424
        conf.files[2].exist = true;
9,355,958✔
425
        filesName[2] = pFileObj[2]->fname;
9,355,958✔
426
      }
427

428
      if (pFileObj[3] != NULL) {
12,770,081✔
429
        conf.files[3].exist = true;
4,384,108✔
430
        conf.files[3].file = *pFileObj[3]->f;
4,384,108✔
431
        filesName[3] = pFileObj[3]->fname;
4,384,108✔
432
      }
433

434
      code = tsdbDataFileReaderOpen(filesName, &conf, &pReader->pFileReader);
12,770,951✔
435
      TSDB_CHECK_CODE(code, lino, _end);
12,769,969!
436

437
      pReader->cost.headFileLoad += 1;
12,769,969✔
438
    }
439

440
    int32_t fid = pReader->status.pCurrentFileset->fid;
203,158,026✔
441
    tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &win.skey, &win.ekey);
203,165,148✔
442

443
    // current file are no longer overlapped with query time window, ignore remain files
444
    if ((asc && win.skey > pReader->info.window.ekey) || (!asc && win.ekey < pReader->info.window.skey)) {
203,125,268✔
445
      tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %s", pReader,
296,009✔
446
                pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr);
447
      *hasNext = false;
296,009✔
448
      break;
296,009✔
449
    }
450

451
    if ((asc && (win.ekey < pReader->info.window.skey)) || ((!asc) && (win.skey > pReader->info.window.ekey))) {
202,844,901✔
452
      pIter->index += step;
4,183,317✔
453
      if ((asc && pIter->index >= pIter->numOfFiles) || ((!asc) && pIter->index < 0)) {
4,102,600!
454
        *hasNext = false;
102,108✔
455
        break;
102,108✔
456
      }
457
      continue;
4,000,492✔
458
    }
459

460
    tsdbDebug("%p file found fid:%d for qrange:%" PRId64 "-%" PRId64 ", %s", pReader, fid, pReader->info.window.skey,
198,709,034✔
461
              pReader->info.window.ekey, pReader->idStr);
462

463
    *hasNext = true;
198,709,536✔
464
    break;
198,701,915✔
465
  }
466

467
_end:
233,047,342✔
468
  if (code != TSDB_CODE_SUCCESS) {
233,070,307!
469
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
470
  }
471
  return code;
233,070,307✔
472
}
473

474
bool shouldFreePkBuf(SBlockLoadSuppInfo* pSupp) {
613,543,209✔
475
  return (pSupp != NULL) && (pSupp->numOfPks > 0) && IS_VAR_DATA_TYPE(pSupp->pk.type);
613,543,209!
476
}
477

478
int32_t resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool needFree, const char* id) {
405,748,656✔
479
  int32_t code = TSDB_CODE_SUCCESS;
405,748,656✔
480
  int32_t lino = 0;
405,748,656✔
481

482
  TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA);
405,748,656!
483

484
  pIter->order = order;
405,748,656✔
485
  pIter->index = -1;
405,839,240✔
486
  pIter->numOfBlocks = 0;
405,843,970✔
487

488
  if (pIter->blockList == NULL) {
405,819,230✔
489
    pIter->blockList = taosArrayInit(4, sizeof(SFileDataBlockInfo));
177,005,942✔
490
    if (pIter->blockList == NULL) {
177,004,258!
491
      tsdbError("%s failed to reset block iter, func:%s at line:%d code:%s", id, __func__, __LINE__, tstrerror(terrno));
×
492
    }
493
    TSDB_CHECK_NULL(pIter->blockList, code, lino, _end, terrno);
177,052,079!
494
  } else {
495
    clearDataBlockIterator(pIter, needFree);
228,728,949✔
496
  }
497

498
_end:
405,868,900✔
499
  if (code != TSDB_CODE_SUCCESS) {
405,868,900!
500
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
501
  }
502
  return code;
405,849,946✔
503
}
504

505
static void initReaderStatus(SReaderStatus* pStatus) {
199,364,493✔
506
  if (pStatus == NULL) {
199,364,493!
507
    return;
×
508
  }
509
  pStatus->pTableIter = NULL;
199,364,493✔
510
  pStatus->loadFromFile = true;
199,379,381✔
511
}
512

513
static int32_t createResBlock(SQueryTableDataCond* pCond, int32_t capacity, SSDataBlock** pResBlock) {
135,605✔
514
  int32_t      code = TSDB_CODE_SUCCESS;
135,605✔
515
  int32_t      lino = 0;
135,605✔
516
  SSDataBlock* pBlock = NULL;
135,605✔
517

518
  TSDB_CHECK_NULL(pCond, code, lino, _end, TSDB_CODE_INVALID_PARA);
135,605!
519
  TSDB_CHECK_CONDITION(capacity >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);
135,605!
520
  TSDB_CHECK_NULL(pResBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
135,605!
521

522
  code = createDataBlock(&pBlock);
135,605✔
523
  TSDB_CHECK_CODE(code, lino, _end);
135,605!
524

525
  for (int32_t i = 0; i < pCond->numOfCols; ++i) {
837,928✔
526
    SColumnInfoData colInfo = {0};
702,207✔
527
    colInfo.info = pCond->colList[i];
702,207✔
528
    code = blockDataAppendColInfo(pBlock, &colInfo);
702,207✔
529
    TSDB_CHECK_CODE(code, lino, _end);
702,323!
530
  }
531

532
  code = blockDataEnsureCapacity(pBlock, capacity);
135,653✔
533
  TSDB_CHECK_CODE(code, lino, _end);
135,653!
534

535
  *pResBlock = pBlock;
135,653✔
536
  pBlock = NULL;
135,653✔
537

538
_end:
135,653✔
539
  if (code != TSDB_CODE_SUCCESS) {
135,653!
540
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
541
  }
542
  if (pBlock) {
135,653!
543
    taosArrayDestroy(pBlock->pDataBlock);
×
544
    taosMemoryFreeClear(pBlock);
×
545
  }
546
  return code;
135,653✔
547
}
548

549
static int32_t tsdbInitReaderLock(STsdbReader* pReader) {
199,314,146✔
550
  int32_t code = TSDB_CODE_SUCCESS;
199,314,146✔
551
  int32_t lino = 0;
199,314,146✔
552

553
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
199,314,146!
554

555
  code = taosThreadMutexInit(&pReader->readerMutex, NULL);
199,314,146✔
556
  tsdbTrace("tsdb/read: %p, post-init read mutex: %p, code: %d", pReader, &pReader->readerMutex, code);
199,239,649✔
557
  TSDB_CHECK_CODE(code, lino, _end);
199,299,301!
558

559
_end:
199,299,301✔
560
  if (code != TSDB_CODE_SUCCESS) {
199,299,301!
561
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
562
  }
563
  return code;
199,283,018✔
564
}
565

566
static int32_t tsdbUninitReaderLock(STsdbReader* pReader) {
199,365,100✔
567
  int32_t code = TSDB_CODE_SUCCESS;
199,365,100✔
568
  int32_t lino = 0;
199,365,100✔
569

570
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
199,365,100!
571

572
  tsdbTrace("tsdb/read: %p, pre-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code);
199,365,100✔
573
  code = taosThreadMutexDestroy(&pReader->readerMutex);
199,365,100✔
574
  tsdbTrace("tsdb/read: %p, post-uninit read mutex: %p, code: %d", pReader, &pReader->readerMutex, code);
199,353,638✔
575
  TSDB_CHECK_CODE(code, lino, _end);
199,355,374!
576

577
_end:
199,355,374✔
578
  if (code != TSDB_CODE_SUCCESS) {
199,355,374!
579
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
580
  }
581
  return code;
199,362,601✔
582
}
583

584
static int32_t tsdbAcquireReader(STsdbReader* pReader) {
998,912,953✔
585
  int32_t code = TSDB_CODE_SUCCESS;
998,912,953✔
586
  int32_t lino = 0;
998,912,953✔
587

588
  TSDB_CHECK_CONDITION((pReader != NULL) && (pReader->idStr != NULL), code, lino, _end, TSDB_CODE_INVALID_PARA);
998,912,953!
589

590
  tsdbTrace("tsdb/read: %s, pre-take read mutex: %p, code: %d", pReader->idStr, &pReader->readerMutex, code);
999,110,849✔
591
  code = taosThreadMutexLock(&pReader->readerMutex);
999,110,849✔
592
  if (code != TSDB_CODE_SUCCESS) {
999,081,786!
593
    tsdbError("tsdb/read:%p, failed to lock reader mutex, code:%s", pReader->idStr, tstrerror(code));
×
594
  } else {
595
    tsdbTrace("tsdb/read: %s, post-take read mutex: %p, code: %d", pReader->idStr, &pReader->readerMutex, code);
999,081,786✔
596
  }
597
  TSDB_CHECK_CODE(code, lino, _end);
998,898,173!
598

599
_end:
998,898,173✔
600
  if (code != TSDB_CODE_SUCCESS) {
998,898,173!
601
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
602
  }
603
  return code;
998,871,704✔
604
}
605

606
static int32_t tsdbTryAcquireReader(STsdbReader* pReader) {
32,775✔
607
  int32_t code = TSDB_CODE_SUCCESS;
32,775✔
608
  int32_t lino = 0;
32,775✔
609

610
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
32,775!
611

612
  code = taosThreadMutexTryLock(&pReader->readerMutex);
32,775✔
613
  if (code != TSDB_CODE_SUCCESS) {
32,775✔
614
    // Failing to acquire the lock is reasonable, not an error
615
    tsdbWarn("tsdb/read: %p, post-trytake read mutex: %p, code: %d", pReader, &pReader->readerMutex, code);
29,325!
616
  } else {
617
    tsdbTrace("tsdb/read: %p, post-trytask read mutex: %p", pReader, &pReader->readerMutex);
3,450!
618
  }
619
  return code;
32,775✔
620

621
_end:
×
622
  if (code != TSDB_CODE_SUCCESS) {
×
623
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
624
  }
625
  return code;
×
626
}
627

628
static int32_t tsdbReleaseReader(STsdbReader* pReader) {
998,692,906✔
629
  int32_t code = TSDB_CODE_SUCCESS;
998,692,906✔
630
  int32_t lino = 0;
998,692,906✔
631

632
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
998,692,906!
633

634
  code = taosThreadMutexUnlock(&pReader->readerMutex);
998,692,906✔
635
  if (code != TSDB_CODE_SUCCESS) {
998,758,351!
636
    tsdbError("tsdb/read: %p post-untake read mutex:%p failed, code:%d", pReader, &pReader->readerMutex, code);
×
637
  } else {
638
    tsdbTrace("tsdb/read: %p, post-untake read mutex: %p", pReader, &pReader->readerMutex);
998,758,351✔
639
  }
640
  TSDB_CHECK_CODE(code, lino, _end);
998,683,563!
641

642
_end:
998,683,563✔
643
  if (code != TSDB_CODE_SUCCESS) {
998,683,563!
644
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
645
  }
646
  return code;
998,733,477✔
647
}
648

649
void tsdbReleaseDataBlock2(void* p) {
38,284,748✔
650
  if (p == NULL) return;
38,284,748!
651

652
  STsdbReader* pReader = (STsdbReader*)p;
38,284,748✔
653
  SReaderStatus* pStatus = &pReader->status;
38,284,748✔
654
  if (!pStatus->composedDataBlock) {
38,285,230!
655
    (void)tsdbReleaseReader(pReader);
12,163,956✔
656
  }
657
}
658

659
static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacity, SSDataBlock* pResBlock,
199,330,152✔
660
                                SQueryTableDataCond* pCond, SBlockLoadSuppInfo* pSup) {
661
  int32_t      code = TSDB_CODE_SUCCESS;
199,330,152✔
662
  int32_t      lino = 0;
199,330,152✔
663
  SSDataBlock* p = NULL;
199,330,152✔
664

665
  TSDB_CHECK_NULL(pResBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
199,330,152!
666
  TSDB_CHECK_CONDITION((pResBlockInfo->pResBlock != NULL) || (pSup != NULL), code, lino, _end, TSDB_CODE_INVALID_PARA);
199,330,152!
667

668
  pResBlockInfo->capacity = capacity;
199,324,068✔
669
  pResBlockInfo->pResBlock = pResBlock;
199,277,975✔
670

671
  if (pResBlockInfo->pResBlock == NULL) {
199,314,347✔
672
    pResBlockInfo->freeBlock = true;
135,653✔
673
    code = createResBlock(pCond, pResBlockInfo->capacity, &pResBlockInfo->pResBlock);
135,653✔
674
    TSDB_CHECK_CODE(code, lino, _end);
135,653!
675
  } else {
676
    pResBlockInfo->freeBlock = false;
199,115,686✔
677
  }
678

679
  if (pSup->numOfPks > 0) {
199,360,977✔
680
    p = pResBlockInfo->pResBlock;
7,229,442✔
681
    p->info.pks[0].type = pSup->pk.type;
7,230,184✔
682
    p->info.pks[1].type = pSup->pk.type;
7,230,926✔
683

684
    if (IS_VAR_DATA_TYPE(pSup->pk.type) && (p->info.pks[0].pData == NULL)) {
7,227,216!
685
      p->info.pks[0].pData = taosMemoryCalloc(1, pSup->pk.bytes);
×
686
      TSDB_CHECK_NULL(p->info.pks[0].pData, code, lino, _end, terrno);
×
687

688
      p->info.pks[1].pData = taosMemoryCalloc(1, pSup->pk.bytes);
×
689
      TSDB_CHECK_NULL(p->info.pks[0].pData, code, lino, _end, terrno);
×
690

691
      p->info.pks[0].nData = pSup->pk.bytes;
×
692
      p->info.pks[1].nData = pSup->pk.bytes;
×
693
    }
694
  }
695

696
_end:
199,311,464✔
697
  if (code != TSDB_CODE_SUCCESS) {
199,311,464!
698
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
699
  }
700
  return code;
199,310,466✔
701
}
702

703
static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void** ppReader, int32_t capacity,
199,366,845✔
704
                                SSDataBlock* pResBlock, const char* idstr) {
705
  int32_t             code = TSDB_CODE_SUCCESS;
199,366,845✔
706
  int32_t             lino = 0;
199,366,845✔
707
  int8_t              level = 0;
199,366,845✔
708
  STsdbReader*        pReader = NULL;
199,378,377✔
709
  SBlockLoadSuppInfo* pSup = NULL;
199,378,377✔
710

711
  TSDB_CHECK_NULL(pVnode, code, lino, _end, TSDB_CODE_INVALID_PARA);
199,378,377!
712
  TSDB_CHECK_NULL(pCond, code, lino, _end, TSDB_CODE_INVALID_PARA);
199,378,377!
713
  TSDB_CHECK_NULL(ppReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
199,378,377!
714

715
  *ppReader = NULL;
199,378,377✔
716
  pReader = (STsdbReader*)taosMemoryCalloc(1, sizeof(*pReader));
199,367,190✔
717
  TSDB_CHECK_NULL(pReader, code, lino, _end, terrno);
199,345,515!
718

719
  initReaderStatus(&pReader->status);
199,345,515✔
720
  getTsdbByRetentions(pVnode, pCond, pVnode->config.tsdbCfg.retentions, idstr, &level, &pReader->pTsdb);
199,373,326✔
721

722
  pReader->info.suid = pCond->suid;
199,379,097✔
723
  pReader->info.order = pCond->order;
199,383,985✔
724
  pReader->info.verRange = getQueryVerRange(pVnode, pCond, level);
199,352,829✔
725
  code = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows, &pReader->info.window);
199,342,879✔
726
  TSDB_CHECK_CODE(code, lino, _end);
199,354,783!
727

728
  if (idstr == NULL) {
199,354,783✔
729
    idstr = "";
16,781,576✔
730
  }
731
  pReader->idStr = taosStrdup(idstr);
199,354,783✔
732
  TSDB_CHECK_NULL(pReader->idStr, code, lino, _end, terrno);
199,345,433!
733

734
  pReader->type = pCond->type;
199,342,334✔
735
  pReader->bFilesetDelimited = false;
199,320,206✔
736
  pReader->blockInfoBuf.numPerBucket = 1000;  // 1000 tables per bucket
199,345,297✔
737

738
  if (pCond->numOfCols <= 0) {
199,361,401!
739
    tsdbError("vgId:%d, invalid column number %d in query cond, %s", TD_VID(pVnode), pCond->numOfCols, idstr);
×
740
    TSDB_CHECK_CONDITION(pCond->numOfCols > 0, code, lino, _end, TSDB_CODE_INVALID_PARA);
×
741
  }
742

743
  // allocate buffer in order to load data blocks from file
744
  pSup = &pReader->suppInfo;
199,310,699✔
745
  pSup->tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID;
199,328,534✔
746
  code = setColumnIdSlotList(pSup, pCond->colList, pCond->pSlotList, pCond->numOfCols);
199,359,658✔
747
  TSDB_CHECK_CODE(code, lino, _end);
199,342,661!
748

749
  code = initResBlockInfo(&pReader->resBlockInfo, capacity, pResBlock, pCond, pSup);
199,342,661✔
750
  TSDB_CHECK_CODE(code, lino, _end);
199,317,147!
751

752
  code = tBlockDataCreate(&pReader->status.fileBlockData);
199,317,147✔
753
  TSDB_CHECK_CODE(code, lino, _end);
199,185,909!
754

755
  if (pReader->suppInfo.colId[0] != PRIMARYKEY_TIMESTAMP_COL_ID) {
199,185,909!
756
    tsdbError("the first column isn't primary timestamp, %d, %s", pReader->suppInfo.colId[0], pReader->idStr);
×
757
    TSDB_CHECK_CONDITION(pReader->suppInfo.colId[0] == PRIMARYKEY_TIMESTAMP_COL_ID, code, lino, _end,
×
758
                         TSDB_CODE_INVALID_PARA);
759
  }
760

761
  pReader->status.pPrimaryTsCol = taosArrayGet(pReader->resBlockInfo.pResBlock->pDataBlock, pSup->slotId[0]);
199,240,895✔
762
  TSDB_CHECK_NULL(pReader->status.pPrimaryTsCol, code, lino, _end, terrno);
199,321,772!
763

764
  int32_t type = pReader->status.pPrimaryTsCol->info.type;
199,195,146✔
765
  if (type != TSDB_DATA_TYPE_TIMESTAMP) {
199,300,450!
766
    tsdbError("the first column isn't primary timestamp in result block, actual: %s, %s", tDataTypes[type].name,
×
767
              pReader->idStr);
768
    TSDB_CHECK_CONDITION(type == TSDB_DATA_TYPE_TIMESTAMP, code, lino, _end, TSDB_CODE_INVALID_PARA);
×
769
  }
770

771
  code = tsdbInitReaderLock(pReader);
199,300,450✔
772
  TSDB_CHECK_CODE(code, lino, _end);
199,305,278!
773

774
  pSup->args = pReader->pTsdb->pVnode->pBse;
199,305,278✔
775

776
  code = tsem_init(&pReader->resumeAfterSuspend, 0, 0);
199,268,096✔
777
  TSDB_CHECK_CODE(code, lino, _end);
199,163,242!
778

779
  *ppReader = pReader;
199,163,242✔
780
  pReader = NULL;
199,170,290✔
781

782
_end:
199,170,290✔
783
  if (code != TSDB_CODE_SUCCESS) {
199,202,501!
784
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
785
  }
786
  if (pReader) {
199,303,822!
787
    tsdbReaderClose2(pReader);
×
788
  }
789
  return code;
199,303,822✔
790
}
791

792
static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFileReader* pFileReader, SArray* pIndexList) {
198,699,317✔
793
  int32_t              code = TSDB_CODE_SUCCESS;
198,699,317✔
794
  int32_t              lino = 0;
198,699,317✔
795
  int32_t              st = 0;
198,699,317✔
796
  int32_t              et1 = 0;
198,699,317✔
797
  int32_t              et2 = 0;
198,699,317✔
798
  int32_t              numOfTables = 0;
198,699,317✔
799
  const TBrinBlkArray* pBlkArray = NULL;
198,699,317✔
800
  STableUidList*       pList = NULL;
198,719,080✔
801
  SBrinBlk*            pBrinBlk = NULL;
198,719,080✔
802

803
  if (pFileReader == NULL) {
198,719,080✔
804
    goto _end;
186,047,136✔
805
  }
806

807
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
12,671,944!
808
  TSDB_CHECK_NULL(pIndexList, code, lino, _end, TSDB_CODE_INVALID_PARA);
12,671,944!
809

810
  st = taosGetTimestampUs();
12,682,685✔
811
  numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);
12,682,685✔
812

813
  code = tsdbDataFileReadBrinBlk(pFileReader, &pBlkArray);
12,681,398✔
814
  TSDB_CHECK_CODE(code, lino, _end);
12,680,034!
815

816
#if 0
817
  LRUHandle* handle = NULL;
818

819
  int32_t    code = tsdbCacheGetBlockIdx(pFileReader->pTsdb->biCache, pFileReader, &handle);
820
  if (code != TSDB_CODE_SUCCESS || handle == NULL) {
821
    goto _end;
822
  }
823

824

825
  SArray* aBlockIdx = (SArray*)taosLRUCacheValue(pFileReader->pTsdb->biCache, handle);
826
  size_t  num = taosArrayGetSize(aBlockIdx);
827
  if (num == 0) {
828
    tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle);
829
    return TSDB_CODE_SUCCESS;
830
  }
831
#endif
832

833
  // todo binary search to the start position
834
  et1 = taosGetTimestampUs();
12,680,300✔
835

836
  pList = &pReader->status.uidList;
12,680,300✔
837

838
  int32_t i = 0;
12,679,751✔
839
  int32_t j = 0;
12,679,751✔
840
  while (i < TARRAY2_SIZE(pBlkArray)) {
21,087,358✔
841
    pBrinBlk = &pBlkArray->data[i];
9,500,112✔
842
    if (pBrinBlk->maxTbid.suid < pReader->info.suid) {
9,498,739✔
843
      i += 1;
13,862✔
844
      continue;
13,862✔
845
    }
846

847
    if (pBrinBlk->minTbid.suid > pReader->info.suid) {  // not include the queried table/super table, quit the loop
9,486,100✔
848
      break;
17,518✔
849
    }
850

851
    TSDB_CHECK_CONDITION(
9,468,569!
852
        (pBrinBlk->minTbid.suid <= pReader->info.suid) && (pBrinBlk->maxTbid.suid >= pReader->info.suid), code, lino,
853
        _end, TSDB_CODE_INTERNAL_ERROR);
854

855
    if (pBrinBlk->maxTbid.suid == pReader->info.suid && pBrinBlk->maxTbid.uid < pList->tableUidList[j]) {
9,468,927✔
856
      i += 1;
235,462✔
857
      continue;
235,462✔
858
    }
859

860
    if (pBrinBlk->minTbid.suid == pReader->info.suid && pBrinBlk->minTbid.uid > pList->tableUidList[numOfTables - 1]) {
9,229,335✔
861
      break;
11,132✔
862
    }
863

864
    const void* p1 = taosArrayPush(pIndexList, pBrinBlk);
9,221,705✔
865
    TSDB_CHECK_NULL(p1, code, lino, _end, terrno);
9,221,705!
866

867
    i += 1;
9,221,705✔
868
    if (pBrinBlk->maxTbid.suid == pReader->info.suid) {
9,221,705✔
869
      while (j < numOfTables && pList->tableUidList[j] < pBrinBlk->maxTbid.uid) {
23,821,971✔
870
        j++;
14,671,630✔
871
      }
872
      if (j >= numOfTables) {
9,148,582✔
873
        break;
1,060,902✔
874
      }
875
    }
876
  }
877

878
  et2 = taosGetTimestampUs();
12,680,358✔
879
  tsdbDebug("load block index for %d/%d tables completed, elapsed time:%.2f ms, set BrinBlk:%.2f ms, size:%.2f Kb %s",
12,680,358✔
880
            numOfTables, (int32_t)pBlkArray->size, (et1 - st) / 1000.0, (et2 - et1) / 1000.0,
881
            pBlkArray->size * sizeof(SBrinBlk) / 1024.0, pReader->idStr);
882

883
  pReader->cost.headFileLoadTime += (et1 - st) / 1000.0;
12,680,358✔
884

885
_end:
198,728,355✔
886
  if (code != TSDB_CODE_SUCCESS) {
198,692,749!
887
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
888
  }
889
  return code;
198,692,749✔
890
}
891

892
static int32_t loadFileBlockBrinInfo(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum,
198,645,497✔
893
                                     SArray* pTableScanInfoList) {
894
  int32_t         code = TSDB_CODE_SUCCESS;
198,645,497✔
895
  int32_t         lino = 0;
198,645,497✔
896
  int64_t         st = 0;
198,645,497✔
897
  bool            asc = false;
198,645,497✔
898
  STimeWindow     w = {0};
198,645,497✔
899
  SBrinRecordIter iter = {0};
198,645,497✔
900
  int32_t         numOfTables = 0;
198,648,605✔
901
  SBrinRecord*    pRecord = NULL;
198,648,605✔
902
  int32_t         k = 0;
198,664,356✔
903
  size_t          sizeInDisk = 0;
198,664,356✔
904

905
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
198,664,356!
906
  TSDB_CHECK_NULL(pBlockNum, code, lino, _end, TSDB_CODE_INVALID_PARA);
198,664,356!
907
  TSDB_CHECK_NULL(pTableScanInfoList, code, lino, _end, TSDB_CODE_INVALID_PARA);
198,664,356!
908

909
  st = taosGetTimestampUs();
198,700,114✔
910
  asc = ASCENDING_TRAVERSE(pReader->info.order);
198,700,114✔
911
  w = pReader->info.window;
198,694,206✔
912
  numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);
198,680,052✔
913

914
  // clear info for the new file
915
  cleanupInfoForNextFileset(pReader->status.pTableMap);
198,686,421✔
916
  initBrinRecordIter(&iter, pReader->pFileReader, pIndexList);
198,701,113✔
917

918
  while (1) {
125,620,921✔
919
    code = getNextBrinRecord(&iter, &pRecord);
324,316,030✔
920
    TSDB_CHECK_CODE(code, lino, _end);
324,233,408!
921

922
    if (pRecord == NULL) {
324,233,408✔
923
      break;
197,548,092✔
924
    }
925

926
    if (pRecord->suid > pReader->info.suid) {
126,685,316✔
927
      break;
70,701✔
928
    }
929

930
    uint64_t uid = pReader->status.uidList.tableUidList[k];
126,681,612✔
931
    if (pRecord->suid < pReader->info.suid) {
126,681,762✔
932
      continue;
76,363✔
933
    }
934

935
    if (uid < pRecord->uid) {  // forward the table uid index
126,603,533✔
936
      while (k < numOfTables && pReader->status.uidList.tableUidList[k] < pRecord->uid) {
29,261,375✔
937
        k += 1;
14,672,935✔
938
      }
939

940
      if (k >= numOfTables) {
14,588,133✔
941
        break;
1,061,949✔
942
      }
943

944
      uid = pReader->status.uidList.tableUidList[k];
13,526,184✔
945
    }
946

947
    if (pRecord->uid < uid) {
125,541,551✔
948
      continue;
84,226,254✔
949
    }
950

951
    TSDB_CHECK_CONDITION((pRecord->suid == pReader->info.suid) && (uid == pRecord->uid), code, lino, _end,
41,300,125!
952
                         TSDB_CODE_INTERNAL_ERROR);
953

954
    STableBlockScanInfo* pScanInfo = NULL;
41,318,601✔
955
    code = getTableBlockScanInfo(pReader->status.pTableMap, uid, &pScanInfo, pReader->idStr);
41,318,457✔
956
    TSDB_CHECK_CODE(code, lino, _end);
41,316,417!
957

958
    // here we should find the first timestamp that is greater than the lastProcKey
959
    // the window is an open interval NOW.
960
    if (asc) {
41,316,417✔
961
      w.skey = pScanInfo->lastProcKey.ts;
40,068,906✔
962
    } else {
963
      w.ekey = pScanInfo->lastProcKey.ts;
1,247,511✔
964
    }
965

966
    // NOTE: specialized for open interval
967
    if (((w.skey < INT64_MAX) && ((w.skey + 1) > w.ekey)) || (w.skey == INT64_MAX)) {
41,316,338!
968
      k += 1;
×
969
      if (k >= numOfTables) {
×
970
        break;
×
971
      } else {
972
        continue;
×
973
      }
974
    }
975

976
    // 1. time range check
977
    if (pRecord->firstKey.key.ts > w.ekey || pRecord->lastKey.key.ts < w.skey) {
41,316,338✔
978
      continue;
481,823✔
979
    }
980
    // The data block's time range must intersect with the query time range
981
    if (pRecord->firstKey.key.ts > pReader->info.window.ekey || pRecord->lastKey.key.ts < pReader->info.window.skey) {
40,833,803!
982
      continue;
520✔
983
    }
984

985
    if (asc) {
40,834,172✔
986
      if (pkCompEx(&pRecord->lastKey.key, &pScanInfo->lastProcKey) <= 0) {
79,422,091!
987
        continue;
1,380✔
988
      }
989
    } else {
990
      if (pkCompEx(&pRecord->firstKey.key, &pScanInfo->lastProcKey) >= 0) {
2,245,166!
991
        continue;
×
992
      }
993
    }
994

995
    // 2. version range check, version range is an CLOSED interval
996
    if (pRecord->minVer > pReader->info.verRange.maxVer || pRecord->maxVer < pReader->info.verRange.minVer) {
40,831,705!
997
      continue;
×
998
    }
999

1000
    if (pScanInfo->pBlockList == NULL) {
40,833,059✔
1001
      pScanInfo->pBlockList = taosArrayInit(4, sizeof(SFileDataBlockInfo));
21,846,851✔
1002
      TSDB_CHECK_NULL(pScanInfo->pBlockList, code, lino, _end, terrno);
21,845,255!
1003
    }
1004

1005
    if (pScanInfo->pBlockIdxList == NULL) {
40,829,122✔
1006
      pScanInfo->pBlockIdxList = taosArrayInit(4, sizeof(STableDataBlockIdx));
3,439,172✔
1007
      TSDB_CHECK_NULL(pScanInfo->pBlockIdxList, code, lino, _end, terrno);
3,439,735!
1008
    }
1009

1010
    SFileDataBlockInfo blockInfo = {.tbBlockIdx = TARRAY_SIZE(pScanInfo->pBlockList)};
40,831,030✔
1011
    code = recordToBlockInfo(&blockInfo, pRecord);
40,834,669✔
1012
    TSDB_CHECK_CODE(code, lino, _end);
40,830,772!
1013
    sizeInDisk += blockInfo.blockSize;
40,830,772✔
1014

1015
    const void* p1 = taosArrayPush(pScanInfo->pBlockList, &blockInfo);
40,830,772✔
1016
    TSDB_CHECK_NULL(p1, code, lino, _end, terrno);
40,833,436!
1017

1018
    // todo: refactor to record the fileset skey/ekey
1019
    if (pScanInfo->filesetWindow.skey > pRecord->firstKey.key.ts) {
40,833,436✔
1020
      pScanInfo->filesetWindow.skey = pRecord->firstKey.key.ts;
21,845,839✔
1021
    }
1022

1023
    if (pScanInfo->filesetWindow.ekey < pRecord->lastKey.key.ts) {
40,834,386✔
1024
      pScanInfo->filesetWindow.ekey = pRecord->lastKey.key.ts;
40,832,068✔
1025
    }
1026

1027
    pBlockNum->numOfBlocks += 1;
40,834,951✔
1028
    if (taosArrayGetSize(pTableScanInfoList) == 0) {
40,833,091✔
1029
      p1 = taosArrayPush(pTableScanInfoList, &pScanInfo);
8,367,473✔
1030
      TSDB_CHECK_NULL(p1, code, lino, _end, terrno);
8,367,473!
1031
    } else {
1032
      STableBlockScanInfo** p = taosArrayGetLast(pTableScanInfoList);
32,466,689✔
1033
      TSDB_CHECK_NULL(p, code, lino, _end, TSDB_CODE_INVALID_PARA);
32,466,689!
1034

1035
      if ((*p)->uid != uid) {
32,466,689✔
1036
        p1 = taosArrayPush(pTableScanInfoList, &pScanInfo);
13,480,041✔
1037
        TSDB_CHECK_NULL(p1, code, lino, _end, terrno);
13,480,041!
1038
      }
1039
    }
1040
  }
1041

1042
  pBlockNum->numOfSttFiles = pReader->status.pCurrentFileset->lvlArr->size;
198,632,752✔
1043
  int32_t total = pBlockNum->numOfSttFiles + pBlockNum->numOfBlocks;
198,667,027✔
1044

1045
  double el = (taosGetTimestampUs() - st) / 1000.0;
198,707,417✔
1046
  tsdbDebug(
198,707,417✔
1047
      "load block of %d tables completed, blocks:%d in %d tables, stt-files:%d, block-info-size:%.2f Kb, elapsed "
1048
      "time:%.2f ms %s",
1049
      numOfTables, pBlockNum->numOfBlocks, (int32_t)taosArrayGetSize(pTableScanInfoList), pBlockNum->numOfSttFiles,
1050
      sizeInDisk / 1000.0, el, pReader->idStr);
1051

1052
  pReader->cost.numOfBlocks += total;
198,707,417✔
1053
  pReader->cost.headFileLoadTime += el;
198,704,182✔
1054

1055
_end:
198,699,784✔
1056
  if (code != TSDB_CODE_SUCCESS) {
198,689,043!
1057
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1058
  }
1059
  clearBrinBlockIter(&iter);
198,689,043✔
1060
  return code;
198,702,155✔
1061
}
1062

1063
static void setBlockAllDumped(SFileBlockDumpInfo* pDumpInfo, int64_t maxKey, int32_t order) {
65,452,431✔
1064
  if (pDumpInfo != NULL) {
65,452,431!
1065
    pDumpInfo->allDumped = true;
65,458,062✔
1066
  }
1067
}
65,455,386✔
1068

1069
static int32_t updateLastKeyInfo(SRowKey* pKey, SFileDataBlockInfo* pBlockInfo, SDataBlockInfo* pInfo, int32_t numOfPks,
36,986,606✔
1070
                                 bool asc) {
1071
  int32_t code = TSDB_CODE_SUCCESS;
36,986,606✔
1072
  int32_t lino = 0;
36,986,606✔
1073

1074
  TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA);
36,986,606!
1075
  TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
36,986,606!
1076

1077
  pKey->ts = asc ? pInfo->window.ekey : pInfo->window.skey;
36,986,606✔
1078
  pKey->numOfPKs = numOfPks;
36,985,982✔
1079
  if (pKey->numOfPKs <= 0) {
36,987,332✔
1080
    goto _end;
36,987,204✔
1081
  }
1082

1083
  TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
826!
1084
  if (IS_NUMERIC_TYPE(pKey->pks[0].type)) {
826!
1085
    VALUE_SET_TRIVIAL_DATUM(pKey->pks, asc ? pBlockInfo->lastPk.val : pBlockInfo->firstPk.val);
×
1086
  } else {
1087
    uint8_t* p = asc ? pBlockInfo->lastPk.pData : pBlockInfo->firstPk.pData;
826!
1088
    pKey->pks[0].nData = asc ? varDataLen(pBlockInfo->lastPk.pData) : varDataLen(pBlockInfo->firstPk.pData);
826!
1089
    TAOS_MEMCPY(pKey->pks[0].pData, p, pKey->pks[0].nData);
826!
1090
  }
1091

1092
_end:
36,988,030✔
1093
  if (code != TSDB_CODE_SUCCESS) {
36,988,030!
1094
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1095
  }
1096
  return code;
36,983,790✔
1097
}
1098

1099
static int32_t doGetValueFromBseBySeq(void* arg, uint8_t* pKey, int32_t keyLen, uint8_t** pValue, int32_t* len) {
24,360✔
1100
  int32_t  code = 0;
24,360✔
1101
  int32_t  lino = 0;
24,360✔
1102
  uint64_t seq = 0;
24,360✔
1103
  if (arg == NULL) {
24,360!
1104
    tsdbError("failed to get value from bse by seq since %s", tstrerror(TSDB_CODE_INVALID_PARA));
×
1105
    return TSDB_CODE_INVALID_PARA;
×
1106
  }
1107

1108
  if (keyLen <= 0) {
24,360!
1109
    *len = 0; 
×
1110
    return code;
×
1111
  } else {
1112
    int32_t unusedRet = tGetU64(pKey, &seq);
24,360✔
1113
  }
1114
 
1115
  if (seq == 0) {
24,360!
1116
    tsdbError("failed to get value from bse by seq since seq is 0");
×
1117
    return TSDB_CODE_INVALID_PARA;
×
1118
  }
1119
  *len = 0;
24,360✔
1120
  code = bseGet((SBse*)arg, seq, pValue, len);
24,360✔
1121
  TSDB_CHECK_CODE(code, lino, _end);
24,360!
1122

1123
_end:
24,360✔
1124
  if (code != 0) {
24,360!
1125
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1126
  }
1127
  return code;
24,360✔
1128
}
1129
static int32_t doReallocBuf(SBlockLoadSuppInfo* pSup, int32_t colIndex, SColumnInfo* pInfo, int32_t len) {
31,320✔
1130
  int32_t code = 0;
31,320✔
1131
  int32_t bytes = pInfo->bytes;
31,320✔
1132
  int32_t extraSize = VARSTR_HEADER_SIZE;
31,320✔
1133

1134
  if (IS_STR_DATA_BLOB(pInfo->type)) {
31,320!
1135
    if (len + BLOBSTR_HEADER_SIZE > TSDB_MAX_BLOB_LEN) {
31,320!
1136
      tsdbWarn("column cid:%d actual data len %d is bigger than schema len %d", pSup->colId[colIndex], len, bytes);
×
1137
      return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER;
×
1138
    }
1139
    extraSize = BLOBSTR_HEADER_SIZE;
31,320✔
1140

1141
  } else {
1142
    if (len + VARSTR_HEADER_SIZE > pInfo->bytes) {
×
1143
      tsdbWarn("column cid:%d actual data len %d is bigger than schema len %d", pSup->colId[colIndex], len, bytes);
×
1144
      return TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER;
×
1145
    }
1146
  }
1147

1148
  char* p = taosMemoryRealloc(pSup->buildBuf[colIndex], len + extraSize);
31,320!
1149
  if (p == NULL) {
31,320!
1150
    return terrno;
×
1151
  }
1152

1153
  pSup->buildBuf[colIndex] = p;
31,320✔
1154

1155
  if (IS_STR_DATA_BLOB(pInfo->type)) {
31,320!
1156
    blobDataSetLen(pSup->buildBuf[colIndex], len);
31,320✔
1157
  } else {
1158
    varDataSetLen(pSup->buildBuf[colIndex], len);
×
1159
  }
1160

1161
  return code;
31,320✔
1162
}
1163
static int32_t doCopyColVal(SColumnInfoData* pColInfoData, int32_t rowIndex, int32_t colIndex, SColVal* pColVal,
2,147,483,647✔
1164
                            SBlockLoadSuppInfo* pSup) {
1165
  int32_t code = TSDB_CODE_SUCCESS;
2,147,483,647✔
1166
int32_t lino = 0;
2,147,483,647✔
1167

1168
  uint8_t* pValue = 0;
2,147,483,647✔
1169
  int32_t  len = 0;
2,147,483,647✔
1170
  TSDB_CHECK_NULL(pColInfoData, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
1171
  TSDB_CHECK_NULL(pColVal, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
1172

1173
  if (IS_VAR_DATA_TYPE(pColVal->value.type)) {
2,147,483,647!
1174
    if (!COL_VAL_IS_VALUE(pColVal)) {
2,147,483,647✔
1175
      colDataSetNULL(pColInfoData, rowIndex);
1,736,542,328!
1176
    } else {
1177
      if (IS_STR_DATA_BLOB(pColVal->value.type)) {
2,147,483,647✔
1178
        if (pColVal->value.nData != 0) {
102,863,396✔
1179
          code = doGetValueFromBseBySeq(pSup->args, pColVal->value.pData, pColVal->value.nData, &pValue, &len);
24,360✔
1180
          if (code != 0) {
24,360!
1181
            colDataSetNULL(pColInfoData, rowIndex);
×
1182
            return 0;
×
1183
          } else {
1184
            TSDB_CHECK_CODE(code, lino, _end);
24,360!
1185
          }
1186
        }
1187

1188
        if (IS_STR_DATA_BLOB(pColVal->value.type)) {
31,320!
1189
          code = doReallocBuf(pSup, colIndex, &pColInfoData->info, len);
31,320✔
1190
        } else {
1191
          varDataSetLen(pSup->buildBuf[colIndex], len);
×
1192
          if ((len + VARSTR_HEADER_SIZE) > pColInfoData->info.bytes) {
×
1193
            tsdbWarn("column cid:%d actual data len %d is bigger than schema len %d", pColVal->cid, len,
×
1194
                     pColInfoData->info.bytes);
1195
            code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER;
×
1196
            TSDB_CHECK_CODE(code, lino, _end);
×
1197
          }
1198
        }
1199

1200
        if (len > 0) {
31,320✔
1201
          if (IS_STR_DATA_BLOB(pColVal->value.type)) {
24,360!
1202
            (void)memcpy(blobDataVal(pSup->buildBuf[colIndex]), pValue, len);
24,360!
1203
          } else {
1204
            (void)memcpy(varDataVal(pSup->buildBuf[colIndex]), pValue, len);
×
1205
          }
1206
        }
1207
        code = colDataSetVal(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false);
31,320✔
1208
        taosMemFreeClear(pValue);
31,320✔
1209

1210
        TSDB_CHECK_CODE(code, lino, _end);
31,320!
1211
      } else {
1212
        TSDB_CHECK_NULL(pSup, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
1213
        varDataSetLen(pSup->buildBuf[colIndex], pColVal->value.nData);
2,147,483,647✔
1214
        if ((pColVal->value.nData + VARSTR_HEADER_SIZE) > pColInfoData->info.bytes) {
2,147,483,647!
1215
          tsdbWarn("column cid:%d actual data len %d is bigger than schema len %d", pColVal->cid, pColVal->value.nData,
×
1216
                   pColInfoData->info.bytes);
1217
          code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER;
×
1218
          TSDB_CHECK_CODE(code, lino, _end);
×
1219
        }
1220

1221
        if (pColVal->value.nData > 0) {  // pData may be null, if nData is 0
2,147,483,647✔
1222
          (void)memcpy(varDataVal(pSup->buildBuf[colIndex]), pColVal->value.pData, pColVal->value.nData);
2,147,483,647!
1223
        }
1224

1225
        code = colDataSetVal(pColInfoData, rowIndex, pSup->buildBuf[colIndex], false);
2,147,483,647✔
1226
        TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
1227
      }
1228
    }
1229
  } else {
1230
    code = colDataSetVal(pColInfoData, rowIndex, VALUE_GET_DATUM(&pColVal->value, pColVal->value.type),
2,147,483,647!
1231
                         !COL_VAL_IS_VALUE(pColVal));
2,147,483,647!
1232
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
1233
  }
1234

1235
_end:
2,147,483,647✔
1236
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
1237
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1238
  }
1239
  return code;
2,147,483,647✔
1240
}
1241

1242
static int32_t getCurrentBlockInfo(SDataBlockIter* pBlockIter, SFileDataBlockInfo** pInfo, const char* id) {
369,651,944✔
1243
  int32_t code = TSDB_CODE_SUCCESS;
369,651,944✔
1244
  int32_t lino = 0;
369,651,944✔
1245

1246
  TSDB_CHECK_NULL(pBlockIter, code, lino, _end, TSDB_CODE_INVALID_PARA);
369,651,944!
1247
  TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
369,651,944!
1248
  TSDB_CHECK_NULL(pBlockIter->blockList, code, lino, _end, TSDB_CODE_INVALID_PARA);
369,651,944!
1249

1250
  *pInfo = NULL;
369,715,604✔
1251

1252
  size_t num = TARRAY_SIZE(pBlockIter->blockList);
369,707,761✔
1253
  if (num == 0) {
369,742,257✔
1254
    // Some callers would attempt to call this function. Filter out certain normal cases and return directly to avoid
1255
    // generating excessive unnecessary error logs.
1256
    TSDB_CHECK_CONDITION(num == pBlockIter->numOfBlocks, code, lino, _end, TSDB_CODE_INVALID_PARA);
190,250,995!
1257
    return TSDB_CODE_INVALID_PARA;
190,257,850✔
1258
  }
1259

1260
  *pInfo = taosArrayGet(pBlockIter->blockList, pBlockIter->index);
179,491,262✔
1261
  TSDB_CHECK_NULL(*pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
179,487,463!
1262

1263
_end:
179,489,287✔
1264
  if (code != TSDB_CODE_SUCCESS) {
179,489,287!
1265
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1266
  }
1267
  return code;
179,480,687✔
1268
}
1269

1270
static int32_t doBinarySearchKey(const TSKEY* keyList, int num, int pos, TSKEY key, int order) {
230,779✔
1271
  // start end position
1272
  int s, e;
1273
  s = pos;
230,779✔
1274

1275
  // check
1276
  if (!(keyList != NULL && pos >= 0 && pos < num && num > 0)) {
230,779!
1277
    return -1;
×
1278
  }
1279
  if (order == TSDB_ORDER_ASC) {
230,779✔
1280
    // find the first position which is smaller than the key
1281
    e = num - 1;
170,378✔
1282
    if (key < keyList[pos]) return -1;
170,378!
1283
    while (1) {
1,956,837✔
1284
      // check can return
1285
      if (key >= keyList[e]) return e;
2,127,215!
1286
      if (key <= keyList[s]) return s;
2,127,215✔
1287
      if (e - s <= 1) return s;
2,123,925✔
1288

1289
      // change start or end position
1290
      int mid = s + (e - s + 1) / 2;
1,959,600✔
1291
      if (keyList[mid] > key)
1,959,600✔
1292
        e = mid;
1,197,080✔
1293
      else if (keyList[mid] < key)
762,520✔
1294
        s = mid;
759,757✔
1295
      else
1296
        return mid;
2,763✔
1297
    }
1298
  } else {  // DESC
1299
    // find the first position which is bigger than the key
1300
    e = 0;
60,401✔
1301
    if (key > keyList[pos]) return -1;
60,401!
1302
    while (1) {
591,159✔
1303
      // check can return
1304
      if (key <= keyList[e]) return e;
651,560!
1305
      if (key >= keyList[s]) return s;
651,560!
1306
      if (s - e <= 1) return s;
651,560✔
1307

1308
      // change start or end position
1309
      int mid = s - (s - e + 1) / 2;
595,880✔
1310
      if (keyList[mid] < key)
595,880✔
1311
        e = mid;
410,179✔
1312
      else if (keyList[mid] > key)
185,701✔
1313
        s = mid;
180,980✔
1314
      else
1315
        return mid;
4,721✔
1316
    }
1317
  }
1318
}
1319

1320
// handle the repeat ts cases.
1321
static int32_t findFirstPos(const int64_t* pTsList, int32_t num, int32_t startPos, bool asc) {
230,779✔
1322
  int32_t i = startPos;
230,779✔
1323
  int64_t startTs = pTsList[startPos];
230,779✔
1324
  if (asc) {
230,779✔
1325
    while (i < num && (pTsList[i] == startTs)) {
340,756!
1326
      i++;
170,378✔
1327
    }
1328
    return i - 1;
170,378✔
1329
  } else {
1330
    while (i >= 0 && (pTsList[i] == startTs)) {
120,802!
1331
      i--;
60,401✔
1332
    }
1333
    return i + 1;
60,401✔
1334
  }
1335
}
1336

1337
static int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData, SBrinRecord* pRecord, int32_t pos) {
25,003,516✔
1338
  // NOTE: reverse the order to find the end position in data block
1339
  int32_t code = TSDB_CODE_SUCCESS;
25,003,516✔
1340
  int32_t lino = 0;
25,003,516✔
1341
  int32_t endPos = -1;
25,003,516✔
1342
  bool    asc = false;
25,003,516✔
1343

1344
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
25,003,516!
1345
  TSDB_CHECK_NULL(pRecord, code, lino, _end, TSDB_CODE_INVALID_PARA);
25,003,516!
1346

1347
  asc = ASCENDING_TRAVERSE(pReader->info.order);
25,003,516✔
1348

1349
  if (asc && pReader->info.window.ekey >= pRecord->lastKey.key.ts) {
25,006,904✔
1350
    endPos = pRecord->numRow - 1;
24,619,021✔
1351
  } else if (!asc && pReader->info.window.skey <= pRecord->firstKey.key.ts) {
387,399✔
1352
    endPos = 0;
276,552✔
1353
  } else {
1354
    int64_t key = asc ? pReader->info.window.ekey : pReader->info.window.skey;
116,413✔
1355
    endPos = doBinarySearchKey(pBlockData->aTSKEY, pRecord->numRow, pos, key, pReader->info.order);
116,413✔
1356
    if (endPos == -1) {
116,413!
1357
      goto _end;
×
1358
    }
1359

1360
    endPos = findFirstPos(pBlockData->aTSKEY, pRecord->numRow, endPos, asc);
116,413✔
1361
  }
1362

1363
  if ((pReader->info.verRange.maxVer >= pRecord->minVer && pReader->info.verRange.maxVer < pRecord->maxVer) ||
25,011,260!
1364
      (pReader->info.verRange.minVer <= pRecord->maxVer && pReader->info.verRange.minVer > pRecord->minVer)) {
25,007,630!
1365
    int32_t i = endPos;
×
1366

1367
    if (asc) {
×
1368
      for (; i >= 0; --i) {
×
1369
        if (pBlockData->aVersion[i] <= pReader->info.verRange.maxVer) {
×
1370
          break;
×
1371
        }
1372
      }
1373
    } else {
1374
      for (; i < pRecord->numRow; ++i) {
×
1375
        if (pBlockData->aVersion[i] >= pReader->info.verRange.minVer) {
×
1376
          break;
×
1377
        }
1378
      }
1379
    }
1380

1381
    endPos = i;
×
1382
  }
1383

1384
_end:
25,007,388✔
1385
  if (code != TSDB_CODE_SUCCESS) {
25,007,388!
1386
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1387
  }
1388
  return endPos;
25,002,548✔
1389
}
1390

1391
static int32_t copyPrimaryTsCol(const SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData,
25,013,196✔
1392
                                int32_t dumpedRows, bool asc) {
1393
  int32_t code = TSDB_CODE_SUCCESS;
25,013,196✔
1394
  int32_t lino = 0;
25,013,196✔
1395

1396
  TSDB_CHECK_CONDITION((pBlockData != NULL) && (pBlockData->aTSKEY != NULL), code, lino, _end, TSDB_CODE_INVALID_PARA);
25,013,196!
1397
  TSDB_CHECK_NULL(pDumpInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
25,012,228!
1398
  TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA);
25,012,228!
1399
  TSDB_CHECK_CONDITION(dumpedRows >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);
25,012,228!
1400

1401
  if (asc) {
25,012,228✔
1402
    TAOS_MEMCPY(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], dumpedRows * sizeof(int64_t));
24,687,105!
1403
  } else {
1404
    int32_t startIndex = pDumpInfo->rowIndex - dumpedRows + 1;
325,123✔
1405
    TAOS_MEMCPY(pColData->pData, &pBlockData->aTSKEY[startIndex], dumpedRows * sizeof(int64_t));
325,365!
1406

1407
    // todo: opt perf by extract the loop
1408
    // reverse the array list
1409
    int32_t  mid = dumpedRows >> 1u;
325,365✔
1410
    int64_t* pts = (int64_t*)pColData->pData;
325,365✔
1411
    for (int32_t j = 0; j < mid; ++j) {
306,494,506✔
1412
      int64_t t = pts[j];
306,169,141✔
1413
      pts[j] = pts[dumpedRows - j - 1];
306,164,656✔
1414
      pts[dumpedRows - j - 1] = t;
306,166,381✔
1415
    }
1416
  }
1417

1418
_end:
325,365✔
1419
  if (code != TSDB_CODE_SUCCESS) {
25,008,840!
1420
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1421
  }
1422
  return code;
25,006,178✔
1423
}
1424

1425
// a faster version of copy procedure.
1426
static int32_t copyNumericCols(const SColData* pData, SFileBlockDumpInfo* pDumpInfo, SColumnInfoData* pColData,
48,577,081✔
1427
                               int32_t dumpedRows, bool asc) {
1428
  int32_t  code = TSDB_CODE_SUCCESS;
48,577,081✔
1429
  int32_t  lino = 0;
48,577,081✔
1430
  uint8_t* p = NULL;
48,577,081✔
1431
  int32_t  step = asc ? 1 : -1;
48,577,081✔
1432

1433
  TSDB_CHECK_NULL(pData, code, lino, _end, TSDB_CODE_INVALID_PARA);
48,577,081!
1434
  TSDB_CHECK_NULL(pDumpInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
48,577,081!
1435
  TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA);
48,577,081!
1436
  TSDB_CHECK_CONDITION(dumpedRows >= 0, code, lino, _end, TSDB_CODE_INVALID_PARA);
48,577,081!
1437

1438
  TSDB_CHECK_CONDITION(pData->type < TSDB_DATA_TYPE_MAX, code, lino, _end, TSDB_CODE_INVALID_PARA);
48,577,081!
1439
  if (asc) {
48,578,775✔
1440
    p = pData->pData + tDataTypes[pData->type].bytes * pDumpInfo->rowIndex;
47,323,656✔
1441
  } else {
1442
    int32_t startIndex = pDumpInfo->rowIndex - dumpedRows + 1;
1,255,119✔
1443
    p = pData->pData + tDataTypes[pData->type].bytes * startIndex;
1,255,119✔
1444
  }
1445

1446
  // make sure it is aligned to 8bit, the allocated memory address is aligned to 256bit
1447

1448
  // 1. copy data in a batch model
1449
  TAOS_MEMCPY(pColData->pData, p, dumpedRows * tDataTypes[pData->type].bytes);
48,585,067!
1450

1451
  // 2. reverse the array list in case of descending order scan data block
1452
  if (!asc) {
48,578,533✔
1453
    switch (pColData->info.type) {
1,255,119!
1454
      case TSDB_DATA_TYPE_TIMESTAMP:
447,696✔
1455
      case TSDB_DATA_TYPE_DOUBLE:
1456
      case TSDB_DATA_TYPE_BIGINT:
1457
      case TSDB_DATA_TYPE_DECIMAL64:
1458
      case TSDB_DATA_TYPE_UBIGINT: {
1459
        int32_t  mid = dumpedRows >> 1u;
447,696✔
1460
        int64_t* pts = (int64_t*)pColData->pData;
447,696✔
1461
        for (int32_t j = 0; j < mid; ++j) {
646,268,076✔
1462
          int64_t t = pts[j];
645,820,380✔
1463
          pts[j] = pts[dumpedRows - j - 1];
645,835,560✔
1464
          pts[dumpedRows - j - 1] = t;
645,843,150✔
1465
        }
1466
        break;
447,696✔
1467
      }
1468

1469
      case TSDB_DATA_TYPE_BOOL:
268,986✔
1470
      case TSDB_DATA_TYPE_TINYINT:
1471
      case TSDB_DATA_TYPE_UTINYINT: {
1472
        int32_t mid = dumpedRows >> 1u;
268,986✔
1473
        int8_t* pts = (int8_t*)pColData->pData;
268,986✔
1474
        for (int32_t j = 0; j < mid; ++j) {
390,024,846✔
1475
          int8_t t = pts[j];
389,755,860✔
1476
          pts[j] = pts[dumpedRows - j - 1];
389,749,995✔
1477
          pts[dumpedRows - j - 1] = t;
388,435,545✔
1478
        }
1479
        break;
268,986✔
1480
      }
1481

1482
      case TSDB_DATA_TYPE_SMALLINT:
179,324✔
1483
      case TSDB_DATA_TYPE_USMALLINT: {
1484
        int32_t  mid = dumpedRows >> 1u;
179,324✔
1485
        int16_t* pts = (int16_t*)pColData->pData;
179,324✔
1486
        for (int32_t j = 0; j < mid; ++j) {
259,911,569✔
1487
          int64_t t = pts[j];
259,732,245✔
1488
          pts[j] = pts[dumpedRows - j - 1];
259,694,640✔
1489
          pts[dumpedRows - j - 1] = t;
259,742,250✔
1490
        }
1491
        break;
179,324✔
1492
      }
1493

1494
      case TSDB_DATA_TYPE_FLOAT:
269,758✔
1495
      case TSDB_DATA_TYPE_INT:
1496
      case TSDB_DATA_TYPE_UINT: {
1497
        int32_t  mid = dumpedRows >> 1u;
269,758✔
1498
        int32_t* pts = (int32_t*)pColData->pData;
269,758✔
1499
        for (int32_t j = 0; j < mid; ++j) {
389,798,524✔
1500
          int32_t t = pts[j];
389,528,766✔
1501
          pts[j] = pts[dumpedRows - j - 1];
389,539,461✔
1502
          pts[dumpedRows - j - 1] = t;
389,558,436✔
1503
        }
1504
        break;
269,758✔
1505
      }
1506
      case TSDB_DATA_TYPE_DECIMAL: {
89,355✔
1507
        int32_t      mid = dumpedRows >> 1u;
89,355✔
1508
        DecimalWord* pDec = (DecimalWord*)pColData->pData;
89,355✔
1509
        DecimalWord  tmp[2] = {0};
89,355✔
1510
        for (int32_t j = 0; j < mid; ++j) {
129,233,205✔
1511
          tmp[0] = pDec[2 * j];
129,143,850✔
1512
          tmp[1] = pDec[2 * j + 1];
129,132,120✔
1513
          pDec[2 * j] = pDec[2 * (dumpedRows - j - 1)];
129,138,675✔
1514
          pDec[2 * j + 1] = pDec[2 * (dumpedRows - j - 1) + 1];
129,142,125✔
1515
          pDec[2 * (dumpedRows - j - 1)] = tmp[0];
129,142,815✔
1516
          pDec[2 * (dumpedRows - j - 1) + 1] = tmp[1];
129,142,815✔
1517
        }
1518
        break;
89,355✔
1519
      }
1520
    }
1521
  }
1522

1523
  // 3. if the  null value exists, check items one-by-one
1524
  if (pData->flag != HAS_VALUE) {
48,578,533✔
1525
    int32_t rowIndex = 0;
6,618,872✔
1526

1527
    for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step, rowIndex++) {
2,147,483,647✔
1528
      uint8_t v = tColDataGetBitValue(pData, j);
2,147,483,647✔
1529
      if (v == 0 || v == 1) {
2,147,483,647✔
1530
        colDataSetNull_f(pColData->nullbitmap, rowIndex);
2,147,483,647!
1531
        pColData->hasNull = true;
2,147,483,647✔
1532
      }
1533
    }
1534
  }
1535

1536
_end:
48,572,483✔
1537
  if (code != TSDB_CODE_SUCCESS) {
48,572,483!
1538
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1539
  }
1540
  return code;
48,572,725✔
1541
}
1542

1543
static void blockInfoToRecord(SBrinRecord* record, SFileDataBlockInfo* pBlockInfo, SBlockLoadSuppInfo* pSupp) {
131,784,807✔
1544
  record->uid = pBlockInfo->uid;
131,784,807✔
1545
  record->firstKey = (STsdbRowKey){.key = {.ts = pBlockInfo->firstKey, .numOfPKs = pSupp->numOfPks}};
131,795,918✔
1546
  record->lastKey = (STsdbRowKey){.key = {.ts = pBlockInfo->lastKey, .numOfPKs = pSupp->numOfPks}};
131,798,053✔
1547

1548
  if (pSupp->numOfPks > 0) {
131,800,213✔
1549
    SValue* pFirst = &record->firstKey.key.pks[0];
10,738✔
1550
    SValue* pLast = &record->lastKey.key.pks[0];
10,738✔
1551

1552
    pFirst->type = pSupp->pk.type;
10,738✔
1553
    pLast->type = pSupp->pk.type;
10,738✔
1554

1555
    if (IS_VAR_DATA_TYPE(pFirst->type)) {
10,738!
1556
      if (IS_STR_DATA_BLOB(pFirst->type)) {
10,738!
1557
        pFirst->pData = (uint8_t*)blobDataVal(pBlockInfo->firstPk.pData);
×
1558
        pFirst->nData = blobDataLen(pBlockInfo->firstPk.pData);
×
1559

1560
        pLast->pData = (uint8_t*)blobDataVal(pBlockInfo->lastPk.pData);
×
1561
        pLast->nData = blobDataLen(pBlockInfo->lastPk.pData);
×
1562

1563
      } else {
1564
        pFirst->pData = (uint8_t*)varDataVal(pBlockInfo->firstPk.pData);
10,738✔
1565
        pFirst->nData = varDataLen(pBlockInfo->firstPk.pData);
10,738✔
1566

1567
        pLast->pData = (uint8_t*)varDataVal(pBlockInfo->lastPk.pData);
10,738✔
1568
        pLast->nData = varDataLen(pBlockInfo->lastPk.pData);
10,738✔
1569
      }
1570
    } else {
1571
      VALUE_SET_TRIVIAL_DATUM(pFirst, pBlockInfo->firstPk.val);
×
1572
      VALUE_SET_TRIVIAL_DATUM(pLast, pBlockInfo->lastPk.val);
×
1573
    }
1574
  }
1575

1576
  record->minVer = pBlockInfo->minVer;
131,799,648✔
1577
  record->maxVer = pBlockInfo->maxVer;
131,800,793✔
1578
  record->blockOffset = pBlockInfo->blockOffset;
131,800,222✔
1579
  record->smaOffset = pBlockInfo->smaOffset;
131,801,434✔
1580
  record->blockSize = pBlockInfo->blockSize;
131,800,488✔
1581
  record->blockKeySize = pBlockInfo->blockKeySize;
131,799,987✔
1582
  record->smaSize = pBlockInfo->smaSize;
131,800,825✔
1583
  record->numRow = pBlockInfo->numRow;
131,798,384✔
1584
  record->count = pBlockInfo->count;
131,797,728✔
1585
}
131,800,301✔
1586

1587
static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, SRowKey* pLastProcKey) {
25,004,968✔
1588
  int32_t             code = TSDB_CODE_SUCCESS;
25,004,968✔
1589
  int32_t             lino = 0;
25,004,968✔
1590
  SReaderStatus*      pStatus = NULL;
25,004,968✔
1591
  SDataBlockIter*     pBlockIter = NULL;
25,004,968✔
1592
  SBlockLoadSuppInfo* pSupInfo = NULL;
25,004,968✔
1593
  SFileBlockDumpInfo* pDumpInfo = NULL;
25,004,968✔
1594
  SBlockData*         pBlockData = NULL;
25,004,968✔
1595
  SFileDataBlockInfo* pBlockInfo = NULL;
25,004,968✔
1596
  SSDataBlock*        pResBlock = NULL;
25,008,114✔
1597
  int32_t             numOfOutputCols = 0;
25,008,114✔
1598
  int64_t             st = 0;
25,008,114✔
1599
  bool                asc = false;
25,008,114✔
1600
  int32_t             step = 0;
25,008,114✔
1601
  SColVal             cv = {0};
25,008,114✔
1602
  SBrinRecord         tmp;
25,010,292✔
1603
  SBrinRecord*        pRecord = NULL;
25,009,082✔
1604

1605
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
25,009,082!
1606

1607
  pStatus = &pReader->status;
25,009,082✔
1608
  pBlockIter = &pStatus->blockIter;
25,008,356✔
1609
  pSupInfo = &pReader->suppInfo;
25,005,210✔
1610

1611
  pSupInfo->args = pReader->pTsdb->pVnode->pBse;
25,009,808✔
1612
  pDumpInfo = &pReader->status.fBlockDumpInfo;
25,013,680✔
1613

1614
  pBlockData = &pStatus->fileBlockData;
25,013,438✔
1615
  pResBlock = pReader->resBlockInfo.pResBlock;
25,007,388✔
1616
  numOfOutputCols = pSupInfo->numOfCols;
25,013,680✔
1617
  st = taosGetTimestampUs();
25,013,438✔
1618
  asc = ASCENDING_TRAVERSE(pReader->info.order);
25,013,438✔
1619
  step = asc ? 1 : -1;
25,013,680✔
1620

1621
  code = getCurrentBlockInfo(pBlockIter, &pBlockInfo, pReader->idStr);
25,013,680✔
1622
  TSDB_CHECK_CODE(code, lino, _end);
25,010,050!
1623

1624
  blockInfoToRecord(&tmp, pBlockInfo, pSupInfo);
25,010,050✔
1625
  pRecord = &tmp;
25,010,050✔
1626

1627
  // no data exists, return directly.
1628
  if (pBlockData->nRow == 0 || pBlockData->aTSKEY == 0) {
25,010,050!
1629
    tsdbWarn("%p no need to copy since no data in blockData, table uid:%" PRIu64 " has been dropped, %s", pReader,
3,388!
1630
             pBlockInfo->uid, pReader->idStr);
1631
    pResBlock->info.rows = 0;
3,388✔
1632
    goto _end;
×
1633
  }
1634

1635
  TSDB_CHECK_CONDITION((pDumpInfo->rowIndex >= 0) && (pDumpInfo->rowIndex < pRecord->numRow), code, lino, _end,
25,007,872!
1636
                       TSDB_CODE_INVALID_PARA);
1637

1638
  // row index of dump info remain the initial position, let's find the appropriate start position.
1639
  if (((pDumpInfo->rowIndex == 0) && asc) || ((pDumpInfo->rowIndex == (pRecord->numRow - 1)) && (!asc))) {
25,007,872!
1640
    if (asc && pReader->info.window.skey <= pRecord->firstKey.key.ts &&
25,001,562✔
1641
        pReader->info.verRange.minVer <= pRecord->minVer) {
24,669,274!
1642
      // pDumpInfo->rowIndex = 0;
1643
    } else if (!asc && pReader->info.window.ekey >= pRecord->lastKey.key.ts &&
335,918✔
1644
               pReader->info.verRange.maxVer >= pRecord->maxVer) {
221,552!
1645
      // pDumpInfo->rowIndex = pRecord->numRow - 1;
1646
    } else {  // find the appropriate the start position in current block, and set it to be the current rowIndex
1647
      int32_t pos = asc ? pRecord->numRow - 1 : 0;
114,366✔
1648
      int32_t order = asc ? TSDB_ORDER_DESC : TSDB_ORDER_ASC;
114,366✔
1649
      int64_t key = asc ? pReader->info.window.skey : pReader->info.window.ekey;
114,366✔
1650
      pDumpInfo->rowIndex = doBinarySearchKey(pBlockData->aTSKEY, pRecord->numRow, pos, key, order);
114,366✔
1651

1652
      if (pDumpInfo->rowIndex < 0) {
114,366!
1653
        tsdbError(
×
1654
            "%p failed to locate the start position in current block, global index:%d, table index:%d, brange:%" PRId64
1655
            "-%" PRId64 ", minVer:%" PRId64 ", maxVer:%" PRId64 " %s",
1656
            pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pRecord->firstKey.key.ts, pRecord->lastKey.key.ts,
1657
            pRecord->minVer, pRecord->maxVer, pReader->idStr);
1658
        code = TSDB_CODE_INVALID_PARA;
×
1659
        TSDB_CHECK_CODE(code, lino, _end);
×
1660
      }
1661

1662
      pDumpInfo->rowIndex = findFirstPos(pBlockData->aTSKEY, pRecord->numRow, pDumpInfo->rowIndex, (!asc));
114,366✔
1663

1664
      TSDB_CHECK_CONDITION(
114,366!
1665
          (pReader->info.verRange.minVer <= pRecord->maxVer && pReader->info.verRange.maxVer >= pRecord->minVer), code,
1666
          lino, _end, TSDB_CODE_INVALID_PARA);
1667

1668
      // find the appropriate start position that satisfies the version requirement.
1669
      if ((pReader->info.verRange.maxVer >= pRecord->minVer && pReader->info.verRange.maxVer < pRecord->maxVer) ||
114,366!
1670
          (pReader->info.verRange.minVer <= pRecord->maxVer && pReader->info.verRange.minVer > pRecord->minVer)) {
114,366!
1671
        int32_t i = pDumpInfo->rowIndex;
×
1672
        if (asc) {
×
1673
          for (; i < pRecord->numRow; ++i) {
×
1674
            if (pBlockData->aVersion[i] >= pReader->info.verRange.minVer) {
×
1675
              break;
×
1676
            }
1677
          }
1678
        } else {
1679
          for (; i >= 0; --i) {
×
1680
            if (pBlockData->aVersion[i] <= pReader->info.verRange.maxVer) {
×
1681
              break;
×
1682
            }
1683
          }
1684
        }
1685

1686
        pDumpInfo->rowIndex = i;
×
1687
      }
1688
    }
1689
  }
1690

1691
  // time window check
1692
  int32_t endIndex = getEndPosInDataBlock(pReader, pBlockData, pRecord, pDumpInfo->rowIndex);
25,008,114✔
1693
  if (endIndex == -1) {
25,001,580!
1694
    setBlockAllDumped(pDumpInfo, pReader->info.window.ekey, pReader->info.order);
×
1695
    goto _end;
×
1696
  }
1697

1698
  endIndex += step;
25,001,580✔
1699
  int32_t dumpedRows = asc ? (endIndex - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex - endIndex);
25,001,580✔
1700
  if (dumpedRows > pReader->resBlockInfo.capacity) {  // output buffer check
25,004,968!
1701
    dumpedRows = pReader->resBlockInfo.capacity;
×
1702
  } else if (dumpedRows <= 0) {  // no qualified rows in current data block, quit directly.
25,002,306!
1703
    setBlockAllDumped(pDumpInfo, pReader->info.window.ekey, pReader->info.order);
×
1704
    goto _end;
×
1705
  }
1706

1707
  int32_t i = 0;
25,002,306✔
1708
  int32_t rowIndex = 0;
25,002,306✔
1709

1710
  SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
25,002,306✔
1711
  TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA);
25,014,164!
1712

1713
  if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) {
25,014,164!
1714
    code = copyPrimaryTsCol(pBlockData, pDumpInfo, pColData, dumpedRows, asc);
25,014,164✔
1715
    TSDB_CHECK_CODE(code, lino, _end);
25,007,388!
1716
    i += 1;
25,007,388✔
1717
  }
1718

1719
  int32_t colIndex = 0;
25,007,388✔
1720
  int32_t num = pBlockData->nColData;
25,007,388✔
1721
  while (i < numOfOutputCols && colIndex < num) {
92,636,861!
1722
    rowIndex = 0;
67,627,779✔
1723

1724
    SColData* pData = tBlockDataGetColDataByIdx(pBlockData, colIndex);
67,627,779✔
1725
    if (pData->cid < pSupInfo->colId[i]) {
67,630,441!
1726
      colIndex += 1;
×
1727
    } else if (pData->cid == pSupInfo->colId[i]) {
67,634,313!
1728
      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
67,636,007✔
1729
      TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA);
67,640,260!
1730

1731
      if (pData->flag == HAS_NONE || pData->flag == HAS_NULL || pData->flag == (HAS_NULL | HAS_NONE)) {
67,640,260✔
1732
        colDataSetNNULL(pColData, 0, dumpedRows);
5,181,946!
1733
      } else {
1734
        if (IS_MATHABLE_TYPE(pColData->info.type)) {
62,460,353✔
1735
          code = copyNumericCols(pData, pDumpInfo, pColData, dumpedRows, asc);
48,578,291✔
1736
          TSDB_CHECK_CODE(code, lino, _end);
48,572,967!
1737
        } else {  // varchar/nchar type
1738
          for (int32_t j = pDumpInfo->rowIndex; rowIndex < dumpedRows; j += step) {
2,147,483,647✔
1739
            code = tColDataGetValue(pData, j, &cv);
2,147,483,647✔
1740
            TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
1741
            code = doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo);
2,147,483,647✔
1742
            TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
1743
          }
1744
        }
1745
      }
1746

1747
      colIndex += 1;
67,625,117✔
1748
      i += 1;
67,625,117✔
1749
    } else {  // the specified column does not exist in file block, fill with null data
1750
      pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
×
1751
      TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA);
×
1752

1753
      colDataSetNNULL(pColData, 0, dumpedRows);
×
1754
      i += 1;
×
1755
    }
1756
  }
1757

1758
  // fill the mis-matched columns with null value
1759
  while (i < numOfOutputCols) {
25,009,082!
1760
    pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
×
1761
    TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA);
×
1762

1763
    colDataSetNNULL(pColData, 0, dumpedRows);
×
1764
    i += 1;
×
1765
  }
1766

1767
  pResBlock->info.dataLoad = 1;
25,009,082✔
1768
  pResBlock->info.rows = dumpedRows;
25,011,018✔
1769
  pDumpInfo->rowIndex += step * dumpedRows;
25,011,018✔
1770

1771
  tColRowGetKeyDeepCopy(pBlockData, pDumpInfo->rowIndex - step, pSupInfo->pkSrcSlot, pLastProcKey);
25,008,356!
1772

1773
  // check if current block are all handled
1774
  if (pDumpInfo->rowIndex >= 0 && pDumpInfo->rowIndex < pRecord->numRow) {
25,130,819✔
1775
    int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex];
116,413✔
1776

1777
    // the remain data has out of query time window, ignore current block
1778
    if (outOfTimeWindow(ts, &pReader->info.window)) {
116,413!
1779
      setBlockAllDumped(pDumpInfo, ts, pReader->info.order);
116,413✔
1780
    }
1781
  } else {
1782
    int64_t ts = asc ? pRecord->lastKey.key.ts : pRecord->firstKey.key.ts;
24,892,911✔
1783
    setBlockAllDumped(pDumpInfo, ts, pReader->info.order);
24,895,573✔
1784
  }
1785

1786
  double elapsedTime = (taosGetTimestampUs() - st) / 1000.0;
25,012,712✔
1787
  pReader->cost.blockLoadTime += elapsedTime;
25,012,712✔
1788

1789
  int32_t unDumpedRows = asc ? pRecord->numRow - pDumpInfo->rowIndex : pDumpInfo->rowIndex + 1;
25,007,630✔
1790
  tsdbDebug("%p copy file block to sdatablock, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64
25,007,146✔
1791
            ", rows:%d, remain:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", uid:%" PRIu64 " elapsed time:%.2f ms, %s",
1792
            pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pRecord->firstKey.key.ts, pRecord->lastKey.key.ts,
1793
            dumpedRows, unDumpedRows, pRecord->minVer, pRecord->maxVer, pBlockInfo->uid, elapsedTime, pReader->idStr);
1794

1795
_end:
25,007,146✔
1796
  if (code != TSDB_CODE_SUCCESS) {
25,010,534!
1797
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1798
  }
1799
  return code;
25,010,534✔
1800
}
1801

1802
static FORCE_INLINE STSchema* getTableSchemaImpl(STsdbReader* pReader, uint64_t uid) {
1803
  int32_t code = TSDB_CODE_SUCCESS;
7,700✔
1804
  int32_t lino = 0;
7,700✔
1805

1806
  TSDB_CHECK_CONDITION((pReader != NULL) && (pReader->info.pSchema == NULL), code, lino, _end, TSDB_CODE_INVALID_PARA);
7,700!
1807

1808
  code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, -1, &pReader->info.pSchema);
7,632✔
1809
  if (code != TSDB_CODE_SUCCESS || pReader->info.pSchema == NULL) {
7,700!
1810
    terrno = code;
×
1811
    tsdbError("failed to get table schema, suid:%"PRIu64",uid:%" PRIu64 ", it may have been dropped, ver:-1, %s", pReader->info.suid, uid, pReader->idStr);
×
1812
  }
1813
  TSDB_CHECK_CODE(code, lino, _end);
7,700!
1814
  TSDB_CHECK_NULL(pReader->info.pSchema, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
7,700!
1815

1816
  code = tsdbRowMergerInit(&pReader->status.merger, pReader->info.pSchema);
7,632✔
1817
  TSDB_CHECK_CODE(code, lino, _end);
7,632!
1818

1819
_end:
7,632✔
1820
  if (code != TSDB_CODE_SUCCESS) {
7,632!
1821
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1822
    terrno = code;
×
1823
    return NULL;
×
1824
  }
1825
  return pReader->info.pSchema;
7,632✔
1826
}
1827

1828
static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockData* pBlockData,
28,141,851✔
1829
                                   uint64_t uid) {
1830
  int32_t             code = TSDB_CODE_SUCCESS;
28,141,851✔
1831
  int32_t             lino = 0;
28,141,851✔
1832
  STSchema*           pSchema = NULL;
28,141,851✔
1833
  SFileDataBlockInfo* pBlockInfo = NULL;
28,141,851✔
1834
  SBlockLoadSuppInfo* pSup = NULL;
28,144,720✔
1835
  SFileBlockDumpInfo* pDumpInfo = NULL;
28,144,720✔
1836
  int64_t             st = 0;
28,144,720✔
1837
  SBrinRecord         tmp;
28,144,720✔
1838
  SBrinRecord*        pRecord = NULL;
28,143,671✔
1839

1840
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
28,143,671!
1841
  TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA);
28,143,671!
1842

1843
  pSchema = pReader->info.pSchema;
28,143,671✔
1844
  st = taosGetTimestampUs();
28,147,317✔
1845
  pSup = &pReader->suppInfo;
28,147,317✔
1846

1847
  tBlockDataReset(pBlockData);
28,147,382✔
1848

1849
  if (pReader->info.pSchema == NULL) {
28,142,238!
1850
    pSchema = getTableSchemaImpl(pReader, uid);
×
1851
    if (pSchema == NULL) {
×
1852
      tsdbError("%p table uid:%" PRIu64 " failed to get tableschema, code:%s, %s", pReader, uid, tstrerror(code),
×
1853
                pReader->idStr);
1854
      TSDB_CHECK_NULL(pSchema, code, lino, _end, terrno);
×
1855
    }
1856
  }
1857

1858
  code = getCurrentBlockInfo(pBlockIter, &pBlockInfo, pReader->idStr);
28,143,206✔
1859
  TSDB_CHECK_CODE(code, lino, _end);
28,145,737!
1860

1861
  pDumpInfo = &pReader->status.fBlockDumpInfo;
28,145,737✔
1862

1863
  blockInfoToRecord(&tmp, pBlockInfo, pSup);
28,147,189✔
1864
  pRecord = &tmp;
28,149,383✔
1865
  code = tsdbDataFileReadBlockDataByColumn(pReader->pFileReader, pRecord, pBlockData, pSchema, &pSup->colId[1],
28,147,722✔
1866
                                           pSup->numOfCols - 1);
28,149,383✔
1867
  if (code != TSDB_CODE_SUCCESS) {
28,142,607!
1868
    tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64
×
1869
              ", rows:%d, code:%s %s",
1870
              pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlockInfo->firstKey, pBlockInfo->lastKey,
1871
              pBlockInfo->numRow, tstrerror(code), pReader->idStr);
1872
    TSDB_CHECK_CODE(code, lino, _end);
×
1873
  }
1874

1875
  double elapsedTime = (taosGetTimestampUs() - st) / 1000.0;
28,141,639✔
1876

1877
  tsdbDebug("%p load file block into buffer, global index:%d, index in table block list:%d, brange:%" PRId64 "-%" PRId64
28,141,639✔
1878
            ", rows:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s",
1879
            pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pRecord->firstKey.key.ts, pRecord->lastKey.key.ts,
1880
            pRecord->numRow, pRecord->minVer, pRecord->maxVer, elapsedTime, pReader->idStr);
1881

1882
  pReader->cost.blockLoadTime += elapsedTime;
28,141,639✔
1883
  pDumpInfo->allDumped = false;
28,147,303✔
1884

1885
_end:
28,146,512✔
1886
  if (code != TSDB_CODE_SUCCESS) {
28,147,624!
1887
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1888
  }
1889
  return code;
28,147,624✔
1890
}
1891

1892
/**
1893
 * This is an two rectangles overlap cases.
1894
 */
1895
static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* pVerRange, SFileDataBlockInfo* pBlock) {
43,767,579✔
1896
  return (pWindow->ekey < pBlock->lastKey && pWindow->ekey >= pBlock->firstKey) ||
44,323,817!
1897
         (pWindow->skey > pBlock->firstKey && pWindow->skey <= pBlock->lastKey) ||
43,212,486!
1898
         (pVerRange->minVer > pBlock->minVer && pVerRange->minVer <= pBlock->maxVer) ||
130,573,317!
1899
         (pVerRange->maxVer < pBlock->maxVer && pVerRange->maxVer >= pBlock->minVer);
43,037,968!
1900
}
1901

1902
static int32_t getNeighborBlockOfTable(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pBlockInfo,
46,900,253✔
1903
                                       STableBlockScanInfo* pScanInfo, int32_t* nextIndex, int32_t order,
1904
                                       SBrinRecord* pRecord, SBlockLoadSuppInfo* pSupInfo, bool* res) {
1905
  int32_t             code = TSDB_CODE_SUCCESS;
46,900,253✔
1906
  int32_t             lino = 0;
46,900,253✔
1907
  bool                asc = false;
46,900,253✔
1908
  int32_t             step = 0;
46,900,253✔
1909
  STableDataBlockIdx* pTableDataBlockIdx = NULL;
46,900,253✔
1910
  SFileDataBlockInfo* p = NULL;
46,900,253✔
1911

1912
  TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
46,900,253!
1913
  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
46,900,253!
1914
  TSDB_CHECK_NULL(res, code, lino, _end, TSDB_CODE_INVALID_PARA);
46,900,253!
1915

1916
  *res = false;
46,900,253✔
1917
  asc = ASCENDING_TRAVERSE(order);
46,900,702✔
1918
  step = asc ? 1 : -1;
46,900,702✔
1919

1920
  if (asc && pBlockInfo->tbBlockIdx >= taosArrayGetSize(pScanInfo->pBlockIdxList) - 1) {
46,900,702✔
1921
    *res = false;
23,338,368✔
1922
  } else if (!asc && pBlockInfo->tbBlockIdx == 0) {
23,562,625✔
1923
    *res = false;
2,236,888✔
1924
  } else {
1925
    TSDB_CHECK_NULL(pBlockIter, code, lino, _end, TSDB_CODE_INVALID_PARA);
21,329,381!
1926
    TSDB_CHECK_NULL(nextIndex, code, lino, _end, TSDB_CODE_INVALID_PARA);
21,329,381!
1927
    TSDB_CHECK_NULL(pRecord, code, lino, _end, TSDB_CODE_INVALID_PARA);
21,329,381!
1928
    TSDB_CHECK_NULL(pSupInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
21,329,381!
1929

1930
    pTableDataBlockIdx = taosArrayGet(pScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx + step);
21,329,381✔
1931
    TSDB_CHECK_NULL(pTableDataBlockIdx, code, lino, _end, TSDB_CODE_INVALID_PARA);
21,328,987!
1932

1933
    p = taosArrayGet(pBlockIter->blockList, pTableDataBlockIdx->globalIndex);
21,328,987✔
1934
    TSDB_CHECK_NULL(p, code, lino, _end, TSDB_CODE_INVALID_PARA);
21,329,096!
1935

1936
    blockInfoToRecord(pRecord, p, pSupInfo);
21,329,096✔
1937

1938
    *nextIndex = pBlockInfo->tbBlockIdx + step;
21,329,490✔
1939
    *res = true;
21,329,381✔
1940
  }
1941

1942
_end:
46,904,746✔
1943
  if (code != TSDB_CODE_SUCCESS) {
46,904,746!
1944
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1945
  }
1946
  return code;
46,903,052✔
1947
}
1948

1949
static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t index,
1,310✔
1950
                                             int32_t step) {
1951
  int32_t              code = TSDB_CODE_SUCCESS;
1,310✔
1952
  int32_t              lino = 0;
1,310✔
1953
  const void*          p = NULL;
1,310✔
1954
  SFileDataBlockInfo   fblock;
1,310✔
1955
  SFileDataBlockInfo*  pBlockInfo = NULL;
1,310✔
1956
  STableBlockScanInfo* pBlockScanInfo = NULL;
1,310✔
1957
  STableDataBlockIdx*  pTableDataBlockIdx = NULL;
1,310✔
1958

1959
  TSDB_CHECK_NULL(pBlockIter, code, lino, _end, TSDB_CODE_INVALID_PARA);
1,310!
1960
  TSDB_CHECK_CONDITION((index >= 0) && (index < pBlockIter->numOfBlocks), code, lino, _end, TSDB_CODE_INVALID_PARA);
1,310!
1961

1962
  p = taosArrayGet(pBlockIter->blockList, index);
1,310✔
1963
  TSDB_CHECK_NULL(p, code, lino, _end, TSDB_CODE_INVALID_PARA);
1,310!
1964

1965
  fblock = *(SFileDataBlockInfo*)p;
1,310✔
1966
  pBlockIter->index += step;
1,310✔
1967

1968
  if (index != pBlockIter->index) {
1,310!
1969
    if (index > pBlockIter->index) {
×
1970
      for (int32_t i = index - 1; i >= pBlockIter->index; --i) {
×
1971
        pBlockInfo = taosArrayGet(pBlockIter->blockList, i);
×
1972
        TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
×
1973

1974
        code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr);
×
1975
        TSDB_CHECK_CODE(code, lino, _end);
×
1976

1977
        pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx);
×
1978
        TSDB_CHECK_NULL(pTableDataBlockIdx, code, lino, _end, TSDB_CODE_INVALID_PARA);
×
1979

1980
        pTableDataBlockIdx->globalIndex = i + 1;
×
1981

1982
        taosArraySet(pBlockIter->blockList, i + 1, pBlockInfo);
×
1983
      }
1984
    } else if (index < pBlockIter->index) {
×
1985
      for (int32_t i = index + 1; i <= pBlockIter->index; ++i) {
×
1986
        pBlockInfo = taosArrayGet(pBlockIter->blockList, i);
×
1987
        TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
×
1988

1989
        code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr);
×
1990
        TSDB_CHECK_CODE(code, lino, _end);
×
1991

1992
        pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, pBlockInfo->tbBlockIdx);
×
1993
        TSDB_CHECK_NULL(pTableDataBlockIdx, code, lino, _end, TSDB_CODE_INVALID_PARA);
×
1994

1995
        pTableDataBlockIdx->globalIndex = i - 1;
×
1996
        taosArraySet(pBlockIter->blockList, i - 1, pBlockInfo);
×
1997
      }
1998
    }
1999

2000
    taosArraySet(pBlockIter->blockList, pBlockIter->index, &fblock);
×
2001
    pBlockScanInfo = NULL;
×
2002
    code = getTableBlockScanInfo(pReader->status.pTableMap, fblock.uid, &pBlockScanInfo, pReader->idStr);
×
2003
    TSDB_CHECK_CODE(code, lino, _end);
×
2004

2005
    pTableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, fblock.tbBlockIdx);
×
2006
    TSDB_CHECK_NULL(pTableDataBlockIdx, code, lino, _end, TSDB_CODE_INVALID_PARA);
×
2007

2008
    pTableDataBlockIdx->globalIndex = pBlockIter->index;
×
2009
  }
2010

2011
_end:
1,310✔
2012
  if (code != TSDB_CODE_SUCCESS) {
1,310!
2013
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2014
  }
2015
  return code;
1,310✔
2016
}
2017

2018
// todo: this attribute could be acquired during extractin the global ordered block list.
2019
static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* pRec, int32_t order, int32_t pkType,
21,329,490✔
2020
                                      int32_t numOfPk) {
2021
  // it is the last block in current file, no chance to overlap with neighbor blocks.
2022
  if (ASCENDING_TRAVERSE(order)) {
21,329,490✔
2023
    if (pBlock->lastKey == pRec->firstKey.key.ts) {
21,101,149✔
2024
      if (numOfPk > 0) {
3,204✔
2025
        SValue v1 = {.type = pkType};
2,478✔
2026
        if (IS_VAR_DATA_TYPE(pkType)) {
2,478!
2027
          if (IS_STR_DATA_BLOB(pkType)) {
2,478!
2028
            v1.pData = (uint8_t*)blobDataVal(pBlock->lastPk.pData), v1.nData = blobDataLen(pBlock->lastPk.pData);
×
2029
          } else {
2030
            v1.pData = (uint8_t*)varDataVal(pBlock->lastPk.pData), v1.nData = varDataLen(pBlock->lastPk.pData);
2,478✔
2031
          }
2032
        } else {
2033
          VALUE_SET_TRIVIAL_DATUM(&v1, pBlock->lastPk.val);
×
2034
        }
2035
        return (tValueCompare(&v1, &pRec->firstKey.key.pks[0]) == 0);
2,478✔
2036
      } else {  // no pk
2037
        return true;
726✔
2038
      }
2039
    } else {
2040
      return false;
21,097,945✔
2041
    }
2042
  } else {
2043
    if (pBlock->firstKey == pRec->lastKey.key.ts) {
228,341✔
2044
      if (numOfPk > 0) {
726!
2045
        SValue v1 = {.type = pkType};
×
2046
        if (IS_VAR_DATA_TYPE(pkType)) {
×
2047
          if (IS_STR_DATA_BLOB(pkType)) {
×
2048
            v1.pData = (uint8_t*)blobDataVal(pBlock->firstPk.pData), v1.nData = blobDataLen(pBlock->firstPk.pData);
×
2049

2050
          } else {
2051
            v1.pData = (uint8_t*)varDataVal(pBlock->firstPk.pData), v1.nData = varDataLen(pBlock->firstPk.pData);
×
2052
          }
2053
        } else {
2054
          VALUE_SET_TRIVIAL_DATUM(&v1, pBlock->firstPk.val);
×
2055
        }
2056
        return (tValueCompare(&v1, &pRec->lastKey.key.pks[0]) == 0);
×
2057
      } else {  // no pk
2058
        return true;
726✔
2059
      }
2060
    } else {
2061
      return false;
227,615✔
2062
    }
2063
  }
2064
}
2065

2066
static int64_t getBoarderKeyInFiles(SFileDataBlockInfo* pBlock, STableBlockScanInfo* pScanInfo, int32_t order) {
37,015,540✔
2067
  bool ascScan = ASCENDING_TRAVERSE(order);
37,015,540✔
2068

2069
  int64_t key = 0;
37,015,540✔
2070
  if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) {
37,015,540✔
2071
    int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey.ts;
291,870✔
2072
    key = ascScan ? TMIN(pBlock->firstKey, keyInStt) : TMAX(pBlock->lastKey, keyInStt);
292,126✔
2073
  } else {
2074
    key = ascScan ? pBlock->firstKey : pBlock->lastKey;
36,731,384✔
2075
  }
2076

2077
  return key;
37,028,350✔
2078
}
2079

2080
static bool bufferDataInFileBlockGap(TSDBKEY keyInBuf, SFileDataBlockInfo* pBlock, STableBlockScanInfo* pScanInfo,
37,020,853✔
2081
                                     int32_t order) {
2082
  bool    ascScan = ASCENDING_TRAVERSE(order);
37,020,853✔
2083
  int64_t key = getBoarderKeyInFiles(pBlock, pScanInfo, order);
37,020,853✔
2084

2085
  return (ascScan && (keyInBuf.ts != TSKEY_INITIAL_VAL && keyInBuf.ts < key)) ||
74,048,459✔
2086
         (!ascScan && (keyInBuf.ts != TSKEY_INITIAL_VAL && keyInBuf.ts > key));
37,023,022!
2087
}
2088

2089
static bool keyOverlapFileBlock(TSDBKEY key, SFileDataBlockInfo* pBlock, SVersionRange* pVerRange) {
43,767,095✔
2090
  return (key.ts >= pBlock->firstKey && key.ts <= pBlock->lastKey) && (pBlock->maxVer >= pVerRange->minVer) &&
46,500,091!
2091
         (pBlock->minVer <= pVerRange->maxVer);
2,736,012✔
2092
}
2093

2094
static int32_t getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pBlockInfo,
43,767,886✔
2095
                                  STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, STsdbReader* pReader) {
2096
  int32_t             code = TSDB_CODE_SUCCESS;
43,767,886✔
2097
  int32_t             lino = 0;
43,767,886✔
2098
  SBrinRecord         rec = {0};
43,767,886✔
2099
  int32_t             neighborIndex = 0;
43,770,064✔
2100
  int32_t             order = 0;
43,769,310✔
2101
  SBlockLoadSuppInfo* pSupInfo = NULL;
43,769,310✔
2102
  SBrinRecord         pRecord;
43,769,310✔
2103
  bool                hasNeighbor = false;
43,769,310✔
2104

2105
  TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
43,768,482!
2106
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
43,768,482!
2107

2108
  order = pReader->info.order;
43,768,482✔
2109
  pSupInfo = &pReader->suppInfo;
43,768,598✔
2110

2111
  code = getNeighborBlockOfTable(&pReader->status.blockIter, pBlockInfo, pScanInfo, &neighborIndex, order, &rec,
43,768,966✔
2112
                                 pSupInfo, &hasNeighbor);
2113
  TSDB_CHECK_CODE(code, lino, _end);
43,769,466!
2114

2115
  // overlap with neighbor
2116
  if (hasNeighbor) {
43,769,466!
2117
    pInfo->overlapWithNeighborBlock =
20,022,869✔
2118
        overlapWithNeighborBlock2(pBlockInfo, &rec, order, pSupInfo->pk.type, pSupInfo->numOfPks);
20,022,869✔
2119
  }
2120

2121
  blockInfoToRecord(&pRecord, pBlockInfo, pSupInfo);
43,769,466✔
2122

2123
  // has duplicated ts of different version in this block
2124
  pInfo->hasDupTs = (pBlockInfo->numRow > pBlockInfo->count) || (pBlockInfo->count <= 0);
43,769,804!
2125
  pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, &pRecord, order);
43,770,981✔
2126

2127
  // todo handle the primary key overlap case
2128
  if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) {
43,767,681✔
2129
    int64_t nextProcKeyInStt = pScanInfo->sttKeyInfo.nextProcKey.ts;
1,809,588✔
2130
    pInfo->overlapWithSttBlock = !(pBlockInfo->lastKey < nextProcKeyInStt || pBlockInfo->firstKey > nextProcKeyInStt);
1,809,844✔
2131
  }
2132

2133
  pInfo->moreThanCapcity = pBlockInfo->numRow > pReader->resBlockInfo.capacity;
43,768,789✔
2134
  pInfo->partiallyRequired = dataBlockPartiallyRequired(&pReader->info.window, &pReader->info.verRange, pBlockInfo);
43,769,327✔
2135
  pInfo->overlapWithKeyInBuf = keyOverlapFileBlock(keyInBuf, pBlockInfo, &pReader->info.verRange);
43,770,660✔
2136

2137
_end:
43,770,083✔
2138
  if (code != TSDB_CODE_SUCCESS) {
43,770,888!
2139
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2140
  }
2141
  return code;
43,770,888✔
2142
}
2143

2144
// 1. the version of all rows should be less than the endVersion
2145
// 2. current block should not overlap with next neighbor block
2146
// 3. current timestamp should not be overlap with each other
2147
// 4. output buffer should be large enough to hold all rows in current block
2148
// 5. delete info should not overlap with current block data
2149
// 6. current block should not contain the duplicated ts
2150
static int32_t fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo,
40,342,004✔
2151
                                   TSDBKEY keyInBuf, bool* load) {
2152
  int32_t              code = TSDB_CODE_SUCCESS;
40,342,004✔
2153
  int32_t              lino = 0;
40,342,004✔
2154
  SDataBlockToLoadInfo info = {0};
40,342,004✔
2155

2156
  TSDB_CHECK_NULL(load, code, lino, _end, TSDB_CODE_INVALID_PARA);
40,343,777!
2157

2158
  *load = false;
40,343,777✔
2159
  code = getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pReader);
40,343,712✔
2160
  TSDB_CHECK_CODE(code, lino, _end);
40,344,605!
2161

2162
  *load = (info.overlapWithNeighborBlock || info.hasDupTs || info.partiallyRequired || info.overlapWithKeyInBuf ||
80,686,209!
2163
           info.moreThanCapcity || info.overlapWithDelInfo || info.overlapWithSttBlock);
80,686,209!
2164

2165
  // log the reason why load the datablock for profile
2166
  if (*load) {
40,341,766!
2167
    tsdbDebug("%p uid:%" PRIu64
3,320,146!
2168
              " need to load the datablock, overlapneighbor:%d, hasDup:%d, partiallyRequired:%d, "
2169
              "overlapWithKey:%d, greaterThanBuf:%d, overlapWithDel:%d, overlapWithSttBlock:%d, %s",
2170
              pReader, pBlockInfo->uid, info.overlapWithNeighborBlock, info.hasDupTs, info.partiallyRequired,
2171
              info.overlapWithKeyInBuf, info.moreThanCapcity, info.overlapWithDelInfo, info.overlapWithSttBlock,
2172
              pReader->idStr);
2173
  }
2174

2175
_end:
40,341,455✔
2176
  if (code != TSDB_CODE_SUCCESS) {
40,340,631!
2177
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2178
  }
2179
  return code;
40,340,631✔
2180
}
2181

2182
static int32_t isCleanFileDataBlock(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo,
3,425,594✔
2183
                                    STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, bool* res) {
2184
  int32_t              code = TSDB_CODE_SUCCESS;
3,425,594✔
2185
  int32_t              lino = 0;
3,425,594✔
2186
  SDataBlockToLoadInfo info = {0};
3,425,594✔
2187

2188
  TSDB_CHECK_NULL(res, code, lino, _end, TSDB_CODE_INVALID_PARA);
3,425,594!
2189

2190
  *res = false;
3,425,594✔
2191
  code = getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pReader);
3,425,673✔
2192
  TSDB_CHECK_CODE(code, lino, _end);
3,425,287!
2193

2194
  *res = !(info.overlapWithNeighborBlock || info.hasDupTs || info.overlapWithKeyInBuf || info.overlapWithDelInfo ||
3,983,385!
2195
           info.overlapWithSttBlock);
558,098!
2196

2197
_end:
3,425,673✔
2198
  if (code != TSDB_CODE_SUCCESS) {
3,425,287!
2199
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2200
  }
2201
  return code;
3,425,287✔
2202
}
2203

2204
static int32_t initRowMergeIfNeeded(STsdbReader* pReader, int64_t uid) {
2,147,483,647✔
2205
  int32_t     code = TSDB_CODE_SUCCESS;
2,147,483,647✔
2206
  int32_t     lino = 0;
2,147,483,647✔
2207
  SRowMerger* pMerger = NULL;
2,147,483,647✔
2208
  STSchema*   ps = NULL;
2,147,483,647✔
2209

2210
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
2211

2212
  pMerger = &pReader->status.merger;
2,147,483,647✔
2213

2214
  if (pMerger->pArray == NULL) {
2,147,483,647✔
2215
    ps = getTableSchemaImpl(pReader, uid);
7,700!
2216
    TSDB_CHECK_NULL(ps, code, lino, _end, terrno);
7,632!
2217
  }
2218

2219
_end:
2,147,483,647✔
2220
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
2221
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2222
  }
2223
  return code;
2,147,483,647✔
2224
}
2225

2226
static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, int64_t endKey) {
827,317,234✔
2227
  int32_t      code = TSDB_CODE_SUCCESS;
827,317,234✔
2228
  int32_t      lino = 0;
827,317,234✔
2229
  int64_t      st = 0;
827,317,234✔
2230
  SSDataBlock* pBlock = NULL;
827,317,234✔
2231

2232
  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
827,317,234!
2233

2234
  if (!(pBlockScanInfo->iiter.hasVal || pBlockScanInfo->iter.hasVal)) {
827,317,234!
2235
    goto _end;
439,429,453✔
2236
  }
2237

2238
  code = initRowMergeIfNeeded(pReader, pBlockScanInfo->uid);
388,350,917✔
2239
  TSDB_CHECK_CODE(code, lino, _end);
388,163,133!
2240

2241
  st = taosGetTimestampUs();
388,289,438✔
2242
  pBlock = pReader->resBlockInfo.pResBlock;
388,289,438✔
2243
  code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->resBlockInfo.capacity, pReader);
388,280,412✔
2244
  TSDB_CHECK_CODE(code, lino, _end);
388,013,986!
2245

2246
  double el = (taosGetTimestampUs() - st) / 1000.0;
388,226,155✔
2247
  updateComposedBlockInfo(pReader, el, pBlockScanInfo);
388,226,155✔
2248

2249
  tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%" PRId64 ", brange:%" PRId64
388,276,085✔
2250
            " - %" PRId64 ", uid:%" PRIu64 ",  %s",
2251
            pReader, el, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey, pBlockScanInfo->uid,
2252
            pReader->idStr);
2253

2254
  pReader->cost.buildmemBlock += el;
388,278,658✔
2255

2256
_end:
827,718,645✔
2257
  if (code != TSDB_CODE_SUCCESS) {
827,718,645!
2258
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2259
  }
2260
  return code;
827,358,639✔
2261
}
2262

2263
static int32_t tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pBlockData, SRowKey* pKey,
2,147,483,647✔
2264
                                               SFileBlockDumpInfo* pDumpInfo, bool* copied) {
2265
  // opt version
2266
  // 1. it is not a border point
2267
  // 2. the direct next point is not an duplicated timestamp
2268
  int32_t code = TSDB_CODE_SUCCESS;
2,147,483,647✔
2269
  int32_t lino = 0;
2,147,483,647✔
2270
  bool    asc = false;
2,147,483,647✔
2271

2272
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
2273
  TSDB_CHECK_NULL(pDumpInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
2274
  TSDB_CHECK_NULL(copied, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
2275

2276
  *copied = false;
2,147,483,647✔
2277
  asc = (pReader->info.order == TSDB_ORDER_ASC);
2,147,483,647✔
2278
  if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && asc) || (pDumpInfo->rowIndex > 0 && (!asc))) {
2,147,483,647✔
2279
    int32_t step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1;
2,147,483,647✔
2280

2281
    SRowKey nextRowKey;
2,147,483,647✔
2282
    TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
2283
    tColRowGetKey(pBlockData, pDumpInfo->rowIndex + step, &nextRowKey);
2,147,483,647✔
2284

2285
    if (pkCompEx(pKey, &nextRowKey) != 0) {  // merge is not needed
2,147,483,647✔
2286
      code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, pBlockData, pDumpInfo->rowIndex);
2,147,483,647✔
2287
      TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
2288
      pDumpInfo->rowIndex += step;
2,147,483,647✔
2289
      *copied = true;
2,147,483,647✔
2290
    }
2291
  }
2292

2293
_end:
1,407,118✔
2294
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
2295
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2296
  }
2297
  return code;
2,147,483,647✔
2298
}
2299

2300
static int32_t nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, int32_t pkSrcSlot,
2,147,483,647✔
2301
                                    SVersionRange* pVerRange) {
2302
  int32_t  code = TSDB_CODE_SUCCESS;
2,147,483,647✔
2303
  int32_t  lino = 0;
2,147,483,647✔
2304
  int32_t  order = 0;
2,147,483,647✔
2305
  int32_t  step = 0;
2,147,483,647✔
2306
  SRowKey* pNextProc = NULL;
2,147,483,647✔
2307

2308
  TSDB_CHECK_NULL(pSttBlockReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
2309
  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
2310

2311
  order = pSttBlockReader->order;
2,147,483,647✔
2312
  step = ASCENDING_TRAVERSE(order) ? 1 : -1;
2,147,483,647✔
2313
  pNextProc = &pScanInfo->sttKeyInfo.nextProcKey;
2,147,483,647✔
2314

2315
  while (1) {
52,315,579✔
2316
    bool hasVal = false;
2,147,483,647✔
2317
    code = tMergeTreeNext(&pSttBlockReader->mergeTree, &hasVal);
2,147,483,647✔
2318
    if (code) {
2,147,483,647!
2319
      tsdbError("failed to iter the next row in stt-file merge tree, code:%s, %s", tstrerror(code),
×
2320
                pSttBlockReader->mergeTree.idStr);
2321
      TSDB_CHECK_CODE(code, lino, _end);
8,526!
2322
    }
2323

2324
    if (!hasVal) {  // the next value will be the accessed key in stt
2,147,483,647✔
2325
      pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA;
292,102,495✔
2326

2327
      // next file, the timestamps in the next file must be greater than those in current
2328
      pNextProc->ts += step;
292,156,702✔
2329
      if (pSttBlockReader->numOfPks > 0) {
292,117,042✔
2330
        if (IS_NUMERIC_TYPE(pNextProc->pks[0].type)) {
13,161,587!
2331
          VALUE_SET_TRIVIAL_DATUM(pNextProc->pks, INT64_MAX);
8,483,216✔
2332
        } else {
2333
          memset(pNextProc->pks[0].pData, 0, pNextProc->pks[0].nData);
4,676,145!
2334
        }
2335
      }
2336
      goto _end;
292,061,279✔
2337
    }
2338

2339
    TSDBROW* pRow = tMergeTreeGetRow(&pSttBlockReader->mergeTree);
2,147,483,647✔
2340
    int64_t  key = pRow->pBlockData->aTSKEY[pRow->iRow];
2,147,483,647✔
2341
    int64_t  ver = pRow->pBlockData->aVersion[pRow->iRow];
2,147,483,647✔
2342

2343
    if (pSttBlockReader->numOfPks == 0) {
2,147,483,647✔
2344
      pSttBlockReader->currentKey.ts = key;
2,147,483,647✔
2345
    } else {
2346
      tColRowGetKeyDeepCopy(pRow->pBlockData, pRow->iRow, pkSrcSlot, &pSttBlockReader->currentKey);
9,649,811!
2347
    }
2348

2349
    tColRowGetKeyDeepCopy(pRow->pBlockData, pRow->iRow, pkSrcSlot, pNextProc);
2,147,483,647!
2350

2351
    if (pScanInfo->delSkyline != NULL && TARRAY_SIZE(pScanInfo->delSkyline) > 0) {
2,147,483,647!
2352
      bool dropped = false;
103,890,216✔
2353
      code = hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, order, pVerRange,
103,895,366✔
2354
                            pSttBlockReader->numOfPks > 0, &dropped);
103,895,216✔
2355
      TSDB_CHECK_CODE(code, lino, _end);
103,884,857!
2356
      if (!dropped) {
103,884,857!
2357
        pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA;
51,569,328✔
2358
        goto _end;
51,572,137✔
2359
      }
2360
    } else {
2361
      pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA;
2,147,483,647✔
2362
      goto _end;
2,147,483,647✔
2363
    }
2364
  }
2365

2366
_end:
2,147,483,647✔
2367
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
2368
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2369
  }
2370
  return code;
2,147,483,647✔
2371
}
2372

2373
static void doPinSttBlock(SSttBlockReader* pSttBlockReader) { tMergeTreePinSttBlock(&pSttBlockReader->mergeTree); }
2,147,483,647✔
2374

2375
static void doUnpinSttBlock(SSttBlockReader* pSttBlockReader) { tMergeTreeUnpinSttBlock(&pSttBlockReader->mergeTree); }
2,147,483,647✔
2376

2377
static int32_t tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SSttBlockReader* pSttBlockReader,
2,147,483,647✔
2378
                                              STableBlockScanInfo* pScanInfo, SRowKey* pSttKey, STsdbReader* pReader,
2379
                                              bool* copied) {
2380
  int32_t  code = TSDB_CODE_SUCCESS;
2,147,483,647✔
2381
  int32_t  lino = 0;
2,147,483,647✔
2382
  SRowKey* pNext = NULL;
2,147,483,647✔
2383

2384
  TSDB_CHECK_NULL(pSttBlockReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
2385
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
2386
  TSDB_CHECK_NULL(copied, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
2387

2388
  *copied = false;
2,147,483,647✔
2389

2390
  // avoid the fetch next row replace the referenced stt block in buffer
2391
  doPinSttBlock(pSttBlockReader);
2,147,483,647✔
2392
  code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange);
2,147,483,647✔
2393
  doUnpinSttBlock(pSttBlockReader);
2,147,483,647✔
2394
  TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
2395

2396
  if (hasDataInSttBlock(pScanInfo)) {
2,147,483,647✔
2397
    pNext = getCurrentKeyInSttBlock(pSttBlockReader);
2,147,483,647!
2398
    if (pkCompEx(pSttKey, pNext) != 0) {
2,147,483,647✔
2399
      code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, fRow->pBlockData, fRow->iRow);
2,147,483,647✔
2400
      *copied = (code == TSDB_CODE_SUCCESS);
2,147,483,647✔
2401
      TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
2402
    }
2403
  } else {
2404
    code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, fRow->pBlockData, fRow->iRow);
128,593,421✔
2405
    *copied = (code == TSDB_CODE_SUCCESS);
130,261,021✔
2406
    TSDB_CHECK_CODE(code, lino, _end);
130,265,469!
2407
  }
2408

2409
_end:
130,265,469✔
2410
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
2411
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2412
  }
2413
  return code;
2,147,483,647✔
2414
}
2415

2416
static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* pReader, uint64_t uid) {
2417
  int32_t   code = TSDB_CODE_SUCCESS;
2,147,483,647✔
2418
  int32_t   lino = 0;
2,147,483,647✔
2419
  STSchema* ps = NULL;
2,147,483,647✔
2420
  void**    p = NULL;
2,147,483,647✔
2421

2422
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
2423

2424
  // always set the newest schema version in pReader->info.pSchema
2425
  if (pReader->info.pSchema == NULL) {
2,147,483,647!
2426
    ps = getTableSchemaImpl(pReader, uid);
×
2427
    TSDB_CHECK_NULL(ps, code, lino, _end, terrno);
×
2428
  }
2429

2430
  if (pReader->info.pSchema && sversion == pReader->info.pSchema->version) {
2,147,483,647!
2431
    ps = pReader->info.pSchema;
2,147,483,647✔
2432
    goto _end;
2,147,483,647✔
2433
  }
2434

2435
  p = tSimpleHashGet(pReader->pSchemaMap, &sversion, sizeof(sversion));
1,981,398,450✔
2436
  if (p != NULL) {
2,114,296,579!
2437
    ps = *(STSchema**)p;
2,109,372,332✔
2438
    goto _end;
2,109,409,055✔
2439
  }
2440

2441
  code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, uid, sversion, &ps);
4,924,247✔
2442
  TSDB_CHECK_CODE(code, lino, _end);
4,925,294!
2443

2444
  code = tSimpleHashPut(pReader->pSchemaMap, &sversion, sizeof(sversion), &ps, POINTER_BYTES);
4,925,294✔
2445
  TSDB_CHECK_CODE(code, lino, _end);
4,925,939!
2446

2447
_end:
2,147,483,647✔
2448
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
2449
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2450
    terrno = code;
×
2451
    return NULL;
×
2452
  }
2453
  return ps;
2,147,483,647✔
2454
}
2455

2456
// deep copy the primary key, to avoid ref invalid buffer if the primary key is varchar or something.
2457
static int32_t primaryKeyDup(SRowKey* pKey) {
2,147,483,647✔
2458
  if (pKey->numOfPKs > 0) {
2,147,483,647✔
2459
    if (!IS_NUMERIC_TYPE(pKey->pks[0].type)) {
3,383,296!
2460
      void* p = taosMemoryMalloc(pKey->pks[0].nData);
3,383,296!
2461
      if (p == NULL)  {
3,383,296!
2462
        tsdbError("%s failed at %s:%d since %s", __func__, __FILE__, __LINE__, tstrerror(terrno));
×
2463
        // memory allocation failed, set pData to NULL to avoid freeing original buffer
2464
        pKey->pks[0].pData = NULL;
×
2465
        return terrno;
×
2466
      }
2467

2468
      TAOS_MEMCPY(p, pKey->pks[0].pData, pKey->pks[0].nData);
3,383,296!
2469
      pKey->pks[0].pData = p;
3,383,296✔
2470
    }
2471
  }
2472

2473
  return 0;
2,147,483,647✔
2474
}
2475

2476
static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow,
2,147,483,647✔
2477
                                     SIterInfo* pIter, SSttBlockReader* pSttBlockReader) {
2478
  int32_t             code = TSDB_CODE_SUCCESS;
2,147,483,647✔
2479
  int32_t             lino = 0;
2,147,483,647✔
2480
  SRowMerger*         pMerger = NULL;
2,147,483,647✔
2481
  SRow*               pTSRow = NULL;
2,147,483,647✔
2482
  SBlockData*         pBlockData = NULL;
2,147,483,647✔
2483
  SFileBlockDumpInfo* pDumpInfo = NULL;
2,147,483,647✔
2484
  SRowKey*            pSttKey = NULL;
2,147,483,647✔
2485
  int32_t             pkSrcSlot = 0;
2,147,483,647✔
2486
  SRowKey             k = {0};
2,147,483,647✔
2487
  STSchema*           pSchema = NULL;
2,147,483,647✔
2488

2489
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
2490
  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
2491
  TSDB_CHECK_NULL(pRow, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
2492

2493
  pMerger = &pReader->status.merger;
2,147,483,647✔
2494
  pMerger->arg = pReader->pTsdb->pVnode->pBse;
2,147,483,647✔
2495

2496
  pBlockData = &pReader->status.fileBlockData;
2,147,483,647✔
2497
  pDumpInfo = &pReader->status.fBlockDumpInfo;
2,147,483,647✔
2498
  pkSrcSlot = pReader->suppInfo.pkSrcSlot;
2,147,483,647✔
2499

2500
  if (hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) {
2,147,483,647✔
2501
    pSttKey = getCurrentKeyInSttBlock(pSttBlockReader);
2,147,483,647✔
2502
  }
2503

2504
  tRowGetKeyEx(pRow, &k);
2,147,483,647✔
2505

2506
  if (pRow->type == TSDBROW_ROW_FMT) {
2,147,483,647✔
2507
    pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid);
2,147,483,647!
2508
    TSDB_CHECK_NULL(pSchema, code, lino, _end, terrno);
2,147,483,647!
2509
  }
2510

2511
  SRowKey* pfKey = &(SRowKey){0};
2,147,483,647✔
2512
  if (hasDataInFileBlock(pBlockData, pDumpInfo)) {
2,147,483,647✔
2513
    tColRowGetKey(pBlockData, pDumpInfo->rowIndex, pfKey);
2,147,483,647✔
2514
    code = primaryKeyDup(pfKey);
2,147,483,647✔
2515
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
2516
  } else {
2517
    pfKey = NULL;
2,147,483,647✔
2518
  }
2519

2520
  TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
2,147,483,647✔
2521

2522
  // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized
2523
  code = initRowMergeIfNeeded(pReader, pBlockScanInfo->uid);
2,147,483,647✔
2524
  TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
2525

2526
  SRowKey minKey = k;
2,147,483,647✔
2527
  if (pReader->info.order == TSDB_ORDER_ASC) {
2,147,483,647✔
2528
    if (pfKey != NULL && pkCompEx(pfKey, &minKey) < 0) {
2,147,483,647✔
2529
      minKey = *pfKey;
2,147,483,647✔
2530
    }
2531

2532
    if (pSttKey != NULL && pkCompEx(pSttKey, &minKey) < 0) {
2,147,483,647✔
2533
      minKey = *pSttKey;
2,147,483,647✔
2534
    }
2535
  } else {
2536
    if (pfKey != NULL && pkCompEx(pfKey, &minKey) > 0) {
66,073,099✔
2537
      minKey = *pfKey;
7,480,728✔
2538
    }
2539

2540
    if (pSttKey != NULL && pkCompEx(pSttKey, &minKey) > 0) {
83,388,196✔
2541
      minKey = *pSttKey;
565,554✔
2542
    }
2543
  }
2544

2545
  // copy the last key before the time of stt reader loading the next stt block, in which the underlying data block may
2546
  // be changed, resulting in the corresponding changing of the value of sttRowKey
2547
  tRowKeyAssign(&pBlockScanInfo->lastProcKey, &minKey);
2,147,483,647✔
2548

2549
  // file block ---> stt block -----> mem
2550
  if (pkCompEx(&pBlockScanInfo->lastProcKey, pfKey) == 0) {
2,147,483,647✔
2551
    code = tsdbRowMergerAdd(pMerger, &fRow, NULL);
2,147,483,647✔
2552
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
2553

2554
    code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pfKey, pReader);
2,147,483,647✔
2555
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
2556
  }
2557

2558
  if (pkCompEx(&pBlockScanInfo->lastProcKey, pSttKey) == 0) {
2,147,483,647✔
2559
    TSDBROW* fRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree);
2,147,483,647✔
2560
    code = tsdbRowMergerAdd(pMerger, fRow1, NULL);
2,147,483,647✔
2561
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
2562
    code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange,
2,147,483,647✔
2563
                                 pReader->idStr);
2,147,483,647✔
2564
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
2565
  }
2566

2567
  if (pkCompEx(&pBlockScanInfo->lastProcKey, &k) == 0) {
2,147,483,647!
2568
    code = tsdbRowMergerAdd(pMerger, pRow, pSchema);
2,092,278,177✔
2569
    TSDB_CHECK_CODE(code, lino, _end);
2,092,705,365!
2570

2571
    code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, &k, pBlockScanInfo->delSkyline, pReader);
2,092,705,365✔
2572
    TSDB_CHECK_CODE(code, lino, _end);
2,092,453,863!
2573
  }
2574

2575
  code = tsdbRowMergerGetRow(pMerger, &pTSRow);
2,147,483,647✔
2576
  TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
2577

2578
  code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo);
2,147,483,647✔
2579

2580
  taosMemoryFreeClear(pTSRow);
2,147,483,647!
2581
  tsdbRowMergerClear(pMerger);
2,147,483,647✔
2582
  TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
2583

2584
_end:
2,147,483,647✔
2585
  clearRowKey(pfKey);
2,147,483,647✔
2586
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
2587
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2588
  }
2589
  return code;
2,147,483,647✔
2590
}
2591

2592
static int32_t mergeFileBlockAndSttBlock(STsdbReader* pReader, SSttBlockReader* pSttBlockReader, SRowKey* pKey,
2,147,483,647✔
2593
                                         STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) {
2594
  int32_t             code = TSDB_CODE_SUCCESS;
2,147,483,647✔
2595
  int32_t             lino = 0;
2,147,483,647✔
2596
  SFileBlockDumpInfo* pDumpInfo = NULL;
2,147,483,647✔
2597
  SRowMerger*         pMerger = NULL;
2,147,483,647✔
2598
  SRow*               pTSRow = NULL;
2,147,483,647✔
2599
  int32_t             pkSrcSlot = 0;
2,147,483,647✔
2600

2601
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
2602
  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
2603
  TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
2604

2605
  pDumpInfo = &pReader->status.fBlockDumpInfo;
2,147,483,647✔
2606
  pMerger = &pReader->status.merger;
2,147,483,647✔
2607
  pMerger->arg = pReader->pTsdb->pVnode->pBse;
2,147,483,647✔
2608

2609
  pkSrcSlot = pReader->suppInfo.pkSrcSlot;
2,147,483,647✔
2610

2611
  // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized
2612
  code = initRowMergeIfNeeded(pReader, pBlockScanInfo->uid);
2,147,483,647✔
2613
  TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
2614

2615
  bool dataInDataFile = hasDataInFileBlock(pBlockData, pDumpInfo);
2,147,483,647✔
2616
  bool dataInSttFile = hasDataInSttBlock(pBlockScanInfo);
2,147,483,647✔
2617

2618
  if (dataInDataFile && (!dataInSttFile)) {
2,147,483,647✔
2619
    // no stt file block available, only data block exists
2620
    code = mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader);
2,147,483,647✔
2621
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
2622
  } else if ((!dataInDataFile) && dataInSttFile) {
2,147,483,647!
2623
    // no data in data file exists
2624
    code = mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader);
2,147,483,647✔
2625
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
2626
  } else if (pBlockScanInfo->cleanSttBlocks && pReader->info.execMode == READER_EXEC_ROWS) {
421,188,186!
2627
    // opt model for count data in stt file, which is not overlap with data blocks in files.
2628
    code = mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader);
×
2629
    TSDB_CHECK_CODE(code, lino, _end);
×
2630
  } else {
2631
    TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA);
424,383,848!
2632
    // row in both stt file blocks and data file blocks
2633
    TSDBROW  fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
424,383,848✔
2634
    SRowKey* pSttKey = getCurrentKeyInSttBlock(pSttBlockReader);
424,415,080!
2635

2636
    int32_t ret = pkCompEx(pKey, pSttKey);
424,681,832✔
2637

2638
    if (ASCENDING_TRAVERSE(pReader->info.order)) {
424,681,832✔
2639
      if (ret < 0) {  // asc
393,408,433✔
2640
        code = mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader);
370,011,078✔
2641
        TSDB_CHECK_CODE(code, lino, _end);
369,780,422!
2642
      } else if (ret > 0) {
23,397,355✔
2643
        code = mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader);
4,705,131✔
2644
        TSDB_CHECK_CODE(code, lino, _end);
4,705,131!
2645
      }
2646
    } else {  // desc
2647
      if (ret > 0) {
31,285,687✔
2648
        code = mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader);
29,137,910✔
2649
        TSDB_CHECK_CODE(code, lino, _end);
29,137,910!
2650
      } else if (ret < 0) {
2,147,777✔
2651
        code = mergeRowsInSttBlocks(pSttBlockReader, pBlockScanInfo, pReader);
211,581✔
2652
        TSDB_CHECK_CODE(code, lino, _end);
211,581!
2653
      }
2654
    }
2655
    if (ret != 0) {
424,463,464✔
2656
      goto _end;
403,832,740✔
2657
    }
2658

2659
    // pKey == pSttKey
2660
    tRowKeyAssign(&pBlockScanInfo->lastProcKey, pKey);
20,630,724✔
2661

2662
    // the following for key == sttKey->key.ts
2663
    // file block ------> stt  block
2664

2665
    code = tsdbRowMergerAdd(pMerger, &fRow, NULL);
20,631,748✔
2666
    TSDB_CHECK_CODE(code, lino, _end);
20,631,492!
2667

2668
    code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader);
20,631,492✔
2669
    TSDB_CHECK_CODE(code, lino, _end);
20,630,468!
2670

2671
    TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree);
20,630,468✔
2672
    code = tsdbRowMergerAdd(pMerger, pRow1, NULL);
20,633,028✔
2673
    TSDB_CHECK_CODE(code, lino, _end);
20,629,188!
2674

2675
    // pSttKey will be changed when sttBlockReader iterates to the next row, so use pKey instead.
2676
    code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange,
20,628,676✔
2677
                                 pReader->idStr);
20,629,188✔
2678
    TSDB_CHECK_CODE(code, lino, _end);
20,631,236!
2679

2680
    code = tsdbRowMergerGetRow(pMerger, &pTSRow);
20,631,236✔
2681
    TSDB_CHECK_CODE(code, lino, _end);
20,634,820!
2682

2683
    code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo);
20,634,820✔
2684

2685
    taosMemoryFreeClear(pTSRow);
20,632,260!
2686
    tsdbRowMergerClear(pMerger);
20,628,420✔
2687
    TSDB_CHECK_CODE(code, lino, _end);
20,628,676!
2688
  }
2689

2690
_end:
2,147,483,647✔
2691
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
2692
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2693
  }
2694
  return code;
2,147,483,647✔
2695
}
2696

2697
static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData,
785,307,462✔
2698
                                     SSttBlockReader* pSttBlockReader) {
2699
  int32_t             code = TSDB_CODE_SUCCESS;
785,307,462✔
2700
  int32_t             lino = 0;
785,307,462✔
2701
  SRowMerger*         pMerger = NULL;
785,307,462✔
2702
  SRow*               pTSRow = NULL;
785,307,462✔
2703
  SFileBlockDumpInfo* pDumpInfo = NULL;
785,307,462✔
2704
  SArray*             pDelList = NULL;
785,307,462✔
2705
  int32_t             pkSrcSlot = 0;
785,307,462✔
2706
  TSDBROW*            pRow = NULL;
785,307,462✔
2707
  TSDBROW*            piRow = NULL;
785,307,462✔
2708
  SRowKey*            pSttKey = NULL;
785,307,462✔
2709

2710
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
785,307,462!
2711
  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
785,307,462!
2712
  TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA);
785,307,462!
2713

2714
  pMerger = &pReader->status.merger;
785,307,462✔
2715
  pDumpInfo = &pReader->status.fBlockDumpInfo;
785,307,462✔
2716
  pDelList = pBlockScanInfo->delSkyline;
785,307,462✔
2717
  pkSrcSlot = pReader->suppInfo.pkSrcSlot;
785,307,462✔
2718

2719
  code = getValidMemRow(&pBlockScanInfo->iter, pDelList, pReader, &pRow);
785,307,462✔
2720
  TSDB_CHECK_CODE(code, lino, _end);
785,307,462!
2721
  code = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader, &piRow);
785,307,462✔
2722
  TSDB_CHECK_CODE(code, lino, _end);
785,307,462!
2723

2724
  if (hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) {
785,307,462!
2725
    pSttKey = getCurrentKeyInSttBlock(pSttBlockReader);
785,307,462✔
2726
  }
2727

2728
  SRowKey* pfKey = &(SRowKey){0};
785,307,462✔
2729
  if (hasDataInFileBlock(pBlockData, pDumpInfo)) {
785,307,462!
2730
    tColRowGetKey(pBlockData, pDumpInfo->rowIndex, pfKey);
×
2731
  } else {
2732
    pfKey = NULL;
785,307,462✔
2733
  }
2734

2735
  SRowKey k = {0}, ik = {0};
785,307,462✔
2736
  tRowGetKeyEx(pRow, &k);
785,307,462!
2737
  tRowGetKeyEx(piRow, &ik);
785,307,462!
2738

2739
  STSchema* pSchema = NULL;
785,307,462✔
2740
  if (pRow->type == TSDBROW_ROW_FMT) {
785,307,462!
2741
    pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid);
785,307,462!
2742
    TSDB_CHECK_NULL(pSchema, code, lino, _end, terrno);
785,307,462!
2743
  }
2744

2745
  STSchema* piSchema = NULL;
785,307,462✔
2746
  if (piRow->type == TSDBROW_ROW_FMT) {
785,307,462!
2747
    piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid);
785,307,462!
2748
    TSDB_CHECK_NULL(pSchema, code, lino, _end, terrno);
785,307,462!
2749
  }
2750

2751
  // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized
2752
  code = initRowMergeIfNeeded(pReader, pBlockScanInfo->uid);
785,307,462✔
2753
  TSDB_CHECK_CODE(code, lino, _end);
785,307,462!
2754

2755
  SRowKey minKey = k;
785,307,462✔
2756
  if (ASCENDING_TRAVERSE(pReader->info.order)) {
785,307,462✔
2757
    if (pkCompEx(&ik, &minKey) < 0) {  // minKey > ik.key.ts) {
785,303,200!
2758
      minKey = ik;
785,303,200✔
2759
    }
2760

2761
    if ((pfKey != NULL) && (pkCompEx(pfKey, &minKey) < 0)) {
785,303,200!
2762
      minKey = *pfKey;
×
2763
    }
2764

2765
    if ((pSttKey != NULL) && (pkCompEx(pSttKey, &minKey) < 0)) {
1,570,606,400!
2766
      minKey = *pSttKey;
785,303,200✔
2767
    }
2768
  } else {
2769
    if (pkCompEx(&ik, &minKey) > 0) {
4,262!
2770
      minKey = ik;
×
2771
    }
2772

2773
    if ((pfKey != NULL) && (pkCompEx(pfKey, &minKey) > 0)) {
4,262!
2774
      minKey = *pfKey;
×
2775
    }
2776

2777
    if ((pSttKey != NULL) && (pkCompEx(pSttKey, &minKey) > 0)) {
8,524!
2778
      minKey = *pSttKey;
×
2779
    }
2780
  }
2781

2782
  tRowKeyAssign(&pBlockScanInfo->lastProcKey, &minKey);
785,307,462✔
2783

2784
  // file block -----> stt block -----> imem -----> mem
2785
  if (pkCompEx(&pBlockScanInfo->lastProcKey, pfKey) == 0) {
1,570,614,924!
2786
    TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
×
2787
    code = tsdbRowMergerAdd(pMerger, &fRow, NULL);
×
2788
    TSDB_CHECK_CODE(code, lino, _end);
×
2789

2790
    code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pfKey, pReader);
×
2791
    TSDB_CHECK_CODE(code, lino, _end);
×
2792
  }
2793

2794
  if (pkCompEx(&pBlockScanInfo->lastProcKey, pSttKey) == 0) {
1,570,614,924!
2795
    TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree);
785,307,462✔
2796
    code = tsdbRowMergerAdd(pMerger, pRow1, NULL);
785,307,462✔
2797
    TSDB_CHECK_CODE(code, lino, _end);
785,307,462!
2798

2799
    code = doMergeRowsInSttBlock(pSttBlockReader, pBlockScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange,
785,307,462✔
2800
                                 pReader->idStr);
785,307,462✔
2801
    TSDB_CHECK_CODE(code, lino, _end);
785,307,462!
2802
  }
2803

2804
  if (pkCompEx(&pBlockScanInfo->lastProcKey, &ik) == 0) {
1,570,614,924!
2805
    code = tsdbRowMergerAdd(pMerger, piRow, piSchema);
4,262✔
2806
    TSDB_CHECK_CODE(code, lino, _end);
4,262!
2807

2808
    code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, &ik, pBlockScanInfo->delSkyline, pReader);
4,262✔
2809
    TSDB_CHECK_CODE(code, lino, _end);
4,262!
2810
  }
2811

2812
  if (pkCompEx(&pBlockScanInfo->lastProcKey, &k) == 0) {
1,570,614,924!
2813
    code = tsdbRowMergerAdd(pMerger, pRow, pSchema);
4,262✔
2814
    TSDB_CHECK_CODE(code, lino, _end);
4,262!
2815

2816
    code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, &k, pBlockScanInfo->delSkyline, pReader);
4,262✔
2817
    TSDB_CHECK_CODE(code, lino, _end);
4,262!
2818
  }
2819

2820
  code = tsdbRowMergerGetRow(pMerger, &pTSRow);
785,307,462✔
2821
  TSDB_CHECK_CODE(code, lino, _end);
785,307,462!
2822

2823
  code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo);
785,307,462✔
2824

2825
  taosMemoryFreeClear(pTSRow);
785,307,462!
2826
  tsdbRowMergerClear(pMerger);
785,307,462✔
2827
  TSDB_CHECK_CODE(code, lino, _end);
785,307,462!
2828

2829
_end:
785,307,462✔
2830
  if (code != TSDB_CODE_SUCCESS) {
785,307,462!
2831
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2832
  }
2833
  return code;
785,307,462✔
2834
}
2835

2836
int32_t doInitMemDataIter(STsdbReader* pReader, STbData** pData, STableBlockScanInfo* pBlockScanInfo, STsdbRowKey* pKey,
970,463,835✔
2837
                          SMemTable* pMem, SIterInfo* pIter, const char* type) {
2838
  int32_t code = TSDB_CODE_SUCCESS;
970,463,835✔
2839
  int32_t lino = 0;
970,463,835✔
2840
  int32_t backward = false;
970,463,835✔
2841

2842
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
970,463,835!
2843
  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
970,463,835!
2844
  TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA);
970,463,835!
2845
  TSDB_CHECK_NULL(type, code, lino, _end, TSDB_CODE_INVALID_PARA);
970,463,835!
2846

2847
  backward = (!ASCENDING_TRAVERSE(pReader->info.order));
970,463,835✔
2848
  pIter->hasVal = false;
970,646,470✔
2849

2850
  if (pMem != NULL) {
970,217,008✔
2851
    TSDB_CHECK_NULL(pData, code, lino, _end, TSDB_CODE_INVALID_PARA);
433,017,594!
2852
    *pData = tsdbGetTbDataFromMemTable(pMem, pReader->info.suid, pBlockScanInfo->uid);
433,017,594✔
2853

2854
    if ((*pData) != NULL) {
433,397,739✔
2855
      code = tsdbTbDataIterCreate((*pData), pKey, backward, &pIter->iter);
376,109,697✔
2856
      if (code == TSDB_CODE_SUCCESS) {
376,001,974!
2857
        pIter->hasVal = (tsdbTbDataIterGet(pIter->iter) != NULL);
751,914,214!
2858

2859
        tsdbDebug("%p uid:%" PRIu64 ", check data in %s from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
375,935,026✔
2860
                  "-%" PRId64 " %s",
2861
                  pReader, pBlockScanInfo->uid, type, pKey->key.ts, pReader->info.order, (*pData)->minKey,
2862
                  (*pData)->maxKey, pReader->idStr);
2863
      } else {
2864
        tsdbError("%p uid:%" PRIu64 ", failed to create iterator for %s, code:%s, %s", pReader, pBlockScanInfo->uid,
×
2865
                  type, tstrerror(code), pReader->idStr);
2866
        TSDB_CHECK_CODE(code, lino, _end);
×
2867
      }
2868
    }
2869
  } else {
2870
    tsdbDebug("%p uid:%" PRIu64 ", no data in %s, %s", pReader, pBlockScanInfo->uid, type, pReader->idStr);
537,199,414✔
2871
  }
2872

2873
_end:
524,170,382✔
2874
  if (code != TSDB_CODE_SUCCESS) {
970,164,159!
2875
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2876
  }
2877
  return code;
970,330,695✔
2878
}
2879

2880
static int32_t doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanInfo* pBlockScanInfo,
94,613,534✔
2881
                                 STsdbReader* pReader) {
2882
  int32_t  code = TSDB_CODE_SUCCESS;
94,613,534✔
2883
  int32_t  lino = 0;
94,613,534✔
2884
  SRowKey  rowKey = {0};
94,613,534✔
2885
  TSDBROW* pRow = NULL;
94,662,885✔
2886

2887
  while (1) {
×
2888
    code = getValidMemRow(pIter, pBlockScanInfo->delSkyline, pReader, &pRow);
94,679,639✔
2889
    TSDB_CHECK_CODE(code, lino, _end);
94,594,483!
2890
    if (!pIter->hasVal) {
94,594,483✔
2891
      break;
52,368,279✔
2892
    }
2893

2894
    tRowGetKeyEx(pRow, &rowKey);
42,296,192✔
2895
    int32_t ret = pkCompEx(pKey, &rowKey);
42,295,177✔
2896
    if (ret == 0) {
42,295,177!
2897
      pIter->hasVal = tsdbTbDataIterNext(pIter->iter);
×
2898
    } else {
2899
      break;
42,295,177✔
2900
    }
2901
  }
2902

2903
_end:
94,663,456✔
2904
  if (code != TSDB_CODE_SUCCESS) {
94,634,153!
2905
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2906
  }
2907
  return code;
94,634,153✔
2908
}
2909

2910
// handle the open interval issue. Find the first row key that is greater than the given one.
2911
static int32_t forwardDataIter(SRowKey* pKey, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {
47,256,711✔
2912
  int32_t code = TSDB_CODE_SUCCESS;
47,256,711✔
2913
  int32_t lino = 0;
47,256,711✔
2914

2915
  TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA);
47,256,711!
2916
  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
47,256,711!
2917

2918
  code = doForwardDataIter(pKey, &pBlockScanInfo->iter, pBlockScanInfo, pReader);
47,256,711✔
2919
  TSDB_CHECK_CODE(code, lino, _end);
47,312,348!
2920
  code = doForwardDataIter(pKey, &pBlockScanInfo->iiter, pBlockScanInfo, pReader);
47,312,348✔
2921
  TSDB_CHECK_CODE(code, lino, _end);
47,358,814!
2922

2923
_end:
47,358,814✔
2924
  if (code != TSDB_CODE_SUCCESS) {
47,358,814!
2925
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2926
  }
2927
  return code;
47,295,784✔
2928
}
2929

2930
static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) {
1,121,072,473✔
2931
  int32_t        code = TSDB_CODE_SUCCESS;
1,121,072,473✔
2932
  int32_t        lino = 0;
1,121,072,473✔
2933
  STbData*       d = NULL;
1,121,072,473✔
2934
  STbData*       di = NULL;
1,121,212,528✔
2935
  bool           asc = false;
1,121,350,115✔
2936
  bool           forward = true;
1,121,350,115✔
2937
  STsdbReadSnap* pSnap = NULL;
1,121,350,115✔
2938
  STimeWindow*   pWindow = NULL;
1,121,350,115✔
2939
  STsdbRowKey    startKey;
1,121,338,097✔
2940

2941
  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
1,121,276,305!
2942
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
1,121,276,305!
2943

2944
  asc = ASCENDING_TRAVERSE(pReader->info.order);
1,121,276,305✔
2945
  pSnap = pReader->pReadSnap;
1,121,479,103✔
2946
  pWindow = &pReader->info.window;
1,121,431,894✔
2947

2948
  if (pBlockScanInfo->iterInit) {
1,121,494,158✔
2949
    goto _end;
636,073,900✔
2950
  }
2951

2952
  startKey.key = pBlockScanInfo->lastProcKey;
485,471,986✔
2953
  startKey.version = asc ? pReader->info.verRange.minVer : pReader->info.verRange.maxVer;
485,310,008✔
2954
  if ((asc && (startKey.key.ts < pWindow->skey)) || ((!asc) && startKey.key.ts > pWindow->ekey)) {
485,324,945✔
2955
    startKey.key.ts = asc ? pWindow->skey : pWindow->ekey;
438,010,588✔
2956
    forward = false;
437,878,395✔
2957
  }
2958

2959
  code = doInitMemDataIter(pReader, &d, pBlockScanInfo, &startKey, pSnap->pMem, &pBlockScanInfo->iter, "mem");
485,282,776✔
2960
  TSDB_CHECK_CODE(code, lino, _end);
485,102,424!
2961

2962
  code = doInitMemDataIter(pReader, &di, pBlockScanInfo, &startKey, pSnap->pIMem, &pBlockScanInfo->iiter, "imem");
485,102,424✔
2963
  TSDB_CHECK_CODE(code, lino, _end);
485,263,314!
2964

2965
  code = loadMemTombData(&pBlockScanInfo->pMemDelData, d, di, pReader->info.verRange.maxVer);
485,263,314✔
2966
  TSDB_CHECK_CODE(code, lino, _end);
485,253,081!
2967

2968
  if (forward) {
485,253,081✔
2969
    code = forwardDataIter(&startKey.key, pBlockScanInfo, pReader);
47,340,668✔
2970
    TSDB_CHECK_CODE(code, lino, _end);
47,316,030!
2971
  }
2972

2973
  pBlockScanInfo->iterInit = true;
485,228,443✔
2974

2975
_end:
1,121,375,068✔
2976
  if (code != TSDB_CODE_SUCCESS) {
1,121,406,781!
2977
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2978
  }
2979
  return code;
1,121,406,781✔
2980
}
2981

2982
static int32_t isValidFileBlockRow(SBlockData* pBlockData, int32_t rowIndex, STableBlockScanInfo* pBlockScanInfo,
2,147,483,647✔
2983
                                   bool asc, STsdbReaderInfo* pInfo, STsdbReader* pReader, bool* valid) {
2984
  int32_t code = TSDB_CODE_SUCCESS;
2,147,483,647✔
2985
  int32_t lino = 0;
2,147,483,647✔
2986

2987
  TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
2988
  TSDB_CHECK_NULL(valid, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
2989

2990
  *valid = false;
2,147,483,647✔
2991
  // it is an multi-table data block
2992
  if (pBlockData->aUid != NULL) {
2,147,483,647!
2993
    uint64_t uid = pBlockData->aUid[rowIndex];
×
2994
    if (uid != pBlockScanInfo->uid) {  // move to next row
×
2995
      *valid = false;
×
2996
      goto _end;
×
2997
    }
2998
  }
2999

3000
  TSDB_CHECK_NULL(pInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
3001
  // check for version and time range
3002
  int64_t ver = pBlockData->aVersion[rowIndex];
2,147,483,647✔
3003
  if (ver > pInfo->verRange.maxVer || ver < pInfo->verRange.minVer) {
2,147,483,647!
3004
    *valid = false;
×
3005
    goto _end;
×
3006
  }
3007

3008
  int64_t ts = pBlockData->aTSKEY[rowIndex];
2,147,483,647✔
3009
  if (ts > pInfo->window.ekey || ts < pInfo->window.skey) {
2,147,483,647✔
3010
    *valid = false;
313,047,494✔
3011
    goto _end;
302,133,708✔
3012
  }
3013

3014
  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
3015
  if ((asc && (ts < pBlockScanInfo->lastProcKey.ts)) || ((!asc) && (ts > pBlockScanInfo->lastProcKey.ts))) {
2,147,483,647!
3016
    *valid = false;
229,976✔
3017
    goto _end;
2,677,545✔
3018
  }
3019

3020
  if (ts == pBlockScanInfo->lastProcKey.ts) {  // todo opt perf
2,147,483,647✔
3021
    SRowKey nextRowKey;                        // lazy eval
345✔
3022
    tColRowGetKey(pBlockData, rowIndex, &nextRowKey);
345!
3023
    if (pkCompEx(&pBlockScanInfo->lastProcKey, &nextRowKey) == 0) {
690!
3024
      *valid = false;
345✔
3025
      goto _end;
345✔
3026
    }
3027
  }
3028

3029
  if (pBlockScanInfo->delSkyline != NULL && TARRAY_SIZE(pBlockScanInfo->delSkyline) > 0) {
2,147,483,647!
3030
    bool dropped = false;
2,147,483,647✔
3031
    code = hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, ts, ver, pInfo->order,
2,147,483,647✔
3032
                          &pInfo->verRange, pReader->suppInfo.numOfPks > 0, &dropped);
2,147,483,647✔
3033
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
3034
    if (dropped) {
2,147,483,647!
3035
      *valid = false;
476,747,852✔
3036
      goto _end;
476,747,852✔
3037
    }
3038
  }
3039

3040
  *valid = true;
2,147,483,647✔
3041

3042
_end:
2,147,483,647✔
3043
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
3044
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3045
  }
3046
  return code;
2,147,483,647✔
3047
}
3048

3049
static void initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) {
444,705,808✔
3050
  int32_t              code = TSDB_CODE_SUCCESS;
444,705,808✔
3051
  int32_t              lino = 0;
444,705,808✔
3052
  int32_t              order = 0;
444,705,808✔
3053
  bool                 asc = false;
444,705,808✔
3054
  int64_t              st = 0;
444,705,808✔
3055
  SSttDataInfoForTable info = (SSttDataInfoForTable){0};
444,705,808✔
3056

3057
  TSDB_CHECK_NULL(pSttBlockReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
444,743,528!
3058
  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
444,743,528!
3059
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
444,743,528!
3060

3061
  order = pReader->info.order;
444,743,528✔
3062
  asc = ASCENDING_TRAVERSE(order);
444,744,477✔
3063

3064
  // the stt block reader has been initialized for this table.
3065
  if (pSttBlockReader->uid == pScanInfo->uid) {
444,744,477✔
3066
    goto _end;
151,043,701✔
3067
  }
3068

3069
  if (pSttBlockReader->uid != 0) {
293,781,045✔
3070
    tMergeTreeClose(&pSttBlockReader->mergeTree);
95,501,823✔
3071
  }
3072

3073
  pSttBlockReader->uid = pScanInfo->uid;
293,737,087✔
3074

3075
  // second or third time init stt block reader
3076
  if (pScanInfo->cleanSttBlocks && (pReader->info.execMode == READER_EXEC_ROWS)) {
293,816,854!
3077
    // only allowed to retrieve clean stt blocks for count once
3078
    if (pScanInfo->sttBlockReturned) {
2,760!
3079
      pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA;
×
3080
      tsdbDebug("uid:%" PRIu64 " set no stt-file data after stt-block retrieved, %s", pScanInfo->uid, pReader->idStr);
×
3081
    }
3082
    goto _end;
2,760✔
3083
  }
3084

3085
  STimeWindow w = pSttBlockReader->window;
293,788,847✔
3086
  if (asc) {
293,686,056✔
3087
    w.skey = pScanInfo->sttKeyInfo.nextProcKey.ts;
282,253,733✔
3088
  } else {
3089
    w.ekey = pScanInfo->sttKeyInfo.nextProcKey.ts;
11,432,323✔
3090
  }
3091

3092
  st = taosGetTimestampUs();
293,796,344✔
3093
  tsdbDebug("init stt block reader, window:%" PRId64 "-%" PRId64 ", uid:%" PRIu64 ", %s", w.skey, w.ekey,
293,796,344✔
3094
            pScanInfo->uid, pReader->idStr);
3095

3096
  SMergeTreeConf conf = {
587,500,981✔
3097
      .uid = pScanInfo->uid,
293,702,795✔
3098
      .suid = pReader->info.suid,
293,709,635✔
3099
      .pTsdb = pReader->pTsdb,
293,733,177✔
3100
      .timewindow = w,
3101
      .verRange = pSttBlockReader->verRange,
3102
      .strictTimeRange = false,
3103
      .pSchema = pReader->info.pSchema,
293,692,856✔
3104
      .pCurrentFileset = pReader->status.pCurrentFileset,
293,681,369✔
3105
      .backward = (pSttBlockReader->order == TSDB_ORDER_DESC),
293,672,456✔
3106
      .pSttFileBlockIterArray = pReader->status.pLDataIterArray,
293,708,639✔
3107
      .pCols = pReader->suppInfo.colId,
293,687,424✔
3108
      .numOfCols = pReader->suppInfo.numOfCols,
293,731,988✔
3109
      .loadTombFn = loadSttTombDataForAll,
3110
      .pCurRowKey = &pScanInfo->sttKeyInfo.nextProcKey,
293,730,057✔
3111
      .pReader = pReader,
3112
      .idstr = pReader->idStr,
293,736,478✔
3113
      .rspRows = (pReader->info.execMode == READER_EXEC_ROWS),
293,653,267✔
3114
  };
3115

3116
  info.pKeyRangeList = taosArrayInit(4, sizeof(SSttKeyRange));
293,754,978✔
3117
  TSDB_CHECK_NULL(info.pKeyRangeList, code, lino, _end, terrno);
293,773,899!
3118

3119
  code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info);
293,773,899✔
3120
  TSDB_CHECK_CODE(code, lino, _end);
293,713,491!
3121

3122
  code = initMemDataIterator(pScanInfo, pReader);
293,713,491✔
3123
  TSDB_CHECK_CODE(code, lino, _end);
293,771,247!
3124

3125
  code = initDelSkylineIterator(pScanInfo, pReader->info.order, &pReader->cost);
293,771,247✔
3126
  TSDB_CHECK_CODE(code, lino, _end);
293,674,306!
3127

3128
  if (conf.rspRows) {
293,674,306!
3129
    pScanInfo->cleanSttBlocks = isCleanSttBlock(info.pKeyRangeList, &pReader->info.window, pScanInfo, order);
9,898,946✔
3130
    if (pScanInfo->cleanSttBlocks) {
9,898,789!
3131
      pScanInfo->numOfRowsInStt = info.numOfRows;
1,466,555✔
3132

3133
      // calculate the time window for data in stt files
3134
      for (int32_t i = 0; i < taosArrayGetSize(info.pKeyRangeList); ++i) {
2,938,641✔
3135
        SSttKeyRange* pKeyRange = taosArrayGet(info.pKeyRangeList, i);
1,471,829✔
3136
        if (pKeyRange == NULL) {
1,471,829!
3137
          continue;
×
3138
        }
3139

3140
        if (pkCompEx(&pScanInfo->sttRange.skey, &pKeyRange->skey) > 0) {
2,943,658!
3141
          tRowKeyAssign(&pScanInfo->sttRange.skey, &pKeyRange->skey);
1,466,812✔
3142
        }
3143

3144
        if (pkCompEx(&pScanInfo->sttRange.ekey, &pKeyRange->ekey) < 0) {
2,943,144!
3145
          tRowKeyAssign(&pScanInfo->sttRange.ekey, &pKeyRange->ekey);
1,471,829✔
3146
        }
3147
      }
3148

3149
      pScanInfo->sttKeyInfo.status = taosArrayGetSize(info.pKeyRangeList) ? STT_FILE_HAS_DATA : STT_FILE_NO_DATA;
1,466,812!
3150

3151
      SRowKey* p = asc ? &pScanInfo->sttRange.skey : &pScanInfo->sttRange.ekey;
1,466,812!
3152
      tRowKeyAssign(&pScanInfo->sttKeyInfo.nextProcKey, p);
1,466,812✔
3153
    } else {                                // not clean stt blocks
3154
      INIT_KEYRANGE(&pScanInfo->sttRange);  // reset the time window
8,430,899✔
3155
      code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange);
8,432,441✔
3156
      if (code != TSDB_CODE_SUCCESS) {
8,430,513!
3157
        pScanInfo->sttBlockReturned = false;
×
3158
        TSDB_CHECK_CODE(code, lino, _end);
×
3159
      }
3160
    }
3161
  } else {
3162
    pScanInfo->cleanSttBlocks = false;
283,775,360✔
3163
    INIT_KEYRANGE(&pScanInfo->sttRange);  // reset the time window
283,851,571✔
3164
    code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange);
283,874,324✔
3165
    if (code != TSDB_CODE_SUCCESS) {
283,733,757!
3166
      pScanInfo->sttBlockReturned = false;
×
3167
      TSDB_CHECK_CODE(code, lino, _end);
×
3168
    }
3169
  }
3170

3171
  pScanInfo->sttBlockReturned = false;
293,631,082✔
3172
  int64_t el = taosGetTimestampUs() - st;
293,770,610✔
3173
  pReader->cost.initSttBlockReader += (el / 1000.0);
293,770,610✔
3174

3175
  tsdbDebug("init stt block reader completed, elapsed time:%" PRId64 "us %s", el, pReader->idStr);
293,758,556✔
3176

3177
_end:
444,801,795✔
3178
  if (code != TSDB_CODE_SUCCESS) {
444,821,706!
3179
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3180
    if (pReader) {
×
3181
      pReader->code = code;
×
3182
    }
3183
  }
3184
  taosArrayDestroy(info.pKeyRangeList);
444,821,706✔
3185
}
444,772,468✔
3186

3187
static bool hasDataInSttBlock(STableBlockScanInfo* pInfo) { return pInfo->sttKeyInfo.status == STT_FILE_HAS_DATA; }
2,147,483,647✔
3188

3189
bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo) {
2,147,483,647✔
3190
  if ((pBlockData->nRow > 0) && (pBlockData->nRow != pDumpInfo->totalRows)) {
2,147,483,647!
3191
    return false;  // this is an invalid result.
×
3192
  }
3193
  return pBlockData->nRow > 0 && (!pDumpInfo->allDumped);
2,147,483,647!
3194
}
3195

3196
int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, SRowKey* pKey,
2,147,483,647✔
3197
                              STsdbReader* pReader) {
3198
  int32_t             code = TSDB_CODE_SUCCESS;
2,147,483,647✔
3199
  int32_t             lino = 0;
2,147,483,647✔
3200
  SRow*               pTSRow = NULL;
2,147,483,647✔
3201
  SRowMerger*         pMerger = NULL;
2,147,483,647✔
3202
  SFileBlockDumpInfo* pDumpInfo = NULL;
2,147,483,647✔
3203
  bool                copied = false;
2,147,483,647✔
3204

3205
  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
3206
  TSDB_CHECK_NULL(pKey, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
3207
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
3208

3209
  pMerger = &pReader->status.merger;
2,147,483,647✔
3210
  pDumpInfo = &pReader->status.fBlockDumpInfo;
2,147,483,647✔
3211

3212
  code = tryCopyDistinctRowFromFileBlock(pReader, pBlockData, pKey, pDumpInfo, &copied);
2,147,483,647✔
3213
  TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
3214

3215
  // merge is not initialized yet, due to the fact that the pReader->info.pSchema is not initialized
3216
  code = initRowMergeIfNeeded(pReader, pBlockScanInfo->uid);
2,147,483,647✔
3217
  TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
3218

3219
  tRowKeyAssign(&pBlockScanInfo->lastProcKey, pKey);
2,147,483,647✔
3220

3221
  if (copied) {
2,147,483,647!
3222
    goto _end;
2,147,483,647✔
3223
  }
3224

3225
  TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex);
1,407,118✔
3226

3227
  code = tsdbRowMergerAdd(pMerger, &fRow, NULL);
1,407,118✔
3228
  TSDB_CHECK_CODE(code, lino, _end);
1,406,504!
3229

3230
  code = doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pKey, pReader);
1,406,504✔
3231
  TSDB_CHECK_CODE(code, lino, _end);
1,406,504!
3232

3233
  code = tsdbRowMergerGetRow(pMerger, &pTSRow);
1,406,504✔
3234
  TSDB_CHECK_CODE(code, lino, _end);
1,406,811!
3235

3236
  code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo);
1,406,811✔
3237

3238
  taosMemoryFreeClear(pTSRow);
1,406,197!
3239
  tsdbRowMergerClear(pMerger);
1,406,504✔
3240
  TSDB_CHECK_CODE(code, lino, _end);
1,406,811!
3241

3242
_end:
2,147,483,647✔
3243
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
3244
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3245
  }
3246
  return code;
2,147,483,647✔
3247
}
3248

3249
int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) {
2,147,483,647✔
3250
  int32_t     code = TSDB_CODE_SUCCESS;
2,147,483,647✔
3251
  int32_t     lino = 0;
2,147,483,647✔
3252
  SRow*       pTSRow = NULL;
2,147,483,647✔
3253
  int32_t     pkSrcSlot = 0;
2,147,483,647✔
3254
  SRowMerger* pMerger = NULL;
2,147,483,647✔
3255
  bool        copied = false;
2,147,483,647✔
3256

3257
  TSDB_CHECK_NULL(pSttBlockReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
3258
  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
3259
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
3260

3261
  pkSrcSlot = pReader->suppInfo.pkSrcSlot;
2,147,483,647✔
3262
  pMerger = &pReader->status.merger;
2,147,483,647✔
3263

3264
  // let's record the last processed key
3265
  tRowKeyAssign(&pScanInfo->lastProcKey, getCurrentKeyInSttBlock(pSttBlockReader));
2,147,483,647✔
3266

3267
  TSDBROW* pRow = tMergeTreeGetRow(&pSttBlockReader->mergeTree);
2,147,483,647✔
3268
  TSDBROW  fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData};
2,147,483,647✔
3269

3270
  if (IS_VAR_DATA_TYPE(pScanInfo->lastProcKey.pks[0].type)) {
2,147,483,647!
3271
    tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", ts:%" PRId64 " pk:%s %s", pRow->pBlockData, pRow->iRow,
2,632,784!
3272
              pSttBlockReader->uid, fRow.pBlockData->aTSKEY[fRow.iRow], pScanInfo->lastProcKey.pks[0].pData,
3273
              pReader->idStr);
3274
  }
3275

3276
  code = tryCopyDistinctRowFromSttBlock(&fRow, pSttBlockReader, pScanInfo, &pScanInfo->lastProcKey, pReader, &copied);
2,147,483,647✔
3277
  TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
3278

3279
  if (copied) {
2,147,483,647✔
3280
    goto _end;
2,147,483,647✔
3281
  }
3282

3283
  code = tsdbRowMergerAdd(pMerger, &fRow, NULL);
70,126,237✔
3284
  TSDB_CHECK_CODE(code, lino, _end);
70,150,651!
3285

3286
  TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree);
70,150,651✔
3287
  code = tsdbRowMergerAdd(pMerger, pRow1, NULL);
70,150,137✔
3288
  TSDB_CHECK_CODE(code, lino, _end);
70,162,218!
3289

3290
  code = doMergeRowsInSttBlock(pSttBlockReader, pScanInfo, pMerger, pkSrcSlot, &pReader->info.verRange, pReader->idStr);
70,162,218✔
3291
  TSDB_CHECK_CODE(code, lino, _end);
70,137,802!
3292

3293
  code = tsdbRowMergerGetRow(pMerger, &pTSRow);
70,137,802✔
3294
  TSDB_CHECK_CODE(code, lino, _end);
70,142,171!
3295

3296
  code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pScanInfo);
70,142,171✔
3297

3298
  taosMemoryFreeClear(pTSRow);
70,132,148!
3299
  tsdbRowMergerClear(pMerger);
70,120,076✔
3300
  TSDB_CHECK_CODE(code, lino, _end);
70,162,733!
3301

3302
_end:
2,147,483,647✔
3303
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
3304
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3305
  }
3306
  return code;
2,147,483,647✔
3307
}
3308

3309
static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo,
2,147,483,647✔
3310
                                          SBlockData* pBlockData, SSttBlockReader* pSttBlockReader) {
3311
  int32_t             code = TSDB_CODE_SUCCESS;
2,147,483,647✔
3312
  int32_t             lino = 0;
2,147,483,647✔
3313
  SFileBlockDumpInfo* pDumpInfo = NULL;
2,147,483,647✔
3314
  TSDBROW*            pRow = NULL;
2,147,483,647✔
3315
  TSDBROW*            piRow = NULL;
2,147,483,647✔
3316
  SRowKey*            pKey = &(SRowKey){0};
2,147,483,647✔
3317

3318
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
3319
  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
3320
  TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
3321

3322
  pDumpInfo = &pReader->status.fBlockDumpInfo;
2,147,483,647✔
3323

3324
  if (hasDataInFileBlock(pBlockData, pDumpInfo)) {
2,147,483,647✔
3325
    tColRowGetKey(pBlockData, pDumpInfo->rowIndex, pKey);
2,147,483,647✔
3326
    code = primaryKeyDup(pKey);
2,147,483,647✔
3327
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
3328
    lino = __LINE__;
2,147,483,647✔
3329
  } else {
3330
    pKey = NULL;
2,147,483,647✔
3331
  }
3332

3333
  if (pBlockScanInfo->iter.hasVal) {
2,147,483,647✔
3334
    code = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader, &pRow);
2,147,483,647✔
3335
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
3336
  }
3337

3338
  if (pBlockScanInfo->iiter.hasVal) {
2,147,483,647!
3339
    code = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader, &piRow);
2,147,483,647✔
3340
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
3341
  }
3342

3343
  if (pRow != NULL && piRow != NULL) {
2,147,483,647✔
3344
    // imem + mem + file + stt block
3345
    // two levels of mem-table does contain the valid rows
3346
    code = doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pSttBlockReader);
785,307,462✔
3347
    TSDB_CHECK_CODE(code, lino, _end);
785,307,462!
3348
  } else if (pBlockScanInfo->iiter.hasVal) {
2,147,483,647!
3349
    // imem + file + stt block
3350
    code = doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, pSttBlockReader);
2,147,483,647✔
3351
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
3352
  } else if (pBlockScanInfo->iter.hasVal) {
2,147,483,647✔
3353
    // mem + file + stt block
3354
    code = doMergeBufAndFileRows(pReader, pBlockScanInfo, pRow, &pBlockScanInfo->iter, pSttBlockReader);
2,147,483,647✔
3355
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
3356
  } else {
3357
    // files data blocks + stt block
3358
    code = mergeFileBlockAndSttBlock(pReader, pSttBlockReader, pKey, pBlockScanInfo, pBlockData);
2,147,483,647✔
3359
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
3360
  }
3361

3362
_end:
2,147,483,647✔
3363
  clearRowKey(pKey);
2,147,483,647✔
3364
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
3365
    tsdbError("%s %s failed at line %d since %s", pReader->idStr, __func__, lino, tstrerror(code));
×
3366
  }
3367
  return code;
2,147,483,647✔
3368
}
3369

3370
static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pBlockScanInfo,
3,133,086✔
3371
                                     STsdbReader* pReader, bool* loadNeighbor) {
3372
  int32_t             code = TSDB_CODE_SUCCESS;
3,133,086✔
3373
  int32_t             lino = 0;
3,133,086✔
3374
  int32_t             order = 0;
3,133,086✔
3375
  SDataBlockIter*     pIter = NULL;
3,133,086✔
3376
  SBlockLoadSuppInfo* pSupInfo = NULL;
3,133,086✔
3377
  int32_t             step = 0;
3,133,086✔
3378
  int32_t             nextIndex = -1;
3,133,086✔
3379
  SBrinRecord         rec = {0};
3,133,393✔
3380
  bool                hasNeighbor = false;
3,133,393✔
3381

3382
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
3,133,086!
3383
  TSDB_CHECK_NULL(loadNeighbor, code, lino, _end, TSDB_CODE_INVALID_PARA);
3,133,086!
3384

3385
  order = pReader->info.order;
3,133,086✔
3386
  pIter = &pReader->status.blockIter;
3,133,086✔
3387
  pSupInfo = &pReader->suppInfo;
3,133,393✔
3388
  step = ASCENDING_TRAVERSE(order) ? 1 : -1;
3,133,086✔
3389

3390
  *loadNeighbor = false;
3,133,086✔
3391
  code = getNeighborBlockOfTable(pIter, pBlockInfo, pBlockScanInfo, &nextIndex, order, &rec, pSupInfo, &hasNeighbor);
3,133,086✔
3392
  TSDB_CHECK_CODE(code, lino, _end);
3,133,700!
3393
  if (!hasNeighbor) {  // do nothing
3,133,700!
3394
    goto _end;
1,827,079✔
3395
  }
3396

3397
  // load next block
3398
  if (overlapWithNeighborBlock2(pBlockInfo, &rec, order, pReader->suppInfo.pk.type, pReader->suppInfo.numOfPks)) {
1,306,621✔
3399
    SReaderStatus*  pStatus = &pReader->status;
1,310✔
3400
    SDataBlockIter* pBlockIter = &pStatus->blockIter;
1,310✔
3401

3402
    // 1. find the next neighbor block in the scan block list
3403
    STableDataBlockIdx* tableDataBlockIdx = taosArrayGet(pBlockScanInfo->pBlockIdxList, nextIndex);
1,310✔
3404
    TSDB_CHECK_NULL(tableDataBlockIdx, code, lino, _end, TSDB_CODE_INVALID_PARA);
1,310!
3405

3406
    // 2. remove it from the scan block list
3407
    int32_t neighborIndex = tableDataBlockIdx->globalIndex;
1,310✔
3408
    code = setFileBlockActiveInBlockIter(pReader, pBlockIter, neighborIndex, step);
1,310✔
3409
    TSDB_CHECK_CODE(code, lino, _end);
1,310!
3410

3411
    // 3. load the neighbor block, and set it to be the currently accessed file data block
3412
    code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pBlockInfo->uid);
1,310✔
3413
    TSDB_CHECK_CODE(code, lino, _end);
1,310!
3414

3415
    // 4. check the data values
3416
    code = initBlockDumpInfo(pReader, pBlockIter);
1,310✔
3417
    TSDB_CHECK_CODE(code, lino, _end);
1,310!
3418
    *loadNeighbor = true;
1,310✔
3419
  }
3420

3421
_end:
3,133,700✔
3422
  if (code != TSDB_CODE_SUCCESS) {
3,133,393!
3423
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3424
  }
3425
  return code;
3,133,393✔
3426
}
3427

3428
void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInfo* pBlockScanInfo) {
535,411,819✔
3429
  SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock;
535,411,819✔
3430

3431
  pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0;
535,667,762!
3432
  pResBlock->info.dataLoad = 1;
535,735,285✔
3433
  pResBlock->info.version = pReader->info.verRange.maxVer;
535,764,801✔
3434

3435
  int32_t code = blockDataUpdateTsWindow(pResBlock, pReader->suppInfo.slotId[0]);
535,603,705✔
3436
  code = blockDataUpdatePkRange(pResBlock, pReader->suppInfo.pkDstSlot, ASCENDING_TRAVERSE(pReader->info.order));
535,699,926✔
3437
  setComposedBlockFlag(pReader, true);
535,469,098✔
3438

3439
  pReader->cost.composedBlocks += 1;
535,608,831✔
3440
  pReader->cost.buildComposedBlockTime += el;
535,690,125✔
3441
}
535,757,219✔
3442

3443
static int32_t buildComposedDataBlock(STsdbReader* pReader) {
3,425,673✔
3444
  int32_t              code = TSDB_CODE_SUCCESS;
3,425,673✔
3445
  int32_t              lino = 0;
3,425,673✔
3446
  bool                 asc = false;
3,425,673✔
3447
  int64_t              st = 0;
3,425,673✔
3448
  double               el = 0;
3,425,673✔
3449
  int32_t              step = 0;
3,425,673✔
3450
  SSDataBlock*         pResBlock = NULL;
3,425,673✔
3451
  SFileDataBlockInfo*  pBlockInfo = NULL;
3,425,673✔
3452
  SSttBlockReader*     pSttBlockReader = NULL;
3,425,287✔
3453
  SFileBlockDumpInfo*  pDumpInfo = NULL;
3,425,287✔
3454
  STableBlockScanInfo* pBlockScanInfo = NULL;
3,425,287✔
3455
  TSDBKEY              keyInBuf;
3,425,673✔
3456

3457
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
3,425,287!
3458

3459
  asc = ASCENDING_TRAVERSE(pReader->info.order);
3,425,287✔
3460
  st = taosGetTimestampUs();
3,425,287✔
3461
  step = asc ? 1 : -1;
3,425,287✔
3462
  pResBlock = pReader->resBlockInfo.pResBlock;
3,425,287✔
3463
  pSttBlockReader = pReader->status.fileIter.pSttBlockReader;
3,425,594✔
3464
  pDumpInfo = &pReader->status.fBlockDumpInfo;
3,425,980✔
3465

3466
  code = getCurrentBlockInfo(&pReader->status.blockIter, &pBlockInfo, pReader->idStr);
3,425,673✔
3467
  if (code != TSDB_CODE_SUCCESS) {
3,425,594!
3468
    return 0;
×
3469
  }
3470

3471
  if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) {
3,425,594!
3472
    setBlockAllDumped(pDumpInfo, pBlockInfo->lastKey, pReader->info.order);
×
3473
    return code;
×
3474
  }
3475

3476
  code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr);
3,425,594✔
3477
  TSDB_CHECK_CODE(code, lino, _end);
3,425,980!
3478

3479
  code = getCurrentKeyInBuf(pBlockScanInfo, pReader, &keyInBuf);
3,425,980✔
3480
  TSDB_CHECK_CODE(code, lino, _end);
3,425,980!
3481

3482
  // it is a clean block, load it directly
3483
  int64_t cap = pReader->resBlockInfo.capacity;
3,425,980✔
3484
  bool    isClean = false;
3,425,673✔
3485
  code = isCleanFileDataBlock(pReader, pBlockInfo, pBlockScanInfo, keyInBuf, &isClean);
3,425,287✔
3486
  TSDB_CHECK_CODE(code, lino, _end);
3,425,980!
3487
  bool directCopy = isClean && (pBlockInfo->numRow <= cap) && (pBlockScanInfo->sttKeyInfo.status == STT_FILE_NO_DATA) &&
3,612,994!
3488
                    ((asc && ((pBlockInfo->lastKey < keyInBuf.ts) || (keyInBuf.ts == INT64_MIN))) ||
83,280!
3489
                     (!asc && (pBlockInfo->lastKey > keyInBuf.ts)));
104,120!
3490
  if (directCopy) {
3,425,594✔
3491
    code = copyBlockDataToSDataBlock(pReader, &pBlockScanInfo->lastProcKey);
187,400✔
3492
    TSDB_CHECK_CODE(code, lino, _end);
187,400!
3493
    goto _end;
187,400✔
3494
  }
3495

3496
  SBlockData* pBlockData = &pReader->status.fileBlockData;
3,238,194✔
3497
  initSttBlockReader(pSttBlockReader, pBlockScanInfo, pReader);
3,238,580✔
3498
  code = pReader->code;
3,238,580✔
3499
  TSDB_CHECK_CODE(code, lino, _end);
340,984!
3500

3501
  while (1) {
2,147,483,647✔
3502
    bool hasBlockData = false;
2,147,483,647✔
3503
    {
3504
      while (pBlockData->nRow > 0 && pBlockData->uid == pBlockScanInfo->uid) {
2,147,483,647!
3505
        // find the first qualified row in data block
3506
        bool valid = false;
2,147,483,647✔
3507
        code =
3508
            isValidFileBlockRow(pBlockData, pDumpInfo->rowIndex, pBlockScanInfo, asc, &pReader->info, pReader, &valid);
2,147,483,647✔
3509
        TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
3510
        if (valid) {
2,147,483,647!
3511
          hasBlockData = true;
2,147,483,647✔
3512
          break;
2,147,483,647✔
3513
        }
3514

3515
        pDumpInfo->rowIndex += step;
781,544,493✔
3516

3517
        if (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0) {
781,551,405✔
3518
          // NOTE: get the new block info
3519
          code = getCurrentBlockInfo(&pReader->status.blockIter, &pBlockInfo, pReader->idStr);
293,193✔
3520
          TSDB_CHECK_CODE(code, lino, _end);
286,319!
3521

3522
          // continue check for the next file block if the last ts in the current block
3523
          // is overlapped with the next neighbor block
3524
          bool loadNeighbor = false;
286,319✔
3525
          code = loadNeighborIfOverlap(pBlockInfo, pBlockScanInfo, pReader, &loadNeighbor);
286,319✔
3526
          if ((!loadNeighbor) || (code != 0)) {
286,319!
3527
            lino = __LINE__;
286,319✔
3528
            setBlockAllDumped(pDumpInfo, pBlockInfo->lastKey, pReader->info.order);
286,319✔
3529
            break;
286,319✔
3530
          }
3531
        }
3532
      }
3533
    }
3534

3535
    // no data in last block and block, no need to proceed.
3536
    if (hasBlockData == false) {
2,147,483,647✔
3537
      break;
286,319✔
3538
    }
3539

3540
    code = buildComposedDataBlockImpl(pReader, pBlockScanInfo, pBlockData, pSttBlockReader);
2,147,483,647✔
3541
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
3542

3543
    // currently loaded file data block is consumed
3544
    if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) {
2,147,483,647!
3545
      setBlockAllDumped(pDumpInfo, pBlockInfo->lastKey, pReader->info.order);
2,627,324✔
3546
      break;
2,845,764✔
3547
    }
3548

3549
    if (pResBlock->info.rows >= pReader->resBlockInfo.capacity) {
2,147,483,647✔
3550
      break;
106,190✔
3551
    }
3552
  }
3553

3554
_end:
199,465✔
3555
  if (pReader) {
3,425,673!
3556
    el = (taosGetTimestampUs() - st) / 1000.0;
3,425,366✔
3557
    updateComposedBlockInfo(pReader, el, pBlockScanInfo);
3,425,366✔
3558
  }
3559

3560
  if (pResBlock && pResBlock->info.rows > 0) {
3,425,980!
3561
    tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64
3,213,060✔
3562
              ", elapsed time:%.2f ms %s",
3563
              pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
3564
              pResBlock->info.rows, el, pReader->idStr);
3565
  }
3566

3567
  if (code != TSDB_CODE_SUCCESS) {
3,425,059!
3568
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3569
  }
3570
  return code;
3,425,059✔
3571
}
3572

3573
void setComposedBlockFlag(STsdbReader* pReader, bool composed) { pReader->status.composedDataBlock = composed; }
573,804,009✔
3574

3575
int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order) {
12,547,838✔
3576
  if (pDelSkyline == NULL) {
12,547,838!
3577
    return 0;
×
3578
  }
3579

3580
  return ASCENDING_TRAVERSE(order) ? 0 : taosArrayGetSize(pDelSkyline) - 1;
12,547,838✔
3581
}
3582

3583
int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SReadCostSummary* pCost) {
1,121,100,563✔
3584
  int32_t code = TSDB_CODE_SUCCESS;
1,121,100,563✔
3585
  int32_t lino = 0;
1,121,100,563✔
3586
  int32_t newDelDataInFile = 0;
1,121,100,563✔
3587
  int64_t st = 0;
1,121,100,563✔
3588
  SArray* pSource = NULL;
1,121,100,563✔
3589

3590
  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
1,121,100,563!
3591

3592
  newDelDataInFile = taosArrayGetSize(pBlockScanInfo->pFileDelData);
1,121,100,563✔
3593
  if (newDelDataInFile == 0 &&
1,121,279,527✔
3594
      ((pBlockScanInfo->delSkyline != NULL) || (TARRAY_SIZE(pBlockScanInfo->pMemDelData) == 0))) {
1,112,150,001✔
3595
    goto _end;
1,108,981,033✔
3596
  }
3597

3598
  st = taosGetTimestampUs();
12,548,145✔
3599

3600
  if (pBlockScanInfo->delSkyline != NULL) {
12,548,145✔
3601
    taosArrayClear(pBlockScanInfo->delSkyline);
2,493,816✔
3602
  } else {
3603
    pBlockScanInfo->delSkyline = taosArrayInit(4, sizeof(TSDBKEY));
10,053,478✔
3604
    TSDB_CHECK_NULL(pBlockScanInfo->delSkyline, code, lino, _end, terrno);
10,053,715!
3605
  }
3606

3607
  pSource = pBlockScanInfo->pFileDelData;
12,547,838✔
3608
  if (pSource == NULL) {
12,547,838✔
3609
    pSource = pBlockScanInfo->pMemDelData;
3,056,514✔
3610
  } else {
3611
    const void* p1 = taosArrayAddAll(pSource, pBlockScanInfo->pMemDelData);
9,491,324✔
3612
    TSDB_CHECK_NULL(p1, code, lino, _end, terrno);
9,491,631!
3613
  }
3614

3615
  code = tsdbBuildDeleteSkyline(pSource, 0, taosArrayGetSize(pSource) - 1, pBlockScanInfo->delSkyline);
12,547,595✔
3616

3617
  taosArrayClear(pBlockScanInfo->pFileDelData);
12,548,145✔
3618
  int32_t index = getInitialDelIndex(pBlockScanInfo->delSkyline, order);
12,548,145✔
3619

3620
  pBlockScanInfo->iter.index = index;
12,547,838✔
3621
  pBlockScanInfo->iiter.index = index;
12,547,838✔
3622
  pBlockScanInfo->fileDelIndex = index;
12,547,294✔
3623
  pBlockScanInfo->sttBlockDelIndex = index;
12,547,838✔
3624
  TSDB_CHECK_CODE(code, lino, _end);
12,546,437!
3625

3626
  TSDB_CHECK_NULL(pCost, code, lino, _end, TSDB_CODE_INVALID_PARA);
12,546,437!
3627
  double el = taosGetTimestampUs() - st;
12,547,838✔
3628
  pCost->createSkylineIterTime = el / 1000.0;
12,547,838✔
3629

3630
_end:
1,121,528,257✔
3631
  if (code != TSDB_CODE_SUCCESS) {
1,121,528,257!
3632
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3633
  }
3634
  return code;
1,121,196,378✔
3635
}
3636

3637
int32_t getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader, TSDBKEY* pkey) {
43,770,839✔
3638
  int32_t  code = TSDB_CODE_SUCCESS;
43,770,839✔
3639
  int32_t  lino = 0;
43,770,839✔
3640
  bool     asc = false;
43,770,839✔
3641
  bool     hasKey = false;
43,770,839✔
3642
  bool     hasIKey = false;
43,770,839✔
3643
  TSDBROW* pRow = NULL;
43,770,839✔
3644
  TSDBROW* pIRow = NULL;
43,772,014✔
3645
  TSDBKEY  key = {.ts = TSKEY_INITIAL_VAL};
43,771,709✔
3646
  TSDBKEY  ikey = {.ts = TSKEY_INITIAL_VAL};
43,771,709✔
3647

3648
  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
43,771,709!
3649
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
43,771,709!
3650
  TSDB_CHECK_NULL(pkey, code, lino, _end, TSDB_CODE_INVALID_PARA);
43,771,709!
3651

3652
  asc = ASCENDING_TRAVERSE(pReader->info.order);
43,771,709✔
3653

3654
  code = getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader, &pRow);
43,771,356✔
3655
  TSDB_CHECK_CODE(code, lino, _end);
43,769,464!
3656
  if (pRow != NULL) {
43,769,464✔
3657
    hasKey = true;
5,038,268✔
3658
    key = TSDBROW_KEY(pRow);
5,038,268!
3659
  }
3660

3661
  code = getValidMemRow(&pScanInfo->iiter, pScanInfo->delSkyline, pReader, &pIRow);
43,769,464✔
3662
  TSDB_CHECK_CODE(code, lino, _end);
43,767,984!
3663
  if (pIRow != NULL) {
43,767,984✔
3664
    hasIKey = true;
1,377,229✔
3665
    ikey = TSDBROW_KEY(pIRow);
1,377,229!
3666
  }
3667

3668
  if (hasKey) {
43,767,984✔
3669
    if (hasIKey) {  // has data in mem & imem
5,038,268✔
3670
      if (asc) {
320,188!
3671
        *pkey = key.ts <= ikey.ts ? key : ikey;
320,188!
3672
      } else {
3673
        *pkey = key.ts <= ikey.ts ? ikey : key;
×
3674
      }
3675
    } else {  // no data in imem
3676
      *pkey = key;
4,718,080✔
3677
    }
3678
  } else {
3679
    // no data in mem & imem, return the initial value
3680
    // only imem has data, return ikey
3681
    *pkey = ikey;
38,729,716✔
3682
  }
3683

3684
_end:
43,769,338✔
3685
  if (code != TSDB_CODE_SUCCESS) {
43,771,100!
3686
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3687
  }
3688
  return code;
43,771,100✔
3689
}
3690

3691
static void prepareDurationForNextFileSet(STsdbReader* pReader) {
5,974,128✔
3692
  if (pReader->status.bProcMemFirstFileset) {
5,974,128!
3693
    pReader->status.prevFilesetStartKey = INT64_MIN;
3,947,834✔
3694
    pReader->status.prevFilesetEndKey = INT64_MAX;
3,947,834✔
3695
    pReader->status.bProcMemFirstFileset = false;
3,943,174✔
3696
  }
3697

3698
  int32_t     fid = pReader->status.pCurrentFileset->fid;
5,954,951✔
3699
  STimeWindow winFid = {0};
5,974,870✔
3700
  tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &winFid.skey, &winFid.ekey);
5,976,068✔
3701

3702
  if (ASCENDING_TRAVERSE(pReader->info.order)) {
5,971,985✔
3703
    pReader->status.bProcMemPreFileset = !(pReader->status.memTableMaxKey < pReader->status.prevFilesetStartKey ||
9,382,237✔
3704
                                           (winFid.skey - 1) < pReader->status.memTableMinKey);
3,917,998✔
3705
  } else {
3706
    pReader->status.bProcMemPreFileset = !(pReader->status.memTableMaxKey < (winFid.ekey + 1) ||
526,922✔
3707
                                           pReader->status.prevFilesetEndKey < pReader->status.memTableMinKey);
24,490!
3708
  }
3709

3710
  if (pReader->status.bProcMemPreFileset) {
5,963,330!
3711
    tsdbDebug("will start pre-fileset %d buffer processing. %s", fid, pReader->idStr);
258,518✔
3712
    pReader->status.procMemUidList.tableUidList = pReader->status.uidList.tableUidList;
258,518✔
3713
    resetPreFilesetMemTableListIndex(&pReader->status);
258,518✔
3714
  }
3715

3716
  if (!pReader->status.bProcMemPreFileset) {
5,973,551!
3717
    if (pReader->notifyFn) {
5,715,033✔
3718
      STsdReaderNotifyInfo info = {0};
5,573,288✔
3719
      info.duration.filesetId = fid;
5,574,442✔
3720
      pReader->notifyFn(TSD_READER_NOTIFY_DURATION_START, &info, pReader->notifyParam);
5,574,442✔
3721
      tsdbDebug("new duration %d start notification when no buffer preceeding fileset, %s", fid, pReader->idStr);
5,579,102✔
3722
    }
3723
  }
3724

3725
  pReader->status.prevFilesetStartKey = winFid.skey;
5,978,211✔
3726
  pReader->status.prevFilesetEndKey = winFid.ekey;
5,977,222✔
3727
}
5,976,645✔
3728

3729
static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SArray* pTableList) {
232,939,498✔
3730
  int32_t        code = TSDB_CODE_SUCCESS;
232,939,498✔
3731
  int32_t        lino = 0;
232,939,498✔
3732
  SReaderStatus* pStatus = NULL;
232,939,498✔
3733
  SArray*        pIndexList = NULL;
232,939,498✔
3734
  size_t         numOfTables = 0;
232,939,498✔
3735

3736
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
232,939,498!
3737
  TSDB_CHECK_NULL(pBlockNum, code, lino, _end, TSDB_CODE_INVALID_PARA);
232,939,498!
3738

3739
  pStatus = &pReader->status;
232,939,498✔
3740
  pBlockNum->numOfBlocks = 0;
232,934,241✔
3741
  pBlockNum->numOfSttFiles = 0;
232,958,199✔
3742

3743
  numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);
232,967,750✔
3744
  pIndexList = taosArrayInit(numOfTables, sizeof(SBrinBlk));
232,964,211✔
3745
  TSDB_CHECK_NULL(pIndexList, code, lino, _end, terrno);
232,937,598!
3746

3747
  while (1) {
141,933✔
3748
    // only check here, since the iterate data in memory is very fast.
3749
    if (pReader->code != TSDB_CODE_SUCCESS) {
233,079,531!
3750
      tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr);
×
3751
      code = pReader->code;
×
3752
      TSDB_CHECK_CODE(code, lino, _end);
×
3753
    }
3754

3755
    bool hasNext = false;
233,101,283✔
3756
    code = filesetIteratorNext(&pStatus->fileIter, pReader, &hasNext);
233,108,724✔
3757
    TSDB_CHECK_CODE(code, lino, _end);
233,055,573!
3758

3759
    if (!hasNext) {  // no data files on disk
233,055,573✔
3760
      break;
34,345,826✔
3761
    }
3762

3763
    taosArrayClear(pIndexList);
198,709,747✔
3764
    code = doLoadBlockIndex(pReader, pReader->pFileReader, pIndexList);
198,716,430✔
3765
    TSDB_CHECK_CODE(code, lino, _end);
198,678,774!
3766

3767
    if (taosArrayGetSize(pIndexList) > 0 || pReader->status.pCurrentFileset->lvlArr->size > 0) {
198,678,774✔
3768
      code = loadFileBlockBrinInfo(pReader, pIndexList, pBlockNum, pTableList);
198,690,929✔
3769
      TSDB_CHECK_CODE(code, lino, _end);
198,719,635!
3770

3771
      if (pBlockNum->numOfBlocks + pBlockNum->numOfSttFiles > 0) {
198,719,635✔
3772
        if (pReader->bFilesetDelimited) {
198,596,843!
3773
          prepareDurationForNextFileSet(pReader);
5,972,560✔
3774
        }
3775
        break;
198,617,478✔
3776
      }
3777
    }
3778

3779
    // no blocks in current file, try next files
3780
  }
3781

3782
  code = loadDataFileTombDataForAll(pReader);
232,969,592✔
3783
  TSDB_CHECK_CODE(code, lino, _end);
232,888,924!
3784

3785
_end:
232,888,924✔
3786
  if (code != TSDB_CODE_SUCCESS) {
232,887,272!
3787
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3788
  }
3789
  if (pIndexList) {
232,883,117!
3790
    taosArrayDestroy(pIndexList);
232,883,117✔
3791
  }
3792
  return code;
232,946,529✔
3793
}
3794

3795
// pTableIter can be NULL, no need to handle the return value
3796
static int32_t resetTableListIndex(SReaderStatus* pStatus, const char* id) {
589,960,451✔
3797
  int32_t        code = TSDB_CODE_SUCCESS;
589,960,451✔
3798
  int32_t        lino = 0;
589,960,451✔
3799
  STableUidList* pList = NULL;
589,960,451✔
3800

3801
  TSDB_CHECK_NULL(pStatus, code, lino, _end, TSDB_CODE_INVALID_PARA);
589,960,451!
3802

3803
  pList = &pStatus->uidList;
589,960,451✔
3804

3805
  pList->currentIndex = 0;
590,005,073✔
3806
  uint64_t uid = pList->tableUidList[0];
590,018,186✔
3807
  pStatus->pTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid));
590,025,197✔
3808
  if (pStatus->pTableIter == NULL) {
590,018,077!
3809
    tsdbError("%s failed to load tableBlockScanInfo for uid:%" PRId64 ", code: internal error", id, uid);
×
3810
    TSDB_CHECK_NULL(pStatus->pTableIter, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
×
3811
  }
3812

3813
_end:
590,002,820✔
3814
  if (code != TSDB_CODE_SUCCESS) {
589,938,267!
3815
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3816
  }
3817
  return code;
589,938,267✔
3818
}
3819

3820
static void resetPreFilesetMemTableListIndex(SReaderStatus* pStatus) {
258,518✔
3821
  STableUidList* pList = &pStatus->procMemUidList;
258,518✔
3822

3823
  pList->currentIndex = 0;
258,518✔
3824
  uint64_t uid = pList->tableUidList[0];
258,518✔
3825
  pStatus->pProcMemTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid));
258,518✔
3826
}
258,518✔
3827

3828
static bool moveToNextTable(STableUidList* pOrderedCheckInfo, SReaderStatus* pStatus) {
732,162,180✔
3829
  pOrderedCheckInfo->currentIndex += 1;
732,162,180✔
3830
  if (pOrderedCheckInfo->currentIndex >= tSimpleHashGetSize(pStatus->pTableMap)) {
732,481,636✔
3831
    pStatus->pTableIter = NULL;
348,537,098✔
3832
    return false;
348,547,536✔
3833
  }
3834

3835
  uint64_t uid = pOrderedCheckInfo->tableUidList[pOrderedCheckInfo->currentIndex];
384,041,191✔
3836
  pStatus->pTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid));
384,006,627✔
3837
  return (pStatus->pTableIter != NULL);
384,080,243✔
3838
}
3839

3840
static bool moveToNextTableForPreFileSetMem(SReaderStatus* pStatus) {
378,792✔
3841
  STableUidList* pUidList = &pStatus->procMemUidList;
378,792✔
3842
  pUidList->currentIndex += 1;
378,792✔
3843
  if (pUidList->currentIndex >= tSimpleHashGetSize(pStatus->pTableMap)) {
378,792✔
3844
    pStatus->pProcMemTableIter = NULL;
258,518✔
3845
    return false;
258,518✔
3846
  }
3847

3848
  uint64_t uid = pUidList->tableUidList[pUidList->currentIndex];
120,274✔
3849
  pStatus->pProcMemTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid));
120,274✔
3850
  return (pStatus->pProcMemTableIter != NULL);
120,274✔
3851
}
3852

3853
static int32_t buildCleanBlockFromSttFiles(STsdbReader* pReader, STableBlockScanInfo* pScanInfo) {
1,466,812✔
3854
  int32_t          code = TSDB_CODE_SUCCESS;
1,466,812✔
3855
  int32_t          lino = 0;
1,466,812✔
3856
  SReaderStatus*   pStatus = NULL;
1,466,812✔
3857
  SSttBlockReader* pSttBlockReader = NULL;
1,466,812✔
3858
  SSDataBlock*     pResBlock = NULL;
1,466,812✔
3859
  SDataBlockInfo*  pInfo = NULL;
1,466,812✔
3860
  bool             asc = false;
1,466,812✔
3861

3862
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
1,466,812!
3863
  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
1,466,812!
3864

3865
  pStatus = &pReader->status;
1,466,812✔
3866
  pSttBlockReader = pStatus->fileIter.pSttBlockReader;
1,466,812✔
3867
  pResBlock = pReader->resBlockInfo.pResBlock;
1,466,555✔
3868

3869
  asc = ASCENDING_TRAVERSE(pReader->info.order);
1,466,555✔
3870
  code = blockDataEnsureCapacity(pResBlock, pScanInfo->numOfRowsInStt);
1,466,555✔
3871
  TSDB_CHECK_CODE(code, lino, _end);
1,466,555!
3872

3873
  pInfo = &pResBlock->info;
1,466,555✔
3874
  pInfo->rows = pScanInfo->numOfRowsInStt;
1,466,555✔
3875
  pInfo->id.uid = pScanInfo->uid;
1,466,812✔
3876
  pInfo->dataLoad = 1;
1,466,812✔
3877
  pInfo->window.skey = pScanInfo->sttRange.skey.ts;
1,466,812✔
3878
  pInfo->window.ekey = pScanInfo->sttRange.ekey.ts;
1,466,812✔
3879

3880
  setComposedBlockFlag(pReader, true);
1,466,812✔
3881

3882
  pScanInfo->sttKeyInfo.nextProcKey.ts = asc ? pScanInfo->sttRange.ekey.ts + 1 : pScanInfo->sttRange.skey.ts - 1;
1,466,812!
3883
  pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA;
1,466,812✔
3884

3885
  if (asc) {
1,466,812!
3886
    tRowKeyAssign(&pScanInfo->lastProcKey, &pScanInfo->sttRange.ekey);
1,466,812✔
3887
  } else {
3888
    tRowKeyAssign(&pScanInfo->lastProcKey, &pScanInfo->sttRange.skey);
×
3889
  }
3890

3891
  pScanInfo->sttBlockReturned = true;
1,466,812✔
3892
  pSttBlockReader->mergeTree.pIter = NULL;
1,466,812✔
3893

3894
  tsdbDebug("%p uid:%" PRId64 " return clean stt block as one, brange:%" PRId64 "-%" PRId64 " rows:%" PRId64 " %s",
1,466,298✔
3895
            pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
3896
            pResBlock->info.rows, pReader->idStr);
3897

3898
_end:
1,465,987✔
3899
  if (code != TSDB_CODE_SUCCESS) {
1,466,812!
3900
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3901
  }
3902
  return code;
1,466,812✔
3903
}
3904

3905
static int32_t buildCleanBlockFromDataFiles(STsdbReader* pReader, STableBlockScanInfo* pScanInfo,
36,984,186✔
3906
                                            SFileDataBlockInfo* pBlockInfo, int32_t blockIndex) {
3907
  int32_t         code = TSDB_CODE_SUCCESS;
36,984,186✔
3908
  int32_t         lino = 0;
36,984,186✔
3909
  SReaderStatus*  pStatus = NULL;
36,984,186✔
3910
  SDataBlockInfo* pInfo = NULL;
36,984,186✔
3911
  bool            asc = false;
36,984,186✔
3912

3913
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
36,984,186!
3914
  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
36,984,186!
3915
  TSDB_CHECK_NULL(pBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
36,984,186!
3916

3917
  // whole data block is required, return it directly
3918
  pStatus = &pReader->status;
36,984,186✔
3919
  pInfo = &pReader->resBlockInfo.pResBlock->info;
36,985,638✔
3920
  asc = ASCENDING_TRAVERSE(pReader->info.order);
36,986,059✔
3921

3922
  pInfo->rows = pBlockInfo->numRow;
36,987,995✔
3923
  pInfo->id.uid = pScanInfo->uid;
36,984,607✔
3924
  pInfo->dataLoad = 0;
36,988,286✔
3925
  pInfo->version = pReader->info.verRange.maxVer;
36,987,802✔
3926
  pInfo->window = (STimeWindow){.skey = pBlockInfo->firstKey, .ekey = pBlockInfo->lastKey};
36,986,122✔
3927

3928
  if (pReader->suppInfo.numOfPks > 0) {
36,987,739✔
3929
    if (IS_NUMERIC_TYPE(pReader->suppInfo.pk.type)) {
826!
3930
      VALUE_SET_TRIVIAL_DATUM(pInfo->pks, pBlockInfo->firstPk.val);
×
3931
      VALUE_SET_TRIVIAL_DATUM(pInfo->pks + 1, pBlockInfo->lastPk.val);
×
3932
    } else {
3933
      (void)memcpy(pInfo->pks[0].pData, varDataVal(pBlockInfo->firstPk.pData), varDataLen(pBlockInfo->firstPk.pData));
826!
3934
      (void)memcpy(pInfo->pks[1].pData, varDataVal(pBlockInfo->lastPk.pData), varDataLen(pBlockInfo->lastPk.pData));
826!
3935

3936
      pInfo->pks[0].nData = varDataLen(pBlockInfo->firstPk.pData);
826✔
3937
      pInfo->pks[1].nData = varDataLen(pBlockInfo->lastPk.pData);
826✔
3938
    }
3939
  }
3940

3941
  // update the last key for the corresponding table
3942
  setComposedBlockFlag(pReader, false);
36,988,058✔
3943
  setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order);
36,985,624✔
3944
  code = updateLastKeyInfo(&pScanInfo->lastProcKey, pBlockInfo, pInfo, pReader->suppInfo.numOfPks, asc);
36,986,848✔
3945
  TSDB_CHECK_CODE(code, lino, _end);
36,984,302!
3946

3947
  tsdbDebug("%p uid:%" PRIu64
36,984,302✔
3948
            " clean file block retrieved from file, global index:%d, "
3949
            "table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s",
3950
            pReader, pScanInfo->uid, blockIndex, pBlockInfo->tbBlockIdx, pBlockInfo->numRow, pBlockInfo->firstKey,
3951
            pBlockInfo->lastKey, pReader->idStr);
3952

3953
_end:
36,902,830✔
3954
  if (code != TSDB_CODE_SUCCESS) {
36,983,155!
3955
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3956
  }
3957
  return code;
36,984,123✔
3958
}
3959

3960
static int32_t doLoadSttBlockSequentially(STsdbReader* pReader) {
342,630,606✔
3961
  int32_t              code = TSDB_CODE_SUCCESS;
342,630,606✔
3962
  int32_t              lino = 0;
342,630,606✔
3963
  SReaderStatus*       pStatus = NULL;
342,630,606✔
3964
  SSttBlockReader*     pSttBlockReader = NULL;
342,630,606✔
3965
  STableUidList*       pUidList = NULL;
342,630,606✔
3966
  SSDataBlock*         pResBlock = NULL;
342,630,606✔
3967
  STableBlockScanInfo* pScanInfo = NULL;
342,630,606✔
3968

3969
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
342,630,606!
3970

3971
  pStatus = &pReader->status;
342,630,606✔
3972
  pSttBlockReader = pStatus->fileIter.pSttBlockReader;
342,629,855✔
3973
  pUidList = &pStatus->uidList;
342,614,378✔
3974

3975
  if (tSimpleHashGetSize(pStatus->pTableMap) == 0) {
342,623,348!
3976
    goto _end;
×
3977
  }
3978

3979
  pResBlock = pReader->resBlockInfo.pResBlock;
342,718,993✔
3980

3981
  while (1) {
78,217,857✔
3982
    if (pReader->code != TSDB_CODE_SUCCESS) {
420,942,566!
3983
      tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr);
×
3984
      code = pReader->code;
×
3985
      TSDB_CHECK_CODE(code, lino, _end);
×
3986
    }
3987

3988
    // load the last data block of current table
3989
    if (pStatus->pTableIter == NULL) {
420,950,429!
3990
      tsdbError("table Iter is null, invalid pScanInfo, try next table %s", pReader->idStr);
×
3991
      bool hasNexTable = moveToNextTable(pUidList, pStatus);
×
3992
      if (!hasNexTable) {
×
3993
        break;
×
3994
      }
3995

3996
      continue;
×
3997
    } else {
3998
      pScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter;
420,823,336✔
3999
    }
4000

4001
    if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pScanInfo->uid, sizeof(pScanInfo->uid))) {
420,945,349✔
4002
      // reset the index in last block when handing a new file
4003
      bool hasNexTable = moveToNextTable(pUidList, pStatus);
1,194,994✔
4004
      if (!hasNexTable) {
1,194,994✔
4005
        break;
148,551✔
4006
      }
4007

4008
      continue;
1,046,443✔
4009
    }
4010

4011
    initSttBlockReader(pSttBlockReader, pScanInfo, pReader);
419,696,603✔
4012
    code = pReader->code;
419,681,916✔
4013
    TSDB_CHECK_CODE(code, lino, _end);
419,746,645!
4014

4015
    if (!hasDataInSttBlock(pScanInfo)) {
419,746,645✔
4016
      bool hasNexTable = moveToNextTable(pUidList, pStatus);
274,295,159✔
4017
      if (!hasNexTable) {
274,231,252✔
4018
        break;
197,059,838✔
4019
      }
4020

4021
      continue;
77,171,414✔
4022
    }
4023

4024
    // if only require the total rows, no need to load data from stt file if it is clean stt blocks
4025
    if (pReader->info.execMode == READER_EXEC_ROWS && pScanInfo->cleanSttBlocks) {
145,454,561!
4026
      code = buildCleanBlockFromSttFiles(pReader, pScanInfo);
1,466,418✔
4027
      TSDB_CHECK_CODE(code, lino, _end);
1,466,418!
4028
      break;
1,466,418✔
4029
    }
4030

4031
    int64_t st = taosGetTimestampUs();
144,003,647✔
4032
    while (1) {
4033
      // no data in stt block and block, no need to proceed.
4034
      if (!hasDataInSttBlock(pScanInfo)) {
2,147,483,647✔
4035
        break;
136,263,418✔
4036
      }
4037

4038
      code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pSttBlockReader);
2,147,483,647✔
4039
      TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
4040

4041
      if (pResBlock->info.rows >= pReader->resBlockInfo.capacity) {
2,147,483,647✔
4042
        break;
7,734,232✔
4043
      }
4044
    }
4045

4046
    double el = (taosGetTimestampUs() - st) / 1000.0;
144,000,931✔
4047
    updateComposedBlockInfo(pReader, el, pScanInfo);
144,000,931✔
4048

4049
    if (pResBlock->info.rows > 0) {
143,968,859!
4050
      tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64
143,985,454✔
4051
                ", elapsed time:%.2f ms %s",
4052
                pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
4053
                pResBlock->info.rows, el, pReader->idStr);
4054
      break;
143,953,389✔
4055
    }
4056

4057
    // current table is exhausted, let's try next table
4058
    bool hasNexTable = moveToNextTable(pUidList, pStatus);
×
4059
    if (!hasNexTable) {
×
4060
      break;
×
4061
    }
4062
  }
4063

4064
_end:
342,628,196✔
4065
  if (code != TSDB_CODE_SUCCESS) {
342,628,196!
4066
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
4067
  }
4068
  return code;
342,589,474✔
4069
}
4070

4071
// current active data block not overlap with the stt-files/stt-blocks
4072
static bool notOverlapWithFiles(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo, bool asc) {
37,017,470✔
4073
  if ((!hasDataInSttBlock(pScanInfo)) || (pScanInfo->cleanSttBlocks == true)) {
37,017,470!
4074
    return true;
36,826,302✔
4075
  } else {
4076
    int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey.ts;
180,164✔
4077
    return (asc && pBlockInfo->lastKey < keyInStt) || (!asc && pBlockInfo->firstKey > keyInStt);
180,164✔
4078
  }
4079
}
4080

4081
static int32_t doBuildDataBlock(STsdbReader* pReader) {
40,673,642✔
4082
  int32_t              code = TSDB_CODE_SUCCESS;
40,673,642✔
4083
  int32_t              lino = 0;
40,673,642✔
4084
  SReaderStatus*       pStatus = NULL;
40,673,642✔
4085
  SDataBlockIter*      pBlockIter = NULL;
40,673,642✔
4086
  STableBlockScanInfo* pScanInfo = NULL;
40,673,642✔
4087
  SFileDataBlockInfo*  pBlockInfo = NULL;
40,673,321✔
4088
  SSttBlockReader*     pSttBlockReader = NULL;
40,674,577✔
4089
  TSDBKEY              keyInBuf;
40,674,577✔
4090
  bool                 asc = false;
40,673,887✔
4091

4092
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
40,673,887!
4093

4094
  pStatus = &pReader->status;
40,673,887✔
4095
  pBlockIter = &pStatus->blockIter;
40,674,061✔
4096
  pSttBlockReader = pReader->status.fileIter.pSttBlockReader;
40,674,887✔
4097
  asc = ASCENDING_TRAVERSE(pReader->info.order);
40,674,314✔
4098

4099
  code = getCurrentBlockInfo(pBlockIter, &pBlockInfo, pReader->idStr);
40,673,577✔
4100
  TSDB_CHECK_CODE(code, lino, _end);
40,674,602!
4101

4102
  if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) {
40,674,602✔
4103
    setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->lastKey, pReader->info.order);
328,839✔
4104
    goto _end;
328,839✔
4105
  }
4106

4107
  code = pReader->code;
40,346,108✔
4108
  TSDB_CHECK_CODE(code, lino, _end);
40,346,608!
4109

4110
  code = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, &pScanInfo, pReader->idStr);
40,346,608✔
4111
  TSDB_CHECK_CODE(code, lino, _end);
40,344,515!
4112

4113
  if (pScanInfo->sttKeyInfo.status == STT_FILE_READER_UNINIT) {
40,344,515✔
4114
    initSttBlockReader(pSttBlockReader, pScanInfo, pReader);
21,764,742✔
4115
    code = pReader->code;
21,764,698✔
4116
    TSDB_CHECK_CODE(code, lino, _end);
21,764,698!
4117
  }
4118

4119
  code = getCurrentKeyInBuf(pScanInfo, pReader, &keyInBuf);
40,344,748✔
4120
  TSDB_CHECK_CODE(code, lino, _end);
40,344,438!
4121
  bool load = false;
40,344,438✔
4122
  code = fileBlockShouldLoad(pReader, pBlockInfo, pScanInfo, keyInBuf, &load);
40,339,997✔
4123
  TSDB_CHECK_CODE(code, lino, _end);
40,337,364!
4124
  if (load) {
40,337,364!
4125
    code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid);
3,320,067✔
4126
    TSDB_CHECK_CODE(code, lino, _end);
3,321,067!
4127

4128
    // build composed data block
4129
    code = buildComposedDataBlock(pReader);
3,321,067✔
4130
    TSDB_CHECK_CODE(code, lino, _end);
3,320,146!
4131
  } else if (bufferDataInFileBlockGap(keyInBuf, pBlockInfo, pScanInfo, pReader->info.order)) {
37,017,297✔
4132
    // data in memory that are earlier than current file block and stt blocks
4133
    // rows in buffer should be less than the file block in asc, greater than file block in desc
4134
    int64_t endKey = getBoarderKeyInFiles(pBlockInfo, pScanInfo, pReader->info.order);
2,415✔
4135
    code = buildDataBlockFromBuf(pReader, pScanInfo, endKey);
2,415✔
4136
    TSDB_CHECK_CODE(code, lino, _end);
2,415!
4137
  } else {
4138
    if (notOverlapWithFiles(pBlockInfo, pScanInfo, asc)) {
37,017,330✔
4139
      int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey.ts;
36,985,727✔
4140

4141
      if ((!hasDataInSttBlock(pScanInfo)) || (asc && pBlockInfo->lastKey < keyInStt) ||
36,987,421✔
4142
          (!asc && pBlockInfo->firstKey > keyInStt)) {
19,369!
4143
        // the stt blocks may located in the gap of different data block, but the whole sttRange may overlap with the
4144
        // data block, so the overlap check is invalid actually.
4145
        code = buildCleanBlockFromDataFiles(pReader, pScanInfo, pBlockInfo, pBlockIter->index);
36,987,434✔
4146
        TSDB_CHECK_CODE(code, lino, _end);
36,983,397!
4147
      } else {  // clean stt block
4148
        TSDB_CHECK_CONDITION((pReader->info.execMode == READER_EXEC_ROWS) && (pSttBlockReader->mergeTree.pIter == NULL),
394!
4149
                             code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
4150
        code = buildCleanBlockFromSttFiles(pReader, pScanInfo);
394✔
4151
        TSDB_CHECK_CODE(code, lino, _end);
394!
4152
      }
4153
    } else {
4154
      SBlockData* pBData = &pReader->status.fileBlockData;
31,680✔
4155
      tBlockDataReset(pBData);
32,890✔
4156

4157
      SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock;
32,890✔
4158

4159
      tsdbDebug("load data in stt block firstly %s", pReader->idStr);
32,890!
4160
      int64_t st = taosGetTimestampUs();
32,890✔
4161

4162
      // let's load data from stt files, make sure clear the cleanStt block flag before load the data from stt files
4163
      initSttBlockReader(pSttBlockReader, pScanInfo, pReader);
32,890✔
4164
      code = pReader->code;
32,890✔
4165
      TSDB_CHECK_CODE(code, lino, _end);
32,890!
4166

4167
      // no data in stt block, no need to proceed.
4168
      while (hasDataInSttBlock(pScanInfo)) {
27,860,360✔
4169
        TSDB_CHECK_CONDITION(pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA, code, lino, _end,
27,838,510!
4170
                             TSDB_CODE_INTERNAL_ERROR);
4171

4172
        code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pSttBlockReader);
27,795,385✔
4173
        TSDB_CHECK_CODE(code, lino, _end);
27,829,885!
4174

4175
        if (pResBlock->info.rows >= pReader->resBlockInfo.capacity) {
27,829,885✔
4176
          break;
2,760✔
4177
        }
4178

4179
        // data in stt now overlaps with current active file data block, need to composed with file data block.
4180
        SRowKey* pSttKey = getCurrentKeyInSttBlock(pSttBlockReader);
27,829,540✔
4181
        if ((pSttKey->ts >= pBlockInfo->firstKey && asc) || (pSttKey->ts <= pBlockInfo->lastKey && (!asc))) {
27,829,540✔
4182
          tsdbDebug("%p lastKeyInStt:%" PRId64 ", overlap with file block, brange:%" PRId64 "-%" PRId64 " %s", pReader,
4,830!
4183
                    pSttKey->ts, pBlockInfo->firstKey, pBlockInfo->lastKey, pReader->idStr);
4184
          break;
4,830✔
4185
        }
4186
      }
4187

4188
      double el = (taosGetTimestampUs() - st) / 1000.0;
32,890✔
4189
      updateComposedBlockInfo(pReader, el, pScanInfo);
32,890✔
4190

4191
      if (pResBlock->info.rows > 0) {
32,890!
4192
        tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%" PRId64
32,890!
4193
                  ", elapsed time:%.2f ms %s",
4194
                  pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
4195
                  pResBlock->info.rows, el, pReader->idStr);
4196
      }
4197
    }
4198
  }
4199

4200
  code = pReader->code;
40,339,242✔
4201
  TSDB_CHECK_CODE(code, lino, _end);
40,343,309!
4202

4203
_end:
40,672,148✔
4204
  if (code != TSDB_CODE_SUCCESS) {
40,672,549!
4205
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
4206
  }
4207
  return code;
40,672,549✔
4208
}
4209

4210
static int32_t buildBlockFromBufferSeqForPreFileset(STsdbReader* pReader, int64_t endKey) {
524,698✔
4211
  int32_t        code = TSDB_CODE_SUCCESS;
524,698✔
4212
  int32_t        lino = 0;
524,698✔
4213
  SReaderStatus* pStatus = NULL;
524,698✔
4214

4215
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
524,698!
4216

4217
  pStatus = &pReader->status;
524,698✔
4218

4219
  tsdbDebug("seq load data blocks from cache that preceeds fileset %d, %s", pReader->status.pCurrentFileset->fid,
524,698✔
4220
            pReader->idStr);
4221

4222
  while (1) {
120,274✔
4223
    if (pReader->code != TSDB_CODE_SUCCESS) {
644,972!
4224
      tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr);
×
4225
      code = pReader->code;
×
4226
      TSDB_CHECK_CODE(code, lino, _end);
×
4227
    }
4228

4229
    STableBlockScanInfo** pBlockScanInfo = pStatus->pProcMemTableIter;
644,972✔
4230
    if (pReader->pIgnoreTables &&
1,289,944!
4231
        taosHashGet(*pReader->pIgnoreTables, &(*pBlockScanInfo)->uid, sizeof((*pBlockScanInfo)->uid))) {
644,972✔
4232
      bool hasNexTable = moveToNextTableForPreFileSetMem(pStatus);
46,320✔
4233
      if (!hasNexTable) {
46,320✔
4234
        break;
18,528✔
4235
      }
4236
      continue;
27,792✔
4237
    }
4238

4239
    code = initMemDataIterator(*pBlockScanInfo, pReader);
598,652✔
4240
    TSDB_CHECK_CODE(code, lino, _end);
598,652!
4241

4242
    code = initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost);
598,652✔
4243
    TSDB_CHECK_CODE(code, lino, _end);
598,652!
4244

4245
    code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey);
598,652✔
4246
    TSDB_CHECK_CODE(code, lino, _end);
598,652!
4247

4248
    if (pReader->resBlockInfo.pResBlock->info.rows > 0) {
598,652✔
4249
      break;
266,180✔
4250
    }
4251

4252
    // current table is exhausted, let's try next table
4253
    bool hasNexTable = moveToNextTableForPreFileSetMem(pStatus);
332,472✔
4254
    if (!hasNexTable) {
332,472✔
4255
      break;
239,990✔
4256
    }
4257
  }
4258

4259
_end:
524,698✔
4260
  if (code != TSDB_CODE_SUCCESS) {
524,698!
4261
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
4262
  }
4263
  return code;
524,698✔
4264
}
4265

4266
static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader, int64_t endKey) {
534,119,545✔
4267
  int32_t        code = TSDB_CODE_SUCCESS;
534,119,545✔
4268
  int32_t        lino = 0;
534,119,545✔
4269
  SReaderStatus* pStatus = NULL;
534,119,545✔
4270
  STableUidList* pUidList = NULL;
534,119,545✔
4271

4272
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
534,119,545!
4273

4274
  pStatus = &pReader->status;
534,119,545✔
4275
  pUidList = &pStatus->uidList;
534,354,779✔
4276

4277
  tsdbDebug("seq load data blocks from cache, %s", pReader->idStr);
534,359,424✔
4278

4279
  while (1) {
305,785,689✔
4280
    if (pReader->code != TSDB_CODE_SUCCESS) {
840,066,264!
4281
      tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", tstrerror(pReader->code), pReader->idStr);
×
4282
      code = pReader->code;
×
4283
      TSDB_CHECK_CODE(code, lino, _end);
×
4284
    }
4285

4286
    STableBlockScanInfo** pBlockScanInfo = pStatus->pTableIter;
840,049,767✔
4287
    if (pBlockScanInfo == NULL || *pBlockScanInfo == NULL) {
840,121,786!
4288
      break;
4289
    }
4290

4291
    if (pReader->pIgnoreTables &&
1,661,084,989✔
4292
        taosHashGet(*pReader->pIgnoreTables, &(*pBlockScanInfo)->uid, sizeof((*pBlockScanInfo)->uid))) {
821,182,780✔
4293
      bool hasNexTable = moveToNextTable(pUidList, pStatus);
13,009,269✔
4294
      if (!hasNexTable) {
13,009,269✔
4295
        break;
305,717✔
4296
      }
4297
      continue;
12,703,552✔
4298
    }
4299

4300
    code = initMemDataIterator(*pBlockScanInfo, pReader);
826,900,429✔
4301
    TSDB_CHECK_CODE(code, lino, _end);
827,034,822!
4302

4303
    code = initDelSkylineIterator(*pBlockScanInfo, pReader->info.order, &pReader->cost);
827,034,822✔
4304
    TSDB_CHECK_CODE(code, lino, _end);
826,853,918!
4305

4306
    code = buildDataBlockFromBuf(pReader, *pBlockScanInfo, endKey);
826,853,918✔
4307
    TSDB_CHECK_CODE(code, lino, _end);
826,713,444!
4308

4309
    if (pReader->resBlockInfo.pResBlock->info.rows > 0) {
826,713,444✔
4310
      break;
383,040,836✔
4311
    }
4312

4313
    // current table is exhausted, let's try next table
4314
    bool hasNexTable = moveToNextTable(pUidList, pStatus);
443,992,882✔
4315
    if (!hasNexTable) {
444,071,472✔
4316
      break;
150,989,335✔
4317
    }
4318
  }
4319

4320
_end:
761✔
4321
  if (code != TSDB_CODE_SUCCESS) {
534,324,112!
4322
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
4323
  }
4324
  return code;
533,977,510✔
4325
}
4326

4327
// set the correct start position in case of the first/last file block, according to the query time window
4328
static int32_t initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) {
230,827,330✔
4329
  int32_t             code = TSDB_CODE_SUCCESS;
230,827,330✔
4330
  int32_t             lino = 0;
230,827,330✔
4331
  SFileDataBlockInfo* pBlockInfo = NULL;
230,827,330✔
4332
  SReaderStatus*      pStatus = NULL;
230,862,315✔
4333
  SFileBlockDumpInfo* pDumpInfo = NULL;
230,862,315✔
4334

4335
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
230,862,315!
4336

4337
  pStatus = &pReader->status;
230,862,315✔
4338
  pDumpInfo = &pStatus->fBlockDumpInfo;
230,865,289✔
4339

4340
  code = getCurrentBlockInfo(pBlockIter, &pBlockInfo, pReader->idStr);
230,857,735✔
4341
  if (code == TSDB_CODE_SUCCESS) {
230,778,894✔
4342
    pDumpInfo->totalRows = pBlockInfo->numRow;
40,656,372✔
4343
    pDumpInfo->rowIndex = ASCENDING_TRAVERSE(pReader->info.order) ? 0 : pBlockInfo->numRow - 1;
40,656,870✔
4344
  } else {
4345
    pDumpInfo->totalRows = 0;
190,122,522✔
4346
    pDumpInfo->rowIndex = 0;
190,164,677✔
4347
    code = TSDB_CODE_SUCCESS;
190,234,744✔
4348
  }
4349

4350
  pDumpInfo->allDumped = false;
230,891,856✔
4351

4352
_end:
230,908,156✔
4353
  if (code != TSDB_CODE_SUCCESS) {
230,911,655!
4354
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
4355
  }
4356
  return code;
230,911,655✔
4357
}
4358

4359
static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBlockIter) {
232,903,502✔
4360
  int32_t      code = TSDB_CODE_SUCCESS;
232,903,502✔
4361
  int32_t      lino = 0;
232,903,502✔
4362
  SBlockNumber num = {0};
232,903,502✔
4363
  SArray*      pTableList = NULL;
232,936,921✔
4364

4365
  pTableList = taosArrayInit(40, POINTER_BYTES);
232,936,921✔
4366
  TSDB_CHECK_NULL(pTableList, code, lino, _end, terrno);
232,950,554!
4367

4368
  code = moveToNextFile(pReader, &num, pTableList);
232,950,554✔
4369
  TSDB_CHECK_CODE(code, lino, _end);
232,917,140!
4370

4371
  // all data files are consumed, try data in buffer
4372
  if (num.numOfBlocks + num.numOfSttFiles == 0) {
232,917,140✔
4373
    pReader->status.loadFromFile = false;
34,345,803✔
4374
    goto _end;
34,347,467✔
4375
  }
4376

4377
  // initialize the block iterator for a new fileset
4378
  if (num.numOfBlocks > 0) {
198,571,337✔
4379
    code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks, pTableList);
8,366,603✔
4380
    TSDB_CHECK_CODE(code, lino, _end);
8,366,989!
4381
  } else {  // no block data, only last block exists
4382
    tBlockDataReset(&pReader->status.fileBlockData);
190,204,734✔
4383
    code = resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo), pReader->idStr);
190,180,741✔
4384
    TSDB_CHECK_CODE(code, lino, _end);
190,237,034!
4385

4386
    code = resetTableListIndex(&pReader->status, pReader->idStr);
190,237,034✔
4387
    TSDB_CHECK_CODE(code, lino, _end);
190,166,365!
4388
  }
4389

4390
  code = initBlockDumpInfo(pReader, pBlockIter);
198,533,354✔
4391
  TSDB_CHECK_CODE(code, lino, _end);
198,524,303!
4392

4393
_end:
232,868,732✔
4394
  if (code != TSDB_CODE_SUCCESS) {
232,905,590!
4395
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
4396
  }
4397
  if (pTableList) {
232,907,519!
4398
    taosArrayDestroy(pTableList);
232,907,519✔
4399
  }
4400
  return code;
232,900,943✔
4401
}
4402

4403
static bool fileBlockPartiallyRead(SFileBlockDumpInfo* pDumpInfo, bool asc) {
41,489,828✔
4404
  return (!pDumpInfo->allDumped) &&
42,365,390!
4405
         ((pDumpInfo->rowIndex > 0 && asc) || (pDumpInfo->rowIndex < (pDumpInfo->totalRows - 1) && (!asc)));
876,565✔
4406
}
4407

4408
typedef enum {
4409
  TSDB_READ_RETURN = 0x1,
4410
  TSDB_READ_CONTINUE = 0x2,
4411
} ERetrieveType;
4412

4413
static int32_t doReadDataFromSttFiles(STsdbReader* pReader, ERetrieveType* pReturnType) {
187,327,046✔
4414
  int32_t         code = TSDB_CODE_SUCCESS;
187,327,046✔
4415
  int32_t         lino = 0;
187,327,046✔
4416
  SSDataBlock*    pResBlock = NULL;
187,327,046✔
4417
  SDataBlockIter* pBlockIter = NULL;
187,327,046✔
4418

4419
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
187,327,046!
4420
  TSDB_CHECK_NULL(pReturnType, code, lino, _end, TSDB_CODE_INVALID_PARA);
187,327,046!
4421

4422
  pResBlock = pReader->resBlockInfo.pResBlock;
187,327,046✔
4423
  pBlockIter = &pReader->status.blockIter;
187,336,818✔
4424

4425
  *pReturnType = TSDB_READ_RETURN;
187,318,086✔
4426

4427
  tsdbDebug("seq load data blocks from stt files %s", pReader->idStr);
187,324,904✔
4428

4429
  while (1) {
4430
    code = doLoadSttBlockSequentially(pReader);
342,665,602✔
4431
    TSDB_CHECK_CODE(code, lino, _end);
342,590,119!
4432

4433
    if (pResBlock->info.rows > 0) {
342,590,119✔
4434
      goto _end;
145,451,750✔
4435
    }
4436

4437
    // all data blocks are checked in this stt file, now let's try the next file set
4438
    TSDB_CHECK_CONDITION(pReader->status.pTableIter == NULL, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
197,204,296!
4439

4440
    code = initForFirstBlockInFile(pReader, pBlockIter);
197,201,613✔
4441
    TSDB_CHECK_CODE(code, lino, _end);
197,198,383!
4442

4443
    // all the data files are completely checked
4444
    if (pReader->status.loadFromFile == false) {
197,198,383✔
4445
      goto _end;
34,082,473✔
4446
    }
4447

4448
    if (pReader->status.bProcMemPreFileset) {
163,137,542!
4449
      code = buildFromPreFilesetBuffer(pReader);
254,514✔
4450
      TSDB_CHECK_CODE(code, lino, _end);
254,514!
4451
      if (pResBlock->info.rows > 0) {
254,514✔
4452
        pReader->status.processingMemPreFileSet = true;
190,000✔
4453
        goto _end;
190,000✔
4454
      }
4455
    }
4456

4457
    if (pBlockIter->numOfBlocks > 0) {  // there are data blocks existed.
162,919,907✔
4458
      *pReturnType = TSDB_READ_CONTINUE;
7,616,081✔
4459
      goto _end;
7,615,839✔
4460
    } else {  // all blocks in data file are checked, let's check the data in stt-files
4461
      code = resetTableListIndex(&pReader->status, pReader->idStr);
155,283,259✔
4462
      TSDB_CHECK_CODE(code, lino, _end);
155,355,277!
4463
    }
4464
  }
4465

4466
_end:
187,340,062✔
4467
  if (code != TSDB_CODE_SUCCESS) {
187,340,062!
4468
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
4469
  }
4470
  return code;
187,297,247✔
4471
}
4472

4473
static int32_t buildBlockFromFiles(STsdbReader* pReader) {
219,957,343✔
4474
  int32_t             code = TSDB_CODE_SUCCESS;
219,957,343✔
4475
  int32_t             lino = 0;
219,957,343✔
4476
  SDataBlockIter*     pBlockIter = NULL;
219,957,343✔
4477
  SSDataBlock*        pResBlock = NULL;
219,957,343✔
4478
  SFileBlockDumpInfo* pDumpInfo = NULL;
219,957,343✔
4479
  SBlockData*         pBlockData = NULL;
219,957,343✔
4480
  const char*         id = NULL;
219,957,343✔
4481
  bool                asc = false;
219,957,343✔
4482

4483
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
219,957,343!
4484

4485
  asc = ASCENDING_TRAVERSE(pReader->info.order);
219,957,343✔
4486

4487
  pBlockIter = &pReader->status.blockIter;
219,986,306✔
4488
  pResBlock = pReader->resBlockInfo.pResBlock;
219,999,688✔
4489
  pDumpInfo = &pReader->status.fBlockDumpInfo;
220,003,317✔
4490
  pBlockData = &pReader->status.fileBlockData;
219,982,352✔
4491
  id = pReader->idStr;
219,975,932✔
4492

4493
  if (pBlockIter->numOfBlocks == 0) {
219,995,053✔
4494
    // let's try to extract data from stt files.
4495
    ERetrieveType type = 0;
179,037,618✔
4496
    code = doReadDataFromSttFiles(pReader, &type);
179,043,971✔
4497
    TSDB_CHECK_CODE(code, lino, _end);
178,990,865!
4498
    if (type == TSDB_READ_RETURN) {
178,984,103✔
4499
      goto _end;
178,883,544✔
4500
    }
4501

4502
    code = doBuildDataBlock(pReader);
100,559✔
4503
    TSDB_CHECK_CODE(code, lino, _end);
100,559!
4504
    if (pResBlock->info.rows > 0) {
100,559✔
4505
      goto _end;
100,317✔
4506
    }
4507
  }
4508

4509
  while (1) {
4510
    if (fileBlockPartiallyRead(pDumpInfo, asc)) {  // file data block is partially loaded
41,485,790✔
4511
      code = buildComposedDataBlock(pReader);
104,913✔
4512
      TSDB_CHECK_CODE(code, lino, _end);
104,913!
4513
    } else {
4514
      // current block are exhausted, try the next file block
4515
      if (pDumpInfo->allDumped) {
41,386,057!
4516
        // try next data block in current file
4517
        bool hasNext = blockIteratorNext(&pReader->status.blockIter);
40,615,794✔
4518
        if (hasNext) {  // check for the next block in the block accessed order list
40,615,028✔
4519
          code = initBlockDumpInfo(pReader, pBlockIter);
32,287,475✔
4520
          TSDB_CHECK_CODE(code, lino, _end);
32,287,219!
4521
        } else {
4522
          // all data blocks in files are checked, let's check the data in last files.
4523
          // data blocks in current file are exhausted, let's try the next file now
4524
          if (pBlockData->uid != 0) {
8,327,553✔
4525
            tBlockDataClear(pBlockData);
7,882,685✔
4526
          }
4527

4528
          tBlockDataReset(pBlockData);
8,328,521✔
4529
          code = resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo), id);
8,328,037✔
4530
          TSDB_CHECK_CODE(code, lino, _end);
8,328,279!
4531

4532
          code = resetTableListIndex(&pReader->status, id);
8,328,279✔
4533
          TSDB_CHECK_CODE(code, lino, _end);
8,327,553!
4534

4535
          ERetrieveType type = 0;
8,327,553✔
4536
          code = doReadDataFromSttFiles(pReader, &type);
8,328,037✔
4537
          TSDB_CHECK_CODE(code, lino, _end);
8,328,037!
4538
          if (type == TSDB_READ_RETURN) {
8,328,037✔
4539
            break;
813,241✔
4540
          }
4541
        }
4542
      }
4543

4544
      code = doBuildDataBlock(pReader);
40,597,203✔
4545
      TSDB_CHECK_CODE(code, lino, _end);
40,570,845!
4546
    }
4547

4548
    if (pResBlock->info.rows > 0) {
40,675,758✔
4549
      break;
40,137,470✔
4550
    }
4551
  }
4552

4553
_end:
219,928,038✔
4554
  if (code != TSDB_CODE_SUCCESS) {
219,928,038!
4555
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
4556
  }
4557
  return code;
219,923,629✔
4558
}
4559

4560
static void getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idStr,
199,359,602✔
4561
                                int8_t* pLevel, STsdb** pTsdb) {
4562
  if (pTsdb == NULL) {
199,359,602!
4563
    return;
×
4564
  }
4565

4566
  *pTsdb = NULL;
199,359,602✔
4567
#if 0
4568
  if (VND_IS_RSMA(pVnode) && !pCond->skipRollup) {
4569
    int8_t  level = 0;
4570
    int8_t  precision = pVnode->config.tsdbCfg.precision;
4571
    int64_t now = taosGetTimestamp(precision);
4572
    int64_t offset = tsQueryRsmaTolerance * ((precision == TSDB_TIME_PRECISION_MILLI)   ? 1L
4573
                                             : (precision == TSDB_TIME_PRECISION_MICRO) ? 1000L
4574
                                                                                        : 1000000L);
4575

4576
    for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) {
4577
      SRetention* pRetention = retentions + level;
4578
      if (pRetention->keep <= 0) {
4579
        if (level > 0) {
4580
          --level;
4581
        }
4582
        break;
4583
      }
4584
      if ((now - pRetention->keep) <= (pCond->twindows.skey + offset)) {
4585
        break;
4586
      }
4587
      ++level;
4588
    }
4589

4590
    const char* str = (idStr != NULL) ? idStr : "";
4591

4592
    if (level == TSDB_RETENTION_L0) {
4593
      *pLevel = TSDB_RETENTION_L0;
4594
      tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L0, str);
4595
      *pTsdb = VND_RSMA0(pVnode);
4596
      return;
4597
    } else if (level == TSDB_RETENTION_L1) {
4598
      *pLevel = TSDB_RETENTION_L1;
4599
      tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L1, str);
4600
      *pTsdb = VND_RSMA1(pVnode);
4601
      return;
4602
    } else {
4603
      *pLevel = TSDB_RETENTION_L2;
4604
      tsdbDebug("vgId:%d, rsma level %d is selected to query %s", TD_VID(pVnode), TSDB_RETENTION_L2, str);
4605
      *pTsdb = VND_RSMA2(pVnode);
4606
      return;
4607
    }
4608
  }
4609
#endif
4610
  *pTsdb = VND_TSDB(pVnode);
199,381,242✔
4611
}
4612

4613
SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level) {
199,361,229✔
4614
  int64_t startVer = (pCond->startVersion == -1) ? 0 : pCond->startVersion;
199,361,229!
4615

4616
  int64_t endVer = 0;
199,364,521✔
4617
  if (pCond->endVersion == -1) {
199,364,521✔
4618
    // user not specified end version, set current maximum version of vnode as the endVersion
4619
    endVer = pVnode->state.applied;
182,875,316✔
4620
  } else {
4621
    endVer = (pCond->endVersion > pVnode->state.applied) ? pVnode->state.applied : pCond->endVersion;
16,490,990✔
4622
  }
4623

4624
  return (SVersionRange){.minVer = startVer, .maxVer = endVer};
199,342,506✔
4625
}
4626

4627
static int32_t reverseSearchStartPos(const SArray* pDelList, int32_t index, int64_t key, bool asc, int32_t* start) {
3,319,300✔
4628
  int32_t code = TSDB_CODE_SUCCESS;
3,319,300✔
4629
  int32_t lino = 0;
3,319,300✔
4630
  size_t  num = 0;
3,319,300✔
4631

4632
  num = taosArrayGetSize(pDelList);
3,319,300✔
4633
  *start = index;
3,319,300✔
4634

4635
  if (asc) {
3,319,300✔
4636
    if (*start >= num - 1) {
2,002,466✔
4637
      *start = num - 1;
93,058✔
4638
    }
4639

4640
    TSDBKEY* p = taosArrayGet(pDelList, *start);
2,002,466✔
4641
    TSDB_CHECK_NULL(p, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,002,466!
4642

4643
    while (p->ts >= key && *start > 0) {
2,102,562✔
4644
      *start -= 1;
100,096✔
4645
    }
4646
  } else {
4647
    if (index <= 0) {
1,316,834!
4648
      *start = 0;
×
4649
    }
4650

4651
    TSDBKEY* p = taosArrayGet(pDelList, *start);
1,316,834✔
4652
    TSDB_CHECK_NULL(p, code, lino, _end, TSDB_CODE_INVALID_PARA);
1,316,834!
4653

4654
    while (p->ts <= key && *start < num - 1) {
1,373,138✔
4655
      *start += 1;
56,304✔
4656
    }
4657
  }
4658

4659
_end:
1,316,834✔
4660
  if (code != TSDB_CODE_SUCCESS) {
3,319,300!
4661
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
4662
  }
4663
  return code;
3,319,300✔
4664
}
4665

4666
int32_t hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order,
2,147,483,647✔
4667
                       SVersionRange* pVerRange, bool hasPk, bool* dropped) {
4668
  int32_t code = TSDB_CODE_SUCCESS;
2,147,483,647✔
4669
  int32_t lino = 0;
2,147,483,647✔
4670
  size_t  num = 0;
2,147,483,647✔
4671
  int32_t step = 0;
2,147,483,647✔
4672
  bool    asc = false;
2,147,483,647✔
4673

4674
  *dropped = false;
2,147,483,647✔
4675

4676
  if (pDelList == NULL || (TARRAY_SIZE(pDelList) == 0)) {
2,147,483,647!
4677
    goto _end;
×
4678
  }
4679

4680
  num = taosArrayGetSize(pDelList);
2,147,483,647✔
4681
  asc = ASCENDING_TRAVERSE(order);
2,147,483,647✔
4682
  step = asc ? 1 : -1;
2,147,483,647✔
4683

4684
  if (hasPk) {  // handle the case where duplicated timestamps existed.
2,147,483,647✔
4685
    code = reverseSearchStartPos(pDelList, *index, key, asc, index);
3,319,300✔
4686
    TSDB_CHECK_CODE(code, lino, _end);
4,041,728!
4687
  }
4688

4689
  if (asc) {
2,147,483,647✔
4690
    if (*index >= num - 1) {
2,147,483,647✔
4691
      TSDBKEY* last = taosArrayGetLast(pDelList);
1,060,572,924✔
4692
      if (last == NULL) {
1,058,721,895!
4693
        goto _end;
×
4694
      }
4695

4696
      if (key > last->ts) {
1,058,721,895✔
4697
        goto _end;
1,057,900,755✔
4698
      } else if (key == last->ts) {
1,772,467!
4699
        TSDBKEY* prev = taosArrayGet(pDelList, num - 2);
1,772,467✔
4700
        if (prev == NULL) {
1,772,467!
4701
          goto _end;
×
4702
        }
4703

4704
        *dropped = (prev->version >= ver && prev->version <= pVerRange->maxVer && prev->version >= pVerRange->minVer);
1,772,467!
4705
      }
4706
    } else {
4707
      TSDBKEY* pCurrent = taosArrayGet(pDelList, *index);
1,670,885,818✔
4708
      TSDBKEY* pNext = taosArrayGet(pDelList, (*index) + 1);
1,672,828,918✔
4709
      if (pCurrent == NULL || pNext == NULL) {
1,671,519,066!
4710
        goto _end;
×
4711
      }
4712

4713
      if (key < pCurrent->ts) {
1,671,519,066✔
4714
        goto _end;
1,028,763,936✔
4715
      }
4716

4717
      if (pCurrent->ts <= key && pNext->ts >= key && pCurrent->version >= ver &&
643,450,118!
4718
          pVerRange->maxVer >= pCurrent->version) {
557,294,307✔
4719
        *dropped = true;
557,295,437✔
4720
        goto _end;
557,295,537✔
4721
      }
4722

4723
      while (pNext->ts <= key && (*index) < num - 1) {
111,339,723✔
4724
        (*index) += 1;
78,806,746✔
4725

4726
        if ((*index) < num - 1) {
78,806,652✔
4727
          pCurrent = taosArrayGet(pDelList, *index);
74,403,454✔
4728
          pNext = taosArrayGet(pDelList, (*index) + 1);
74,402,366✔
4729
          if (pCurrent == NULL || pNext == NULL) {
74,402,366!
4730
            break;
4731
          }
4732

4733
          // it is not a consecutive deletion range, ignore it
4734
          if (pCurrent->version == 0 && pNext->version > 0) {
74,402,366!
4735
            continue;
764,043✔
4736
          }
4737

4738
          if (pCurrent->ts <= key && pNext->ts >= key && pCurrent->version >= ver &&
73,638,323!
4739
              pVerRange->maxVer >= pCurrent->version) {
53,635,803!
4740
            *dropped = true;
53,636,347✔
4741
            break;
53,635,803✔
4742
          }
4743
        }
4744
      }
4745
    }
4746
  } else {
4747
    if (*index <= 0) {
1,874,141,824!
4748
      TSDBKEY* pFirst = taosArrayGet(pDelList, 0);
×
4749
      if (pFirst == NULL) {
×
4750
        goto _end;
×
4751
      }
4752

4753
      if (key < pFirst->ts) {
×
4754
        goto _end;
×
4755
      } else if (key == pFirst->ts) {
×
4756
        *dropped = pFirst->version >= ver;
×
4757
      } else {
4758
        tsdbError("unexpected error, key:%" PRId64 ", first:%" PRId64, key, pFirst->ts);
×
4759
      }
4760
    } else {
4761
      TSDBKEY* pCurrent = taosArrayGet(pDelList, *index);
1,874,124,325✔
4762
      TSDBKEY* pPrev = taosArrayGet(pDelList, (*index) - 1);
1,873,468,880✔
4763
      if (pCurrent == NULL || pPrev == NULL) {
1,871,318,038!
4764
        goto _end;
×
4765
      }
4766

4767
      if (key > pCurrent->ts) {
1,871,318,038✔
4768
        goto _end;
950,588,074✔
4769
      }
4770

4771
      if (pPrev->ts <= key && pCurrent->ts >= key && pPrev->version >= ver) {
923,514,454!
4772
        *dropped = true;
55,191,484✔
4773
        goto _end;
55,081,578✔
4774
      }
4775

4776
      while (pPrev->ts >= key && (*index) > 1) {
878,632,142✔
4777
        (*index) += step;
69,485,483✔
4778

4779
        if ((*index) >= 1) {
69,485,483!
4780
          pCurrent = taosArrayGet(pDelList, *index);
69,485,483✔
4781
          pPrev = taosArrayGet(pDelList, (*index) - 1);
69,485,483✔
4782
          if (pCurrent == NULL || pPrev == NULL) {
69,485,483!
4783
            break;
4784
          }
4785

4786
          // it is not a consecutive deletion range, ignore it
4787
          if (pCurrent->version > 0 && pPrev->version == 0) {
69,485,483✔
4788
            continue;
335,481✔
4789
          }
4790

4791
          if (pPrev->ts <= key && pCurrent->ts >= key && pPrev->version >= ver) {
69,150,002!
4792
            *dropped = true;
59,221,133✔
4793
            break;
59,221,133✔
4794
          }
4795
        }
4796
      }
4797
    }
4798
  }
4799

4800
_end:
809,328,096✔
4801
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
4802
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
4803
  }
4804
  return code;
2,147,483,647✔
4805
}
4806

4807
FORCE_INLINE int32_t getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader, TSDBROW** pRes) {
4808
  int32_t  code = TSDB_CODE_SUCCESS;
2,147,483,647✔
4809
  int32_t  lino = 0;
2,147,483,647✔
4810
  int32_t  order = 0;
2,147,483,647✔
4811
  TSDBROW* pRow = NULL;
2,147,483,647✔
4812
  TSDBKEY  key;
4813

4814
  TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
4815
  TSDB_CHECK_NULL(pRes, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
4816

4817
  *pRes = NULL;
2,147,483,647✔
4818

4819
  if (!pIter->hasVal) {
2,147,483,647!
4820
    goto _end;
2,147,483,647✔
4821
  }
4822

4823
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
4824
  order = pReader->info.order;
2,147,483,647✔
4825
  pRow = tsdbTbDataIterGet(pIter->iter);
2,147,483,647!
4826

4827
  TSDBROW_INIT_KEY(pRow, key);
2,147,483,647!
4828
  if (outOfTimeWindow(key.ts, &pReader->info.window)) {
2,147,483,647!
4829
    pIter->hasVal = false;
17,930,640✔
4830
    goto _end;
22,404,411✔
4831
  }
4832

4833
  // it is a valid data version
4834
  if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) {
2,147,483,647!
4835
    if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) {
2,147,483,647!
4836
      *pRes = pRow;
2,147,483,647✔
4837
      goto _end;
2,147,483,647✔
4838
    } else {
4839
      bool dropped = false;
204,486,092✔
4840
      code = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange,
204,508,292✔
4841
                            pReader->suppInfo.numOfPks > 0, &dropped);
204,485,235✔
4842
      TSDB_CHECK_CODE(code, lino, _end);
204,447,606!
4843
      if (!dropped) {
204,447,606!
4844
        *pRes = pRow;
198,205,765✔
4845
        goto _end;
198,210,565✔
4846
      }
4847
    }
4848
  }
4849

4850
  while (1) {
4851
    pIter->hasVal = tsdbTbDataIterNext(pIter->iter);
204,716,239✔
4852
    if (!pIter->hasVal) {
204,719,617!
4853
      goto _end;
1,403,002✔
4854
    }
4855

4856
    pRow = tsdbTbDataIterGet(pIter->iter);
203,316,325!
4857

4858
    TSDBROW_INIT_KEY(pRow, key);
203,317,522!
4859
    if (outOfTimeWindow(key.ts, &pReader->info.window)) {
203,312,633!
4860
      pIter->hasVal = false;
117,507✔
4861
      goto _end;
119,442✔
4862
    }
4863

4864
    if (key.version <= pReader->info.verRange.maxVer && key.version >= pReader->info.verRange.minVer) {
203,199,957!
4865
      if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) {
194,681,696!
4866
        *pRes = pRow;
250✔
4867
        goto _end;
×
4868
      } else {
4869
        bool dropped = false;
194,681,496✔
4870
        code = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange,
194,681,058✔
4871
                              pReader->suppInfo.numOfPks > 0, &dropped);
194,681,646✔
4872
        TSDB_CHECK_CODE(code, lino, _end);
194,679,246!
4873
        if (!dropped) {
194,679,246!
4874
          *pRes = pRow;
4,747,642✔
4875
          goto _end;
4,747,642✔
4876
        }
4877
      }
4878
    }
4879
  }
4880

4881
_end:
2,147,483,647✔
4882
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
4883
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
4884
  }
4885
  return code;
2,147,483,647✔
4886
}
4887

4888
int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArray* pDelList, STsdbReader* pReader) {
2,147,483,647✔
4889
  int32_t     code = TSDB_CODE_SUCCESS;
2,147,483,647✔
4890
  int32_t     lino = 0;
2,147,483,647✔
4891
  SRowMerger* pMerger = NULL;
2,147,483,647✔
4892

4893
  TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
4894
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
4895

4896
  pMerger = &pReader->status.merger;
2,147,483,647✔
4897

4898
  while (1) {
52,110,027✔
4899
    pIter->hasVal = tsdbTbDataIterNext(pIter->iter);
2,147,483,647✔
4900
    if (!pIter->hasVal) {
2,147,483,647✔
4901
      break;
5,392,072✔
4902
    }
4903

4904
    // data exists but not valid
4905
    TSDBROW* pRow = NULL;
2,147,483,647!
4906
    code = getValidMemRow(pIter, pDelList, pReader, &pRow);
2,147,483,647✔
4907
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
4908
    if (pRow == NULL) {
2,147,483,647✔
4909
      break;
61,721✔
4910
    }
4911

4912
    // ts is not identical, quit
4913
    if (TSDBROW_TS(pRow) != pCurKey->ts) {
2,147,483,647✔
4914
      break;
2,147,483,647✔
4915
    }
4916

4917
    if (pCurKey->numOfPKs > 0) {
52,126,259✔
4918
      SRowKey nextKey = {0};
682,602✔
4919
      tRowGetKeyEx(pRow, &nextKey);
682,602!
4920
      if (pkCompEx(pCurKey, &nextKey) != 0) {
682,602✔
4921
        break;
360,519✔
4922
      }
4923
    }
4924

4925
    STSchema* pTSchema = NULL;
52,109,513✔
4926
    if (pRow->type == TSDBROW_ROW_FMT) {
52,109,513✔
4927
      pTSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, uid);
51,891,622!
4928
      TSDB_CHECK_NULL(pTSchema, code, lino, _end, terrno);
51,889,052!
4929
    }
4930

4931
    code = tsdbRowMergerAdd(pMerger, pRow, pTSchema);
52,101,032✔
4932
    TSDB_CHECK_CODE(code, lino, _end);
52,110,027!
4933
  }
4934

4935
_end:
2,147,483,647✔
4936
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
4937
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
4938
  }
4939
  return code;
2,147,483,647✔
4940
}
4941

4942
static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t* rowIndex, SRowKey* pKey, SRowMerger* pMerger,
2,147,483,647✔
4943
                                          SVersionRange* pVerRange, int32_t step) {
4944
  int32_t code = TSDB_CODE_SUCCESS;
2,147,483,647✔
4945
  int32_t lino = 0;
2,147,483,647✔
4946

4947
  TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
4948
  TSDB_CHECK_NULL(rowIndex, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
4949

4950
  while ((*rowIndex) < pBlockData->nRow && (*rowIndex) >= 0) {
2,147,483,647!
4951
    SRowKey cur;
2,147,483,647✔
4952
    tColRowGetKey(pBlockData, (*rowIndex), &cur);
2,147,483,647✔
4953
    if (pkCompEx(&cur, pKey) != 0) {
2,147,483,647✔
4954
      break;
2,147,483,647✔
4955
    }
4956

4957
    if (pBlockData->aVersion[(*rowIndex)] > pVerRange->maxVer ||
3,379,692!
4958
        pBlockData->aVersion[(*rowIndex)] < pVerRange->minVer) {
1,310!
4959
      (*rowIndex) += step;
×
4960
      continue;
×
4961
    }
4962

4963
    TSDBROW fRow = tsdbRowFromBlockData(pBlockData, (*rowIndex));
1,310✔
4964
    code = tsdbRowMergerAdd(pMerger, &fRow, NULL);
1,310✔
4965
    TSDB_CHECK_CODE(code, lino, _end);
1,310!
4966
    (*rowIndex) += step;
1,310✔
4967
  }
4968

4969
_end:
×
4970
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
4971
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
4972
  }
4973
  return code;
2,147,483,647✔
4974
}
4975

4976
typedef enum {
4977
  CHECK_FILEBLOCK_CONT = 0x1,
4978
  CHECK_FILEBLOCK_QUIT = 0x2,
4979
} CHECK_FILEBLOCK_STATE;
4980

4981
static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanInfo* pScanInfo,
2,846,767✔
4982
                                         SFileDataBlockInfo* pFBlock, SRowMerger* pMerger, SRowKey* pKey,
4983
                                         CHECK_FILEBLOCK_STATE* state) {
4984
  int32_t             code = TSDB_CODE_SUCCESS;
2,846,767✔
4985
  int32_t             lino = 0;
2,846,767✔
4986
  SFileBlockDumpInfo* pDumpInfo = NULL;
2,846,767✔
4987
  SBlockData*         pBlockData = NULL;
2,846,767✔
4988
  bool                asc = false;
2,846,767✔
4989
  SVersionRange*      pVerRange = NULL;
2,846,767✔
4990
  bool                loadNeighbor = true;
2,846,767✔
4991
  int32_t             step = 0;
2,846,767✔
4992

4993
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,846,767!
4994
  TSDB_CHECK_NULL(state, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,846,767!
4995

4996
  pDumpInfo = &pReader->status.fBlockDumpInfo;
2,846,767✔
4997
  pBlockData = &pReader->status.fileBlockData;
2,847,074✔
4998
  asc = ASCENDING_TRAVERSE(pReader->info.order);
2,847,074✔
4999
  pVerRange = &pReader->info.verRange;
2,847,381✔
5000
  step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1;
2,846,767✔
5001

5002
  *state = CHECK_FILEBLOCK_QUIT;
2,847,381✔
5003
  code = loadNeighborIfOverlap(pFBlock, pScanInfo, pReader, &loadNeighbor);
2,847,381✔
5004
  TSDB_CHECK_CODE(code, lino, _end);
2,847,074!
5005

5006
  if (loadNeighbor) {
2,847,074!
5007
    code = doMergeRowsInFileBlockImpl(pBlockData, &pDumpInfo->rowIndex, pKey, pMerger, pVerRange, step);
1,310✔
5008
    TSDB_CHECK_CODE(code, lino, _end);
1,310!
5009
    if ((pDumpInfo->rowIndex >= pDumpInfo->totalRows && asc) || (pDumpInfo->rowIndex < 0 && !asc)) {
1,310!
5010
      *state = CHECK_FILEBLOCK_CONT;
×
5011
    }
5012
  }
5013

5014
_end:
2,847,074✔
5015
  if (code != TSDB_CODE_SUCCESS) {
2,846,767!
5016
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5017
  }
5018
  return code;
2,846,767✔
5019
}
5020

5021
int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey,
2,147,483,647✔
5022
                                STsdbReader* pReader) {
5023
  int32_t             code = TSDB_CODE_SUCCESS;
2,147,483,647✔
5024
  int32_t             lino = 0;
2,147,483,647✔
5025
  SFileBlockDumpInfo* pDumpInfo = NULL;
2,147,483,647✔
5026
  SRowMerger*         pMerger = NULL;
2,147,483,647✔
5027
  bool                asc = false;
2,147,483,647✔
5028
  int32_t             step = 0;
2,147,483,647✔
5029
  SVersionRange*      pRange = NULL;
2,147,483,647✔
5030

5031
  TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
5032
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
5033

5034
  pDumpInfo = &pReader->status.fBlockDumpInfo;
2,147,483,647✔
5035
  pMerger = &pReader->status.merger;
2,147,483,647✔
5036
  asc = ASCENDING_TRAVERSE(pReader->info.order);
2,147,483,647✔
5037
  step = asc ? 1 : -1;
2,147,483,647✔
5038
  pRange = &pReader->info.verRange;
2,147,483,647✔
5039

5040
  pDumpInfo->rowIndex += step;
2,147,483,647✔
5041
  if ((pDumpInfo->rowIndex <= pBlockData->nRow - 1 && asc) || (pDumpInfo->rowIndex >= 0 && !asc)) {
2,147,483,647✔
5042
    code = doMergeRowsInFileBlockImpl(pBlockData, &pDumpInfo->rowIndex, pKey, pMerger, pRange, step);
2,147,483,647✔
5043
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
5044
  }
5045

5046
  // all rows are consumed, let's try next file block
5047
  if ((pDumpInfo->rowIndex >= pBlockData->nRow && asc) || (pDumpInfo->rowIndex < 0 && !asc)) {
2,147,483,647!
5048
    while (1) {
×
5049
      SFileDataBlockInfo* pFileBlockInfo = NULL;
2,847,074✔
5050
      code = getCurrentBlockInfo(&pReader->status.blockIter, &pFileBlockInfo, pReader->idStr);
2,847,074✔
5051
      TSDB_CHECK_CODE(code, lino, _end);
2,847,074!
5052

5053
      if (pFileBlockInfo == NULL) {
2,847,074!
5054
        break;
×
5055
      }
5056

5057
      CHECK_FILEBLOCK_STATE st = CHECK_FILEBLOCK_QUIT;
2,847,074✔
5058
      code = checkForNeighborFileBlock(pReader, pScanInfo, pFileBlockInfo, pMerger, pKey, &st);
2,847,074✔
5059
      TSDB_CHECK_CODE(code, lino, _end);
2,846,767!
5060
      if (st == CHECK_FILEBLOCK_QUIT) {
2,846,767!
5061
        break;
2,846,767✔
5062
      }
5063
    }
5064
  }
5065

5066
_end:
2,147,483,647✔
5067
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
5068
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5069
  }
5070
  return code;
2,147,483,647✔
5071
}
5072

5073
int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, SRowMerger* pMerger,
2,147,483,647✔
5074
                              int32_t pkSrcSlot, SVersionRange* pVerRange, const char* idStr) {
5075
  int32_t  code = TSDB_CODE_SUCCESS;
2,147,483,647✔
5076
  int32_t  lino = 0;
2,147,483,647✔
5077
  SRowKey* pRowKey = NULL;
2,147,483,647✔
5078
  SRowKey* pNextKey = NULL;
2,147,483,647✔
5079

5080
  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
5081

5082
  pRowKey = &pScanInfo->lastProcKey;
2,147,483,647✔
5083

5084
  while (1) {
685,834,281✔
5085
    code = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pkSrcSlot, pVerRange);
2,147,483,647✔
5086
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
5087
    if (!hasDataInSttBlock(pScanInfo)) {
2,147,483,647✔
5088
      break;
6,076,726✔
5089
    }
5090

5091
    pNextKey = getCurrentKeyInSttBlock(pSttBlockReader);
2,147,483,647!
5092

5093
    int32_t ret = pkCompEx(pRowKey, pNextKey);
2,147,483,647✔
5094
    if (ret == 0) {
2,147,483,647✔
5095
      TSDBROW* pRow1 = tMergeTreeGetRow(&pSttBlockReader->mergeTree);
685,821,688✔
5096
      code = tsdbRowMergerAdd(pMerger, pRow1, NULL);
685,830,169✔
5097
      TSDB_CHECK_CODE(code, lino, _end);
685,834,281!
5098
    } else {
5099
      tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid,
2,147,483,647✔
5100
                pScanInfo->sttBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline),
5101
                pScanInfo->sttKeyInfo.nextProcKey.ts, idStr);
5102
      break;
2,147,483,647✔
5103
    }
5104
  }
5105

5106
_end:
2,147,483,647✔
5107
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
5108
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5109
  }
5110
  return code;
2,147,483,647✔
5111
}
5112

5113
int32_t doMergeMemTableMultiRows(TSDBROW* pRow, SRowKey* pKey, uint64_t uid, SIterInfo* pIter, SArray* pDelList,
2,147,483,647✔
5114
                                 TSDBROW* pResRow, STsdbReader* pReader, bool* freeTSRow) {
5115
  int32_t     code = TSDB_CODE_SUCCESS;
2,147,483,647✔
5116
  int32_t     lino = 0;
2,147,483,647✔
5117
  SRowMerger* pMerger = NULL;
2,147,483,647✔
5118
  TSDBROW*    pNextRow = NULL;
2,147,483,647✔
5119
  STSchema*   pTSchema = NULL;
2,147,483,647✔
5120
  STSchema*   pTSchema1 = NULL;
2,147,483,647✔
5121
  TSDBROW     current;
2,147,483,647✔
5122

5123
  TSDB_CHECK_NULL(pRow, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
5124
  TSDB_CHECK_NULL(pIter, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
5125
  TSDB_CHECK_NULL(pResRow, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
5126
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
5127
  TSDB_CHECK_NULL(freeTSRow, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
5128

5129
  pMerger = &pReader->status.merger;
2,147,483,647✔
5130
  pMerger->arg = pReader->pTsdb->pVnode->pBse;
2,147,483,647✔
5131

5132
  current = *pRow;
2,147,483,647✔
5133

5134
  {  // if the timestamp of the next valid row has a different ts, return current row directly
5135
    pIter->hasVal = tsdbTbDataIterNext(pIter->iter);
2,147,483,647✔
5136

5137
    if (!pIter->hasVal) {
2,147,483,647✔
5138
      *pResRow = *pRow;
320,492,847✔
5139
      *freeTSRow = false;
320,461,597✔
5140
      goto _end;
320,471,845✔
5141
    } else {  // has next point in mem/imem
5142
      code = getValidMemRow(pIter, pDelList, pReader, &pNextRow);
2,147,483,647✔
5143
      TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
5144
      if (pNextRow == NULL) {
2,147,483,647✔
5145
        *pResRow = current;
18,736,364✔
5146
        *freeTSRow = false;
18,736,381✔
5147
        goto _end;
18,735,020✔
5148
      }
5149

5150
      if (TSDBROW_TS(&current) != TSDBROW_TS(pNextRow)) {
2,147,483,647✔
5151
        *pResRow = current;
2,147,483,647✔
5152
        *freeTSRow = false;
2,147,483,647✔
5153
        goto _end;
2,147,483,647✔
5154
      }
5155

5156
      if (pKey->numOfPKs > 0) {
152,048,372✔
5157
        SRowKey nextRowKey = {0};
93,626,081✔
5158
        tRowGetKeyEx(pNextRow, &nextRowKey);
93,625,826!
5159
        if (pkCompEx(pKey, &nextRowKey) != 0) {
93,624,295✔
5160
          *pResRow = current;
93,005,965✔
5161
          *freeTSRow = false;
93,006,475✔
5162
          goto _end;
93,007,979✔
5163
        }
5164
      }
5165
    }
5166
  }
5167

5168
  // start to merge duplicated rows
5169
  if (current.type == TSDBROW_ROW_FMT) {  // get the correct schema for row-wise data in memory
59,055,520✔
5170
    pTSchema = doGetSchemaForTSRow(current.pTSRow->sver, pReader, uid);
59,055,474!
5171
    TSDB_CHECK_NULL(pTSchema, code, lino, _end, terrno);
59,059,838!
5172
  }
5173

5174
  code = tsdbRowMergerAdd(pMerger, &current, pTSchema);
59,059,884✔
5175
  TSDB_CHECK_CODE(code, lino, _end);
59,032,046!
5176

5177
  if (pNextRow->type == TSDBROW_ROW_FMT) {  // get the correct schema for row-wise data in memory
59,032,046!
5178
    pTSchema1 = doGetSchemaForTSRow(pNextRow->pTSRow->sver, pReader, uid);
59,076,438!
5179
    TSDB_CHECK_NULL(pTSchema1, code, lino, _end, terrno);
59,091,712!
5180
  }
5181

5182
  code = tsdbRowMergerAdd(pMerger, pNextRow, pTSchema1);
59,044,312✔
5183
  TSDB_CHECK_CODE(code, lino, _end);
59,066,102!
5184

5185
  code = doMergeRowsInBuf(pIter, uid, pKey, pDelList, pReader);
59,066,102✔
5186
  TSDB_CHECK_CODE(code, lino, _end);
59,030,485!
5187

5188
  code = tsdbRowMergerGetRow(pMerger, &pResRow->pTSRow);
59,030,485✔
5189
  TSDB_CHECK_CODE(code, lino, _end);
59,051,814!
5190

5191
  pResRow->type = TSDBROW_ROW_FMT;
59,051,814✔
5192
  tsdbRowMergerClear(pMerger);
59,061,564✔
5193
  *freeTSRow = true;
59,023,385✔
5194

5195
_end:
2,147,483,647✔
5196
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
5197
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5198
  }
5199
  return code;
2,147,483,647✔
5200
}
5201

5202
int32_t doMergeMemIMemRows(TSDBROW* pRow, SRowKey* pRowKey, TSDBROW* piRow, SRowKey* piRowKey,
4,937,920✔
5203
                           STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, SRow** pTSRow) {
5204
  int32_t     code = TSDB_CODE_SUCCESS;
4,937,920✔
5205
  int32_t     lino = 0;
4,937,920✔
5206
  SRowMerger* pMerger = NULL;
4,937,920✔
5207
  STSchema*   pSchema = NULL;
4,937,920✔
5208
  STSchema*   piSchema = NULL;
4,937,920✔
5209

5210
  TSDB_CHECK_NULL(pRow, code, lino, _end, TSDB_CODE_INVALID_PARA);
4,937,920!
5211
  TSDB_CHECK_NULL(piRow, code, lino, _end, TSDB_CODE_INVALID_PARA);
4,937,920!
5212
  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
4,937,920!
5213
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
4,937,920!
5214

5215
  pMerger = &pReader->status.merger;
4,937,920✔
5216
  pMerger->arg = pReader->pTsdb->pVnode->pBse;
4,937,920✔
5217

5218
  if (pRow->type == TSDBROW_ROW_FMT) {
4,937,920!
5219
    pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid);
4,937,920!
5220
    TSDB_CHECK_NULL(pSchema, code, lino, _end, terrno);
4,937,920!
5221
  }
5222

5223
  if (piRow->type == TSDBROW_ROW_FMT) {
4,937,920!
5224
    piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid);
4,937,920!
5225
    TSDB_CHECK_NULL(piSchema, code, lino, _end, terrno);
4,937,920!
5226
  }
5227

5228
  if (ASCENDING_TRAVERSE(pReader->info.order)) {  // ascending order imem --> mem
4,937,920✔
5229
    code = tsdbRowMergerAdd(pMerger, piRow, piSchema);
4,937,216✔
5230
    TSDB_CHECK_CODE(code, lino, _end);
4,937,216!
5231

5232
    code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, piRowKey, pBlockScanInfo->delSkyline, pReader);
4,937,216✔
5233
    TSDB_CHECK_CODE(code, lino, _end);
4,937,216!
5234

5235
    code = tsdbRowMergerAdd(pMerger, pRow, pSchema);
4,937,216✔
5236
    TSDB_CHECK_CODE(code, lino, _end);
4,937,216!
5237

5238
    code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, pRowKey, pBlockScanInfo->delSkyline, pReader);
4,937,216✔
5239
    TSDB_CHECK_CODE(code, lino, _end);
4,937,216!
5240
  } else {
5241
    code = tsdbRowMergerAdd(pMerger, pRow, pSchema);
704✔
5242
    TSDB_CHECK_CODE(code, lino, _end);
704!
5243

5244
    code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, pRowKey, pBlockScanInfo->delSkyline, pReader);
704✔
5245
    TSDB_CHECK_CODE(code, lino, _end);
704!
5246

5247
    code = tsdbRowMergerAdd(pMerger, piRow, piSchema);
704✔
5248
    TSDB_CHECK_CODE(code, lino, _end);
704!
5249

5250
    code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, piRowKey, pBlockScanInfo->delSkyline, pReader);
704✔
5251
    TSDB_CHECK_CODE(code, lino, _end);
704!
5252
  }
5253

5254
  tRowKeyAssign(&pBlockScanInfo->lastProcKey, pRowKey);
4,937,920✔
5255

5256
  code = tsdbRowMergerGetRow(pMerger, pTSRow);
4,937,920✔
5257
  tsdbRowMergerClear(pMerger);
4,937,920✔
5258
  TSDB_CHECK_CODE(code, lino, _end);
4,937,920!
5259

5260
_end:
4,937,920✔
5261
  if (code != TSDB_CODE_SUCCESS) {
4,937,920!
5262
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5263
  }
5264
  return code;
4,937,920✔
5265
}
5266

5267
static int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, TSDBROW* pResRow,
2,147,483,647✔
5268
                                   int64_t endKey, bool* freeTSRow) {
5269
  int32_t    code = TSDB_CODE_SUCCESS;
2,147,483,647✔
5270
  int32_t    lino = 0;
2,147,483,647✔
5271
  TSDBROW*   pRow = NULL;
2,147,483,647✔
5272
  TSDBROW*   piRow = NULL;
2,147,483,647✔
5273
  SArray*    pDelList = NULL;
2,147,483,647✔
5274
  uint64_t   uid = 0;
2,147,483,647✔
5275
  SIterInfo* piter = NULL;
2,147,483,647✔
5276
  SIterInfo* piiter = NULL;
2,147,483,647✔
5277
  SRowKey    rowKey = {0};
2,147,483,647✔
5278
  SRowKey    irowKey = {0};
2,147,483,647✔
5279
  bool       asc = false;
2,147,483,647✔
5280

5281
  TSDB_CHECK_NULL(pBlockScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
5282
  TSDB_CHECK_NULL(pResRow, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
5283
  TSDB_CHECK_NULL(freeTSRow, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
5284

5285
  code = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader, &pRow);
2,147,483,647!
5286
  TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
5287
  code = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader, &piRow);
2,147,483,647!
5288
  TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
5289

5290
  pDelList = pBlockScanInfo->delSkyline;
2,147,483,647✔
5291
  uid = pBlockScanInfo->uid;
2,147,483,647✔
5292
  piter = &pBlockScanInfo->iter;
2,147,483,647✔
5293
  piiter = &pBlockScanInfo->iiter;
2,147,483,647✔
5294

5295
  // todo refactor
5296
  asc = ASCENDING_TRAVERSE(pReader->info.order);
2,147,483,647✔
5297
  if (piter->hasVal) {
2,147,483,647✔
5298
    tRowGetKeyEx(pRow, &rowKey);
2,147,483,647✔
5299
    if ((rowKey.ts >= endKey && asc) || (rowKey.ts <= endKey && !asc)) {
2,147,483,647!
5300
      pRow = NULL;
43,973✔
5301
    }
5302
  }
5303

5304
  if (piiter->hasVal) {
2,147,483,647!
5305
    tRowGetKeyEx(piRow, &irowKey);
2,147,483,647!
5306
    if ((irowKey.ts >= endKey && asc) || (irowKey.ts <= endKey && !asc)) {
2,147,483,647!
5307
      piRow = NULL;
1,380✔
5308
    }
5309
  }
5310

5311
  if (pRow != NULL && piRow != NULL) {
2,147,483,647✔
5312
    if (rowKey.numOfPKs == 0) {
2,005,006,212✔
5313
      if ((rowKey.ts > irowKey.ts && asc) || (rowKey.ts < irowKey.ts && (!asc))) {  // ik.ts < k.ts
2,004,849,636!
5314
        code = doMergeMemTableMultiRows(piRow, &irowKey, uid, piiter, pDelList, pResRow, pReader, freeTSRow);
1,998,420,876✔
5315
        TSDB_CHECK_CODE(code, lino, _end);
1,999,775,767!
5316
      } else if ((rowKey.ts < irowKey.ts && asc) || (rowKey.ts > irowKey.ts && (!asc))) {
6,529,846!
5317
        code = doMergeMemTableMultiRows(pRow, &rowKey, uid, piter, pDelList, pResRow, pReader, freeTSRow);
207,612✔
5318
        TSDB_CHECK_CODE(code, lino, _end);
207,612!
5319
      } else {  // ik.ts == k.ts
5320
        pResRow->type = TSDBROW_ROW_FMT;
6,325,253✔
5321
        code = doMergeMemIMemRows(pRow, &rowKey, piRow, &irowKey, pBlockScanInfo, pReader, &pResRow->pTSRow);
4,937,664✔
5322
        TSDB_CHECK_CODE(code, lino, _end);
4,937,920!
5323
        *freeTSRow = true;
4,937,920✔
5324
      }
5325
    } else {
5326
      int32_t ret = pkCompEx(&rowKey, &irowKey);
×
5327
      if (ret != 0) {
×
5328
        if ((ret > 0 && asc) || (ret < 0 && (!asc))) {  // ik.ts < k.ts
×
5329
          code = doMergeMemTableMultiRows(piRow, &irowKey, uid, piiter, pDelList, pResRow, pReader, freeTSRow);
×
5330
          TSDB_CHECK_CODE(code, lino, _end);
×
5331
        } else if ((ret < 0 && asc) || (ret > 0 && (!asc))) {
×
5332
          code = doMergeMemTableMultiRows(pRow, &rowKey, uid, piter, pDelList, pResRow, pReader, freeTSRow);
×
5333
          TSDB_CHECK_CODE(code, lino, _end);
×
5334
        }
5335
      } else {  // ik.ts == k.ts
5336
        pResRow->type = TSDBROW_ROW_FMT;
×
5337
        code = doMergeMemIMemRows(pRow, &rowKey, piRow, &irowKey, pBlockScanInfo, pReader, &pResRow->pTSRow);
×
5338
        TSDB_CHECK_CODE(code, lino, _end);
×
5339
        *freeTSRow = true;
×
5340
      }
5341
    }
5342
  } else if (piter->hasVal && pRow != NULL) {
2,147,483,647✔
5343
    code = doMergeMemTableMultiRows(pRow, &rowKey, uid, piter, pDelList, pResRow, pReader, freeTSRow);
2,147,483,647✔
5344
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
5345
  } else if (piiter->hasVal && piRow != NULL) {
2,147,483,647!
5346
    code = doMergeMemTableMultiRows(piRow, &irowKey, uid, piiter, pDelList, pResRow, pReader, freeTSRow);
2,147,483,647✔
5347
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
5348
  }
5349

5350
_end:
2,147,483,647✔
5351
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
5352
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5353
  }
5354
  return code;
2,147,483,647✔
5355
}
5356

5357
int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, STableBlockScanInfo* pScanInfo) {
2,147,483,647✔
5358
  int32_t             code = TSDB_CODE_SUCCESS;
2,147,483,647✔
5359
  int32_t             lino = 0;
2,147,483,647✔
5360
  int32_t             outputRowIndex = 0;
2,147,483,647✔
5361
  int64_t             uid = 0;
2,147,483,647✔
5362
  SBlockLoadSuppInfo* pSupInfo = NULL;
2,147,483,647✔
5363
  STSchema*           pSchema = NULL;
2,147,483,647✔
5364
  SColVal             colVal = {0};
2,147,483,647✔
5365
  SColVal             dstVal = {0};
2,147,483,647✔
5366
  int32_t             i = 0, j = 0;
2,147,483,647✔
5367

5368
  TSDB_CHECK_NULL(pBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
5369
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
5370
  TSDB_CHECK_NULL(pTSRow, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
5371
  TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
5372

5373
  outputRowIndex = pBlock->info.rows;
2,147,483,647✔
5374
  uid = pScanInfo->uid;
2,147,483,647✔
5375

5376
  pSupInfo = &pReader->suppInfo;
2,147,483,647✔
5377
  pSupInfo->args = pReader->pTsdb->pVnode->pBse;
2,147,483,647✔
5378

5379
  pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid);
2,147,483,647!
5380
  TSDB_CHECK_NULL(pSchema, code, lino, _end, terrno);
2,147,483,647!
5381

5382
  if (pSupInfo->colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) {
2,147,483,647✔
5383
    SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
2,147,483,647✔
5384
    TSDB_CHECK_NULL(pColData, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
5385

5386
    ((int64_t*)pColData->pData)[outputRowIndex] = pTSRow->ts;
2,147,483,647✔
5387
    i += 1;
2,147,483,647✔
5388
  }
5389

5390
  while (i < pSupInfo->numOfCols && j < pSchema->numOfCols) {
2,147,483,647✔
5391
    col_id_t colId = pSupInfo->colId[i];
2,147,483,647✔
5392

5393
    if (colId == pSchema->columns[j].colId) {
2,147,483,647✔
5394
      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
2,147,483,647✔
5395
      TSDB_CHECK_NULL(pColInfoData, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
5396

5397
      code = tRowGet(pTSRow, pSchema, j, &colVal);
2,147,483,647✔
5398
      TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
5399
      code = doCopyColVal(pColInfoData, outputRowIndex, i, &colVal, pSupInfo);
2,147,483,647✔
5400
      TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
5401
      i += 1;
2,147,483,647✔
5402
      j += 1;
2,147,483,647✔
5403
    } else if (colId < pSchema->columns[j].colId) {
2,147,483,647✔
5404
      SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
22,320,000✔
5405
      TSDB_CHECK_NULL(pColInfoData, code, lino, _end, TSDB_CODE_INVALID_PARA);
22,320,000!
5406

5407
      colDataSetNULL(pColInfoData, outputRowIndex);
22,320,000!
5408
      i += 1;
22,320,000✔
5409
    } else if (colId > pSchema->columns[j].colId) {
2,147,483,647!
5410
      j += 1;
2,147,483,647✔
5411
    }
5412
  }
5413

5414
  // set null value since current column does not exist in the "pSchema"
5415
  while (i < pSupInfo->numOfCols) {
2,147,483,647✔
5416
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pSupInfo->slotId[i]);
4,231,571✔
5417
    TSDB_CHECK_NULL(pColInfoData, code, lino, _end, TSDB_CODE_INVALID_PARA);
4,231,571!
5418

5419
    colDataSetNULL(pColInfoData, outputRowIndex);
4,231,571!
5420
    i += 1;
4,231,571✔
5421
  }
5422

5423
  pBlock->info.dataLoad = 1;
2,147,483,647✔
5424
  pBlock->info.rows += 1;
2,147,483,647✔
5425

5426
_end:
2,147,483,647✔
5427
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
5428
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5429
  }
5430
  return code;
2,147,483,647✔
5431
}
5432
int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData,
2,147,483,647✔
5433
                                 int32_t rowIndex) {
5434
  int32_t             code = TSDB_CODE_SUCCESS;
2,147,483,647✔
5435
  int32_t             lino = 0;
2,147,483,647✔
5436
  int32_t             i = 0, j = 0;
2,147,483,647✔
5437
  int32_t             outputRowIndex = 0;
2,147,483,647✔
5438
  SBlockLoadSuppInfo* pSupInfo = NULL;
2,147,483,647✔
5439
  SColVal             cv = {0};
2,147,483,647✔
5440
  int32_t             numOfInputCols = 0;
2,147,483,647✔
5441
  int32_t             numOfOutputCols = 0;
2,147,483,647✔
5442

5443
  TSDB_CHECK_NULL(pResBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
5444
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
5445
  TSDB_CHECK_NULL(pBlockData, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,147,483,647!
5446

5447
  outputRowIndex = pResBlock->info.rows;
2,147,483,647✔
5448

5449
  pSupInfo = &pReader->suppInfo;
2,147,483,647✔
5450
  pSupInfo->args = pReader->pTsdb->pVnode->pBse;
2,147,483,647✔
5451

5452
  ((int64_t*)pReader->status.pPrimaryTsCol->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex];
2,147,483,647✔
5453
  i += 1;
2,147,483,647✔
5454

5455
  numOfInputCols = pBlockData->nColData;
2,147,483,647✔
5456
  numOfOutputCols = pSupInfo->numOfCols;
2,147,483,647✔
5457

5458
  while (i < numOfOutputCols && j < numOfInputCols) {
2,147,483,647✔
5459
    SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j);
2,147,483,647✔
5460
    if (pData->cid < pSupInfo->colId[i]) {
2,147,483,647✔
5461
      j += 1;
220,885,500✔
5462
      continue;
220,885,500✔
5463
    }
5464
    SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, pSupInfo->slotId[i]);
2,147,483,647✔
5465
    if (pData->cid == pSupInfo->colId[i]) {
2,147,483,647!
5466
      code = tColDataGetValue(pData, rowIndex, &cv);
2,147,483,647✔
5467
      TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
5468

5469
      code = doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo);
2,147,483,647✔
5470
      TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
5471
      j += 1;
2,147,483,647✔
5472
    } else if (pData->cid > pCol->info.colId) {
×
5473
      // the specified column does not exist in file block, fill with null data
5474
      colDataSetNULL(pCol, outputRowIndex);
×
5475
    }
5476

5477
    i += 1;
2,147,483,647✔
5478
  }
5479

5480
  while (i < numOfOutputCols) {
2,147,483,647✔
5481
    SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]);
2,016,742✔
5482
    TSDB_CHECK_NULL(pCol, code, lino, _end, TSDB_CODE_INVALID_PARA);
2,016,742!
5483

5484
    colDataSetNULL(pCol, outputRowIndex);
2,016,742!
5485
    i += 1;
2,020,102✔
5486
  }
5487

5488
  pResBlock->info.dataLoad = 1;
2,147,483,647✔
5489
  pResBlock->info.rows += 1;
2,147,483,647✔
5490

5491
_end:
2,147,483,647✔
5492
  if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
5493
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5494
  }
5495
  return code;
2,147,483,647✔
5496
}
5497

5498
int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity,
387,967,404✔
5499
                                  STsdbReader* pReader) {
5500
  int32_t      code = TSDB_CODE_SUCCESS;
387,967,404✔
5501
  int32_t      lino = 0;
387,967,404✔
5502
  SSDataBlock* pBlock = NULL;
387,967,404✔
5503

5504
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
387,967,404!
5505

5506
  pBlock = pReader->resBlockInfo.pResBlock;
387,967,404✔
5507

5508
  do {
2,147,483,647✔
5509
    TSDBROW row = {.type = -1};
2,147,483,647✔
5510
    bool    freeTSRow = false;
2,147,483,647✔
5511
    code = tsdbGetNextRowInMem(pBlockScanInfo, pReader, &row, endKey, &freeTSRow);
2,147,483,647✔
5512
    TSDB_CHECK_CODE(code, lino, _end);
2,147,483,647!
5513

5514
    if (row.type == -1) {
2,147,483,647✔
5515
      break;
4,978,696✔
5516
    }
5517

5518
    if (row.type == TSDBROW_ROW_FMT) {
2,147,483,647✔
5519
      code = doAppendRowFromTSRow(pBlock, pReader, row.pTSRow, pBlockScanInfo);
2,147,483,647✔
5520
      if (code != TSDB_CODE_SUCCESS) {
2,147,483,647!
5521
        if (freeTSRow) {
×
5522
          taosMemoryFreeClear(row.pTSRow);
×
5523
        }
5524
        TSDB_CHECK_CODE(code, lino, _end);
×
5525
      }
5526
      pBlockScanInfo->lastProcKey.ts = row.pTSRow->ts;
2,147,483,647✔
5527
      pBlockScanInfo->lastProcKey.numOfPKs = row.pTSRow->numOfPKs;
2,147,483,647✔
5528
      if (row.pTSRow->numOfPKs > 0) {
2,147,483,647✔
5529
        code = tRowGetPrimaryKeyDeepCopy(row.pTSRow, &pBlockScanInfo->lastProcKey);
157,138,571✔
5530
        if (code != TSDB_CODE_SUCCESS) {
157,135,538!
5531
          if (freeTSRow) {
×
5532
            taosMemoryFreeClear(row.pTSRow);
×
5533
          }
5534
          TSDB_CHECK_CODE(code, lino, _end);
×
5535
        }
5536
      }
5537

5538
      if (freeTSRow) {
2,147,483,647!
5539
        taosMemoryFreeClear(row.pTSRow);
63,989,488!
5540
      }
5541
    } else {
5542
      code = doAppendRowFromFileBlock(pBlock, pReader, row.pBlockData, row.iRow);
25,835✔
5543
      TSDB_CHECK_CODE(code, lino, _end);
237,070,786!
5544

5545
      tColRowGetKeyDeepCopy(row.pBlockData, row.iRow, pReader->suppInfo.pkSrcSlot, &pBlockScanInfo->lastProcKey);
237,070,786!
5546
    }
5547

5548
    // no data in buffer, return immediately
5549
    if (!(pBlockScanInfo->iter.hasVal || pBlockScanInfo->iiter.hasVal)) {
2,147,483,647!
5550
      break;
339,206,236✔
5551
    }
5552

5553
    if (pBlock->info.rows >= capacity) {
2,147,483,647✔
5554
      break;
43,935,324✔
5555
    }
5556
  } while (1);
5557

5558
_end:
327,476,773✔
5559
  if (code != TSDB_CODE_SUCCESS) {
327,476,773!
5560
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5561
  }
5562
  return code;
388,022,166✔
5563
}
5564

5565
// TODO refactor: with createDataBlockScanInfo
5566
int32_t tsdbSetTableList2(void* ptr, const void* pTableList, int32_t num) {
30,003,892✔
5567
  int32_t               code = TSDB_CODE_SUCCESS;
30,003,892✔
5568
  int32_t               lino = 0;
30,003,892✔
5569
  int32_t               size = 0;
30,003,892✔
5570
  STableBlockScanInfo** p = NULL;
30,003,892✔
5571
  STableUidList*        pUidList = NULL;
30,003,892✔
5572
  int32_t               iter = 0;
30,003,892✔
5573
  bool                  acquired = false;
30,004,961✔
5574

5575
  STsdbReader* pReader = (STsdbReader*)ptr;
30,004,961✔
5576
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
30,004,961!
5577
  size = tSimpleHashGetSize(pReader->status.pTableMap);
30,004,961✔
5578

5579
  code = tsdbAcquireReader(pReader);
30,003,683✔
5580
  TSDB_CHECK_CODE(code, lino, _end);
30,005,797!
5581
  acquired = true;
30,005,797✔
5582

5583
  while ((p = tSimpleHashIterate(pReader->status.pTableMap, p, &iter)) != NULL) {
60,018,243✔
5584
    clearBlockScanInfo(*p);
30,012,163✔
5585
  }
5586

5587
  tSimpleHashClear(pReader->status.pTableMap);
30,004,455✔
5588

5589
  if (size < num) {
30,004,339✔
5590
    code = ensureBlockScanInfoBuf(&pReader->blockInfoBuf, num);
4,230✔
5591
    TSDB_CHECK_CODE(code, lino, _end);
4,230!
5592

5593
    char* p1 = taosMemoryRealloc(pReader->status.uidList.tableUidList, sizeof(uint64_t) * num);
4,230!
5594
    TSDB_CHECK_NULL(p1, code, lino, _end, terrno);
4,230!
5595

5596
    pReader->status.uidList.tableUidList = (uint64_t*)p1;
4,230✔
5597
  }
5598

5599
  pUidList = &pReader->status.uidList;
30,004,339✔
5600
  pUidList->currentIndex = 0;
30,004,473✔
5601

5602
  STableKeyInfo* pList = (STableKeyInfo*)pTableList;
30,003,751✔
5603
  for (int32_t i = 0; i < num; ++i) {
60,013,811✔
5604
    pUidList->tableUidList[i] = pList[i].uid;
30,009,101✔
5605

5606
    STableBlockScanInfo* pInfo = NULL;
30,008,960✔
5607
    code = getPosInBlockInfoBuf(&pReader->blockInfoBuf, i, &pInfo);
30,009,011✔
5608
    TSDB_CHECK_CODE(code, lino, _end);
30,010,757!
5609

5610
    code = initTableBlockScanInfo(pInfo, pList[i].uid, pReader->status.pTableMap, pReader);
30,010,757✔
5611
    TSDB_CHECK_CODE(code, lino, _end);
30,008,690!
5612
  }
5613

5614
_end:
30,004,311✔
5615
  if (code != TSDB_CODE_SUCCESS) {
30,005,426!
5616
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5617
  }
5618
  if (acquired) {
30,002,358!
5619
    (void)tsdbReleaseReader(pReader);
30,002,358✔
5620
  }
5621
  return code;
30,006,436✔
5622
}
5623

5624
uint64_t tsdbGetReaderMaxVersion2(STsdbReader* pReader) { return pReader->info.verRange.maxVer; }
×
5625

5626
static int32_t doOpenReaderImpl(STsdbReader* pReader) {
177,066,800✔
5627
  int32_t         code = TSDB_CODE_SUCCESS;
177,066,800✔
5628
  int32_t         lino = 0;
177,066,800✔
5629
  SReaderStatus*  pStatus = NULL;
177,066,800✔
5630
  SDataBlockIter* pBlockIter = NULL;
177,066,800✔
5631

5632
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
177,066,800!
5633

5634
  pStatus = &pReader->status;
177,066,800✔
5635
  pBlockIter = &pStatus->blockIter;
177,083,278✔
5636

5637
  if (pReader->bFilesetDelimited) {
177,084,348!
5638
    getMemTableTimeRange(pReader, &pReader->status.memTableMaxKey, &pReader->status.memTableMinKey);
27,337,842✔
5639
    pReader->status.bProcMemFirstFileset = true;
27,341,271✔
5640
  }
5641

5642
  code = initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader);
177,100,845✔
5643
  TSDB_CHECK_CODE(code, lino, _end);
176,985,667!
5644

5645
  code = resetDataBlockIterator(&pStatus->blockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo),
176,986,953✔
5646
                                pReader->idStr);
176,985,667✔
5647
  TSDB_CHECK_CODE(code, lino, _end);
177,065,305!
5648

5649
  if (pStatus->fileIter.numOfFiles == 0) {
177,065,305✔
5650
    pStatus->loadFromFile = false;
143,399,353✔
5651
  } else {
5652
    code = initForFirstBlockInFile(pReader, pBlockIter);
33,640,683✔
5653
  }
5654

5655
  if (!pStatus->loadFromFile) {
177,030,770✔
5656
    code = resetTableListIndex(pStatus, pReader->idStr);
143,601,675✔
5657
    TSDB_CHECK_CODE(code, lino, _end);
143,561,199!
5658
  }
5659

5660
_end:
176,999,884✔
5661
  if (code != TSDB_CODE_SUCCESS) {
176,999,884!
5662
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5663
  }
5664
  return code;
176,976,358✔
5665
}
5666

5667
static void freeSchemaFunc(void* param) {
4,926,560✔
5668
  void** p = (void**)param;
4,926,560✔
5669
  taosMemoryFreeClear(*p);
4,926,560!
5670
}
4,926,791✔
5671

5672
static void clearSharedPtr(STsdbReader* p) {
4,189,118✔
5673
  if (p) {
4,189,118!
5674
    p->status.pTableMap = NULL;
4,189,118✔
5675
    p->status.uidList.tableUidList = NULL;
4,189,118✔
5676
    p->info.pSchema = NULL;
4,189,118✔
5677
    p->pReadSnap = NULL;
4,189,118✔
5678
    p->pSchemaMap = NULL;
4,189,118✔
5679
  }
5680
}
4,189,118✔
5681

5682
static int32_t setSharedPtr(STsdbReader* pDst, const STsdbReader* pSrc) {
4,189,118✔
5683
  pDst->status.pTableMap = pSrc->status.pTableMap;
4,189,118✔
5684
  pDst->status.uidList = pSrc->status.uidList;
4,189,118✔
5685
  pDst->info.pSchema = pSrc->info.pSchema;
4,189,118✔
5686
  pDst->pSchemaMap = pSrc->pSchemaMap;
4,189,118✔
5687
  pDst->pReadSnap = pSrc->pReadSnap;
4,189,118✔
5688
  pDst->pReadSnap->pfSetArray = pSrc->pReadSnap->pfSetArray;
4,189,118✔
5689

5690
  if (pDst->info.pSchema) {
4,189,118!
5691
    return tsdbRowMergerInit(&pDst->status.merger, pDst->info.pSchema);
4,189,118✔
5692
  }
5693

5694
  return TSDB_CODE_SUCCESS;
×
5695
}
5696

5697
// ====================================== EXPOSED APIs ======================================
5698
int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables,
195,145,871✔
5699
                        SSDataBlock* pResBlock, void** ppReader, const char* idstr, SHashObj** pIgnoreTables) {
5700
  int32_t      code = TSDB_CODE_SUCCESS;
195,145,871✔
5701
  int32_t      lino = 0;
195,145,871✔
5702
  STimeWindow  window = {0};
195,145,871✔
5703
  SVnodeCfg*   pConf = NULL;
195,145,871✔
5704
  STsdbReader* pReader = NULL;
195,145,871✔
5705
  int32_t      capacity = 0;
195,145,871✔
5706

5707
  window = pCond->twindows;
195,145,871✔
5708
  pConf = &(((SVnode*)pVnode)->config);
195,192,925✔
5709

5710
  capacity = pConf->tsdbCfg.maxRows;
195,143,990✔
5711
  if (pResBlock != NULL) {
195,172,681✔
5712
    code = blockDataEnsureCapacity(pResBlock, capacity);
195,034,593✔
5713
    TSDB_CHECK_CODE(code, lino, _end);
195,053,480!
5714
  }
5715

5716
  code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr);
195,191,568✔
5717
  TSDB_CHECK_CODE(code, lino, _end);
195,074,094!
5718

5719
  // check for query time window
5720
  pReader = *ppReader;
195,074,094✔
5721
  if (isEmptyQueryTimeWindow(&pReader->info.window) && pCond->type == TIMEWINDOW_RANGE_CONTAINED) {
195,084,998✔
5722
    tsdbDebug("%p query window not overlaps with the data set, no result returned, %s", pReader, pReader->idStr);
797,566✔
5723
    goto _end;
797,960✔
5724
  }
5725

5726
  if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) {
194,239,109✔
5727
    // update the SQueryTableDataCond to create inner reader
5728
    int32_t order = pCond->order;
2,094,559✔
5729
    if (order == TSDB_ORDER_ASC) {
2,094,559!
5730
      pCond->twindows = pCond->extTwindows[0];
2,094,559✔
5731
      pCond->order = TSDB_ORDER_DESC;
2,094,559✔
5732
    } else {
5733
      pCond->twindows = pCond->extTwindows[1];
×
5734
      pCond->order = TSDB_ORDER_ASC;
×
5735
    }
5736

5737
    // here we only need one more row, so the capacity is set to be ONE.
5738
    code = tsdbReaderCreate(pVnode, pCond, (void**)&((STsdbReader*)pReader)->innerReader[0], 1, pResBlock, idstr);
2,094,559✔
5739
    TSDB_CHECK_CODE(code, lino, _end);
2,094,559!
5740

5741
    if (order == TSDB_ORDER_ASC) {
2,094,559!
5742
      pCond->twindows = pCond->extTwindows[1];
2,094,559✔
5743
    } else {
5744
      pCond->twindows = pCond->extTwindows[0];
×
5745
    }
5746
    pCond->order = order;
2,094,559✔
5747

5748
    code = tsdbReaderCreate(pVnode, pCond, (void**)&((STsdbReader*)pReader)->innerReader[1], 1, pResBlock, idstr);
2,094,559✔
5749
    TSDB_CHECK_CODE(code, lino, _end);
2,094,559✔
5750
  }
2,094,559!
5751

5752
  // NOTE: the endVersion in pCond is the data version not schema version, so pCond->endVersion is not correct here.
5753
  //  no valid error code set in metaGetTbTSchema, so let's set the error code here.
5754
  //  we should proceed in case of tmq processing.
5755
  if (pCond->suid != 0) {
5756
    code = metaGetTbTSchemaMaybeNull(pReader->pTsdb->pVnode->pMeta, pReader->info.suid, -1, 1, &pReader->info.pSchema);
194,350,744✔
5757
    TSDB_CHECK_CODE(code, lino, _end);
150,691,657✔
5758
    if (pReader->info.pSchema == NULL) {
150,627,319!
5759
      tsdbWarn("failed to get table schema, suid:%" PRIu64 ", ver:-1, %s", pReader->info.suid, pReader->idStr);
150,627,319✔
5760
    }
148,985!
5761
  } else if (numOfTables > 0) {
5762
    STableKeyInfo* pKey = pTableList;
43,627,860!
5763
    code = metaGetTbTSchemaMaybeNull(pReader->pTsdb->pVnode->pMeta, pKey->uid, -1, 1, &pReader->info.pSchema);
43,656,906✔
5764
    TSDB_CHECK_CODE(code, lino, _end);
43,656,906✔
5765
    if (pReader->info.pSchema == NULL) {
43,654,467!
5766
      tsdbWarn("failed to get table schema, uid:%" PRIu64 ", ver:-1, %s", pKey->uid, pReader->idStr);
43,654,467!
5767
    }
×
5768
  }
5769

5770
  if (pReader->info.pSchema != NULL) {
5771
    code = tsdbRowMergerInit(&pReader->status.merger, pReader->info.pSchema);
194,286,491✔
5772
    TSDB_CHECK_CODE(code, lino, _end);
194,206,387✔
5773
  }
194,158,563!
5774

5775
  pReader->pSchemaMap = tSimpleHashInit(8, taosFastHash);
5776
  if (pReader->pSchemaMap == NULL) {
194,291,424✔
5777
    tsdbError("failed init schema hash for reader %s", pReader->idStr);
194,287,915!
5778
    TSDB_CHECK_NULL(pReader->pSchemaMap, code, lino, _end, terrno);
×
5779
  }
×
5780

5781
  tSimpleHashSetFreeFp(pReader->pSchemaMap, freeSchemaFunc);
5782
  if (pReader->info.pSchema != NULL) {
194,281,786✔
5783
    code = updateBlockSMAInfo(pReader->info.pSchema, &pReader->suppInfo);
194,281,074✔
5784
    TSDB_CHECK_CODE(code, lino, _end);
194,168,641✔
5785
  }
194,181,734✔
5786

5787
  STsdbReader* p = (pReader->innerReader[0] != NULL) ? pReader->innerReader[0] : pReader;
5788

194,268,373✔
5789
  code = createDataBlockScanInfo(p, &pReader->blockInfoBuf, pTableList, &pReader->status.uidList, numOfTables,
5790
                                 &pReader->status.pTableMap);
194,298,050✔
5791
  TSDB_CHECK_CODE(code, lino, _end);
5792

194,280,781!
5793
  pReader->status.pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
5794
  TSDB_CHECK_NULL(pReader->status.pLDataIterArray, code, lino, _end, terrno);
194,280,781✔
5795

194,328,114!
5796
  pReader->flag = READER_STATUS_SUSPEND;
5797
  pReader->info.execMode = pCond->notLoadData ? READER_EXEC_ROWS : READER_EXEC_DATA;
194,320,612✔
5798

194,310,818!
5799
  pReader->pIgnoreTables = pIgnoreTables;
5800
  tsdbDebug("%p total numOfTable:%d, window:%" PRId64 " - %" PRId64 ", verRange:%" PRId64 " - %" PRId64
194,365,183✔
5801
            " in this query %s",
194,294,244✔
5802
            pReader, numOfTables, pReader->info.window.skey, pReader->info.window.ekey, pReader->info.verRange.minVer,
5803
            pReader->info.verRange.maxVer, pReader->idStr);
5804

5805
  // NOTE: simulate the suspend operation after the reader is suspended, ref: TD-38007.
5806
  // tsdbReaderSuspend2(pReader);
5807

5808
_end:
5809
  if (code != TSDB_CODE_SUCCESS) {
189,038,269✔
5810
    tsdbError("%s failed at line %d since %s, %s", __func__, lino, tstrerror(code), idstr);
195,119,650✔
5811
    tsdbReaderClose2(*ppReader);
6,603!
5812
    *ppReader = NULL;  // reset the pointer value.
6,603✔
5813
  }
6,603✔
5814
  return code;
5815
}
195,119,650✔
5816

5817
void tsdbReaderClose2(void* ptr) {
5818
  if (ptr == NULL) {
266,073,673✔
5819
    return;
266,073,673✔
5820
  }
66,682,838✔
5821
  STsdbReader* pReader = (STsdbReader*)ptr;
5822
  int32_t code = tsdbAcquireReader(pReader);
199,390,835✔
5823
  if (code) {
199,390,835✔
5824
    return;
199,394,770!
5825
  }
×
5826

5827
  {
5828
    if (pReader->innerReader[0] != NULL || pReader->innerReader[1] != NULL) {
5829
      STsdbReader* p = pReader->innerReader[0];
199,394,770!
5830
      clearSharedPtr(p);
2,093,553✔
5831

2,094,559✔
5832
      p = pReader->innerReader[1];
5833
      clearSharedPtr(p);
2,094,559✔
5834

2,094,559✔
5835
      tsdbReaderClose2(pReader->innerReader[0]);
5836
      tsdbReaderClose2(pReader->innerReader[1]);
2,094,559✔
5837
    }
2,094,559✔
5838
  }
5839

5840
  SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo;
5841
  TARRAY2_DESTROY(&pSupInfo->colAggArray, NULL);
199,393,093✔
5842

199,392,512!
5843
  if (pSupInfo->buildBuf) {
5844
    for (int32_t i = 0; i < pSupInfo->numOfCols; ++i) {
199,390,885✔
5845
      if (pSupInfo->buildBuf[i] != NULL) {
1,023,058,231✔
5846
        taosMemoryFreeClear(pSupInfo->buildBuf[i]);
823,687,395✔
5847
      }
100,423,155!
5848
    }
5849
  }
5850

5851
  if (pReader->resBlockInfo.freeBlock) {
5852
    blockDataDestroy(pReader->resBlockInfo.pResBlock);
199,382,100!
5853
    pReader->resBlockInfo.pResBlock = NULL;
135,557✔
5854
  }
135,537✔
5855

5856
  taosMemoryFree(pSupInfo->colId);
5857
  tBlockDataDestroy(&pReader->status.fileBlockData);
199,390,514✔
5858
  cleanupDataBlockIterator(&pReader->status.blockIter, shouldFreePkBuf(&pReader->suppInfo));
199,389,768✔
5859

199,369,479✔
5860
  size_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);
5861
  if (pReader->status.pTableMap != NULL) {
199,382,438✔
5862
    destroyAllBlockScanInfo(&pReader->status.pTableMap);
199,387,655✔
5863
  }
194,396,348✔
5864
  clearBlockScanInfoBuf(&pReader->blockInfoBuf);
5865

199,364,371✔
5866
  if (pReader->pFileReader != NULL) {
5867
    tsdbDataFileReaderClose(&pReader->pFileReader);
199,367,189✔
5868
  }
4,011,699✔
5869

5870
  SReadCostSummary* pCost = &pReader->cost;
5871
  SFilesetIter*     pFilesetIter = &pReader->status.fileIter;
199,369,100✔
5872
  if (pFilesetIter->pSttBlockReader != NULL) {
199,382,044✔
5873
    SSttBlockReader* pSttBlockReader = pFilesetIter->pSttBlockReader;
199,379,560✔
5874
    tMergeTreeClose(&pSttBlockReader->mergeTree);
177,095,672✔
5875

177,104,842✔
5876
    clearRowKey(&pSttBlockReader->currentKey);
5877
    taosMemoryFree(pSttBlockReader);
177,095,111✔
5878
  }
177,091,216✔
5879

5880
  destroySttBlockReader(pReader->status.pLDataIterArray, &pCost->sttCost);
5881
  pReader->status.pLDataIterArray = NULL;
199,361,921✔
5882
  taosMemoryFreeClear(pReader->status.uidList.tableUidList);
199,352,684✔
5883

199,359,596✔
5884
  tsdbTrace("tsdb/reader-close: %p, untake snapshot", pReader);
5885
  void* p = pReader->pReadSnap;
199,347,080✔
5886
  if ((p == atomic_val_compare_exchange_ptr((void**)&pReader->pReadSnap, p, NULL)) && (p != NULL)) {
199,347,080✔
5887
    tsdbUntakeReadSnap2(pReader, p, true);
199,333,909!
5888
    pReader->pReadSnap = NULL;
173,824,775✔
5889
  }
173,768,230✔
5890

5891
  code = tsem_destroy(&pReader->resumeAfterSuspend);
5892
  if (code != 0) {
199,340,341✔
5893
    tsdbWarn("faild to destroy tsem"); 
199,357,247!
5894
  }
×
5895
  (void)tsdbReleaseReader(pReader);
5896
  (void)tsdbUninitReaderLock(pReader);
199,357,247✔
5897

199,373,125✔
5898
  tsdbDebug(
5899
      "%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64
199,369,347✔
5900
      " SMA-time:%.2f ms, fileBlocks:%" PRId64
5901
      ", fileBlocks-load-time:%.2f ms, "
5902
      "build in-memory-block-time:%.2f ms, sttBlocks:%" PRId64 ", sttBlocks-time:%.2f ms, sttStatisBlock:%" PRId64
5903
      ", stt-statis-Block-time:%.2f ms, composed-blocks:%" PRId64
5904
      ", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb, createTime:%.2f ms,createSkylineIterTime:%.2f "
5905
      "ms, initSttBlockReader:%.2fms, %s",
5906
      pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, pCost->numOfBlocks,
5907
      pCost->blockLoadTime, pCost->buildmemBlock, pCost->sttCost.loadBlocks, pCost->sttCost.blockElapsedTime,
5908
      pCost->sttCost.loadStatisBlocks, pCost->sttCost.statisElapsedTime, pCost->composedBlocks,
5909
      pCost->buildComposedBlockTime, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pCost->createScanInfoList,
5910
      pCost->createSkylineIterTime, pCost->initSttBlockReader, pReader->idStr);
5911

5912
  taosMemoryFree(pReader->idStr);
5913

199,364,725✔
5914
  tsdbRowMergerCleanup(&pReader->status.merger);
5915
  taosMemoryFree(pReader->info.pSchema);
199,339,283✔
5916

199,332,477✔
5917
  tSimpleHashCleanup(pReader->pSchemaMap);
5918
  taosMemoryFreeClear(pReader);
199,339,549✔
5919
}
199,348,896!
5920

5921
static int32_t doSuspendCurrentReader(STsdbReader* pCurrentReader) {
5922
  int32_t               code = TSDB_CODE_SUCCESS;
3,450✔
5923
  int32_t               lino = 0;
3,450✔
5924
  SReaderStatus*        pStatus = NULL;
3,450✔
5925
  STableBlockScanInfo** p = NULL;
3,450✔
5926

3,450✔
5927
  TSDB_CHECK_NULL(pCurrentReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
5928

3,450!
5929
  pStatus = &pCurrentReader->status;
5930

3,450✔
5931
  if (pStatus->loadFromFile) {
5932
    tsdbDataFileReaderClose(&pCurrentReader->pFileReader);
3,450!
5933

3,450✔
5934
    SReadCostSummary* pCost = &pCurrentReader->cost;
5935
    destroySttBlockReader(pStatus->pLDataIterArray, &pCost->sttCost);
3,450✔
5936
    pStatus->pLDataIterArray = taosArrayInit(4, POINTER_BYTES);
3,450✔
5937
    TSDB_CHECK_NULL(pStatus->pLDataIterArray, code, lino, _end, terrno);
3,450✔
5938
  }
3,450!
5939

5940
  // resetDataBlockScanInfo excluding lastKey
5941
  int32_t step = ASCENDING_TRAVERSE(pCurrentReader->info.order) ? 1 : -1;
5942
  int32_t iter = 0;
3,450!
5943
  while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) {
3,450✔
5944
    STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p;
6,900✔
5945
    clearBlockScanInfoLoadInfo(pInfo);
3,450✔
5946
  }
3,450✔
5947

5948
  pStatus->uidList.currentIndex = 0;
5949
  initReaderStatus(pStatus);
3,450✔
5950

3,450✔
5951
_end:
5952
  if (code != TSDB_CODE_SUCCESS) {
3,450✔
5953
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
3,450!
5954
  }
×
5955
  return code;
5956
}
3,450✔
5957

5958
int32_t tsdbReaderSuspend2(STsdbReader* pReader) {
5959
  int32_t code = TSDB_CODE_SUCCESS;
3,450✔
5960
  int32_t lino = 0;
3,450✔
5961

3,450✔
5962
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
5963

3,450!
5964
  // save reader's base state & reset top state to be reconstructed from base state
5965
  pReader->status.suspendInvoked = true;  // record the suspend status
5966

3,450✔
5967
  if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) {
5968
    if (pReader->step == EXTERNAL_ROWS_PREV) {
3,450!
5969
      code = doSuspendCurrentReader(pReader->innerReader[0]);
×
5970
    } else if (pReader->step == EXTERNAL_ROWS_MAIN) {
×
5971
      code = doSuspendCurrentReader(pReader);
×
5972
    } else {
×
5973
      code = doSuspendCurrentReader(pReader->innerReader[1]);
5974
    }
×
5975
  } else {
5976
    code = doSuspendCurrentReader(pReader);
5977
  }
3,450✔
5978

5979
  // make sure only release once
5980
  void* p = pReader->pReadSnap;
5981
  if ((p == atomic_val_compare_exchange_ptr((void**)&pReader->pReadSnap, p, NULL)) && (p != NULL)) {
3,450✔
5982
    tsdbUntakeReadSnap2(pReader, p, false);
3,450!
5983
    pReader->pReadSnap = NULL;
3,450✔
5984
  }
3,450✔
5985

5986
  if (pReader->bFilesetDelimited) {
5987
    pReader->status.memTableMinKey = INT64_MAX;
3,450!
5988
    pReader->status.memTableMaxKey = INT64_MIN;
×
5989
  }
×
5990
  pReader->flag = READER_STATUS_SUSPEND;
5991

3,450✔
5992
  if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) {
5993
    clearSharedPtr(pReader->innerReader[0]);
3,450!
5994
    clearSharedPtr(pReader->innerReader[1]);
×
5995
  }
×
5996

5997
#if SUSPEND_RESUME_TEST
5998
  tsem_post(&pReader->resumeAfterSuspend);
5999
#endif
6000

6001
  tsdbDebug("reader: %p suspended in this query %s, step:%d", pReader, pReader->idStr, pReader->step);
6002

3,450!
6003
_end:
6004
  if (code != TSDB_CODE_SUCCESS) {
3,450✔
6005
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
3,450!
6006
  }
×
6007
  return code;
6008
}
3,450✔
6009

6010
static int32_t tsdbSetQueryReseek(void* pQHandle) {
6011
  int32_t      code = 0;
32,775✔
6012
  STsdbReader* pReader = pQHandle;
32,775✔
6013

32,775✔
6014
  code = tsdbTryAcquireReader(pReader);
6015
  if (code == 0) {
32,775✔
6016
    if (pReader->flag == READER_STATUS_SUSPEND) {
32,775✔
6017
      code = tsdbReleaseReader(pReader);
3,450!
6018
      return code;
×
6019
    }
×
6020

6021
    code = tsdbReaderSuspend2(pReader);
6022
    (void)tsdbReleaseReader(pReader);
3,450✔
6023
    return code;
3,450✔
6024
  } else if (code == EBUSY) {
3,450✔
6025
    return TSDB_CODE_VND_QUERY_BUSY;
29,325!
6026
  } else {
29,325✔
6027
    terrno = TAOS_SYSTEM_ERROR(code);
6028
    return TSDB_CODE_FAILED;
×
6029
  }
×
6030
}
6031

6032
int32_t tsdbReaderResume2(STsdbReader* pReader) {
6033
  int32_t               code = TSDB_CODE_SUCCESS;
194,381,185✔
6034
  int32_t               lino = 0;
194,381,185✔
6035
  STableBlockScanInfo** pBlockScanInfo = NULL;
194,381,185✔
6036
  int32_t               numOfTables = 0;
194,381,185✔
6037

194,381,185✔
6038
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
6039

194,381,185!
6040
  pBlockScanInfo = pReader->status.pTableIter;
6041

194,381,185✔
6042
  //  restore reader's state, task snapshot
6043
  numOfTables = tSimpleHashGetSize(pReader->status.pTableMap);
6044
  if (numOfTables > 0) {
194,390,216✔
6045
    tsdbTrace("tsdb/reader: %p, take snapshot", pReader);
194,354,296✔
6046
    code = tsdbTakeReadSnap2(pReader, tsdbSetQueryReseek, &pReader->pReadSnap, pReader->idStr);
173,720,496✔
6047
    TSDB_CHECK_CODE(code, lino, _end);
173,720,496✔
6048

173,752,795!
6049
    // open reader failure may cause the flag still to be READER_STATUS_SUSPEND, which may cause suspend reader failure.
6050
    // So we need to set it A.S.A.P
6051
    pReader->flag = READER_STATUS_NORMAL;
6052

173,752,795✔
6053
    if (pReader->type == TIMEWINDOW_RANGE_CONTAINED) {
6054
      code = doOpenReaderImpl(pReader);
173,761,434✔
6055
      TSDB_CHECK_CODE(code, lino, _end);
171,654,616✔
6056
    } else {
171,654,958!
6057
      STsdbReader* pPrevReader = pReader->innerReader[0];
6058
      STsdbReader* pNextReader = pReader->innerReader[1];
2,094,559✔
6059

2,094,559✔
6060
      // we need only one row
6061
      pPrevReader->resBlockInfo.capacity = 1;
6062
      code = setSharedPtr(pPrevReader, pReader);
2,094,559✔
6063
      TSDB_CHECK_CODE(code, lino, _end);
2,094,559✔
6064

2,094,559!
6065
      pNextReader->resBlockInfo.capacity = 1;
6066
      code = setSharedPtr(pNextReader, pReader);
2,094,559✔
6067
      TSDB_CHECK_CODE(code, lino, _end);
2,094,559✔
6068

2,094,559!
6069
      if (pReader->step == 0 || pReader->step == EXTERNAL_ROWS_PREV) {
6070
        code = doOpenReaderImpl(pPrevReader);
2,094,559!
6071
        TSDB_CHECK_CODE(code, lino, _end);
2,094,559✔
6072
      } else if (pReader->step == EXTERNAL_ROWS_MAIN) {
2,094,559!
6073
        code = doOpenReaderImpl(pReader);
×
6074
        TSDB_CHECK_CODE(code, lino, _end);
×
6075
      } else {
×
6076
        code = doOpenReaderImpl(pNextReader);
6077
        TSDB_CHECK_CODE(code, lino, _end);
×
6078
      }
×
6079
    }
6080
  }
6081

6082
  tsdbDebug("reader: %p resumed uid %" PRIu64 ", numOfTable:%" PRId32 ", in this query %s", pReader,
6083
            pBlockScanInfo ? (*pBlockScanInfo)->uid : 0, numOfTables, pReader->idStr);
194,383,317!
6084

6085
_end:
6086
  if (code != TSDB_CODE_SUCCESS) {
189,127,852✔
6087
    tsdbError("%s failed at line %d since %s, %s", __func__, lino, tstrerror(code),
194,383,827!
6088
              (pReader && pReader->idStr) ? pReader->idStr : "");
×
6089
  }
6090
  return code;
6091
}
194,369,950✔
6092

6093
static int32_t buildFromPreFilesetBuffer(STsdbReader* pReader) {
6094
  int32_t        code = TSDB_CODE_SUCCESS;
524,698✔
6095
  int32_t        lino = 0;
524,698✔
6096
  SReaderStatus* pStatus = NULL;
524,698✔
6097
  SSDataBlock*   pBlock = NULL;
524,698✔
6098
  int32_t        fid = 0;
524,698✔
6099
  STimeWindow    win = {0};
524,698✔
6100

524,698✔
6101
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
6102

524,698!
6103
  pStatus = &pReader->status;
6104
  pBlock = pReader->resBlockInfo.pResBlock;
524,698✔
6105
  fid = pReader->status.pCurrentFileset->fid;
524,698✔
6106
  tsdbFidKeyRange(fid, pReader->pTsdb->keepCfg.days, pReader->pTsdb->keepCfg.precision, &win.skey, &win.ekey);
524,698✔
6107

524,698✔
6108
  int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? win.skey : win.ekey;
6109
  code = buildBlockFromBufferSeqForPreFileset(pReader, endKey);
524,698✔
6110
  TSDB_CHECK_CODE(code, lino, _end);
524,698✔
6111
  if (pBlock->info.rows <= 0) {
524,698!
6112
    tsdbDebug("finished pre-fileset %d buffer processing. %s", fid, pReader->idStr);
524,698✔
6113
    pStatus->bProcMemPreFileset = false;
258,518✔
6114
    pStatus->processingMemPreFileSet = false;
258,518✔
6115
    if (pReader->notifyFn) {
258,518✔
6116
      STsdReaderNotifyInfo info = {0};
258,518✔
6117
      info.duration.filesetId = fid;
257,482✔
6118
      pReader->notifyFn(TSD_READER_NOTIFY_DURATION_START, &info, pReader->notifyParam);
257,482✔
6119
      tsdbDebug("new duration %d start notification when buffer pre-fileset, %s", fid, pReader->idStr);
257,482✔
6120
    }
257,482✔
6121
  }
6122

6123
_end:
6124
  if (code != TSDB_CODE_SUCCESS) {
524,698✔
6125
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
524,698!
6126
  }
×
6127
  return code;
6128
}
524,698✔
6129

6130
static int32_t doTsdbNextDataBlockFilesetDelimited(STsdbReader* pReader) {
6131
  int32_t        code = TSDB_CODE_SUCCESS;
125,000,627✔
6132
  int32_t        lino = 0;
125,000,627✔
6133
  SReaderStatus* pStatus = NULL;
125,000,627✔
6134
  SSDataBlock*   pBlock = NULL;
125,000,627✔
6135

125,000,627✔
6136
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
6137

125,000,627!
6138
  pStatus = &pReader->status;
6139
  pBlock = pReader->resBlockInfo.pResBlock;
125,000,627✔
6140

125,000,976✔
6141
  if (pStatus->loadFromFile) {
6142
    if (pStatus->bProcMemPreFileset) {
124,993,112!
6143
      code = buildFromPreFilesetBuffer(pReader);
15,615,714!
6144
      TSDB_CHECK_CODE(code, lino, _end);
270,184✔
6145
      if (pBlock->info.rows > 0) {
270,184!
6146
        goto _end;
270,184✔
6147
      }
76,180✔
6148
    }
6149

6150
    code = buildBlockFromFiles(pReader);
6151
    TSDB_CHECK_CODE(code, lino, _end);
15,538,380✔
6152

15,529,170!
6153
    tsdbTrace("block from file rows: %" PRId64 ", will process pre-file set buffer: %d. %s", pBlock->info.rows,
6154
              pStatus->bProcMemFirstFileset, pReader->idStr);
15,529,170!
6155
    if (pStatus->bProcMemPreFileset) {
6156
      if (pBlock->info.rows > 0) {
15,529,170!
6157
        if (pReader->notifyFn && !pReader->status.processingMemPreFileSet) {
190,000!
6158
          int32_t              fid = pReader->status.pCurrentFileset->fid;
190,000!
6159
          STsdReaderNotifyInfo info = {0};
×
6160
          info.duration.filesetId = fid;
×
6161
          pReader->notifyFn(TSD_READER_NOTIFY_NEXT_DURATION_BLOCK, &info, pReader->notifyParam);
×
6162
        }
×
6163
      } else {
6164
        pStatus->bProcMemPreFileset = false;
6165
      }
×
6166
    }
6167

6168
    if (pBlock->info.rows <= 0) {
6169
      code = resetTableListIndex(&pReader->status, pReader->idStr);
15,531,478✔
6170
      TSDB_CHECK_CODE(code, lino, _end);
3,947,639✔
6171

3,947,639!
6172
      int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN;
6173
      code = buildBlockFromBufferSequentially(pReader, endKey);
3,947,639✔
6174
      TSDB_CHECK_CODE(code, lino, _end);
3,946,927✔
6175
    }
3,947,062!
6176
  } else {  // no data in files, let's try the buffer
6177
    int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN;
6178
    code = buildBlockFromBufferSequentially(pReader, endKey);
109,380,216✔
6179
    TSDB_CHECK_CODE(code, lino, _end);
109,384,859✔
6180
  }
109,373,562!
6181

6182
_end:
6183
  if (code != TSDB_CODE_SUCCESS) {
109,373,562✔
6184
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
124,981,220!
6185
  }
×
6186
  return code;
6187
}
124,975,269✔
6188

6189
static int32_t doTsdbNextDataBlockFilesFirst(STsdbReader* pReader) {
6190
  int32_t        code = TSDB_CODE_SUCCESS;
595,266,324✔
6191
  int32_t        lino = 0;
595,266,324✔
6192
  SReaderStatus* pStatus = NULL;
595,266,324✔
6193
  SSDataBlock*   pBlock = NULL;
595,266,324✔
6194

595,266,324✔
6195
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
6196

595,266,324!
6197
  pStatus = &pReader->status;
6198
  pBlock = pReader->resBlockInfo.pResBlock;
595,266,324✔
6199

595,350,645✔
6200
  if (pStatus->loadFromFile) {
6201
    code = buildBlockFromFiles(pReader);
595,328,888✔
6202
    TSDB_CHECK_CODE(code, lino, _end);
204,435,948✔
6203

204,388,745!
6204
    if (pBlock->info.rows <= 0) {
6205
      code = resetTableListIndex(&pReader->status, pReader->idStr);
204,388,745✔
6206
      TSDB_CHECK_CODE(code, lino, _end);
30,136,827✔
6207

30,134,069!
6208
      int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN;
6209
      code = buildBlockFromBufferSequentially(pReader, endKey);
30,134,069✔
6210
      TSDB_CHECK_CODE(code, lino, _end);
30,133,707✔
6211
    }
30,135,513!
6212
  } else {  // no data in files, let's try the buffer
6213
    int64_t endKey = (ASCENDING_TRAVERSE(pReader->info.order)) ? INT64_MAX : INT64_MIN;
6214
    code = buildBlockFromBufferSequentially(pReader, endKey);
390,718,085✔
6215
    TSDB_CHECK_CODE(code, lino, _end);
390,855,709✔
6216
  }
390,527,404!
6217

6218
_end:
6219
  if (code != TSDB_CODE_SUCCESS) {
390,527,404✔
6220
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
594,969,631!
6221
  }
×
6222
  return code;
6223
}
594,894,198✔
6224

6225
static int32_t doTsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) {
6226
  int32_t      code = TSDB_CODE_SUCCESS;
740,573,593✔
6227
  int32_t      lino = 0;
740,573,593✔
6228
  SSDataBlock* pBlock = NULL;
740,573,593✔
6229

740,573,593✔
6230
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
6231

740,573,593!
6232
  // cleanup the data that belongs to the previous data block
6233
  pBlock = pReader->resBlockInfo.pResBlock;
6234
  blockDataCleanup(pBlock);
740,573,593✔
6235

740,807,762✔
6236
  *hasNext = false;
6237

740,958,628✔
6238
  SReaderStatus* pStatus = &pReader->status;
6239
  if (tSimpleHashGetSize(pStatus->pTableMap) == 0) {
740,971,946✔
6240
    goto _end;
740,967,740✔
6241
  }
20,589,985✔
6242

6243
  if (!pReader->bFilesetDelimited) {
6244
    code = doTsdbNextDataBlockFilesFirst(pReader);
720,355,584✔
6245
  } else {
595,359,396✔
6246
    code = doTsdbNextDataBlockFilesetDelimited(pReader);
6247
  }
125,005,771✔
6248

6249
  *hasNext = pBlock->info.rows > 0;
6250
  TSDB_CHECK_CODE(code, lino, _end);
719,866,137✔
6251

720,096,605!
6252
_end:
6253
  if (code != TSDB_CODE_SUCCESS) {
720,096,605✔
6254
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
740,686,590!
6255
  }
×
6256
  return code;
6257
}
740,580,051✔
6258

6259
int32_t tsdbNextDataBlock2(void* p, bool* hasNext) {
6260
  int32_t        code = TSDB_CODE_SUCCESS;
741,565,254✔
6261
  int32_t        lino = 0;
741,565,254✔
6262
  SReaderStatus* pStatus = NULL;
741,565,254✔
6263
  bool           acquired = false;
741,565,254✔
6264

741,565,254✔
6265
  STsdbReader* pReader = (STsdbReader*)p;
6266
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
741,565,254✔
6267
  TSDB_CHECK_NULL(hasNext, code, lino, _end, TSDB_CODE_INVALID_PARA);
741,565,254!
6268

741,565,254!
6269
  *hasNext = false;
6270

741,565,254✔
6271
  code = pReader->code;
6272
  TSDB_CHECK_CODE(code, lino, _end);
741,792,930✔
6273

741,780,143!
6274
  if (isEmptyQueryTimeWindow(&pReader->info.window) || pReader->step == EXTERNAL_ROWS_NEXT) {
6275
    goto _end;
741,780,143✔
6276
  }
2,258,048✔
6277

6278
  pStatus = &pReader->status;
6279

739,492,170✔
6280
  // NOTE: the following codes is used to perform test for suspend/resume for tsdbReader when it blocks the commit
6281
  // the data should be ingested in round-robin and all the child tables should be createted before ingesting data
6282
  // the version range of query will be used to identify the correctness of suspend/resume functions.
6283
  // this function will be blocked before loading the SECOND block from vnode-buffer, and restart itself from sst-files
6284
#if SUSPEND_RESUME_TEST
6285
  if (!pReader->status.suspendInvoked && !pReader->status.loadFromFile) {
6286
    tsem_wait(&pReader->resumeAfterSuspend);
6287
  }
6288
#endif
6289

6290
  code = tsdbAcquireReader(pReader);
6291
  TSDB_CHECK_CODE(code, lino, _end);
739,529,387✔
6292
  acquired = true;
739,267,007!
6293

739,267,007✔
6294
  tsdbTrace("tsdb/read: %p, take read mutex, code: %d", pReader, code);
6295

739,267,007✔
6296
  if (pReader->flag == READER_STATUS_SUSPEND) {
6297
    code = tsdbReaderResume2(pReader);
739,267,007✔
6298
    TSDB_CHECK_CODE(code, lino, _end);
167,079,952✔
6299
  }
167,078,113!
6300

6301
  if (pReader->innerReader[0] != NULL && pReader->step == 0) {
6302
    code = doTsdbNextDataBlock2(pReader->innerReader[0], hasNext);
739,334,172✔
6303
    TSDB_CHECK_CODE(code, lino, _end);
2,094,559✔
6304

451,621✔
6305
    pReader->step = EXTERNAL_ROWS_PREV;
6306
    if (*hasNext) {
1,642,938✔
6307
      pStatus = &pReader->innerReader[0]->status;
1,642,938!
6308
      if (pStatus->composedDataBlock) {
6309
        tsdbTrace("tsdb/read: %p, unlock read mutex", pReader);
6310
        code = tsdbReleaseReader(pReader);
2,094,559✔
6311
        acquired = false;
2,094,559!
6312
        TSDB_CHECK_CODE(code, lino, _end);
1,410,609✔
6313
      }
1,410,609!
6314

1,410,609✔
6315
      return code;
1,410,609✔
6316
    }
1,410,609✔
6317
  }
1,410,609!
6318

6319
  if (pReader->step == EXTERNAL_ROWS_PREV) {
6320
    // prepare for the main scan
1,410,609✔
6321
    if (tSimpleHashGetSize(pReader->status.pTableMap) > 0) {
6322
      code = doOpenReaderImpl(pReader);
6323
    }
6324

737,983,577✔
6325
    int32_t step = 1;
2,094,559✔
6326
    resetAllDataBlockScanInfo(pReader->status.pTableMap, pReader->innerReader[0]->info.window.ekey, step);
6327
    TSDB_CHECK_CODE(code, lino, _end);
1,552,079!
6328

1,552,079✔
6329
    pReader->step = EXTERNAL_ROWS_MAIN;
6330
  }
6331

1,552,079✔
6332
  code = doTsdbNextDataBlock2(pReader, hasNext);
1,552,079✔
6333
  TSDB_CHECK_CODE(code, lino, _end);
1,552,079!
6334

6335
  if (*hasNext) {
6336
    if (pStatus->composedDataBlock) {
2,094,559✔
6337
      tsdbTrace("tsdb/read: %p, unlock read mutex", pReader);
6338
      code = tsdbReleaseReader(pReader);
6339
      acquired = false;
6340
      TSDB_CHECK_CODE(code, lino, _end);
737,943,810✔
6341
    }
737,520,460✔
6342
    return code;
737,225,615!
6343
  }
6344

542,480✔
6345
  if (pReader->step == EXTERNAL_ROWS_MAIN && pReader->innerReader[1] != NULL) {
6346
    // prepare for the next row scan
6347
    if (tSimpleHashGetSize(pReader->status.pTableMap) > 0) {
737,768,095✔
6348
      code = doOpenReaderImpl(pReader->innerReader[1]);
565,771,235✔
6349
    }
529,049,282✔
6350

529,049,282✔
6351
    int32_t step = -1;
528,740,430✔
6352
    resetAllDataBlockScanInfo(pReader->innerReader[1]->status.pTableMap, pReader->info.window.ekey, step);
528,740,430!
6353
    TSDB_CHECK_CODE(code, lino, _end);
6354

565,664,752✔
6355
    code = doTsdbNextDataBlock2(pReader->innerReader[1], hasNext);
6356
    TSDB_CHECK_CODE(code, lino, _end);
6357

172,113,916!
6358
    pReader->step = EXTERNAL_ROWS_NEXT;
2,094,559✔
6359
    if (*hasNext) {
361,658✔
6360
      pStatus = &pReader->innerReader[1]->status;
6361
      if (pStatus->composedDataBlock) {
6362
        tsdbTrace("tsdb/read: %p, unlock read mutex", pReader);
1,732,901!
6363
        code = tsdbReleaseReader(pReader);
1,732,901✔
6364
        acquired = false;
6365
        TSDB_CHECK_CODE(code, lino, _end);
6366
      }
1,732,901✔
6367

1,732,901✔
6368
      return code;
1,732,901!
6369
    }
6370
  }
1,732,901✔
6371

1,732,901!
6372
  tsdbTrace("tsdb/read: %p, unlock read mutex", pReader);
6373
  code = tsdbReleaseReader(pReader);
6374
  acquired = false;
2,094,559✔
6375
  TSDB_CHECK_CODE(code, lino, _end);
2,094,559!
6376

1,471,162✔
6377
_end:
1,471,162!
6378
  if (code != TSDB_CODE_SUCCESS) {
1,471,162✔
6379
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
1,471,162✔
6380
  }
1,471,162✔
6381
  if (acquired) {
1,471,162!
6382
    tsdbTrace("tsdb/read: %p, unlock read mutex", pReader);
6383
    (void)tsdbReleaseReader(pReader);
6384
  }
1,471,162✔
6385
  return code;
6386
}
6387

6388
static int32_t doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols,
170,463,922✔
6389
                                SColumnDataAgg* pTsAgg) {
170,463,922✔
6390
  // do fill all null column value SMA info
170,452,820✔
6391
  int32_t code = TSDB_CODE_SUCCESS;
170,452,820!
6392
  int32_t lino = 0;
6393
  int32_t i = 0, j = 0;
170,452,820✔
6394
  int32_t size = 0;
172,710,868!
6395

×
6396
  TSDB_CHECK_NULL(pSup, code, lino, _end, TSDB_CODE_INVALID_PARA);
6397

172,710,367!
6398
  size = (int32_t)TARRAY2_SIZE(&pSup->colAggArray);
×
6399
  code = TARRAY2_INSERT_PTR(&pSup->colAggArray, 0, pTsAgg);
×
6400
  TSDB_CHECK_CODE(code, lino, _end);
6401

172,718,527✔
6402
  size++;
6403

6404
  while (j < numOfCols && i < size) {
13,538,371✔
6405
    SColumnDataAgg* pAgg = &pSup->colAggArray.data[i];
6406
    if (pAgg->colId == pSup->colId[j]) {
6407
      i += 1;
13,538,371✔
6408
      j += 1;
13,538,371✔
6409
    } else if (pAgg->colId < pSup->colId[j]) {
13,538,371✔
6410
      i += 1;
13,538,371✔
6411
    } else if (pSup->colId[j] < pAgg->colId) {
6412
      if (pSup->colId[j] != PRIMARYKEY_TIMESTAMP_COL_ID) {
13,538,371!
6413
        SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows};
6414
        code = TARRAY2_INSERT_PTR(&pSup->colAggArray, i, &nullColAgg);
13,538,371✔
6415
        TSDB_CHECK_CODE(code, lino, _end);
13,538,115!
6416

13,538,371!
6417
        i += 1;
6418
        size++;
13,538,371✔
6419
      }
6420
      j += 1;
94,315,835✔
6421
    }
80,777,976✔
6422
  }
80,776,794✔
6423

48,672,755✔
6424
  while (j < numOfCols) {
48,672,755✔
6425
    if (pSup->colId[j] != PRIMARYKEY_TIMESTAMP_COL_ID) {
32,104,315!
6426
      SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows};
32,104,965✔
6427
      code = TARRAY2_INSERT_PTR(&pSup->colAggArray, i, &nullColAgg);
×
6428
      TSDB_CHECK_CODE(code, lino, _end);
×
6429

×
6430
      i += 1;
×
6431
    }
×
6432
    j++;
6433
  }
×
6434

×
6435
_end:
6436
  if (code != TSDB_CODE_SUCCESS) {
×
6437
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
6438
  }
6439
  return code;
6440
}
13,538,729✔
6441

870!
6442
int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool* hasNullSMA) {
870✔
6443
  int32_t             code = TSDB_CODE_SUCCESS;
870!
6444
  int32_t             lino = 0;
870!
6445
  SColumnDataAgg**    pBlockSMA = NULL;
6446
  SFileDataBlockInfo* pBlockInfo = NULL;
870✔
6447

6448
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
870✔
6449
  TSDB_CHECK_NULL(pDataBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
6450
  TSDB_CHECK_NULL(allHave, code, lino, _end, TSDB_CODE_INVALID_PARA);
6451

13,537,859✔
6452
  pBlockSMA = &pDataBlock->pBlockAgg;
13,537,859!
6453
  *allHave = false;
×
6454
  *pBlockSMA = NULL;
6455

13,537,721✔
6456
  if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) {
6457
    goto _end;
6458
  }
105,860,932✔
6459

105,860,932✔
6460
  // there is no statistics data for composed block
105,860,932✔
6461
  if (pReader->status.composedDataBlock || (!pReader->suppInfo.smaValid)) {
105,860,932✔
6462
    goto _end;
105,860,932✔
6463
  }
6464

105,866,076!
6465
  code = getCurrentBlockInfo(&pReader->status.blockIter, &pBlockInfo, pReader->idStr);
105,866,076!
6466
  TSDB_CHECK_CODE(code, lino, _end);
105,866,076!
6467

6468
  SBlockLoadSuppInfo* pSup = &pReader->suppInfo;
105,866,076✔
6469

105,859,734✔
6470
  SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock;
105,874,780✔
6471
  if (pResBlock->info.id.uid != pBlockInfo->uid) {
6472
    goto _end;
105,864,219✔
6473
  }
2,970✔
6474

6475
  //  int64_t st = taosGetTimestampUs();
6476
  TARRAY2_CLEAR(&pSup->colAggArray, 0);
6477

105,855,248!
6478
  SBrinRecord pRecord;
92,307,399✔
6479
  blockInfoToRecord(&pRecord, pBlockInfo, pSup);
6480
  code = tsdbDataFileReadBlockSma(pReader->pFileReader, &pRecord, &pSup->colAggArray);
6481
  if (code != TSDB_CODE_SUCCESS) {
13,538,371✔
6482
    tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pBlockInfo->uid, tstrerror(code),
13,538,371!
6483
              pReader->idStr);
6484
    TSDB_CHECK_CODE(code, lino, _end);
13,538,371✔
6485
  }
6486

13,538,371✔
6487
  if (pSup->colAggArray.size > 0) {
13,538,371!
6488
    *allHave = true;
×
6489
  } else {
6490
    *pBlockSMA = NULL;
6491
    goto _end;
6492
  }
13,538,371✔
6493

6494
  // always load the first primary timestamp column data
13,538,371✔
6495
  SColumnDataAgg* pTsAgg = &pSup->tsColAgg;
13,537,977✔
6496

13,538,371✔
6497
  pTsAgg->numOfNull = 0;
13,537,327!
6498
  pTsAgg->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
×
6499
  pTsAgg->min = pResBlock->info.window.skey;
6500
  pTsAgg->max = pResBlock->info.window.ekey;
×
6501

6502
  // update the number of NULL data rows
6503
  size_t numOfCols = pSup->numOfCols;
13,537,327!
6504

13,537,977✔
6505
  if (pResBlock->pBlockAgg == NULL) {
6506
    size_t num = taosArrayGetSize(pResBlock->pDataBlock);
×
6507
    pResBlock->pBlockAgg = taosMemoryCalloc(num, sizeof(SColumnDataAgg));
×
6508
    TSDB_CHECK_NULL(pResBlock->pBlockAgg, code, lino, _end, terrno);
6509
    for (int i = 0; i < num; ++i) {
6510
      pResBlock->pBlockAgg[i].colId = -1;
6511
    }
13,537,977✔
6512
  }
6513

13,537,977✔
6514
  // do fill all null column value SMA info
13,537,583✔
6515
  code = doFillNullColSMA(pSup, pBlockInfo->numRow, numOfCols, pTsAgg);
13,537,721✔
6516
  TSDB_CHECK_CODE(code, lino, _end);
13,537,465✔
6517

6518
  size_t size = pSup->colAggArray.size;
6519

13,537,977✔
6520
  int32_t i = 0, j = 0;
6521
  while (j < numOfCols && i < size) {
13,537,859!
6522
    SColumnDataAgg* pAgg = &pSup->colAggArray.data[i];
13,537,859✔
6523
    if (pAgg->colId == pSup->colId[j]) {
13,537,465!
6524
      pResBlock->pBlockAgg[pSup->slotId[j]] = *pAgg;
13,537,327!
6525
      i += 1;
65,657,202✔
6526
      j += 1;
52,118,831✔
6527
    } else if (pAgg->colId < pSup->colId[j]) {
6528
      i += 1;
6529
    } else if (pSup->colId[j] < pAgg->colId) {
6530
      pResBlock->pBlockAgg[pSup->slotId[j]].colId = -1;
6531
      *allHave = false;
13,538,115✔
6532
      j += 1;
13,537,721!
6533
    }
6534
  }
13,537,721✔
6535

6536
  *pBlockSMA = pResBlock->pBlockAgg;
13,537,721✔
6537
  pReader->cost.smaDataLoad += 1;
94,325,531✔
6538

80,781,644✔
6539
  //  double elapsedTime = (taosGetTimestampUs() - st) / 1000.0;
80,781,526✔
6540
  pReader->cost.smaLoadTime += 0;  // elapsedTime;
48,679,673✔
6541

48,683,869✔
6542
  tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pBlockInfo->uid, pReader->idStr);
48,683,869✔
6543

32,103,685!
6544
_end:
32,103,941✔
6545
  if (code != TSDB_CODE_SUCCESS) {
×
6546
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
6547
  }
×
6548
  return code;
×
6549
}
6550

6551
static int32_t doRetrieveDataBlock(STsdbReader* pReader, SSDataBlock** pBlock) {
6552
  int32_t              code = TSDB_CODE_SUCCESS;
13,543,887✔
6553
  int32_t              lino = 0;
13,538,371✔
6554
  SReaderStatus*       pStatus = NULL;
6555
  SFileDataBlockInfo*  pBlockInfo = NULL;
6556
  STableBlockScanInfo* pBlockScanInfo = NULL;
13,537,583✔
6557
  bool                 reset = false;
6558

13,537,465!
6559
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
6560
  TSDB_CHECK_NULL(pBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
105,847,834✔
6561

105,860,242!
6562
  pStatus = &pReader->status;
×
6563
  *pBlock = NULL;
6564

105,860,242✔
6565
  code = getCurrentBlockInfo(&pStatus->blockIter, &pBlockInfo, pReader->idStr);
6566
  TSDB_CHECK_CODE(code, lino, _end);
6567

24,817,747✔
6568
  code = pReader->code;
24,817,747✔
6569
  TSDB_CHECK_CODE(code, lino, _end);
24,817,747✔
6570

24,817,747✔
6571
  code = getTableBlockScanInfo(pStatus->pTableMap, pBlockInfo->uid, &pBlockScanInfo, pReader->idStr);
24,817,747✔
6572
  TSDB_CHECK_CODE(code, lino, _end);
24,820,893✔
6573

24,818,957✔
6574
  reset = true;
6575
  code = doLoadFileBlockData(pReader, &pStatus->blockIter, &pStatus->fileBlockData, pBlockScanInfo->uid);
24,818,957!
6576
  TSDB_CHECK_CODE(code, lino, _end);
24,818,957!
6577

6578
  code = copyBlockDataToSDataBlock(pReader, &pBlockScanInfo->lastProcKey);
24,818,957✔
6579
  TSDB_CHECK_CODE(code, lino, _end);
24,824,765✔
6580

6581
  *pBlock = pReader->resBlockInfo.pResBlock;
24,824,765✔
6582

24,813,086!
6583
_end:
6584
  if (code != TSDB_CODE_SUCCESS) {
24,813,086✔
6585
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
24,823,008!
6586
    if (reset) {
6587
      tBlockDataReset(&pStatus->fileBlockData);
24,823,008✔
6588
    }
24,825,554!
6589
  }
6590
  return code;
24,825,554✔
6591
}
24,825,554✔
6592

24,820,956!
6593
int32_t tsdbRetrieveDataBlock2(void* p, SSDataBlock** pBlock, SArray* pIdList) {
6594
  int32_t      code = TSDB_CODE_SUCCESS;
24,820,956✔
6595
  int32_t      lino = 0;
24,821,924!
6596
  STsdbReader* pTReader = NULL;
6597

24,821,924✔
6598
  STsdbReader* pReader = (STsdbReader*)p;
6599
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
24,822,408✔
6600
  TSDB_CHECK_NULL(pBlock, code, lino, _end, TSDB_CODE_INVALID_PARA);
24,823,860!
6601

×
6602
  *pBlock = NULL;
×
6603

×
6604
  pTReader = pReader;
6605
  if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) {
6606
    if (pReader->step == EXTERNAL_ROWS_PREV) {
24,823,860✔
6607
      pTReader = pReader->innerReader[0];
6608
    } else if (pReader->step == EXTERNAL_ROWS_NEXT) {
6609
      pTReader = pReader->innerReader[1];
530,233,847✔
6610
    }
530,233,847✔
6611
  }
530,233,847✔
6612

530,233,847✔
6613
  SReaderStatus* pStatus = &pTReader->status;
6614
  if (pStatus->composedDataBlock || pReader->info.execMode == READER_EXEC_ROWS) {
530,233,847✔
6615
    //    tsdbReaderSuspend2(pReader);
530,233,847!
6616
    //    tsdbReaderResume2(pReader);
530,233,847!
6617
    *pBlock = pTReader->resBlockInfo.pResBlock;
6618
    goto _end;
530,233,847✔
6619
  }
6620

530,606,692✔
6621
  code = doRetrieveDataBlock(pTReader, pBlock);
530,606,692✔
6622

3,943,161✔
6623
  tsdbTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader);
1,410,609✔
6624
  (void)tsdbReleaseReader(pReader);
2,532,552✔
6625
  TSDB_CHECK_CODE(code, lino, _end);
1,471,162✔
6626

6627
  //  tsdbReaderSuspend2(pReader);
6628
  //  tsdbReaderResume2(pReader);
6629
_end:
530,538,072✔
6630
  if (code != TSDB_CODE_SUCCESS) {
530,479,832!
6631
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
6632
  }
6633
  return code;
505,859,490✔
6634
}
505,791,756✔
6635

6636
int32_t tsdbReaderReset2(void* p, SQueryTableDataCond* pCond) {
6637
  int32_t code = TSDB_CODE_SUCCESS;
24,817,747✔
6638
  int32_t lino = 0;
6639
  bool    acquired = false;
24,818,778✔
6640

24,818,778✔
6641
  STsdbReader* pReader = (STsdbReader*)p;
24,818,536!
6642
  tsdbTrace("tsdb/reader-reset: %p, take read mutex", pReader);
6643
  code = tsdbAcquireReader(pReader);
6644
  TSDB_CHECK_CODE(code, lino, _end);
6645
  acquired = true;
24,818,536✔
6646

530,610,292!
6647
  if (pReader->flag == READER_STATUS_SUSPEND) {
×
6648
    code = tsdbReaderResume2(pReader);
6649
    TSDB_CHECK_CODE(code, lino, _end);
530,216,277✔
6650
  }
6651

6652
  if (isEmptyQueryTimeWindow(&pReader->info.window) || pReader->pReadSnap == NULL) {
30,243,963✔
6653
    tsdbDebug("tsdb reader reset return %p, %s", pReader->pReadSnap, pReader->idStr);
30,243,963✔
6654
    code = tsdbReleaseReader(pReader);
30,243,963✔
6655
    acquired = false;
30,243,963✔
6656
    TSDB_CHECK_CODE(code, lino, _end);
6657
    goto _end;
30,243,963✔
6658
  }
30,243,963!
6659

30,243,963✔
6660
  SReaderStatus*  pStatus = &pReader->status;
30,245,006!
6661
  SDataBlockIter* pBlockIter = &pStatus->blockIter;
30,245,006✔
6662

6663
  TSDB_CHECK_NULL(pCond, code, lino, _end, TSDB_CODE_INVALID_PARA);
30,245,006✔
6664
  pReader->info.order = pCond->order;
27,251,573✔
6665
  pReader->type = TIMEWINDOW_RANGE_CONTAINED;
27,251,573!
6666
  code = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows, &pReader->info.window);
6667
  TSDB_CHECK_CODE(code, lino, _end);
6668
  pStatus->loadFromFile = true;
30,245,006✔
6669
  pStatus->pTableIter = NULL;
32,215!
6670

32,215✔
6671
  // allocate buffer in order to load data blocks from file
31,597✔
6672
  memset(&pReader->suppInfo.tsColAgg, 0, sizeof(SColumnDataAgg));
31,597!
6673

31,597✔
6674
  pReader->suppInfo.tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID;
6675
  tsdbDataFileReaderClose(&pReader->pFileReader);
6676

30,212,791✔
6677
  int32_t numOfTables = tSimpleHashGetSize(pStatus->pTableMap);
30,212,860✔
6678

6679
  code = initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->pfSetArray, pReader);
30,213,409!
6680
  TSDB_CHECK_CODE(code, lino, _end);
30,213,409✔
6681

30,212,860✔
6682
  code = resetDataBlockIterator(pBlockIter, pReader->info.order, shouldFreePkBuf(&pReader->suppInfo), pReader->idStr);
30,212,281✔
6683
  TSDB_CHECK_CODE(code, lino, _end);
30,210,574!
6684

30,210,574✔
6685
  code = resetTableListIndex(&pReader->status, pReader->idStr);
30,211,123✔
6686
  TSDB_CHECK_CODE(code, lino, _end);
6687

6688
  bool    asc = ASCENDING_TRAVERSE(pReader->info.order);
30,211,564!
6689
  int32_t step = asc ? 1 : -1;
6690
  int64_t ts = 0;
30,211,833✔
6691
  if (asc) {
30,212,258✔
6692
    ts = (pReader->info.window.skey > INT64_MIN) ? pReader->info.window.skey - 1 : pReader->info.window.skey;
6693
  } else {
30,212,807✔
6694
    ts = (pReader->info.window.ekey < INT64_MAX) ? pReader->info.window.ekey + 1 : pReader->info.window.ekey;
6695
  }
30,212,327✔
6696
  resetAllDataBlockScanInfo(pStatus->pTableMap, ts, step);
30,212,197!
6697

6698
  // no data in files, let's try buffer in memory
30,212,197✔
6699
  if (pStatus->fileIter.numOfFiles == 0) {
30,211,763!
6700
    pStatus->loadFromFile = false;
6701
    code = resetTableListIndex(pStatus, pReader->idStr);
30,211,763✔
6702
    TSDB_CHECK_CODE(code, lino, _end);
30,212,501!
6703
  } else {
6704
    code = initForFirstBlockInFile(pReader, pBlockIter);
30,212,501✔
6705
    if (code != TSDB_CODE_SUCCESS) {
30,212,036✔
6706
      tsdbError("%p reset reader failed, numOfTables:%d, query range:%" PRId64 " - %" PRId64 " in query %s", pReader,
30,212,036✔
6707
                numOfTables, pReader->info.window.skey, pReader->info.window.ekey, pReader->idStr);
30,212,036✔
6708
      TSDB_CHECK_CODE(code, lino, _end);
29,490,569!
6709
    }
6710
  }
721,467✔
6711

6712
  tsdbDebug("%p reset reader, suid:%" PRIu64 ", numOfTables:%d, skey:%" PRId64 ", query range:%" PRId64 " - %" PRId64
30,209,528✔
6713
            " in query %s",
6714
            pReader, pReader->info.suid, numOfTables, pCond->twindows.skey, pReader->info.window.skey,
6715
            pReader->info.window.ekey, pReader->idStr);
30,212,542✔
6716

28,158,356✔
6717
  code = tsdbReleaseReader(pReader);
28,158,264✔
6718
  acquired = false;
28,158,866!
6719
  TSDB_CHECK_CODE(code, lino, _end);
6720

2,053,257✔
6721
_end:
2,050,998!
6722
  if (code != TSDB_CODE_SUCCESS) {
×
6723
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
6724
  }
×
6725
  if (acquired) {
6726
    (void)tsdbReleaseReader(pReader);
6727
  }
6728
  return code;
30,209,864✔
6729
}
6730

6731
static int32_t getBucketIndex(int32_t startRow, int32_t bucketRange, int32_t numOfRows, int32_t numOfBucket) {
6732
  if (numOfRows < startRow) {
6733
    return 0;
30,209,864✔
6734
  }
30,210,630✔
6735
  int32_t bucketIndex = ((numOfRows - startRow) / bucketRange);
30,210,630!
6736
  if (bucketIndex == numOfBucket) {
6737
    bucketIndex -= 1;
30,210,630✔
6738
  }
30,242,227!
6739
  return bucketIndex;
×
6740
}
6741

30,242,442!
NEW
6742
void tsdbGetDataBlock(STsdbReader* pReader, SSDataBlock** pBlock) {
×
6743
  *pBlock = pReader->resBlockInfo.pResBlock;
6744
}
30,243,340✔
6745

6746
void tsdbSetDataBlock(STsdbReader* pReader, SSDataBlock* pBlock) {
6747
  pReader->resBlockInfo.pResBlock = pBlock;
81,883✔
6748
  if (pBlock) {
81,883!
NEW
6749
    pReader->status.pPrimaryTsCol = taosArrayGet(pBlock->pDataBlock,  pReader->suppInfo.slotId[0]);
×
6750
  }
6751
}
81,883!
6752

81,883!
UNCOV
6753
int32_t tsdbGetFileBlocksDistInfo2(STsdbReader* pReader, STableBlockDistInfo* pTableBlockInfo) {
×
6754
  int32_t       code = TSDB_CODE_SUCCESS;
6755
  int32_t       lino = 0;
81,883✔
6756
  const int32_t numOfBuckets = 20.0;
6757
  bool          acquired = false;
6758

15,587✔
6759
  TSDB_CHECK_NULL(pTableBlockInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
15,587✔
6760

15,587✔
6761
  pTableBlockInfo->totalSize = 0;
15,587✔
6762
  pTableBlockInfo->totalRows = 0;
15,587✔
6763
  pTableBlockInfo->numOfVgroups = 1;
6764

15,587!
6765
  // find the start data block in file
6766
  code = tsdbAcquireReader(pReader);
15,587✔
6767
  TSDB_CHECK_CODE(code, lino, _end);
16,237✔
6768
  acquired = true;
16,237✔
6769

6770
  if (pReader->flag == READER_STATUS_SUSPEND) {
6771
    code = tsdbReaderResume2(pReader);
16,237✔
6772
    TSDB_CHECK_CODE(code, lino, _end);
16,237!
6773
  }
16,237✔
6774

6775
  SMergeTreeConf conf = {
16,237!
6776
      .pReader = pReader,
16,237✔
6777
      .pSchema = pReader->info.pSchema,
16,237!
6778
      .pCols = pReader->suppInfo.colId,
6779
      .numOfCols = pReader->suppInfo.numOfCols,
6780
      .suid = pReader->info.suid,
16,237✔
6781
  };
6782

16,237✔
6783
  SReaderStatus* pStatus = &pReader->status;
16,237✔
6784
  if (pStatus->pCurrentFileset != NULL) {
15,732✔
6785
    pTableBlockInfo->numOfSttRows += tsdbGetRowsInSttFiles(pStatus->pCurrentFileset, pStatus->pLDataIterArray,
16,237✔
6786
                                                           pReader->pTsdb, &conf, pReader->idStr);
6787
  }
6788

16,237✔
6789
  STsdbCfg* pc = &pReader->pTsdb->pVnode->config.tsdbCfg;
16,237✔
6790
  pTableBlockInfo->defMinRows = pc->minRows;
7,700✔
6791
  pTableBlockInfo->defMaxRows = pc->maxRows;
7,700✔
6792

6793
  int32_t bucketRange = ceil(((double)(pc->maxRows - pc->minRows)) / numOfBuckets);
6794

15,732✔
6795
  pTableBlockInfo->numOfFiles += 1;
16,237✔
6796

16,237✔
6797
  int32_t numOfTables = (int32_t)tSimpleHashGetSize(pStatus->pTableMap);
6798

16,237!
6799
  SDataBlockIter* pBlockIter = &pStatus->blockIter;
6800
  pTableBlockInfo->numOfFiles += pStatus->fileIter.numOfFiles;
16,237✔
6801

6802
  if (pBlockIter->numOfBlocks > 0) {
15,732✔
6803
    pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks;
6804
  }
15,732✔
6805

15,732✔
6806
  pTableBlockInfo->numOfTables = numOfTables;
6807
  bool hasNext = (pBlockIter->numOfBlocks > 0);
15,732✔
6808

1,981✔
6809
  while (true) {
6810
    if (hasNext) {
6811
      SFileDataBlockInfo* pBlockInfo = NULL;
15,732✔
6812
      code = getCurrentBlockInfo(pBlockIter, &pBlockInfo, pReader->idStr);
16,237✔
6813
      TSDB_CHECK_CODE(code, lino, _end);
6814

6815
      int32_t numOfRows = pBlockInfo->numRow;
111,897✔
6816

81,883✔
6817
      pTableBlockInfo->totalRows += numOfRows;
81,883✔
6818

81,883!
6819
      if (numOfRows > pTableBlockInfo->maxRows) {
6820
        pTableBlockInfo->maxRows = numOfRows;
81,883✔
6821
      }
6822

81,883✔
6823
      if (numOfRows < pTableBlockInfo->minRows) {
6824
        pTableBlockInfo->minRows = numOfRows;
81,883✔
6825
      }
3,139✔
6826

6827
      pTableBlockInfo->totalSize += pBlockInfo->blockSize;
6828

81,883✔
6829
      int32_t bucketIndex = getBucketIndex(pTableBlockInfo->defMinRows, bucketRange, numOfRows, numOfBuckets);
3,139✔
6830
      pTableBlockInfo->blockRowsHisto[bucketIndex]++;
6831

6832
      hasNext = blockIteratorNext(&pStatus->blockIter);
81,883✔
6833
    } else {
6834
      code = initForFirstBlockInFile(pReader, pBlockIter);
81,883✔
6835
      TSDB_CHECK_CODE(code, lino, _end);
81,883✔
6836
      if (pStatus->loadFromFile == false) {
6837
        break;
81,883✔
6838
      }
6839

30,014✔
6840
      // add the data in stt files of new fileset
30,519!
6841
      if (pStatus->pCurrentFileset != NULL) {
30,519!
6842
        pTableBlockInfo->numOfSttRows += tsdbGetRowsInSttFiles(pStatus->pCurrentFileset, pStatus->pLDataIterArray,
16,237✔
6843
                                                               pReader->pTsdb, &conf, pReader->idStr);
6844
      }
6845

6846
      pTableBlockInfo->numOfBlocks += pBlockIter->numOfBlocks;
14,282!
6847
      hasNext = (pBlockIter->numOfBlocks > 0);
14,282✔
6848
    }
14,282✔
6849
  }
6850

6851
  // record the data in stt files
14,282✔
6852
_end:
14,282✔
6853
  if (code != TSDB_CODE_SUCCESS) {
6854
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
6855
  }
6856
  if (acquired) {
6857
    (void)tsdbReleaseReader(pReader);
16,237✔
6858
  }
16,237!
UNCOV
6859
  return code;
×
6860
}
6861

16,237!
6862
static void getMemTableTimeRange(STsdbReader* pReader, int64_t* pMaxKey, int64_t* pMinKey) {
16,237✔
6863
  SReaderStatus* pStatus = &pReader->status;
6864

15,732✔
6865
  int32_t iter = 0;
6866
  int64_t maxKey = INT64_MIN;
6867
  int64_t minKey = INT64_MAX;
27,339,516✔
6868

27,339,516✔
6869
  void* pHashIter = tSimpleHashIterate(pStatus->pTableMap, NULL, &iter);
6870
  while (pHashIter != NULL) {
27,341,884✔
6871
    STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)pHashIter;
27,336,099✔
6872

27,336,099✔
6873
    STbData* d = NULL;
6874
    if (pReader->pReadSnap->pMem != NULL) {
27,336,099✔
6875
      d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->info.suid, pBlockScanInfo->uid);
129,721,275✔
6876
      if (d != NULL) {
102,374,949✔
6877
        if (d->maxKey > maxKey) {
6878
          maxKey = d->maxKey;
102,368,533✔
6879
        }
102,368,533✔
6880
        if (d->minKey < minKey) {
91,344,119✔
6881
          minKey = d->minKey;
91,352,197✔
6882
        }
86,154,040✔
6883
      }
20,918,222✔
6884
    }
6885

86,152,332✔
6886
    STbData* di = NULL;
20,077,914✔
6887
    if (pReader->pReadSnap->pIMem != NULL) {
6888
      di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->info.suid, pBlockScanInfo->uid);
6889
      if (di != NULL) {
6890
        if (di->maxKey > maxKey) {
6891
          maxKey = di->maxKey;
102,377,362✔
6892
        }
102,377,362✔
6893
        if (di->minKey < minKey) {
5,582✔
6894
          minKey = di->minKey;
5,582!
6895
        }
5,582✔
6896
      }
3,262✔
6897
    }
6898

5,582✔
6899
    // current table is exhausted, let's try the next table
3,918✔
6900
    pHashIter = tSimpleHashIterate(pStatus->pTableMap, pHashIter, &iter);
6901
  }
6902

6903
  *pMaxKey = maxKey;
6904
  *pMinKey = minKey;
6905
}
102,379,766✔
6906

6907
int64_t tsdbGetNumOfRowsInMemTable2(STsdbReader* pReader, uint32_t* rows) {
6908
  int32_t        code = TSDB_CODE_SUCCESS;
27,346,326✔
6909
  int32_t        lino = 0;
27,346,730✔
6910
  SReaderStatus* pStatus = NULL;
27,344,022✔
6911
  bool           acquired = false;
6912

16,237✔
6913
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
16,237✔
6914
  TSDB_CHECK_NULL(rows, code, lino, _end, TSDB_CODE_INVALID_PARA);
16,237✔
6915

16,237✔
6916
  *rows = 0;
16,237✔
6917
  pStatus = &pReader->status;
6918
  code = tsdbAcquireReader(pReader);
16,237!
6919
  TSDB_CHECK_CODE(code, lino, _end);
16,237!
6920
  acquired = true;
6921

16,237✔
6922
  if (pReader->flag == READER_STATUS_SUSPEND) {
16,237✔
6923
    code = tsdbReaderResume2(pReader);
16,237✔
6924
    TSDB_CHECK_CODE(code, lino, _end);
16,237!
6925
  }
16,237✔
6926

6927
  int32_t iter = 0;
16,237✔
6928
  pStatus->pTableIter = tSimpleHashIterate(pStatus->pTableMap, NULL, &iter);
4,434✔
6929

4,434!
6930
  while (pStatus->pTableIter != NULL) {
6931
    STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter;
6932

16,237✔
6933
    STbData* d = NULL;
16,237✔
6934
    if (pReader->pReadSnap->pMem != NULL) {
6935
      d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->info.suid, pBlockScanInfo->uid);
75,396✔
6936
      if (d != NULL) {
59,159✔
6937
        *rows += tsdbGetNRowsInTbData(d);
6938
      }
58,654✔
6939
    }
58,654✔
6940

6,187✔
6941
    STbData* di = NULL;
6,692✔
6942
    if (pReader->pReadSnap->pIMem != NULL) {
5,920✔
6943
      di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->info.suid, pBlockScanInfo->uid);
6944
      if (di != NULL) {
6945
        *rows += tsdbGetNRowsInTbData(di);
6946
      }
59,159✔
6947
    }
59,159✔
6948

1,741✔
6949
    // current table is exhausted, let's try the next table
1,741!
6950
    pStatus->pTableIter = tSimpleHashIterate(pStatus->pTableMap, pStatus->pTableIter, &iter);
1,741✔
6951
  }
6952

6953
_end:
6954
  if (code != TSDB_CODE_SUCCESS) {
6955
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
59,159✔
6956
  }
6957
  if (acquired) {
6958
    (void)tsdbReleaseReader(pReader);
16,237✔
6959
  }
16,237!
UNCOV
6960
  return code;
×
6961
}
6962

16,237!
6963
int32_t tsdbGetTableSchema(SMeta* pMeta, int64_t uid, STSchema** pSchema, int64_t* suid, SSchemaWrapper** pTagSchema) {
16,237✔
6964
  SMetaReader mr = {0};
6965
  metaReaderDoInit(&mr, pMeta, META_READER_LOCK);
16,237✔
6966
  int32_t code = metaReaderGetTableEntryByUidCache(&mr, uid);
6967
  if (code != TSDB_CODE_SUCCESS) {
6968
    code = TSDB_CODE_TDB_INVALID_TABLE_ID;
80,312✔
6969
    metaReaderClear(&mr);
80,312✔
6970
    return code;
80,312✔
6971
  }
80,312✔
6972

80,312!
UNCOV
6973
  *suid = 0;
×
UNCOV
6974

×
UNCOV
6975
  if (mr.me.type == TSDB_CHILD_TABLE) {
×
6976
    tDecoderClear(&mr.coder);
6977
    *suid = mr.me.ctbEntry.suid;
6978
    code = metaReaderGetTableEntryByUidCache(&mr, *suid);
80,312✔
6979
    if (code != TSDB_CODE_SUCCESS) {
6980
      code = TSDB_CODE_TDB_INVALID_TABLE_ID;
80,312✔
6981
      metaReaderClear(&mr);
26,825✔
6982
      return code;
26,825✔
6983
    }
26,825✔
6984
  } else if (mr.me.type == TSDB_NORMAL_TABLE) {  // do nothing
26,825!
6985
  } else if (mr.me.type == TSDB_SUPER_TABLE) {
×
6986
    tDecoderClear(&mr.coder);
×
UNCOV
6987
    *suid = uid;
×
6988
    code = metaReaderGetTableEntryByUidCache(&mr, *suid);
6989
    if (code != TSDB_CODE_SUCCESS) {
53,487✔
6990
      code = TSDB_CODE_TDB_INVALID_TABLE_ID;
2,952!
6991
      metaReaderClear(&mr);
2,952✔
6992
      return code;
2,952✔
6993
    }
2,952✔
6994
    *pTagSchema = tCloneSSchemaWrapper(&mr.me.stbEntry.schemaTag);
2,952!
UNCOV
6995
  } else {
×
6996
    code = TSDB_CODE_INVALID_PARA;
×
6997
    tsdbError("invalid mr.me.type:%d, code:%s", mr.me.type, tstrerror(code));
×
6998
    metaReaderClear(&mr);
6999
    return code;
2,952✔
7000
  }
UNCOV
7001

×
UNCOV
7002
  metaReaderClear(&mr);
×
UNCOV
7003

×
UNCOV
7004
  // get the newest table schema version
×
7005
  code = metaGetTbTSchemaEx(pMeta, *suid, uid, -1, pSchema);
7006
  return code;
7007
}
80,312✔
7008

7009
int32_t tsdbTakeReadSnap2(STsdbReader* pReader, _query_reseek_func_t reseek, STsdbReadSnap** ppSnap, const char* id) {
7010
  int32_t        code = 0;
80,312✔
7011
  STsdb*         pTsdb = pReader->pTsdb;
80,312✔
7012
  SVersionRange* pRange = &pReader->info.verRange;
7013
  int32_t        lino = 0;
7014
  *ppSnap = NULL;
174,656,003✔
7015

174,656,003✔
7016
  // lock
174,656,003✔
7017
  code = taosThreadMutexLock(&pTsdb->mutex);
174,691,634✔
7018
  if (code != TSDB_CODE_SUCCESS) {
174,710,771✔
7019
    tsdbError("%s failed to lock tsdb, code:%s", id, tstrerror(code));
174,710,771✔
7020
    return code;
7021
  }
7022

174,675,834✔
7023
  // alloc
174,669,528!
UNCOV
7024
  STsdbReadSnap* pSnap = (STsdbReadSnap*)taosMemoryCalloc(1, sizeof(STsdbReadSnap));
×
UNCOV
7025
  if (pSnap == NULL) {
×
7026
    (void)taosThreadMutexUnlock(&pTsdb->mutex);
7027
    TSDB_CHECK_NULL(pSnap, code, lino, _exit, terrno);
7028
  }
7029

174,669,528✔
7030
  // take snapshot
174,701,650!
UNCOV
7031
  if (pTsdb->mem && (pRange->minVer <= pTsdb->mem->maxVer && pRange->maxVer >= pTsdb->mem->minVer)) {
×
UNCOV
7032
    pSnap->pMem = pTsdb->mem;
×
7033
    pSnap->pNode = taosMemoryMalloc(sizeof(*pSnap->pNode));
7034
    if (pSnap->pNode == NULL) {
7035
      (void)taosThreadMutexUnlock(&pTsdb->mutex);
7036
      TSDB_CHECK_NULL(pSnap->pNode, code, lino, _exit, terrno);
174,701,650!
7037
    }
144,401,348✔
7038

144,406,404✔
7039
    pSnap->pNode->pQHandle = pReader;
144,373,672!
UNCOV
7040
    pSnap->pNode->reseek = reseek;
×
UNCOV
7041

×
7042
    code = tsdbRefMemTable(pTsdb->mem, pSnap->pNode);
7043
    TSDB_CHECK_CODE(code, lino, _exit);
7044
  }
144,384,503✔
7045

144,378,343✔
7046
  if (pTsdb->imem && (pRange->minVer <= pTsdb->imem->maxVer && pRange->maxVer >= pTsdb->imem->minVer)) {
7047
    pSnap->pIMem = pTsdb->imem;
144,395,912✔
7048
    pSnap->pINode = taosMemoryMalloc(sizeof(*pSnap->pINode));
144,380,465!
7049
    if (pSnap->pINode == NULL) {
7050
      code = terrno;
7051

174,655,781!
7052
      if (pTsdb->mem && pSnap->pNode) {
1,504,169✔
7053
        tsdbUnrefMemTable(pTsdb->mem, pSnap->pNode, true);  // unref the previous refed mem
1,504,169!
7054
      }
1,504,169!
UNCOV
7055

×
7056
      (void)taosThreadMutexUnlock(&pTsdb->mutex);
UNCOV
7057
      goto _exit;
×
UNCOV
7058
    }
×
7059

7060
    pSnap->pINode->pQHandle = pReader;
UNCOV
7061
    pSnap->pINode->reseek = reseek;
×
UNCOV
7062

×
7063
    code = tsdbRefMemTable(pTsdb->imem, pSnap->pINode);
7064
    TSDB_CHECK_CODE(code, lino, _exit);
7065
  }
1,504,169✔
7066

1,504,169✔
7067
  // fs
7068
  code = tsdbFSCreateRefSnapshotWithoutLock(pTsdb->pFS, &pSnap->pfSetArray);
1,504,169✔
7069
  if (code) {
1,504,169!
7070
    if (pSnap->pNode) {
7071
      tsdbUnrefMemTable(pTsdb->mem, pSnap->pNode, true);  // unref the previous refed mem
7072
    }
7073

174,697,451✔
7074
    if (pSnap->pINode) {
174,704,156!
UNCOV
7075
      tsdbUnrefMemTable(pTsdb->imem, pSnap->pINode, true);
×
UNCOV
7076
    }
×
7077

7078
    (void)taosThreadMutexUnlock(&pTsdb->mutex);
UNCOV
7079
    TSDB_CHECK_CODE(code, lino, _exit);
×
UNCOV
7080
  }
×
7081

7082
  // unlock
UNCOV
7083
  (void)taosThreadMutexUnlock(&pTsdb->mutex);
×
UNCOV
7084
  *ppSnap = pSnap;
×
7085

7086
  tsdbTrace("%s vgId:%d, take read snapshot", id, TD_VID(pTsdb->pVnode));
7087
  return code;
7088

174,704,156✔
7089
_exit:
174,738,820✔
7090
  tsdbError("%s vgId:%d take read snapshot failed, line:%d code:%s", id, TD_VID(pTsdb->pVnode), lino, tstrerror(code));
7091

174,740,416✔
7092
  if (pSnap) {
174,693,413✔
7093
    if (pSnap->pNode) taosMemoryFree(pSnap->pNode);
7094
    if (pSnap->pINode) taosMemoryFree(pSnap->pINode);
×
7095
    taosMemoryFree(pSnap);
×
7096
  }
UNCOV
7097
  return code;
×
UNCOV
7098
}
×
UNCOV
7099

×
UNCOV
7100
void tsdbUntakeReadSnap2(STsdbReader* pReader, STsdbReadSnap* pSnap, bool proactive) {
×
7101
  STsdb* pTsdb = pReader->pTsdb;
UNCOV
7102

×
7103
  if (pSnap) {
7104
    if (pSnap->pMem) {
7105
      tsdbUnrefMemTable(pSnap->pMem, pSnap->pNode, proactive);
174,722,111✔
7106
    }
174,722,111✔
7107

7108
    if (pSnap->pIMem) {
174,727,580!
7109
      tsdbUnrefMemTable(pSnap->pIMem, pSnap->pINode, proactive);
174,729,149✔
7110
    }
144,415,088✔
7111

7112
    if (pSnap->pNode) taosMemoryFree(pSnap->pNode);
7113
    if (pSnap->pINode) taosMemoryFree(pSnap->pINode);
174,741,847✔
7114

1,504,169✔
7115
    tsdbFSDestroyRefSnapshot(&pSnap->pfSetArray);
7116

7117
    taosMemoryFree(pSnap);
174,731,641✔
7118
  }
174,716,550!
7119
  tsdbTrace("vgId:%d, untake read snapshot", TD_VID(pTsdb->pVnode));
7120
}
174,723,540✔
7121

7122
// if failed, do nothing
174,710,220✔
7123
int32_t tsdbReaderSetId(void* p, const char* idstr) {
7124
  int32_t      code = TSDB_CODE_SUCCESS;
174,684,789✔
7125
  int32_t      lino = 0;
174,684,789✔
7126
  STsdbReader* pReader = (STsdbReader*)p;
7127

7128
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
54,553✔
7129

54,553✔
7130
  taosMemoryFreeClear(pReader->idStr);
54,553✔
7131
  pReader->idStr = taosStrdup(idstr);
54,553✔
7132
  TSDB_CHECK_NULL(pReader->idStr, code, lino, _end, terrno);
7133

54,553!
7134
  pReader->status.fileIter.pSttBlockReader->mergeTree.idStr = pReader->idStr;
7135

54,553!
7136
_end:
54,708!
7137
  if (code != TSDB_CODE_SUCCESS) {
54,547!
7138
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
7139
  }
54,616✔
7140
  return code;
7141
}
54,478✔
7142

54,478!
UNCOV
7143
void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/
×
7144
}
7145

54,478✔
7146
void tsdbSetFilesetDelimited(STsdbReader* pReader) { pReader->bFilesetDelimited = true; }
7147

7148
void tsdbReaderSetNotifyCb(STsdbReader* pReader, TsdReaderNotifyCbFn notifyFn, void* param) {
63,253✔
7149
  pReader->notifyFn = notifyFn;
63,253✔
7150
  pReader->notifyParam = param;
7151
}
27,328,438✔
7152

7153
#if 0
26,974,405✔
7154
static int32_t tsdbEncodeLastProcKeys(const SSHashObj* pTableMap, void** buf) {
26,974,405✔
7155
  void* p = NULL;
26,981,385✔
7156
  int32_t iter = 0;
26,976,794✔
7157
  int32_t len = 0;
7158

7159
  len += taosEncodeFixedI32(buf, tSimpleHashGetSize(pTableMap)); // number of tables
7160

7161
  while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) {
7162
    uint64_t uid = *(uint64_t*)tSimpleHashGetKey(p, NULL);
7163
    SRowKey* pKey = &(*(STableBlockScanInfo**)p)->lastProcKey;
7164

7165
    len += taosEncodeFixedU64(buf, uid);            // table uid
7166
    len += taosEncodeFixedI64(buf, pKey->ts);       // last read ts
7167
    len += taosEncodeFixedU8(buf, pKey->numOfPKs);  // number of pks
7168

7169
    if (pKey->numOfPKs > 0) {
7170
      int8_t type = pKey->pks[0].type;
7171
      len += taosEncodeFixedI8(buf, type);  // pk data type
7172

7173
      uint8_t* pData = 0;
7174
      uint32_t nData = 0;
7175
      if (IS_VAR_DATA_TYPE(type)) {
7176
        pData = pKey->pks[0].pData;
7177
        nData = pKey->pks[0].nData;
7178
        len += taosEncodeFixedU32(buf, nData);  // data length
7179
      } else {
7180
        pData = (uint8_t*)&pKey->pks[0].val;
7181
        nData = tDataTypes[pKey->pks[0].type].bytes;
7182
      }
7183
      len += taosEncodeBinary(buf, pData, nData); // data value
7184
    }
7185
  }
7186

7187
  return len;
7188
}
7189

7190
static int32_t tsdbDecodeLastProcKeys(const void* buf, int32_t len, SSHashObj* pTableMap) {
7191
  int32_t code = TSDB_CODE_SUCCESS;
7192
  int32_t lino = 0;
7193
  const void *begin = buf;
7194

7195
  TSDB_CHECK_NULL(buf, code, lino, _end, TSDB_CODE_INVALID_PARA);
7196
  TSDB_CHECK_NULL(pTableMap, code, lino, _end, TSDB_CODE_INVALID_PARA);
7197

7198
  int32_t numOfTables = 0;
7199
  buf = taosDecodeFixedI32(buf, &numOfTables);
7200

7201
  TSDB_CHECK_CONDITION(numOfTables == tSimpleHashGetSize(pTableMap), code, lino, _end, TSDB_CODE_INVALID_PARA);
7202

7203
  for (int32_t i = 0; i < numOfTables; ++i) {
7204
    uint64_t uid = 0;
7205
    buf = taosDecodeFixedU64(buf, &uid);
7206

7207
    STableBlockScanInfo** pScanInfo = tSimpleHashGet(pTableMap, &uid, sizeof(uid));
7208
    if (pScanInfo == NULL) {
7209
      tsdbError("failed to find scan info of uid: %" PRIu64 " when decoding last proc keys", uid);
7210
      TSDB_CHECK_NULL(pScanInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
7211
    }
7212
    SRowKey* pKey = &(*pScanInfo)->lastProcKey;
7213

7214
    buf = taosDecodeFixedI64(buf, &pKey->ts);
7215
    uint8_t numOfPks = 0;
7216
    buf = taosDecodeFixedU8(buf, &numOfPks);
7217
    if (numOfPks != pKey->numOfPKs) {
7218
      tsdbError("numOfPks of uid %" PRIu64 " mismatch: [%u/%u]", uid, numOfPks, pKey->numOfPKs);
7219
      TSDB_CHECK_CONDITION(numOfPks == pKey->numOfPKs, code, lino, _end, TSDB_CODE_INVALID_PARA);
7220
    }
7221

7222
    if (numOfPks > 0) {
7223
      int8_t type = 0;
7224
      buf = taosDecodeFixedI8(buf, &type);
7225
      if (type != pKey->pks[0].type) {
7226
        tsdbError("pk type of uid %" PRIu64 " mismatch: [%d/%d]", uid, type, pKey->pks[0].type);
7227
        TSDB_CHECK_CONDITION(type == pKey->pks[0].type, code, lino, _end, TSDB_CODE_INVALID_PARA);
7228
      }
7229

7230
      uint8_t *pData = 0;
7231
      uint32_t nData = 0;
7232
      if (IS_VAR_DATA_TYPE(type)) {
7233
        pData = pKey->pks[0].pData;
7234
        buf = taosDecodeFixedU32(buf, &nData);
7235
        pKey->pks[0].nData = nData;
7236
      } else {
7237
        pKey->pks[0].val = 0;
7238
        pData = (uint8_t*)&pKey->pks[0].val;
7239
        nData = tDataTypes[type].bytes;
7240
      }
7241
      buf = taosDecodeBinary(buf, (void **)&pData, nData);
7242
    }
7243
  }
7244

7245
  TSDB_CHECK_CONDITION(buf == begin + len , code, lino, _end, TSDB_CODE_INVALID_PARA);
7246

7247
_end:
7248
  if (code != TSDB_CODE_SUCCESS) {
7249
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
7250
  }
7251
  return code;
7252
}
7253
#endif
7254

7255
int32_t tsdbReaderGetProgress(const STsdbReader* pReader, void** pBuf, uint64_t* pLen) {
7256
  int32_t code = TSDB_CODE_SUCCESS;
7257
#if 0
7258
  int32_t              lino = 0;
7259
  void*                buf = NULL;
UNCOV
7260
  int32_t len = 0;
×
UNCOV
7261

×
7262
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
7263
  TSDB_CHECK_NULL(pBuf, code, lino, _end, TSDB_CODE_INVALID_PARA);
7264
  TSDB_CHECK_NULL(pLen, code, lino, _end, TSDB_CODE_INVALID_PARA);
7265

7266
  *pBuf = NULL;
7267
  *pLen = tsdbEncodeLastProcKeys(pReader->status.pTableMap, NULL);
7268

7269
  buf = taosMemoryMalloc(*pLen);
7270
  TSDB_CHECK_NULL(buf, code, lino, _end, terrno);
7271
  *pBuf = buf;
7272
  len = tsdbEncodeLastProcKeys(pReader->status.pTableMap, &buf);
7273
  TSDB_CHECK_CONDITION(len == *pLen, code, lino, _end, TSDB_CODE_INTERNAL_ERROR);
7274
  buf = NULL;
7275

7276
_end:
7277
  if (code != TSDB_CODE_SUCCESS) {
7278
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
7279
  }
7280
  if (buf != NULL) {
7281
    taosMemoryFreeClear(buf);
7282
  }
7283
#endif
7284
  return code;
7285
}
7286

7287
int32_t tsdbReaderSetProgress(STsdbReader* pReader, const void* buf, uint64_t len) {
7288
  int32_t code = TSDB_CODE_SUCCESS;
UNCOV
7289
#if 0
×
7290
  int32_t lino = 0;
7291

UNCOV
7292
  if (len == 0) {
×
UNCOV
7293
    goto _end;
×
7294
  }
7295

7296
  TSDB_CHECK_NULL(pReader, code, lino, _end, TSDB_CODE_INVALID_PARA);
7297
  TSDB_CHECK_NULL(buf, code, lino, _end, TSDB_CODE_INVALID_PARA);
7298

7299
  code = tsdbDecodeLastProcKeys(buf, len, pReader->status.pTableMap);
7300
  TSDB_CHECK_CODE(code, lino, _end);
7301

7302
_end:
7303
  if (code != TSDB_CODE_SUCCESS) {
7304
    tsdbError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
7305
  }
7306
#endif
7307
  return code;
7308
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc