• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

taosdata / TDengine / #3971

29 Apr 2025 09:40AM UTC coverage: 62.459% (+0.2%) from 62.291%
#3971

push

travis-ci

web-flow
Merge pull request #30938 from taosdata/fix/taosdCmdArgs

fix: taosd cmd args

155466 of 317399 branches covered (48.98%)

Branch coverage included in aggregate %.

1 of 4 new or added lines in 1 file covered. (25.0%)

336 existing lines in 103 files now uncovered.

240465 of 316504 relevant lines covered (75.98%)

6514139.43 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

62.66
/source/libs/executor/src/scanoperator.c
1
/*
2
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3
 *
4
 * This program is free software: you can use, redistribute, and/or modify
5
 * it under the terms of the GNU Affero General Public License, version 3
6
 * or later ("AGPL"), as published by the Free Software Foundation.
7
 *
8
 * This program is distributed in the hope that it will be useful, but WITHOUT
9
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10
 * FITNESS FOR A PARTICULAR PURPOSE.
11
 *
12
 * You should have received a copy of the GNU Affero General Public License
13
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
14
 */
15

16
#include "executorInt.h"
17
#include "filter.h"
18
#include "function.h"
19
#include "functionMgt.h"
20
#include "os.h"
21
#include "querynodes.h"
22
#include "streamexecutorInt.h"
23
#include "systable.h"
24
#include "tname.h"
25

26
#include "tdatablock.h"
27
#include "tmsg.h"
28
#include "ttime.h"
29

30
#include "operator.h"
31
#include "query.h"
32
#include "querytask.h"
33
#include "tcompare.h"
34
#include "thash.h"
35
#include "ttypes.h"
36

37
#include "storageapi.h"
38
#include "wal.h"
39
#include "function.h"
40

41
int32_t scanDebug = 0;
42

43
#define MULTI_READER_MAX_TABLE_NUM     5000
44
#define SET_REVERSE_SCAN_FLAG(_info)   ((_info)->scanFlag = REVERSE_SCAN)
45
#define SWITCH_ORDER(n)                (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC))
46
#define STREAM_SCAN_OP_NAME            "StreamScanOperator"
47
#define STREAM_SCAN_OP_STATE_NAME      "StreamScanFillHistoryState"
48
#define STREAM_SCAN_OP_CHECKPOINT_NAME "StreamScanOperator_Checkpoint"
49

50
typedef struct STableMergeScanExecInfo {
51
  SFileBlockLoadRecorder blockRecorder;
52
  SSortExecInfo          sortExecInfo;
53
} STableMergeScanExecInfo;
54

55
typedef struct STableMergeScanSortSourceParam {
56
  SOperatorInfo* pOperator;
57
  int32_t        readerIdx;
58
  uint64_t       uid;
59
  STsdbReader*   reader;
60
} STableMergeScanSortSourceParam;
61

62
typedef struct STableCountScanOperatorInfo {
63
  SReadHandle  readHandle;
64
  SSDataBlock* pRes;
65

66
  STableCountScanSupp supp;
67

68
  int32_t currGrpIdx;
69
  SArray* stbUidList;  // when group by db_name and/or stable_name
70
} STableCountScanOperatorInfo;
71

72
static bool    processBlockWithProbability(const SSampleExecInfo* pInfo);
73
static int32_t doTableCountScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes);
74

75
bool processBlockWithProbability(const SSampleExecInfo* pInfo) {
3,362,238✔
76
#if 0
77
  if (pInfo->sampleRatio == 1) {
78
    return true;
79
  }
80

81
  uint32_t val = taosRandR((uint32_t*) &pInfo->seed);
82
  return (val % ((uint32_t)(1/pInfo->sampleRatio))) == 0;
83
#else
84
  return true;
3,362,238✔
85
#endif
86
}
87

88
static void switchCtxOrder(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
191✔
89
  for (int32_t i = 0; i < numOfOutput; ++i) {
191!
90
    SWITCH_ORDER(pCtx[i].order);
×
91
  }
92
}
191✔
93

94
static int32_t overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockInfo, int32_t order, bool* overlap) {
3,182,334✔
95
  int32_t     code = TSDB_CODE_SUCCESS;
3,182,334✔
96
  STimeWindow w = {0};
3,182,334✔
97

98
  // 0 by default, which means it is not a interval operator of the upstream operator.
99
  if (pInterval->interval == 0) {
3,182,334✔
100
    *overlap = false;
2,803,184✔
101
    return code;
2,803,184✔
102
  }
103

104
  if (order == TSDB_ORDER_ASC) {
379,150✔
105
    w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.skey);
360,537✔
106
    if (w.ekey < pBlockInfo->window.skey) {
360,468!
107
      qError("w.ekey:%" PRId64 " < pBlockInfo->window.skey:%" PRId64, w.ekey, pBlockInfo->window.skey);
×
108
      return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
×
109
    }
110

111
    if (w.ekey < pBlockInfo->window.ekey) {
360,468✔
112
      *overlap = true;
326,266✔
113
      return code;
326,266✔
114
    }
115

116
    while (1) {
117
      getNextTimeWindow(pInterval, &w, order);
34,208✔
118
      if (w.skey > pBlockInfo->window.ekey) {
34,163✔
119
        break;
33,640✔
120
      }
121

122
      if (w.ekey <= pBlockInfo->window.ekey) {
523!
123
        qError("w.ekey:%" PRId64 " <= pBlockInfo->window.ekey:%" PRId64, w.ekey, pBlockInfo->window.ekey);
×
124
        return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
×
125
      }
126
      if (TMAX(w.skey, pBlockInfo->window.skey) <= pBlockInfo->window.ekey) {
523✔
127
        *overlap = true;
517✔
128
        return code;
517✔
129
      }
130
    }
131
  } else {
132
    w = getAlignQueryTimeWindow(pInterval, pBlockInfo->window.ekey);
18,613✔
133
    if (w.skey > pBlockInfo->window.ekey) {
19,304!
134
      qError("w.skey:%" PRId64 " > pBlockInfo->window.skey:%" PRId64, w.skey, pBlockInfo->window.ekey);
×
135
      return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
×
136
    }
137

138
    if (w.skey > pBlockInfo->window.skey) {
19,304✔
139
      *overlap = true;
17,701✔
140
      return code;
17,701✔
141
    }
142

143
    while (1) {
144
      getNextTimeWindow(pInterval, &w, order);
1,603✔
145
      if (w.ekey < pBlockInfo->window.skey) {
1,598✔
146
        break;
1,574✔
147
      }
148

149
      if (w.skey >= pBlockInfo->window.skey) {
24!
150
        qError("w.skey:%" PRId64 " >= pBlockInfo->window.skey:%" PRId64, w.skey, pBlockInfo->window.skey);
×
151
        return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
×
152
      }
153
      if (pBlockInfo->window.skey <= TMIN(w.ekey, pBlockInfo->window.ekey)) {
24!
154
        *overlap = true;
24✔
155
        return code;
24✔
156
      }
157
    }
158
  }
159

160
  *overlap = false;
35,214✔
161
  return code;
35,214✔
162
}
163

164
// this function is for table scanner to extract temporary results of upstream aggregate results.
165
static SResultRow* getTableGroupOutputBuf(SOperatorInfo* pOperator, uint64_t groupId, SFilePage** pPage) {
494,678✔
166
  if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
494,678!
167
    return NULL;
×
168
  }
169

170
  int64_t buf[2] = {0};
494,678✔
171
  SET_RES_WINDOW_KEY((char*)buf, &groupId, sizeof(groupId), groupId);
494,678✔
172

173
  STableScanInfo* pTableScanInfo = pOperator->info;
494,678✔
174

175
  SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pTableScanInfo->base.pdInfo.pAggSup->pResultRowHashTable,
494,678✔
176
                                                               buf, GET_RES_WINDOW_KEY_LEN(sizeof(groupId)));
177

178
  if (p1 == NULL) {
495,367✔
179
    return NULL;
273,101✔
180
  }
181

182
  *pPage = getBufPage(pTableScanInfo->base.pdInfo.pAggSup->pResultBuf, p1->pageId);
222,266✔
183
  if (NULL == *pPage) {
222,108!
184
    return NULL;
×
185
  }
186

187
  return (SResultRow*)((char*)(*pPage) + p1->offset);
222,108✔
188
}
189

190
static int32_t insertTableToScanIgnoreList(STableScanInfo* pTableScanInfo, uint64_t uid) {
4,116✔
191
  if (NULL == pTableScanInfo->pIgnoreTables) {
4,116✔
192
    int32_t tableNum = taosArrayGetSize(pTableScanInfo->base.pTableListInfo->pTableList);
2,205✔
193
    pTableScanInfo->pIgnoreTables =
2,205✔
194
        taosHashInit(tableNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
2,205✔
195
    if (NULL == pTableScanInfo->pIgnoreTables) {
2,205!
196
      return terrno;
×
197
    }
198
  }
199

200
  int32_t tempRes = taosHashPut(pTableScanInfo->pIgnoreTables, &uid, sizeof(uid), &pTableScanInfo->scanTimes,
4,116✔
201
                                sizeof(pTableScanInfo->scanTimes));
202
  if (tempRes != TSDB_CODE_SUCCESS && tempRes != TSDB_CODE_DUP_KEY) {
4,120!
203
    qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(tempRes));
×
204
    return tempRes;
×
205
  }
206
  return TSDB_CODE_SUCCESS;
4,120✔
207
}
208

209
static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* pBlockInfo, uint32_t* status) {
3,676,882✔
210
  STableScanInfo* pTableScanInfo = pOperator->info;
3,676,882✔
211
  int32_t         code = TSDB_CODE_SUCCESS;
3,676,882✔
212

213
  if (pTableScanInfo->base.pdInfo.pExprSup == NULL) {
3,676,882✔
214
    return TSDB_CODE_SUCCESS;
3,184,130✔
215
  }
216

217
  SExprSupp* pSup1 = pTableScanInfo->base.pdInfo.pExprSup;
492,752✔
218

219
  SFilePage*  pPage = NULL;
492,752✔
220
  SResultRow* pRow = getTableGroupOutputBuf(pOperator, pBlockInfo->id.groupId, &pPage);
492,752✔
221

222
  if (pRow == NULL) {
495,193✔
223
    return TSDB_CODE_SUCCESS;
273,126✔
224
  }
225

226
  bool notLoadBlock = true;
222,067✔
227
  for (int32_t i = 0; i < pSup1->numOfExprs; ++i) {
242,634✔
228
    int32_t functionId = pSup1->pCtx[i].functionId;
238,524✔
229

230
    SResultRowEntryInfo* pEntry = getResultEntryInfo(pRow, i, pTableScanInfo->base.pdInfo.pExprSup->rowEntryInfoOffset);
238,524✔
231

232
    EFuncDataRequired reqStatus = fmFuncDynDataRequired(functionId, pEntry, pBlockInfo);
238,463✔
233
    if (reqStatus != FUNC_DATA_REQUIRED_NOT_LOAD) {
238,497✔
234
      notLoadBlock = false;
217,930✔
235
      break;
217,930✔
236
    }
237
  }
238

239
  // release buffer pages
240
  releaseBufPage(pTableScanInfo->base.pdInfo.pAggSup->pResultBuf, pPage);
222,040✔
241

242
  if (notLoadBlock) {
221,939✔
243
    *status = FUNC_DATA_REQUIRED_NOT_LOAD;
4,116✔
244
    code = insertTableToScanIgnoreList(pTableScanInfo, pBlockInfo->id.uid);
4,116✔
245
  }
246

247
  return code;
221,976✔
248
}
249

250
static int32_t doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg* pColsAgg, int32_t numOfCols,
31,187✔
251
                                  int32_t numOfRows, bool* keep) {
252
  if (pColsAgg == NULL || pFilterInfo == NULL) {
31,187!
253
    *keep = true;
×
254
    return TSDB_CODE_SUCCESS;
×
255
  }
256

257
  return filterRangeExecute(pFilterInfo, pColsAgg, numOfCols, numOfRows, keep);
31,187✔
258
}
259

260
static int32_t doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo,
930,315✔
261
                              bool* pLoad) {
262
  SStorageAPI* pAPI = &pTaskInfo->storageAPI;
930,315✔
263
  bool         allColumnsHaveAgg = true;
930,315✔
264
  bool         hasNullSMA = false;
930,315✔
265
  if (pLoad != NULL) {
930,315!
266
    *pLoad = false;
930,780✔
267
  }
268

269
  int32_t code = pAPI->tsdReader.tsdReaderRetrieveBlockSMAInfo(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg,
930,315✔
270
                                                               &hasNullSMA);
271
  if (code != TSDB_CODE_SUCCESS) {
930,363!
272
    return code;
×
273
  }
274

275
  if (!allColumnsHaveAgg || hasNullSMA) {
930,363!
276
    *pLoad = false;
886,456✔
277
  } else {
278
    *pLoad = true;
43,907✔
279
  }
280

281
  return code;
930,363✔
282
}
283

284
static int32_t doSetTagColumnData(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo,
4,225,676✔
285
                                  int32_t rows) {
286
  int32_t    code = 0;
4,225,676✔
287
  SExprSupp* pSup = &pTableScanInfo->pseudoSup;
4,225,676✔
288
  if (pSup->numOfExprs > 0) {
4,225,676✔
289
    code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, rows,
1,468,926✔
290
                                  pTaskInfo, &pTableScanInfo->metaCache);
291
    // ignore the table not exists error, since this table may have been dropped during the scan procedure.
292
    if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
1,469,027!
293
      if (pTaskInfo->streamInfo.pState) blockDataCleanup(pBlock);
×
294
      code = 0;
×
295
    }
296
  }
297

298
  return code;
4,226,689✔
299
}
300

301
bool applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) {
7,128,061✔
302
  SLimit*     pLimit = &pLimitInfo->limit;
7,128,061✔
303
  const char* id = GET_TASKID(pTaskInfo);
7,128,061✔
304

305
  if (pLimitInfo->remainOffset > 0) {
7,128,061✔
306
    if (pLimitInfo->remainOffset >= pBlock->info.rows) {
47,283✔
307
      pLimitInfo->remainOffset -= pBlock->info.rows;
21,891✔
308
      blockDataEmpty(pBlock);
21,891✔
309
      qDebug("current block ignore due to offset, current:%" PRId64 ", %s", pLimitInfo->remainOffset, id);
21,891✔
310
      return false;
21,891✔
311
    } else {
312
      int32_t code = blockDataTrimFirstRows(pBlock, pLimitInfo->remainOffset);
25,392✔
313
      if (code != TSDB_CODE_SUCCESS) {
25,392!
314
        qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
×
315
        pTaskInfo->code = code;
×
316
        T_LONG_JMP(pTaskInfo->env, code);
×
317
      }
318
      pLimitInfo->remainOffset = 0;
25,392✔
319
    }
320
  }
321

322
  if (pLimit->limit != -1 && pLimit->limit <= (pLimitInfo->numOfOutputRows + pBlock->info.rows)) {
7,106,170✔
323
    // limit the output rows
324
    int32_t keep = (int32_t)(pLimit->limit - pLimitInfo->numOfOutputRows);
148,469✔
325
    blockDataKeepFirstNRows(pBlock, keep);
148,469✔
326

327
    pLimitInfo->numOfOutputRows += pBlock->info.rows;
148,475✔
328
    qDebug("output limit %" PRId64 " has reached, %s", pLimit->limit, id);
148,475✔
329
    return true;
148,475✔
330
  }
331

332
  pLimitInfo->numOfOutputRows += pBlock->info.rows;
6,957,701✔
333
  return false;
6,957,701✔
334
}
335

336
static bool isDynVtbScan(SOperatorInfo* pOperator) {
1,141✔
337
  return pOperator->dynamicTask && ((STableScanInfo*)(pOperator->info))->virtualStableScan;
1,141!
338
}
339

340
static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableScanInfo, SSDataBlock* pBlock,
3,864,484✔
341
                             uint32_t* status) {
342
  int32_t        code = TSDB_CODE_SUCCESS;
3,864,484✔
343
  int32_t        lino = 0;
3,864,484✔
344
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
3,864,484✔
345
  SStorageAPI*   pAPI = &pTaskInfo->storageAPI;
3,864,484✔
346
  bool           loadSMA = false;
3,864,484✔
347

348
  SFileBlockLoadRecorder* pCost = &pTableScanInfo->readRecorder;
3,864,484✔
349

350
  pCost->totalBlocks += 1;
3,864,484✔
351
  pCost->totalRows += pBlock->info.rows;
3,864,484✔
352
  *status = pTableScanInfo->dataBlockLoadFlag;
3,864,484✔
353

354
  if (pOperator->exprSupp.pFilterInfo != NULL) {
3,864,484✔
355
    (*status) = FUNC_DATA_REQUIRED_DATA_LOAD;
681,677✔
356
  } else {
357
    bool overlap = false;
3,182,807✔
358
    int  ret =
359
        overlapWithTimeWindow(&pTableScanInfo->pdInfo.interval, &pBlock->info, pTableScanInfo->cond.order, &overlap);
3,182,807✔
360
    if (ret != TSDB_CODE_SUCCESS) {
3,182,434!
361
      return ret;
×
362
    }
363
    if (overlap) {
3,182,434✔
364
      (*status) = FUNC_DATA_REQUIRED_DATA_LOAD;
344,522✔
365
    }
366
  }
367

368
  SDataBlockInfo* pBlockInfo = &pBlock->info;
3,864,111✔
369
  taosMemoryFreeClear(pBlock->pBlockAgg);
3,864,111!
370

371
  if (*status == FUNC_DATA_REQUIRED_FILTEROUT) {
3,864,111!
372
    qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo),
×
373
           pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
374
    pCost->filterOutBlocks += 1;
×
375
    pCost->totalRows += pBlock->info.rows;
×
376
    pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader);
×
377
    return TSDB_CODE_SUCCESS;
×
378
  } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) {
3,864,111✔
379
    qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", uid:%" PRIu64,
170,720✔
380
           GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows,
381
           pBlockInfo->id.uid);
382
    code = doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows);
170,720✔
383
    pCost->skipBlocks += 1;
170,702✔
384
    pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader);
170,702✔
385
    return code;
170,725✔
386
  } else if (*status == FUNC_DATA_REQUIRED_SMA_LOAD) {
3,693,391✔
387
    pCost->loadBlockStatis += 1;
249,182✔
388
    loadSMA = true;  // mark the operation of load sma;
249,182✔
389
    bool success = true;
249,182✔
390
    code = doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo, &success);
249,182✔
391
    if (code) {
249,246!
392
      pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader);
×
393
      qError("%s failed to retrieve sma info", GET_TASKID(pTaskInfo));
×
394
      QUERY_CHECK_CODE(code, lino, _end);
×
395
    }
396

397
    if (success) {  // failed to load the block sma data, data block statistics does not exist, load data block instead
249,008✔
398
      qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo),
12,720✔
399
             pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
400
      code = doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows);
12,720✔
401
      pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader);
12,721✔
402
      return code;
15,556✔
403
    } else {
404
      qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo));
236,288✔
405
      *status = FUNC_DATA_REQUIRED_DATA_LOAD;
236,276✔
406
    }
407
  }
408

409
  if (*status != FUNC_DATA_REQUIRED_DATA_LOAD) {
3,680,485!
410
    pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader);
×
411
    qError("%s loadDataBlock invalid status:%d", GET_TASKID(pTaskInfo), *status);
×
412
    return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
×
413
  }
414

415
  // try to filter data block according to sma info
416
  if (pOperator->exprSupp.pFilterInfo != NULL && (!loadSMA)) {
3,680,485!
417
    bool success = true;
681,224✔
418
    code = doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo, &success);
681,224✔
419
    if (code) {
681,463!
420
      pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader);
×
421
      qError("%s failed to retrieve sma info", GET_TASKID(pTaskInfo));
×
422
      QUERY_CHECK_CODE(code, lino, _end);
×
423
    }
424

425
    if (success) {
680,882✔
426
      size_t size = taosArrayGetSize(pBlock->pDataBlock);
31,187✔
427
      bool   keep = false;
31,187✔
428
      code = doFilterByBlockSMA(pOperator->exprSupp.pFilterInfo, pBlock->pBlockAgg, size, pBlockInfo->rows, &keep);
31,187✔
429
      if (code) {
31,187!
430
        pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader);
×
431
        qError("%s failed to do filter by block sma, code:%s", GET_TASKID(pTaskInfo), tstrerror(code));
×
432
        QUERY_CHECK_CODE(code, lino, _end);
×
433
      }
434

435
      if (!keep) {
31,187✔
436
        qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64,
5!
437
               GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
438
        pCost->filterOutBlocks += 1;
5✔
439
        (*status) = FUNC_DATA_REQUIRED_FILTEROUT;
5✔
440
        taosMemoryFreeClear(pBlock->pBlockAgg);
5!
441

442
        pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader);
5✔
443
        return TSDB_CODE_SUCCESS;
5✔
444
      }
445
    }
446
  }
447

448
  // free the sma info, since it should not be involved in *later computing process.
449
  taosMemoryFreeClear(pBlock->pBlockAgg);
3,680,138!
450

451
  // try to filter data block according to current results
452
  code = doDynamicPruneDataBlock(pOperator, pBlockInfo, status);
3,680,138✔
453
  if (code) {
3,679,333✔
454
    pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader);
1,914✔
455
    QUERY_CHECK_CODE(code, lino, _end);
×
456
  }
457

458
  if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) {
3,677,419✔
459
    qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64,
4,118✔
460
           GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows);
461
    pCost->skipBlocks += 1;
4,118✔
462
    pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->dataReader);
4,118✔
463

464
    STableScanInfo* p1 = pOperator->info;
4,119✔
465
    if (taosHashGetSize(p1->pIgnoreTables) == taosArrayGetSize(p1->base.pTableListInfo->pTableList)) {
4,119✔
466
      *status = FUNC_DATA_REQUIRED_ALL_FILTEROUT;
1,400✔
467
    } else {
468
      *status = FUNC_DATA_REQUIRED_FILTEROUT;
2,719✔
469
    }
470
    return TSDB_CODE_SUCCESS;
4,119✔
471
  }
472

473
  pCost->totalCheckedRows += pBlock->info.rows;
3,673,301✔
474
  pCost->loadBlocks += 1;
3,673,301✔
475

476
  SSDataBlock* p = NULL;
3,673,301✔
477
  code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pTableScanInfo->dataReader, &p, NULL);
3,673,301✔
478
  if (p == NULL || code != TSDB_CODE_SUCCESS || p != pBlock) {
3,673,193!
479
    return code;
×
480
  }
481

482
  if ((pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) && ((STableScanInfo *)pOperator->info)->ignoreTag) {
3,675,681!
483
    // do nothing
484
  } else {
485
    // dyn vtb scan do not read tag from origin tables.
486
    code = doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows);
3,675,681✔
487
    if (code) {
3,674,714!
488
      return code;
×
489
    }
490
  }
491

492
  // restore the previous value
493
  pCost->totalRows -= pBlock->info.rows;
3,674,714✔
494

495
  if (pOperator->exprSupp.pFilterInfo != NULL) {
3,674,714✔
496
    code = doFilter(pBlock, pOperator->exprSupp.pFilterInfo, &pTableScanInfo->matchInfo);
681,596✔
497
    QUERY_CHECK_CODE(code, lino, _end);
681,608✔
498

499
    int64_t st = taosGetTimestampUs();
681,486✔
500
    double  el = (taosGetTimestampUs() - st) / 1000.0;
681,665✔
501
    pTableScanInfo->readRecorder.filterTime += el;
681,665✔
502

503
    if (pBlock->info.rows == 0) {
681,665✔
504
      pCost->filterOutBlocks += 1;
106,675✔
505
      qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", elapsed time:%.2f ms",
106,675✔
506
             GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, el);
507
    } else {
508
      qDebug("%s data block filter applied, elapsed time:%.2f ms", GET_TASKID(pTaskInfo), el);
574,990✔
509
    }
510
  }
511

512
  bool limitReached = applyLimitOffset(&pTableScanInfo->limitInfo, pBlock, pTaskInfo);
3,674,787✔
513
  if (limitReached) {  // set operator flag is done
3,672,584✔
514
    setOperatorCompleted(pOperator);
24,981✔
515
  }
516

517
  pCost->totalRows += pBlock->info.rows;
3,674,657✔
518

519
_end:
3,674,658✔
520
  if (code != TSDB_CODE_SUCCESS) {
3,674,658✔
521
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
1!
522
  }
523
  return code;
3,674,352✔
524
}
525

526
static void prepareForDescendingScan(STableScanBase* pTableScanInfo, SqlFunctionCtx* pCtx, int32_t numOfOutput) {
191✔
527
  SET_REVERSE_SCAN_FLAG(pTableScanInfo);
191✔
528

529
  switchCtxOrder(pCtx, numOfOutput);
191✔
530
  pTableScanInfo->cond.order = TSDB_ORDER_DESC;
191✔
531
  STimeWindow* pTWindow = &pTableScanInfo->cond.twindows;
191✔
532
  TSWAP(pTWindow->skey, pTWindow->ekey);
191✔
533
}
191✔
534

535
typedef struct STableCachedVal {
536
  const char* pName;
537
  STag*       pTags;
538
} STableCachedVal;
539

540
static void freeTableCachedVal(void* param) {
1,036,499✔
541
  if (param == NULL) {
1,036,499!
542
    return;
×
543
  }
544

545
  STableCachedVal* pVal = param;
1,036,499✔
546
  taosMemoryFree((void*)pVal->pName);
1,036,499!
547
  taosMemoryFree(pVal->pTags);
1,037,860!
548
  taosMemoryFree(pVal);
1,037,845!
549
}
550

551
static int32_t createTableCacheVal(const SMetaReader* pMetaReader, STableCachedVal** ppResVal) {
1,037,101✔
552
  int32_t          code = TSDB_CODE_SUCCESS;
1,037,101✔
553
  int32_t          lino = 0;
1,037,101✔
554
  STableCachedVal* pVal = taosMemoryMalloc(sizeof(STableCachedVal));
1,037,101!
555
  QUERY_CHECK_NULL(pVal, code, lino, _end, terrno);
1,037,456!
556

557
  pVal->pTags = NULL;
1,037,456✔
558
  pVal->pName = taosStrdup(pMetaReader->me.name);
1,037,456!
559
  QUERY_CHECK_NULL(pVal->pName, code, lino, _end, terrno);
1,037,200!
560

561
  // only child table has tag value
562
  if (pMetaReader->me.type == TSDB_CHILD_TABLE || pMetaReader->me.type == TSDB_VIRTUAL_CHILD_TABLE) {
1,037,200✔
563
    STag* pTag = (STag*)pMetaReader->me.ctbEntry.pTags;
1,035,800✔
564
    pVal->pTags = taosMemoryMalloc(pTag->len);
1,035,800✔
565
    QUERY_CHECK_NULL(pVal->pTags, code, lino, _end, terrno);
1,034,926!
566
    memcpy(pVal->pTags, pTag, pTag->len);
1,034,926✔
567
  }
568

569
  (*ppResVal) = pVal;
1,036,326✔
570

571
_end:
1,036,326✔
572
  if (code != TSDB_CODE_SUCCESS) {
1,036,326!
573
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
574
    freeTableCachedVal(pVal);
×
575
  }
576
  return code;
1,036,385✔
577
}
578

579
// const void *key, size_t keyLen, void *value
580
static void freeCachedMetaItem(const void* key, size_t keyLen, void* value, void* ud) {
1,036,422✔
581
  (void)key;
582
  (void)keyLen;
583
  (void)ud;
584
  freeTableCachedVal(value);
1,036,422✔
585
}
1,037,841✔
586

587
static void doSetNullValue(SSDataBlock* pBlock, const SExprInfo* pExpr, int32_t numOfExpr) {
×
588
  for (int32_t j = 0; j < numOfExpr; ++j) {
×
589
    int32_t dstSlotId = pExpr[j].base.resSchema.slotId;
×
590

591
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotId);
×
592
    colDataSetNNULL(pColInfoData, 0, pBlock->info.rows);
×
593
  }
594
}
×
595

596
static void freeTableCachedValObj(STableCachedVal* pVal) {
×
597
  taosMemoryFree((void*)pVal->pName);
×
598
  taosMemoryFree(pVal->pTags);
×
599
}
×
600

601
int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock,
1,845,634✔
602
                               int32_t rows, SExecTaskInfo* pTask, STableMetaCacheInfo* pCache) {
603
  int32_t          code = TSDB_CODE_SUCCESS;
1,845,634✔
604
  int32_t          lino = 0;
1,845,634✔
605
  bool             freeReader = false;
1,845,634✔
606
  LRUHandle*       h = NULL;
1,845,634✔
607
  STableCachedVal  val = {0};
1,845,634✔
608
  SMetaReader      mr = {0};
1,845,634✔
609
  const char*      idStr = pTask->id.str;
1,845,634✔
610
  int32_t          insertRet = TAOS_LRU_STATUS_OK;
1,845,634✔
611
  STableCachedVal* pVal = NULL;
1,845,634✔
612

613
  // currently only the tbname pseudo column
614
  if (numOfExpr <= 0) {
1,845,634✔
615
    return TSDB_CODE_SUCCESS;
5,274✔
616
  }
617

618
  // todo: opt if only require the vgId and the vgVer;
619

620
  // backup the rows
621
  int32_t backupRows = pBlock->info.rows;
1,840,360✔
622
  pBlock->info.rows = rows;
1,840,360✔
623

624
  // todo refactor: extract method
625
  // the handling of the null data should be packed in the extracted method
626

627
  // 1. check if it is existed in meta cache
628
  if (pCache == NULL) {
1,840,360✔
629
    pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, META_READER_LOCK, &pHandle->api.metaFn);
870✔
630
    code = pHandle->api.metaReaderFn.getEntryGetUidCache(&mr, pBlock->info.id.uid);
870✔
631
    if (code != TSDB_CODE_SUCCESS) {
870!
632
      // when encounter the TSDB_CODE_PAR_TABLE_NOT_EXIST error, we proceed.
633
      if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
×
634
        qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s",
×
635
              pBlock->info.id.uid, tstrerror(code), idStr);
636

637
        // append null value before return to caller, since the caller will ignore this error code and proceed
638
        doSetNullValue(pBlock, pExpr, numOfExpr);
×
639
      } else {
640
        qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(code), idStr);
×
641
      }
642
      pHandle->api.metaReaderFn.clearReader(&mr);
×
643
      return code;
×
644
    }
645

646
    pHandle->api.metaReaderFn.readerReleaseLock(&mr);
870✔
647

648
    val.pName = mr.me.name;
870✔
649
    val.pTags = (STag*)mr.me.ctbEntry.pTags;
870✔
650

651
    freeReader = true;
870✔
652
  } else {
653
    pCache->metaFetch += 1;
1,839,490✔
654

655
    h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid));
1,839,490✔
656
    if (h == NULL) {
1,840,700✔
657
      pHandle->api.metaReaderFn.initReader(&mr, pHandle->vnode, META_READER_LOCK, &pHandle->api.metaFn);
1,037,808✔
658
      freeReader = true;
1,037,698✔
659
      code = pHandle->api.metaReaderFn.getEntryGetUidCache(&mr, pBlock->info.id.uid);
1,037,698✔
660
      if (code != TSDB_CODE_SUCCESS) {
1,036,448!
661
        if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
×
662
          qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s",
×
663
                pBlock->info.id.uid, tstrerror(code), idStr);
664
          // append null value before return to caller, since the caller will ignore this error code and proceed
665
          doSetNullValue(pBlock, pExpr, numOfExpr);
×
666
        } else {
667
          qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(code),
×
668
                 idStr);
669
        }
670
        pHandle->api.metaReaderFn.clearReader(&mr);
×
671
        return code;
×
672
      }
673

674
      pHandle->api.metaReaderFn.readerReleaseLock(&mr);
1,036,448✔
675

676
      code = createTableCacheVal(&mr, &pVal);
1,037,450✔
677
      QUERY_CHECK_CODE(code, lino, _end);
1,036,256!
678

679
      val = *pVal;
1,036,256✔
680
    } else {
681
      pCache->cacheHit += 1;
802,892✔
682
      STableCachedVal* pValTmp = taosLRUCacheValue(pCache->pTableMetaEntryCache, h);
802,892✔
683
      val = *pValTmp;
802,749✔
684

685
      bool bRes = taosLRUCacheRelease(pCache->pTableMetaEntryCache, h, false);
802,749✔
686
      qTrace("release LRU cache, res %d", bRes);
803,040✔
687
    }
688

689
    qDebug("retrieve table meta from cache:%" PRIu64 ", hit:%" PRIu64 " miss:%" PRIu64 ", %s", pCache->metaFetch,
1,839,296✔
690
           pCache->cacheHit, (pCache->metaFetch - pCache->cacheHit), idStr);
691
  }
692

693
  for (int32_t j = 0; j < numOfExpr; ++j) {
7,151,460✔
694
    const SExprInfo* pExpr1 = &pExpr[j];
5,309,638✔
695
    int32_t          dstSlotId = pExpr1->base.resSchema.slotId;
5,309,638✔
696

697
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotId);
5,309,638✔
698
    colInfoDataCleanup(pColInfoData, pBlock->info.rows);
5,307,079✔
699

700
    int32_t functionId = pExpr1->pExpr->_function.functionId;
5,306,001✔
701

702
    // this is to handle the tbname
703
    if (fmIsScanPseudoColumnFunc(functionId)) {
5,306,001✔
704
      int32_t fType = pExpr1->pExpr->_function.functionType;
849,404✔
705
      if (fType == FUNCTION_TYPE_TBNAME) {
849,404✔
706
        code = setTbNameColData(pBlock, pColInfoData, functionId, val.pName);
840,912✔
707
        QUERY_CHECK_CODE(code, lino, _end);
841,395!
708
      } else if (fType == FUNCTION_TYPE_VGID) {
8,492✔
709
        code = setVgIdColData(pBlock, pColInfoData, functionId, pTask->id.vgId);
4,247✔
710
        QUERY_CHECK_CODE(code, lino, _end);
4,246!
711
      } else if (fType == FUNCTION_TYPE_VGVER) {
4,245!
712
        code = setVgVerColData(pBlock, pColInfoData, functionId, pBlock->info.version);
4,245✔
713
        QUERY_CHECK_CODE(code, lino, _end);
4,247!
714
      }
715
    } else {  // these are tags
716
      STagVal tagVal = {0};
4,456,578✔
717
      tagVal.cid = pExpr1->base.pParam[0].pCol->colId;
4,456,578✔
718
      const char* p = pHandle->api.metaFn.extractTagVal(val.pTags, pColInfoData->info.type, &tagVal);
4,456,578✔
719

720
      char* data = NULL;
4,456,642✔
721
      if (pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL) {
4,456,642✔
722
        data = tTagValToData((const STagVal*)p, false);
4,409,776✔
723
      } else {
724
        data = (char*)p;
46,866✔
725
      }
726

727
      bool isNullVal = (data == NULL) || (pColInfoData->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data));
4,457,577!
728
      if (isNullVal) {
4,457,577✔
729
        colDataSetNNULL(pColInfoData, 0, pBlock->info.rows);
44,450✔
730
      } else if (pColInfoData->info.type != TSDB_DATA_TYPE_JSON) {
4,413,127✔
731
        code = colDataSetNItems(pColInfoData, 0, data, pBlock->info.rows, false);
4,409,893✔
732
        if (IS_VAR_DATA_TYPE(((const STagVal*)p)->type)) {
4,414,154!
733
          char* tmp = taosMemoryCalloc(1, varDataLen(data) + 1);
1,524,954!
734
          if (tmp != NULL){
1,527,530!
735
            memcpy(tmp, varDataVal(data), varDataLen(data));
1,527,561✔
736
            qDebug("get tag value:%s, cid:%d, table name:%s, uid%"PRId64, tmp, tagVal.cid, val.pName, pBlock->info.id.uid);
1,527,561✔
737
            taosMemoryFree(tmp);
1,527,565✔
738
          }
739
          taosMemoryFree(data);
1,527,406!
740
        }
741
        QUERY_CHECK_CODE(code, lino, _end);
4,413,727!
742
      } else {  // todo opt for json tag
743
        for (int32_t i = 0; i < pBlock->info.rows; ++i) {
6,118✔
744
          code = colDataSetVal(pColInfoData, i, data, false);
2,884✔
745
          QUERY_CHECK_CODE(code, lino, _end);
2,884!
746
        }
747
      }
748
    }
749
  }
750

751
  // restore the rows
752
  pBlock->info.rows = backupRows;
1,841,822✔
753

754
_end:
1,841,822✔
755

756
  if (NULL != pVal) {
1,841,822✔
757
    insertRet = taosLRUCacheInsert(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(uint64_t), pVal,
1,037,324✔
758
                                   sizeof(STableCachedVal), freeCachedMetaItem, NULL, NULL, TAOS_LRU_PRIORITY_LOW, NULL);
759
    if (insertRet != TAOS_LRU_STATUS_OK) {
1,037,259!
760
      qWarn("failed to put meta into lru cache, code:%d, %s", insertRet, idStr);
×
761
    }
762
  }
763

764
  if (freeReader) {
1,841,757✔
765
    pHandle->api.metaReaderFn.clearReader(&mr);
1,038,254✔
766
  }
767
  if (code != TSDB_CODE_SUCCESS) {
1,841,792!
768
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
769
  }
770
  return code;
1,840,810✔
771
}
772

773
int32_t setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId,
840,734✔
774
                         const char* name) {
775
  int32_t                     code = TSDB_CODE_SUCCESS;
840,734✔
776
  int32_t                     lino = 0;
840,734✔
777
  struct SScalarFuncExecFuncs fpSet = {0};
840,734✔
778
  code = fmGetScalarFuncExecFuncs(functionId, &fpSet);
840,734✔
779
  QUERY_CHECK_CODE(code, lino, _end);
840,764!
780

781
  size_t len = TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE;
840,764✔
782
  char   buf[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
840,764✔
783
  STR_TO_VARSTR(buf, name)
840,764✔
784

785
  SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, len, 1);
840,764✔
786

787
  code = colInfoDataEnsureCapacity(&infoData, 1, false);
841,807✔
788
  QUERY_CHECK_CODE(code, lino, _end);
841,505!
789

790
  code = colDataSetVal(&infoData, 0, buf, false);
841,505✔
791
  QUERY_CHECK_CODE(code, lino, _end);
841,507!
792

793
  SScalarParam srcParam = {.numOfRows = pBlock->info.rows, .columnData = &infoData};
841,507✔
794
  SScalarParam param = {.columnData = pColInfoData};
841,507✔
795

796
  if (fpSet.process != NULL) {
841,507✔
797
    code = fpSet.process(&srcParam, 1, &param);
841,492✔
798
    QUERY_CHECK_CODE(code, lino, _end);
841,809!
799
  } else {
800
    qError("failed to get the corresponding callback function, functionId:%d", functionId);
15!
801
  }
802

803
  colDataDestroy(&infoData);
841,824✔
804

805
_end:
841,535✔
806
  if (code != TSDB_CODE_SUCCESS) {
841,535!
807
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
808
  }
809
  return code;
841,416✔
810
}
811

812
int32_t setVgIdColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int32_t vgId) {
4,247✔
813
  int32_t                     code = TSDB_CODE_SUCCESS;
4,247✔
814
  int32_t                     lino = 0;
4,247✔
815
  struct SScalarFuncExecFuncs fpSet = {0};
4,247✔
816
  code = fmGetScalarFuncExecFuncs(functionId, &fpSet);
4,247✔
817
  QUERY_CHECK_CODE(code, lino, _end);
4,247!
818

819
  SColumnInfoData infoData = createColumnInfoData(pColInfoData->info.type, pColInfoData->info.bytes, 1);
4,247✔
820

821
  code = colInfoDataEnsureCapacity(&infoData, 1, false);
4,247✔
822
  QUERY_CHECK_CODE(code, lino, _end);
4,247!
823

824
  code = colDataSetVal(&infoData, 0, (const char*)&vgId, false);
4,247✔
825
  QUERY_CHECK_CODE(code, lino, _end);
4,247!
826

827
  SScalarParam srcParam = {.numOfRows = pBlock->info.rows, .columnData = &infoData};
4,247✔
828
  SScalarParam param = {.columnData = pColInfoData};
4,247✔
829

830
  if (fpSet.process != NULL) {
4,247!
831
    code = fpSet.process(&srcParam, 1, &param);
4,247✔
832
    QUERY_CHECK_CODE(code, lino, _end);
4,247!
833
  } else {
834
    qError("failed to get the corresponding callback function, functionId:%d", functionId);
×
835
  }
836

837
_end:
×
838
  colDataDestroy(&infoData);
4,247✔
839
  if (code != TSDB_CODE_SUCCESS) {
4,246!
840
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
841
  }
842
  return code;
4,246✔
843
}
844

845
int32_t setVgVerColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int64_t vgVer) {
4,245✔
846
  int32_t                     code = TSDB_CODE_SUCCESS;
4,245✔
847
  int32_t                     lino = 0;
4,245✔
848
  struct SScalarFuncExecFuncs fpSet = {0};
4,245✔
849
  code = fmGetScalarFuncExecFuncs(functionId, &fpSet);
4,245✔
850
  QUERY_CHECK_CODE(code, lino, _end);
4,245!
851

852
  SColumnInfoData infoData = createColumnInfoData(pColInfoData->info.type, pColInfoData->info.bytes, 1);
4,245✔
853

854
  code = colInfoDataEnsureCapacity(&infoData, 1, false);
4,247✔
855
  QUERY_CHECK_CODE(code, lino, _end);
4,247!
856

857
  code = colDataSetVal(&infoData, 0, (const char*)&vgVer, false);
4,247✔
858
  QUERY_CHECK_CODE(code, lino, _end);
4,247!
859

860
  SScalarParam srcParam = {.numOfRows = pBlock->info.rows, .columnData = &infoData};
4,247✔
861
  SScalarParam param = {.columnData = pColInfoData};
4,247✔
862

863
  if (fpSet.process != NULL) {
4,247✔
864
    code = fpSet.process(&srcParam, 1, &param);
4,246✔
865
    QUERY_CHECK_CODE(code, lino, _end);
4,247!
866
  } else {
867
    qError("failed to get the corresponding callback function, functionId:%d", functionId);
1!
868
  }
869

870
_end:
1✔
871
  colDataDestroy(&infoData);
4,248✔
872
  if (code != TSDB_CODE_SUCCESS) {
4,247!
873
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
874
  }
875
  return code;
4,247✔
876
}
877

878
static int32_t initNextGroupScan(STableScanInfo* pInfo, STableKeyInfo** pKeyInfo, int32_t* size) {
1,129,640✔
879
  int32_t code = TSDB_CODE_SUCCESS;
1,129,640✔
880
  int32_t lino = 0;
1,129,640✔
881
  code = tableListGetGroupList(pInfo->base.pTableListInfo, pInfo->currentGroupId, pKeyInfo, size);
1,129,640✔
882
  QUERY_CHECK_CODE(code, lino, _end);
1,131,108!
883

884
  pInfo->tableStartIndex = TARRAY_ELEM_IDX(pInfo->base.pTableListInfo->pTableList, *pKeyInfo);
1,131,108✔
885
  pInfo->tableEndIndex = (pInfo->tableStartIndex + (*size) - 1);
1,131,108✔
886
  pInfo->pResBlock->info.blankFill = false;
1,131,108✔
887
  taosMemoryFreeClear(pInfo->pResBlock->pBlockAgg);
1,131,108!
888

889
  if (!pInfo->needCountEmptyTable) {
1,131,108✔
890
    pInfo->countState = TABLE_COUNT_STATE_END;
998,642✔
891
  } else {
892
    pInfo->countState = TABLE_COUNT_STATE_SCAN;
132,466✔
893
  }
894

895
_end:
1,131,108✔
896
  if (code != TSDB_CODE_SUCCESS) {
1,131,108!
897
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
898
  }
899
  return code;
1,130,545✔
900
}
901

902
void markGroupProcessed(STableScanInfo* pInfo, uint64_t groupId) {
3,616,211✔
903
  if (pInfo->countState == TABLE_COUNT_STATE_END) {
3,616,211✔
904
    return;
2,975,672✔
905
  }
906
  if (pInfo->base.pTableListInfo->groupOffset) {
640,539✔
907
    pInfo->countState = TABLE_COUNT_STATE_PROCESSED;
406✔
908
  } else {
909
    int32_t code = taosHashRemove(pInfo->base.pTableListInfo->remainGroups, &groupId, sizeof(groupId));
640,133✔
910
    if (code != TSDB_CODE_SUCCESS) {
641,814✔
911
      qDebug("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
135,276✔
912
    }
913
  }
914
}
915

916
static SSDataBlock* getOneRowResultBlock(SExecTaskInfo* pTaskInfo, STableScanBase* pBase, SSDataBlock* pBlock,
368,201✔
917
                                         const STableKeyInfo* tbInfo) {
918
  blockDataEmpty(pBlock);
368,201✔
919
  pBlock->info.rows = 1;
368,287✔
920
  pBlock->info.id.uid = tbInfo->uid;
368,287✔
921
  pBlock->info.id.groupId = tbInfo->groupId;
368,287✔
922
  pBlock->info.blankFill = true;
368,287✔
923

924
  // only one row: set all col data to null & hasNull
925
  int32_t col_num = blockDataGetNumOfCols(pBlock);
368,287✔
926
  for (int32_t i = 0; i < col_num; ++i) {
1,488,193✔
927
    SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
1,120,323✔
928
    colDataSetNULL(pColInfoData, 0);
929
  }
930

931
  // set tag/tbname
932
  terrno = doSetTagColumnData(pBase, pBlock, pTaskInfo, 1);
367,870✔
933

934
  return pBlock;
368,035✔
935
}
936

937
static SSDataBlock* getBlockForEmptyTable(SOperatorInfo* pOperator, const STableKeyInfo* tbInfo) {
353,719✔
938
  STableScanInfo* pTableScanInfo = pOperator->info;
353,719✔
939
  SSDataBlock*    pBlock =
940
      getOneRowResultBlock(pOperator->pTaskInfo, &pTableScanInfo->base, pTableScanInfo->pResBlock, tbInfo);
353,719✔
941

942
  pOperator->resultInfo.totalRows++;
353,595✔
943
  return pBlock;
353,595✔
944
}
945

946
static int32_t doTableScanImplNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
4,366,868✔
947
  int32_t         code = TSDB_CODE_SUCCESS;
4,366,868✔
948
  int32_t         lino = 0;
4,366,868✔
949
  STableScanInfo* pTableScanInfo = pOperator->info;
4,366,868✔
950
  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
4,366,868✔
951
  SStorageAPI*    pAPI = &pTaskInfo->storageAPI;
4,366,868✔
952
  SSDataBlock*    pBlock = pTableScanInfo->pResBlock;
4,366,868✔
953
  bool            hasNext = false;
4,366,868✔
954
  int64_t         st = taosGetTimestampUs();
4,368,364✔
955

956
  QRY_PARAM_CHECK(ppRes);
4,368,364!
957
  pBlock->info.dataLoad = false;
4,368,364✔
958

959
  while (true) {
95,039✔
960
    code = pAPI->tsdReader.tsdNextDataBlock(pTableScanInfo->base.dataReader, &hasNext);
4,463,403✔
961
    if (code != TSDB_CODE_SUCCESS) {
4,463,275!
962
      pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->base.dataReader);
×
963
      QUERY_CHECK_CODE(code, lino, _end);
809!
964
    }
965

966
    if (!hasNext) {
4,464,792✔
967
      break;
1,102,504✔
968
    }
969

970
    if (isTaskKilled(pTaskInfo)) {
3,364,527✔
971
      pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->base.dataReader);
232✔
972
      code = pTaskInfo->code;
804✔
973
      goto _end;
804✔
974
    }
975

976
    if (pOperator->status == OP_EXEC_DONE) {
3,364,772✔
977
      pAPI->tsdReader.tsdReaderReleaseDataBlock(pTableScanInfo->base.dataReader);
840✔
978
      break;
840✔
979
    }
980

981
    // process this data block based on the probabilities
982
    bool processThisBlock = processBlockWithProbability(&pTableScanInfo->sample);
3,363,932✔
983
    if (!processThisBlock) {
3,362,758!
984
      continue;
95,039✔
985
    }
986

987
    if (pBlock->info.id.uid) {
3,362,758!
988
      pBlock->info.id.groupId = tableListGetTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid);
3,363,849✔
989
    }
990

991
    uint32_t status = 0;
3,363,604✔
992
    code = loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status);
3,363,604✔
993
    QUERY_CHECK_CODE(code, lino, _end);
3,362,070✔
994

995
    if (status == FUNC_DATA_REQUIRED_ALL_FILTEROUT) {
3,362,069✔
996
      break;
1,399✔
997
    }
998

999
    // current block is filter out according to filter condition, continue load the next block
1000
    if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) {
3,360,670✔
1001
      continue;
95,039✔
1002
    }
1003

1004
    pOperator->resultInfo.totalRows = pTableScanInfo->base.readRecorder.totalRows;
3,265,631✔
1005
    pTableScanInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
3,265,436✔
1006

1007
    pOperator->cost.totalCost = pTableScanInfo->base.readRecorder.elapsedTime;
3,265,436✔
1008
    pBlock->info.scanFlag = pTableScanInfo->base.scanFlag;
3,265,436✔
1009

1010
    (*ppRes) = pBlock;
3,265,436✔
1011
    return code;
3,265,436✔
1012
  }
1013

1014
_end:
1,103,313✔
1015
  if (code != TSDB_CODE_SUCCESS) {
1,103,313✔
1016
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
809!
1017
    pTaskInfo->code = code;
809✔
1018
    T_LONG_JMP(pTaskInfo->env, code);
809!
1019
  }
1020
  return code;
1,102,504✔
1021
}
1022

1023
static int32_t doGroupedTableScan(SOperatorInfo* pOperator, SSDataBlock** pBlock) {
4,736,565✔
1024
  int32_t         code = TSDB_CODE_SUCCESS;
4,736,565✔
1025
  int32_t         lino = 0;
4,736,565✔
1026
  STableScanInfo* pTableScanInfo = pOperator->info;
4,736,565✔
1027
  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
4,736,565✔
1028
  SStorageAPI*    pAPI = &pTaskInfo->storageAPI;
4,736,565✔
1029

1030
  QRY_PARAM_CHECK(pBlock);
4,736,565!
1031

1032
  // The read handle is not initialized yet, since no qualified tables exists
1033
  if (pTableScanInfo->base.dataReader == NULL || pOperator->status == OP_EXEC_DONE) {
4,736,565!
1034
    return code;
16,047✔
1035
  }
1036

1037
  // do the ascending order traverse in the first place.
1038
  while (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) {
5,750,136✔
1039
    SSDataBlock* p = NULL;
4,122,874✔
1040
    code = doTableScanImplNext(pOperator, &p);
4,122,874✔
1041
    QUERY_CHECK_CODE(code, lino, _end);
4,119,634!
1042

1043
    if (p != NULL) {
4,119,634✔
1044
      markGroupProcessed(pTableScanInfo, p->info.id.groupId);
3,091,178✔
1045
      *pBlock = p;
3,091,450✔
1046
      return code;
3,091,450✔
1047
    }
1048

1049
    pTableScanInfo->scanTimes += 1;
1,028,456✔
1050
    taosHashClear(pTableScanInfo->pIgnoreTables);
1,028,456✔
1051

1052
    if (pTableScanInfo->scanTimes < pTableScanInfo->scanInfo.numOfAsc) {
1,029,635✔
1053
      setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
944✔
1054
      pTableScanInfo->base.scanFlag = MAIN_SCAN;
927✔
1055
      pTableScanInfo->base.dataBlockLoadFlag = FUNC_DATA_REQUIRED_DATA_LOAD;
927✔
1056
      qDebug("start to repeat ascending order scan data blocks due to query func required, %s", GET_TASKID(pTaskInfo));
927✔
1057

1058
      // do prepare for the next round table scan operation
1059
      code = pAPI->tsdReader.tsdReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond);
927✔
1060
      QUERY_CHECK_CODE(code, lino, _end);
927!
1061
    }
1062
  }
1063

1064
  int32_t total = pTableScanInfo->scanInfo.numOfAsc + pTableScanInfo->scanInfo.numOfDesc;
1,627,262✔
1065
  if (pTableScanInfo->scanTimes < total) {
1,627,262✔
1066
    if (pTableScanInfo->base.cond.order == TSDB_ORDER_ASC) {
245,355✔
1067
      prepareForDescendingScan(&pTableScanInfo->base, pOperator->exprSupp.pCtx, 0);
191✔
1068
      code = pAPI->tsdReader.tsdReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond);
191✔
1069
      QUERY_CHECK_CODE(code, lino, _end);
191!
1070
      qDebug("%s start to descending order scan data blocks due to query func required", GET_TASKID(pTaskInfo));
191✔
1071
    }
1072

1073
    while (pTableScanInfo->scanTimes < total) {
317,970✔
1074
      SSDataBlock* p = NULL;
245,262✔
1075
      code = doTableScanImplNext(pOperator, &p);
245,262✔
1076
      QUERY_CHECK_CODE(code, lino, _end);
245,190!
1077

1078
      if (p != NULL) {
245,190✔
1079
        markGroupProcessed(pTableScanInfo, p->info.id.groupId);
172,588✔
1080
        *pBlock = p;
172,735✔
1081
        return code;
172,735✔
1082
      }
1083

1084
      pTableScanInfo->scanTimes += 1;
72,602✔
1085
      taosHashClear(pTableScanInfo->pIgnoreTables);
72,602✔
1086

1087
      if (pTableScanInfo->scanTimes < total) {
72,596✔
1088
        setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED);
9✔
1089
        pTableScanInfo->base.scanFlag = MAIN_SCAN;
×
1090

1091
        qDebug("%s start to repeat descending order scan data blocks", GET_TASKID(pTaskInfo));
×
1092
        code = pAPI->tsdReader.tsdReaderResetStatus(pTableScanInfo->base.dataReader, &pTableScanInfo->base.cond);
×
1093
        QUERY_CHECK_CODE(code, lino, _end);
×
1094
      }
1095
    }
1096
  }
1097

1098
  if (pTableScanInfo->countState < TABLE_COUNT_STATE_END) {
1,454,615✔
1099
    STableListInfo* pTableListInfo = pTableScanInfo->base.pTableListInfo;
482,245✔
1100
    if (pTableListInfo->groupOffset) {  // group by tbname, group by tag + sort
482,245✔
1101
      if (pTableScanInfo->countState < TABLE_COUNT_STATE_PROCESSED) {
198!
1102
        pTableScanInfo->countState = TABLE_COUNT_STATE_PROCESSED;
×
1103
        STableKeyInfo* pStart =
1104
            (STableKeyInfo*)tableListGetInfo(pTableScanInfo->base.pTableListInfo, pTableScanInfo->tableStartIndex);
×
1105

1106
        if (NULL == pStart) {
×
1107
          return code;
×
1108
        }
1109

1110
        *pBlock = getBlockForEmptyTable(pOperator, pStart);
×
1111
        return code;
×
1112
      }
1113
    } else {  // group by tag + no sort
1114
      int32_t numOfTables = 0;
482,047✔
1115
      code = tableListGetSize(pTableListInfo, &numOfTables);
482,047✔
1116
      QUERY_CHECK_CODE(code, lino, _end);
482,110!
1117

1118
      if (pTableScanInfo->tableEndIndex + 1 >= numOfTables) {
482,110✔
1119
        // get empty group, mark processed & rm from hash
1120
        void* pIte = taosHashIterate(pTableListInfo->remainGroups, NULL);
477,726✔
1121
        if (pIte != NULL) {
478,207✔
1122
          size_t        keySize = 0;
353,899✔
1123
          uint64_t*     pGroupId = taosHashGetKey(pIte, &keySize);
353,899✔
1124
          STableKeyInfo info = {.uid = *(uint64_t*)pIte, .groupId = *pGroupId};
353,933✔
1125
          taosHashCancelIterate(pTableListInfo->remainGroups, pIte);
353,933✔
1126
          markGroupProcessed(pTableScanInfo, *pGroupId);
353,932✔
1127
          *pBlock = getBlockForEmptyTable(pOperator, &info);
353,781✔
1128

1129
          return code;
353,589✔
1130
        }
1131
      }
1132
    }
1133
    pTableScanInfo->countState = TABLE_COUNT_STATE_END;
128,890✔
1134
  }
1135

1136
_end:
972,370✔
1137
  if (code != TSDB_CODE_SUCCESS) {
1,101,260!
1138
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1139
    pTaskInfo->code = code;
×
1140
  }
1141

1142
  return code;
1,101,260✔
1143
}
1144

1145
static int32_t createTableListInfoFromParam(SOperatorInfo* pOperator) {
1,141✔
1146
  STableScanInfo*          pInfo = pOperator->info;
1,141✔
1147
  SExecTaskInfo*           pTaskInfo = pOperator->pTaskInfo;
1,141✔
1148
  int32_t                  code = 0;
1,141✔
1149
  STableListInfo*          pListInfo = pInfo->base.pTableListInfo;
1,141✔
1150
  STableScanOperatorParam* pParam = (STableScanOperatorParam*)pOperator->pOperatorGetParam->value;
1,141✔
1151
  int32_t                  num = taosArrayGetSize(pParam->pUidList);
1,141✔
1152
  if (num <= 0) {
1,143!
1153
    qError("empty table scan uid list");
×
1154
    return TSDB_CODE_INVALID_PARA;
×
1155
  }
1156

1157
  qDebug("vgId:%d add total %d dynamic tables to scan, tableSeq:%d, exist num:%" PRId64 ", operator status:%d",
1,143✔
1158
         pTaskInfo->id.vgId, num, pParam->tableSeq, (int64_t)taosArrayGetSize(pListInfo->pTableList),
1159
         pOperator->status);
1160

1161
  if (pParam->tableSeq) {
1,142✔
1162
    pListInfo->oneTableForEachGroup = true;
48✔
1163
    if (taosArrayGetSize(pListInfo->pTableList) > 0) {
48!
1164
      taosHashClear(pListInfo->map);
×
1165
      taosArrayClear(pListInfo->pTableList);
×
1166
      pOperator->status = OP_EXEC_DONE;
1✔
1167
    }
1168
  } else {
1169
    pListInfo->oneTableForEachGroup = false;
1,094✔
1170
    pListInfo->numOfOuputGroups = 1;
1,094✔
1171
  }
1172

1173
  STableKeyInfo info = {.groupId = 0};
1,143✔
1174
  int32_t       tableIdx = 0;
1,143✔
1175
  for (int32_t i = 0; i < num; ++i) {
69,281✔
1176
    uint64_t* pUid = taosArrayGet(pParam->pUidList, i);
68,143✔
1177
    if (!pUid) {
68,066!
1178
      qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));
×
1179
      return terrno;
×
1180
    }
1181

1182
    if (taosHashPut(pListInfo->map, pUid, sizeof(uint64_t), &tableIdx, sizeof(int32_t))) {
68,066✔
1183
      if (TSDB_CODE_DUP_KEY == terrno) {
64,822✔
1184
        continue;
64,559✔
1185
      }
1186
      return terrno;
17✔
1187
    }
1188

1189
    info.uid = *pUid;
3,811✔
1190
    void* p = taosArrayPush(pListInfo->pTableList, &info);
3,811✔
1191
    if (p == NULL) {
3,868!
1192
      return terrno;
×
1193
    }
1194

1195
    tableIdx++;
3,868✔
1196
    qDebug("add dynamic table scan uid:%" PRIu64 ", %s", info.uid, GET_TASKID(pTaskInfo));
3,868✔
1197
  }
1198

1199
  return code;
1,138✔
1200
}
1201

1202
static int32_t doInitReader(STableScanInfo* pInfo, SExecTaskInfo* pTaskInfo, SStorageAPI* pAPI, int32_t* pNum,
1,129,532✔
1203
                            STableKeyInfo** pList) {
1204
  const char* idStr = GET_TASKID(pTaskInfo);
1,129,532✔
1205
  int32_t     code = initNextGroupScan(pInfo, pList, pNum);
1,129,532✔
1206
  if (code) {
1,129,735!
1207
    qError("%s failed to init groupScan Info, code:%s at line:%d", idStr, tstrerror(code), __LINE__);
×
1208
    return code;
×
1209
  }
1210

1211
  if (pInfo->base.dataReader != NULL) {
1,129,931!
1212
    qError("%s tsdb reader should be null", idStr);
×
1213
    return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
×
1214
  }
1215

1216
  code = pAPI->tsdReader.tsdReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, *pList, *pNum, pInfo->pResBlock,
1,129,931✔
1217
                                       (void**)&pInfo->base.dataReader, idStr, &pInfo->pIgnoreTables);
1,129,931✔
1218
  if (code) {
1,130,890✔
1219
    qError("%s failed to open tsdbReader, code:%s at line:%d", idStr, tstrerror(code), __LINE__);
16!
1220
  }
1221

1222
  return code;
1,130,735✔
1223
}
1224

1225
int compareColIdPair(const void* elem1, const void* elem2) {
×
1226
  SColIdPair* node1 = (SColIdPair*)elem1;
×
1227
  SColIdPair* node2 = (SColIdPair*)elem2;
×
1228

1229
  if (node1->orgColId < node2->orgColId) {
×
1230
    return -1;
×
1231
  }
1232

1233
  return node1->orgColId > node2->orgColId;
×
1234
}
1235

1236
static int32_t createVTableScanInfoFromParam(SOperatorInfo* pOperator) {
×
1237
  int32_t                  code = 0;
×
1238
  int32_t                  lino = 0;
×
1239
  STableScanInfo*          pInfo = pOperator->info;
×
1240
  SExecTaskInfo*           pTaskInfo = pOperator->pTaskInfo;
×
1241
  SStorageAPI*             pAPI = &pTaskInfo->storageAPI;
×
1242
  STableListInfo*          pListInfo = pInfo->base.pTableListInfo;
×
1243
  STableScanOperatorParam* pParam = (STableScanOperatorParam*)pOperator->pOperatorGetParam->value;
×
1244
  SMetaReader              orgTable = {0};
×
1245
  SMetaReader              superTable = {0};
×
1246
  SSchemaWrapper*          schema = NULL;
×
1247
  SArray*                  pColArray = NULL;
×
1248
  SArray*                  pBlockColArray = NULL;
×
1249
  int32_t                  num = 0;
×
1250
  STableKeyInfo*           pList = NULL;
×
1251

1252
  cleanupQueryTableDataCond(&pInfo->base.cond);
×
1253
  if (pAPI->tsdReader.tsdReaderClose) {
×
1254
    pAPI->tsdReader.tsdReaderClose(pInfo->base.dataReader);
×
1255
  }
1256

1257
  pAPI->metaReaderFn.initReader(&orgTable, pInfo->base.readHandle.vnode, META_READER_LOCK, &pAPI->metaFn);
×
1258
  code = pAPI->metaReaderFn.getTableEntryByName(&orgTable, strstr(pParam->pOrgTbInfo->tbName, ".") + 1);
×
1259
  QUERY_CHECK_CODE(code, lino, _return);
×
1260
  switch (orgTable.me.type) {
×
1261
    case TSDB_CHILD_TABLE:
×
1262
      pAPI->metaReaderFn.initReader(&superTable, pInfo->base.readHandle.vnode, META_READER_LOCK, &pAPI->metaFn);
×
1263
      code = pAPI->metaReaderFn.getTableEntryByUid(&superTable, orgTable.me.ctbEntry.suid);
×
1264
      QUERY_CHECK_CODE(code, lino, _return);
×
1265
      schema = &superTable.me.stbEntry.schemaRow;
×
1266
      break;
×
1267
    case TSDB_NORMAL_TABLE:
×
1268
      schema = &orgTable.me.ntbEntry.schemaRow;
×
1269
      break;
×
1270
    default:
×
1271
      qError("invalid table type:%d", orgTable.me.type);
×
1272
      return TSDB_CODE_INVALID_PARA;
×
1273
      break;
1274
  }
1275

1276
  pColArray = taosArrayInit(schema->nCols, sizeof(SColIdPair));
×
1277
  QUERY_CHECK_NULL(pColArray, code, lino, _return, terrno);
×
1278
  pBlockColArray = taosArrayInit(schema->nCols, sizeof(int32_t));
×
1279
  QUERY_CHECK_NULL(pBlockColArray, code, lino, _return, terrno);
×
1280

1281
  // virtual table's origin table scan do not has ts column.
1282
  SColIdPair tsPair = {.vtbColId = PRIMARYKEY_TIMESTAMP_COL_ID, .orgColId = PRIMARYKEY_TIMESTAMP_COL_ID};
×
1283
  QUERY_CHECK_NULL(taosArrayPush(pColArray, &tsPair), code, lino, _return, terrno);
×
1284

1285
  for (int32_t i = 0; i < taosArrayGetSize(pParam->pOrgTbInfo->colMap); ++i) {
×
1286
    SColIdNameKV *kv = taosArrayGet(pParam->pOrgTbInfo->colMap, i);
×
1287
    for (int32_t j = 0; j < schema->nCols; j++) {
×
1288
      if (strncmp(kv->colName, schema->pSchema[j].name, strlen(schema->pSchema[j].name)) == 0) {
×
1289
        SColIdPair pPair = {.vtbColId = kv->colId, .orgColId = (col_id_t)(j + 1)};
×
1290
        QUERY_CHECK_NULL(taosArrayPush(pColArray, &pPair), code, lino, _return, terrno);
×
1291
        break;
×
1292
      }
1293
    }
1294
  }
1295

1296
  for (int32_t i = 0; i < taosArrayGetSize(pColArray); i++) {
×
1297
    SColIdPair *pPair = (SColIdPair*)taosArrayGet(pColArray, i);
×
1298
    for (int32_t j = 0; j < taosArrayGetSize(pInfo->base.matchInfo.pList); j++) {
×
1299
      SColMatchItem *pItem = taosArrayGet(pInfo->base.matchInfo.pList, j);
×
1300
      if (pItem->colId == pPair->vtbColId) {
×
1301
        SColIdPair tmpPair = {.orgColId = pPair->orgColId, .vtbColId = pItem->dstSlotId};
×
1302
        QUERY_CHECK_NULL(taosArrayPush(pBlockColArray, &tmpPair), code, lino, _return, terrno);
×
1303
        break;
×
1304
      }
1305
    }
1306
  }
1307

1308
  if (pInfo->pResBlock) {
×
1309
    blockDataDestroy(pInfo->pResBlock);
×
1310
    pInfo->pResBlock = NULL;
×
1311
  }
1312
  taosArraySort(pColArray, compareColIdPair);
×
1313
  taosArraySort(pBlockColArray, compareColIdPair);
×
1314
  code = createOneDataBlockWithColArray(pInfo->pOrgBlock, pBlockColArray, &pInfo->pResBlock);
×
1315
  QUERY_CHECK_CODE(code, lino, _return);
×
1316
  code = initQueryTableDataCondWithColArray(&pInfo->base.cond, &pInfo->base.orgCond, &pInfo->base.readHandle, pColArray);
×
1317
  QUERY_CHECK_CODE(code, lino, _return);
×
1318
  pInfo->base.cond.twindows.skey = pParam->window.ekey + 1;
×
1319
  pInfo->base.cond.suid = orgTable.me.type == TSDB_CHILD_TABLE ? superTable.me.uid : 0;
×
1320
  pInfo->currentGroupId = 0;
×
1321
  pInfo->base.dataReader = NULL;
×
1322
  pInfo->ignoreTag = true;
×
1323

1324
  pListInfo->oneTableForEachGroup = true;
×
1325
  taosHashClear(pListInfo->map);
×
1326
  taosArrayClear(pListInfo->pTableList);
×
1327

1328
  uint64_t      pUid = orgTable.me.uid;
×
1329
  STableKeyInfo info = {.groupId = 0, .uid = pUid};
×
1330
  int32_t       tableIdx = 0;
×
1331
  code = taosHashPut(pListInfo->map, &pUid, sizeof(uint64_t), &tableIdx, sizeof(int32_t));
×
1332
  QUERY_CHECK_CODE(code, lino, _return);
×
1333
  QUERY_CHECK_NULL(taosArrayPush(pListInfo->pTableList, &info), code, lino, _return, terrno);
×
1334
  qDebug("add dynamic table scan uid:%" PRIu64 ", %s", info.uid, GET_TASKID(pTaskInfo));
×
1335

1336
  pOperator->status = OP_OPENED;
×
1337

1338
  taosRLockLatch(&pTaskInfo->lock);
×
1339
  code = doInitReader(pInfo, pTaskInfo, pAPI, &num, &pList);
×
1340
  taosRUnLockLatch(&pTaskInfo->lock);
×
1341
  QUERY_CHECK_CODE(code, lino, _return);
×
1342

1343
  if (pInfo->pResBlock->info.capacity > pOperator->resultInfo.capacity) {
×
1344
    pOperator->resultInfo.capacity = pInfo->pResBlock->info.capacity;
×
1345
  }
1346

1347
  pInfo->currentGroupId = -1;
×
1348
_return:
×
1349
  if (code) {
×
1350
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1351
  }
1352
  taosArrayDestroy(pColArray);
×
1353
  taosArrayDestroy(pBlockColArray);
×
1354
  pAPI->metaReaderFn.clearReader(&superTable);
×
1355
  pAPI->metaReaderFn.clearReader(&orgTable);
×
1356
  return code;
×
1357
}
1358

1359
static int32_t startNextGroupScan(SOperatorInfo* pOperator, SSDataBlock** pResult) {
1,101,157✔
1360
  int32_t         code = TSDB_CODE_SUCCESS;
1,101,157✔
1361
  int32_t         lino = 0;
1,101,157✔
1362
  STableScanInfo* pInfo = pOperator->info;
1,101,157✔
1363
  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
1,101,157✔
1364
  SStorageAPI*    pAPI = &pTaskInfo->storageAPI;
1,101,157✔
1365
  int32_t         numOfTables = 0;
1,101,157✔
1366

1367
  QRY_PARAM_CHECK(pResult);
1,101,157!
1368

1369
  code = tableListGetSize(pInfo->base.pTableListInfo, &numOfTables);
1,101,157✔
1370
  QUERY_CHECK_CODE(code, lino, _end);
1,101,591!
1371

1372
  if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) {
1,101,591✔
1373
    setOperatorCompleted(pOperator);
1,100,531✔
1374
    if (pOperator->dynamicTask) {
1,100,634✔
1375
      taosArrayClear(pInfo->base.pTableListInfo->pTableList);
1,132✔
1376
      taosHashClear(pInfo->base.pTableListInfo->map);
1,131✔
1377
    }
1378
    return code;
1,100,616✔
1379
  }
1380

1381
  // reset value for the next group data output
1382
  pOperator->status = OP_OPENED;
807✔
1383
  resetLimitInfoForNextGroup(&pInfo->base.limitInfo);
807✔
1384

1385
  int32_t        num = 0;
909✔
1386
  STableKeyInfo* pList = NULL;
909✔
1387
  code = initNextGroupScan(pInfo, &pList, &num);
909✔
1388
  QUERY_CHECK_CODE(code, lino, _end);
909!
1389

1390
  code = pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, pList, num);
909✔
1391
  QUERY_CHECK_CODE(code, lino, _end);
909!
1392

1393
  code = pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond);
909✔
1394
  QUERY_CHECK_CODE(code, lino, _end);
909!
1395
  pInfo->scanTimes = 0;
909✔
1396

1397
  code = doGroupedTableScan(pOperator, pResult);
909✔
1398
  QUERY_CHECK_CODE(code, lino, _end);
908!
1399

1400
  if (*pResult != NULL) {
908✔
1401
    if (pOperator->dynamicTask) {
804✔
1402
      (*pResult)->info.id.groupId = (*pResult)->info.id.uid;
24✔
1403
    }
1404
    return code;
804✔
1405
  }
1406

1407
_end:
104✔
1408
  if (code != TSDB_CODE_SUCCESS) {
104!
1409
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1410
    pTaskInfo->code = code;
×
1411
  }
1412
  return code;
104✔
1413
}
1414

1415
static int32_t groupSeqTableScan(SOperatorInfo* pOperator, SSDataBlock** pResBlock) {
4,693,902✔
1416
  int32_t         code = TSDB_CODE_SUCCESS;
4,693,902✔
1417
  int32_t         lino = 0;
4,693,902✔
1418
  STableScanInfo* pInfo = pOperator->info;
4,693,902✔
1419
  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
4,693,902✔
1420
  SStorageAPI*    pAPI = &pTaskInfo->storageAPI;
4,693,902✔
1421
  int32_t         num = 0;
4,693,902✔
1422
  STableKeyInfo*  pList = NULL;
4,693,902✔
1423
  SSDataBlock*    pResult = NULL;
4,693,902✔
1424
  const char*     idStr = GET_TASKID(pTaskInfo);
4,693,902✔
1425

1426
  QRY_PARAM_CHECK(pResBlock);
4,693,902!
1427

1428
  if (pInfo->currentGroupId == -1) {
4,693,902✔
1429
    if ((++pInfo->currentGroupId) >= tableListGetOutputGroups(pInfo->base.pTableListInfo)) {
1,129,580!
1430
      setOperatorCompleted(pOperator);
×
1431
      return code;
×
1432
    }
1433

1434
    taosRLockLatch(&pTaskInfo->lock);
1,129,635✔
1435
    code = doInitReader(pInfo, pTaskInfo, pAPI, &num, &pList);
1,131,068✔
1436
    taosRUnLockLatch(&pTaskInfo->lock);
1,130,687✔
1437
    QUERY_CHECK_CODE(code, lino, _end);
1,131,400✔
1438

1439
    if (pInfo->filesetDelimited) {
1,131,384✔
1440
      pAPI->tsdReader.tsdSetFilesetDelimited(pInfo->base.dataReader);
1,117✔
1441
    }
1442

1443
    if (pInfo->pResBlock->info.capacity > pOperator->resultInfo.capacity) {
1,131,383!
1444
      pOperator->resultInfo.capacity = pInfo->pResBlock->info.capacity;
×
1445
    }
1446
  }
1447

1448
  code = doGroupedTableScan(pOperator, &pResult);
4,695,705✔
1449
  QUERY_CHECK_CODE(code, lino, _end);
4,695,116!
1450

1451
  if (pResult != NULL) {
4,695,116✔
1452
    if (pOperator->dynamicTask) {
3,596,066✔
1453
      pResult->info.id.groupId = pResult->info.id.uid;
3,429✔
1454
    }
1455

1456
    *pResBlock = pResult;
3,596,066✔
1457
    return code;
3,596,066✔
1458
  }
1459

1460
  while (true) {
1461
    code = startNextGroupScan(pOperator, &pResult);
1,099,154✔
1462
    QUERY_CHECK_CODE(code, lino, _end);
1,101,588!
1463

1464
    if (pResult || pOperator->status == OP_EXEC_DONE) {
1,101,588✔
1465
      *pResBlock = pResult;
1,101,484✔
1466
      return code;
1,101,484✔
1467
    }
1468
  }
1469

1470
_end:
16✔
1471
  if (code != TSDB_CODE_SUCCESS) {
16!
1472
    qError("%s %s failed at line %d since %s", idStr, __func__, lino, tstrerror(code));
16!
1473
    pTaskInfo->code = code;
16✔
1474
  }
1475

1476
  return code;
16✔
1477
}
1478

1479
int32_t doTableScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
4,715,348✔
1480
  int32_t         code = TSDB_CODE_SUCCESS;
4,715,348✔
1481
  int32_t         lino = 0;
4,715,348✔
1482
  STableScanInfo* pInfo = pOperator->info;
4,715,348✔
1483
  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
4,715,348✔
1484
  SStorageAPI*    pAPI = &pTaskInfo->storageAPI;
4,715,348✔
1485
  QRY_PARAM_CHECK(ppRes);
4,715,348!
1486
  qTrace("%s call", __FUNCTION__);
4,715,348✔
1487
  if (pOperator->pOperatorGetParam) {
4,714,529✔
1488
    pOperator->dynamicTask = true;
1,142✔
1489
    if (isDynVtbScan(pOperator)) {
1,142!
1490
      code = createVTableScanInfoFromParam(pOperator);
×
1491

1492
      freeOperatorParam(pOperator->pOperatorGetParam, OP_GET_PARAM);
×
1493
      pOperator->pOperatorGetParam = NULL;
×
1494
      QUERY_CHECK_CODE(code, lino, _end);
×
1495

1496
      SSDataBlock* result = NULL;
×
1497
      while (true) {
1498
        code = startNextGroupScan(pOperator, &result);
×
1499
        QUERY_CHECK_CODE(code, lino, _end);
×
1500

1501
        if (result || pOperator->status == OP_EXEC_DONE) {
×
1502
          SSDataBlock* res = NULL;
×
1503
          if (result) {
×
1504
            code = createOneDataBlockWithTwoBlock(result, pInfo->pOrgBlock, &res);
×
1505
            QUERY_CHECK_CODE(code, lino, _end);
×
1506
            pInfo->pResBlock = res;
×
1507
            blockDataDestroy(result);
×
1508
          }
1509
          (*ppRes) = res;
×
1510
          return code;
×
1511
        }
1512
      }
1513
    } else {
1514
      code = createTableListInfoFromParam(pOperator);
1,142✔
1515
      freeOperatorParam(pOperator->pOperatorGetParam, OP_GET_PARAM);
1,143✔
1516
      pOperator->pOperatorGetParam = NULL;
1,143✔
1517
      QUERY_CHECK_CODE(code, lino, _end);
1,143!
1518

1519
      if (pOperator->status == OP_EXEC_DONE) {
1,143✔
1520
        pInfo->currentGroupId = -1;
24✔
1521
        pOperator->status = OP_OPENED;
24✔
1522
        SSDataBlock* result = NULL;
24✔
1523

1524
        while (true) {
1525
          code = startNextGroupScan(pOperator, &result);
24✔
1526
          QUERY_CHECK_CODE(code, lino, _end);
24!
1527

1528
          if (result || pOperator->status == OP_EXEC_DONE) {
24!
1529
            (*ppRes) = result;
24✔
1530
            return code;
24✔
1531
          }
1532
        }
1533
      }
1534
    }
1535
  }
1536

1537
  // scan table one by one sequentially
1538
  if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) {
4,714,506✔
1539
    int32_t       numOfTables = 0;
20,526✔
1540
    STableKeyInfo tInfo = {0};
20,526✔
1541
    pInfo->countState = TABLE_COUNT_STATE_END;
20,526✔
1542

1543
    while (1) {
19,025✔
1544
      SSDataBlock* result = NULL;
39,551✔
1545
      code = doGroupedTableScan(pOperator, &result);
39,551✔
1546
      QUERY_CHECK_CODE(code, lino, _end);
39,541!
1547

1548
      if (result || (pOperator->status == OP_EXEC_DONE) || isTaskKilled(pTaskInfo)) {
39,541!
1549
        (*ppRes) = result;
20,301✔
1550
        return code;
20,501✔
1551
      }
1552

1553
      // if no data, switch to next table and continue scan
1554
      pInfo->currentTable++;
19,240✔
1555

1556
      taosRLockLatch(&pTaskInfo->lock);
19,240✔
1557
      numOfTables = 0;
19,241✔
1558
      code = tableListGetSize(pInfo->base.pTableListInfo, &numOfTables);
19,241✔
1559
      if (code != TSDB_CODE_SUCCESS) {
19,241!
1560
        taosRUnLockLatch(&pTaskInfo->lock);
×
1561
        TSDB_CHECK_CODE(code, lino, _end);
×
1562
      }
1563

1564
      if (pInfo->currentTable >= numOfTables) {
19,241✔
1565
        qDebug("all table checked in table list, total:%d, return NULL, %s", numOfTables, GET_TASKID(pTaskInfo));
200✔
1566
        taosRUnLockLatch(&pTaskInfo->lock);
200✔
1567
        (*ppRes) = NULL;
200✔
1568
        return code;
200✔
1569
      }
1570

1571
      STableKeyInfo* tmp = (STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->currentTable);
19,041✔
1572
      if (!tmp) {
19,040!
1573
        taosRUnLockLatch(&pTaskInfo->lock);
×
1574
        (*ppRes) = NULL;
×
1575
        QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
×
1576
      }
1577

1578
      tInfo = *tmp;
19,040✔
1579
      taosRUnLockLatch(&pTaskInfo->lock);
19,040✔
1580

1581
      code = pAPI->tsdReader.tsdSetQueryTableList(pInfo->base.dataReader, &tInfo, 1);
19,042✔
1582
      QUERY_CHECK_CODE(code, lino, _end);
19,039!
1583
      qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d/%d %s", tInfo.uid, numOfTables,
19,039✔
1584
             pInfo->currentTable, numOfTables, GET_TASKID(pTaskInfo));
1585

1586
      code = pAPI->tsdReader.tsdReaderResetStatus(pInfo->base.dataReader, &pInfo->base.cond);
19,039✔
1587
      QUERY_CHECK_CODE(code, lino, _end);
19,025!
1588
      pInfo->scanTimes = 0;
19,025✔
1589
    }
1590
  } else {  // scan table group by group sequentially
1591
    code = groupSeqTableScan(pOperator, ppRes);
4,693,980✔
1592
    QUERY_CHECK_CODE(code, lino, _end);
4,695,485✔
1593
  }
1594

1595
_end:
4,695,469✔
1596
  if (code != TSDB_CODE_SUCCESS) {
4,695,485✔
1597
    qError("%s %s failed at line %d since %s", GET_TASKID(pTaskInfo), __func__, lino, tstrerror(code));
16!
1598
    pTaskInfo->code = code;
16✔
1599
    T_LONG_JMP(pTaskInfo->env, code);
16!
1600
  }
1601

1602
  return code;
4,695,469✔
1603
}
1604

1605
static int32_t getTableScannerExecInfo(struct SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) {
17,873✔
1606
  SFileBlockLoadRecorder* pRecorder = taosMemoryCalloc(1, sizeof(SFileBlockLoadRecorder));
17,873!
1607
  if (!pRecorder) {
17,876!
1608
    return terrno;
×
1609
  }
1610
  STableScanInfo* pTableScanInfo = pOptr->info;
17,876✔
1611
  *pRecorder = pTableScanInfo->base.readRecorder;
17,876✔
1612
  *pOptrExplain = pRecorder;
17,876✔
1613
  *len = sizeof(SFileBlockLoadRecorder);
17,876✔
1614
  return 0;
17,876✔
1615
}
1616

1617
static void destroyTableScanBase(STableScanBase* pBase, TsdReader* pAPI) {
1,345,234✔
1618
  cleanupQueryTableDataCond(&pBase->cond);
1,345,234✔
1619
  cleanupQueryTableDataCond(&pBase->orgCond);
1,345,654✔
1620

1621
  if (pAPI->tsdReaderClose) {
1,345,660!
1622
    pAPI->tsdReaderClose(pBase->dataReader);
1,345,665✔
1623
  }
1624
  pBase->dataReader = NULL;
1,345,679✔
1625

1626
  if (pBase->matchInfo.pList != NULL) {
1,345,679!
1627
    taosArrayDestroy(pBase->matchInfo.pList);
1,345,720✔
1628
  }
1629

1630
  tableListDestroy(pBase->pTableListInfo);
1,345,732✔
1631
  taosLRUCacheCleanup(pBase->metaCache.pTableMetaEntryCache);
1,345,661✔
1632
  cleanupExprSupp(&pBase->pseudoSup);
1,345,713✔
1633
}
1,345,752✔
1634

1635
static void destroyTableScanOperatorInfo(void* param) {
1,171,655✔
1636
  STableScanInfo* pTableScanInfo = (STableScanInfo*)param;
1,171,655✔
1637
  blockDataDestroy(pTableScanInfo->pResBlock);
1,171,655✔
1638
  blockDataDestroy(pTableScanInfo->pOrgBlock);
1,172,191✔
1639
  taosHashCleanup(pTableScanInfo->pIgnoreTables);
1,172,182✔
1640
  destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI);
1,171,939✔
1641
  taosMemoryFreeClear(param);
1,172,094!
1642
}
1,172,169✔
1643

1644
static void resetClolumnReserve(SSDataBlock* pBlock, int32_t dataRequireFlag) {
1,170,028✔
1645
  if (pBlock && dataRequireFlag == FUNC_DATA_REQUIRED_NOT_LOAD) {
1,170,028!
1646
    int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
73,142✔
1647
    for (int32_t i = 0; i < numOfCols; ++i) {
167,737✔
1648
      SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i);
94,633✔
1649
      if (pCol) {
94,608!
1650
        pCol->info.noData = true;
94,620✔
1651
      }
1652
    }
1653
  }
1654
}
1,169,990✔
1655

1656
int32_t createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
1,170,885✔
1657
                                    STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo,
1658
                                    SOperatorInfo** pOptrInfo) {
1659
  QRY_PARAM_CHECK(pOptrInfo);
1,170,885!
1660

1661
  int32_t         code = TSDB_CODE_SUCCESS;
1,170,885✔
1662
  int32_t         lino = 0;
1,170,885✔
1663
  STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
1,170,885!
1664
  SOperatorInfo*  pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
1,171,370!
1665
  if (pInfo == NULL || pOperator == NULL) {
1,171,546!
1666
    code = terrno;
×
1667
    goto _error;
×
1668
  }
1669

1670
  SScanPhysiNode*     pScanNode = &pTableScanNode->scan;
1,171,598✔
1671
  SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc;
1,171,598✔
1672

1673
  int32_t numOfCols = 0;
1,171,598✔
1674
  code =
1675
      extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->base.matchInfo);
1,171,598✔
1676
  QUERY_CHECK_CODE(code, lino, _error);
1,170,888!
1677

1678
  initLimitInfo(pScanNode->node.pLimit, pScanNode->node.pSlimit, &pInfo->base.limitInfo);
1,170,888✔
1679
  code = initQueryTableDataCond(&pInfo->base.cond, pTableScanNode, readHandle);
1,171,440✔
1680
  QUERY_CHECK_CODE(code, lino, _error);
1,171,585!
1681

1682
  if (pScanNode->pScanPseudoCols != NULL) {
1,171,585✔
1683
    SExprSupp* pSup = &pInfo->base.pseudoSup;
344,107✔
1684
    pSup->pExprInfo = NULL;
344,107✔
1685
    code = createExprInfo(pScanNode->pScanPseudoCols, NULL, &pSup->pExprInfo, &pSup->numOfExprs);
344,107✔
1686
    QUERY_CHECK_CODE(code, lino, _error);
343,892!
1687

1688
    pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset,
343,892✔
1689
                                      &pTaskInfo->storageAPI.functionStore);
1690
    QUERY_CHECK_NULL(pSup->pCtx, code, lino, _error, terrno);
343,650✔
1691
  }
1692

1693
  pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]};
1,170,624✔
1694
  pInfo->base.scanFlag = (pInfo->scanInfo.numOfAsc > 1) ? PRE_SCAN : MAIN_SCAN;
1,170,624✔
1695

1696
  pInfo->base.pdInfo.interval = extractIntervalInfo(pTableScanNode);
1,170,624✔
1697
  pInfo->base.readHandle = *readHandle;
1,170,318✔
1698
  pInfo->base.dataBlockLoadFlag = pTableScanNode->dataRequired;
1,170,318✔
1699

1700
  pInfo->sample.sampleRatio = pTableScanNode->ratio;
1,170,318✔
1701
  pInfo->sample.seed = taosGetTimestampSec();
1,170,318✔
1702

1703
  pInfo->base.readerAPI = pTaskInfo->storageAPI.tsdReader;
1,170,616✔
1704
  initResultSizeInfo(&pOperator->resultInfo, 4096);
1,170,616✔
1705
  pInfo->pResBlock = createDataBlockFromDescNode(pDescNode);
1,169,954✔
1706
  resetClolumnReserve(pInfo->pResBlock, pInfo->base.dataBlockLoadFlag);
1,170,468✔
1707
  QUERY_CHECK_NULL(pInfo->pResBlock, code, lino, _error, terrno);
1,170,787!
1708

1709
  code = prepareDataBlockBuf(pInfo->pResBlock, &pInfo->base.matchInfo);
1,170,787✔
1710
  QUERY_CHECK_CODE(code, lino, _error);
1,170,902!
1711

1712
  pInfo->virtualStableScan = pScanNode->virtualStableScan;
1,170,902✔
1713
  if (pScanNode->node.dynamicOp && pScanNode->virtualStableScan) {
1,170,902!
1714
    TSWAP(pInfo->pOrgBlock, pInfo->pResBlock);
×
1715
    pInfo->pResBlock = NULL;
×
1716
    memcpy(&pInfo->base.orgCond, &pInfo->base.cond, sizeof(SQueryTableDataCond));
×
1717
    memset(&pInfo->base.cond, 0, sizeof(SQueryTableDataCond));
×
1718
  }
1719

1720
  code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
1,170,902✔
1721
  QUERY_CHECK_CODE(code, lino, _error);
1,170,153✔
1722

1723
  pInfo->currentGroupId = -1;
1,169,931✔
1724

1725
  pInfo->tableEndIndex = -1;
1,169,931✔
1726
  pInfo->assignBlockUid = pTableScanNode->assignBlockUid;
1,169,931✔
1727
  pInfo->hasGroupByTag = pTableScanNode->pGroupTags ? true : false;
1,169,931✔
1728

1729
  setOperatorInfo(pOperator, "TableScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo,
1,169,931✔
1730
                  pTaskInfo);
1731
  pOperator->exprSupp.numOfExprs = numOfCols;
1,169,474✔
1732

1733
  pInfo->needCountEmptyTable = tsCountAlwaysReturnValue && pTableScanNode->needCountEmptyTable;
1,169,474✔
1734
  pInfo->ignoreTag = false;
1,169,474✔
1735

1736
  pInfo->base.pTableListInfo = pTableListInfo;
1,169,474✔
1737
  pInfo->base.metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5);
1,169,474✔
1738
  if (pInfo->base.metaCache.pTableMetaEntryCache == NULL) {
1,171,345!
1739
    code = terrno;
×
1740
    QUERY_CHECK_CODE(code, lino, _error);
×
1741
  }
1742

1743
  pInfo->filesetDelimited = pTableScanNode->filesetDelimited;
1,171,345✔
1744

1745
  taosLRUCacheSetStrictCapacity(pInfo->base.metaCache.pTableMetaEntryCache, false);
1,171,345✔
1746
  pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableScanNext, NULL, destroyTableScanOperatorInfo,
1,171,841✔
1747
                                         optrDefaultBufFn, getTableScannerExecInfo, optrDefaultGetNextExtFn, NULL);
1748

1749
  // for non-blocking operator, the open cost is always 0
1750
  pOperator->cost.openCost = 0;
1,170,624✔
1751
  *pOptrInfo = pOperator;
1,170,624✔
1752
  return TSDB_CODE_SUCCESS;
1,170,624✔
1753

1754
_error:
222✔
1755
  if (pInfo != NULL) {
222!
1756
    pInfo->base.pTableListInfo = NULL;  // this attribute will be destroy outside of this function
222✔
1757
    destroyTableScanOperatorInfo(pInfo);
222✔
1758
  }
1759

1760
  if (pOperator != NULL) {
222!
1761
    pOperator->info = NULL;
222✔
1762
    destroyOperator(pOperator);
222✔
1763
  }
1764
  pTaskInfo->code = code;
222✔
1765
  return code;
222✔
1766
}
1767

1768
int32_t createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) {
×
1769
  QRY_PARAM_CHECK(pOptrInfo);
×
1770

1771
  int32_t         code = 0;
×
1772
  STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo));
×
1773
  SOperatorInfo*  pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
×
1774
  if (pInfo == NULL || pOperator == NULL) {
×
1775
    code = terrno;
×
1776
    goto _end;
×
1777
  }
1778

1779
  pInfo->base.dataReader = pReadHandle;
×
1780
  //  pInfo->prevGroupId       = -1;
1781

1782
  setOperatorInfo(pOperator, "TableSeqScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, false, OP_NOT_OPENED,
×
1783
                  pInfo, pTaskInfo);
1784
  pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableScanImplNext, NULL, NULL, optrDefaultBufFn, NULL,
×
1785
                                         optrDefaultGetNextExtFn, NULL);
1786
  *pOptrInfo = pOperator;
×
1787
  return code;
×
1788

1789
_end:
×
1790
  if (pInfo != NULL) {
×
1791
    taosMemoryFree(pInfo);
×
1792
  }
1793

1794
  if (pOperator != NULL) {
×
1795
    taosMemoryFree(pOperator);
×
1796
  }
1797

1798
  pTaskInfo->code = code;
×
1799
  return code;
×
1800
}
1801

1802
FORCE_INLINE void doClearBufferedBlocks(SStreamScanInfo* pInfo) {
×
1803
  qDebug("clear buff blocks:%d", (int32_t)taosArrayGetSize(pInfo->pBlockLists));
17,014!
1804
  taosArrayClear(pInfo->pBlockLists);
17,014✔
1805
  pInfo->validBlockIndex = 0;
17,014✔
1806
}
17,014✔
1807

1808
static bool isSessionWindow(SStreamScanInfo* pInfo) {
974✔
1809
  return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION ||
1,644✔
1810
         pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION ||
1,604✔
1811
         pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION;
630!
1812
}
1813

1814
static bool isStateWindow(SStreamScanInfo* pInfo) {
654✔
1815
  return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE ||
779✔
1816
         pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT;
125✔
1817
}
1818

1819
static bool isIntervalWindow(SStreamScanInfo* pInfo) {
8,564✔
1820
  return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL ||
15,480✔
1821
         pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL ||
6,916✔
1822
         pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL ||
5,949!
1823
         pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL ||
5,949!
1824
         pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_INTERVAL ||
5,949✔
1825
         pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_SEMI_INTERVAL ||
19,489!
1826
         pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_CONTINUE_FINAL_INTERVAL;
4,009!
1827
}
1828

1829
static bool isSignleIntervalWindow(SStreamScanInfo* pInfo) {
1,321✔
1830
  return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL;
1,321✔
1831
}
1832

1833
static bool isSlidingWindow(SStreamScanInfo* pInfo) {
6,490✔
1834
  return isIntervalWindow(pInfo) && pInfo->interval.interval != pInfo->interval.sliding;
6,490✔
1835
}
1836

1837
static bool isCountSlidingWindow(SStreamScanInfo* pInfo) {
6,173✔
1838
  return pInfo->windowSup.pStreamAggSup &&
10,823✔
1839
         (pInfo->windowSup.pStreamAggSup->windowCount != pInfo->windowSup.pStreamAggSup->windowSliding);
4,650✔
1840
}
1841

1842
static bool isCountWindow(SStreamScanInfo* pInfo) {
100✔
1843
  return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT;
100✔
1844
}
1845

1846
static bool isTimeSlice(SStreamScanInfo* pInfo) {
82✔
1847
  return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERP_FUNC;
82✔
1848
}
1849

1850
static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) {
6,490✔
1851
  SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex);
6,490✔
1852
  uint64_t*        groupCol = (uint64_t*)pColInfo->pData;
6,490✔
1853
  if (colDataIsNull_s(pColInfo, rowIndex)) {
12,980✔
1854
    pInfo->igCheckGroupId = true;
4,310✔
1855
  } else {
1856
    pInfo->groupId = groupCol[rowIndex];
2,180✔
1857
  }
1858
}
6,490✔
1859

1860
void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin, int64_t ver) {
6,491✔
1861
  pTableScanInfo->base.cond.twindows = *pWin;
6,491✔
1862
  pTableScanInfo->base.cond.startVersion = 0;
6,491✔
1863
  pTableScanInfo->base.cond.endVersion = ver;
6,491✔
1864
  pTableScanInfo->scanTimes = 0;
6,491✔
1865
  pTableScanInfo->currentGroupId = -1;
6,491✔
1866
  pTableScanInfo->tableEndIndex = -1;
6,491✔
1867
  pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader);
6,491✔
1868
  pTableScanInfo->base.dataReader = NULL;
6,491✔
1869
  pTableScanInfo->scanMode = TABLE_SCAN__BLOCK_ORDER;
6,491✔
1870
}
6,491✔
1871

1872
SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbUid, TSKEY startTs, TSKEY endTs,
409✔
1873
                                int64_t maxVersion) {
1874
  int32_t       code = TSDB_CODE_SUCCESS;
409✔
1875
  int32_t       lino = 0;
409✔
1876
  STableKeyInfo tblInfo = {.uid = tbUid, .groupId = 0};
409✔
1877

1878
  STableScanInfo*     pTableScanInfo = pTableScanOp->info;
409✔
1879
  SQueryTableDataCond cond = pTableScanInfo->base.cond;
409✔
1880

1881
  cond.startVersion = -1;
409✔
1882
  cond.endVersion = maxVersion;
409✔
1883
  cond.twindows = (STimeWindow){.skey = startTs, .ekey = endTs};
409✔
1884

1885
  SExecTaskInfo* pTaskInfo = pTableScanOp->pTaskInfo;
409✔
1886
  SStorageAPI*   pAPI = &pTaskInfo->storageAPI;
409✔
1887

1888
  SSDataBlock* pBlock = pTableScanInfo->pResBlock;
409✔
1889
  STsdbReader* pReader = NULL;
409✔
1890
  code = pAPI->tsdReader.tsdReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock,
409✔
1891
                                       (void**)&pReader, GET_TASKID(pTaskInfo), NULL);
409✔
1892
  QUERY_CHECK_CODE(code, lino, _end);
409!
1893

1894
  bool hasNext = false;
409✔
1895
  code = pAPI->tsdReader.tsdNextDataBlock(pReader, &hasNext);
409✔
1896
  QUERY_CHECK_CODE(code, lino, _end);
409!
1897

1898
  if (hasNext) {
409✔
1899
    SSDataBlock* p = NULL;
393✔
1900
    code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pReader, &p, NULL);
393✔
1901
    QUERY_CHECK_CODE(code, lino, _end);
393!
1902

1903
    code = doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, pBlock->info.rows);
393✔
1904
    QUERY_CHECK_CODE(code, lino, _end);
393!
1905

1906
    pBlock->info.id.groupId = tableListGetTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid);
393✔
1907
  }
1908

1909
_end:
16✔
1910
  pAPI->tsdReader.tsdReaderClose(pReader);
409✔
1911
  qDebug("retrieve prev rows:%" PRId64 ", skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64
409!
1912
         ", suid:%" PRIu64,
1913
         pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid);
1914
  if (code != TSDB_CODE_SUCCESS) {
409!
1915
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
1916
    terrno = code;
×
1917
    return NULL;
×
1918
  }
1919

1920
  return pBlock->info.rows > 0 ? pBlock : NULL;
409✔
1921
}
1922

1923
bool comparePrimaryKey(SColumnInfoData* pCol, int32_t rowId, void* pVal) {
2✔
1924
  // coverity scan
1925
  if (!pVal || !pCol) {
2!
1926
    qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_INVALID_PARA));
×
1927
    return false;
×
1928
  }
1929
  void* pData = colDataGetData(pCol, rowId);
2!
1930
  if (IS_VAR_DATA_TYPE(pCol->info.type)) {
2!
1931
    int32_t colLen = varDataLen(pData);
×
1932
    int32_t keyLen = varDataLen(pVal);
×
1933
    if (pCol->info.type == TSDB_DATA_TYPE_JSON) {
×
1934
      colLen = getJsonValueLen(pData);
×
1935
      keyLen = getJsonValueLen(pVal);
×
1936
    }
1937

1938
    if (colLen == keyLen && memcmp(pData, pVal, colLen) == 0) {
×
1939
      return true;
×
1940
    }
1941
  } else {
1942
    if (memcmp(pData, pVal, pCol->info.bytes) == 0) {
2!
1943
      return true;
2✔
1944
    }
1945
  }
1946
  return false;
×
1947
}
1948

1949
static bool hasPrimaryKeyCol(SStreamScanInfo* pInfo) { return pInfo->primaryKeyIndex != -1; }
10,012,126✔
1950

1951
static uint64_t getGroupIdByCol(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion, void* pVal) {
351✔
1952
  SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, uid, ts, ts, maxVersion);
351✔
1953
  if (!pPreRes || pPreRes->info.rows == 0) {
351!
1954
    if (terrno != TSDB_CODE_SUCCESS) {
16!
1955
      qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));
×
1956
    }
1957
    return 0;
16✔
1958
  }
1959

1960
  int32_t rowId = 0;
335✔
1961
  if (hasPrimaryKeyCol(pInfo)) {
335✔
1962
    SColumnInfoData* pPkCol = taosArrayGet(pPreRes->pDataBlock, pInfo->primaryKeyIndex);
2✔
1963
    for (; rowId < pPreRes->info.rows; rowId++) {
2!
1964
      if (comparePrimaryKey(pPkCol, rowId, pVal)) {
2!
1965
        break;
2✔
1966
      }
1967
    }
1968
  }
1969
  if (rowId >= pPreRes->info.rows) {
335!
1970
    qInfo("===stream===read preversion data of primary key failed. ts:%" PRId64 ",version:%" PRId64, ts, maxVersion);
×
1971
    return 0;
×
1972
  }
1973
  return calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pPreRes, rowId);
335✔
1974
}
1975

1976
static uint64_t getGroupIdByUid(SStreamScanInfo* pInfo, uint64_t uid) {
1,366✔
1977
  STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
1,366✔
1978
  return tableListGetTableGroupId(pTableScanInfo->base.pTableListInfo, uid);
1,366✔
1979
}
1980

1981
static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion, void* pVal) {
1,369✔
1982
  if (pInfo->partitionSup.needCalc) {
1,369✔
1983
    return getGroupIdByCol(pInfo, uid, ts, maxVersion, pVal);
351✔
1984
  }
1985

1986
  return getGroupIdByUid(pInfo, uid);
1,018✔
1987
}
1988

1989
void prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex, bool* pRes) {
12,708✔
1990
  int32_t code = TSDB_CODE_SUCCESS;
12,708✔
1991
  int32_t lino = 0;
12,708✔
1992
  if (pBlock->info.rows == 0) {
12,708✔
1993
    if (pRes) {
1!
1994
      (*pRes) = false;
×
1995
    }
1996
    goto _end;
1✔
1997
  }
1998
  if ((*pRowIndex) == pBlock->info.rows) {
12,707✔
1999
    if (pRes) {
6,217!
2000
      (*pRes) = false;
6,217✔
2001
    }
2002
    goto _end;
6,217✔
2003
  }
2004

2005
  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
6,490✔
2006
  TSKEY*           startData = (TSKEY*)pStartTsCol->pData;
6,490✔
2007
  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
6,490✔
2008
  TSKEY*           endData = (TSKEY*)pEndTsCol->pData;
6,490✔
2009
  STimeWindow      win = {.skey = startData[*pRowIndex], .ekey = endData[*pRowIndex]};
6,490✔
2010
  SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
6,490✔
2011
  uint64_t*        gpData = (uint64_t*)pGpCol->pData;
6,490✔
2012
  uint64_t         groupId = gpData[*pRowIndex];
6,490✔
2013

2014
  SColumnInfoData* pCalStartTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
6,490✔
2015
  TSKEY*           calStartData = (TSKEY*)pCalStartTsCol->pData;
6,490✔
2016
  SColumnInfoData* pCalEndTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
6,490✔
2017
  TSKEY*           calEndData = (TSKEY*)pCalEndTsCol->pData;
6,490✔
2018

2019
  setGroupId(pInfo, pBlock, GROUPID_COLUMN_INDEX, *pRowIndex);
6,490✔
2020
  if (isSlidingWindow(pInfo) || isCountSlidingWindow(pInfo)) {
6,490✔
2021
    pInfo->updateWin.skey = calStartData[*pRowIndex];
325✔
2022
    pInfo->updateWin.ekey = calEndData[*pRowIndex];
325✔
2023
  }
2024
  (*pRowIndex)++;
6,490✔
2025

2026
  for (; *pRowIndex < pBlock->info.rows; (*pRowIndex)++) {
6,491✔
2027
    if (win.skey == startData[*pRowIndex] && groupId == gpData[*pRowIndex]) {
275✔
2028
      win.ekey = TMAX(win.ekey, endData[*pRowIndex]);
1✔
2029
      continue;
1✔
2030
    }
2031

2032
    if (win.skey == endData[*pRowIndex] && groupId == gpData[*pRowIndex]) {
274!
2033
      win.skey = TMIN(win.skey, startData[*pRowIndex]);
×
2034
      continue;
×
2035
    }
2036
    break;
274✔
2037
  }
2038

2039
  STableScanInfo* pTScanInfo = pInfo->pTableScanOp->info;
6,490✔
2040

2041
  int64_t maxVer = -1;
6,490✔
2042
  if (!isRecalculateOperator(&pInfo->basic)) {
6,490!
2043
    // coverity scan
2044
    QUERY_CHECK_NULL(pInfo->pUpdateInfo, code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
6,491!
2045

2046
    qDebug("prepare range scan start:%" PRId64 ",end:%" PRId64 ",maxVer:%" PRId64, win.skey, win.ekey,
6,491✔
2047
          pInfo->pUpdateInfo->maxDataVersion);
2048
    maxVer = pInfo->pUpdateInfo->maxDataVersion;
6,491✔
2049
  }
2050
  resetTableScanInfo(pInfo->pTableScanOp->info, &win, maxVer);
6,491✔
2051
  pInfo->pTableScanOp->status = OP_OPENED;
6,491✔
2052
  if (pRes) {
6,491✔
2053
    (*pRes) = true;
274✔
2054
  }
2055

2056
_end:
6,217✔
2057
  if (code != TSDB_CODE_SUCCESS) {
12,709!
2058
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2059
  }
2060
}
12,709✔
2061

2062
STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, uint64_t* gpIdCol, SInterval* pInterval,
941✔
2063
                             SDataBlockInfo* pDataBlockInfo, int32_t* pRowIndex, bool hasGroup) {
2064
  SResultRowInfo dumyInfo = {0};
941✔
2065
  dumyInfo.cur.pageId = -1;
941✔
2066
  STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC);
941✔
2067
  STimeWindow endWin = win;
941✔
2068
  STimeWindow preWin = win;
941✔
2069
  uint64_t    groupId = gpIdCol[*pRowIndex];
941✔
2070

2071
  while (1) {
2072
    if (hasGroup) {
941✔
2073
      (*pRowIndex) += 1;
123✔
2074
    } else {
2075
      while ((groupId == gpIdCol[(*pRowIndex)] && startTsCol[*pRowIndex] <= endWin.ekey)) {
859✔
2076
        (*pRowIndex) += 1;
825✔
2077
        if ((*pRowIndex) == pDataBlockInfo->rows) {
825✔
2078
          break;
784✔
2079
        }
2080
      }
2081
    }
2082

2083
    do {
2084
      preWin = endWin;
1,724✔
2085
      getNextTimeWindow(pInterval, &endWin, TSDB_ORDER_ASC);
1,724✔
2086
    } while (endTsCol[(*pRowIndex) - 1] >= endWin.skey);
1,724✔
2087
    endWin = preWin;
941✔
2088
    if (win.ekey == endWin.ekey || (*pRowIndex) == pDataBlockInfo->rows || groupId != gpIdCol[*pRowIndex]) {
941!
2089
      win.ekey = endWin.ekey;
941✔
2090
      return win;
941✔
2091
    }
2092
    win.ekey = endWin.ekey;
×
2093
  }
2094
}
2095

2096
int32_t doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex,
8,327✔
2097
                    SSDataBlock** ppRes) {
2098
  qDebug("do stream range scan. windows index:%d", *pRowIndex);
8,327✔
2099

2100
  int32_t code = TSDB_CODE_SUCCESS;
8,327✔
2101
  int32_t lino = 0;
8,327✔
2102
  bool    prepareRes = true;
8,327✔
2103

2104
  while (1) {
467✔
2105
    SSDataBlock* pResult = NULL;
8,794✔
2106
    code = doTableScanNext(pInfo->pTableScanOp, &pResult);
8,794✔
2107
    QUERY_CHECK_CODE(code, lino, _end);
17,121!
2108

2109
    if (!pResult) {
8,794✔
2110
      prepareRangeScan(pInfo, pSDB, pRowIndex, &prepareRes);
6,491✔
2111
      // scan next window data
2112
      code = doTableScanNext(pInfo->pTableScanOp, &pResult);
6,491✔
2113
      QUERY_CHECK_CODE(code, lino, _end);
6,491!
2114
    }
2115

2116
    if (!pResult) {
8,794✔
2117
      if (prepareRes) {
6,336✔
2118
        continue;
119✔
2119
      }
2120
      blockDataCleanup(pSDB);
6,217✔
2121
      *pRowIndex = 0;
6,216✔
2122
      pInfo->updateWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
6,216✔
2123
      STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
6,216✔
2124
      pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader);
6,216✔
2125
      pTableScanInfo->base.dataReader = NULL;
6,217✔
2126
      (*ppRes) = NULL;
6,217✔
2127
      goto _end;
6,217✔
2128
    }
2129

2130
    code = doFilter(pResult, pInfo->pTableScanOp->exprSupp.pFilterInfo, NULL);
2,458✔
2131
    QUERY_CHECK_CODE(code, lino, _end);
2,458!
2132
    if (pResult->info.rows == 0) {
2,458!
2133
      continue;
×
2134
    }
2135

2136
    if (pInfo->igCheckGroupId == true) {
2,458✔
2137
      pResult->info.calWin = pInfo->updateWin;
592✔
2138
      (*ppRes) = pResult;
592✔
2139
      goto _end;
592✔
2140
    }
2141

2142
    if (pInfo->partitionSup.needCalc) {
1,866✔
2143
      SSDataBlock* tmpBlock = NULL;
414✔
2144
      code = createOneDataBlock(pResult, true, &tmpBlock);
414✔
2145
      QUERY_CHECK_CODE(code, lino, _end);
690!
2146

2147
      blockDataCleanup(pResult);
414✔
2148
      for (int32_t i = 0; i < tmpBlock->info.rows; i++) {
1,148✔
2149
        if (calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, tmpBlock, i) == pInfo->groupId) {
734✔
2150
          for (int32_t j = 0; j < pInfo->pTableScanOp->exprSupp.numOfExprs; j++) {
4,656✔
2151
            SColumnInfoData* pSrcCol = taosArrayGet(tmpBlock->pDataBlock, j);
4,217✔
2152
            SColumnInfoData* pDestCol = taosArrayGet(pResult->pDataBlock, j);
4,217✔
2153
            bool             isNull = colDataIsNull(pSrcCol, tmpBlock->info.rows, i, NULL);
4,217✔
2154
            char*            pSrcData = NULL;
4,217✔
2155
            if (!isNull) pSrcData = colDataGetData(pSrcCol, i);
4,217!
2156
            code = colDataSetVal(pDestCol, pResult->info.rows, pSrcData, isNull);
4,217✔
2157
            QUERY_CHECK_CODE(code, lino, _end);
4,217!
2158
          }
2159
          pResult->info.rows++;
439✔
2160
        }
2161
      }
2162

2163
      blockDataDestroy(tmpBlock);
414✔
2164

2165
      if (pResult->info.rows > 0) {
414✔
2166
        pResult->info.calWin = pInfo->updateWin;
276✔
2167
        (*ppRes) = pResult;
276✔
2168
        goto _end;
276✔
2169
      }
2170
    } else if (pResult->info.id.groupId == pInfo->groupId) {
1,452✔
2171
      pResult->info.calWin = pInfo->updateWin;
1,242✔
2172
      (*ppRes) = pResult;
1,242✔
2173
      goto _end;
1,242✔
2174
    }
2175
  }
2176

2177
_end:
8,327✔
2178
  if (code != TSDB_CODE_SUCCESS) {
8,327!
2179
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2180
  }
2181
  return code;
8,327✔
2182
}
2183

2184
static void getPreSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
2✔
2185
                                SSessionKey* pKey) {
2186
  pKey->win.skey = startTs;
2✔
2187
  pKey->win.ekey = endTs;
2✔
2188
  pKey->groupId = groupId;
2✔
2189

2190
  void*   pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, pKey);
2✔
2191
  int32_t code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, pKey, NULL, 0);
2✔
2192
  if (code != TSDB_CODE_SUCCESS) {
2!
2193
    SET_SESSION_WIN_KEY_INVALID(pKey);
×
2194
  }
2195

2196
  taosMemoryFree(pCur);
2!
2197
}
2✔
2198

2199
int32_t appendOneRowToSpecialBlockImpl(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, TSKEY* pCalStartTs,
6,704✔
2200
                                       TSKEY* pCalEndTs, uint64_t* pUid, uint64_t* pGp, void* pTbName, void* pPkData) {
2201
  int32_t          code = TSDB_CODE_SUCCESS;
6,704✔
2202
  int32_t          lino = 0;
6,704✔
2203
  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
6,704✔
2204
  SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
6,704✔
2205
  SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
6,704✔
2206
  SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
6,704✔
2207
  SColumnInfoData* pCalStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
6,704✔
2208
  SColumnInfoData* pCalEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
6,704✔
2209
  SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
6,704✔
2210
  code = colDataSetVal(pStartTsCol, pBlock->info.rows, (const char*)pStartTs, false);
6,704✔
2211
  QUERY_CHECK_CODE(code, lino, _end);
6,703!
2212

2213
  code = colDataSetVal(pEndTsCol, pBlock->info.rows, (const char*)pEndTs, false);
6,703✔
2214
  QUERY_CHECK_CODE(code, lino, _end);
6,703!
2215

2216
  code = colDataSetVal(pUidCol, pBlock->info.rows, (const char*)pUid, pUid == NULL);
6,703✔
2217
  QUERY_CHECK_CODE(code, lino, _end);
6,703!
2218

2219
  code = colDataSetVal(pGpCol, pBlock->info.rows, (const char*)pGp, pGp == NULL);
6,703✔
2220
  QUERY_CHECK_CODE(code, lino, _end);
6,704!
2221

2222
  code = colDataSetVal(pCalStartCol, pBlock->info.rows, (const char*)pCalStartTs, false);
6,704✔
2223
  QUERY_CHECK_CODE(code, lino, _end);
6,704!
2224

2225
  code = colDataSetVal(pCalEndCol, pBlock->info.rows, (const char*)pCalEndTs, false);
6,704✔
2226
  QUERY_CHECK_CODE(code, lino, _end);
6,704!
2227

2228
  code = colDataSetVal(pTableCol, pBlock->info.rows, (const char*)pTbName, pTbName == NULL);
6,704✔
2229
  QUERY_CHECK_CODE(code, lino, _end);
6,704!
2230

2231
  if (taosArrayGetSize(pBlock->pDataBlock) > PRIMARY_KEY_COLUMN_INDEX) {
6,704✔
2232
    SColumnInfoData* pPkCol = taosArrayGet(pBlock->pDataBlock, PRIMARY_KEY_COLUMN_INDEX);
3✔
2233
    code = colDataSetVal(pPkCol, pBlock->info.rows, (const char*)pPkData, pPkData == NULL);
3✔
2234
    QUERY_CHECK_CODE(code, lino, _end);
3!
2235
  }
2236
  pBlock->info.rows++;
6,704✔
2237

2238
_end:
6,704✔
2239
  if (code != TSDB_CODE_SUCCESS) {
6,704!
2240
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2241
  }
2242
  return code;
6,704✔
2243
}
2244

2245
int32_t appendPkToSpecialBlock(SSDataBlock* pBlock, TSKEY* pTsArray, SColumnInfoData* pPkCol, int32_t rowId,
1,077✔
2246
                               uint64_t* pUid, uint64_t* pGp, void* pTbName) {
2247
  void* pVal = NULL;
1,077✔
2248
  if (pPkCol) {
1,077✔
2249
    pVal = colDataGetData(pPkCol, rowId);
9!
2250
  }
2251
  return appendOneRowToSpecialBlockImpl(pBlock, pTsArray + rowId, pTsArray + rowId, pTsArray + rowId, pTsArray + rowId,
1,077✔
2252
                                        pUid, pGp, pTbName, pVal);
2253
}
2254

2255
static int32_t getPreVersionDataBlock(uint64_t uid, TSKEY startTs, TSKEY endTs, int64_t version, char* taskIdStr,
7✔
2256
                                      SStreamScanInfo* pInfo, SSDataBlock* pBlock) {
2257
  int32_t      code = TSDB_CODE_SUCCESS;
7✔
2258
  int32_t      lino = 0;
7✔
2259
  SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, uid, startTs, endTs, version);
7✔
2260
  printDataBlock(pPreRes, "pre res", taskIdStr);
7✔
2261
  blockDataCleanup(pBlock);
7✔
2262
  if (!pPreRes) {
7!
2263
    if (terrno != TSDB_CODE_SUCCESS) {
×
2264
      qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));
×
2265
    }
2266
    goto _end;
×
2267
  }
2268
  code = blockDataEnsureCapacity(pBlock, pPreRes->info.rows);
7✔
2269
  QUERY_CHECK_CODE(code, lino, _end);
7!
2270

2271
  SColumnInfoData* pTsCol = (SColumnInfoData*)taosArrayGet(pPreRes->pDataBlock, pInfo->primaryTsIndex);
7✔
2272
  SColumnInfoData* pPkCol = NULL;
7✔
2273
  if (hasPrimaryKeyCol(pInfo)) {
7✔
2274
    pPkCol = (SColumnInfoData*)taosArrayGet(pPreRes->pDataBlock, pInfo->primaryKeyIndex);
2✔
2275
  }
2276
  for (int32_t i = 0; i < pPreRes->info.rows; i++) {
28✔
2277
    uint64_t groupId = calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pPreRes, i);
21✔
2278
    code = appendPkToSpecialBlock(pBlock, (TSKEY*)pTsCol->pData, pPkCol, i, &uid, &groupId, NULL);
21✔
2279
    QUERY_CHECK_CODE(code, lino, _end);
21!
2280
  }
2281
  printDataBlock(pBlock, "new delete", taskIdStr);
7✔
2282

2283
_end:
7✔
2284
  if (code != TSDB_CODE_SUCCESS) {
7!
2285
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2286
  }
2287
  return code;
7✔
2288
}
2289

2290
static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock,
560✔
2291
                                        EStreamType mode) {
2292
  int32_t code = TSDB_CODE_SUCCESS;
560✔
2293
  int32_t lino = 0;
560✔
2294
  if (pSrcBlock->info.rows == 0) {
560!
2295
    return TSDB_CODE_SUCCESS;
×
2296
  }
2297
  SSHashObj*       pScanRange = tSimpleHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
560✔
2298
  SExecTaskInfo*   pTaskInfo = pInfo->pStreamScanOp->pTaskInfo;
560✔
2299
  SColumnInfoData* pStartTsCol = taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX);
560✔
2300
  TSKEY*           startData = (TSKEY*)pStartTsCol->pData;
560✔
2301
  SColumnInfoData* pEndTsCol = taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX);
560✔
2302
  TSKEY*           endData = (TSKEY*)pEndTsCol->pData;
560✔
2303
  SColumnInfoData* pUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX);
560✔
2304
  uint64_t*        uidCol = (uint64_t*)pUidCol->pData;
560✔
2305
  SColumnInfoData* pGpCol = taosArrayGet(pSrcBlock->pDataBlock, GROUPID_COLUMN_INDEX);
560✔
2306
  SColumnInfoData* pSrcPkCol = NULL;
560✔
2307
  uint64_t*        pSrcGp = (uint64_t*)pGpCol->pData;
560✔
2308
  if (taosArrayGetSize(pSrcBlock->pDataBlock) > PRIMARY_KEY_COLUMN_INDEX) {
560✔
2309
    pSrcPkCol = taosArrayGet(pSrcBlock->pDataBlock, PRIMARY_KEY_COLUMN_INDEX);
1✔
2310
  }
2311
  int64_t ver = pSrcBlock->info.version - 1;
560✔
2312

2313
  if (pInfo->partitionSup.needCalc &&
560✔
2314
      (startData[0] != endData[0] || (hasPrimaryKeyCol(pInfo) && mode == STREAM_DELETE_DATA))) {
228✔
2315
    code = getPreVersionDataBlock(uidCol[0], startData[0], endData[0], ver, GET_TASKID(pTaskInfo), pInfo, pSrcBlock);
4✔
2316
    QUERY_CHECK_CODE(code, lino, _end);
4!
2317
    startData = (TSKEY*)pStartTsCol->pData;
4✔
2318
    endData = (TSKEY*)pEndTsCol->pData;
4✔
2319
    uidCol = (uint64_t*)pUidCol->pData;
4✔
2320
    pSrcGp = (uint64_t*)pGpCol->pData;
4✔
2321
  }
2322
  blockDataCleanup(pDestBlock);
560✔
2323
  code = blockDataEnsureCapacity(pDestBlock, pSrcBlock->info.rows);
560✔
2324
  QUERY_CHECK_CODE(code, lino, _end);
560!
2325

2326
  SColumnInfoData* pDestStartCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX);
560✔
2327
  SColumnInfoData* pDestEndCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX);
560✔
2328
  SColumnInfoData* pDestUidCol = taosArrayGet(pDestBlock->pDataBlock, UID_COLUMN_INDEX);
560✔
2329
  SColumnInfoData* pDestGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX);
560✔
2330
  SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
560✔
2331
  SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
560✔
2332
  SColumnInfoData* pDestTableNameInxCol = taosArrayGet(pDestBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
560✔
2333
  for (int32_t i = 0; i < pSrcBlock->info.rows; i++) {
1,166✔
2334
    uint64_t groupId = pSrcGp[i];
606✔
2335
    if (groupId == 0) {
606✔
2336
      void* pVal = NULL;
519✔
2337
      if (hasPrimaryKeyCol(pInfo) && pSrcPkCol) {
519!
2338
        pVal = colDataGetData(pSrcPkCol, i);
1!
2339
      }
2340
      groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], ver, pVal);
519✔
2341
    }
2342
    // gap must be 0.
2343
    SSessionKey startWin = {0};
606✔
2344
    getCurSessionWindow(pInfo->windowSup.pStreamAggSup, startData[i], startData[i], groupId, &startWin);
606✔
2345
    if (IS_INVALID_SESSION_WIN_KEY(startWin)) {
606✔
2346
      // window has been closed.
2347
      continue;
52✔
2348
    }
2349
    SSessionKey endWin = {0};
559✔
2350
    getCurSessionWindow(pInfo->windowSup.pStreamAggSup, endData[i], endData[i], groupId, &endWin);
559✔
2351
    if (IS_INVALID_SESSION_WIN_KEY(endWin)) {
559✔
2352
      getPreSessionWindow(pInfo->windowSup.pStreamAggSup, endData[i], endData[i], groupId, &endWin);
2✔
2353
    }
2354
    if (IS_INVALID_SESSION_WIN_KEY(startWin)) {
559!
2355
      // window has been closed.
2356
      qError("generate session scan range failed. rang start:%" PRIx64 ", end:%" PRIx64, startData[i], endData[i]);
×
2357
      continue;
×
2358
    }
2359

2360
    SSessionKey checkKey = {.groupId = groupId, .win.skey = startWin.win.skey, .win.ekey = endWin.win.ekey};
559✔
2361
    if (tSimpleHashGet(pScanRange, &checkKey, sizeof(SSessionKey)) != NULL) {
559✔
2362
      continue;
5✔
2363
    }
2364
    code = tSimpleHashPut(pScanRange, &checkKey, sizeof(SSessionKey), NULL, 0);
554✔
2365
    QUERY_CHECK_CODE(code, lino, _end);
554!
2366

2367
    code = colDataSetVal(pDestStartCol, i, (const char*)&startWin.win.skey, false);
554✔
2368
    QUERY_CHECK_CODE(code, lino, _end);
554!
2369

2370
    code = colDataSetVal(pDestEndCol, i, (const char*)&endWin.win.ekey, false);
554✔
2371
    QUERY_CHECK_CODE(code, lino, _end);
554!
2372

2373
    colDataSetNULL(pDestUidCol, i);
554!
2374
    code = colDataSetVal(pDestGpCol, i, (const char*)&groupId, false);
554✔
2375
    QUERY_CHECK_CODE(code, lino, _end);
554!
2376

2377
    colDataSetNULL(pDestCalStartTsCol, i);
554!
2378
    colDataSetNULL(pDestTableNameInxCol, i);
554!
2379
    pDestBlock->info.rows++;
554✔
2380
  }
2381

2382
_end:
560✔
2383
  tSimpleHashCleanup(pScanRange);
560✔
2384
  if (code != TSDB_CODE_SUCCESS) {
560!
2385
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2386
  }
2387
  return code;
560✔
2388
}
2389

2390
static int32_t generateCountScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock,
7✔
2391
                                      EStreamType mode) {
2392
  int32_t code = TSDB_CODE_SUCCESS;
7✔
2393
  int32_t lino = 0;
7✔
2394
  blockDataCleanup(pDestBlock);
7✔
2395
  if (pSrcBlock->info.rows == 0) {
7!
2396
    return TSDB_CODE_SUCCESS;
×
2397
  }
2398
  SExecTaskInfo*   pTaskInfo = pInfo->pStreamScanOp->pTaskInfo;
7✔
2399
  SColumnInfoData* pStartTsCol = taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX);
7✔
2400
  TSKEY*           startData = (TSKEY*)pStartTsCol->pData;
7✔
2401
  SColumnInfoData* pEndTsCol = taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX);
7✔
2402
  TSKEY*           endData = (TSKEY*)pEndTsCol->pData;
7✔
2403
  SColumnInfoData* pUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX);
7✔
2404
  uint64_t*        uidCol = (uint64_t*)pUidCol->pData;
7✔
2405
  SColumnInfoData* pGpCol = taosArrayGet(pSrcBlock->pDataBlock, GROUPID_COLUMN_INDEX);
7✔
2406
  uint64_t*        pSrcGp = (uint64_t*)pGpCol->pData;
7✔
2407
  SColumnInfoData* pSrcPkCol = NULL;
7✔
2408
  if (taosArrayGetSize(pSrcBlock->pDataBlock) > PRIMARY_KEY_COLUMN_INDEX) {
7!
2409
    pSrcPkCol = taosArrayGet(pSrcBlock->pDataBlock, PRIMARY_KEY_COLUMN_INDEX);
×
2410
  }
2411
  int64_t ver = pSrcBlock->info.version - 1;
7✔
2412

2413
  if (pInfo->partitionSup.needCalc &&
7✔
2414
      (startData[0] != endData[0] || (hasPrimaryKeyCol(pInfo) && mode == STREAM_DELETE_DATA))) {
1!
2415
    code = getPreVersionDataBlock(uidCol[0], startData[0], endData[0], ver, GET_TASKID(pTaskInfo), pInfo, pSrcBlock);
1✔
2416
    QUERY_CHECK_CODE(code, lino, _end);
1!
2417
    startData = (TSKEY*)pStartTsCol->pData;
1✔
2418
    endData = (TSKEY*)pEndTsCol->pData;
1✔
2419
    uidCol = (uint64_t*)pUidCol->pData;
1✔
2420
    pSrcGp = (uint64_t*)pGpCol->pData;
1✔
2421
  }
2422

2423
  code = blockDataEnsureCapacity(pDestBlock, pSrcBlock->info.rows);
7✔
2424
  QUERY_CHECK_CODE(code, lino, _end);
7!
2425

2426
  SColumnInfoData* pDestStartCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX);
7✔
2427
  SColumnInfoData* pDestEndCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX);
7✔
2428
  SColumnInfoData* pDestUidCol = taosArrayGet(pDestBlock->pDataBlock, UID_COLUMN_INDEX);
7✔
2429
  SColumnInfoData* pDestGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX);
7✔
2430
  SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
7✔
2431
  SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
7✔
2432
  SColumnInfoData* pDestTableNameInxCol = taosArrayGet(pDestBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
7✔
2433
  for (int32_t i = 0; i < pSrcBlock->info.rows; i++) {
16✔
2434
    uint64_t groupId = pSrcGp[i];
9✔
2435
    if (groupId == 0) {
9✔
2436
      void* pVal = NULL;
6✔
2437
      if (hasPrimaryKeyCol(pInfo) && pSrcPkCol) {
6!
2438
        pVal = colDataGetData(pSrcPkCol, i);
×
2439
      }
2440
      groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], ver, pVal);
6✔
2441
    }
2442
    SSessionKey startWin = {.win.skey = startData[i], .win.ekey = endData[i], .groupId = groupId};
9✔
2443
    SSessionKey range = {0};
9✔
2444
    getCountWinRange(pInfo->windowSup.pStreamAggSup, &startWin, mode, &range);
9✔
2445
    code = colDataSetVal(pDestStartCol, i, (const char*)&range.win.skey, false);
9✔
2446
    QUERY_CHECK_CODE(code, lino, _end);
9!
2447

2448
    code = colDataSetVal(pDestEndCol, i, (const char*)&range.win.ekey, false);
9✔
2449
    QUERY_CHECK_CODE(code, lino, _end);
9!
2450

2451
    colDataSetNULL(pDestUidCol, i);
9!
2452
    code = colDataSetVal(pDestGpCol, i, (const char*)&groupId, false);
9✔
2453
    QUERY_CHECK_CODE(code, lino, _end);
9!
2454

2455
    code = colDataSetVal(pDestCalStartTsCol, i, (const char*)&range.win.skey, false);
9✔
2456
    QUERY_CHECK_CODE(code, lino, _end);
9!
2457

2458
    code = colDataSetVal(pDestCalEndTsCol, i, (const char*)&range.win.ekey, false);
9✔
2459
    QUERY_CHECK_CODE(code, lino, _end);
9!
2460

2461
    colDataSetNULL(pDestTableNameInxCol, i);
9!
2462

2463
    pDestBlock->info.rows++;
9✔
2464
  }
2465

2466
_end:
7✔
2467
  if (code != TSDB_CODE_SUCCESS) {
7!
2468
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2469
  }
2470
  return code;
7✔
2471
}
2472

2473
static int32_t setDelRangeEndKey(SStreamAggSupporter* pAggSup, SStreamFillSupporter* pFillSup, SWinKey* pEndKey, STimeWindow* pScanRange, bool* pRes) {
24✔
2474
  int32_t     code = TSDB_CODE_SUCCESS;
24✔
2475
  int32_t     lino = 0;
24✔
2476
  SSlicePoint nextPoint = {.key.groupId = pEndKey->groupId};
24✔
2477
  int32_t     vLen = 0;
24✔
2478
  int32_t     winCode = TSDB_CODE_SUCCESS;
24✔
2479
  code = pAggSup->stateStore.streamStateFillGetNext(pAggSup->pState, pEndKey, &nextPoint.key, (void**)&nextPoint.pResPos, &vLen, &winCode);
24✔
2480
  QUERY_CHECK_CODE(code, lino, _end);
24!
2481
  if (winCode == TSDB_CODE_SUCCESS) {
24✔
2482
    setPointBuff(&nextPoint, pFillSup);
16✔
2483
    if (HAS_ROW_DATA(nextPoint.pLeftRow) && pEndKey->ts < nextPoint.pLeftRow->key) {
16!
2484
      pScanRange->ekey = nextPoint.pLeftRow->key;
8✔
2485
      *pRes = true;
8✔
2486
    } else if (pEndKey->ts < nextPoint.pRightRow->key) {
8✔
2487
      pScanRange->ekey = nextPoint.pRightRow->key;
2✔
2488
      *pRes = true;
2✔
2489
    } else {
2490
      *pEndKey = nextPoint.key;
6✔
2491
      pScanRange->ekey = TMAX(nextPoint.pRightRow->key, nextPoint.key.ts);
6✔
2492
      *pRes = false;
6✔
2493
    }
2494
  }
2495

2496
_end:
8✔
2497
  if (code != TSDB_CODE_SUCCESS) {
24!
2498
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2499
  }
2500
  return code;
24✔
2501
}
2502

2503
int32_t getTimeSliceWinRange(SStreamAggSupporter* pAggSup, SStreamFillSupporter* pFillSup, SInterval* pInterval, TSKEY start, TSKEY end,
15✔
2504
                             int64_t groupId, STimeWindow* pScanRange) {
2505
  int32_t        code = TSDB_CODE_SUCCESS;
15✔
2506
  int32_t        lino = 0;
15✔
2507
  int32_t        winCode = TSDB_CODE_SUCCESS;
15✔
2508
  SResultRowInfo dumyInfo = {0};
15✔
2509
  dumyInfo.cur.pageId = -1;
15✔
2510
  STimeWindow sWin = getActiveTimeWindow(NULL, &dumyInfo, start, pInterval, TSDB_ORDER_ASC);
15✔
2511
  SWinKey     startKey = {.groupId = groupId, .ts = sWin.skey};
15✔
2512

2513
  sWin = getActiveTimeWindow(NULL, &dumyInfo, end, pInterval, TSDB_ORDER_ASC);
15✔
2514
  SWinKey endKey = {.groupId = groupId, .ts = sWin.ekey};
15✔
2515

2516
  SSlicePoint prevPoint = {.key.groupId = groupId};
15✔
2517
  SSlicePoint nextPoint = {.key.groupId = groupId};
15✔
2518
  int32_t     vLen = 0;
15✔
2519
  code = pAggSup->stateStore.streamStateFillGetPrev(pAggSup->pState, &startKey, &prevPoint.key, (void**)&prevPoint.pResPos, &vLen, &winCode);
15✔
2520
  QUERY_CHECK_CODE(code, lino, _end);
15!
2521
  if (winCode == TSDB_CODE_SUCCESS) {
15✔
2522
    setPointBuff(&prevPoint, pFillSup);
10✔
2523
    if (HAS_ROW_DATA(prevPoint.pRightRow)) {
10!
2524
      pScanRange->skey = prevPoint.pRightRow->key;
9✔
2525
    } else {
2526
      pScanRange->skey = prevPoint.pLeftRow->key;
1✔
2527
    }
2528
  } else {
2529
    pScanRange->skey = startKey.ts;
5✔
2530
  }
2531

2532
  bool res = false;
15✔
2533
  SWinKey curKey = endKey;
15✔
2534
  code = setDelRangeEndKey(pAggSup, pFillSup, &curKey, pScanRange, &res);
15✔
2535
  QUERY_CHECK_CODE(code, lino, _end);
15!
2536
  if (res == false) {
15✔
2537
    code = setDelRangeEndKey(pAggSup, pFillSup, &curKey, pScanRange, &res);
9✔
2538
    QUERY_CHECK_CODE(code, lino, _end);
9!
2539
  }
2540
  if (res == false) {
15✔
2541
    pScanRange->ekey = TMAX(endKey.ts, pScanRange->ekey);
5✔
2542
  }
2543

2544
_end:
10✔
2545
  if (code != TSDB_CODE_SUCCESS) {
15!
2546
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2547
  }
2548
  return code;
15✔
2549
}
2550

2551
static int32_t generateTimeSliceScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock,
15✔
2552
                                          EStreamType mode) {
2553
  int32_t code = TSDB_CODE_SUCCESS;
15✔
2554
  int32_t lino = 0;
15✔
2555
  blockDataCleanup(pDestBlock);
15✔
2556
  if (pSrcBlock->info.rows == 0) {
15!
2557
    return TSDB_CODE_SUCCESS;
×
2558
  }
2559
  SExecTaskInfo*   pTaskInfo = pInfo->pStreamScanOp->pTaskInfo;
15✔
2560
  SColumnInfoData* pStartTsCol = taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX);
15✔
2561
  TSKEY*           startData = (TSKEY*)pStartTsCol->pData;
15✔
2562
  SColumnInfoData* pEndTsCol = taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX);
15✔
2563
  TSKEY*           endData = (TSKEY*)pEndTsCol->pData;
15✔
2564
  SColumnInfoData* pUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX);
15✔
2565
  uint64_t*        uidCol = (uint64_t*)pUidCol->pData;
15✔
2566
  SColumnInfoData* pGpCol = taosArrayGet(pSrcBlock->pDataBlock, GROUPID_COLUMN_INDEX);
15✔
2567
  uint64_t*        pSrcGp = (uint64_t*)pGpCol->pData;
15✔
2568
  SColumnInfoData* pSrcPkCol = NULL;
15✔
2569
  if (taosArrayGetSize(pSrcBlock->pDataBlock) > PRIMARY_KEY_COLUMN_INDEX) {
15!
2570
    pSrcPkCol = taosArrayGet(pSrcBlock->pDataBlock, PRIMARY_KEY_COLUMN_INDEX);
×
2571
  }
2572
  int64_t ver = pSrcBlock->info.version - 1;
15✔
2573

2574
  if (pInfo->partitionSup.needCalc &&
15!
2575
      (startData[0] != endData[0] || (hasPrimaryKeyCol(pInfo) && mode == STREAM_DELETE_DATA))) {
×
2576
    code = getPreVersionDataBlock(uidCol[0], startData[0], endData[0], ver, GET_TASKID(pTaskInfo), pInfo, pSrcBlock);
×
2577
    QUERY_CHECK_CODE(code, lino, _end);
×
2578
    startData = (TSKEY*)pStartTsCol->pData;
×
2579
    endData = (TSKEY*)pEndTsCol->pData;
×
2580
    uidCol = (uint64_t*)pUidCol->pData;
×
2581
    pSrcGp = (uint64_t*)pGpCol->pData;
×
2582
  }
2583

2584
  code = blockDataEnsureCapacity(pDestBlock, pSrcBlock->info.rows);
15✔
2585
  QUERY_CHECK_CODE(code, lino, _end);
15!
2586

2587
  SColumnInfoData* pDestStartCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX);
15✔
2588
  SColumnInfoData* pDestEndCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX);
15✔
2589
  SColumnInfoData* pDestUidCol = taosArrayGet(pDestBlock->pDataBlock, UID_COLUMN_INDEX);
15✔
2590
  SColumnInfoData* pDestGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX);
15✔
2591
  SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
15✔
2592
  SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
15✔
2593
  for (int32_t i = 0; i < pSrcBlock->info.rows; i++) {
30✔
2594
    uint64_t groupId = pSrcGp[i];
15✔
2595
    if (groupId == 0) {
15!
2596
      void* pVal = NULL;
15✔
2597
      if (hasPrimaryKeyCol(pInfo) && pSrcPkCol) {
15!
2598
        pVal = colDataGetData(pSrcPkCol, i);
×
2599
      }
2600
      groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], ver, pVal);
15✔
2601
    }
2602

2603
    STimeWindow scanRange = {0};
15✔
2604
    code = getTimeSliceWinRange(pInfo->windowSup.pStreamAggSup, pInfo->pFillSup, &pInfo->interval, startData[i], endData[i], groupId,
15✔
2605
                                &scanRange);
2606
    QUERY_CHECK_CODE(code, lino, _end);
15!
2607

2608
    code = colDataSetVal(pDestStartCol, i, (const char*)&scanRange.skey, false);
15✔
2609
    QUERY_CHECK_CODE(code, lino, _end);
15!
2610

2611
    code = colDataSetVal(pDestEndCol, i, (const char*)&scanRange.ekey, false);
15✔
2612
    QUERY_CHECK_CODE(code, lino, _end);
15!
2613

2614
    colDataSetNULL(pDestUidCol, i);
15!
2615
    code = colDataSetVal(pDestGpCol, i, (const char*)&groupId, false);
15✔
2616
    QUERY_CHECK_CODE(code, lino, _end);
15!
2617

2618
    code = colDataSetVal(pDestCalStartTsCol, i, (const char*)&scanRange.skey, false);
15✔
2619
    QUERY_CHECK_CODE(code, lino, _end);
15!
2620

2621
    code = colDataSetVal(pDestCalEndTsCol, i, (const char*)&scanRange.ekey, false);
15✔
2622
    QUERY_CHECK_CODE(code, lino, _end);
15!
2623

2624
    pDestBlock->info.rows++;
15✔
2625
  }
2626

2627
_end:
15✔
2628
  if (code != TSDB_CODE_SUCCESS) {
15!
2629
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2630
  }
2631
  return code;
15✔
2632
}
2633

2634
static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock,
889✔
2635
                                         EStreamType mode) {
2636
  int32_t code = TSDB_CODE_SUCCESS;
889✔
2637
  int32_t lino = 0;
889✔
2638
  blockDataCleanup(pDestBlock);
889✔
2639
  if (pSrcBlock->info.rows == 0) {
889!
2640
    return TSDB_CODE_SUCCESS;
×
2641
  }
2642
  SSHashObj*       pScanRange = tSimpleHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
889✔
2643
  SExecTaskInfo*   pTaskInfo = pInfo->pStreamScanOp->pTaskInfo;
889✔
2644
  SColumnInfoData* pSrcStartTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX);
889✔
2645
  SColumnInfoData* pSrcEndTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX);
889✔
2646
  SColumnInfoData* pSrcUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX);
889✔
2647
  SColumnInfoData* pSrcGpCol = taosArrayGet(pSrcBlock->pDataBlock, GROUPID_COLUMN_INDEX);
889✔
2648
  SColumnInfoData* pSrcPkCol = NULL;
889✔
2649
  if (taosArrayGetSize(pSrcBlock->pDataBlock) > PRIMARY_KEY_COLUMN_INDEX) {
889✔
2650
    pSrcPkCol = taosArrayGet(pSrcBlock->pDataBlock, PRIMARY_KEY_COLUMN_INDEX);
1✔
2651
  }
2652

2653
  uint64_t* srcUidData = (uint64_t*)pSrcUidCol->pData;
889✔
2654
  TSKEY*    srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData;
889✔
2655
  TSKEY*    srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData;
889✔
2656
  int64_t   ver = pSrcBlock->info.version - 1;
889✔
2657

2658
  if (pInfo->partitionSup.needCalc &&
889✔
2659
      (srcStartTsCol[0] != srcEndTsCol[0] || (hasPrimaryKeyCol(pInfo) && mode == STREAM_DELETE_DATA))) {
105✔
2660
    code = getPreVersionDataBlock(srcUidData[0], srcStartTsCol[0], srcEndTsCol[0], ver, GET_TASKID(pTaskInfo), pInfo,
2✔
2661
                                  pSrcBlock);
2662
    QUERY_CHECK_CODE(code, lino, _end);
2!
2663

2664
    srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData;
2✔
2665
    srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData;
2✔
2666
    srcUidData = (uint64_t*)pSrcUidCol->pData;
2✔
2667
  }
2668

2669
  uint64_t* srcGp = (uint64_t*)pSrcGpCol->pData;
889✔
2670
  code = blockDataEnsureCapacity(pDestBlock, pSrcBlock->info.rows);
889✔
2671
  QUERY_CHECK_CODE(code, lino, _end);
889!
2672

2673
  SColumnInfoData* pStartTsCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX);
889✔
2674
  SColumnInfoData* pEndTsCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX);
889✔
2675
  SColumnInfoData* pDeUidCol = taosArrayGet(pDestBlock->pDataBlock, UID_COLUMN_INDEX);
889✔
2676
  SColumnInfoData* pGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX);
889✔
2677
  SColumnInfoData* pCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
889✔
2678
  SColumnInfoData* pCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
889✔
2679
  SColumnInfoData* pDestTableNameInxCol = taosArrayGet(pDestBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
889✔
2680
  for (int32_t i = 0; i < pSrcBlock->info.rows;) {
1,830✔
2681
    uint64_t srcUid = srcUidData[i];
941✔
2682
    uint64_t groupId = srcGp[i];
941✔
2683
    if (groupId == 0) {
941✔
2684
      void* pVal = NULL;
828✔
2685
      if (hasPrimaryKeyCol(pInfo) && pSrcPkCol) {
828!
2686
        pVal = colDataGetData(pSrcPkCol, i);
1!
2687
      }
2688
      groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], ver, pVal);
828✔
2689
    }
2690
    TSKEY calStartTs = srcStartTsCol[i];
941✔
2691
    code = colDataSetVal(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false);
941✔
2692
    QUERY_CHECK_CODE(code, lino, _end);
941!
2693

2694
    STimeWindow win = getSlidingWindow(srcStartTsCol, srcEndTsCol, srcGp, &pInfo->interval, &pSrcBlock->info, &i,
941✔
2695
                                       pInfo->partitionSup.needCalc);
941✔
2696
    TSKEY       calEndTs = srcStartTsCol[i - 1];
941✔
2697
    code = colDataSetVal(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false);
941✔
2698
    QUERY_CHECK_CODE(code, lino, _end);
941!
2699

2700
    code = colDataSetVal(pDeUidCol, pDestBlock->info.rows, (const char*)(&srcUid), false);
941✔
2701
    QUERY_CHECK_CODE(code, lino, _end);
941!
2702

2703
    SSessionKey checkKey = {.groupId = groupId, .win = win};
941✔
2704
    if (tSimpleHashGet(pScanRange, &checkKey, sizeof(SSessionKey)) != NULL) {
941!
2705
      continue;
×
2706
    }
2707
    code = tSimpleHashPut(pScanRange, &checkKey, sizeof(SSessionKey), NULL, 0);
941✔
2708
    QUERY_CHECK_CODE(code, lino, _end);
941!
2709

2710
    code = colDataSetVal(pStartTsCol, pDestBlock->info.rows, (const char*)(&win.skey), false);
941✔
2711
    QUERY_CHECK_CODE(code, lino, _end);
941!
2712

2713
    code = colDataSetVal(pEndTsCol, pDestBlock->info.rows, (const char*)(&win.ekey), false);
941✔
2714
    QUERY_CHECK_CODE(code, lino, _end);
941!
2715

2716
    code = colDataSetVal(pGpCol, pDestBlock->info.rows, (const char*)(&groupId), false);
941✔
2717
    QUERY_CHECK_CODE(code, lino, _end);
941!
2718

2719
    colDataSetNULL(pDestTableNameInxCol,  pDestBlock->info.rows);
941!
2720

2721
    pDestBlock->info.rows++;
941✔
2722
  }
2723

2724
_end:
889✔
2725
  tSimpleHashCleanup(pScanRange);
889✔
2726
  if (code != TSDB_CODE_SUCCESS) {
889!
2727
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2728
  }
2729
  return code;
889✔
2730
}
2731

2732
int32_t calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t rowId) {
532,757✔
2733
  int32_t        code = TSDB_CODE_SUCCESS;
532,757✔
2734
  int32_t        lino = 0;
532,757✔
2735
  SExecTaskInfo* pTaskInfo = pInfo->pStreamScanOp->pTaskInfo;
532,757✔
2736
  const char*    idStr = GET_TASKID(pTaskInfo);
532,757✔
2737

2738
  blockDataCleanup(pInfo->pCreateTbRes);
532,757✔
2739

2740
  if (pInfo->tbnameCalSup.numOfExprs == 0 && pInfo->tagCalSup.numOfExprs == 0) {
532,794✔
2741
    pBlock->info.parTbName[0] = 0;
524,645✔
2742
    if (pInfo->hasPart == false) {
524,645✔
2743
      pInfo->stateStore.streamStateSetParNameInvalid(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState);
522,691✔
2744
      clearParTbNameHashPtr(pTaskInfo->pRoot, idStr, &pTaskInfo->storageAPI);
522,635✔
2745
    }
2746
  } else {
2747
    code = appendCreateTableRow(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, &pInfo->tbnameCalSup,
8,149✔
2748
                                &pInfo->tagCalSup, pBlock->info.id.groupId, pBlock, rowId, pInfo->pCreateTbRes,
2749
                                &pInfo->stateStore);
2750
    QUERY_CHECK_CODE(code, lino, _end);
8,162!
2751
  }
2752

2753
  qTrace("%s %s child_table_name:%s,groupId:%" PRIu64, idStr, __func__, pBlock->info.parTbName, pBlock->info.id.groupId);
532,820✔
2754
_end:
529,796✔
2755
  if (code != TSDB_CODE_SUCCESS) {
532,820!
2756
    qError("%s %s failed at line %d since %s", idStr, __func__, lino, tstrerror(code));
×
2757
  }
2758
  return code;
532,731✔
2759
}
2760

2761
static int32_t generatePartitionDelResBlock(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) {
51✔
2762
  int32_t          code = TSDB_CODE_SUCCESS;
51✔
2763
  int32_t          lino = 0;
51✔
2764
  SColumnInfoData* pSrcStartTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX);
51✔
2765
  SColumnInfoData* pSrcEndTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX);
51✔
2766
  SColumnInfoData* pSrcUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX);
51✔
2767
  uint64_t*        srcUidData = (uint64_t*)pSrcUidCol->pData;
51✔
2768
  SColumnInfoData* pSrcGpCol = taosArrayGet(pSrcBlock->pDataBlock, GROUPID_COLUMN_INDEX);
51✔
2769
  uint64_t*        srcGp = (uint64_t*)pSrcGpCol->pData;
51✔
2770

2771
  TSKEY*  srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData;
51✔
2772
  TSKEY*  srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData;
51✔
2773
  int64_t ver = pSrcBlock->info.version - 1;
51✔
2774
  for (int32_t delI = 0; delI < pSrcBlock->info.rows; delI++) {
102✔
2775
    uint64_t     groupId = 0;
51✔
2776
    uint64_t     srcUid = srcUidData[delI];
51✔
2777
    char         tbname[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN] = {0};
51✔
2778
    SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, srcUid, srcStartTsCol[delI], srcEndTsCol[delI], ver);
51✔
2779
    if (!pPreRes) {
51!
2780
      qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));
×
2781
      continue;
×
2782
    }
2783
    code = blockDataEnsureCapacity(pDestBlock, pDestBlock->info.rows + pPreRes->info.rows);
51✔
2784
    QUERY_CHECK_CODE(code, lino, _end);
51!
2785
    for (int32_t preJ = 0; preJ < pPreRes->info.rows; preJ++) {
107✔
2786
      groupId = calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pPreRes, preJ);
56✔
2787
      if (pInfo->pPartTbnameSup) {
56!
2788
        void*   parTbname = NULL;
56✔
2789
        int32_t winCode = TSDB_CODE_SUCCESS;
56✔
2790
        code = pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId,
56✔
2791
                                                       &parTbname, false, &winCode);
2792
        QUERY_CHECK_CODE(code, lino, _end);
56!
2793

2794
        if (winCode != TSDB_CODE_SUCCESS) {
56✔
2795
          code = calBlockTbName(pInfo, pPreRes, preJ);
53✔
2796
          QUERY_CHECK_CODE(code, lino, _end);
53!
2797
          memcpy(varDataVal(tbname), pPreRes->info.parTbName, strlen(pPreRes->info.parTbName));
53✔
2798
        } else {
2799
          memcpy(varDataVal(tbname), parTbname, TSDB_TABLE_NAME_LEN);
3✔
2800
        }
2801
        varDataSetLen(tbname, strlen(varDataVal(tbname)));
56✔
2802
        pInfo->stateStore.streamStateFreeVal(parTbname);
56✔
2803
      }
2804
      code = appendDataToSpecialBlock(pDestBlock, srcStartTsCol + delI, srcEndTsCol + delI, srcUidData + delI, &groupId,
56✔
2805
                                      tbname[0] == 0 ? NULL : tbname);
56✔
2806
      QUERY_CHECK_CODE(code, lino, _end);
56!
2807
    }
2808
  }
2809

2810
_end:
51✔
2811
  if (code != TSDB_CODE_SUCCESS) {
51!
2812
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2813
  }
2814
  return code;
51✔
2815
}
2816

2817
static int32_t generateDeleteResultBlockImpl(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) {
1✔
2818
  int32_t          code = TSDB_CODE_SUCCESS;
1✔
2819
  int32_t          lino = 0;
1✔
2820
  SColumnInfoData* pSrcStartTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX);
1✔
2821
  SColumnInfoData* pSrcEndTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX);
1✔
2822
  SColumnInfoData* pSrcUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX);
1✔
2823
  uint64_t*        srcUidData = (uint64_t*)pSrcUidCol->pData;
1✔
2824
  SColumnInfoData* pSrcGpCol = taosArrayGet(pSrcBlock->pDataBlock, GROUPID_COLUMN_INDEX);
1✔
2825
  uint64_t*        srcGp = (uint64_t*)pSrcGpCol->pData;
1✔
2826
  SColumnInfoData* pSrcPkCol = NULL;
1✔
2827
  if (taosArrayGetSize(pSrcBlock->pDataBlock) > PRIMARY_KEY_COLUMN_INDEX) {
1!
2828
    pSrcPkCol = taosArrayGet(pSrcBlock->pDataBlock, PRIMARY_KEY_COLUMN_INDEX);
×
2829
  }
2830

2831
  TSKEY*  srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData;
1✔
2832
  TSKEY*  srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData;
1✔
2833
  int64_t ver = pSrcBlock->info.version - 1;
1✔
2834
  for (int32_t i = 0; i < pSrcBlock->info.rows; i++) {
2✔
2835
    uint64_t srcUid = srcUidData[i];
1✔
2836
    uint64_t groupId = srcGp[i];
1✔
2837
    char     tbname[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN] = {0};
1✔
2838
    if (groupId == 0) {
1!
2839
      void* pVal = NULL;
1✔
2840
      if (hasPrimaryKeyCol(pInfo) && pSrcPkCol) {
1!
2841
        pVal = colDataGetData(pSrcPkCol, i);
×
2842
      }
2843
      groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], ver, pVal);
1✔
2844
    }
2845
    if (pInfo->tbnameCalSup.pExprInfo) {
1!
2846
      void*   parTbname = NULL;
1✔
2847
      int32_t winCode = TSDB_CODE_SUCCESS;
1✔
2848
      code = pInfo->stateStore.streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, groupId,
1✔
2849
                                                     &parTbname, false, &winCode);
2850
      QUERY_CHECK_CODE(code, lino, _end);
1!
2851

2852
      if (winCode != TSDB_CODE_SUCCESS) {
1!
2853
        SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, srcUid, srcStartTsCol[i], srcStartTsCol[i], ver);
×
2854
        if (!pPreRes) {
×
2855
          qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));
×
2856
          continue;
×
2857
        }
2858
        printDataBlock(pPreRes, "pre res", GET_TASKID(pInfo->pStreamScanOp->pTaskInfo));
×
2859
        code = calBlockTbName(pInfo, pPreRes, 0);
×
2860
        QUERY_CHECK_CODE(code, lino, _end);
×
2861
        memcpy(varDataVal(tbname), pPreRes->info.parTbName, strlen(pPreRes->info.parTbName));
×
2862
      } else {
2863
        memcpy(varDataVal(tbname), parTbname, TSDB_TABLE_NAME_LEN);
1✔
2864
      }
2865
      varDataSetLen(tbname, strlen(varDataVal(tbname)));
1✔
2866
      pInfo->stateStore.streamStateFreeVal(parTbname);
1✔
2867
    }
2868
    code = appendDataToSpecialBlock(pDestBlock, srcStartTsCol + i, srcEndTsCol + i, srcUidData + i, &groupId,
1✔
2869
                                    tbname[0] == 0 ? NULL : tbname);
1!
2870
    QUERY_CHECK_CODE(code, lino, _end);
1!
2871
  }
2872

2873
_end:
1✔
2874
  if (code != TSDB_CODE_SUCCESS) {
1!
2875
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2876
  }
2877
  return code;
1✔
2878
}
2879

2880
int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) {
52✔
2881
  blockDataCleanup(pDestBlock);
52✔
2882
  int32_t rows = pSrcBlock->info.rows;
52✔
2883
  if (rows == 0) {
52!
2884
    return TSDB_CODE_SUCCESS;
×
2885
  }
2886
  int32_t code = blockDataEnsureCapacity(pDestBlock, rows);
52✔
2887
  if (code != TSDB_CODE_SUCCESS) {
52!
2888
    return code;
×
2889
  }
2890
  if (pInfo->partitionSup.needCalc) {
52✔
2891
    return generatePartitionDelResBlock(pInfo, pSrcBlock, pDestBlock);
51✔
2892
  }
2893
  return generateDeleteResultBlockImpl(pInfo, pSrcBlock, pDestBlock);
1✔
2894
}
2895

2896
int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock, EStreamType type) {
1,520✔
2897
  int32_t code = TSDB_CODE_SUCCESS;
1,520✔
2898
  int32_t lino = 0;
1,520✔
2899
  if (isIntervalWindow(pInfo)) {
1,520✔
2900
    code = generateIntervalScanRange(pInfo, pSrcBlock, pDestBlock, type);
889✔
2901
    QUERY_CHECK_CODE(code, lino, _end);
889!
2902
  } else if (isSessionWindow(pInfo) || isStateWindow(pInfo)) {
631✔
2903
    code = generateSessionScanRange(pInfo, pSrcBlock, pDestBlock, type);
560✔
2904
    QUERY_CHECK_CODE(code, lino, _end);
560!
2905
  } else if (isCountWindow(pInfo)) {
71✔
2906
    code = generateCountScanRange(pInfo, pSrcBlock, pDestBlock, type);
7✔
2907
    QUERY_CHECK_CODE(code, lino, _end);
7!
2908
  } else if (isTimeSlice(pInfo)) {
64✔
2909
    code = generateTimeSliceScanRange(pInfo, pSrcBlock, pDestBlock, type);
15✔
2910
    QUERY_CHECK_CODE(code, lino, _end);
15!
2911
  } else {
2912
    code = generateDeleteResultBlock(pInfo, pSrcBlock, pDestBlock);
49✔
2913
    QUERY_CHECK_CODE(code, lino, _end);
49!
2914
  }
2915
  pDestBlock->info.type = STREAM_CLEAR;
1,520✔
2916
  pDestBlock->info.version = pSrcBlock->info.version;
1,520✔
2917
  pDestBlock->info.dataLoad = 1;
1,520✔
2918
  code = blockDataUpdateTsWindow(pDestBlock, 0);
1,520✔
2919
  QUERY_CHECK_CODE(code, lino, _end);
1,520!
2920

2921
_end:
1,520✔
2922
  if (code != TSDB_CODE_SUCCESS) {
1,520!
2923
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
2924
  }
2925
  return code;
1,520✔
2926
}
2927

2928
int32_t appendDataToSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp,
5,621✔
2929
                                 void* pTbName) {
2930
  return appendOneRowToSpecialBlockImpl(pBlock, pStartTs, pEndTs, pStartTs, pEndTs, pUid, pGp, pTbName, NULL);
5,621✔
2931
}
2932

2933
bool checkExpiredData(SStateStore* pAPI, SUpdateInfo* pUpdateInfo, STimeWindowAggSupp* pTwSup, uint64_t tableId,
2,085✔
2934
                      TSKEY ts, void* pPkVal, int32_t len) {
2935
  bool isExpired = false;
2,085✔
2936
  bool isInc = pAPI->isIncrementalTimeStamp(pUpdateInfo, tableId, ts, pPkVal, len);
2,085✔
2937
  if (!isInc) {
2,085✔
2938
    isExpired = isOverdue(ts, pTwSup);
126✔
2939
  }
2940
  return isExpired;
2,085✔
2941
}
2942

2943
int32_t checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock* pBlock, bool out) {
12,896✔
2944
  int32_t code = TSDB_CODE_SUCCESS;
12,896✔
2945
  int32_t lino = 0;
12,896✔
2946
  if (out) {
12,896✔
2947
    blockDataCleanup(pInfo->pUpdateDataRes);
11,393✔
2948
    code = blockDataEnsureCapacity(pInfo->pUpdateDataRes, pBlock->info.rows * 2);
11,392✔
2949
    QUERY_CHECK_CODE(code, lino, _end);
11,392!
2950
  }
2951
  SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
12,895✔
2952
  TSKEY*           tsCol = (TSKEY*)pColDataInfo->pData;
12,895✔
2953
  SColumnInfoData* pPkColDataInfo = NULL;
12,895✔
2954
  if (hasPrimaryKeyCol(pInfo)) {
12,895✔
2955
    pPkColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryKeyIndex);
32✔
2956
  }
2957

2958
  bool tableInserted = pInfo->stateStore.updateInfoIsTableInserted(pInfo->pUpdateInfo, pBlock->info.id.uid);
12,895✔
2959
  for (int32_t rowId = 0; rowId < pBlock->info.rows; rowId++) {
9,992,100!
2960
    SResultRowInfo dumyInfo;
2961
    dumyInfo.cur.pageId = -1;
9,997,884✔
2962
    bool        isClosed = false;
9,997,884✔
2963
    STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX};
9,997,884✔
2964
    bool        overDue = isOverdue(tsCol[rowId], &pInfo->twAggSup);
9,997,884✔
2965
    if (pInfo->igExpired && overDue) {
9,994,595✔
2966
      continue;
21✔
2967
    }
2968

2969
    if (tableInserted && overDue) {
9,994,574✔
2970
      win = getActiveTimeWindow(NULL, &dumyInfo, tsCol[rowId], &pInfo->interval, TSDB_ORDER_ASC);
2,928✔
2971
      isClosed = isCloseWindow(&win, &pInfo->twAggSup);
2,928✔
2972
    }
2973
    // must check update info first.
2974
    void*   pPkVal = NULL;
9,994,574✔
2975
    int32_t pkLen = 0;
9,994,574✔
2976
    if (hasPrimaryKeyCol(pInfo)) {
9,994,574✔
2977
      pPkVal = colDataGetData(pPkColDataInfo, rowId);
32!
2978
      pkLen = colDataGetRowLength(pPkColDataInfo, rowId);
32✔
2979
    }
2980
    bool update =
2981
        pInfo->stateStore.updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.id.uid, tsCol[rowId], pPkVal, pkLen);
9,999,661✔
2982
    bool isDeleted =
9,979,182✔
2983
        isClosed && isSignleIntervalWindow(pInfo) &&
9,992,160!
2984
        isDeletedStreamWindow(&win, pBlock->info.id.groupId, pInfo->pState, &pInfo->twAggSup, &pInfo->stateStore);
12,978✔
2985
    if ((update || isDeleted) && out) {
9,979,182!
2986
      qDebug("stream update check not pass, update %d, deleted Win %d", update, isDeleted);
1,056✔
2987
      uint64_t gpId = 0;
1,056✔
2988
      code = appendPkToSpecialBlock(pInfo->pUpdateDataRes, tsCol, pPkColDataInfo, rowId, &pBlock->info.id.uid, &gpId,
1,056✔
2989
                                    NULL);
2990
      QUERY_CHECK_CODE(code, lino, _end);
1,056!
2991
      if (isDeleted && pInfo->partitionSup.needCalc) {
1,056!
2992
        gpId = calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pBlock, rowId);
×
2993
        code = appendPkToSpecialBlock(pInfo->pUpdateDataRes, tsCol, pPkColDataInfo, rowId, &pBlock->info.id.uid, &gpId,
×
2994
                                      NULL);
2995
        QUERY_CHECK_CODE(code, lino, _end);
×
2996
      }
2997
    }
2998
  }
UNCOV
2999
  if (out && pInfo->pUpdateDataRes->info.rows > 0) {
×
3000
    pInfo->pUpdateDataRes->info.version = pBlock->info.version;
970✔
3001
    pInfo->pUpdateDataRes->info.dataLoad = 1;
970✔
3002
    code = blockDataUpdateTsWindow(pInfo->pUpdateDataRes, 0);
970✔
3003
    QUERY_CHECK_CODE(code, lino, _end);
970!
3004
    pInfo->pUpdateDataRes->info.type = pInfo->partitionSup.needCalc ? STREAM_DELETE_DATA : STREAM_CLEAR;
970✔
3005
  }
3006

UNCOV
3007
_end:
×
UNCOV
3008
  if (code != TSDB_CODE_SUCCESS) {
×
3009
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3010
  }
3011
  return code;
12,898✔
3012
}
3013

3014
int32_t doBlockDataWindowFilter(SSDataBlock* pBlock, int32_t tsIndex, STimeWindow* pWindow, const char* id) {
529,774✔
3015
  int32_t code = TSDB_CODE_SUCCESS;
529,774✔
3016
  int32_t lino = 0;
529,774✔
3017
  bool*   p = NULL;
529,774✔
3018
  if (pWindow->skey != INT64_MIN || pWindow->ekey != INT64_MAX) {
529,774✔
3019
    p = taosMemoryCalloc(pBlock->info.rows, sizeof(bool));
955!
3020
    QUERY_CHECK_NULL(p, code, lino, _end, terrno);
1,014!
3021
    bool hasUnqualified = false;
1,014✔
3022

3023
    SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, tsIndex);
1,014✔
3024

3025
    if (pWindow->skey != INT64_MIN) {
1,014✔
3026
      qDebug("%s filter for additional history window, skey:%" PRId64, id, pWindow->skey);
723!
3027

3028
      QUERY_CHECK_NULL(pCol->pData, code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
723!
3029
      for (int32_t i = 0; i < pBlock->info.rows; ++i) {
1,446✔
3030
        int64_t* ts = (int64_t*)colDataGetData(pCol, i);
723!
3031
        p[i] = (*ts >= pWindow->skey);
723✔
3032

3033
        if (!p[i]) {
723✔
3034
          hasUnqualified = true;
691✔
3035
        }
3036
      }
3037
    } else if (pWindow->ekey != INT64_MAX) {
291!
3038
      qDebug("%s filter for additional history window, ekey:%" PRId64, id, pWindow->ekey);
291!
3039
      for (int32_t i = 0; i < pBlock->info.rows; ++i) {
582✔
3040
        int64_t* ts = (int64_t*)colDataGetData(pCol, i);
291!
3041
        p[i] = (*ts <= pWindow->ekey);
291✔
3042

3043
        if (!p[i]) {
291✔
3044
          hasUnqualified = true;
32✔
3045
        }
3046
      }
3047
    }
3048

3049
    if (hasUnqualified) {
1,014✔
3050
      code = trimDataBlock(pBlock, pBlock->info.rows, p);
723✔
3051
      QUERY_CHECK_CODE(code, lino, _end);
723!
3052
    }
3053
  }
3054

3055
_end:
529,833✔
3056
  taosMemoryFree(p);
529,833!
3057
  if (code != TSDB_CODE_SUCCESS) {
529,815!
3058
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3059
  }
3060
  return code;
529,804✔
3061
}
3062

3063
static int32_t doBlockDataPrimaryKeyFilter(SSDataBlock* pBlock, STqOffsetVal* offset) {
×
3064
  int32_t code = TSDB_CODE_SUCCESS;
×
3065
  int32_t lino = 0;
×
3066
  if (pBlock->info.window.skey != offset->ts || offset->primaryKey.type == 0) {
×
3067
    return code;
×
3068
  }
3069
  bool* p = taosMemoryCalloc(pBlock->info.rows, sizeof(bool));
×
3070
  QUERY_CHECK_NULL(p, code, lino, _end, terrno);
×
3071
  bool hasUnqualified = false;
×
3072

3073
  SColumnInfoData* pColTs = taosArrayGet(pBlock->pDataBlock, 0);
×
3074
  SColumnInfoData* pColPk = taosArrayGet(pBlock->pDataBlock, 1);
×
3075

3076
  qDebug("doBlockDataWindowFilter primary key, ts:%" PRId64 " %" PRId64, offset->ts,
×
3077
         VALUE_GET_TRIVIAL_DATUM(&offset->primaryKey));
3078
  QUERY_CHECK_CONDITION((pColPk->info.type == offset->primaryKey.type), code, lino, _end,
×
3079
                        TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
3080

3081
  __compar_fn_t func = getComparFunc(pColPk->info.type, 0);
×
3082
  QUERY_CHECK_NULL(func, code, lino, _end, terrno);
×
3083
  for (int32_t i = 0; i < pBlock->info.rows; ++i) {
×
3084
    int64_t* ts = (int64_t*)colDataGetData(pColTs, i);
×
3085
    void*    data = colDataGetData(pColPk, i);
×
3086
    if (IS_VAR_DATA_TYPE(pColPk->info.type)) {
×
3087
      void* tmq = taosMemoryMalloc(offset->primaryKey.nData + VARSTR_HEADER_SIZE);
×
3088
      QUERY_CHECK_NULL(tmq, code, lino, _end, terrno);
×
3089
      memcpy(varDataVal(tmq), offset->primaryKey.pData, offset->primaryKey.nData);
×
3090
      varDataLen(tmq) = offset->primaryKey.nData;
×
3091
      p[i] = (*ts > offset->ts) || (func(data, tmq) > 0);
×
3092
      taosMemoryFree(tmq);
×
3093
    } else {
3094
      p[i] = (*ts > offset->ts) || (func(data, VALUE_GET_DATUM(&offset->primaryKey, pColPk->info.type)) > 0);
×
3095
    }
3096

3097
    if (!p[i]) {
×
3098
      hasUnqualified = true;
×
3099
    }
3100
  }
3101

3102
  if (hasUnqualified) {
×
3103
    code = trimDataBlock(pBlock, pBlock->info.rows, p);
×
3104
    QUERY_CHECK_CODE(code, lino, _end);
×
3105
  }
3106

3107
_end:
×
3108
  taosMemoryFree(p);
×
3109
  if (code != TSDB_CODE_SUCCESS) {
×
3110
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3111
  }
3112
  return code;
×
3113
}
3114

3115
// re-build the delete block, ONLY according to the split timestamp
3116
int32_t rebuildDeleteBlockData(SSDataBlock* pBlock, STimeWindow* pWindow, const char* id) {
981✔
3117
  int32_t code = TSDB_CODE_SUCCESS;
981✔
3118
  int32_t lino = 0;
981✔
3119
  int32_t numOfRows = pBlock->info.rows;
981✔
3120
  bool*   p = taosMemoryCalloc(numOfRows, sizeof(bool));
981!
3121
  QUERY_CHECK_NULL(p, code, lino, _end, terrno);
981!
3122

3123
  bool    hasUnqualified = false;
981✔
3124
  int64_t skey = pWindow->skey;
981✔
3125
  int64_t ekey = pWindow->ekey;
981✔
3126

3127
  SColumnInfoData* pSrcStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
981✔
3128
  uint64_t*        tsStartCol = (uint64_t*)pSrcStartCol->pData;
981✔
3129
  SColumnInfoData* pSrcEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
981✔
3130
  uint64_t*        tsEndCol = (uint64_t*)pSrcEndCol->pData;
981✔
3131

3132
  if (pWindow->skey != INT64_MIN) {
981!
3133
    for (int32_t i = 0; i < numOfRows; i++) {
×
3134
      if (tsStartCol[i] < skey) {
×
3135
        tsStartCol[i] = skey;
×
3136
      }
3137

3138
      if (tsEndCol[i] >= skey) {
×
3139
        p[i] = true;
×
3140
      } else {  // this row should be removed, since it is not in this query time window, which is [skey, INT64_MAX]
3141
        hasUnqualified = true;
×
3142
      }
3143
    }
3144
  } else if (pWindow->ekey != INT64_MAX) {
981!
3145
    for (int32_t i = 0; i < numOfRows; ++i) {
×
3146
      if (tsEndCol[i] > ekey) {
×
3147
        tsEndCol[i] = ekey;
×
3148
      }
3149

3150
      if (tsStartCol[i] <= ekey) {
×
3151
        p[i] = true;
×
3152
      } else {
3153
        hasUnqualified = true;
×
3154
      }
3155
    }
3156
  }
3157

3158
  if (hasUnqualified) {
981!
3159
    code = trimDataBlock(pBlock, pBlock->info.rows, p);
×
3160
    qDebug("%s re-build delete datablock, start key revised to:%" PRId64 ", rows:%" PRId64, id, skey,
×
3161
           pBlock->info.rows);
3162
    QUERY_CHECK_CODE(code, lino, _end);
×
3163
  } else {
3164
    qDebug("%s not update the delete block", id);
981!
3165
  }
3166

3167
_end:
×
3168
  taosMemoryFree(p);
981!
3169
  if (code != TSDB_CODE_SUCCESS) {
981!
3170
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3171
  }
3172
  return code;
981✔
3173
}
3174

3175
int32_t colIdComparFn(const void* param1, const void* param2) {
4,170,697✔
3176
  int32_t p1 = *(int32_t*)param1;
4,170,697✔
3177
  int32_t p2 = *(int32_t*)param2;
4,170,697✔
3178

3179
  if (p1 == p2) {
4,170,697!
3180
    return 0;
×
3181
  } else {
3182
    return (p1 < p2) ? -1 : 1;
4,170,697✔
3183
  }
3184
}
3185

3186
int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock, STimeWindow* pTimeWindow, bool filter) {
529,844✔
3187
  int32_t         code = TSDB_CODE_SUCCESS;
529,844✔
3188
  int32_t         lino = 0;
529,844✔
3189
  SDataBlockInfo* pBlockInfo = &pInfo->pRes->info;
529,844✔
3190
  SOperatorInfo*  pOperator = pInfo->pStreamScanOp;
529,844✔
3191
  SExecTaskInfo*  pTaskInfo = pOperator->pTaskInfo;
529,844✔
3192
  const char*     id = GET_TASKID(pTaskInfo);
529,844✔
3193
  bool            isVtableSourceScan = (pTaskInfo->pSubplan->pVTables != NULL);
529,844✔
3194

3195
  code = blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows);
529,844✔
3196
  QUERY_CHECK_CODE(code, lino, _end);
529,854!
3197

3198
  pBlockInfo->rows = pBlock->info.rows;
529,854✔
3199
  pBlockInfo->id.uid = pBlock->info.id.uid;
529,854✔
3200
  pBlockInfo->type = STREAM_NORMAL;
529,854✔
3201
  pBlockInfo->version = pBlock->info.version;
529,854✔
3202

3203
  STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
529,854✔
3204
  if (!isVtableSourceScan) {
529,854✔
3205
    pBlockInfo->id.groupId = tableListGetTableGroupId(pTableScanInfo->base.pTableListInfo, pBlock->info.id.uid);
529,849✔
3206
  } else {
3207
    // use original table uid as groupId for vtable
3208
    pBlockInfo->id.groupId = pBlock->info.id.groupId;
5✔
3209
  }
3210

3211
  SArray* pColList = taosArrayInit(4, sizeof(int32_t));
529,890✔
3212
  QUERY_CHECK_NULL(pColList, code, lino, _end, terrno);
529,881!
3213

3214
  // todo extract method
3215
  for (int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) {
2,748,995✔
3216
    SColMatchItem* pColMatchInfo = taosArrayGet(pInfo->matchInfo.pList, i);
2,219,188✔
3217
    if (!pColMatchInfo->needOutput) {
2,219,135✔
3218
      continue;
2✔
3219
    }
3220

3221
    bool colExists = false;
2,219,133✔
3222
    for (int32_t j = 0; j < blockDataGetNumOfCols(pBlock); ++j) {
6,438,595✔
3223
      SColumnInfoData* pResCol = NULL;
6,438,573✔
3224
      code = bdGetColumnInfoData(pBlock, j, &pResCol);
6,438,573✔
3225
      QUERY_CHECK_CODE(code, lino, _end);
6,438,322!
3226

3227
      if (pResCol->info.colId == pColMatchInfo->colId) {
6,438,322✔
3228
        SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->dstSlotId);
2,218,860✔
3229
        code = colDataAssign(pDst, pResCol, pBlock->info.rows, &pInfo->pRes->info);
2,218,790✔
3230
        QUERY_CHECK_CODE(code, lino, _end);
2,219,016!
3231

3232
        colExists = true;
2,219,016✔
3233
        void* tmp = taosArrayPush(pColList, &pColMatchInfo->dstSlotId);
2,219,016✔
3234
        QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
2,219,102!
3235
        break;
2,219,102✔
3236
      }
3237
    }
3238

3239
    // the required column does not exists in submit block, let's set it to be all null value
3240
    if (!colExists) {
2,219,108✔
3241
      SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->dstSlotId);
4✔
3242
      colDataSetNNULL(pDst, 0, pBlockInfo->rows);
4!
3243
      void* tmp = taosArrayPush(pColList, &pColMatchInfo->dstSlotId);
4✔
3244
      QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
4!
3245
    }
3246
  }
3247

3248
  // currently only the tbname pseudo column
3249
  if (pInfo->numOfPseudoExpr > 0 && !isVtableSourceScan) {
529,724!
3250
    code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes,
370,759✔
3251
                                  pBlockInfo->rows, pTaskInfo, &pTableScanInfo->base.metaCache);
370,759✔
3252
    // ignore the table not exists error, since this table may have been dropped during the scan procedure.
3253
    if (code) {
370,776!
3254
      QUERY_CHECK_CODE(code, lino, _end);
×
3255
    }
3256

3257
    // reset the error code.
3258
    terrno = 0;
370,776✔
3259

3260
    for (int32_t i = 0; i < pInfo->numOfPseudoExpr; ++i) {
1,379,594✔
3261
      void* tmp = taosArrayPush(pColList, &pInfo->pPseudoExpr[i].base.resSchema.slotId);
1,008,815✔
3262
      QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
1,008,818!
3263
    }
3264
  }
3265

3266
  taosArraySort(pColList, colIdComparFn);
529,744✔
3267

3268
  int32_t i = 0, j = 0;
529,860✔
3269
  while (i < taosArrayGetSize(pColList)) {
3,757,892✔
3270
    int32_t slot1 = *(int32_t*)taosArrayGet(pColList, i);
3,227,987✔
3271
    if (slot1 > j) {
3,228,032!
3272
      SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, j);
×
3273
      colDataSetNNULL(pDst, 0, pBlockInfo->rows);
×
3274
      j += 1;
×
3275
    } else {
3276
      i += 1;
3,228,032✔
3277
      j += 1;
3,228,032✔
3278
    }
3279
  }
3280

3281
  while (j < taosArrayGetSize(pInfo->pRes->pDataBlock)) {
530,391✔
3282
    SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, j);
566✔
3283
    colDataSetNNULL(pDst, 0, pBlockInfo->rows);
566✔
3284
    j += 1;
566✔
3285
  }
3286

3287
  taosArrayDestroy(pColList);
529,820✔
3288

3289
  if (filter) {
529,865✔
3290
    code = doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
502,828✔
3291
    QUERY_CHECK_CODE(code, lino, _end);
502,754!
3292
  }
3293

3294
  // filter the block extracted from WAL files, according to the time window apply additional time window filter
3295
  code = doBlockDataWindowFilter(pInfo->pRes, pInfo->primaryTsIndex, pTimeWindow, id);
529,791✔
3296
  QUERY_CHECK_CODE(code, lino, _end);
529,795!
3297
  pInfo->pRes->info.dataLoad = 1;
529,795✔
3298

3299
  code = blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
529,795✔
3300
  QUERY_CHECK_CODE(code, lino, _end);
529,835!
3301
  if (pInfo->pRes->info.rows == 0) {
529,835✔
3302
    return 0;
3,052✔
3303
  }
3304

3305
  code = calBlockTbName(pInfo, pInfo->pRes, 0);
526,783✔
3306
  QUERY_CHECK_CODE(code, lino, _end);
526,695!
3307

3308
_end:
526,695✔
3309
  if (code != TSDB_CODE_SUCCESS) {
526,695!
3310
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3311
  }
3312
  return code;
526,704✔
3313
}
3314

3315
static int32_t processPrimaryKey(SSDataBlock* pBlock, bool hasPrimaryKey, STqOffsetVal* offset) {
22,809✔
3316
  int32_t code = TSDB_CODE_SUCCESS;
22,809✔
3317
  int32_t lino = 0;
22,809✔
3318
  SValue  val = {0};
22,809✔
3319
  if (hasPrimaryKey) {
22,809!
3320
    code = doBlockDataPrimaryKeyFilter(pBlock, offset);
×
3321
    if (code != TSDB_CODE_SUCCESS) {
×
3322
      qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
×
3323
      return code;
×
3324
    }
3325
    SColumnInfoData* pColPk = taosArrayGet(pBlock->pDataBlock, 1);
×
3326

3327
    if (pBlock->info.rows < 1) {
×
3328
      return code;
×
3329
    }
3330
    void* tmp = colDataGetData(pColPk, pBlock->info.rows - 1);
×
3331
    val.type = pColPk->info.type;
×
3332
    if (IS_VAR_DATA_TYPE(pColPk->info.type)) {
×
3333
      val.pData = taosMemoryMalloc(varDataLen(tmp));
×
3334
      QUERY_CHECK_NULL(val.pData, code, lino, _end, terrno);
×
3335
      val.nData = varDataLen(tmp);
×
3336
      memcpy(val.pData, varDataVal(tmp), varDataLen(tmp));
×
3337
    } else {
3338
      valueSetDatum(&val, pColPk->info.type, tmp, pColPk->info.bytes);
×
3339
    }
3340
  }
3341
  tqOffsetResetToData(offset, pBlock->info.id.uid, pBlock->info.window.ekey, val);
45,638!
3342

3343
_end:
22,829✔
3344
  if (code != TSDB_CODE_SUCCESS) {
22,829!
3345
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3346
  }
3347
  return code;
22,818✔
3348
}
3349

3350
static int32_t doQueueScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
603,190✔
3351
  int32_t        code = TSDB_CODE_SUCCESS;
603,190✔
3352
  int32_t        lino = 0;
603,190✔
3353
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
603,190✔
3354
  SStorageAPI*   pAPI = &pTaskInfo->storageAPI;
603,190✔
3355

3356
  SStreamScanInfo* pInfo = pOperator->info;
603,190✔
3357
  const char*      id = GET_TASKID(pTaskInfo);
603,190✔
3358

3359
  qDebug("start to exec queue scan, %s", id);
603,190✔
3360

3361
  if (isTaskKilled(pTaskInfo)) {
603,189!
3362
    (*ppRes) = NULL;
×
3363
    return pTaskInfo->code;
×
3364
  }
3365

3366
  if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) {
603,196✔
3367
    while (1) {
×
3368
      SSDataBlock* pResult = NULL;
20,526✔
3369
      code = doTableScanNext(pInfo->pTableScanOp, &pResult);
20,526✔
3370
      QUERY_CHECK_CODE(code, lino, _end);
20,501!
3371

3372
      if (pResult && pResult->info.rows > 0) {
20,501!
3373
        bool hasPrimaryKey = pAPI->tqReaderFn.tqGetTablePrimaryKey(pInfo->tqReader);
20,304✔
3374
        code = processPrimaryKey(pResult, hasPrimaryKey, &pTaskInfo->streamInfo.currentOffset);
20,319✔
3375
        QUERY_CHECK_CODE(code, lino, _end);
20,306!
3376
        qDebug("tmqsnap doQueueScan get data utid:%" PRId64, pResult->info.id.uid);
20,306✔
3377
        if (pResult->info.rows > 0) {
20,314!
3378
          (*ppRes) = pResult;
20,314✔
3379
          return code;
20,314✔
3380
        }
3381
      } else {
3382
        break;
3383
      }
3384
    }
3385

3386
    STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
197✔
3387
    pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader);
197✔
3388

3389
    pTSInfo->base.dataReader = NULL;
200✔
3390
    int64_t validVer = pTaskInfo->streamInfo.snapshotVer + 1;
200✔
3391
    qDebug("queue scan tsdb over, switch to wal ver %" PRId64, validVer);
200✔
3392
    if (pAPI->tqReaderFn.tqReaderSeek(pInfo->tqReader, validVer, pTaskInfo->id.str) < 0) {
200✔
3393
      (*ppRes) = NULL;
106✔
3394
      return code;
106✔
3395
    }
3396

3397
    tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, validVer);
94✔
3398
  }
3399

3400
  if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__LOG) {
582,764!
3401
    while (1) {
2,329✔
3402
      bool hasResult =
3403
          pAPI->tqReaderFn.tqReaderNextBlockInWal(pInfo->tqReader, id, pTaskInfo->streamInfo.sourceExcluded);
585,093✔
3404

3405
      SSDataBlock*       pRes = pAPI->tqReaderFn.tqGetResultBlock(pInfo->tqReader);
585,063✔
3406
      struct SWalReader* pWalReader = pAPI->tqReaderFn.tqReaderGetWalReader(pInfo->tqReader);
585,058✔
3407

3408
      // curVersion move to next
3409
      tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pWalReader->curVersion);
585,055✔
3410

3411
      // use ts to pass time when replay, because ts not used if type is log
3412
      pTaskInfo->streamInfo.currentOffset.ts = pAPI->tqReaderFn.tqGetResultBlockTime(pInfo->tqReader);
585,055✔
3413

3414
      if (hasResult) {
585,052✔
3415
        qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, pRes->info.rows,
502,821✔
3416
               pTaskInfo->streamInfo.currentOffset.version);
3417
        blockDataCleanup(pInfo->pRes);
502,835✔
3418
        STimeWindow defaultWindow = {.skey = INT64_MIN, .ekey = INT64_MAX};
502,798✔
3419
        code = setBlockIntoRes(pInfo, pRes, &defaultWindow, true);
502,798✔
3420
        QUERY_CHECK_CODE(code, lino, _end);
502,678!
3421
        qDebug("doQueueScan after filter get data from log %" PRId64 " rows, version:%" PRId64, pInfo->pRes->info.rows,
502,678✔
3422
               pTaskInfo->streamInfo.currentOffset.version);
3423
        if (pInfo->pRes->info.rows > 0) {
502,697✔
3424
          (*ppRes) = pInfo->pRes;
500,368✔
3425
          return code;
500,368✔
3426
        }
3427
      } else {
3428
        qDebug("doQueueScan get none from log, return, version:%" PRId64, pTaskInfo->streamInfo.currentOffset.version);
82,231✔
3429
        (*ppRes) = NULL;
82,235✔
3430
        return code;
82,235✔
3431
      }
3432
    }
3433
  } else {
3434
    qError("unexpected streamInfo prepare type: %d", pTaskInfo->streamInfo.currentOffset.type);
×
3435
    (*ppRes) = NULL;
×
3436
    return code;
×
3437
  }
3438

3439
_end:
×
3440
  if (code != TSDB_CODE_SUCCESS) {
×
3441
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3442
    pTaskInfo->code = code;
×
3443
    T_LONG_JMP(pTaskInfo->env, code);
×
3444
  }
3445
  (*ppRes) = NULL;
×
3446
  return code;
×
3447
}
3448

3449
static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
×
3450
  SSDataBlock* pRes = NULL;
×
3451
  int32_t      code = doQueueScanNext(pOperator, &pRes);
×
3452
  return pRes;
×
3453
}
3454

3455
int32_t filterDelBlockByUid(SSDataBlock* pDst, const SSDataBlock* pSrc, STqReader* pReader, SStoreTqReader* pReaderFn) {
981✔
3456
  int32_t    code = TSDB_CODE_SUCCESS;
981✔
3457
  int32_t    lino = 0;
981✔
3458
  int32_t    rows = pSrc->info.rows;
981✔
3459
  code = blockDataEnsureCapacity(pDst, rows);
981✔
3460
  QUERY_CHECK_CODE(code, lino, _end);
981!
3461

3462
  SColumnInfoData* pSrcStartCol = taosArrayGet(pSrc->pDataBlock, START_TS_COLUMN_INDEX);
981✔
3463
  uint64_t*        startCol = (uint64_t*)pSrcStartCol->pData;
981✔
3464
  SColumnInfoData* pSrcEndCol = taosArrayGet(pSrc->pDataBlock, END_TS_COLUMN_INDEX);
981✔
3465
  uint64_t*        endCol = (uint64_t*)pSrcEndCol->pData;
981✔
3466
  SColumnInfoData* pSrcUidCol = taosArrayGet(pSrc->pDataBlock, UID_COLUMN_INDEX);
981✔
3467
  uint64_t*        uidCol = (uint64_t*)pSrcUidCol->pData;
981✔
3468

3469
  SColumnInfoData* pDstStartCol = taosArrayGet(pDst->pDataBlock, START_TS_COLUMN_INDEX);
981✔
3470
  SColumnInfoData* pDstEndCol = taosArrayGet(pDst->pDataBlock, END_TS_COLUMN_INDEX);
981✔
3471
  SColumnInfoData* pDstUidCol = taosArrayGet(pDst->pDataBlock, UID_COLUMN_INDEX);
981✔
3472

3473
  int32_t j = 0;
981✔
3474
  for (int32_t i = 0; i < rows; i++) {
1,969✔
3475
    if (pReaderFn->tqReaderIsQueriedTable(pReader, uidCol[i])) {
988✔
3476
      code = colDataSetVal(pDstStartCol, j, (const char*)&startCol[i], false);
561✔
3477
      QUERY_CHECK_CODE(code, lino, _end);
561!
3478

3479
      code = colDataSetVal(pDstEndCol, j, (const char*)&endCol[i], false);
561✔
3480
      QUERY_CHECK_CODE(code, lino, _end);
561!
3481

3482
      code = colDataSetVal(pDstUidCol, j, (const char*)&uidCol[i], false);
561✔
3483
      QUERY_CHECK_CODE(code, lino, _end);
561!
3484

3485
      colDataSetNULL(taosArrayGet(pDst->pDataBlock, GROUPID_COLUMN_INDEX), j);
561✔
3486
      colDataSetNULL(taosArrayGet(pDst->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX), j);
561✔
3487
      colDataSetNULL(taosArrayGet(pDst->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX), j);
561✔
3488
      colDataSetNULL(taosArrayGet(pDst->pDataBlock, TABLE_NAME_COLUMN_INDEX), j);
561✔
3489
      j++;
561✔
3490
    }
3491
  }
3492

3493
  uint32_t cap = pDst->info.capacity;
981✔
3494
  pDst->info = pSrc->info;
981✔
3495
  pDst->info.rows = j;
981✔
3496
  pDst->info.capacity = cap;
981✔
3497

3498
_end:
981✔
3499
  if (code != TSDB_CODE_SUCCESS) {
981!
3500
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3501
  }
3502
  return code;
981✔
3503
}
3504

3505
// for partition by tag
3506
int32_t setBlockGroupIdByUid(SStreamScanInfo* pInfo, SSDataBlock* pBlock) {
981✔
3507
  int32_t          code = TSDB_CODE_SUCCESS;
981✔
3508
  int32_t          lino = 0;
981✔
3509
  SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
981✔
3510
  TSKEY*           startTsCol = (TSKEY*)pStartTsCol->pData;
981✔
3511
  SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
981✔
3512
  uint64_t*        gpCol = (uint64_t*)pGpCol->pData;
981✔
3513
  SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
981✔
3514
  uint64_t*        uidCol = (uint64_t*)pUidCol->pData;
981✔
3515
  int32_t          rows = pBlock->info.rows;
981✔
3516
  if (!pInfo->partitionSup.needCalc) {
981✔
3517
    for (int32_t i = 0; i < rows; i++) {
929✔
3518
      uint64_t groupId = getGroupIdByUid(pInfo, uidCol[i]);
348✔
3519
      code = colDataSetVal(pGpCol, i, (const char*)&groupId, false);
348✔
3520
      QUERY_CHECK_CODE(code, lino, _end);
348!
3521
    }
3522
  }
3523

3524
_end:
981✔
3525
  if (code != TSDB_CODE_SUCCESS) {
981!
3526
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3527
  }
3528
  return code;
981✔
3529
}
3530

3531
int32_t doCheckUpdate(SStreamScanInfo* pInfo, TSKEY endKey, SSDataBlock* pBlock) {
26,312✔
3532
  int32_t code = TSDB_CODE_SUCCESS;
26,312✔
3533
  int32_t lino = 0;
26,312✔
3534
  if (pInfo->pUpdateInfo) {
26,312✔
3535
    pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version);
25,880✔
3536
  }
3537
  if (!pInfo->igCheckUpdate && pInfo->pUpdateInfo) {
26,312✔
3538
    code = checkUpdateData(pInfo, true, pBlock, true);
11,393✔
3539
    QUERY_CHECK_CODE(code, lino, _end);
11,395!
3540
    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, endKey);
11,395✔
3541
    if (pInfo->pUpdateDataRes->info.rows > 0) {
11,395✔
3542
      pInfo->updateResIndex = 0;
970✔
3543
      if (pInfo->pUpdateDataRes->info.type == STREAM_CLEAR) {
970✔
3544
        pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES;
794✔
3545
      } else if (pInfo->pUpdateDataRes->info.type == STREAM_INVERT) {
176!
3546
        pInfo->scanMode = STREAM_SCAN_FROM_RES;
×
3547
        // return pInfo->pUpdateDataRes;
3548
      } else if (pInfo->pUpdateDataRes->info.type == STREAM_DELETE_DATA) {
176!
3549
        pInfo->scanMode = STREAM_SCAN_FROM_DELETE_DATA;
176✔
3550
      }
3551
    }
3552
  }
3553

3554
_end:
25,344✔
3555
  if (code != TSDB_CODE_SUCCESS) {
26,314!
3556
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3557
  }
3558
  return code;
26,314✔
3559
}
3560

3561
int32_t streamScanOperatorEncode(SStreamScanInfo* pInfo, void** pBuff, int32_t* pLen) {
36✔
3562
  int32_t   code = TSDB_CODE_SUCCESS;
36✔
3563
  int32_t   lino = 0;
36✔
3564
  int32_t   len = 0;
36✔
3565
  SEncoder* pEnCoder = NULL;
36✔
3566
  SEncoder* pScanEnCoder = NULL;
36✔
3567

3568
  len += encodeSTimeWindowAggSupp(NULL, &pInfo->twAggSup);
36✔
3569
  SEncoder encoder = {0};
36✔
3570
  pEnCoder = &encoder;
36✔
3571
  tEncoderInit(pEnCoder, NULL, 0);
36✔
3572
  if (tStartEncode(pEnCoder) != 0) {
36!
3573
    code = TSDB_CODE_STREAM_INTERNAL_ERROR;
×
3574
    QUERY_CHECK_CODE(code, lino, _end);
×
3575
  }
3576
  code = pInfo->stateStore.updateInfoSerialize(pEnCoder, pInfo->pUpdateInfo);
36✔
3577
  QUERY_CHECK_CODE(code, lino, _end);
36!
3578

3579
  if (tEncodeI64(pEnCoder, pInfo->lastScanRange.skey) < 0) {
72!
3580
    code = TSDB_CODE_STREAM_INTERNAL_ERROR;
×
3581
    QUERY_CHECK_CODE(code, lino, _end);
×
3582
  }
3583
  if (tEncodeI64(pEnCoder, pInfo->lastScanRange.ekey) < 0) {
72!
3584
    code = TSDB_CODE_STREAM_INTERNAL_ERROR;
×
3585
    QUERY_CHECK_CODE(code, lino, _end);
×
3586
  }
3587

3588
  tEndEncode(pEnCoder);
36✔
3589
  len += encoder.pos;
36✔
3590
  tEncoderClear(pEnCoder);
36✔
3591
  pEnCoder = NULL;
36✔
3592

3593
  *pBuff = taosMemoryCalloc(1, len);
36!
3594
  if (!(*pBuff)) {
36!
3595
    code = terrno;
×
3596
    QUERY_CHECK_CODE(code, lino, _end);
×
3597
  }
3598
  void* buf = *pBuff;
36✔
3599
  int32_t stwLen = encodeSTimeWindowAggSupp(&buf, &pInfo->twAggSup);
36✔
3600

3601
  SEncoder scanEncoder = {0};
36✔
3602
  pScanEnCoder = &scanEncoder;
36✔
3603
  tEncoderInit(pScanEnCoder, buf, len - stwLen);
36✔
3604
  if (tStartEncode(pScanEnCoder) != 0) {
36!
3605
    code = TSDB_CODE_FAILED;
×
3606
    QUERY_CHECK_CODE(code, lino, _end);
×
3607
  }
3608
  code = pInfo->stateStore.updateInfoSerialize(pScanEnCoder, pInfo->pUpdateInfo);
36✔
3609
  QUERY_CHECK_CODE(code, lino, _end);
36!
3610

3611
  if (tEncodeI64(pScanEnCoder, pInfo->lastScanRange.skey) < 0) {
72!
3612
    code = TSDB_CODE_STREAM_INTERNAL_ERROR;
×
3613
    QUERY_CHECK_CODE(code, lino, _end);
×
3614
  }
3615
  if (tEncodeI64(pScanEnCoder, pInfo->lastScanRange.ekey) < 0) {
72!
3616
    code = TSDB_CODE_STREAM_INTERNAL_ERROR;
×
3617
    QUERY_CHECK_CODE(code, lino, _end);
×
3618
  }
3619

3620
  qDebug("%s last scan range %d. %" PRId64 ",%" PRId64, __func__, __LINE__, pInfo->lastScanRange.skey, pInfo->lastScanRange.ekey);
36✔
3621

3622
  *pLen = len;
36✔
3623

3624
_end:
36✔
3625
  if (code != TSDB_CODE_SUCCESS) {
36!
3626
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3627
  }
3628
  if (pEnCoder != NULL) {
36!
3629
    tEndEncode(pEnCoder);
×
3630
    tEncoderClear(pEnCoder);
×
3631
  }
3632
  if (pScanEnCoder != NULL) {
36!
3633
    tEndEncode(pScanEnCoder);
36✔
3634
    tEncoderClear(pScanEnCoder);
36✔
3635
  }
3636
  return code;
36✔
3637
}
3638

3639
void streamScanOperatorSaveCheckpoint(SStreamScanInfo* pInfo) {
907✔
3640
  int32_t code = TSDB_CODE_SUCCESS;
907✔
3641
  int32_t lino = 0;
907✔
3642
  if (!pInfo->pState) {
907!
3643
    return;
×
3644
  }
3645
  if (needSaveStreamOperatorInfo(&pInfo->basic)) {
907✔
3646
    void*   pBuf = NULL;
36✔
3647
    int32_t len = 0;
36✔
3648
    code = streamScanOperatorEncode(pInfo, &pBuf, &len);
36✔
3649
    QUERY_CHECK_CODE(code, lino, _end);
36!
3650
    pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_CHECKPOINT_NAME,
36✔
3651
                                          strlen(STREAM_SCAN_OP_CHECKPOINT_NAME), pBuf, len);
3652
    taosMemoryFree(pBuf);
36!
3653
    saveStreamOperatorStateComplete(&pInfo->basic);
36✔
3654
  }
3655

3656
_end:
871✔
3657
  if (code != TSDB_CODE_SUCCESS) {
907!
3658
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3659
  }
3660
}
3661

3662
// other properties are recovered from the execution plan
3663
void streamScanOperatorDecode(void* pBuff, int32_t len, SStreamScanInfo* pInfo) {
6✔
3664
  int32_t   code = TSDB_CODE_SUCCESS;
6✔
3665
  int32_t   lino = 0;
6✔
3666
  SDecoder* pDeCoder = NULL;
6✔
3667
  if (!pBuff || len == 0) {
6!
3668
    lino = __LINE__;
×
3669
    goto _end;
×
3670
  }
3671
  void* buf = pBuff;
6✔
3672
  buf = decodeSTimeWindowAggSupp(buf, &pInfo->twAggSup);
6✔
3673
  int32_t tlen = len - encodeSTimeWindowAggSupp(NULL, &pInfo->twAggSup);
6✔
3674
  if (tlen == 0) {
6!
3675
    lino = __LINE__;
×
3676
    goto _end;
×
3677
  }
3678

3679
  SDecoder decoder = {0};
6✔
3680
  pDeCoder = &decoder;
6✔
3681
  tDecoderInit(pDeCoder, buf, tlen);
6✔
3682
  if (tStartDecode(pDeCoder) < 0) {
6!
3683
    lino = __LINE__;
×
3684
    goto _end;
×
3685
  }
3686

3687
  void* pUpInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo));
6!
3688
  if (!pUpInfo) {
6!
3689
    lino = __LINE__;
×
3690
    goto _end;
×
3691
  }
3692
  code = pInfo->stateStore.updateInfoDeserialize(pDeCoder, pUpInfo);
6✔
3693
  if (code == TSDB_CODE_SUCCESS) {
6!
3694
    pInfo->stateStore.updateInfoDestroy(pInfo->pUpdateInfo);
6✔
3695
    pInfo->pUpdateInfo = pUpInfo;
6✔
3696
    qDebug("%s line:%d. stream scan updateinfo deserialize success", __func__, __LINE__);
6!
3697
  } else {
UNCOV
3698
    taosMemoryFree(pUpInfo);
×
UNCOV
3699
    code = TSDB_CODE_SUCCESS;
×
UNCOV
3700
    qDebug("%s line:%d. stream scan did not have updateinfo", __func__, __LINE__);
×
3701
  }
3702

3703
  if (tDecodeIsEnd(pDeCoder)) {
6!
3704
    lino = __LINE__;
×
3705
    goto _end;
×
3706
  }
3707

3708
  SET_WIN_KEY_INVALID(pInfo->lastScanRange.skey);
6✔
3709
  SET_WIN_KEY_INVALID(pInfo->lastScanRange.ekey);
6✔
3710

3711
  if (tDecodeI64(pDeCoder, &pInfo->lastScanRange.skey) < 0) {
12!
3712
    lino = __LINE__;
×
3713
    goto _end;
×
3714
  }
3715

3716
  if (tDecodeI64(pDeCoder, &pInfo->lastScanRange.ekey) < 0) {
12!
3717
    lino = __LINE__;
×
3718
    goto _end;
×
3719
  }
3720
  qDebug("%s last scan range %d. %" PRId64 ",%" PRId64, __func__, __LINE__, pInfo->lastScanRange.skey, pInfo->lastScanRange.ekey);
6!
3721

UNCOV
3722
_end:
×
3723
  if (pDeCoder != NULL) {
6!
3724
    tEndDecode(pDeCoder);
6✔
3725
    tDecoderClear(pDeCoder);
6✔
3726
  }
3727
  qInfo("%s end at line %d", __func__, lino);
6!
3728
}
6✔
3729

3730
bool hasScanRange(SStreamScanInfo* pInfo) {
40,097✔
3731
  SStreamAggSupporter* pSup = pInfo->windowSup.pStreamAggSup;
40,097✔
3732
  return pSup && pSup->pScanBlock->info.rows > 0 && (isStateWindow(pInfo) || isCountWindow(pInfo));
40,097!
3733
}
3734

3735
bool isStreamWindow(SStreamScanInfo* pInfo) {
554✔
3736
  return isIntervalWindow(pInfo) || isSessionWindow(pInfo) || isStateWindow(pInfo) || isCountWindow(pInfo) ||
572✔
3737
         isTimeSlice(pInfo);
18✔
3738
}
3739

3740
int32_t copyGetResultBlock(SSDataBlock* dest, TSKEY start, TSKEY end) {
4,311✔
3741
  int32_t code = blockDataEnsureCapacity(dest, 1);
4,311✔
3742
  if (code != TSDB_CODE_SUCCESS) {
4,311!
3743
    return code;
×
3744
  }
3745
  return appendDataToSpecialBlock(dest, &start, &end, NULL, NULL, NULL);
4,311✔
3746
}
3747

3748
int32_t deletePartName(SStateStore* pStore, SStreamState* pState, SSDataBlock* pBlock, int32_t *deleteNum) {
1✔
3749
  int32_t code = TSDB_CODE_SUCCESS;
1✔
3750
  int32_t lino = 0;
1✔
3751
  for (int32_t i = 0; i < pBlock->info.rows; i++) {
2✔
3752
    // uid is the same as gid
3753
    SColumnInfoData* pGpIdCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX);
1✔
3754
    SColumnInfoData* pTbnameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
1✔
3755
    int64_t*         gpIdCol = (int64_t*)pGpIdCol->pData;
1✔
3756
    void*            pParName = NULL;
1✔
3757
    int32_t          winCode = 0;
1✔
3758
    code = pStore->streamStateGetParName(pState, gpIdCol[i],
1✔
3759
                                                   &pParName, false, &winCode);
3760
    if (TSDB_CODE_SUCCESS == code && winCode != 0) {
1!
3761
      qDebug("delete stream part Name for:%"PRId64 " not found", gpIdCol[i]);
1!
3762
      colDataSetNULL(pTbnameCol, i);
1!
3763
      continue;
1✔
3764
    }
3765
    (*deleteNum)++;
×
3766
    QUERY_CHECK_CODE(code, lino, _end);
×
3767
    char varTbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE + 1] = {0};
×
3768
    varDataSetLen(varTbName, strlen(pParName));
×
3769
    int64_t len = tsnprintf(varTbName + VARSTR_HEADER_SIZE, TSDB_TABLE_NAME_LEN + 1, "%s", pParName);
×
3770
    code = colDataSetVal(pTbnameCol, i, varTbName, false);
×
3771
    qDebug("delete stream part for:%"PRId64 " res tb: %s", gpIdCol[i], (char*)pParName);
×
3772
    pStore->streamStateFreeVal(pParName);
×
3773
    QUERY_CHECK_CODE(code, lino, _end);
×
3774
    code = pStore->streamStateDeleteParName(pState, gpIdCol[i]);
×
3775
    QUERY_CHECK_CODE(code, lino, _end);
×
3776
    pBlock->info.id.groupId = gpIdCol[i];
×
3777
    // currently, only one valid row in pBlock
3778
    memcpy(pBlock->info.parTbName, varTbName + VARSTR_HEADER_SIZE, TSDB_TABLE_NAME_LEN + 1);
×
3779
  }
3780

3781
_end:
1✔
3782
  if (code != TSDB_CODE_SUCCESS) {
1!
3783
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
3784
  }
3785
  return code;
1✔
3786
}
3787

3788
static int32_t doStreamScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
63,122✔
3789
  // NOTE: this operator does never check if current status is done or not
3790
  int32_t        code = TSDB_CODE_SUCCESS;
63,122✔
3791
  int32_t        lino = 0;
63,122✔
3792
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
63,122✔
3793
  const char*    id = GET_TASKID(pTaskInfo);
63,122✔
3794

3795
  SStorageAPI*     pAPI = &pTaskInfo->storageAPI;
63,122✔
3796
  SStreamScanInfo* pInfo = pOperator->info;
63,122✔
3797
  SStreamTaskInfo* pStreamInfo = &pTaskInfo->streamInfo;
63,122✔
3798
  SSHashObj*       pVtableInfos = pTaskInfo->pSubplan->pVTables;
63,122✔
3799

3800
  qDebug("%s stream scan started, %s, recoverStep:%d", __FUNCTION__, id, pStreamInfo->recoverStep);
63,122✔
3801

3802
  if (pVtableInfos != NULL && pStreamInfo->recoverStep != STREAM_RECOVER_STEP__NONE) {
63,123!
3803
    qError("stream vtable source scan should not have recovery step: %d", pStreamInfo->recoverStep);
×
3804
    pStreamInfo->recoverStep = STREAM_RECOVER_STEP__NONE;
×
3805
  }
3806

3807
  if (pVtableInfos != NULL && !pInfo->igCheckUpdate) {
63,123!
3808
    qError("stream vtable source scan should have igCheckUpdate");
×
3809
    pInfo->igCheckUpdate = false;
×
3810
  }
3811

3812
  if (pStreamInfo->recoverStep == STREAM_RECOVER_STEP__PREPARE1 ||
63,123✔
3813
      pStreamInfo->recoverStep == STREAM_RECOVER_STEP__PREPARE2) {
62,234✔
3814
    STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
1,989✔
3815
    memcpy(&pTSInfo->base.cond, &pStreamInfo->tableCond, sizeof(SQueryTableDataCond));
1,989✔
3816

3817
    if (pStreamInfo->recoverStep == STREAM_RECOVER_STEP__PREPARE1) {
1,989✔
3818
      pTSInfo->base.cond.startVersion = pStreamInfo->fillHistoryVer.minVer;
889✔
3819
      pTSInfo->base.cond.endVersion = pStreamInfo->fillHistoryVer.maxVer;
889✔
3820

3821
      pTSInfo->base.cond.twindows = pStreamInfo->fillHistoryWindow;
889✔
3822
      qDebug("stream scan step1, verRange:%" PRId64 "-%" PRId64 " window:%" PRId64 "-%" PRId64 ", %s",
889✔
3823
             pTSInfo->base.cond.startVersion, pTSInfo->base.cond.endVersion, pTSInfo->base.cond.twindows.skey,
3824
             pTSInfo->base.cond.twindows.ekey, id);
3825
      pStreamInfo->recoverStep = STREAM_RECOVER_STEP__SCAN1;
889✔
3826
      pStreamInfo->recoverScanFinished = false;
889✔
3827
    } else {
3828
      pTSInfo->base.cond.startVersion = pStreamInfo->fillHistoryVer.minVer;
1,100✔
3829
      pTSInfo->base.cond.endVersion = pStreamInfo->fillHistoryVer.maxVer;
1,100✔
3830
      pTSInfo->base.cond.twindows = pStreamInfo->fillHistoryWindow;
1,100✔
3831
      qDebug("stream scan step2 (scan wal), verRange:%" PRId64 " - %" PRId64 ", window:%" PRId64 "-%" PRId64 ", %s",
1,100✔
3832
             pTSInfo->base.cond.startVersion, pTSInfo->base.cond.endVersion, pTSInfo->base.cond.twindows.skey,
3833
             pTSInfo->base.cond.twindows.ekey, id);
3834
      pStreamInfo->recoverStep = STREAM_RECOVER_STEP__NONE;
1,100✔
3835
    }
3836

3837
    pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader);
1,989✔
3838

3839
    pTSInfo->base.dataReader = NULL;
1,989✔
3840
    pInfo->pTableScanOp->status = OP_OPENED;
1,989✔
3841

3842
    pTSInfo->scanTimes = 0;
1,989✔
3843
    pTSInfo->currentGroupId = -1;
1,989✔
3844
  }
3845

3846
  if (pStreamInfo->recoverStep == STREAM_RECOVER_STEP__SCAN1) {
63,123✔
3847
    if (isTaskKilled(pTaskInfo)) {
5,860✔
3848
      qInfo("===stream===stream scan is killed. task id:%s, code %s", id, tstrerror(pTaskInfo->code));
1!
3849
      (*ppRes) = NULL;
1✔
3850
      return code;
1✔
3851
    }
3852

3853
    switch (pInfo->scanMode) {
5,859✔
3854
      case STREAM_SCAN_FROM_RES: {
1,087✔
3855
        pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
1,087✔
3856
        printSpecDataBlock(pInfo->pRecoverRes, getStreamOpName(pOperator->operatorType), "recover",
1,087✔
3857
                           GET_TASKID(pTaskInfo));
1,087✔
3858
        (*ppRes) = pInfo->pRecoverRes;
1,087✔
3859
        return code;
1,087✔
3860
      } break;
3861
      default:
4,772✔
3862
        break;
4,772✔
3863
    }
3864

3865
    code = doTableScanNext(pInfo->pTableScanOp, &pInfo->pRecoverRes);
4,772✔
3866
    QUERY_CHECK_CODE(code, lino, _end);
4,772!
3867

3868
    if (pInfo->pRecoverRes != NULL) {
4,772✔
3869
      code = calBlockTbName(pInfo, pInfo->pRecoverRes, 0);
3,884✔
3870
      QUERY_CHECK_CODE(code, lino, _end);
3,884!
3871
      if (!pInfo->igCheckUpdate && pInfo->pUpdateInfo) {
3,884✔
3872
        TSKEY maxTs = INT64_MIN;
3,073✔
3873
        code = pAPI->stateStore.updateInfoFillBlockData(pInfo->pUpdateInfo, pInfo->pRecoverRes, pInfo->primaryTsIndex,
3,073✔
3874
                                                        pInfo->primaryKeyIndex, &maxTs);
3875
        QUERY_CHECK_CODE(code, lino, _end);
3,073!
3876
        pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
3,073✔
3877
      }
3878
      if (pInfo->pCreateTbRes->info.rows > 0) {
3,884✔
3879
        pInfo->scanMode = STREAM_SCAN_FROM_RES;
1,087✔
3880
        printSpecDataBlock(pInfo->pCreateTbRes, getStreamOpName(pOperator->operatorType), "recover",
1,087✔
3881
                           GET_TASKID(pTaskInfo));
1,087✔
3882
        (*ppRes) = pInfo->pCreateTbRes;
1,087✔
3883
        return code;
1,087✔
3884
      }
3885

3886
      qDebug("stream recover scan get block, rows %" PRId64, pInfo->pRecoverRes->info.rows);
2,797✔
3887
      printSpecDataBlock(pInfo->pRecoverRes, getStreamOpName(pOperator->operatorType), "recover",
2,797✔
3888
                         GET_TASKID(pTaskInfo));
2,797✔
3889
      (*ppRes) = pInfo->pRecoverRes;
2,797✔
3890
      return code;
2,797✔
3891
    }
3892
    pStreamInfo->recoverStep = STREAM_RECOVER_STEP__NONE;
888✔
3893
    STableScanInfo* pTSInfo = pInfo->pTableScanOp->info;
888✔
3894
    pAPI->tsdReader.tsdReaderClose(pTSInfo->base.dataReader);
888✔
3895

3896
    pTSInfo->base.dataReader = NULL;
888✔
3897

3898
    pTSInfo->base.cond.startVersion = -1;
888✔
3899
    pTSInfo->base.cond.endVersion = -1;
888✔
3900

3901
    pStreamInfo->recoverScanFinished = true;
888✔
3902
    (*ppRes) = NULL;
888✔
3903
    return code;
888✔
3904
  }
3905

3906
  size_t total = taosArrayGetSize(pInfo->pBlockLists);
57,263✔
3907
// TODO: refactor
3908
FETCH_NEXT_BLOCK:
57,696✔
3909
  if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) {
57,696✔
3910
    if (pVtableInfos != NULL) {
11,991!
3911
      qInfo("stream vtable source scan would ignore all data blocks");
×
3912
      pInfo->validBlockIndex = total;
×
3913
    }
3914
    if (pInfo->validBlockIndex >= total) {
11,991✔
3915
      doClearBufferedBlocks(pInfo);
3916
      (*ppRes) = NULL;
2,344✔
3917
      return code;
2,344✔
3918
    }
3919

3920
    int32_t current = pInfo->validBlockIndex++;
9,647✔
3921
    qDebug("process %d/%d input data blocks, %s", current, (int32_t)total, id);
9,647✔
3922

3923
    SPackedData* pPacked = taosArrayGet(pInfo->pBlockLists, current);
9,647✔
3924
    QUERY_CHECK_NULL(pPacked, code, lino, _end, terrno);
9,647!
3925

3926
    SSDataBlock* pBlock = pPacked->pDataBlock;
9,647✔
3927
    if (pBlock->info.parTbName[0]) {
9,647✔
3928
      code =
3929
          pAPI->stateStore.streamStatePutParName(pStreamInfo->pState, pBlock->info.id.groupId, pBlock->info.parTbName);
56✔
3930
      QUERY_CHECK_CODE(code, lino, _end);
56!
3931
    }
3932

3933
    // TODO move into scan
3934
    pBlock->info.calWin.skey = INT64_MIN;
9,647✔
3935
    pBlock->info.calWin.ekey = INT64_MAX;
9,647✔
3936
    pBlock->info.dataLoad = 1;
9,647✔
3937
    if (pInfo->pUpdateInfo) {
9,647✔
3938
      pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pBlock->info.version);
6,628✔
3939
    }
3940

3941
    code = blockDataUpdateTsWindow(pBlock, 0);
9,647✔
3942
    QUERY_CHECK_CODE(code, lino, _end);
9,647!
3943
    switch (pBlock->info.type) {
9,647!
3944
      case STREAM_NORMAL:
2,030✔
3945
      case STREAM_GET_ALL:
3946
        printDataBlock(pBlock, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
2,030✔
3947
        setStreamOperatorState(&pInfo->basic, pBlock->info.type);
2,030✔
3948
        (*ppRes) = pBlock;
2,030✔
3949
        return code;
2,030✔
3950
      case STREAM_RETRIEVE: {
394✔
3951
        pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
394✔
3952
        pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RETRIEVE;
394✔
3953
        code = copyDataBlock(pInfo->pUpdateRes, pBlock);
394✔
3954
        QUERY_CHECK_CODE(code, lino, _end);
394!
3955
        pInfo->updateResIndex = 0;
394✔
3956
        prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex, NULL);
394✔
3957
        pAPI->stateStore.updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo);
394✔
3958
      } break;
394✔
3959
      case STREAM_DELETE_DATA: {
981✔
3960
        printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "delete recv", GET_TASKID(pTaskInfo));
981✔
3961
        SSDataBlock* pDelBlock = NULL;
981✔
3962
        if (pInfo->tqReader) {
981!
3963
          code = createSpecialDataBlock(STREAM_DELETE_DATA, &pDelBlock);
981✔
3964
          QUERY_CHECK_CODE(code, lino, _end);
981!
3965

3966
          code = filterDelBlockByUid(pDelBlock, pBlock, pInfo->tqReader, &pInfo->readerFn);
981✔
3967
          QUERY_CHECK_CODE(code, lino, _end);
981!
3968
        } else {
3969
          pDelBlock = pBlock;
×
3970
        }
3971

3972
        code = setBlockGroupIdByUid(pInfo, pDelBlock);
981✔
3973
        QUERY_CHECK_CODE(code, lino, _end);
981!
3974
        code = rebuildDeleteBlockData(pDelBlock, &pStreamInfo->fillHistoryWindow, id);
981✔
3975
        QUERY_CHECK_CODE(code, lino, _end);
981!
3976
        printSpecDataBlock(pDelBlock, getStreamOpName(pOperator->operatorType), "delete recv filtered",
981✔
3977
                           GET_TASKID(pTaskInfo));
981✔
3978
        if (pDelBlock->info.rows == 0) {
981✔
3979
          if (pInfo->tqReader) {
427!
3980
            blockDataDestroy(pDelBlock);
427✔
3981
          }
3982
          goto FETCH_NEXT_BLOCK;
428✔
3983
        }
3984

3985
        if (!isStreamWindow(pInfo)) {
554✔
3986
          code = generateDeleteResultBlock(pInfo, pDelBlock, pInfo->pDeleteDataRes);
3✔
3987
          QUERY_CHECK_CODE(code, lino, _end);
3!
3988
          if (pInfo->partitionSup.needCalc) {
3✔
3989
            pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA;
2✔
3990
          } else {
3991
            pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT;
1✔
3992
          }
3993
          blockDataDestroy(pDelBlock);
3✔
3994

3995
          if (pInfo->pDeleteDataRes->info.rows > 0) {
3!
3996
            printSpecDataBlock(pInfo->pDeleteDataRes, getStreamOpName(pOperator->operatorType), "delete result",
3✔
3997
                               GET_TASKID(pTaskInfo));
3✔
3998
            setStreamOperatorState(&pInfo->basic, pInfo->pDeleteDataRes->info.type);
3✔
3999
            (*ppRes) = pInfo->pDeleteDataRes;
3✔
4000
            return code;
553✔
4001
          } else {
4002
            goto FETCH_NEXT_BLOCK;
×
4003
          }
4004
        } else {
4005
          pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
551✔
4006
          pInfo->updateResIndex = 0;
551✔
4007
          code = generateScanRange(pInfo, pDelBlock, pInfo->pUpdateRes, STREAM_DELETE_DATA);
551✔
4008
          QUERY_CHECK_CODE(code, lino, _end);
551!
4009
          prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex, NULL);
551✔
4010
          code = copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes);
551✔
4011
          QUERY_CHECK_CODE(code, lino, _end);
551!
4012
          pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA;
551✔
4013
          if (pInfo->tqReader) {
551!
4014
            blockDataDestroy(pDelBlock);
551✔
4015
          }
4016
          if (pInfo->pDeleteDataRes->info.rows > 0) {
551✔
4017
            pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
550✔
4018
            printSpecDataBlock(pInfo->pDeleteDataRes, getStreamOpName(pOperator->operatorType), "delete result",
550✔
4019
                               GET_TASKID(pTaskInfo));
550✔
4020
            setStreamOperatorState(&pInfo->basic, pInfo->pDeleteDataRes->info.type);
550✔
4021
            (*ppRes) = pInfo->pDeleteDataRes;
550✔
4022
            return code;
550✔
4023
          } else {
4024
            goto FETCH_NEXT_BLOCK;
1✔
4025
          }
4026
        }
4027
      } break;
4028
      case STREAM_GET_RESULT: {
4,311✔
4029
        pInfo->blockType = STREAM_INPUT__DATA_SUBMIT;
4,311✔
4030
        pInfo->updateResIndex = 0;
4,311✔
4031
        pInfo->lastScanRange = pBlock->info.window;
4,311✔
4032
        TSKEY endKey = getNextTimeWindowStart(&pInfo->interval, pBlock->info.window.skey, TSDB_ORDER_ASC) - 1;
4,311✔
4033
        if (pInfo->useGetResultRange == true) {
4,309✔
4034
          endKey = pBlock->info.window.ekey;
890✔
4035
        }
4036
        code = copyGetResultBlock(pInfo->pUpdateRes, pBlock->info.window.skey, endKey);
4,309✔
4037
        QUERY_CHECK_CODE(code, lino, _end);
4,310!
4038
        pInfo->pUpdateInfo->maxDataVersion = -1;
4,310✔
4039
        prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex, NULL);
4,310✔
4040
        pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
4,311✔
4041
      } break;
4,311✔
4042
      case STREAM_DROP_CHILD_TABLE: {
1✔
4043
        int32_t deleteNum = 0;
1✔
4044
        code = deletePartName(&pInfo->stateStore, pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, pBlock, &deleteNum);
1✔
4045
        QUERY_CHECK_CODE(code, lino, _end);
1!
4046
        if (deleteNum == 0) {
1!
4047
          printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "block recv", GET_TASKID(pTaskInfo));
1✔
4048
          qDebug("===stream=== ignore block type 18, delete num is 0");
1!
4049
          goto FETCH_NEXT_BLOCK;
1✔
4050
        }
4051
      } break;
×
4052
      case STREAM_CHECKPOINT: {
×
4053
        qError("stream check point error. msg type: STREAM_INPUT__DATA_BLOCK");
×
4054
      } break;
×
4055
      default:
1,930✔
4056
        break;
1,930✔
4057
    }
4058
    printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "block recv", GET_TASKID(pTaskInfo));
6,635✔
4059
    setStreamOperatorState(&pInfo->basic, pBlock->info.type);
6,635✔
4060
    (*ppRes) = pBlock;
6,635✔
4061
    return code;
6,635✔
4062
  } else if (pInfo->blockType == STREAM_INPUT__DATA_SUBMIT) {
45,705✔
4063
    qDebug("stream scan mode:%d, %s", pInfo->scanMode, id);
43,891✔
4064
    if (pVtableInfos != NULL && pInfo->scanMode != STREAM_SCAN_FROM_READERHANDLE) {
43,891!
4065
      qError("stream vtable source scan should not have scan mode: %d", pInfo->scanMode);
×
4066
      pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
×
4067
    }
4068
    switch (pInfo->scanMode) {
43,891✔
4069
      case STREAM_SCAN_FROM_RES: {
733✔
4070
        pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
733✔
4071
        code = doCheckUpdate(pInfo, pInfo->pRes->info.window.ekey, pInfo->pRes);
733✔
4072
        QUERY_CHECK_CODE(code, lino, _end);
733!
4073
        setStreamOperatorState(&pInfo->basic, pInfo->pRes->info.type);
733✔
4074
        code = doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
733✔
4075
        QUERY_CHECK_CODE(code, lino, _end);
733!
4076
        pInfo->pRes->info.dataLoad = 1;
733✔
4077
        code = blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
733✔
4078
        QUERY_CHECK_CODE(code, lino, _end);
733!
4079
        if (pInfo->pRes->info.rows > 0) {
733!
4080
          printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
733✔
4081
          (*ppRes) = pInfo->pRes;
733✔
4082
          return code;
733✔
4083
        }
4084
      } break;
×
4085
      case STREAM_SCAN_FROM_DELETE_DATA: {
175✔
4086
        code = generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes, STREAM_PARTITION_DELETE_DATA);
175✔
4087
        QUERY_CHECK_CODE(code, lino, _end);
175!
4088
        if (pInfo->pUpdateRes->info.rows > 0) {
175✔
4089
          prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex, NULL);
159✔
4090
          pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
159✔
4091
          code = copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes);
159✔
4092
          QUERY_CHECK_CODE(code, lino, _end);
159!
4093
          pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA;
159✔
4094
          (*ppRes) = pInfo->pDeleteDataRes;
159✔
4095
          return code;
159✔
4096
        }
4097
        qError("%s===stream=== %s failed at line %d since pInfo->pUpdateRes is empty", GET_TASKID(pTaskInfo), __func__,
16!
4098
               __LINE__);
4099
        blockDataCleanup(pInfo->pUpdateDataRes);
16✔
4100
        pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
16✔
4101
      } break;
16✔
4102
      case STREAM_SCAN_FROM_UPDATERES: {
794✔
4103
        code = generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes, STREAM_CLEAR);
794✔
4104
        QUERY_CHECK_CODE(code, lino, _end);
794!
4105
        if (pInfo->pUpdateRes->info.rows > 0) {
794✔
4106
          prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex, NULL);
779✔
4107
          pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
779✔
4108
          (*ppRes) = pInfo->pUpdateRes;
779✔
4109
          return code;
779✔
4110
        }
4111
        qError("%s===stream=== %s failed at line %d since pInfo->pUpdateRes is empty", GET_TASKID(pTaskInfo), __func__,
15!
4112
               __LINE__);
4113
        blockDataCleanup(pInfo->pUpdateDataRes);
15✔
4114
        pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
15✔
4115
      } break;
15✔
4116
      case STREAM_SCAN_FROM_DATAREADER_RANGE:
8,340✔
4117
      case STREAM_SCAN_FROM_DATAREADER_RETRIEVE: {
4118
        if (pInfo->pRangeScanRes != NULL) {
8,340✔
4119
          (*ppRes) = pInfo->pRangeScanRes;
13✔
4120
          pInfo->pRangeScanRes = NULL;
13✔
4121
          return code;
2,123✔
4122
        } 
4123
        SSDataBlock* pSDB = NULL;
8,327✔
4124
        code = doRangeScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex, &pSDB);
8,327✔
4125
        QUERY_CHECK_CODE(code, lino, _end);
8,327!
4126
        if (pSDB) {
8,327✔
4127
          STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info;
2,110✔
4128
          pSDB->info.type = pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE ? STREAM_NORMAL : STREAM_PULL_DATA;
2,110✔
4129
          if (!pInfo->igCheckUpdate && pInfo->pUpdateInfo) {
2,110!
4130
            code = checkUpdateData(pInfo, true, pSDB, false);
1,503✔
4131
            QUERY_CHECK_CODE(code, lino, _end);
1,503!
4132
          }
4133

4134
          code = calBlockTbName(pInfo, pSDB, 0);
2,110✔
4135
          QUERY_CHECK_CODE(code, lino, _end);
2,110!
4136

4137
          printSpecDataBlock(pSDB, getStreamOpName(pOperator->operatorType), "update", GET_TASKID(pTaskInfo));
2,110✔
4138

4139
          if (pInfo->pCreateTbRes->info.rows > 0) {
2,110✔
4140
            printSpecDataBlock(pInfo->pCreateTbRes, getStreamOpName(pOperator->operatorType), "update",
13✔
4141
                               GET_TASKID(pTaskInfo));
13✔
4142
            (*ppRes) = pInfo->pCreateTbRes;
13✔
4143
            pInfo->pRangeScanRes = pSDB;
13✔
4144
            return code;
13✔
4145
          }
4146

4147
          (*ppRes) = pSDB;
2,097✔
4148
          return code;
2,097✔
4149
        }
4150
        blockDataCleanup(pInfo->pUpdateDataRes);
6,217✔
4151
        pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
6,217✔
4152
      } break;
6,217✔
4153
      default:
33,849✔
4154
        break;
33,849✔
4155
    }
4156

4157
    if (hasScanRange(pInfo)) {
40,097✔
4158
      pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE;
24✔
4159
      pInfo->updateResIndex = 0;
24✔
4160
      SStreamAggSupporter* pSup = pInfo->windowSup.pStreamAggSup;
24✔
4161
      code = copyDataBlock(pInfo->pUpdateRes, pSup->pScanBlock);
24✔
4162
      QUERY_CHECK_CODE(code, lino, _end);
24!
4163
      blockDataCleanup(pSup->pScanBlock);
24✔
4164
      prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex, NULL);
24✔
4165
      pInfo->pUpdateRes->info.type = STREAM_DELETE_DATA;
24✔
4166
      printSpecDataBlock(pInfo->pUpdateRes, getStreamOpName(pOperator->operatorType), "rebuild", GET_TASKID(pTaskInfo));
24✔
4167
      (*ppRes) = pInfo->pUpdateRes;
24✔
4168
      return code;
24✔
4169
    }
4170

4171
    SDataBlockInfo* pBlockInfo = &pInfo->pRes->info;
40,073✔
4172
    int32_t         totalBlocks = taosArrayGetSize(pInfo->pBlockLists);
40,073✔
4173

4174
  NEXT_SUBMIT_BLK:
40,073✔
4175
    while (1) {
4176
      if (pInfo->readerFn.tqReaderCurrentBlockConsumed(pInfo->tqReader)) {
92,656✔
4177
        if (pInfo->validBlockIndex >= totalBlocks) {
66,349✔
4178
          pAPI->stateStore.updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo);
13,763✔
4179
          doClearBufferedBlocks(pInfo);
4180

4181
          qDebug("stream scan return empty, all %d submit blocks consumed, %s", totalBlocks, id);
13,763✔
4182
          (*ppRes) = NULL;
13,763✔
4183
          return code;
13,763✔
4184
        }
4185

4186
        int32_t      current = pInfo->validBlockIndex++;
52,586✔
4187
        SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current);
52,586✔
4188
        QUERY_CHECK_NULL(pSubmit, code, lino, _end, terrno);
52,586!
4189

4190
        qDebug("set %d/%d as the input submit block, %s", current + 1, totalBlocks, id);
52,586✔
4191
        if (pAPI->tqReaderFn.tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver, NULL) <
52,588!
4192
            0) {
4193
          qError("submit msg messed up when initializing stream submit block %p, current %d/%d, %s", pSubmit, current,
×
4194
                 totalBlocks, id);
4195
          continue;
×
4196
        }
4197
      }
4198

4199
      blockDataCleanup(pInfo->pRes);
78,891✔
4200

4201
      while (pAPI->tqReaderFn.tqNextBlockImpl(pInfo->tqReader, id)) {
79,621✔
4202
        SSDataBlock* pRes = NULL;
27,037✔
4203

4204
        code = pAPI->tqReaderFn.tqRetrieveBlock(pInfo->tqReader, &pRes, id);
27,037✔
4205
        qDebug("retrieve data from submit completed code:%s rows:%" PRId64 " %s", tstrerror(code), pRes->info.rows, id);
27,037✔
4206

4207
        if (code != TSDB_CODE_SUCCESS || pRes->info.rows == 0) {
27,037!
4208
          qDebug("retrieve data failed, try next block in submit block, %s", id);
×
4209
          continue;
723✔
4210
        }
4211

4212
        code = setBlockIntoRes(pInfo, pRes, &pStreamInfo->fillHistoryWindow, false);
27,037✔
4213
        if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
27,035!
4214
          pInfo->pRes->info.rows = 0;
×
4215
          code = TSDB_CODE_SUCCESS;
×
4216
        } else {
4217
          QUERY_CHECK_CODE(code, lino, _end);
27,035!
4218
        }
4219

4220
        if (pInfo->pRes->info.rows == 0) {
27,035✔
4221
          continue;
723✔
4222
        }
4223

4224
        if (pVtableInfos != NULL && pInfo->pCreateTbRes->info.rows > 0) {
26,312!
4225
          qError("stream vtable source scan should not have create table res");
×
4226
          blockDataCleanup(pInfo->pCreateTbRes);
×
4227
        }
4228

4229
        if (pInfo->pCreateTbRes->info.rows > 0) {
26,312✔
4230
          pInfo->scanMode = STREAM_SCAN_FROM_RES;
733✔
4231
          qDebug("create table res exists, rows:%" PRId64 " return from stream scan, %s",
733!
4232
                 pInfo->pCreateTbRes->info.rows, id);
4233
          (*ppRes) = pInfo->pCreateTbRes;
733✔
4234
          return code;
733✔
4235
        }
4236

4237
        code = doCheckUpdate(pInfo, pBlockInfo->window.ekey, pInfo->pRes);
25,579✔
4238
        QUERY_CHECK_CODE(code, lino, _end);
25,581!
4239
        setStreamOperatorState(&pInfo->basic, pInfo->pRes->info.type);
25,581✔
4240
        if (pVtableInfos == NULL) {
25,581!
4241
          // filter should be applied in merge task for vtables
4242
          code = doFilter(pInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
25,581✔
4243
          QUERY_CHECK_CODE(code, lino, _end);
25,581!
4244
        }
4245

4246
        code = blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex);
25,581✔
4247
        QUERY_CHECK_CODE(code, lino, _end);
25,581!
4248

4249
        int64_t numOfUpdateRes = pInfo->pUpdateDataRes->info.rows;
25,581✔
4250
        qDebug("%s %" PRId64 " rows in datablock, update res:%" PRId64, id, pBlockInfo->rows, numOfUpdateRes);
25,581✔
4251
        if (pBlockInfo->rows > 0 || numOfUpdateRes > 0) {
25,581✔
4252
          break;
4253
        }
4254
      }
4255

4256
      if (pBlockInfo->rows > 0 || pInfo->pUpdateDataRes->info.rows > 0) {
78,158✔
4257
        break;
4258
      } else {
4259
        continue;
52,583✔
4260
      }
4261
    }
4262

4263
    // record the scan action.
4264
    pInfo->numOfExec++;
25,575✔
4265
    pOperator->resultInfo.totalRows += pBlockInfo->rows;
25,575✔
4266

4267
    qDebug("stream scan completed, and return source rows:%" PRId64 ", %s", pBlockInfo->rows, id);
25,575✔
4268
    if (pBlockInfo->rows > 0) {
25,577✔
4269
      printDataBlock(pInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo));
25,573✔
4270
      (*ppRes) = pInfo->pRes;
25,573✔
4271
      return code;
25,573✔
4272
    }
4273

4274
    if (pInfo->pUpdateDataRes->info.rows > 0) {
4!
4275
      goto FETCH_NEXT_BLOCK;
4✔
4276
    }
4277

4278
    goto NEXT_SUBMIT_BLK;
×
4279
  } else if (pInfo->blockType == STREAM_INPUT__CHECKPOINT) {
1,814!
4280
    if (pInfo->validBlockIndex >= total || pVtableInfos != NULL) {
1,814!
4281
      doClearBufferedBlocks(pInfo);
4282
      (*ppRes) = NULL;
907✔
4283
      return code;
907✔
4284
    }
4285

4286
    int32_t current = pInfo->validBlockIndex++;
907✔
4287
    qDebug("process %d/%d input data blocks, %s", current, (int32_t)total, id);
907✔
4288

4289
    SPackedData* pData = taosArrayGet(pInfo->pBlockLists, current);
907✔
4290
    QUERY_CHECK_NULL(pData, code, lino, _end, terrno);
907!
4291
    SSDataBlock* pBlock = taosArrayGet(pData->pDataBlock, 0);
907✔
4292
    QUERY_CHECK_NULL(pBlock, code, lino, _end, terrno);
907!
4293

4294
    if (pBlock->info.type == STREAM_CHECKPOINT) {
907!
4295
      streamScanOperatorSaveCheckpoint(pInfo);
907✔
4296
    }
4297
    // printDataBlock(pInfo->pCheckpointRes, "stream scan ck", GET_TASKID(pTaskInfo));
4298
    (*ppRes) = pInfo->pCheckpointRes;
907✔
4299
    return code;
907✔
4300
  } else {
4301
    qError("stream scan error, invalid block type %d, %s", pInfo->blockType, id);
×
4302
    code = TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
×
4303
  }
4304

4305
_end:
×
4306
  if (code != TSDB_CODE_SUCCESS) {
×
4307
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
4308
    pTaskInfo->code = code;
×
4309
    T_LONG_JMP(pTaskInfo->env, code);
×
4310
  }
4311
  (*ppRes) = NULL;
×
4312
  return code;
×
4313
}
4314

4315
static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) {
×
4316
  SSDataBlock* pRes = NULL;
×
4317
  int32_t      code = doStreamScanNext(pOperator, &pRes);
×
4318
  return pRes;
×
4319
}
4320
int32_t extractTableIdList(const STableListInfo* pTableListInfo, SArray** ppArrayRes) {
4,628✔
4321
  int32_t code = TSDB_CODE_SUCCESS;
4,628✔
4322
  int32_t lino = 0;
4,628✔
4323
  SArray* tableIdList = taosArrayInit(4, sizeof(uint64_t));
4,628✔
4324
  QUERY_CHECK_NULL(tableIdList, code, lino, _end, terrno);
4,646!
4325

4326
  // Transfer the Array of STableKeyInfo into uid list.
4327
  int32_t size = 0;
4,646✔
4328
  code = tableListGetSize(pTableListInfo, &size);
4,646✔
4329
  QUERY_CHECK_CODE(code, lino, _end);
4,647!
4330
  for (int32_t i = 0; i < size; ++i) {
52,011✔
4331
    STableKeyInfo* pkeyInfo = tableListGetInfo(pTableListInfo, i);
47,360✔
4332
    QUERY_CHECK_NULL(pkeyInfo, code, lino, _end, terrno);
47,337!
4333
    void* tmp = taosArrayPush(tableIdList, &pkeyInfo->uid);
47,337✔
4334
    QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
47,364!
4335
  }
4336

4337
  (*ppArrayRes) = tableIdList;
4,651✔
4338

4339
_end:
4,651✔
4340
  if (code != TSDB_CODE_SUCCESS) {
4,651!
4341
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
4342
  }
4343
  return code;
4,647✔
4344
}
4345

4346
static int32_t doRawScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
5,038✔
4347
  int32_t        code = TSDB_CODE_SUCCESS;
5,038✔
4348
  int32_t        lino = 0;
5,038✔
4349
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
5,038✔
4350
  SStorageAPI*   pAPI = &pTaskInfo->storageAPI;
5,038✔
4351

4352
  SStreamRawScanInfo* pInfo = pOperator->info;
5,038✔
4353
  pTaskInfo->streamInfo.btMetaRsp.batchMetaReq = NULL;  // use batchMetaReq != NULL to judge if data is meta
5,038✔
4354
  pTaskInfo->streamInfo.btMetaRsp.batchMetaLen = NULL;
5,038✔
4355

4356
  qDebug("tmqsnap doRawScan called");
5,038✔
4357
  if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) {
5,038✔
4358
    bool hasNext = false;
5,021✔
4359
    if (pInfo->dataReader && pInfo->sContext->withMeta != ONLY_META) {
5,021!
4360
      code = pAPI->tsdReader.tsdNextDataBlock(pInfo->dataReader, &hasNext);
5,016✔
4361
      if (code != TSDB_CODE_SUCCESS) {
5,013!
4362
        pAPI->tsdReader.tsdReaderReleaseDataBlock(pInfo->dataReader);
×
4363
        QUERY_CHECK_CODE(code, lino, _end);
×
4364
      }
4365
    }
4366

4367
    if (pInfo->dataReader && hasNext) {
5,018✔
4368
      if (isTaskKilled(pTaskInfo)) {
2,509!
4369
        pAPI->tsdReader.tsdReaderReleaseDataBlock(pInfo->dataReader);
×
4370
        return code;
2,508✔
4371
      }
4372

4373
      SSDataBlock* pBlock = NULL;
2,509✔
4374
      code = pAPI->tsdReader.tsdReaderRetrieveDataBlock(pInfo->dataReader, &pBlock, NULL);
2,509✔
4375
      QUERY_CHECK_CODE(code, lino, _end);
2,508!
4376

4377
      if (pBlock && pBlock->info.rows > 0) {
2,508!
4378
        bool hasPrimaryKey = pAPI->snapshotFn.taosXGetTablePrimaryKey(pInfo->sContext);
2,508✔
4379
        code = processPrimaryKey(pBlock, hasPrimaryKey, &pTaskInfo->streamInfo.currentOffset);
2,508✔
4380
        QUERY_CHECK_CODE(code, lino, _end);
2,508!
4381
        qDebug("tmqsnap doRawScan get data uid:%" PRId64, pBlock->info.id.uid);
2,508✔
4382
        (*ppRes) = pBlock;
2,508✔
4383
        return code;
2,508✔
4384
      }
4385
    }
4386

4387
    SMetaTableInfo mtInfo = {0};
2,509✔
4388
    code = pAPI->snapshotFn.getMetaTableInfoFromSnapshot(pInfo->sContext, &mtInfo);
2,509✔
4389
    QUERY_CHECK_CODE(code, lino, _end);
2,510!
4390
    if (code != 0) {
2,510!
4391
      destroyMetaTableInfo(&mtInfo);
4392
      QUERY_CHECK_CODE(code, lino, _end);
×
4393
    }
4394
    STqOffsetVal offset = {0};
2,510✔
4395
    if (mtInfo.uid == 0 || pInfo->sContext->withMeta == ONLY_META) {  // read snapshot done, change to get data from wal
2,510!
4396
      qDebug("tmqsnap read snapshot done, change to get data from wal");
31✔
4397
      tqOffsetResetToLog(&offset, pInfo->sContext->snapVersion + 1);
31✔
4398
    } else {
4399
      SValue val = {0};
2,479✔
4400
      tqOffsetResetToData(&offset, mtInfo.uid, INT64_MIN, val);
2,479!
4401
      qDebug("tmqsnap change get data uid:%" PRId64, mtInfo.uid);
2,479✔
4402
    }
4403
    destroyMetaTableInfo(&mtInfo);
4404
    code = qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType);
2,511✔
4405
    QUERY_CHECK_CODE(code, lino, _end);
2,509!
4406
    (*ppRes) = NULL;
2,509✔
4407
    return code;
2,509✔
4408
  } else if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_META) {
17!
4409
    SSnapContext* sContext = pInfo->sContext;
17✔
4410
    for (int32_t i = 0; i < tmqRowSize; i++) {
148!
4411
      void*   data = NULL;
148✔
4412
      int32_t dataLen = 0;
148✔
4413
      int16_t type = 0;
148✔
4414
      int64_t uid = 0;
148✔
4415
      if (pAPI->snapshotFn.getTableInfoFromSnapshot(sContext, &data, &dataLen, &type, &uid) < 0) {
148!
4416
        qError("tmqsnap getTableInfoFromSnapshot error");
×
4417
        taosMemoryFreeClear(data);
×
4418
        break;
17✔
4419
      }
4420

4421
      if (!sContext->queryMeta) {  // change to get data next poll request
148✔
4422
        STqOffsetVal offset = {0};
17✔
4423
        SValue       val = {0};
17!
4424
        tqOffsetResetToData(&offset, 0, INT64_MIN, val);
17✔
4425
        code = qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType);
17✔
4426
        QUERY_CHECK_CODE(code, lino, _end);
17!
4427
        break;
17✔
4428
      } else {
4429
        tqOffsetResetToMeta(&pTaskInfo->streamInfo.currentOffset, uid);
131✔
4430
        SMqMetaRsp tmpMetaRsp = {0};
131✔
4431
        tmpMetaRsp.resMsgType = type;
131✔
4432
        tmpMetaRsp.metaRspLen = dataLen;
131✔
4433
        tmpMetaRsp.metaRsp = data;
131✔
4434
        if (!pTaskInfo->streamInfo.btMetaRsp.batchMetaReq) {
131✔
4435
          pTaskInfo->streamInfo.btMetaRsp.batchMetaReq = taosArrayInit(4, POINTER_BYTES);
15✔
4436
          QUERY_CHECK_NULL(pTaskInfo->streamInfo.btMetaRsp.batchMetaReq, code, lino, _end, terrno);
15!
4437

4438
          pTaskInfo->streamInfo.btMetaRsp.batchMetaLen = taosArrayInit(4, sizeof(int32_t));
15✔
4439
          QUERY_CHECK_NULL(pTaskInfo->streamInfo.btMetaRsp.batchMetaLen, code, lino, _end, terrno);
15!
4440
        }
4441
        int32_t  tempRes = TSDB_CODE_SUCCESS;
131✔
4442
        uint32_t len = 0;
131✔
4443
        tEncodeSize(tEncodeMqMetaRsp, &tmpMetaRsp, len, tempRes);
131!
4444
        if (TSDB_CODE_SUCCESS != tempRes) {
131!
4445
          qError("tmqsnap tEncodeMqMetaRsp error");
×
4446
          taosMemoryFreeClear(data);
×
4447
          break;
×
4448
        }
4449

4450
        int32_t tLen = sizeof(SMqRspHead) + len;
131✔
4451
        void*   tBuf = taosMemoryCalloc(1, tLen);
131!
4452
        QUERY_CHECK_NULL(tBuf, code, lino, _end, terrno);
131!
4453

4454
        void*    metaBuff = POINTER_SHIFT(tBuf, sizeof(SMqRspHead));
131✔
4455
        SEncoder encoder = {0};
131✔
4456
        tEncoderInit(&encoder, metaBuff, len);
131✔
4457
        int32_t tempLen = tEncodeMqMetaRsp(&encoder, &tmpMetaRsp);
131✔
4458
        if (tempLen < 0) {
131!
4459
          qError("tmqsnap tEncodeMqMetaRsp error");
×
4460
          tEncoderClear(&encoder);
×
4461
          taosMemoryFreeClear(tBuf);
×
4462
          taosMemoryFreeClear(data);
×
4463
          break;
×
4464
        }
4465
        taosMemoryFreeClear(data);
131!
4466
        void* tmp = taosArrayPush(pTaskInfo->streamInfo.btMetaRsp.batchMetaReq, &tBuf);
131✔
4467
        QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
131!
4468

4469
        tmp = taosArrayPush(pTaskInfo->streamInfo.btMetaRsp.batchMetaLen, &tLen);
131✔
4470
        QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
131!
4471
      }
4472
    }
4473

4474
    (*ppRes) = NULL;
17✔
4475
    return code;
17✔
4476
  }
4477

4478
_end:
×
4479
  if (code != TSDB_CODE_SUCCESS) {
×
4480
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
4481
    pTaskInfo->code = code;
×
4482
    T_LONG_JMP(pTaskInfo->env, code);
×
4483
  }
4484

4485
  (*ppRes) = NULL;
×
4486
  return code;
×
4487
}
4488

4489
static void destroyRawScanOperatorInfo(void* param) {
341✔
4490
  SStreamRawScanInfo* pRawScan = (SStreamRawScanInfo*)param;
341✔
4491
  pRawScan->pAPI->tsdReader.tsdReaderClose(pRawScan->dataReader);
341✔
4492
  pRawScan->pAPI->snapshotFn.destroySnapshot(pRawScan->sContext);
342✔
4493
  tableListDestroy(pRawScan->pTableListInfo);
342✔
4494
  taosMemoryFree(pRawScan);
342!
4495
}
342✔
4496

4497
// for subscribing db or stb (not including column),
4498
// if this scan is used, meta data can be return
4499
// and schemas are decided when scanning
4500
int32_t createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) {
332✔
4501
  // create operator
4502
  // create tb reader
4503
  // create meta reader
4504
  // create tq reader
4505

4506
  QRY_PARAM_CHECK(pOptrInfo);
332!
4507
  int32_t code = TSDB_CODE_SUCCESS;
332✔
4508
  int32_t lino = 0;
332✔
4509

4510
  SStreamRawScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamRawScanInfo));
332!
4511
  SOperatorInfo*      pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
342!
4512
  if (pInfo == NULL || pOperator == NULL) {
342!
4513
    code = terrno;
×
4514
    lino = __LINE__;
×
4515
    goto _end;
×
4516
  }
4517

4518
  pInfo->pTableListInfo = tableListCreate();
342✔
4519
  QUERY_CHECK_NULL(pInfo->pTableListInfo, code, lino, _end, terrno);
341!
4520
  pInfo->vnode = pHandle->vnode;
341✔
4521
  pInfo->pAPI = &pTaskInfo->storageAPI;
341✔
4522

4523
  pInfo->sContext = pHandle->sContext;
341✔
4524
  setOperatorInfo(pOperator, "RawScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, false, OP_NOT_OPENED, pInfo,
341✔
4525
                  pTaskInfo);
4526

4527
  pOperator->fpSet = createOperatorFpSet(NULL, doRawScanNext, NULL, destroyRawScanOperatorInfo, optrDefaultBufFn, NULL,
342✔
4528
                                         optrDefaultGetNextExtFn, NULL);
4529
  *pOptrInfo = pOperator;
342✔
4530
  return code;
342✔
4531

4532
_end:
×
4533
  if (code != TSDB_CODE_SUCCESS) {
×
4534
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
4535
  }
4536
  taosMemoryFree(pInfo);
×
4537
  taosMemoryFree(pOperator);
×
4538
  pTaskInfo->code = code;
×
4539
  return code;
×
4540
}
4541

4542
void destroyStreamScanOperatorInfo(void* param) {
4,846✔
4543
  if (param == NULL) {
4,846!
4544
    return;
×
4545
  }
4546

4547
  SStreamScanInfo* pStreamScan = (SStreamScanInfo*)param;
4,846✔
4548
  if (pStreamScan->pTableScanOp && pStreamScan->pTableScanOp->info) {
4,846!
4549
    destroyOperator(pStreamScan->pTableScanOp);
4,648✔
4550
  }
4551

4552
  if (pStreamScan->tqReader != NULL && pStreamScan->readerFn.tqReaderClose != NULL) {
4,846!
4553
    pStreamScan->readerFn.tqReaderClose(pStreamScan->tqReader);
4,648✔
4554
  }
4555
  if (pStreamScan->pVtableMergeHandles) {
4,845!
4556
    taosHashCleanup(pStreamScan->pVtableMergeHandles);
×
4557
    pStreamScan->pVtableMergeHandles = NULL;
×
4558
  }
4559
  if (pStreamScan->pVtableMergeBuf) {
4,845!
4560
    destroyDiskbasedBuf(pStreamScan->pVtableMergeBuf);
×
4561
    pStreamScan->pVtableMergeBuf = NULL;
×
4562
  }
4563
  if (pStreamScan->pVtableReadyHandles) {
4,845!
4564
    taosArrayDestroy(pStreamScan->pVtableReadyHandles);
×
4565
    pStreamScan->pVtableReadyHandles = NULL;
×
4566
  }
4567
  if (pStreamScan->pTableListInfo) {
4,845!
4568
    tableListDestroy(pStreamScan->pTableListInfo);
×
4569
    pStreamScan->pTableListInfo = NULL;
×
4570
  }
4571
  if (pStreamScan->matchInfo.pList) {
4,845!
4572
    taosArrayDestroy(pStreamScan->matchInfo.pList);
4,845✔
4573
  }
4574
  if (pStreamScan->pPseudoExpr) {
4,846✔
4575
    destroyExprInfo(pStreamScan->pPseudoExpr, pStreamScan->numOfPseudoExpr);
2,369✔
4576
    taosMemoryFree(pStreamScan->pPseudoExpr);
2,369!
4577
  }
4578

4579
  cleanupExprSupp(&pStreamScan->tbnameCalSup);
4,846✔
4580
  cleanupExprSupp(&pStreamScan->tagCalSup);
4,844✔
4581

4582
  if (pStreamScan->stateStore.updateInfoDestroy) {
4,846!
4583
    pStreamScan->stateStore.updateInfoDestroy(pStreamScan->pUpdateInfo);
4,846✔
4584
  }
4585

4586
  blockDataDestroy(pStreamScan->pRes);
4,846✔
4587
  blockDataDestroy(pStreamScan->pUpdateRes);
4,845✔
4588
  blockDataDestroy(pStreamScan->pDeleteDataRes);
4,846✔
4589
  blockDataDestroy(pStreamScan->pUpdateDataRes);
4,844✔
4590
  blockDataDestroy(pStreamScan->pCreateTbRes);
4,845✔
4591
  taosArrayDestroy(pStreamScan->pBlockLists);
4,844✔
4592
  blockDataDestroy(pStreamScan->pCheckpointRes);
4,846✔
4593

4594
  taosMemoryFree(pStreamScan);
4,846!
4595
}
4596

4597
void streamScanReleaseState(SOperatorInfo* pOperator) {
841✔
4598
  int32_t          code = TSDB_CODE_SUCCESS;
841✔
4599
  int32_t          lino = 0;
841✔
4600
  SStreamScanInfo* pInfo = pOperator->info;
841✔
4601
  void*            pBuff = NULL;
841✔
4602
  SEncoder*        pEnCoder = NULL;
841✔
4603
  SEncoder*        pScanEnCoder = NULL;
841✔
4604
  if (!pInfo->pState) {
841!
4605
    return;
181✔
4606
  }
4607
  if (!pInfo->pUpdateInfo) {
841✔
4608
    qDebug("stask:%s streamScanReleaseState cancel", GET_TASKID(pOperator->pTaskInfo));
181✔
4609
    return;
181✔
4610
  }
4611
  int32_t  len = 0;
660✔
4612
  SEncoder encoder = {0};
660✔
4613
  pEnCoder = &encoder;
660✔
4614
  tEncoderInit(pEnCoder, NULL, 0);
660✔
4615
  if (tStartEncode(pEnCoder) != 0) {
660!
4616
    code = TSDB_CODE_FAILED;
×
4617
    QUERY_CHECK_CODE(code, lino, _end);
×
4618
  }
4619
  code = pInfo->stateStore.updateInfoSerialize(pEnCoder, pInfo->pUpdateInfo);
660✔
4620
  QUERY_CHECK_CODE(code, lino, _end);
660!
4621

4622
  tEndEncode(pEnCoder);
660✔
4623
  len += encoder.pos;
660✔
4624
  tEncoderClear(pEnCoder);
660✔
4625
  pEnCoder = NULL;
660✔
4626

4627
  pBuff = taosMemoryCalloc(1, len);
660!
4628
  if (!pBuff) {
660!
4629
    code = terrno;
×
4630
    QUERY_CHECK_CODE(code, lino, _end);
×
4631
  }
4632

4633
  SEncoder scanEncoder = {0};
660✔
4634
  pScanEnCoder = &scanEncoder;
660✔
4635
  tEncoderInit(pScanEnCoder, pBuff, len);
660✔
4636
  if (tStartEncode(pScanEnCoder) != 0) {
660!
4637
    code = TSDB_CODE_FAILED;
×
4638
    QUERY_CHECK_CODE(code, lino, _end);
×
4639
  }
4640
  code = pInfo->stateStore.updateInfoSerialize(pScanEnCoder, pInfo->pUpdateInfo);
660✔
4641
  QUERY_CHECK_CODE(code, lino, _end);
660!
4642

4643
  tEndEncode(pScanEnCoder);
660✔
4644
  tEncoderClear(pScanEnCoder);
660✔
4645
  pScanEnCoder = NULL;
660✔
4646

4647
  pInfo->stateStore.streamStateSaveInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME, strlen(STREAM_SCAN_OP_STATE_NAME),
660✔
4648
                                        pBuff, len);
4649
_end:
660✔
4650
  if (code != TSDB_CODE_SUCCESS) {
660!
4651
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
4652
  }
4653
  if (pEnCoder != NULL) {
660!
4654
    tEndEncode(pEnCoder);
×
4655
    tEncoderClear(pEnCoder);
×
4656
  }
4657
  if (pScanEnCoder != NULL) {
660!
4658
    tEndEncode(pScanEnCoder);
×
4659
    tEncoderClear(pScanEnCoder);
×
4660
  }
4661
  taosMemoryFree(pBuff);
660!
4662
}
4663

4664
void streamScanReloadState(SOperatorInfo* pOperator) {
841✔
4665
  int32_t          code = TSDB_CODE_SUCCESS;
841✔
4666
  int32_t          lino = 0;
841✔
4667
  SDecoder*        pDeCoder = NULL;
841✔
4668
  SStreamScanInfo* pInfo = pOperator->info;
841✔
4669
  if (!pInfo->pState) {
841!
4670
    return;
181✔
4671
  }
4672
  if (!pInfo->pUpdateInfo) {
841✔
4673
    qDebug("stask:%s streamScanReloadState cancel", GET_TASKID(pOperator->pTaskInfo));
181✔
4674
    return;
181✔
4675
  }
4676
  void*   pBuff = NULL;
660✔
4677
  int32_t len = 0;
660✔
4678
  code = pInfo->stateStore.streamStateGetInfo(pInfo->pState, STREAM_SCAN_OP_STATE_NAME,
660✔
4679
                                              strlen(STREAM_SCAN_OP_STATE_NAME), &pBuff, &len);
4680
  QUERY_CHECK_CODE(code, lino, _end);
660!
4681

4682
  SUpdateInfo* pUpInfo = taosMemoryCalloc(1, sizeof(SUpdateInfo));
660!
4683
  if (!pUpInfo) {
660!
4684
    code = terrno;
×
4685
    QUERY_CHECK_CODE(code, lino, _end);
×
4686
  }
4687

4688
  SDecoder decoder = {0};
660✔
4689
  pDeCoder = &decoder;
660✔
4690
  tDecoderInit(pDeCoder, pBuff, len);
660✔
4691
  if (tStartDecode(pDeCoder) < 0) {
660!
4692
    lino = __LINE__;
×
4693
    goto _end;
×
4694
  }
4695
  int32_t winCode = pInfo->stateStore.updateInfoDeserialize(pDeCoder, pUpInfo);
660✔
4696
  taosMemoryFree(pBuff);
660!
4697
  if (winCode == TSDB_CODE_SUCCESS && pInfo->pUpdateInfo) {
660!
4698
    if (pInfo->pUpdateInfo->minTS < 0) {
660✔
4699
      pInfo->stateStore.updateInfoDestroy(pInfo->pUpdateInfo);
659✔
4700
      pInfo->pUpdateInfo = pUpInfo;
659✔
4701
    } else {
4702
      pInfo->stateStore.windowSBfDelete(pInfo->pUpdateInfo, 1);
1✔
4703
      code = pInfo->stateStore.windowSBfAdd(pInfo->pUpdateInfo, 1);
1✔
4704
      QUERY_CHECK_CODE(code, lino, _end);
1!
4705

4706
      QUERY_CHECK_CONDITION((pInfo->pUpdateInfo->minTS > pUpInfo->minTS), code, lino, _end,
1!
4707
                            TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
4708
      pInfo->pUpdateInfo->maxDataVersion = TMAX(pInfo->pUpdateInfo->maxDataVersion, pUpInfo->maxDataVersion);
1✔
4709
      SHashObj* curMap = pInfo->pUpdateInfo->pMap;
1✔
4710
      void*     pIte = taosHashIterate(curMap, NULL);
1✔
4711
      while (pIte != NULL) {
2✔
4712
        size_t   keySize = 0;
1✔
4713
        int64_t* pUid = taosHashGetKey(pIte, &keySize);
1✔
4714
        code = taosHashPut(pUpInfo->pMap, pUid, sizeof(int64_t), pIte, sizeof(TSKEY));
1✔
4715
        if (code == TSDB_CODE_DUP_KEY) {
1!
4716
          code = TSDB_CODE_SUCCESS;
×
4717
        }
4718
        QUERY_CHECK_CODE(code, lino, _end);
1!
4719

4720
        pIte = taosHashIterate(curMap, pIte);
1✔
4721
      }
4722
      taosHashCleanup(curMap);
1✔
4723
      pInfo->pUpdateInfo->pMap = pUpInfo->pMap;
1✔
4724
      pUpInfo->pMap = NULL;
1✔
4725
      pInfo->stateStore.updateInfoDestroy(pUpInfo);
1✔
4726
    }
4727
  } else {
4728
    pInfo->stateStore.updateInfoDestroy(pUpInfo);
×
4729
  }
4730

4731
_end:
660✔
4732
  if (pDeCoder != NULL) {
660!
4733
    tEndDecode(pDeCoder);
660✔
4734
    tDecoderClear(pDeCoder);
660✔
4735
  }
4736
  if (code != TSDB_CODE_SUCCESS) {
660!
4737
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
4738
  }
4739
}
4740

4741
int32_t addPrimaryKeyCol(SSDataBlock* pBlock, uint8_t type, int32_t bytes) {
43✔
4742
  int32_t code = TSDB_CODE_SUCCESS;
43✔
4743
  int32_t lino = 0;
43✔
4744
  pBlock->info.rowSize += bytes;
43✔
4745
  SColumnInfoData infoData = {0};
43✔
4746
  infoData.info.type = type;
43✔
4747
  infoData.info.bytes = bytes;
43✔
4748
  void* tmp = taosArrayPush(pBlock->pDataBlock, &infoData);
43✔
4749
  QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
43!
4750

4751
_end:
43✔
4752
  if (code != TSDB_CODE_SUCCESS) {
43!
4753
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
4754
  }
4755
  return code;
43✔
4756
}
4757

4758
static SSDataBlock* createStreamVtableBlock(SColMatchInfo *pMatchInfo, const char *idstr) {
×
4759
  int32_t code = TSDB_CODE_SUCCESS;
×
4760
  int32_t lino = 0;
×
4761
  SSDataBlock *pRes = NULL;
×
4762

4763
  QUERY_CHECK_NULL(pMatchInfo, code, lino, _end, TSDB_CODE_INVALID_PARA);
×
4764

4765
  code = createDataBlock(&pRes);
×
4766
  QUERY_CHECK_CODE(code, lino, _end);
×
4767
  int32_t numOfOutput = taosArrayGetSize(pMatchInfo->pList);
×
4768
  for (int32_t i = 0; i < numOfOutput; ++i) {
×
4769
    SColMatchItem*  pItem = taosArrayGet(pMatchInfo->pList, i);
×
4770
    if (!pItem->needOutput) {
×
4771
      continue;
×
4772
    }
4773
    SColumnInfoData colInfo = createColumnInfoData(pItem->dataType.type, pItem->dataType.bytes, pItem->colId);
×
4774
    code = blockDataAppendColInfo(pRes, &colInfo);
×
4775
    QUERY_CHECK_CODE(code, lino, _end);
×
4776
  }
4777

4778
_end:
×
4779
  if (code != TSDB_CODE_SUCCESS) {
×
4780
    qError("%s failed at line %d since %s, id: %s", __func__, lino, tstrerror(code), idstr);
×
4781
    if (pRes != NULL) {
×
4782
      blockDataDestroy(pRes);
×
4783
    }
4784
    pRes = NULL;
×
4785
    terrno = code;
×
4786
  }
4787
  return pRes;
×
4788
}
4789

4790
static int32_t createStreamNormalScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode,
4,846✔
4791
                                                  SNode* pTagCond, STableListInfo* pTableListInfo,
4792
                                                  SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) {
4793
  QRY_PARAM_CHECK(pOptrInfo);
4,846!
4794

4795
  int32_t          code = TSDB_CODE_SUCCESS;
4,846✔
4796
  int32_t          lino = 0;
4,846✔
4797
  SArray*          pColIds = NULL;
4,846✔
4798
  SStreamScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamScanInfo));
4,846!
4799
  SOperatorInfo*   pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
4,842!
4800
  SStorageAPI*     pAPI = &pTaskInfo->storageAPI;
4,846✔
4801
  const char*      idstr = pTaskInfo->id.str;
4,846✔
4802
  SSHashObj*       pVtableInfos = pTaskInfo->pSubplan->pVTables;
4,846✔
4803

4804
  if (pInfo == NULL || pOperator == NULL) {
4,846!
4805
    code = terrno;
×
4806
    goto _error;
×
4807
  }
4808

4809
  SScanPhysiNode*     pScanPhyNode = &pTableScanNode->scan;
4,846✔
4810
  SDataBlockDescNode* pDescNode = pScanPhyNode->node.pOutputDataBlockDesc;
4,846✔
4811

4812
  pInfo->pTagCond = pTagCond;
4,846✔
4813
  pInfo->pGroupTags = pTableScanNode->pGroupTags;
4,846✔
4814

4815
  int32_t numOfCols = 0;
4,846✔
4816
  code = extractColMatchInfo(pScanPhyNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo);
4,846✔
4817
  if (code != TSDB_CODE_SUCCESS) {
4,841!
4818
    goto _error;
×
4819
  }
4820

4821
  SDataType pkType = {0};
4,841✔
4822
  pInfo->primaryKeyIndex = -1;
4,841✔
4823
  pInfo->basic.primaryPkIndex = -1;
4,841✔
4824
  int32_t numOfOutput = taosArrayGetSize(pInfo->matchInfo.pList);
4,841✔
4825
  pColIds = taosArrayInit(numOfOutput, sizeof(int16_t));
4,842✔
4826
  QUERY_CHECK_NULL(pColIds, code, lino, _error, terrno);
4,846!
4827

4828
  for (int32_t i = 0; i < numOfOutput; ++i) {
27,305✔
4829
    SColMatchItem* id = taosArrayGet(pInfo->matchInfo.pList, i);
22,457✔
4830
    QUERY_CHECK_NULL(id, code, lino, _error, terrno);
22,457!
4831

4832
    int16_t colId = id->colId;
22,457✔
4833
    void*   tmp = taosArrayPush(pColIds, &colId);
22,459✔
4834
    QUERY_CHECK_NULL(tmp, code, lino, _error, terrno);
22,459!
4835

4836
    if (id->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
22,459✔
4837
      pInfo->primaryTsIndex = id->dstSlotId;
4,647✔
4838
    }
4839
    if (id->isPk) {
22,459✔
4840
      pInfo->primaryKeyIndex = id->dstSlotId;
43✔
4841
      pInfo->basic.primaryPkIndex = id->dstSlotId;
43✔
4842
      pkType = id->dataType;
43✔
4843
    }
4844
  }
4845

4846
  pInfo->pPartTbnameSup = NULL;
4,848✔
4847
  if (pTableScanNode->pSubtable != NULL) {
4,848✔
4848
    SExprInfo* pSubTableExpr = taosMemoryCalloc(1, sizeof(SExprInfo));
1,167!
4849
    if (pSubTableExpr == NULL) {
1,167!
4850
      code = terrno;
×
4851
      goto _error;
×
4852
    }
4853

4854
    pInfo->tbnameCalSup.pExprInfo = pSubTableExpr;
1,167✔
4855
    code = createExprFromOneNode(pSubTableExpr, pTableScanNode->pSubtable, 0);
1,167✔
4856
    QUERY_CHECK_CODE(code, lino, _error);
1,167!
4857

4858
    if (initExprSupp(&pInfo->tbnameCalSup, pSubTableExpr, 1, &pTaskInfo->storageAPI.functionStore) != 0) {
1,167!
4859
      goto _error;
×
4860
    }
4861
  }
4862

4863
  if (pTableScanNode->pTags != NULL) {
4,848✔
4864
    int32_t    numOfTags;
4865
    SExprInfo* pTagExpr = createExpr(pTableScanNode->pTags, &numOfTags);
1,071✔
4866
    if (pTagExpr == NULL) {
1,072!
4867
      goto _error;
×
4868
    }
4869
    code = initExprSupp(&pInfo->tagCalSup, pTagExpr, numOfTags, &pTaskInfo->storageAPI.functionStore);
1,072✔
4870
    if (code != 0) {
1,072!
4871
      goto _error;
×
4872
    }
4873
  }
4874

4875
  pInfo->pBlockLists = taosArrayInit(4, sizeof(SPackedData));
4,849✔
4876
  TSDB_CHECK_NULL(pInfo->pBlockLists, code, lino, _error, terrno);
4,844!
4877

4878
  if (pHandle->vnode) {
4,844✔
4879
    SOperatorInfo* pTableScanOp = NULL;
4,646✔
4880
    code = createTableScanOperatorInfo(pTableScanNode, pHandle, pTableListInfo, pTaskInfo, &pTableScanOp);
4,646✔
4881
    if (pTableScanOp == NULL || code != 0) {
4,645!
4882
      qError("createTableScanOperatorInfo error, code:%d", pTaskInfo->code);
×
4883
      goto _error;
×
4884
    }
4885

4886
    STableScanInfo* pTSInfo = (STableScanInfo*)pTableScanOp->info;
4,646✔
4887
    if (pHandle->version > 0) {
4,646✔
4888
      pTSInfo->base.cond.endVersion = pHandle->version;
1,244✔
4889
    }
4890

4891
    STableKeyInfo* pList = NULL;
4,646✔
4892
    int32_t        num = 0;
4,646✔
4893
    code = tableListGetGroupList(pTableListInfo, 0, &pList, &num);
4,646✔
4894
    QUERY_CHECK_CODE(code, lino, _error);
4,646!
4895

4896
    if (pHandle->initTableReader) {
4,646✔
4897
      pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER;
1,243✔
4898
      pTSInfo->base.dataReader = NULL;
1,243✔
4899
    }
4900

4901
    if (pHandle->initTqReader) {
4,646!
4902
      pInfo->tqReader = pAPI->tqReaderFn.tqReaderOpen(pHandle->vnode);
4,646✔
4903
      QUERY_CHECK_NULL(pInfo->tqReader, code, lino, _error, terrno);
4,648!
4904
    } else {
4905
      pInfo->tqReader = pHandle->tqReader;
×
4906
      QUERY_CHECK_NULL(pInfo->tqReader, code, lino, _error, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
×
4907
    }
4908

4909
    if (pVtableInfos != NULL) {
4,648!
4910
      // save vtable info into tqReader for vtable source scan
4911
      SSDataBlock* pResBlock = createStreamVtableBlock(&pInfo->matchInfo, idstr);
×
4912
      QUERY_CHECK_CODE(code, lino, _error);
×
4913
      code = pAPI->tqReaderFn.tqReaderSetVtableInfo(pInfo->tqReader, pHandle->vnode, pAPI, pVtableInfos, &pResBlock,
×
4914
                                                    idstr);
4915
      QUERY_CHECK_CODE(code, lino, _error);
×
4916
    }
4917

4918
    pInfo->pUpdateInfo = NULL;
4,648✔
4919
    pInfo->pTableScanOp = pTableScanOp;
4,648✔
4920
    if (pInfo->pTableScanOp->pTaskInfo->streamInfo.pState) {
4,648✔
4921
      pAPI->stateStore.streamStateSetNumber(pInfo->pTableScanOp->pTaskInfo->streamInfo.pState, -1,
3,403✔
4922
                                            pInfo->primaryTsIndex);
4923
    }
4924

4925
    pInfo->readHandle = *pHandle;
4,648✔
4926
    pTaskInfo->streamInfo.snapshotVer = pHandle->version;
4,648✔
4927
    pInfo->pCreateTbRes = buildCreateTableBlock(&pInfo->tbnameCalSup, &pInfo->tagCalSup);
4,648✔
4928
    QUERY_CHECK_NULL(pInfo->pCreateTbRes, code, lino, _error, terrno);
4,648!
4929
    pInfo->hasPart = false;
4,648✔
4930

4931
    code = blockDataEnsureCapacity(pInfo->pCreateTbRes, 8);
4,648✔
4932
    QUERY_CHECK_CODE(code, lino, _error);
4,648!
4933

4934
    // set the extract column id to streamHandle
4935
    code = pAPI->tqReaderFn.tqReaderSetColIdList(pInfo->tqReader, pColIds, idstr);
4,648✔
4936
    QUERY_CHECK_CODE(code, lino, _error);
4,648!
4937

4938
    SArray* tableIdList = NULL;
4,648✔
4939
    code = extractTableIdList(((STableScanInfo*)(pInfo->pTableScanOp->info))->base.pTableListInfo, &tableIdList);
4,648✔
4940
    QUERY_CHECK_CODE(code, lino, _error);
4,647!
4941
    code = pAPI->tqReaderFn.tqReaderSetQueryTableList(pInfo->tqReader, tableIdList, idstr);
4,647✔
4942
    QUERY_CHECK_CODE(code, lino, _error);
4,648!
4943
    taosArrayDestroy(tableIdList);
4,648✔
4944
    memcpy(&pTaskInfo->streamInfo.tableCond, &pTSInfo->base.cond, sizeof(SQueryTableDataCond));
4,648✔
4945
  } else {
4946
    taosArrayDestroy(pColIds);
198✔
4947
    tableListDestroy(pTableListInfo);
198✔
4948
  }
4949

4950
  // clear the local variable to avoid repeatly free
4951
  pColIds = NULL;
4,846✔
4952

4953
  // create the pseduo columns info
4954
  if (pTableScanNode->scan.pScanPseudoCols != NULL) {
4,846✔
4955
    code = createExprInfo(pTableScanNode->scan.pScanPseudoCols, NULL, &pInfo->pPseudoExpr, &pInfo->numOfPseudoExpr);
2,369✔
4956
    QUERY_CHECK_CODE(code, lino, _error);
2,369!
4957
  }
4958

4959
  code = filterInitFromNode((SNode*)pScanPhyNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
4,846✔
4960
  QUERY_CHECK_CODE(code, lino, _error);
4,846!
4961

4962
  pInfo->pRes = createDataBlockFromDescNode(pDescNode);
4,846✔
4963
  QUERY_CHECK_NULL(pInfo->pRes, code, lino, _error, terrno);
4,846!
4964
  code = createSpecialDataBlock(STREAM_CLEAR, &pInfo->pUpdateRes);
4,846✔
4965
  QUERY_CHECK_CODE(code, lino, _error);
4,846!
4966

4967
  pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
4,846✔
4968
  pInfo->windowSup = (SWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN};
4,846✔
4969
  pInfo->groupId = 0;
4,846✔
4970
  pInfo->igCheckGroupId = false;
4,846✔
4971
  pInfo->pStreamScanOp = pOperator;
4,846✔
4972
  pInfo->deleteDataIndex = 0;
4,846✔
4973
  code = createSpecialDataBlock(STREAM_DELETE_DATA, &pInfo->pDeleteDataRes);
4,846✔
4974
  QUERY_CHECK_CODE(code, lino, _error);
4,846!
4975

4976
  pInfo->updateWin = (STimeWindow){.skey = INT64_MAX, .ekey = INT64_MAX};
4,846✔
4977
  code = createSpecialDataBlock(STREAM_CLEAR, &pInfo->pUpdateDataRes);
4,846✔
4978
  QUERY_CHECK_CODE(code, lino, _error);
4,845!
4979

4980
  if (hasPrimaryKeyCol(pInfo)) {
4,845✔
4981
    code = addPrimaryKeyCol(pInfo->pUpdateDataRes, pkType.type, pkType.bytes);
43✔
4982
    QUERY_CHECK_CODE(code, lino, _error);
43!
4983

4984
    pInfo->pkColType = pkType.type;
43✔
4985
    pInfo->pkColLen = pkType.bytes;
43✔
4986
  }
4987

4988
  pInfo->assignBlockUid = pTableScanNode->assignBlockUid;
4,845✔
4989
  pInfo->partitionSup.needCalc = false;
4,845✔
4990
  pInfo->igCheckUpdate = pTableScanNode->igCheckUpdate;
4,845✔
4991
  pInfo->igExpired = pTableScanNode->igExpired;
4,845✔
4992
  pInfo->twAggSup.maxTs = INT64_MIN;
4,845✔
4993
  pInfo->pState = pTaskInfo->streamInfo.pState;
4,845✔
4994
  pInfo->stateStore = pTaskInfo->storageAPI.stateStore;
4,845✔
4995
  pInfo->readerFn = pTaskInfo->storageAPI.tqReaderFn;
4,845✔
4996
  pInfo->pFillSup = NULL;
4,845✔
4997
  pInfo->useGetResultRange = false;
4,845✔
4998
  pInfo->pRangeScanRes = NULL;
4,845✔
4999

5000
  code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);
4,845✔
5001
  QUERY_CHECK_CODE(code, lino, _error);
4,846!
5002

5003
  SET_WIN_KEY_INVALID(pInfo->lastScanRange.skey);
4,846✔
5004
  SET_WIN_KEY_INVALID(pInfo->lastScanRange.ekey);
4,846✔
5005
  // for stream
5006
  if (pTaskInfo->streamInfo.pState) {
4,846✔
5007
    void*   buff = NULL;
3,601✔
5008
    int32_t len = 0;
3,601✔
5009
    int32_t res = pAPI->stateStore.streamStateGetInfo(pTaskInfo->streamInfo.pState, STREAM_SCAN_OP_CHECKPOINT_NAME,
3,601✔
5010
                                                      strlen(STREAM_SCAN_OP_CHECKPOINT_NAME), &buff, &len);
5011
    if (res == TSDB_CODE_SUCCESS) {
3,600✔
5012
      streamScanOperatorDecode(buff, len, pInfo);
6✔
5013
      taosMemoryFree(buff);
6!
5014
    }
5015
  }
5016

5017
  setOperatorInfo(pOperator, STREAM_SCAN_OP_NAME, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, false, OP_NOT_OPENED, pInfo,
4,845✔
5018
                  pTaskInfo);
5019
  pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock);
4,846✔
5020

5021
  __optr_fn_t nextFn = (pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM) ? doStreamScanNext : doQueueScanNext;
4,846✔
5022
  pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, nextFn, NULL, destroyStreamScanOperatorInfo, optrDefaultBufFn,
4,846✔
5023
                                         NULL, optrDefaultGetNextExtFn, NULL);
5024
  setOperatorStreamStateFn(pOperator, streamScanReleaseState, streamScanReloadState);
4,846✔
5025

5026
  *pOptrInfo = pOperator;
4,845✔
5027
  return code;
4,845✔
5028

5029
_error:
×
5030
  if (pColIds != NULL) {
×
5031
    taosArrayDestroy(pColIds);
×
5032
  }
5033

5034
  if (pInfo != NULL && pInfo->pTableScanOp != NULL) {
×
5035
    STableScanInfo* p = (STableScanInfo*)pInfo->pTableScanOp->info;
×
5036
    if (p != NULL) {
×
5037
      p->base.pTableListInfo = NULL;
×
5038
    }
5039
    destroyStreamScanOperatorInfo(pInfo);
×
5040
  }
5041

5042
  if (pOperator != NULL) {
×
5043
    pOperator->info = NULL;
×
5044
    destroyOperator(pOperator);
×
5045
  }
5046
  pTaskInfo->code = code;
×
5047
  return code;
×
5048
}
5049

5050
int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond,
4,828✔
5051
                                     STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo,
5052
                                     SOperatorInfo** pOptrInfo) {
5053
  if (pTableScanNode->triggerType == STREAM_TRIGGER_CONTINUOUS_WINDOW_CLOSE) {
4,828!
5054
    return createStreamDataScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTableListInfo, pTaskInfo, pOptrInfo);
×
5055
  } else {
5056
    return createStreamNormalScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTableListInfo, pTaskInfo, pOptrInfo);
4,828✔
5057
  }
5058
}
5059

5060
static int32_t doTagScanOneTable(SOperatorInfo* pOperator, SSDataBlock* pRes, SMetaReader* mr, SStorageAPI* pAPI) {
14,129✔
5061
  int32_t        code = TSDB_CODE_SUCCESS;
14,129✔
5062
  int32_t        lino = 0;
14,129✔
5063
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
14,129✔
5064
  STagScanInfo*  pInfo = pOperator->info;
14,129✔
5065
  SExprInfo*     pExprInfo = &pOperator->exprSupp.pExprInfo[0];
14,129✔
5066
  int32_t        count = pRes->info.rows;
14,129✔
5067

5068
  STableKeyInfo* item = tableListGetInfo(pInfo->pTableListInfo, pInfo->curPos);
14,129✔
5069
  if (!item) {
14,129!
5070
    qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno),
×
5071
           GET_TASKID(pTaskInfo));
5072
    tDecoderClear(&(*mr).coder);
×
5073
    goto _end;
×
5074
  }
5075

5076
  code = pAPI->metaReaderFn.getTableEntryByUid(mr, item->uid);
14,129✔
5077
  tDecoderClear(&(*mr).coder);
14,078✔
5078
  if (code != TSDB_CODE_SUCCESS) {
14,165!
5079
    qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno),
×
5080
           GET_TASKID(pTaskInfo));
5081
    goto _end;
×
5082
  }
5083

5084
  char str[512];
5085
  for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
28,947✔
5086
    SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId);
14,814✔
5087

5088
    // refactor later
5089
    if (FUNCTION_TYPE_TBNAME == pExprInfo[j].pExpr->_function.functionType) {
14,813✔
5090
      STR_TO_VARSTR(str, (*mr).me.name);
14,162✔
5091
      code = colDataSetVal(pDst, (count), str, false);
14,162✔
5092
      QUERY_CHECK_CODE(code, lino, _end);
14,131!
5093
    } else if (FUNCTION_TYPE_TBUID == pExprInfo[j].pExpr->_function.functionType) {
651✔
5094
      code = colDataSetVal(pDst, (count), (char*)&(*mr).me.uid, false);
150✔
5095
      QUERY_CHECK_CODE(code, lino, _end);
150!
5096
    } else if (FUNCTION_TYPE_VGID == pExprInfo[j].pExpr->_function.functionType) {
501✔
5097
      code = colDataSetVal(pDst, (count), (char*)&pTaskInfo->id.vgId, false);
150✔
5098
      QUERY_CHECK_CODE(code, lino, _end);
150!
5099
    } else {  // it is a tag value
5100
      STagVal val = {0};
351✔
5101
      val.cid = pExprInfo[j].base.pParam[0].pCol->colId;
351✔
5102
      const char* p = pAPI->metaFn.extractTagVal((*mr).me.ctbEntry.pTags, pDst->info.type, &val);
351✔
5103

5104
      char* data = NULL;
351✔
5105
      if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL) {
351!
5106
        data = tTagValToData((const STagVal*)p, false);
351✔
5107
      } else {
5108
        data = (char*)p;
×
5109
      }
5110

5111
      code = colDataSetVal(pDst, (count), data,
351✔
5112
                           (data == NULL) || (pDst->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data)));
351!
5113
      QUERY_CHECK_CODE(code, lino, _end);
351!
5114

5115
      if ((pDst->info.type != TSDB_DATA_TYPE_JSON) && (p != NULL) && IS_VAR_DATA_TYPE(((const STagVal*)p)->type) &&
351!
5116
          (data != NULL)) {
5117
        taosMemoryFree(data);
272!
5118
      }
5119
    }
5120
  }
5121

5122
_end:
14,133✔
5123
  if (code != TSDB_CODE_SUCCESS) {
14,133!
5124
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5125
    pTaskInfo->code = code;
×
5126
  } else {
5127
    pRes->info.rows++;
14,133✔
5128
  }
5129

5130
  return code;
14,133✔
5131
}
5132

5133
static void tagScanFreeUidTag(void* p) {
68,636✔
5134
  STUidTagInfo* pInfo = p;
68,636✔
5135
  if (pInfo->pTagVal != NULL) {
68,636!
5136
    taosMemoryFree(pInfo->pTagVal);
68,668!
5137
  }
5138
}
68,684✔
5139

5140
static int32_t tagScanCreateResultData(SDataType* pType, int32_t numOfRows, SScalarParam* pParam) {
15,410✔
5141
  SColumnInfoData* pColumnData = taosMemoryCalloc(1, sizeof(SColumnInfoData));
15,410!
5142
  if (pColumnData == NULL) {
15,428!
5143
    return terrno;
×
5144
  }
5145

5146
  pColumnData->info.type = pType->type;
15,428✔
5147
  pColumnData->info.bytes = pType->bytes;
15,428✔
5148
  pColumnData->info.scale = pType->scale;
15,428✔
5149
  pColumnData->info.precision = pType->precision;
15,428✔
5150

5151
  int32_t code = colInfoDataEnsureCapacity(pColumnData, numOfRows, true);
15,428✔
5152
  if (code != TSDB_CODE_SUCCESS) {
15,426✔
5153
    terrno = code;
2✔
5154
    taosMemoryFree(pColumnData);
×
5155
    return terrno;
×
5156
  }
5157

5158
  pParam->columnData = pColumnData;
15,424✔
5159
  pParam->colAlloced = true;
15,424✔
5160
  return TSDB_CODE_SUCCESS;
15,424✔
5161
}
5162

5163
static EDealRes tagScanRewriteTagColumn(SNode** pNode, void* pContext) {
96,985✔
5164
  int32_t                code = TSDB_CODE_SUCCESS;
96,985✔
5165
  int32_t                lino = 0;
96,985✔
5166
  STagScanFilterContext* pCtx = (STagScanFilterContext*)pContext;
96,985✔
5167
  SColumnNode*           pSColumnNode = NULL;
96,985✔
5168
  if (QUERY_NODE_COLUMN == nodeType((*pNode))) {
96,985✔
5169
    pSColumnNode = *(SColumnNode**)pNode;
27,100✔
5170
  } else if (QUERY_NODE_FUNCTION == nodeType((*pNode))) {
69,885!
5171
    SFunctionNode* pFuncNode = *(SFunctionNode**)(pNode);
×
5172
    if (pFuncNode->funcType == FUNCTION_TYPE_TBNAME) {
×
5173
      pSColumnNode = NULL;
×
5174
      pCtx->code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pSColumnNode);
×
5175
      if (NULL == pSColumnNode) {
×
5176
        return DEAL_RES_ERROR;
×
5177
      }
5178
      pSColumnNode->colId = -1;
×
5179
      pSColumnNode->colType = COLUMN_TYPE_TBNAME;
×
5180
      pSColumnNode->node.resType.type = TSDB_DATA_TYPE_VARCHAR;
×
5181
      pSColumnNode->node.resType.bytes = TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE;
×
5182
      nodesDestroyNode(*pNode);
×
5183
      *pNode = (SNode*)pSColumnNode;
×
5184
    } else {
5185
      return DEAL_RES_CONTINUE;
×
5186
    }
5187
  } else {
5188
    return DEAL_RES_CONTINUE;
69,885✔
5189
  }
5190

5191
  void* data = taosHashGet(pCtx->colHash, &pSColumnNode->colId, sizeof(pSColumnNode->colId));
27,100✔
5192
  if (!data) {
27,091✔
5193
    code = taosHashPut(pCtx->colHash, &pSColumnNode->colId, sizeof(pSColumnNode->colId), pNode, sizeof((*pNode)));
15,412✔
5194
    if (code == TSDB_CODE_DUP_KEY) {
15,439!
5195
      code = TSDB_CODE_SUCCESS;
×
5196
    }
5197
    QUERY_CHECK_CODE(code, lino, _end);
15,439!
5198
    pSColumnNode->slotId = pCtx->index++;
15,439✔
5199
    SColumnInfo cInfo = {.colId = pSColumnNode->colId,
15,439✔
5200
                         .type = pSColumnNode->node.resType.type,
15,439✔
5201
                         .bytes = pSColumnNode->node.resType.bytes};
15,439✔
5202
    void*       tmp = taosArrayPush(pCtx->cInfoList, &cInfo);
15,439✔
5203
    QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
15,434!
5204
  } else {
5205
    SColumnNode* col = *(SColumnNode**)data;
11,679✔
5206
    pSColumnNode->slotId = col->slotId;
11,679✔
5207
  }
5208

5209
_end:
27,113✔
5210
  if (code != TSDB_CODE_SUCCESS) {
27,113!
5211
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5212
    return DEAL_RES_ERROR;
×
5213
  }
5214
  return DEAL_RES_CONTINUE;
27,113✔
5215
}
5216

5217
static int32_t tagScanFilterByTagCond(SArray* aUidTags, SNode* pTagCond, SArray* aFilterIdxs, void* pVnode,
15,423✔
5218
                                      SStorageAPI* pAPI, STagScanInfo* pInfo) {
5219
  int32_t code = TSDB_CODE_SUCCESS;
15,423✔
5220
  int32_t lino = 0;
15,423✔
5221
  int32_t numOfTables = taosArrayGetSize(aUidTags);
15,423✔
5222
  SArray* pBlockList = NULL;
15,424✔
5223

5224
  SSDataBlock* pResBlock = createTagValBlockForFilter(pInfo->filterCtx.cInfoList, numOfTables, aUidTags, pVnode, pAPI);
15,424✔
5225
  QUERY_CHECK_NULL(pResBlock, code, lino, _end, terrno);
15,411!
5226

5227
  pBlockList = taosArrayInit(1, POINTER_BYTES);
15,411✔
5228
  QUERY_CHECK_NULL(pBlockList, code, lino, _end, terrno);
15,426!
5229

5230
  void* tmp = taosArrayPush(pBlockList, &pResBlock);
15,418✔
5231
  QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
15,418!
5232

5233
  SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)};
15,418✔
5234

5235
  SScalarParam output = {0};
15,418✔
5236
  code = tagScanCreateResultData(&type, numOfTables, &output);
15,418✔
5237
  QUERY_CHECK_CODE(code, lino, _end);
15,425!
5238

5239
  code = scalarCalculate(pTagCond, pBlockList, &output);
15,425✔
5240
  QUERY_CHECK_CODE(code, lino, _end);
15,396!
5241

5242
  bool* result = (bool*)output.columnData->pData;
15,396✔
5243
  for (int32_t i = 0; i < numOfTables; ++i) {
38,541✔
5244
    if (result[i]) {
23,163✔
5245
      void* tmp = taosArrayPush(aFilterIdxs, &i);
22,139✔
5246
      QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
22,139!
5247
    }
5248
  }
5249

5250
  colDataDestroy(output.columnData);
15,378✔
5251
  taosMemoryFreeClear(output.columnData);
15,420!
5252

5253
_end:
×
5254
  blockDataDestroy(pResBlock);
15,415✔
5255
  taosArrayDestroy(pBlockList);
15,420✔
5256
  if (code != TSDB_CODE_SUCCESS) {
15,416!
5257
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5258
  }
5259
  return code;
15,404✔
5260
}
5261

5262
static int32_t tagScanFillOneCellWithTag(SOperatorInfo* pOperator, const STUidTagInfo* pUidTagInfo,
283,216✔
5263
                                         SExprInfo* pExprInfo, SColumnInfoData* pColInfo, int rowIndex,
5264
                                         const SStorageAPI* pAPI, void* pVnode) {
5265
  int32_t code = TSDB_CODE_SUCCESS;
283,216✔
5266
  int32_t lino = 0;
283,216✔
5267
  if (QUERY_NODE_FUNCTION == pExprInfo->pExpr->nodeType) {
283,216✔
5268
    if (FUNCTION_TYPE_TBNAME == pExprInfo->pExpr->_function.functionType) {  // tbname
135,227!
5269
      char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
×
5270
      STR_TO_VARSTR(str, "ctbidx");
×
5271

5272
      code = colDataSetVal(pColInfo, rowIndex, str, false);
×
5273
      QUERY_CHECK_CODE(code, lino, _end);
×
5274
    } else if (FUNCTION_TYPE_TBUID == pExprInfo->pExpr->_function.functionType) {
135,227✔
5275
      code = colDataSetVal(pColInfo, rowIndex, (char*)&pUidTagInfo->uid, false);
67,653✔
5276
      QUERY_CHECK_CODE(code, lino, _end);
67,680!
5277
    } else if (FUNCTION_TYPE_VGID == pExprInfo->pExpr->_function.functionType) {
67,574!
5278
      code = colDataSetVal(pColInfo, rowIndex, (char*)&pOperator->pTaskInfo->id.vgId, false);
67,663✔
5279
      QUERY_CHECK_CODE(code, lino, _end);
67,668!
5280
    }
5281
  } else {
5282
    STagVal tagVal = {0};
147,989✔
5283
    tagVal.cid = pExprInfo->base.pParam[0].pCol->colId;
147,989✔
5284
    if (pUidTagInfo->pTagVal == NULL) {
147,989!
5285
      colDataSetNULL(pColInfo, rowIndex);
×
5286
    } else {
5287
      const char* p = pAPI->metaFn.extractTagVal(pUidTagInfo->pTagVal, pColInfo->info.type, &tagVal);
147,989✔
5288

5289
      if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) {
148,385!
5290
        colDataSetNULL(pColInfo, rowIndex);
35!
5291
      } else if (pColInfo->info.type == TSDB_DATA_TYPE_JSON) {
148,350!
5292
        code = colDataSetVal(pColInfo, rowIndex, p, false);
×
5293
        QUERY_CHECK_CODE(code, lino, _end);
×
5294
      } else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) {
184,183!
5295
        char* tmp = taosMemoryMalloc(tagVal.nData + VARSTR_HEADER_SIZE + 1);
35,845!
5296
        QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
35,820!
5297

5298
        varDataSetLen(tmp, tagVal.nData);
35,820✔
5299
        memcpy(tmp + VARSTR_HEADER_SIZE, tagVal.pData, tagVal.nData);
35,820✔
5300
        code = colDataSetVal(pColInfo, rowIndex, tmp, false);
35,820✔
5301
        taosMemoryFree(tmp);
35,836!
5302
        QUERY_CHECK_CODE(code, lino, _end);
35,833!
5303
      } else {
5304
        code = colDataSetVal(pColInfo, rowIndex, (const char*)&tagVal.i64, false);
112,505✔
5305
        QUERY_CHECK_CODE(code, lino, _end);
112,492!
5306
      }
5307
    }
5308
  }
5309

5310
_end:
283,619✔
5311
  if (code != TSDB_CODE_SUCCESS) {
283,619!
5312
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5313
  }
5314
  return code;
283,269✔
5315
}
5316

5317
static int32_t tagScanFillResultBlock(SOperatorInfo* pOperator, SSDataBlock* pRes, SArray* aUidTags,
43,980✔
5318
                                      SArray* aFilterIdxs, bool ignoreFilterIdx, SStorageAPI* pAPI) {
5319
  int32_t       code = TSDB_CODE_SUCCESS;
43,980✔
5320
  int32_t       lino = 0;
43,980✔
5321
  STagScanInfo* pInfo = pOperator->info;
43,980✔
5322
  SExprInfo*    pExprInfo = &pOperator->exprSupp.pExprInfo[0];
43,980✔
5323
  if (!ignoreFilterIdx) {
43,980✔
5324
    size_t szTables = taosArrayGetSize(aFilterIdxs);
15,401✔
5325
    for (int i = 0; i < szTables; ++i) {
37,543✔
5326
      int32_t       idx = *(int32_t*)taosArrayGet(aFilterIdxs, i);
22,161✔
5327
      STUidTagInfo* pUidTagInfo = taosArrayGet(aUidTags, idx);
22,167✔
5328
      QUERY_CHECK_NULL(pUidTagInfo, code, lino, _end, terrno);
22,164!
5329
      for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
126,727✔
5330
        SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId);
104,583✔
5331
        QUERY_CHECK_NULL(pDst, code, lino, _end, terrno);
104,541!
5332
        code = tagScanFillOneCellWithTag(pOperator, pUidTagInfo, &pExprInfo[j], pDst, i, pAPI, pInfo->readHandle.vnode);
104,541✔
5333
        QUERY_CHECK_CODE(code, lino, _end);
104,563!
5334
      }
5335
    }
5336
  } else {
5337
    size_t szTables = taosArrayGetSize(aUidTags);
28,579✔
5338
    for (int i = 0; i < szTables; ++i) {
74,121✔
5339
      STUidTagInfo* pUidTagInfo = taosArrayGet(aUidTags, i);
45,533✔
5340
      QUERY_CHECK_NULL(pUidTagInfo, code, lino, _end, terrno);
45,523✔
5341
      for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) {
224,235✔
5342
        SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId);
178,722✔
5343
        QUERY_CHECK_NULL(pDst, code, lino, _end, terrno);
178,687!
5344
        code = tagScanFillOneCellWithTag(pOperator, pUidTagInfo, &pExprInfo[j], pDst, i, pAPI, pInfo->readHandle.vnode);
178,687✔
5345
        QUERY_CHECK_CODE(code, lino, _end);
178,713!
5346
      }
5347
    }
5348
  }
5349

5350
_end:
28,588✔
5351
  if (code != TSDB_CODE_SUCCESS) {
43,970!
5352
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5353
  }
5354
  return code;
44,005✔
5355
}
5356

5357
static int32_t doTagScanFromCtbIdxNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
87,479✔
5358
  int32_t        code = TSDB_CODE_SUCCESS;
87,479✔
5359
  int32_t        lino = 0;
87,479✔
5360
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
87,479✔
5361
  SStorageAPI*   pAPI = &pTaskInfo->storageAPI;
87,479✔
5362

5363
  STagScanInfo* pInfo = pOperator->info;
87,479✔
5364
  SSDataBlock*  pRes = pInfo->pRes;
87,479✔
5365

5366
  QRY_PARAM_CHECK(ppRes);
87,479!
5367

5368
  if (pOperator->status == OP_EXEC_DONE) {
87,479✔
5369
    return TSDB_CODE_SUCCESS;
43,352✔
5370
  }
5371
  blockDataCleanup(pRes);
44,127✔
5372

5373
  if (pInfo->pCtbCursor == NULL) {
44,201!
5374
    pInfo->pCtbCursor = pAPI->metaFn.openCtbCursor(pInfo->readHandle.vnode, pInfo->suid, 1);
44,208✔
5375
    QUERY_CHECK_NULL(pInfo->pCtbCursor, code, lino, _end, terrno);
44,205!
5376
  } else {
5377
    code = pAPI->metaFn.resumeCtbCursor(pInfo->pCtbCursor, 0);
×
5378
    QUERY_CHECK_CODE(code, lino, _end);
×
5379
  }
5380

5381
  SArray* aUidTags = pInfo->aUidTags;
44,208✔
5382
  SArray* aFilterIdxs = pInfo->aFilterIdxs;
44,208✔
5383
  int32_t count = 0;
44,208✔
5384
  bool    ctbCursorFinished = false;
44,208✔
5385
  while (1) {
632✔
5386
    taosArrayClearEx(aUidTags, tagScanFreeUidTag);
44,840✔
5387
    taosArrayClear(aFilterIdxs);
44,841✔
5388

5389
    int32_t numTables = 0;
44,786✔
5390
    while (numTables < pOperator->resultInfo.capacity) {
113,435!
5391
      SMCtbCursor* pCur = pInfo->pCtbCursor;
113,439✔
5392
      tb_uid_t     uid = pAPI->metaFn.ctbCursorNext(pInfo->pCtbCursor);
113,439✔
5393
      if (uid == 0) {
113,465✔
5394
        ctbCursorFinished = true;
44,851✔
5395
        break;
44,851✔
5396
      }
5397
      STUidTagInfo info = {.uid = uid, .pTagVal = pCur->pVal};
68,614✔
5398
      info.pTagVal = taosMemoryMalloc(pCur->vLen);
68,614!
5399
      QUERY_CHECK_NULL(info.pTagVal, code, lino, _end, terrno);
68,694!
5400

5401
      memcpy(info.pTagVal, pCur->pVal, pCur->vLen);
68,694✔
5402
      void* tmp = taosArrayPush(aUidTags, &info);
68,649✔
5403
      QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
68,649!
5404
      ++numTables;
68,649✔
5405
    }
5406

5407
    if (numTables == 0) {
44,847✔
5408
      break;
815✔
5409
    }
5410
    bool ignoreFilterIdx = true;
44,032✔
5411
    if (pInfo->pTagCond != NULL) {
44,032✔
5412
      ignoreFilterIdx = false;
15,424✔
5413
      code = tagScanFilterByTagCond(aUidTags, pInfo->pTagCond, aFilterIdxs, pInfo->readHandle.vnode, pAPI, pInfo);
15,424✔
5414
      QUERY_CHECK_CODE(code, lino, _end);
15,399!
5415
    } else {
5416
      ignoreFilterIdx = true;
28,608✔
5417
    }
5418

5419
    code = tagScanFillResultBlock(pOperator, pRes, aUidTags, aFilterIdxs, ignoreFilterIdx, pAPI);
44,007✔
5420
    QUERY_CHECK_CODE(code, lino, _end);
44,006!
5421

5422
    count = ignoreFilterIdx ? taosArrayGetSize(aUidTags) : taosArrayGetSize(aFilterIdxs);
44,006✔
5423

5424
    if (count != 0) {
43,994✔
5425
      break;
43,362✔
5426
    }
5427
  }
5428

5429
  if (count > 0) {
44,177✔
5430
    pAPI->metaFn.pauseCtbCursor(pInfo->pCtbCursor);
43,368✔
5431
  }
5432
  if (count == 0 || ctbCursorFinished) {
44,211!
5433
    pAPI->metaFn.closeCtbCursor(pInfo->pCtbCursor);
44,211✔
5434
    pInfo->pCtbCursor = NULL;
44,222✔
5435
    setOperatorCompleted(pOperator);
44,222✔
5436
  }
5437

5438
  pRes->info.rows = count;
44,205✔
5439
  bool bLimitReached = applyLimitOffset(&pInfo->limitInfo, pRes, pTaskInfo);
44,205✔
5440
  if (bLimitReached) {
44,176!
5441
    setOperatorCompleted(pOperator);
×
5442
  }
5443

5444
  pOperator->resultInfo.totalRows += pRes->info.rows;
44,198✔
5445
  (*ppRes) = (pRes->info.rows == 0) ? NULL : pInfo->pRes;
44,198✔
5446

5447
_end:
44,198✔
5448
  if (code != TSDB_CODE_SUCCESS) {
44,198!
5449
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5450
    pTaskInfo->code = code;
×
5451
    T_LONG_JMP(pTaskInfo->env, code);
×
5452
  }
5453

5454
  return code;
44,198✔
5455
}
5456

5457
static int32_t doTagScanFromMetaEntryNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
1,216✔
5458
  int32_t code = TSDB_CODE_SUCCESS;
1,216✔
5459
  int32_t lino = 0;
1,216✔
5460
  if (pOperator->status == OP_EXEC_DONE) {
1,216✔
5461
    (*ppRes) = NULL;
564✔
5462
    return code;
564✔
5463
  }
5464

5465
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
652✔
5466
  SStorageAPI*   pAPI = &pTaskInfo->storageAPI;
652✔
5467

5468
  STagScanInfo* pInfo = pOperator->info;
652✔
5469
  SExprInfo*    pExprInfo = &pOperator->exprSupp.pExprInfo[0];
652✔
5470
  SSDataBlock*  pRes = pInfo->pRes;
652✔
5471
  blockDataCleanup(pRes);
652✔
5472

5473
  int32_t size = 0;
655✔
5474
  code = tableListGetSize(pInfo->pTableListInfo, &size);
655✔
5475
  QUERY_CHECK_CODE(code, lino, _end);
655!
5476

5477
  if (size == 0) {
655✔
5478
    setTaskStatus(pTaskInfo, TASK_COMPLETED);
93✔
5479
    (*ppRes) = NULL;
93✔
5480
    return code;
93✔
5481
  }
5482

5483
  SMetaReader mr = {0};
562✔
5484
  pAPI->metaReaderFn.initReader(&mr, pInfo->readHandle.vnode, META_READER_LOCK, &pAPI->metaFn);
562✔
5485
  pRes->info.rows = 0;
566✔
5486

5487
  while (pInfo->curPos < size && pRes->info.rows < pOperator->resultInfo.capacity) {
14,701!
5488
    code = doTagScanOneTable(pOperator, pRes, &mr, &pTaskInfo->storageAPI);
14,132✔
5489
    if (code != TSDB_CODE_OUT_OF_MEMORY && code != TSDB_CODE_QRY_REACH_QMEM_THRESHOLD && code != TSDB_CODE_QRY_QUERY_MEM_EXHAUSTED) {
14,134!
5490
      // ignore other error
5491
      code = TSDB_CODE_SUCCESS;
14,137✔
5492
    }
5493
    QUERY_CHECK_CODE(code, lino, _end);
14,134!
5494

5495
    if (++pInfo->curPos >= size) {
14,134✔
5496
      setOperatorCompleted(pOperator);
565✔
5497
    }
5498
  }
5499

5500
  pAPI->metaReaderFn.clearReader(&mr);
569✔
5501
  bool bLimitReached = applyLimitOffset(&pInfo->limitInfo, pRes, pTaskInfo);
566✔
5502
  if (bLimitReached) {
565!
5503
    setOperatorCompleted(pOperator);
×
5504
  }
5505

5506
  // qDebug("QInfo:0x%" PRIx64 ", create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count);
5507
  if (pOperator->status == OP_EXEC_DONE) {
565!
5508
    setTaskStatus(pTaskInfo, TASK_COMPLETED);
565✔
5509
  }
5510

5511
  pOperator->resultInfo.totalRows += pRes->info.rows;
566✔
5512

5513
  (*ppRes) = (pRes->info.rows == 0) ? NULL : pInfo->pRes;
566!
5514

5515
_end:
566✔
5516
  if (code != TSDB_CODE_SUCCESS) {
566!
5517
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5518
    pTaskInfo->code = code;
×
5519
    T_LONG_JMP(pTaskInfo->env, code);
×
5520
  }
5521
  return code;
566✔
5522
}
5523

5524
static SSDataBlock* doTagScanFromMetaEntry(SOperatorInfo* pOperator) {
×
5525
  SSDataBlock* pRes = NULL;
×
5526
  int32_t      code = doTagScanFromMetaEntryNext(pOperator, &pRes);
×
5527
  return pRes;
×
5528
}
5529

5530
static void destroyTagScanOperatorInfo(void* param) {
44,757✔
5531
  STagScanInfo* pInfo = (STagScanInfo*)param;
44,757✔
5532
  if (pInfo->pCtbCursor != NULL && pInfo->pStorageAPI != NULL) {
44,757!
5533
    pInfo->pStorageAPI->metaFn.closeCtbCursor(pInfo->pCtbCursor);
×
5534
  }
5535
  taosHashCleanup(pInfo->filterCtx.colHash);
44,757✔
5536
  taosArrayDestroy(pInfo->filterCtx.cInfoList);
44,856✔
5537
  taosArrayDestroy(pInfo->aFilterIdxs);
44,860✔
5538
  taosArrayDestroyEx(pInfo->aUidTags, tagScanFreeUidTag);
44,873✔
5539

5540
  blockDataDestroy(pInfo->pRes);
44,872✔
5541
  taosArrayDestroy(pInfo->matchInfo.pList);
44,870✔
5542
  tableListDestroy(pInfo->pTableListInfo);
44,872✔
5543

5544
  pInfo->pRes = NULL;
44,867✔
5545
  pInfo->pTableListInfo = NULL;
44,867✔
5546
  taosMemoryFreeClear(param);
44,867!
5547
}
44,882✔
5548

5549
int32_t createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pTagScanNode,
44,804✔
5550
                                  STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond,
5551
                                  SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) {
5552
  QRY_PARAM_CHECK(pOptrInfo);
44,804!
5553

5554
  int32_t         code = TSDB_CODE_SUCCESS;
44,804✔
5555
  int32_t         lino = 0;
44,804✔
5556
  SScanPhysiNode* pPhyNode = (SScanPhysiNode*)pTagScanNode;
44,804✔
5557
  STagScanInfo*   pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo));
44,804!
5558
  SOperatorInfo*  pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
44,811!
5559
  if (pInfo == NULL || pOperator == NULL) {
44,837!
5560
    code = terrno;
×
5561
    goto _error;
×
5562
  }
5563

5564
  SDataBlockDescNode* pDescNode = pPhyNode->node.pOutputDataBlockDesc;
44,838✔
5565

5566
  int32_t    numOfExprs = 0;
44,838✔
5567
  SExprInfo* pExprInfo = NULL;
44,838✔
5568

5569
  code = createExprInfo(pPhyNode->pScanPseudoCols, NULL, &pExprInfo, &numOfExprs);
44,838✔
5570
  QUERY_CHECK_CODE(code, lino, _error);
44,822!
5571

5572
  code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs, &pTaskInfo->storageAPI.functionStore);
44,822✔
5573
  QUERY_CHECK_CODE(code, lino, _error);
44,781!
5574

5575
  int32_t num = 0;
44,781✔
5576
  code = extractColMatchInfo(pPhyNode->pScanPseudoCols, pDescNode, &num, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo);
44,781✔
5577
  QUERY_CHECK_CODE(code, lino, _error);
44,789!
5578

5579
  pInfo->pTagCond = pTagCond;
44,789✔
5580
  pInfo->pTagIndexCond = pTagIndexCond;
44,789✔
5581
  pInfo->suid = pPhyNode->suid;
44,789✔
5582
  pInfo->pStorageAPI = &pTaskInfo->storageAPI;
44,789✔
5583

5584
  pInfo->pTableListInfo = pTableListInfo;
44,789✔
5585
  pInfo->pRes = createDataBlockFromDescNode(pDescNode);
44,789✔
5586
  QUERY_CHECK_NULL(pInfo->pRes, code, lino, _error, terrno);
44,834!
5587

5588
  pInfo->readHandle = *pReadHandle;
44,834✔
5589
  pInfo->curPos = 0;
44,834✔
5590

5591
  initLimitInfo(pPhyNode->node.pLimit, pPhyNode->node.pSlimit, &pInfo->limitInfo);
44,834✔
5592
  setOperatorInfo(pOperator, "TagScanOperator", QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, false, OP_NOT_OPENED, pInfo,
44,843✔
5593
                  pTaskInfo);
5594
  initResultSizeInfo(&pOperator->resultInfo, 4096);
44,719✔
5595
  code = blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
44,747✔
5596
  QUERY_CHECK_CODE(code, lino, _error);
44,875!
5597

5598
  if (pTagScanNode->onlyMetaCtbIdx) {
44,875✔
5599
    pInfo->aUidTags = taosArrayInit(pOperator->resultInfo.capacity, sizeof(STUidTagInfo));
44,219✔
5600
    QUERY_CHECK_NULL(pInfo->aUidTags, code, lino, _error, terrno);
44,200!
5601

5602
    pInfo->aFilterIdxs = taosArrayInit(pOperator->resultInfo.capacity, sizeof(int32_t));
44,200✔
5603
    QUERY_CHECK_NULL(pInfo->aFilterIdxs, code, lino, _error, terrno);
44,196!
5604

5605
    pInfo->filterCtx.colHash =
44,193✔
5606
        taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK);
44,196✔
5607
    QUERY_CHECK_NULL(pInfo->filterCtx.colHash, code, lino, _error, terrno);
44,193!
5608

5609
    pInfo->filterCtx.cInfoList = taosArrayInit(4, sizeof(SColumnInfo));
44,193✔
5610
    QUERY_CHECK_NULL(pInfo->filterCtx.cInfoList, code, lino, _error, terrno);
44,221!
5611

5612
    if (pInfo->pTagCond != NULL) {
44,221✔
5613
      nodesRewriteExprPostOrder(&pTagCond, tagScanRewriteTagColumn, (void*)&pInfo->filterCtx);
15,435✔
5614
    }
5615
  }
5616
  __optr_fn_t tagScanNextFn = (pTagScanNode->onlyMetaCtbIdx) ? doTagScanFromCtbIdxNext : doTagScanFromMetaEntryNext;
44,867✔
5617
  pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, tagScanNextFn, NULL, destroyTagScanOperatorInfo,
44,867✔
5618
                                         optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
5619
  *pOptrInfo = pOperator;
44,846✔
5620
  return code;
44,846✔
5621

5622
_error:
×
5623
  if (pInfo) {
×
5624
    pInfo->pTableListInfo = NULL;
×
5625
  }
5626

5627
  if (pInfo != NULL) destroyTagScanOperatorInfo(pInfo);
×
5628
  if (pOperator != NULL) {
×
5629
    pOperator->info = NULL;
×
5630
    destroyOperator(pOperator);
×
5631
  }
5632
  return code;
×
5633
}
5634

5635
// table merge scan operator
5636

5637
static int32_t subTblRowCompareTsFn(const void* pLeft, const void* pRight, void* param) {
913,631✔
5638
  int32_t                 left = *(int32_t*)pLeft;
913,631✔
5639
  int32_t                 right = *(int32_t*)pRight;
913,631✔
5640
  STmsSubTablesMergeInfo* pInfo = (STmsSubTablesMergeInfo*)param;
913,631✔
5641

5642
  int32_t leftIdx = pInfo->aInputs[left].rowIdx;
913,631✔
5643
  int32_t rightIdx = pInfo->aInputs[right].rowIdx;
913,631✔
5644

5645
  if (leftIdx == -1) {
913,631✔
5646
    return 1;
768,270✔
5647
  } else if (rightIdx == -1) {
145,361✔
5648
    return -1;
1,440✔
5649
  }
5650

5651
  int64_t leftTs = pInfo->aInputs[left].aTs[leftIdx];
143,921✔
5652
  int64_t rightTs = pInfo->aInputs[right].aTs[rightIdx];
143,921✔
5653
  int32_t ret = leftTs > rightTs ? 1 : ((leftTs < rightTs) ? -1 : 0);
143,921!
5654
  if (pInfo->pTsOrderInfo->order == TSDB_ORDER_DESC) {
143,921!
5655
    ret = -1 * ret;
×
5656
  }
5657
  return ret;
143,921✔
5658
}
5659

5660
static int32_t subTblRowCompareTsPkFn(const void* pLeft, const void* pRight, void* param) {
×
5661
  int32_t                 left = *(int32_t*)pLeft;
×
5662
  int32_t                 right = *(int32_t*)pRight;
×
5663
  STmsSubTablesMergeInfo* pInfo = (STmsSubTablesMergeInfo*)param;
×
5664

5665
  int32_t leftIdx = pInfo->aInputs[left].rowIdx;
×
5666
  int32_t rightIdx = pInfo->aInputs[right].rowIdx;
×
5667

5668
  if (leftIdx == -1) {
×
5669
    return 1;
×
5670
  } else if (rightIdx == -1) {
×
5671
    return -1;
×
5672
  }
5673

5674
  int64_t leftTs = pInfo->aInputs[left].aTs[leftIdx];
×
5675
  int64_t rightTs = pInfo->aInputs[right].aTs[rightIdx];
×
5676
  int32_t ret = leftTs > rightTs ? 1 : ((leftTs < rightTs) ? -1 : 0);
×
5677
  if (pInfo->pTsOrderInfo->order == TSDB_ORDER_DESC) {
×
5678
    ret = -1 * ret;
×
5679
  }
5680
  if (ret == 0 && pInfo->pPkOrderInfo) {
×
5681
    ret = tsortComparBlockCell(pInfo->aInputs[left].pInputBlock, pInfo->aInputs[right].pInputBlock, leftIdx, rightIdx,
×
5682
                               pInfo->pPkOrderInfo);
×
5683
  }
5684
  return ret;
×
5685
}
5686

5687
int32_t dumpQueryTableCond(const SQueryTableDataCond* src, SQueryTableDataCond* dst) {
11,176✔
5688
  int32_t code = TSDB_CODE_SUCCESS;
11,176✔
5689
  int32_t lino = 0;
11,176✔
5690
  memcpy((void*)dst, (void*)src, sizeof(SQueryTableDataCond));
11,176✔
5691
  dst->colList = taosMemoryCalloc(src->numOfCols, sizeof(SColumnInfo));
11,176!
5692
  QUERY_CHECK_NULL(dst->colList, code, lino, _end, terrno);
11,209!
5693
  for (int i = 0; i < src->numOfCols; i++) {
36,990✔
5694
    dst->colList[i] = src->colList[i];
25,781✔
5695
  }
5696

5697
_end:
11,209✔
5698
  if (code != TSDB_CODE_SUCCESS) {
11,209!
5699
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5700
  }
5701
  return code;
11,206✔
5702
}
5703

5704
static int32_t fetchNextSubTableBlockFromReader(SOperatorInfo* pOperator, STmsSubTableInput* pInput,
16,806✔
5705
                                                bool* pSubTableHasBlock) {
5706
  int32_t code = 0;
16,806✔
5707

5708
  STableMergeScanInfo*    pInfo = pOperator->info;
16,806✔
5709
  SReadHandle*            pHandle = &pInfo->base.readHandle;
16,806✔
5710
  STmsSubTablesMergeInfo* pSubTblsInfo = pInfo->pSubTablesMergeInfo;
16,806✔
5711
  SExecTaskInfo*          pTaskInfo = pOperator->pTaskInfo;
16,806✔
5712
  const SStorageAPI*      pAPI = &pTaskInfo->storageAPI;
16,806✔
5713

5714
  blockDataCleanup(pInput->pReaderBlock);
16,806✔
5715
  if (!pInput->bInMemReader) {
16,803✔
5716
    code = pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, &pInput->tblCond, pInput->pKeyInfo, 1, pInput->pReaderBlock,
8,483✔
5717
                                         (void**)&pInput->pReader, GET_TASKID(pTaskInfo), NULL);
8,483✔
5718
    if (code != 0) {
8,486!
5719
      return code;
×
5720
    }
5721
  }
5722

5723
  pInfo->base.dataReader = pInput->pReader;
16,806✔
5724

5725
  while (true) {
18✔
5726
    bool hasNext = false;
16,824✔
5727
    code = pAPI->tsdReader.tsdNextDataBlock(pInfo->base.dataReader, &hasNext);
16,824✔
5728
    if (code != 0) {
16,829!
5729
      pAPI->tsdReader.tsdReaderReleaseDataBlock(pInfo->base.dataReader);
×
5730
      pInfo->base.dataReader = NULL;
×
5731
      return code;
×
5732
    }
5733

5734
    if (!hasNext || isTaskKilled(pTaskInfo)) {
16,829!
5735
      if (isTaskKilled(pTaskInfo)) {
11,217!
5736
        pAPI->tsdReader.tsdReaderReleaseDataBlock(pInfo->base.dataReader);
×
5737
        pInfo->base.dataReader = NULL;
×
5738
        return code;
×
5739
      }
5740

5741
      *pSubTableHasBlock = false;
11,221✔
5742
      break;
11,221✔
5743
    }
5744

5745
    if (pInput->tblCond.order == TSDB_ORDER_ASC) {
5,612✔
5746
      pInput->tblCond.twindows.skey = pInput->pReaderBlock->info.window.ekey + 1;
5,530✔
5747
    } else {
5748
      pInput->tblCond.twindows.ekey = pInput->pReaderBlock->info.window.skey - 1;
82✔
5749
    }
5750

5751
    uint32_t status = 0;
5,612✔
5752
    code = loadDataBlock(pOperator, &pInfo->base, pInput->pReaderBlock, &status);
5,612✔
5753
    if (code != 0) {
5,612!
5754
      pInfo->base.dataReader = NULL;
×
5755
      return code;
×
5756
    }
5757

5758
    if (status == FUNC_DATA_REQUIRED_ALL_FILTEROUT) {
5,612!
5759
      *pSubTableHasBlock = false;
×
5760
      break;
×
5761
    }
5762
    if (status == FUNC_DATA_REQUIRED_FILTEROUT || pInput->pReaderBlock->info.rows == 0) {
5,612!
5763
      continue;
18✔
5764
    }
5765

5766
    *pSubTableHasBlock = true;
5,594✔
5767
    break;
5,594✔
5768
  }
5769

5770
  if (*pSubTableHasBlock) {
16,815✔
5771
    pInput->pReaderBlock->info.id.groupId =
11,189✔
5772
        tableListGetTableGroupId(pInfo->base.pTableListInfo, pInput->pReaderBlock->info.id.uid);
5,594✔
5773
    pOperator->resultInfo.totalRows += pInput->pReaderBlock->info.rows;
5,595✔
5774
  }
5775
  if (!pInput->bInMemReader || !*pSubTableHasBlock) {
16,816✔
5776
    pAPI->tsdReader.tsdReaderClose(pInput->pReader);
12,662✔
5777
    pInput->pReader = NULL;
12,661✔
5778
  }
5779

5780
  pInfo->base.dataReader = NULL;
16,815✔
5781
  return TSDB_CODE_SUCCESS;
16,815✔
5782
}
5783

5784
static int32_t setGroupStartEndIndex(STableMergeScanInfo* pInfo) {
6,995✔
5785
  pInfo->bGroupProcessed = false;
6,995✔
5786

5787
  int32_t numOfTables = 0;
6,995✔
5788
  int32_t code = tableListGetSize(pInfo->base.pTableListInfo, &numOfTables);
6,995✔
5789
  if (code != TSDB_CODE_SUCCESS) {
7,030!
5790
    qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
×
5791
    return code;
×
5792
  }
5793

5794
  int32_t i = pInfo->tableStartIndex + 1;
7,034✔
5795
  for (; i < numOfTables; ++i) {
11,203✔
5796
    STableKeyInfo* tableKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, i);
4,397✔
5797
    if (!tableKeyInfo) {
4,399!
UNCOV
5798
      qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));
×
UNCOV
5799
      return terrno;
×
5800
    }
5801
    if (tableKeyInfo->groupId != pInfo->groupId) {
4,399✔
5802
      break;
230✔
5803
    }
5804
  }
5805
  pInfo->tableEndIndex = i - 1;
7,036✔
5806
  return TSDB_CODE_SUCCESS;
7,036✔
5807
}
5808

5809
static int32_t openSubTablesMergeSort(STmsSubTablesMergeInfo* pSubTblsInfo) {
7,049✔
5810
  for (int32_t i = 0; i < pSubTblsInfo->numSubTables; ++i) {
18,269✔
5811
    STmsSubTableInput* pInput = pSubTblsInfo->aInputs + i;
11,220✔
5812
    if (pInput->rowIdx == -1) {
11,220✔
5813
      continue;
5,625✔
5814
    }
5815

5816
    if (pInput->type == SUB_TABLE_MEM_BLOCK) {
5,595!
5817
      pInput->rowIdx = 0;
5,595✔
5818
      pInput->pageIdx = -1;
5,595✔
5819
    }
5820

5821
    pInput->pInputBlock = (pInput->type == SUB_TABLE_MEM_BLOCK) ? pInput->pReaderBlock : pInput->pPageBlock;
5,595!
5822
    SColumnInfoData* col = taosArrayGet(pInput->pInputBlock->pDataBlock, pSubTblsInfo->pTsOrderInfo->slotId);
5,595✔
5823
    if (!col) {
5,595!
5824
      qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));
×
5825
      return terrno;
×
5826
    }
5827
    pInput->aTs = (int64_t*)col->pData;
5,595✔
5828
  }
5829

5830
  __merge_compare_fn_t mergeCompareFn = (!pSubTblsInfo->pPkOrderInfo) ? subTblRowCompareTsFn : subTblRowCompareTsPkFn;
7,049!
5831
  return tMergeTreeCreate(&pSubTblsInfo->pTree, pSubTblsInfo->numSubTables, pSubTblsInfo, mergeCompareFn);
7,049✔
5832
}
5833

5834
static int32_t initSubTablesMergeInfo(STableMergeScanInfo* pInfo) {
6,996✔
5835
  int32_t code = setGroupStartEndIndex(pInfo);
6,996✔
5836
  if (code != TSDB_CODE_SUCCESS) {
7,032!
5837
    qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
×
5838
    return code;
×
5839
  }
5840
  STmsSubTablesMergeInfo* pSubTblsInfo = taosMemoryCalloc(1, sizeof(STmsSubTablesMergeInfo));
7,032!
5841
  if (pSubTblsInfo == NULL) {
7,040!
5842
    return terrno;
×
5843
  }
5844
  pSubTblsInfo->pTsOrderInfo = taosArrayGet(pInfo->pSortInfo, 0);
7,040✔
5845
  if (!pSubTblsInfo->pTsOrderInfo) {
7,039!
5846
    qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));
×
5847
    return terrno;
×
5848
  }
5849
  if (taosArrayGetSize(pInfo->pSortInfo) == 2) {
7,039✔
5850
    pSubTblsInfo->pPkOrderInfo = taosArrayGet(pInfo->pSortInfo, 1);
6✔
5851
    if (!pSubTblsInfo->pPkOrderInfo) {
×
5852
      qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));
×
5853
      return terrno;
×
5854
    }
5855
  } else {
5856
    pSubTblsInfo->pPkOrderInfo = NULL;
7,028✔
5857
  }
5858
  pSubTblsInfo->numSubTables = pInfo->tableEndIndex - pInfo->tableStartIndex + 1;
7,028✔
5859
  pSubTblsInfo->aInputs = taosMemoryCalloc(pSubTblsInfo->numSubTables, sizeof(STmsSubTableInput));
7,028!
5860
  if (pSubTblsInfo->aInputs == NULL) {
7,027!
5861
    taosMemoryFree(pSubTblsInfo);
×
5862
    return terrno;
×
5863
  }
5864
  int32_t bufPageSize = pInfo->bufPageSize;
7,027✔
5865
  int32_t inMemSize = (pSubTblsInfo->numSubTables - pSubTblsInfo->numTableBlocksInMem) * bufPageSize;
7,027✔
5866
  code = createDiskbasedBuf(&pSubTblsInfo->pBlocksBuf, pInfo->bufPageSize, inMemSize, "blocksExternalBuf", tsTempDir);
7,027✔
5867
  if (code != TSDB_CODE_SUCCESS) {
7,044✔
5868
    taosMemoryFree(pSubTblsInfo->aInputs);
12!
5869
    taosMemoryFree(pSubTblsInfo);
×
5870
    return code;
×
5871
  }
5872
  pSubTblsInfo->numTableBlocksInMem = pSubTblsInfo->numSubTables;
7,032✔
5873
  pSubTblsInfo->numInMemReaders = pSubTblsInfo->numSubTables;
7,032✔
5874

5875
  pInfo->pSubTablesMergeInfo = pSubTblsInfo;
7,032✔
5876
  return TSDB_CODE_SUCCESS;
7,032✔
5877
}
5878

5879
static int32_t initSubTableInputs(SOperatorInfo* pOperator, STableMergeScanInfo* pInfo) {
7,019✔
5880
  int32_t        code = TSDB_CODE_SUCCESS;
7,019✔
5881
  int32_t        lino = 0;
7,019✔
5882
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
7,019✔
5883
  SReadHandle*   pHandle = &pInfo->base.readHandle;
7,019✔
5884
  SStorageAPI*   pAPI = &pTaskInfo->storageAPI;
7,019✔
5885

5886
  STmsSubTablesMergeInfo* pSubTblsInfo = pInfo->pSubTablesMergeInfo;
7,019✔
5887

5888
  for (int32_t i = 0; i < pSubTblsInfo->numSubTables; ++i) {
18,238✔
5889
    STmsSubTableInput* pInput = pSubTblsInfo->aInputs + i;
11,182✔
5890
    pInput->type = SUB_TABLE_MEM_BLOCK;
11,182✔
5891

5892
    code = dumpQueryTableCond(&pInfo->base.cond, &pInput->tblCond);
11,182✔
5893
    QUERY_CHECK_CODE(code, lino, _end);
11,208!
5894

5895
    code = createOneDataBlock(pInfo->pResBlock, false, &pInput->pReaderBlock);
11,208✔
5896
    QUERY_CHECK_CODE(code, lino, _end);
11,216!
5897

5898
    code = createOneDataBlock(pInfo->pResBlock, false, &pInput->pPageBlock);
11,216✔
5899
    QUERY_CHECK_CODE(code, lino, _end);
11,217!
5900

5901
    STableKeyInfo* keyInfo = tableListGetInfo(pInfo->base.pTableListInfo, i + pInfo->tableStartIndex);
11,217✔
5902
    pInput->pKeyInfo = keyInfo;
11,215✔
5903

5904
    if (isTaskKilled(pTaskInfo)) {
11,215!
5905
      T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
×
5906
    }
5907

5908
    if (i + 1 < pSubTblsInfo->numInMemReaders) {
11,214✔
5909
      code = pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, &pInput->tblCond, keyInfo, 1, pInput->pReaderBlock,
4,171✔
5910
                                           (void**)&pInput->pReader, GET_TASKID(pTaskInfo), NULL);
4,171✔
5911
      QUERY_CHECK_CODE(code, lino, _end);
4,170!
5912
      pInput->bInMemReader = true;
4,170✔
5913
    } else {
5914
      pInput->pReader = NULL;
7,043✔
5915
      pInput->bInMemReader = false;
7,043✔
5916
    }
5917
    bool hasNext = true;
11,213✔
5918
    code = fetchNextSubTableBlockFromReader(pOperator, pInput, &hasNext);
11,213✔
5919
    QUERY_CHECK_CODE(code, lino, _end);
11,219!
5920
    if (!hasNext) {
11,219✔
5921
      pInput->rowIdx = -1;
5,625✔
5922
      ++pSubTblsInfo->numSubTablesCompleted;
5,625✔
5923
      continue;
5,625✔
5924
    } else {
5925
      pInput->rowIdx = 0;
5,594✔
5926
      pInput->pageIdx = -1;
5,594✔
5927
    }
5928
  }
5929

5930
_end:
7,056✔
5931
  if (code != TSDB_CODE_SUCCESS) {
7,056!
5932
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5933
  }
5934
  return code;
7,050✔
5935
}
5936

5937
static void adjustSubTableFromMemBlock(SOperatorInfo* pOperatorInfo, STmsSubTablesMergeInfo* pSubTblsInfo) {
5,594✔
5938
  int32_t              code = TSDB_CODE_SUCCESS;
5,594✔
5939
  int32_t              lino = 0;
5,594✔
5940
  SExecTaskInfo*       pTaskInfo = pOperatorInfo->pTaskInfo;
5,594✔
5941
  STableMergeScanInfo* pInfo = pOperatorInfo->info;
5,594✔
5942
  STmsSubTableInput*   pInput = pSubTblsInfo->aInputs + tMergeTreeGetChosenIndex(pSubTblsInfo->pTree);
5,594✔
5943
  bool                 hasNext = true;
5,594✔
5944
  code = fetchNextSubTableBlockFromReader(pOperatorInfo, pInput, &hasNext);
5,594✔
5945
  QUERY_CHECK_CODE(code, lino, _end);
5,595!
5946

5947
  if (!hasNext) {
5,595!
5948
    pInput->rowIdx = -1;
5,595✔
5949
    ++pSubTblsInfo->numSubTablesCompleted;
5,595✔
5950
  } else {
5951
    pInput->rowIdx = 0;
×
5952
  }
5953

5954
_end:
5,595✔
5955
  if (code != TSDB_CODE_SUCCESS) {
5,595!
5956
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
5957
    pTaskInfo->code = code;
×
5958
    T_LONG_JMP(pTaskInfo->env, code);
×
5959
  }
5960
}
5,595✔
5961

5962
static int32_t adjustSubTableForNextRow(SOperatorInfo* pOperatorInfo, STmsSubTablesMergeInfo* pSubTblsInfo) {
1,005,777✔
5963
  STableMergeScanInfo* pInfo = pOperatorInfo->info;
1,005,777✔
5964
  STmsSubTableInput*   pInput = pSubTblsInfo->aInputs + tMergeTreeGetChosenIndex(pSubTblsInfo->pTree);
1,005,777✔
5965

5966
  SSDataBlock* pInputBlock = (pInput->type == SUB_TABLE_MEM_BLOCK) ? pInput->pReaderBlock : pInput->pPageBlock;
1,005,777✔
5967
  if (pInput->rowIdx < pInputBlock->info.rows - 1) {
1,005,777✔
5968
    ++pInput->rowIdx;
1,000,251✔
5969
  } else if (pInput->rowIdx == pInputBlock->info.rows - 1) {
5,526!
5970
    if (pInput->type == SUB_TABLE_MEM_BLOCK) {
5,594!
5971
      adjustSubTableFromMemBlock(pOperatorInfo, pSubTblsInfo);
5,594✔
5972
    }
5973
    if (pInput->rowIdx != -1) {
5,595!
5974
      SColumnInfoData* col = taosArrayGet(pInputBlock->pDataBlock, pSubTblsInfo->pTsOrderInfo->slotId);
×
5975
      if (!col) {
×
5976
        qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(terrno));
×
5977
        return terrno;
×
5978
      }
5979
      pInput->pInputBlock = pInputBlock;
×
5980
      pInput->aTs = (int64_t*)col->pData;
×
5981
    }
5982
  }
5983

5984
  return tMergeTreeAdjust(pSubTblsInfo->pTree, tMergeTreeGetAdjustIndex(pSubTblsInfo->pTree));
1,005,778✔
5985
}
5986

5987
static int32_t appendChosenRowToDataBlock(STmsSubTablesMergeInfo* pSubTblsInfo, SSDataBlock* pBlock) {
1,003,992✔
5988
  int32_t            code = TSDB_CODE_SUCCESS;
1,003,992✔
5989
  int32_t            lino = 0;
1,003,992✔
5990
  STmsSubTableInput* pInput = pSubTblsInfo->aInputs + tMergeTreeGetChosenIndex(pSubTblsInfo->pTree);
1,003,992✔
5991
  SSDataBlock*       pInputBlock = (pInput->type == SUB_TABLE_MEM_BLOCK) ? pInput->pReaderBlock : pInput->pPageBlock;
1,003,992!
5992

5993
  for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
4,069,700✔
5994
    SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
3,002,956✔
5995
    QUERY_CHECK_NULL(pColInfo, code, lino, _end, terrno);
3,046,352!
5996

5997
    SColumnInfoData* pSrcColInfo = taosArrayGet(pInputBlock->pDataBlock, i);
3,046,352✔
5998
    QUERY_CHECK_NULL(pSrcColInfo, code, lino, _end, terrno);
3,029,531!
5999
    bool isNull = colDataIsNull(pSrcColInfo, pInputBlock->info.rows, pInput->rowIdx, NULL);
3,044,825✔
6000

6001
    if (isNull) {
3,044,825✔
6002
      code = colDataSetVal(pColInfo, pBlock->info.rows, NULL, true);
286,755✔
6003
      QUERY_CHECK_CODE(code, lino, _end);
287,004!
6004
    } else {
6005
      if (pSrcColInfo->pData != NULL) {
2,758,070✔
6006
        char* pData = colDataGetData(pSrcColInfo, pInput->rowIdx);
2,755,103!
6007
        code = colDataSetVal(pColInfo, pBlock->info.rows, pData, false);
2,755,103✔
6008
        QUERY_CHECK_CODE(code, lino, _end);
2,775,737!
6009
      }
6010
    }
6011
  }
6012
  pBlock->info.dataLoad = 1;
1,006,503✔
6013
  pBlock->info.scanFlag = pInputBlock->info.scanFlag;
1,006,503✔
6014
  pBlock->info.rows += 1;
1,006,503✔
6015

6016
_end:
1,006,503✔
6017
  if (code != TSDB_CODE_SUCCESS) {
1,006,503!
6018
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
6019
  }
6020
  return code;
1,006,017✔
6021
}
6022

6023
static int32_t getSubTablesSortedBlock(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t capacity,
11,204✔
6024
                                       SSDataBlock** pResBlock) {
6025
  int32_t                 code = TSDB_CODE_SUCCESS;
11,204✔
6026
  int32_t                 lino = 0;
11,204✔
6027
  STableMergeScanInfo*    pInfo = pOperator->info;
11,204✔
6028
  SExecTaskInfo*          pTaskInfo = pOperator->pTaskInfo;
11,204✔
6029
  STmsSubTablesMergeInfo* pSubTblsInfo = pInfo->pSubTablesMergeInfo;
11,204✔
6030
  bool                    finished = false;
11,204✔
6031

6032
  QRY_PARAM_CHECK(pResBlock);
11,204!
6033

6034
  blockDataCleanup(pBlock);
11,204✔
6035

6036
  while (true) {
×
6037
    while (true) {
6038
      if (pSubTblsInfo->numSubTablesCompleted >= pSubTblsInfo->numSubTables) {
1,016,168✔
6039
        finished = true;
11,205✔
6040
        break;
11,205✔
6041
      }
6042

6043
      code = appendChosenRowToDataBlock(pSubTblsInfo, pBlock);
1,004,963✔
6044
      QUERY_CHECK_CODE(code, lino, _end);
1,005,839!
6045

6046
      code = adjustSubTableForNextRow(pOperator, pSubTblsInfo);
1,005,839✔
6047
      QUERY_CHECK_CODE(code, lino, _end);
1,004,876!
6048

6049
      if (pBlock->info.rows >= capacity) {
1,004,876!
6050
        break;
×
6051
      }
6052
    }
6053

6054
    if (isTaskKilled(pTaskInfo)) {
11,205!
6055
      return pTaskInfo->code;
×
6056
    }
6057

6058
    bool limitReached = applyLimitOffset(&pInfo->limitInfo, pBlock, pTaskInfo);
11,205✔
6059
    if (finished || limitReached || pBlock->info.rows > 0) {
11,204!
6060
      break;
6061
    }
6062
  }
6063

6064
  if (pBlock->info.rows > 0) {
11,204✔
6065
    *pResBlock = pBlock;
4,155✔
6066
  }
6067

6068
_end:
7,049✔
6069
  if (code != TSDB_CODE_SUCCESS) {
11,204!
6070
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
6071
    pTaskInfo->code = code;
×
6072
  }
6073
  return code;
11,204✔
6074
}
6075

6076
static int32_t startSubTablesTableMergeScan(SOperatorInfo* pOperator) {
7,009✔
6077
  int32_t              code = TSDB_CODE_SUCCESS;
7,009✔
6078
  int32_t              lino = 0;
7,009✔
6079
  STableMergeScanInfo* pInfo = pOperator->info;
7,009✔
6080

6081
  code = initSubTablesMergeInfo(pInfo);
7,009✔
6082
  QUERY_CHECK_CODE(code, lino, _end);
7,031!
6083

6084
  code = initSubTableInputs(pOperator, pInfo);
7,031✔
6085
  QUERY_CHECK_CODE(code, lino, _end);
7,050!
6086

6087
  code = openSubTablesMergeSort(pInfo->pSubTablesMergeInfo);
7,050✔
6088
  QUERY_CHECK_CODE(code, lino, _end);
7,049!
6089

6090
_end:
7,049✔
6091
  if (code != TSDB_CODE_SUCCESS) {
7,049!
6092
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
6093
  }
6094
  return code;
7,049✔
6095
}
6096

6097
static void stopSubTablesTableMergeScan(STableMergeScanInfo* pInfo) {
180,589✔
6098
  STmsSubTablesMergeInfo* pSubTblsInfo = pInfo->pSubTablesMergeInfo;
180,589✔
6099
  if (pSubTblsInfo != NULL) {
180,589✔
6100
    tMergeTreeDestroy(&pSubTblsInfo->pTree);
7,049✔
6101

6102
    for (int32_t i = 0; i < pSubTblsInfo->numSubTables; ++i) {
18,271✔
6103
      STmsSubTableInput* pInput = pSubTblsInfo->aInputs + i;
11,221✔
6104
      taosMemoryFree(pInput->tblCond.colList);
11,221!
6105
      blockDataDestroy(pInput->pReaderBlock);
11,220✔
6106
      blockDataDestroy(pInput->pPageBlock);
11,221✔
6107
      taosArrayDestroy(pInput->aBlockPages);
11,221✔
6108
      pInfo->base.readerAPI.tsdReaderClose(pInput->pReader);
11,221✔
6109
      pInput->pReader = NULL;
11,221✔
6110
    }
6111

6112
    destroyDiskbasedBuf(pSubTblsInfo->pBlocksBuf);
7,050✔
6113
    taosMemoryFree(pSubTblsInfo->aInputs);
7,050!
6114

6115
    taosMemoryFree(pSubTblsInfo);
7,050!
6116
    pInfo->pSubTablesMergeInfo = NULL;
7,050✔
6117

6118
    // taosMemoryTrim(0);
6119
  }
6120
}
180,590✔
6121

6122
int32_t doTableMergeScanParaSubTablesNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
11,053✔
6123
  QRY_PARAM_CHECK(ppRes);
11,053!
6124

6125
  int32_t lino = 0;
11,053✔
6126
  int32_t tableListSize = 0;
11,053✔
6127
  int64_t st = taosGetTimestampUs();
11,086✔
6128

6129
  SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
11,086✔
6130
  STableMergeScanInfo* pInfo = pOperator->info;
11,086✔
6131

6132
  if (pOperator->status == OP_EXEC_DONE) {
11,086!
6133
    return TSDB_CODE_SUCCESS;
×
6134
  }
6135

6136
  int32_t code = pOperator->fpSet._openFn(pOperator);
11,086✔
6137
  QUERY_CHECK_CODE(code, lino, _end);
11,061!
6138

6139
  code = tableListGetSize(pInfo->base.pTableListInfo, &tableListSize);
11,061✔
6140
  QUERY_CHECK_CODE(code, lino, _end);
11,074!
6141

6142
  if (!pInfo->hasGroupId) {
11,074✔
6143
    pInfo->hasGroupId = true;
6,928✔
6144

6145
    if (tableListSize == 0) {
6,928✔
6146
      setOperatorCompleted(pOperator);
130✔
6147
      (*ppRes) = NULL;
130✔
6148
      return code;
130✔
6149
    }
6150

6151
    pInfo->tableStartIndex = 0;
6,798✔
6152
    STableKeyInfo* pTmpGpId = (STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex);
6,798✔
6153
    QUERY_CHECK_NULL(pTmpGpId, code, lino, _end, terrno);
6,804!
6154
    pInfo->groupId = pTmpGpId->groupId;
6,804✔
6155
    code = startSubTablesTableMergeScan(pOperator);
6,804✔
6156
    QUERY_CHECK_CODE(code, lino, _end);
6,819!
6157
  }
6158

6159
  SSDataBlock* pBlock = NULL;
10,965✔
6160
  while (pInfo->tableStartIndex < tableListSize) {
11,195!
6161
    if (isTaskKilled(pTaskInfo)) {
11,204!
6162
      break;
×
6163
    }
6164

6165
    code = getSubTablesSortedBlock(pOperator, pInfo->pResBlock, pOperator->resultInfo.capacity, &pBlock);
11,204✔
6166
    QUERY_CHECK_CODE(code, lino, _end);
11,204!
6167

6168
    if (pBlock == NULL && !pInfo->bGroupProcessed && pInfo->needCountEmptyTable) {
11,204!
6169
      STableKeyInfo* tbInfo = tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex);
×
6170
      QUERY_CHECK_NULL(tbInfo, code, lino, _end, terrno);
×
6171

6172
      pBlock = getOneRowResultBlock(pTaskInfo, &pInfo->base, pInfo->pResBlock, tbInfo);
×
6173
    }
6174

6175
    if (pBlock != NULL) {
11,204✔
6176
      pBlock->info.id.groupId = pInfo->groupId;
4,155✔
6177
      pOperator->resultInfo.totalRows += pBlock->info.rows;
4,155✔
6178
      pInfo->bGroupProcessed = true;
4,155✔
6179
      break;
4,155✔
6180
    } else {
6181
      // Data of this group are all dumped, let's try the next group
6182
      stopSubTablesTableMergeScan(pInfo);
7,049✔
6183
      if (pInfo->tableEndIndex >= tableListSize - 1) {
7,050✔
6184
        setOperatorCompleted(pOperator);
6,820✔
6185
        break;
6,820✔
6186
      }
6187

6188
      pInfo->tableStartIndex = pInfo->tableEndIndex + 1;
230✔
6189
      STableKeyInfo* pTmpGpId = tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex);
230✔
6190
      QUERY_CHECK_NULL(pTmpGpId, code, lino, _end, terrno);
230!
6191

6192
      pInfo->groupId = pTmpGpId->groupId;
230✔
6193
      code = startSubTablesTableMergeScan(pOperator);
230✔
6194
      QUERY_CHECK_CODE(code, lino, _end);
230!
6195
      resetLimitInfoForNextGroup(&pInfo->limitInfo);
230✔
6196
    }
6197
  }
6198

6199
  pOperator->cost.totalCost += (taosGetTimestampUs() - st) / 1000.0;
10,975✔
6200

6201
_end:
10,975✔
6202
  if (code != TSDB_CODE_SUCCESS) {
10,975!
6203
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
6204
    pTaskInfo->code = code;
×
6205
    T_LONG_JMP(pTaskInfo->env, code);
×
6206
  } else {
6207
    (*ppRes) = pBlock;
10,975✔
6208
  }
6209

6210
  return code;
10,975✔
6211
}
6212

6213
static void tableMergeScanDoSkipTable(uint64_t uid, void* pTableMergeOpInfo) {
33,479✔
6214
  int32_t              code = TSDB_CODE_SUCCESS;
33,479✔
6215
  int32_t              lino = 0;
33,479✔
6216
  SOperatorInfo*       pOperator = (SOperatorInfo*)pTableMergeOpInfo;
33,479✔
6217
  STableMergeScanInfo* pInfo = pOperator->info;
33,479✔
6218
  SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
33,479✔
6219

6220
  if (pInfo->mSkipTables == NULL) {
33,479✔
6221
    pInfo->mSkipTables = taosHashInit(pInfo->tableEndIndex - pInfo->tableStartIndex + 1,
3,058✔
6222
                                      taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_NO_LOCK);
6223
    QUERY_CHECK_NULL(pInfo->mSkipTables, code, lino, _end, terrno);
3,058!
6224
  }
6225
  int bSkip = 1;
33,479✔
6226
  if (pInfo->mSkipTables != NULL) {
33,479!
6227
    code = taosHashPut(pInfo->mSkipTables, &uid, sizeof(uid), &bSkip, sizeof(bSkip));
33,479✔
6228
    if (code == TSDB_CODE_DUP_KEY) {
33,501!
6229
      code = TSDB_CODE_SUCCESS;
×
6230
    }
6231
    QUERY_CHECK_CODE(code, lino, _end);
33,501!
6232
  }
6233

6234
_end:
33,501✔
6235
  if (code != TSDB_CODE_SUCCESS) {
33,501!
6236
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
6237
    pTaskInfo->code = code;
×
6238
    T_LONG_JMP(pTaskInfo->env, code);
×
6239
  }
6240
}
33,501✔
6241

6242
static int32_t doGetBlockForTableMergeScan(SOperatorInfo* pOperator, bool* pFinished, bool* pSkipped) {
701,113✔
6243
  STableMergeScanInfo* pInfo = pOperator->info;
701,113✔
6244
  SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
701,113✔
6245
  SStorageAPI*         pAPI = &pTaskInfo->storageAPI;
701,113✔
6246
  SSDataBlock*         pBlock = pInfo->pReaderBlock;
701,113✔
6247
  int32_t              code = 0;
701,113✔
6248
  bool                 hasNext = false;
701,113✔
6249
  STsdbReader*         reader = pInfo->base.dataReader;
701,113✔
6250

6251
  code = pAPI->tsdReader.tsdNextDataBlock(reader, &hasNext);
701,113✔
6252
  if (code != 0) {
701,593!
6253
    pAPI->tsdReader.tsdReaderReleaseDataBlock(reader);
×
6254
    qError("table merge scan fetch next data block error code: %d, %s", code, GET_TASKID(pTaskInfo));
×
6255
    pTaskInfo->code = code;
×
6256
    return code;
×
6257
  }
6258

6259
  if (!hasNext || isTaskKilled(pTaskInfo)) {
701,593!
6260
    if (isTaskKilled(pTaskInfo)) {
206,690!
6261
      qInfo("table merge scan fetch next data block found task killed. %s", GET_TASKID(pTaskInfo));
×
6262
      pAPI->tsdReader.tsdReaderReleaseDataBlock(reader);
×
6263
    }
6264
    *pFinished = true;
206,808✔
6265
    return code;
206,808✔
6266
  }
6267

6268
  uint32_t status = 0;
494,869✔
6269
  code = loadDataBlock(pOperator, &pInfo->base, pBlock, &status);
494,869✔
6270

6271
  if (code != TSDB_CODE_SUCCESS) {
494,319!
6272
    qInfo("table merge scan load datablock code %d, %s", code, GET_TASKID(pTaskInfo));
×
6273
    pTaskInfo->code = code;
×
6274
    return code;
×
6275
  }
6276

6277
  if (status == FUNC_DATA_REQUIRED_ALL_FILTEROUT) {
494,353!
6278
    *pFinished = true;
×
6279
    return code;
×
6280
  }
6281

6282
  // current block is filter out according to filter condition, continue load the next block
6283
  if (status == FUNC_DATA_REQUIRED_FILTEROUT || pBlock->info.rows == 0) {
494,353!
6284
    *pSkipped = true;
22,695✔
6285
    return code;
22,695✔
6286
  }
6287

6288
  return code;
471,658✔
6289
}
6290

6291
static int32_t getBlockForTableMergeScan(void* param, SSDataBlock** ppBlock) {
722,986✔
6292
  STableMergeScanSortSourceParam* source = param;
722,986✔
6293

6294
  SOperatorInfo*       pOperator = source->pOperator;
722,986✔
6295
  STableMergeScanInfo* pInfo = pOperator->info;
722,986✔
6296
  SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
722,986✔
6297
  SSDataBlock*         pBlock = NULL;
722,986✔
6298
  int64_t              st = taosGetTimestampUs();
723,191✔
6299
  int32_t              code = TSDB_CODE_SUCCESS;
723,191✔
6300

6301
  while (true) {
6302
    if (pInfo->rtnNextDurationBlocks) {
782,741✔
6303
      qDebug("%s table merge scan return already fetched new duration blocks. index %d num of blocks %d",
81,392✔
6304
             GET_TASKID(pTaskInfo), pInfo->nextDurationBlocksIdx, pInfo->numNextDurationBlocks);
6305

6306
      if (pInfo->nextDurationBlocksIdx < pInfo->numNextDurationBlocks) {
81,402✔
6307
        pBlock = pInfo->nextDurationBlocks[pInfo->nextDurationBlocksIdx];
36,878✔
6308
        ++pInfo->nextDurationBlocksIdx;
36,878✔
6309
      } else {
6310
        for (int32_t i = 0; i < pInfo->numNextDurationBlocks; ++i) {
81,428✔
6311
          blockDataDestroy(pInfo->nextDurationBlocks[i]);
36,897✔
6312
          pInfo->nextDurationBlocks[i] = NULL;
36,904✔
6313
        }
6314

6315
        pInfo->rtnNextDurationBlocks = false;
44,531✔
6316
        pInfo->nextDurationBlocksIdx = 0;
44,531✔
6317
        pInfo->numNextDurationBlocks = 0;
44,531✔
6318
        continue;
44,531✔
6319
      }
6320
    } else {
6321
      bool bFinished = false;
701,349✔
6322
      bool bSkipped = false;
701,349✔
6323

6324
      code = doGetBlockForTableMergeScan(pOperator, &bFinished, &bSkipped);
701,349✔
6325
      if (code != 0) {
700,986!
6326
        return code;
44,476✔
6327
      }
6328

6329
      pBlock = pInfo->pReaderBlock;
700,986✔
6330
      qDebug("%s table merge scan fetch block. finished %d skipped %d next-duration-block %d new-fileset %d",
700,986✔
6331
             GET_TASKID(pTaskInfo), bFinished, bSkipped, pInfo->bNextDurationBlockEvent, pInfo->bNewFilesetEvent);
6332
      if (bFinished) {
701,042✔
6333
        pInfo->bNewFilesetEvent = false;
206,798✔
6334
        break;
206,798✔
6335
      }
6336

6337
      if (pInfo->bNextDurationBlockEvent || pInfo->bNewFilesetEvent) {
494,244!
6338
        if (!bSkipped) {
44,471✔
6339
          code = createOneDataBlock(pBlock, true, &pInfo->nextDurationBlocks[pInfo->numNextDurationBlocks]);
36,866✔
6340
          if (code) {
36,871!
6341
            *ppBlock = NULL;
×
6342
            return code;
×
6343
          }
6344

6345
          ++pInfo->numNextDurationBlocks;
36,871✔
6346
          if (pInfo->numNextDurationBlocks > 2) {
36,871!
6347
            qError("%s table merge scan prefetch %d next duration blocks. end early.", GET_TASKID(pTaskInfo),
×
6348
                   pInfo->numNextDurationBlocks);
6349
            pInfo->bNewFilesetEvent = false;
×
6350
            break;
×
6351
          }
6352
        }
6353

6354
        if (pInfo->bNewFilesetEvent) {
44,476!
6355
          pInfo->rtnNextDurationBlocks = true;
44,476✔
6356
          *ppBlock = NULL;
44,476✔
6357
          return code;
44,476✔
6358
        }
6359

6360
        if (pInfo->bNextDurationBlockEvent) {
×
6361
          pInfo->bNextDurationBlockEvent = false;
×
6362
          continue;
15,019✔
6363
        }
6364
      }
6365
      if (bSkipped) continue;
449,773✔
6366
    }
6367

6368
    pBlock->info.id.groupId = tableListGetTableGroupId(pInfo->base.pTableListInfo, pBlock->info.id.uid);
471,632✔
6369

6370
    pOperator->resultInfo.totalRows += pBlock->info.rows;
471,635✔
6371
    pInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
471,735✔
6372
    *ppBlock = pBlock;
471,735✔
6373

6374
    return code;
471,735✔
6375
  }
6376

6377
  *ppBlock = NULL;
206,798✔
6378
  return code;
206,798✔
6379
}
6380

6381
int32_t generateSortByTsPkInfo(SArray* colMatchInfo, int32_t order, SArray** ppSortArray) {
172,865✔
6382
  int32_t code = TSDB_CODE_SUCCESS;
172,865✔
6383
  int32_t lino = 0;
172,865✔
6384
  SArray* pSortInfo = taosArrayInit(1, sizeof(SBlockOrderInfo));
172,865✔
6385
  QUERY_CHECK_NULL(pSortInfo, code, lino, _end, terrno);
173,327!
6386
  SBlockOrderInfo biTs = {0};
173,389✔
6387
  SBlockOrderInfo biPk = {0};
173,389✔
6388

6389
  int32_t tsTargetSlotId = 0;
173,389✔
6390
  int32_t pkTargetSlotId = -1;
173,389✔
6391
  for (int32_t i = 0; i < taosArrayGetSize(colMatchInfo); ++i) {
852,058✔
6392
    SColMatchItem* colInfo = taosArrayGet(colMatchInfo, i);
678,876✔
6393
    QUERY_CHECK_NULL(colInfo, code, lino, _end, terrno);
678,790!
6394
    if (colInfo->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
678,888✔
6395
      tsTargetSlotId = colInfo->dstSlotId;
173,306✔
6396
      biTs.order = order;
173,306✔
6397
      biTs.slotId = tsTargetSlotId;
173,306✔
6398
      biTs.nullFirst = (order == TSDB_ORDER_ASC);
173,306✔
6399
      biTs.compFn = getKeyComparFunc(TSDB_DATA_TYPE_TIMESTAMP, order);
173,306✔
6400
    }
6401
    // TODO: order by just ts
6402
    if (colInfo->isPk) {
678,678✔
6403
      pkTargetSlotId = colInfo->dstSlotId;
13,098✔
6404
      biPk.order = order;
13,098✔
6405
      biPk.slotId = pkTargetSlotId;
13,098✔
6406
      biPk.nullFirst = (order == TSDB_ORDER_ASC);
13,098✔
6407
      biPk.compFn = getKeyComparFunc(colInfo->dataType.type, order);
13,098✔
6408
    }
6409
  }
6410

6411
  void* tmp = taosArrayPush(pSortInfo, &biTs);
173,202✔
6412
  QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
173,202!
6413
  if (pkTargetSlotId != -1) {
173,202✔
6414
    tmp = taosArrayPush(pSortInfo, &biPk);
13,092✔
6415
    QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
13,092!
6416
  }
6417

6418
  (*ppSortArray) = pSortInfo;
173,205✔
6419

6420
_end:
173,205✔
6421
  if (code != TSDB_CODE_SUCCESS) {
173,205!
6422
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
6423
  }
6424
  return code;
172,701✔
6425
}
6426

6427
void tableMergeScanTsdbNotifyCb(ETsdReaderNotifyType type, STsdReaderNotifyInfo* info, void* param) {
145,409✔
6428
  STableMergeScanInfo* pTmsInfo = param;
145,409✔
6429
  if (type == TSD_READER_NOTIFY_DURATION_START) {
145,409!
6430
    pTmsInfo->bNewFilesetEvent = true;
145,502✔
6431
  } else if (type == TSD_READER_NOTIFY_NEXT_DURATION_BLOCK) {
×
6432
    pTmsInfo->bNextDurationBlockEvent = true;
×
6433
  }
6434
  qDebug("table merge scan receive notification. type %d, fileset %d", type, info->duration.filesetId);
145,409✔
6435

6436
  return;
145,539✔
6437
}
6438

6439
int32_t startDurationForGroupTableMergeScan(SOperatorInfo* pOperator) {
251,136✔
6440
  STableMergeScanInfo* pInfo = pOperator->info;
251,136✔
6441
  SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
251,136✔
6442

6443
  int32_t code = TSDB_CODE_SUCCESS;
251,136✔
6444
  int32_t lino = 0;
251,136✔
6445
  int32_t numOfTable = pInfo->tableEndIndex - pInfo->tableStartIndex + 1;
251,136✔
6446

6447
  qDebug("%s table merge scan start duration ", GET_TASKID(pTaskInfo));
251,136✔
6448
  pInfo->bNewFilesetEvent = false;
251,137✔
6449
  pInfo->bNextDurationBlockEvent = false;
251,137✔
6450

6451
  pInfo->sortBufSize = 2048 * pInfo->bufPageSize;
251,137✔
6452
  int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize;
251,137✔
6453

6454
  pInfo->pSortHandle = NULL;
251,137✔
6455
  code = tsortCreateSortHandle(pInfo->pSortInfo, SORT_BLOCK_TS_MERGE, pInfo->bufPageSize, numOfBufPage,
251,137✔
6456
                               pInfo->pSortInputBlock, pTaskInfo->id.str, 0, 0, 0, &pInfo->pSortHandle);
251,137✔
6457
  if (code) {
251,102!
6458
    return code;
×
6459
  }
6460

6461
  if (pInfo->bSortRowId && numOfTable != 1) {
251,102✔
6462
    int32_t memSize = 512 * 1024 * 1024;
28,673✔
6463
    code = tsortSetSortByRowId(pInfo->pSortHandle, memSize);
28,673✔
6464
    QUERY_CHECK_CODE(code, lino, _end);
28,689!
6465
  }
6466

6467
  tsortSetMergeLimit(pInfo->pSortHandle, pInfo->mergeLimit);
251,118✔
6468
  tsortSetMergeLimitReachedFp(pInfo->pSortHandle, tableMergeScanDoSkipTable, pOperator);
250,842✔
6469
  tsortSetAbortCheckFn(pInfo->pSortHandle, isTaskKilled, pOperator->pTaskInfo);
250,709✔
6470

6471
  tsortSetFetchRawDataFp(pInfo->pSortHandle, getBlockForTableMergeScan, NULL, NULL);
250,742✔
6472
  QUERY_CHECK_CODE(code, lino, _end);
250,840!
6473

6474
  STableMergeScanSortSourceParam* param = taosMemoryCalloc(1, sizeof(STableMergeScanSortSourceParam));
250,840!
6475
  QUERY_CHECK_NULL(param, code, lino, _end, terrno);
251,294!
6476
  param->pOperator = pOperator;
251,294✔
6477

6478
  SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource));
251,294!
6479
  if (ps == NULL) {
251,291!
6480
    taosMemoryFree(param);
×
6481
    QUERY_CHECK_NULL(ps, code, lino, _end, terrno);
×
6482
  }
6483

6484
  ps->param = param;
251,291✔
6485
  ps->onlyRef = false;
251,291✔
6486
  code = tsortAddSource(pInfo->pSortHandle, ps);
251,291✔
6487
  QUERY_CHECK_CODE(code, lino, _end);
250,870!
6488

6489
  if (numOfTable == 1) {
250,870✔
6490
    tsortSetSingleTableMerge(pInfo->pSortHandle);
172,584✔
6491
  } else {
6492
    code = tsortOpen(pInfo->pSortHandle);
78,286✔
6493
    QUERY_CHECK_CODE(code, lino, _end);
78,436!
6494
  }
6495

6496
_end:
78,436✔
6497
  if (code != TSDB_CODE_SUCCESS) {
250,946!
6498
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
6499
  }
6500
  return code;
250,986✔
6501
}
6502

6503
void stopDurationForGroupTableMergeScan(SOperatorInfo* pOperator) {
251,066✔
6504
  STableMergeScanInfo* pInfo = pOperator->info;
251,066✔
6505
  SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
251,066✔
6506
  qDebug("%s table merge scan stop duration ", GET_TASKID(pTaskInfo));
251,066✔
6507

6508
  SSortExecInfo sortExecInfo = tsortGetSortExecInfo(pInfo->pSortHandle);
251,067✔
6509
  pInfo->sortExecInfo.sortMethod = sortExecInfo.sortMethod;
251,137✔
6510
  pInfo->sortExecInfo.sortBuffer = sortExecInfo.sortBuffer;
251,137✔
6511
  pInfo->sortExecInfo.loops += sortExecInfo.loops;
251,137✔
6512
  pInfo->sortExecInfo.readBytes += sortExecInfo.readBytes;
251,137✔
6513
  pInfo->sortExecInfo.writeBytes += sortExecInfo.writeBytes;
251,137✔
6514

6515
  tsortDestroySortHandle(pInfo->pSortHandle);
251,137✔
6516
  pInfo->pSortHandle = NULL;
251,218✔
6517
}
251,218✔
6518

6519
void startGroupTableMergeScan(SOperatorInfo* pOperator) {
205,863✔
6520
  int32_t              code = TSDB_CODE_SUCCESS;
205,863✔
6521
  int32_t              lino = 0;
205,863✔
6522
  STableMergeScanInfo* pInfo = pOperator->info;
205,863✔
6523
  SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
205,863✔
6524
  SReadHandle*         pHandle = &pInfo->base.readHandle;
205,863✔
6525
  SStorageAPI*         pAPI = &pTaskInfo->storageAPI;
205,863✔
6526
  qDebug("%s table merge scan start group %" PRIu64, GET_TASKID(pTaskInfo), pInfo->groupId);
205,863✔
6527

6528
  {
6529
    int32_t numOfTables = 0;
205,863✔
6530
    code = tableListGetSize(pInfo->base.pTableListInfo, &numOfTables);
205,863✔
6531
    QUERY_CHECK_CODE(code, lino, _end);
206,479!
6532

6533
    int32_t i = pInfo->tableStartIndex + 1;
206,479✔
6534
    for (; i < numOfTables; ++i) {
565,474✔
6535
      STableKeyInfo* tableKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, i);
411,656✔
6536
      QUERY_CHECK_NULL(tableKeyInfo, code, lino, _end, terrno);
411,653✔
6537
      if (tableKeyInfo->groupId != pInfo->groupId) {
411,641✔
6538
        break;
52,646✔
6539
      }
6540
    }
6541
    pInfo->tableEndIndex = i - 1;
206,464✔
6542
  }
6543
  pInfo->bGroupProcessed = false;
206,464✔
6544
  int32_t tableStartIdx = pInfo->tableStartIndex;
206,464✔
6545
  int32_t tableEndIdx = pInfo->tableEndIndex;
206,464✔
6546

6547
  int32_t        numOfTable = tableEndIdx - tableStartIdx + 1;
206,464✔
6548
  STableKeyInfo* startKeyInfo = tableListGetInfo(pInfo->base.pTableListInfo, tableStartIdx);
206,464✔
6549
  code = pAPI->tsdReader.tsdReaderOpen(pHandle->vnode, &pInfo->base.cond, startKeyInfo, numOfTable, pInfo->pReaderBlock,
206,407✔
6550
                                       (void**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), &pInfo->mSkipTables);
206,407✔
6551
  QUERY_CHECK_CODE(code, lino, _end);
206,562!
6552
  if (pInfo->filesetDelimited) {
206,562!
6553
    pAPI->tsdReader.tsdSetFilesetDelimited(pInfo->base.dataReader);
206,710✔
6554
  }
6555
  pAPI->tsdReader.tsdSetSetNotifyCb(pInfo->base.dataReader, tableMergeScanTsdbNotifyCb, pInfo);
206,323✔
6556

6557
  code = startDurationForGroupTableMergeScan(pOperator);
206,705✔
6558
  QUERY_CHECK_CODE(code, lino, _end);
206,404!
6559

6560
_end:
206,404✔
6561
  if (code != TSDB_CODE_SUCCESS) {
206,404!
6562
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
6563
    pTaskInfo->code = code;
×
6564
    T_LONG_JMP(pTaskInfo->env, code);
×
6565
  }
6566
}
206,404✔
6567

6568
void stopGroupTableMergeScan(SOperatorInfo* pOperator) {
206,646✔
6569
  STableMergeScanInfo* pInfo = pOperator->info;
206,646✔
6570
  SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
206,646✔
6571
  SStorageAPI*         pAPI = &pTaskInfo->storageAPI;
206,646✔
6572

6573
  stopDurationForGroupTableMergeScan(pOperator);
206,646✔
6574

6575
  if (pInfo->base.dataReader != NULL) {
206,695✔
6576
    pAPI->tsdReader.tsdReaderClose(pInfo->base.dataReader);
206,685✔
6577
    pInfo->base.dataReader = NULL;
206,709✔
6578
  }
6579
  for (int32_t i = 0; i < pInfo->numNextDurationBlocks; ++i) {
206,719!
6580
    if (pInfo->nextDurationBlocks[i]) {
×
6581
      blockDataDestroy(pInfo->nextDurationBlocks[i]);
×
6582
      pInfo->nextDurationBlocks[i] = NULL;
×
6583
    }
6584
    pInfo->numNextDurationBlocks = 0;
×
6585
    pInfo->nextDurationBlocksIdx = 0;
×
6586
  }
6587
  resetLimitInfoForNextGroup(&pInfo->limitInfo);
206,719✔
6588
  taosHashCleanup(pInfo->mSkipTables);
206,660✔
6589
  pInfo->mSkipTables = NULL;
206,633✔
6590
  qDebug("%s table merge scan stop group %" PRIu64, GET_TASKID(pTaskInfo), pInfo->groupId);
206,633✔
6591
}
206,633✔
6592

6593
// all data produced by this function only belongs to one group
6594
// slimit/soffset does not need to be concerned here, since this function only deal with data within one group.
6595
SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock* pResBlock, int32_t capacity,
438,504✔
6596
                                              SOperatorInfo* pOperator) {
6597
  STableMergeScanInfo* pInfo = pOperator->info;
438,504✔
6598
  SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
438,504✔
6599
  STupleHandle*        pTupleHandle = NULL;
438,504✔
6600

6601
  blockDataCleanup(pResBlock);
438,504✔
6602

6603
  while (1) {
×
6604
    while (1) {
56,943,869✔
6605
      pTupleHandle = NULL;
57,382,792✔
6606
      int32_t code = tsortNextTuple(pHandle, &pTupleHandle);
57,382,792✔
6607
      if (code != TSDB_CODE_SUCCESS) {
56,061,583✔
6608
        qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
16,617!
6609
        T_LONG_JMP(pOperator->pTaskInfo->env, code);
16,617!
6610
      }
6611
      if (pTupleHandle == NULL) {
56,044,966✔
6612
        break;
401,597✔
6613
      }
6614

6615
      code = tsortAppendTupleToBlock(pInfo->pSortHandle, pResBlock, pTupleHandle);
55,643,369✔
6616
      if (code != TSDB_CODE_SUCCESS) {
56,971,741!
6617
        qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
×
6618
        T_LONG_JMP(pOperator->pTaskInfo->env, terrno);
×
6619
      }
6620

6621
      if (pResBlock->info.rows >= capacity) {
56,981,280✔
6622
        break;
37,411✔
6623
      }
6624
    }
6625

6626
    if (tsortIsClosed(pHandle)) {
439,008!
6627
      terrno = TSDB_CODE_TSC_QUERY_CANCELLED;
×
6628
      T_LONG_JMP(pOperator->pTaskInfo->env, terrno);
×
6629
    }
6630

6631
    bool limitReached = applyLimitOffset(&pInfo->limitInfo, pResBlock, pTaskInfo);
439,268✔
6632
    qDebug("%s get sorted row block, rows:%" PRId64 ", limit:%" PRId64, GET_TASKID(pTaskInfo), pResBlock->info.rows,
439,148✔
6633
           pInfo->limitInfo.numOfOutputRows);
6634
    if (pTupleHandle == NULL || limitReached || pResBlock->info.rows > 0) {
439,131!
6635
      break;
6636
    }
6637
  }
6638
  return (pResBlock->info.rows > 0) ? pResBlock : NULL;
439,131✔
6639
}
6640

6641
int32_t doTableMergeScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
353,403✔
6642
  if (pOperator->status == OP_EXEC_DONE) {
353,403✔
6643
    (*ppRes) = NULL;
11✔
6644
    return TSDB_CODE_SUCCESS;
11✔
6645
  }
6646

6647
  int32_t              code = TSDB_CODE_SUCCESS;
353,392✔
6648
  int32_t              lino = 0;
353,392✔
6649
  SExecTaskInfo*       pTaskInfo = pOperator->pTaskInfo;
353,392✔
6650
  STableMergeScanInfo* pInfo = pOperator->info;
353,392✔
6651

6652
  code = pOperator->fpSet._openFn(pOperator);
353,392✔
6653
  QUERY_CHECK_CODE(code, lino, _end);
353,628!
6654

6655
  int64_t st = taosGetTimestampUs();
354,280✔
6656

6657
  int32_t tableListSize = 0;
354,280✔
6658
  code = tableListGetSize(pInfo->base.pTableListInfo, &tableListSize);
354,280✔
6659
  QUERY_CHECK_CODE(code, lino, _end);
353,899!
6660

6661
  if (!pInfo->hasGroupId) {
353,899✔
6662
    pInfo->hasGroupId = true;
166,146✔
6663

6664
    if (tableListSize == 0) {
166,146✔
6665
      setOperatorCompleted(pOperator);
12,476✔
6666
      (*ppRes) = NULL;
12,480✔
6667
      return code;
12,480✔
6668
    }
6669
    pInfo->tableStartIndex = 0;
153,670✔
6670
    STableKeyInfo* tmp = (STableKeyInfo*)tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex);
153,670✔
6671
    QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
153,665!
6672
    pInfo->groupId = tmp->groupId;
153,665✔
6673
    startGroupTableMergeScan(pOperator);
153,665✔
6674
  }
6675

6676
  SSDataBlock* pBlock = NULL;
341,581✔
6677
  while (pInfo->tableStartIndex < tableListSize) {
438,616!
6678
    if (isTaskKilled(pTaskInfo)) {
438,616✔
6679
      goto _end;
101✔
6680
    }
6681

6682
    pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pInfo->pResBlock, pOperator->resultInfo.capacity,
438,630✔
6683
                                              pOperator);
6684
    if (pBlock == NULL && !pInfo->bGroupProcessed && pInfo->needCountEmptyTable) {
439,067✔
6685
      STableKeyInfo* tbInfo = tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex);
14,443✔
6686
      QUERY_CHECK_NULL(tbInfo, code, lino, _end, terrno);
14,445!
6687
      pBlock = getOneRowResultBlock(pTaskInfo, &pInfo->base, pInfo->pResBlock, tbInfo);
14,445✔
6688
    }
6689
    if (pBlock != NULL) {
439,127✔
6690
      pBlock->info.id.groupId = pInfo->groupId;
187,952✔
6691
      pOperator->resultInfo.totalRows += pBlock->info.rows;
187,952✔
6692
      pInfo->bGroupProcessed = true;
187,952✔
6693
      break;
187,952✔
6694
    } else {
6695
      if (pInfo->bNewFilesetEvent) {
251,175✔
6696
        stopDurationForGroupTableMergeScan(pOperator);
44,518✔
6697
        code = startDurationForGroupTableMergeScan(pOperator);
44,522✔
6698
        QUERY_CHECK_CODE(code, lino, _end);
44,466!
6699
      } else {
6700
        // Data of this group are all dumped, let's try the next group
6701
        stopGroupTableMergeScan(pOperator);
206,657✔
6702
        if (pInfo->tableEndIndex >= tableListSize - 1) {
206,631✔
6703
          setOperatorCompleted(pOperator);
153,962✔
6704
          break;
153,965✔
6705
        }
6706

6707
        pInfo->tableStartIndex = pInfo->tableEndIndex + 1;
52,669✔
6708
        STableKeyInfo* tmp = tableListGetInfo(pInfo->base.pTableListInfo, pInfo->tableStartIndex);
52,669✔
6709
        QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
52,717!
6710
        pInfo->groupId = tmp->groupId;
52,717✔
6711
        startGroupTableMergeScan(pOperator);
52,717✔
6712
        resetLimitInfoForNextGroup(&pInfo->limitInfo);
52,657✔
6713
      }
6714
    }
6715
  }
6716

6717
  pOperator->cost.totalCost += (taosGetTimestampUs() - st) / 1000.0;
341,915✔
6718

6719
_end:
342,016✔
6720
  if (code != TSDB_CODE_SUCCESS) {
342,016!
6721
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
6722
    pTaskInfo->code = code;
×
6723
    T_LONG_JMP(pTaskInfo->env, code);
×
6724
  } else {
6725
    (*ppRes) = pBlock;
342,016✔
6726
  }
6727

6728
  return code;
342,016✔
6729
}
6730

6731
static SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
×
6732
  SSDataBlock* pRes = NULL;
×
6733
  int32_t      code = doTableMergeScanNext(pOperator, &pRes);
×
6734
  return pRes;
×
6735
}
6736

6737
void destroyTableMergeScanOperatorInfo(void* param) {
173,520✔
6738
  STableMergeScanInfo* pTableScanInfo = (STableMergeScanInfo*)param;
173,520✔
6739

6740
  // start one reader variable
6741
  if (pTableScanInfo->base.readerAPI.tsdReaderClose != NULL) {
173,520!
6742
    pTableScanInfo->base.readerAPI.tsdReaderClose(pTableScanInfo->base.dataReader);
173,532✔
6743
    pTableScanInfo->base.dataReader = NULL;
173,536✔
6744
  }
6745

6746
  for (int32_t i = 0; i < pTableScanInfo->numNextDurationBlocks; ++i) {
173,524!
6747
    if (pTableScanInfo->nextDurationBlocks[i] != NULL) {
×
6748
      blockDataDestroy(pTableScanInfo->nextDurationBlocks[i]);
×
6749
      pTableScanInfo->nextDurationBlocks[i] = NULL;
×
6750
    }
6751
  }
6752

6753
  tsortDestroySortHandle(pTableScanInfo->pSortHandle);
173,524✔
6754
  pTableScanInfo->pSortHandle = NULL;
173,527✔
6755
  taosHashCleanup(pTableScanInfo->mSkipTables);
173,527✔
6756
  pTableScanInfo->mSkipTables = NULL;
173,529✔
6757
  blockDataDestroy(pTableScanInfo->pSortInputBlock);
173,529✔
6758
  pTableScanInfo->pSortInputBlock = NULL;
173,555✔
6759
  // end one reader variable
6760

6761
  cleanupQueryTableDataCond(&pTableScanInfo->base.cond);
173,555✔
6762
  destroyTableScanBase(&pTableScanInfo->base, &pTableScanInfo->base.readerAPI);
173,552✔
6763

6764
  blockDataDestroy(pTableScanInfo->pResBlock);
173,534✔
6765
  pTableScanInfo->pResBlock = NULL;
173,555✔
6766

6767
  // remove it from the task->result list
6768
  blockDataDestroy(pTableScanInfo->pReaderBlock);
173,555✔
6769
  pTableScanInfo->pReaderBlock = NULL;
173,558✔
6770
  taosArrayDestroy(pTableScanInfo->pSortInfo);
173,558✔
6771

6772
  stopSubTablesTableMergeScan(pTableScanInfo);
173,557✔
6773

6774
  taosMemoryFreeClear(param);
173,539!
6775
}
173,545✔
6776

6777
int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) {
3,241✔
6778
  if (pOptr == NULL) {
3,241!
6779
    qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(TSDB_CODE_INVALID_PARA));
×
6780
    return TSDB_CODE_INVALID_PARA;
×
6781
  }
6782
  // TODO: merge these two info into one struct
6783
  STableMergeScanExecInfo* execInfo = taosMemoryCalloc(1, sizeof(STableMergeScanExecInfo));
3,241!
6784
  if (!execInfo) {
3,241!
6785
    return terrno;
×
6786
  }
6787
  STableMergeScanInfo* pInfo = pOptr->info;
3,241✔
6788
  execInfo->blockRecorder = pInfo->base.readRecorder;
3,241✔
6789
  execInfo->sortExecInfo = pInfo->sortExecInfo;
3,241✔
6790

6791
  *pOptrExplain = execInfo;
3,241✔
6792
  *len = sizeof(STableMergeScanExecInfo);
3,241✔
6793

6794
  return TSDB_CODE_SUCCESS;
3,241✔
6795
}
6796

6797
int32_t createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
172,435✔
6798
                                         STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo,
6799
                                         SOperatorInfo** pOptrInfo) {
6800
  QRY_PARAM_CHECK(pOptrInfo);
172,435!
6801

6802
  int32_t              code = TSDB_CODE_SUCCESS;
172,435✔
6803
  int32_t              lino = 0;
172,435✔
6804
  STableMergeScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableMergeScanInfo));
172,435!
6805
  SOperatorInfo*       pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
173,271!
6806
  if (pInfo == NULL || pOperator == NULL) {
172,911!
6807
    code = terrno;
×
6808
    goto _error;
×
6809
  }
6810

6811
  SDataBlockDescNode* pDescNode = pTableScanNode->scan.node.pOutputDataBlockDesc;
172,971✔
6812

6813
  int32_t numOfCols = 0;
172,971✔
6814
  code = extractColMatchInfo(pTableScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID,
172,971✔
6815
                             &pInfo->base.matchInfo);
6816
  QUERY_CHECK_CODE(code, lino, _error);
172,579!
6817

6818
  code = initQueryTableDataCond(&pInfo->base.cond, pTableScanNode, readHandle);
172,579✔
6819
  QUERY_CHECK_CODE(code, lino, _error);
173,244!
6820

6821
  if (pTableScanNode->scan.pScanPseudoCols != NULL) {
173,244✔
6822
    SExprSupp* pSup = &pInfo->base.pseudoSup;
81,894✔
6823
    code = createExprInfo(pTableScanNode->scan.pScanPseudoCols, NULL, &pSup->pExprInfo, &pSup->numOfExprs);
81,894✔
6824
    QUERY_CHECK_CODE(code, lino, _error);
81,850!
6825

6826
    pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset,
81,850✔
6827
                                      &pTaskInfo->storageAPI.functionStore);
6828
    QUERY_CHECK_NULL(pSup->pCtx, code, lino, _error, terrno);
81,523!
6829
  }
6830

6831
  pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]};
172,873✔
6832

6833
  pInfo->base.metaCache.pTableMetaEntryCache = taosLRUCacheInit(1024 * 128, -1, .5);
172,873✔
6834
  QUERY_CHECK_NULL(pInfo->base.metaCache.pTableMetaEntryCache, code, lino, _error, terrno);
173,192!
6835

6836
  pInfo->base.readerAPI = pTaskInfo->storageAPI.tsdReader;
173,192✔
6837
  pInfo->base.dataBlockLoadFlag = FUNC_DATA_REQUIRED_DATA_LOAD;
173,192✔
6838
  pInfo->base.scanFlag = MAIN_SCAN;
173,192✔
6839
  pInfo->base.readHandle = *readHandle;
173,192✔
6840

6841
  pInfo->readIdx = -1;
173,192✔
6842

6843
  pInfo->base.limitInfo.limit.limit = -1;
173,192✔
6844
  pInfo->base.limitInfo.slimit.limit = -1;
173,192✔
6845
  pInfo->base.pTableListInfo = pTableListInfo;
173,192✔
6846

6847
  pInfo->sample.sampleRatio = pTableScanNode->ratio;
173,192✔
6848
  pInfo->sample.seed = taosGetTimestampSec();
173,192✔
6849

6850
  code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
173,296✔
6851
  QUERY_CHECK_CODE(code, lino, _error);
173,107!
6852

6853
  initLimitInfo(pTableScanNode->scan.node.pLimit, pTableScanNode->scan.node.pSlimit, &pInfo->limitInfo);
173,107✔
6854

6855
  pInfo->mergeLimit = -1;
173,249✔
6856
  bool hasLimit = pInfo->limitInfo.limit.limit != -1 || pInfo->limitInfo.limit.offset != -1;
173,249!
6857
  if (hasLimit) {
173,249✔
6858
    pInfo->mergeLimit = pInfo->limitInfo.limit.offset != -1 ? pInfo->limitInfo.limit.limit + pInfo->limitInfo.limit.offset : pInfo->limitInfo.limit.limit;
15,303✔
6859
    pInfo->mSkipTables = NULL;
15,303✔
6860
  }
6861

6862
  initResultSizeInfo(&pOperator->resultInfo, 1024);
173,249✔
6863
  pInfo->pResBlock = createDataBlockFromDescNode(pDescNode);
172,826✔
6864
  QUERY_CHECK_NULL(pInfo->pResBlock, code, lino, _error, terrno);
173,011!
6865
  code = blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity);
173,011✔
6866
  QUERY_CHECK_CODE(code, lino, _error);
173,376!
6867
  if (!hasLimit && blockDataGetRowSize(pInfo->pResBlock) >= 256 && !pTableScanNode->smallDataTsSort) {
173,376!
6868
    pInfo->bSortRowId = true;
45,534✔
6869
  } else {
6870
    pInfo->bSortRowId = false;
127,710✔
6871
  }
6872

6873
  code = prepareDataBlockBuf(pInfo->pResBlock, &pInfo->base.matchInfo);
173,244✔
6874
  QUERY_CHECK_CODE(code, lino, _error);
173,192!
6875

6876
  code = generateSortByTsPkInfo(pInfo->base.matchInfo.pList, pInfo->base.cond.order, &pInfo->pSortInfo);
173,192✔
6877
  QUERY_CHECK_CODE(code, lino, _error);
172,845!
6878

6879
  code = createOneDataBlock(pInfo->pResBlock, false, &pInfo->pReaderBlock);
172,845✔
6880
  QUERY_CHECK_CODE(code, lino, _error);
173,318!
6881

6882
  pInfo->needCountEmptyTable = tsCountAlwaysReturnValue && pTableScanNode->needCountEmptyTable;
173,318✔
6883

6884
  int32_t  rowSize = pInfo->pResBlock->info.rowSize;
173,318✔
6885
  uint32_t nCols = taosArrayGetSize(pInfo->pResBlock->pDataBlock);
173,318✔
6886

6887
  pInfo->bufPageSize = getProperSortPageSize(rowSize, nCols);
173,094✔
6888

6889
  // start one reader variable
6890
  code = createOneDataBlock(pInfo->pResBlock, false, &pInfo->pSortInputBlock);
173,240✔
6891
  QUERY_CHECK_CODE(code, lino, _error);
173,367!
6892

6893
  if (!tsExperimental) {
173,367!
6894
    pInfo->filesetDelimited = false;
×
6895
  } else {
6896
    pInfo->filesetDelimited = pTableScanNode->filesetDelimited;
173,367✔
6897
  }
6898
  // end one reader variable
6899
  setOperatorInfo(pOperator, "TableMergeScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, false, OP_NOT_OPENED,
173,367✔
6900
                  pInfo, pTaskInfo);
6901
  pOperator->exprSupp.numOfExprs = numOfCols;
173,150✔
6902

6903
  pOperator->fpSet = createOperatorFpSet(
173,150✔
6904
      optrDummyOpenFn, pTableScanNode->paraTablesSort ? doTableMergeScanParaSubTablesNext : doTableMergeScanNext, NULL,
173,150✔
6905
      destroyTableMergeScanOperatorInfo, optrDefaultBufFn, getTableMergeScanExplainExecInfo, optrDefaultGetNextExtFn,
6906
      NULL);
6907
  pOperator->cost.openCost = 0;
173,049✔
6908

6909
  *pOptrInfo = pOperator;
173,049✔
6910
  return code;
173,049✔
6911

6912
_error:
×
6913
  if (code != TSDB_CODE_SUCCESS) {
×
6914
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
6915
  }
6916
  pTaskInfo->code = code;
×
6917
  if (pInfo != NULL) {
×
6918
    pInfo->base.pTableListInfo = NULL;
×
6919
    destroyTableMergeScanOperatorInfo(pInfo);
×
6920
  }
6921
  if (pOperator != NULL) {
×
6922
    pOperator->info = NULL;
×
6923
    destroyOperator(pOperator);
×
6924
  }
6925
  return code;
×
6926
}
6927

6928
// ====================================================================================================================
6929
// TableCountScanOperator
6930
static void        destoryTableCountScanOperator(void* param);
6931
static int32_t     buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp,
6932
                                                  SSDataBlock* pRes, char* dbName, tb_uid_t stbUid, SStorageAPI* pAPI);
6933
static int32_t     buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp,
6934
                                                  SSDataBlock* pRes, char* dbName, SStorageAPI* pAPI);
6935
static int32_t     buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
6936
                                             STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName);
6937
static int32_t     buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
6938
                                               STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName);
6939
static int32_t     buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
6940
                                          STableCountScanSupp* pSupp, SSDataBlock* pRes);
6941
static void        buildSysDbGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
6942
                                               STableCountScanSupp* pSupp, SSDataBlock* pRes, size_t infodbTableNum,
6943
                                               size_t perfdbTableNum);
6944
static void        buildSysDbFilterTableCount(SOperatorInfo* pOperator, STableCountScanSupp* pSupp, SSDataBlock* pRes,
6945
                                              size_t infodbTableNum, size_t perfdbTableNum);
6946
static const char* GROUP_TAG_DB_NAME = "db_name";
6947
static const char* GROUP_TAG_STABLE_NAME = "stable_name";
6948

6949
int32_t tblCountScanGetGroupTagsSlotId(const SNodeList* scanCols, STableCountScanSupp* supp) {
×
6950
  if (scanCols != NULL) {
×
6951
    SNode* pNode = NULL;
×
6952
    FOREACH(pNode, scanCols) {
×
6953
      if (nodeType(pNode) != QUERY_NODE_TARGET) {
×
6954
        return TSDB_CODE_QRY_SYS_ERROR;
×
6955
      }
6956
      STargetNode* targetNode = (STargetNode*)pNode;
×
6957
      if (nodeType(targetNode->pExpr) != QUERY_NODE_COLUMN) {
×
6958
        return TSDB_CODE_QRY_SYS_ERROR;
×
6959
      }
6960
      SColumnNode* colNode = (SColumnNode*)(targetNode->pExpr);
×
6961
      if (strcmp(colNode->colName, GROUP_TAG_DB_NAME) == 0) {
×
6962
        supp->dbNameSlotId = targetNode->slotId;
×
6963
      } else if (strcmp(colNode->colName, GROUP_TAG_STABLE_NAME) == 0) {
×
6964
        supp->stbNameSlotId = targetNode->slotId;
×
6965
      }
6966
    }
6967
  }
6968
  return TSDB_CODE_SUCCESS;
×
6969
}
6970

6971
int32_t tblCountScanGetCountSlotId(const SNodeList* pseudoCols, STableCountScanSupp* supp) {
×
6972
  if (pseudoCols != NULL) {
×
6973
    SNode* pNode = NULL;
×
6974
    FOREACH(pNode, pseudoCols) {
×
6975
      if (nodeType(pNode) != QUERY_NODE_TARGET) {
×
6976
        return TSDB_CODE_QRY_SYS_ERROR;
×
6977
      }
6978
      STargetNode* targetNode = (STargetNode*)pNode;
×
6979
      if (nodeType(targetNode->pExpr) != QUERY_NODE_FUNCTION) {
×
6980
        return TSDB_CODE_QRY_SYS_ERROR;
×
6981
      }
6982
      SFunctionNode* funcNode = (SFunctionNode*)(targetNode->pExpr);
×
6983
      if (funcNode->funcType == FUNCTION_TYPE_TABLE_COUNT) {
×
6984
        supp->tbCountSlotId = targetNode->slotId;
×
6985
      }
6986
    }
6987
  }
6988
  return TSDB_CODE_SUCCESS;
×
6989
}
6990

6991
int32_t tblCountScanGetInputs(SNodeList* groupTags, SName* tableName, STableCountScanSupp* supp) {
×
6992
  if (groupTags != NULL) {
×
6993
    SNode* pNode = NULL;
×
6994
    FOREACH(pNode, groupTags) {
×
6995
      if (nodeType(pNode) != QUERY_NODE_COLUMN) {
×
6996
        return TSDB_CODE_QRY_SYS_ERROR;
×
6997
      }
6998
      SColumnNode* colNode = (SColumnNode*)pNode;
×
6999
      if (strcmp(colNode->colName, GROUP_TAG_DB_NAME) == 0) {
×
7000
        supp->groupByDbName = true;
×
7001
      }
7002
      if (strcmp(colNode->colName, GROUP_TAG_STABLE_NAME) == 0) {
×
7003
        supp->groupByStbName = true;
×
7004
      }
7005
    }
7006
  } else {
7007
    tstrncpy(supp->dbNameFilter, tNameGetDbNameP(tableName), TSDB_DB_NAME_LEN);
×
7008
    tstrncpy(supp->stbNameFilter, tNameGetTableName(tableName), TSDB_TABLE_NAME_LEN);
×
7009
  }
7010
  return TSDB_CODE_SUCCESS;
×
7011
}
7012

7013
int32_t getTableCountScanSupp(SNodeList* groupTags, SName* tableName, SNodeList* scanCols, SNodeList* pseudoCols,
×
7014
                              STableCountScanSupp* supp, SExecTaskInfo* taskInfo) {
7015
  int32_t code = 0;
×
7016
  code = tblCountScanGetInputs(groupTags, tableName, supp);
×
7017
  if (code != TSDB_CODE_SUCCESS) {
×
7018
    qError("%s get table count scan supp. get inputs error", GET_TASKID(taskInfo));
×
7019
    return code;
×
7020
  }
7021

7022
  supp->dbNameSlotId = -1;
×
7023
  supp->stbNameSlotId = -1;
×
7024
  supp->tbCountSlotId = -1;
×
7025

7026
  code = tblCountScanGetGroupTagsSlotId(scanCols, supp);
×
7027
  if (code != TSDB_CODE_SUCCESS) {
×
7028
    qError("%s get table count scan supp. get group tags slot id error", GET_TASKID(taskInfo));
×
7029
    return code;
×
7030
  }
7031

7032
  code = tblCountScanGetCountSlotId(pseudoCols, supp);
×
7033
  if (code != TSDB_CODE_SUCCESS) {
×
7034
    qError("%s get table count scan supp. get count error", GET_TASKID(taskInfo));
×
7035
    return code;
×
7036
  }
7037
  return code;
×
7038
}
7039

7040
int32_t createTableCountScanOperatorInfo(SReadHandle* readHandle, STableCountScanPhysiNode* pTblCountScanNode,
×
7041
                                         SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) {
7042
  QRY_PARAM_CHECK(pOptrInfo);
×
7043

7044
  int32_t                      code = TSDB_CODE_SUCCESS;
×
7045
  int32_t                      lino = 0;
×
7046
  SScanPhysiNode*              pScanNode = &pTblCountScanNode->scan;
×
7047
  STableCountScanOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableCountScanOperatorInfo));
×
7048
  SOperatorInfo*               pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
×
7049
  if (!pInfo || !pOperator) {
×
7050
    code = terrno;
×
7051
    goto _error;
×
7052
  }
7053

7054
  pInfo->readHandle = *readHandle;
×
7055

7056
  SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc;
×
7057
  initResultSizeInfo(&pOperator->resultInfo, 1);
×
7058
  pInfo->pRes = createDataBlockFromDescNode(pDescNode);
×
7059
  QUERY_CHECK_NULL(pInfo->pRes, code, lino, _error, terrno);
×
7060

7061
  code = blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
×
7062
  QUERY_CHECK_CODE(code, lino, _error);
×
7063

7064
  code = getTableCountScanSupp(pTblCountScanNode->pGroupTags, &pTblCountScanNode->scan.tableName,
×
7065
                               pTblCountScanNode->scan.pScanCols, pTblCountScanNode->scan.pScanPseudoCols, &pInfo->supp,
7066
                               pTaskInfo);
7067
  QUERY_CHECK_CODE(code, lino, _error);
×
7068

7069
  setOperatorInfo(pOperator, "TableCountScanOperator", QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, false, OP_NOT_OPENED,
×
7070
                  pInfo, pTaskInfo);
7071
  pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableCountScanNext, NULL, destoryTableCountScanOperator,
×
7072
                                         optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL);
7073
  *pOptrInfo = pOperator;
×
7074
  return code;
×
7075

7076
_error:
×
7077
  if (pInfo != NULL) {
×
7078
    destoryTableCountScanOperator(pInfo);
×
7079
  }
7080
  if (pOperator != NULL) {
×
7081
    pOperator->info = NULL;
×
7082
    destroyOperator(pOperator);
×
7083
  }
7084
  pTaskInfo->code = code;
×
7085
  return code;
×
7086
}
7087

7088
int32_t fillTableCountScanDataBlock(STableCountScanSupp* pSupp, char* dbName, char* stbName, int64_t count,
×
7089
                                    SSDataBlock* pRes) {
7090
  int32_t code = TSDB_CODE_SUCCESS;
×
7091
  int32_t lino = 0;
×
7092
  if (pSupp->dbNameSlotId != -1) {
×
7093
    QUERY_CHECK_CONDITION((strlen(dbName) > 0), code, lino, _end, TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR);
×
7094
    SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->dbNameSlotId);
×
7095
    QUERY_CHECK_NULL(colInfoData, code, lino, _end, terrno);
×
7096

7097
    char varDbName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
×
7098
    tstrncpy(varDataVal(varDbName), dbName, TSDB_DB_NAME_LEN);
×
7099

7100
    varDataSetLen(varDbName, strlen(dbName));
×
7101
    code = colDataSetVal(colInfoData, 0, varDbName, false);
×
7102
    QUERY_CHECK_CODE(code, lino, _end);
×
7103
  }
7104

7105
  if (pSupp->stbNameSlotId != -1) {
×
7106
    SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->stbNameSlotId);
×
7107
    QUERY_CHECK_NULL(colInfoData, code, lino, _end, terrno);
×
7108
    if (strlen(stbName) != 0) {
×
7109
      char varStbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
×
7110
      tstrncpy(varDataVal(varStbName), stbName, TSDB_TABLE_NAME_LEN);
×
7111
      varDataSetLen(varStbName, strlen(stbName));
×
7112
      code = colDataSetVal(colInfoData, 0, varStbName, false);
×
7113
      QUERY_CHECK_CODE(code, lino, _end);
×
7114
    } else {
7115
      colDataSetNULL(colInfoData, 0);
7116
    }
7117
  }
7118

7119
  if (pSupp->tbCountSlotId != -1) {
×
7120
    SColumnInfoData* colInfoData = taosArrayGet(pRes->pDataBlock, pSupp->tbCountSlotId);
×
7121
    QUERY_CHECK_NULL(colInfoData, code, lino, _end, terrno);
×
7122
    code = colDataSetVal(colInfoData, 0, (char*)&count, false);
×
7123
    QUERY_CHECK_CODE(code, lino, _end);
×
7124
  }
7125
  pRes->info.rows = 1;
×
7126

7127
_end:
×
7128
  if (code != TSDB_CODE_SUCCESS) {
×
7129
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
7130
  }
7131
  return code;
×
7132
}
7133

7134
static SSDataBlock* buildSysDbTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo) {
×
7135
  STableCountScanSupp* pSupp = &pInfo->supp;
×
7136
  SSDataBlock*         pRes = pInfo->pRes;
×
7137

7138
  size_t infodbTableNum;
7139
  getInfosDbMeta(NULL, &infodbTableNum);
×
7140
  infodbTableNum -= 1;
×
7141
  size_t perfdbTableNum;
7142
  getPerfDbMeta(NULL, &perfdbTableNum);
×
7143

7144
  if (pSupp->groupByDbName || pSupp->groupByStbName) {
×
7145
    buildSysDbGroupedTableCount(pOperator, pInfo, pSupp, pRes, infodbTableNum, perfdbTableNum);
×
7146
    return (pRes->info.rows > 0) ? pRes : NULL;
×
7147
  } else {
7148
    buildSysDbFilterTableCount(pOperator, pSupp, pRes, infodbTableNum, perfdbTableNum);
×
7149
    return (pRes->info.rows > 0) ? pRes : NULL;
×
7150
  }
7151
}
7152

7153
static void buildSysDbFilterTableCount(SOperatorInfo* pOperator, STableCountScanSupp* pSupp, SSDataBlock* pRes,
×
7154
                                       size_t infodbTableNum, size_t perfdbTableNum) {
7155
  int32_t        code = TSDB_CODE_SUCCESS;
×
7156
  int32_t        lino = 0;
×
7157
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
×
7158
  if (strcmp(pSupp->dbNameFilter, TSDB_INFORMATION_SCHEMA_DB) == 0) {
×
7159
    code = fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes);
×
7160
    QUERY_CHECK_CODE(code, lino, _end);
×
7161
  } else if (strcmp(pSupp->dbNameFilter, TSDB_PERFORMANCE_SCHEMA_DB) == 0) {
×
7162
    code = fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes);
×
7163
    QUERY_CHECK_CODE(code, lino, _end);
×
7164
  } else if (strlen(pSupp->dbNameFilter) == 0) {
×
7165
    code = fillTableCountScanDataBlock(pSupp, "", "", infodbTableNum + perfdbTableNum, pRes);
×
7166
    QUERY_CHECK_CODE(code, lino, _end);
×
7167
  }
7168

7169
_end:
×
7170
  if (code != TSDB_CODE_SUCCESS) {
×
7171
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
7172
    pTaskInfo->code = code;
×
7173
    T_LONG_JMP(pTaskInfo->env, code);
×
7174
  }
7175
  setOperatorCompleted(pOperator);
×
7176
}
×
7177

7178
static void buildSysDbGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
×
7179
                                        STableCountScanSupp* pSupp, SSDataBlock* pRes, size_t infodbTableNum,
7180
                                        size_t perfdbTableNum) {
7181
  int32_t        code = TSDB_CODE_SUCCESS;
×
7182
  int32_t        lino = 0;
×
7183
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
×
7184
  if (pInfo->currGrpIdx == 0) {
×
7185
    uint64_t groupId = 0;
×
7186
    if (pSupp->groupByDbName) {
×
7187
      groupId = calcGroupId(TSDB_INFORMATION_SCHEMA_DB, strlen(TSDB_INFORMATION_SCHEMA_DB));
×
7188
    } else {
7189
      groupId = calcGroupId("", 0);
×
7190
    }
7191

7192
    pRes->info.id.groupId = groupId;
×
7193
    code = fillTableCountScanDataBlock(pSupp, TSDB_INFORMATION_SCHEMA_DB, "", infodbTableNum, pRes);
×
7194
    QUERY_CHECK_CODE(code, lino, _end);
×
7195
  } else if (pInfo->currGrpIdx == 1) {
×
7196
    uint64_t groupId = 0;
×
7197
    if (pSupp->groupByDbName) {
×
7198
      groupId = calcGroupId(TSDB_PERFORMANCE_SCHEMA_DB, strlen(TSDB_PERFORMANCE_SCHEMA_DB));
×
7199
    } else {
7200
      groupId = calcGroupId("", 0);
×
7201
    }
7202

7203
    pRes->info.id.groupId = groupId;
×
7204
    code = fillTableCountScanDataBlock(pSupp, TSDB_PERFORMANCE_SCHEMA_DB, "", perfdbTableNum, pRes);
×
7205
    QUERY_CHECK_CODE(code, lino, _end);
×
7206
  } else {
7207
    setOperatorCompleted(pOperator);
×
7208
  }
7209
  pInfo->currGrpIdx++;
×
7210

7211
_end:
×
7212
  if (code != TSDB_CODE_SUCCESS) {
×
7213
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
7214
    pTaskInfo->code = code;
×
7215
    T_LONG_JMP(pTaskInfo->env, code);
×
7216
  }
7217
}
×
7218

7219
static int32_t doTableCountScanNext(SOperatorInfo* pOperator, SSDataBlock** ppRes) {
×
7220
  int32_t                      code = TSDB_CODE_SUCCESS;
×
7221
  SExecTaskInfo*               pTaskInfo = pOperator->pTaskInfo;
×
7222
  STableCountScanOperatorInfo* pInfo = pOperator->info;
×
7223
  STableCountScanSupp*         pSupp = &pInfo->supp;
×
7224
  SSDataBlock*                 pRes = pInfo->pRes;
×
7225

7226
  blockDataCleanup(pRes);
×
7227
  QRY_PARAM_CHECK(ppRes);
×
7228

7229
  if (pOperator->status == OP_EXEC_DONE) {
×
7230
    return code;
×
7231
  }
7232

7233
  if (pInfo->readHandle.mnd != NULL) {
×
7234
    (*ppRes) = buildSysDbTableCount(pOperator, pInfo);
×
7235
    return code;
×
7236
  }
7237

7238
  code = buildVnodeDbTableCount(pOperator, pInfo, pSupp, pRes);
×
7239
  if (code != TSDB_CODE_SUCCESS) {
×
7240
    qError("%s failed since %s", __func__, tstrerror(code));
×
7241
    pTaskInfo->code = code;
×
7242
    T_LONG_JMP(pTaskInfo->env, code);
×
7243
  }
7244
  if (pRes->info.rows > 0) {
×
7245
    *ppRes = pRes;
×
7246
  }
7247

7248
  return code;
×
7249
}
7250

7251
static int32_t buildVnodeDbTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
×
7252
                                      STableCountScanSupp* pSupp, SSDataBlock* pRes) {
7253
  int32_t        code = TSDB_CODE_SUCCESS;
×
7254
  int32_t        lino = 0;
×
7255
  const char*    db = NULL;
×
7256
  int32_t        vgId = 0;
×
7257
  char           dbName[TSDB_DB_NAME_LEN] = {0};
×
7258
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
×
7259
  SStorageAPI*   pAPI = &pTaskInfo->storageAPI;
×
7260

7261
  // get dbname
7262
  pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, &db, &vgId, NULL, NULL);
×
7263
  SName sn = {0};
×
7264

7265
  code = tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB);
×
7266
  QUERY_CHECK_CODE(code, lino, _end);
×
7267

7268
  code = tNameGetDbName(&sn, dbName);
×
7269
  QUERY_CHECK_CODE(code, lino, _end);
×
7270

7271
  if (pSupp->groupByDbName || pSupp->groupByStbName) {
×
7272
    code = buildVnodeGroupedTableCount(pOperator, pInfo, pSupp, pRes, vgId, dbName);
×
7273
  } else {
7274
    code = buildVnodeFilteredTbCount(pOperator, pInfo, pSupp, pRes, dbName);
×
7275
  }
7276

7277
_end:
×
7278
  if (code != TSDB_CODE_SUCCESS) {
×
7279
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
7280
    pTaskInfo->code = code;
×
7281
  }
7282

7283
  return code;
×
7284
}
7285

7286
static int32_t buildVnodeGroupedTableCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
×
7287
                                           STableCountScanSupp* pSupp, SSDataBlock* pRes, int32_t vgId, char* dbName) {
7288
  int32_t        code = TSDB_CODE_SUCCESS;
×
7289
  int32_t        lino = 0;
×
7290
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
×
7291
  SStorageAPI*   pAPI = &pTaskInfo->storageAPI;
×
7292

7293
  if (pSupp->groupByStbName) {
×
7294
    if (pInfo->stbUidList == NULL) {
×
7295
      pInfo->stbUidList = taosArrayInit(16, sizeof(tb_uid_t));
×
7296
      QUERY_CHECK_NULL(pInfo->stbUidList, code, lino, _end, terrno);
×
7297
      code = pAPI->metaFn.storeGetTableList(pInfo->readHandle.vnode, TSDB_SUPER_TABLE, pInfo->stbUidList);
×
7298
      QUERY_CHECK_CODE(code, lino, _end);
×
7299
    }
7300

7301
    if (pInfo->currGrpIdx < taosArrayGetSize(pInfo->stbUidList)) {
×
7302
      tb_uid_t stbUid = *(tb_uid_t*)taosArrayGet(pInfo->stbUidList, pInfo->currGrpIdx);
×
7303
      code = buildVnodeGroupedStbTableCount(pInfo, pSupp, pRes, dbName, stbUid, pAPI);
×
7304
      QUERY_CHECK_CODE(code, lino, _end);
×
7305

7306
      pInfo->currGrpIdx++;
×
7307
    } else if (pInfo->currGrpIdx == taosArrayGetSize(pInfo->stbUidList)) {
×
7308
      code = buildVnodeGroupedNtbTableCount(pInfo, pSupp, pRes, dbName, pAPI);
×
7309
      QUERY_CHECK_CODE(code, lino, _end);
×
7310

7311
      pInfo->currGrpIdx++;
×
7312
    } else {
7313
      setOperatorCompleted(pOperator);
×
7314
    }
7315
  } else {
7316
    uint64_t groupId = calcGroupId(dbName, strlen(dbName));
×
7317
    pRes->info.id.groupId = groupId;
×
7318

7319
    int64_t dbTableCount = 0;
×
7320
    pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, &dbTableCount, NULL);
×
7321
    code = fillTableCountScanDataBlock(pSupp, dbName, "", dbTableCount, pRes);
×
7322
    QUERY_CHECK_CODE(code, lino, _end);
×
7323
    setOperatorCompleted(pOperator);
×
7324
  }
7325

7326
_end:
×
7327
  if (code != TSDB_CODE_SUCCESS) {
×
7328
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
7329
    pTaskInfo->code = code;
×
7330
  }
7331
  return code;
×
7332
}
7333

7334
static int32_t buildVnodeFilteredTbCount(SOperatorInfo* pOperator, STableCountScanOperatorInfo* pInfo,
×
7335
                                         STableCountScanSupp* pSupp, SSDataBlock* pRes, char* dbName) {
7336
  int32_t        code = TSDB_CODE_SUCCESS;
×
7337
  int32_t        lino = 0;
×
7338
  SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
×
7339
  SStorageAPI*   pAPI = &pTaskInfo->storageAPI;
×
7340

7341
  if (strlen(pSupp->dbNameFilter) != 0) {
×
7342
    if (strlen(pSupp->stbNameFilter) != 0) {
×
7343
      uint64_t uid = 0;
×
7344
      code = pAPI->metaFn.getTableUidByName(pInfo->readHandle.vnode, pSupp->stbNameFilter, &uid);
×
7345
      QUERY_CHECK_CODE(code, lino, _end);
×
7346

7347
      int64_t numOfChildTables = 0;
×
7348
      code = pAPI->metaFn.getNumOfChildTables(pInfo->readHandle.vnode, uid, &numOfChildTables, NULL, NULL);
×
7349
      QUERY_CHECK_CODE(code, lino, _end);
×
7350

7351
      code = fillTableCountScanDataBlock(pSupp, dbName, pSupp->stbNameFilter, numOfChildTables, pRes);
×
7352
      QUERY_CHECK_CODE(code, lino, _end);
×
7353
    } else {
7354
      int64_t tbNumVnode = 0;
×
7355
      pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, &tbNumVnode, NULL);
×
7356
      code = fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes);
×
7357
      QUERY_CHECK_CODE(code, lino, _end);
×
7358
    }
7359
  } else {
7360
    int64_t tbNumVnode = 0;
×
7361
    pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, &tbNumVnode, NULL);
×
7362
    code = fillTableCountScanDataBlock(pSupp, dbName, "", tbNumVnode, pRes);
×
7363
    QUERY_CHECK_CODE(code, lino, _end);
×
7364
  }
7365

7366
_end:
×
7367
  if (code != TSDB_CODE_SUCCESS) {
×
7368
    pTaskInfo->code = code;
×
7369
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
7370
  }
7371

7372
  setOperatorCompleted(pOperator);
×
7373
  return code;
×
7374
}
7375

7376
static int32_t buildVnodeGroupedNtbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp,
×
7377
                                              SSDataBlock* pRes, char* dbName, SStorageAPI* pAPI) {
7378
  int32_t code = TSDB_CODE_SUCCESS;
×
7379
  int32_t lino = 0;
×
7380
  char    fullStbName[TSDB_TABLE_FNAME_LEN] = {0};
×
7381
  if (pSupp->groupByDbName) {
×
7382
    snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, "");
×
7383
  }
7384

7385
  uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName));
×
7386
  pRes->info.id.groupId = groupId;
×
7387

7388
  int64_t numOfTables = 0;
×
7389
  pAPI->metaFn.getBasicInfo(pInfo->readHandle.vnode, NULL, NULL, NULL, &numOfTables);
×
7390

7391
  if (numOfTables != 0) {
×
7392
    code = fillTableCountScanDataBlock(pSupp, dbName, "", numOfTables, pRes);
×
7393
  }
7394

7395
_end:
×
7396
  if (code != TSDB_CODE_SUCCESS) {
×
7397
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
7398
  }
7399
  return code;
×
7400
}
7401

7402
static int32_t buildVnodeGroupedStbTableCount(STableCountScanOperatorInfo* pInfo, STableCountScanSupp* pSupp,
×
7403
                                              SSDataBlock* pRes, char* dbName, tb_uid_t stbUid, SStorageAPI* pAPI) {
7404
  int32_t code = TSDB_CODE_SUCCESS;
×
7405
  int32_t lino = 0;
×
7406
  char    stbName[TSDB_TABLE_NAME_LEN] = {0};
×
7407
  code = pAPI->metaFn.getTableNameByUid(pInfo->readHandle.vnode, stbUid, stbName);
×
7408
  QUERY_CHECK_CODE(code, lino, _end);
×
7409

7410
  char fullStbName[TSDB_TABLE_FNAME_LEN] = {0};
×
7411
  if (pSupp->groupByDbName) {
×
7412
    (void)snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s.%s", dbName, varDataVal(stbName));
×
7413
  } else {
7414
    (void)snprintf(fullStbName, TSDB_TABLE_FNAME_LEN, "%s", varDataVal(stbName));
×
7415
  }
7416

7417
  uint64_t groupId = calcGroupId(fullStbName, strlen(fullStbName));
×
7418
  pRes->info.id.groupId = groupId;
×
7419

7420
  int64_t ctbNum = 0;
×
7421
  code = pAPI->metaFn.getNumOfChildTables(pInfo->readHandle.vnode, stbUid, &ctbNum, NULL, NULL);
×
7422
  QUERY_CHECK_CODE(code, lino, _end);
×
7423
  code = fillTableCountScanDataBlock(pSupp, dbName, varDataVal(stbName), ctbNum, pRes);
×
7424
  QUERY_CHECK_CODE(code, lino, _end);
×
7425

7426
_end:
×
7427
  if (code != TSDB_CODE_SUCCESS) {
×
7428
    qError("%s failed at line %d since %s", __func__, lino, tstrerror(code));
×
7429
  }
7430
  return code;
×
7431
}
7432

7433
static void destoryTableCountScanOperator(void* param) {
×
7434
  STableCountScanOperatorInfo* pTableCountScanInfo = param;
×
7435
  blockDataDestroy(pTableCountScanInfo->pRes);
×
7436

7437
  taosArrayDestroy(pTableCountScanInfo->stbUidList);
×
7438
  taosMemoryFreeClear(param);
×
7439
}
×
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc