• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

taosdata / TDengine / #5049

11 May 2026 06:30AM UTC coverage: 73.313% (+0.09%) from 73.222%
#5049

push

travis-ci

web-flow
feat: refactor taosdump code to improve backup speed and compression ratio (#35292)

6625 of 8435 new or added lines in 28 files covered. (78.54%)

2491 existing lines in 142 files now uncovered.

281233 of 383605 relevant lines covered (73.31%)

132489999.79 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

69.18
/source/libs/executor/src/dataDispatcher.c
1
/*
2
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3
 *
4
 * This program is free software: you can use, redistribute, and/or modify
5
 * it under the terms of the GNU Affero General Public License, version 3
6
 * or later ("AGPL"), as published by the Free Software Foundation.
7
 *
8
 * This program is distributed in the hope that it will be useful, but WITHOUT
9
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10
 * FITNESS FOR A PARTICULAR PURPOSE.
11
 *
12
 * You should have received a copy of the GNU Affero General Public License
13
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
14
 */
15

16
#include "dataSinkInt.h"
17
#include "dataSinkMgt.h"
18
#include "executorInt.h"
19
#include "planner.h"
20
#include "tcompression.h"
21
#include "tdatablock.h"
22
#include "tglobal.h"
23
#include "tqueue.h"
24

25
extern SDataSinkStat gDataSinkStat;
26

27
typedef struct SDataDispatchBuf {
28
  int32_t useSize;
29
  int32_t allocSize;
30
  char*   pData;
31
} SDataDispatchBuf;
32

33
typedef struct SDataCacheEntry {
34
  int32_t rawLen;
35
  int32_t dataLen;
36
  int32_t numOfRows;
37
  int32_t numOfCols;
38
  int8_t  compressed;
39
  char    data[];
40
} SDataCacheEntry;
41

42
typedef struct SDataDispatchHandle {
43
  SDataSinkHandle     sink;
44
  SDataSinkManager*   pManager;
45
  SDataBlockDescNode* pSchema;
46
  SDataSinkNode*      pSinkNode;
47
  STaosQueue*         pDataBlocks;
48
  SDataDispatchBuf    nextOutput;
49
  int32_t             outPutColCounts;
50
  int32_t             status;
51
  bool                queryEnd;
52
  uint64_t            useconds;
53
  uint64_t            cachedSize;
54
  uint64_t            flags;
55
  void*               pCompressBuf;
56
  int32_t             bufSize;
57
  bool                dynamicSchema;
58
  TdThreadMutex       mutex;
59
} SDataDispatchHandle;
60

61
static int32_t inputSafetyCheck(SDataDispatchHandle* pHandle, const SInputData* pInput)  {
836,075,549✔
62
  if(tsSafetyCheckLevel == TSDB_SAFETY_CHECK_LEVELL_NEVER) {
836,075,549✔
63
    return TSDB_CODE_SUCCESS;
×
64
  }
65
  if (pInput == NULL || pInput->pData == NULL || pInput->pData->info.rows <= 0) {
836,075,549✔
66
    qError("invalid input data");
18,276✔
67
    return TSDB_CODE_QRY_INVALID_INPUT;
×
68
  }
69
  SDataBlockDescNode* pSchema = pHandle->pSchema;
836,046,551✔
70
  if (pSchema == NULL || pSchema->totalRowSize != pInput->pData->info.rowSize) {
836,011,678✔
UNCOV
71
    qError("invalid schema");
×
72
    return TSDB_CODE_QRY_INVALID_INPUT;
×
73
  }
74

75
  if (pHandle->outPutColCounts > taosArrayGetSize(pInput->pData->pDataBlock)) {
836,057,504✔
76
    qError("invalid column number, schema:%d, input:%zu", pHandle->outPutColCounts, taosArrayGetSize(pInput->pData->pDataBlock));
×
77
    return TSDB_CODE_QRY_INVALID_INPUT;
×
78
  }
79

80
  SNode*  pNode;
81
  int32_t colNum = 0;
836,025,561✔
82
  FOREACH(pNode, pHandle->pSchema->pSlots) {
2,147,483,647✔
83
    SSlotDescNode* pSlotDesc = (SSlotDescNode*)pNode;
2,147,483,647✔
84
    if (pSlotDesc->output) {
2,147,483,647✔
85
      SColumnInfoData* pColInfoData = taosArrayGet(pInput->pData->pDataBlock, colNum);
2,147,483,647✔
86
      if (pColInfoData == NULL) {
2,147,483,647✔
87
        return -1;
×
88
      }
89
      if (pColInfoData->info.bytes < 0) {
2,147,483,647✔
90
        qError("invalid column bytes, schema:%d, input:%d", pSlotDesc->dataType.bytes, pColInfoData->info.bytes);
×
91
        return TSDB_CODE_TSC_INTERNAL_ERROR;
×
92
      }
93
      if (!IS_VAR_DATA_TYPE(pColInfoData->info.type) &&
2,147,483,647✔
94
          TYPE_BYTES[pColInfoData->info.type] != pColInfoData->info.bytes) {
2,147,483,647✔
95
        qError("invalid column bytes, schema:%d, input:%d", TYPE_BYTES[pColInfoData->info.type],
×
96
               pColInfoData->info.bytes);
97
        return TSDB_CODE_TSC_INTERNAL_ERROR;
×
98
      }
99
      if (pColInfoData->info.type != pSlotDesc->dataType.type) {
2,147,483,647✔
100
        qError("invalid column type, schema:%d, input:%d", pSlotDesc->dataType.type, pColInfoData->info.type);
×
101
        return TSDB_CODE_QRY_INVALID_INPUT;
×
102
      }
103
      if (pColInfoData->info.bytes != pSlotDesc->dataType.bytes) {
2,147,483,647✔
104
        qError("invalid column bytes, schema:%d, input:%d", pSlotDesc->dataType.bytes, pColInfoData->info.bytes);
×
105
        return TSDB_CODE_QRY_INVALID_INPUT;
×
106
      }
107

108
      if (IS_INVALID_TYPE(pColInfoData->info.type)) {
2,147,483,647✔
109
        qError("invalid column type, type:%d", pColInfoData->info.type);
111,061✔
110
        return TSDB_CODE_TSC_INTERNAL_ERROR;
×
111
      }
112
      ++colNum;
2,147,483,647✔
113
    }
114
  }
115

116

117
  return TSDB_CODE_SUCCESS;
836,052,568✔
118
}
119

120
static int32_t inputSafetyCheckForDynamicSchema(const SInputData* pInput)  {
31,796,679✔
121
  if(tsSafetyCheckLevel == TSDB_SAFETY_CHECK_LEVELL_NEVER) {
31,796,679✔
122
    return TSDB_CODE_SUCCESS;
×
123
  }
124
  if (pInput == NULL || pInput->pData == NULL || pInput->pData->info.rows <= 0) {
31,796,679✔
125
    qError("invalid input data");
×
126
    return TSDB_CODE_QRY_INVALID_INPUT;
×
127
  }
128

129
  for (int32_t colNum = 0; colNum < taosArrayGetSize(pInput->pData->pDataBlock); colNum++) {
116,305,818✔
130
    SColumnInfoData* pColInfoData = taosArrayGet(pInput->pData->pDataBlock, colNum);
84,509,139✔
131
    if (pColInfoData == NULL) {
84,509,139✔
132
      return -1;
×
133
    }
134
    if (pColInfoData->info.bytes < 0) {
84,509,139✔
135
      qError("invalid column bytes, input:%d", pColInfoData->info.bytes);
×
136
      return TSDB_CODE_TSC_INTERNAL_ERROR;
×
137
    }
138
    if (!IS_VAR_DATA_TYPE(pColInfoData->info.type) &&
84,509,139✔
139
        TYPE_BYTES[pColInfoData->info.type] != pColInfoData->info.bytes) {
70,748,137✔
140
      qError("invalid column bytes, schema:%d, input:%d", TYPE_BYTES[pColInfoData->info.type],
×
141
             pColInfoData->info.bytes);
142
      return TSDB_CODE_TSC_INTERNAL_ERROR;
×
143
    }
144

145
    if (IS_INVALID_TYPE(pColInfoData->info.type)) {
84,509,139✔
146
      qError("invalid column type, type:%d", pColInfoData->info.type);
×
147
      return TSDB_CODE_TSC_INTERNAL_ERROR;
×
148
    }
149
  }
150

151
  return TSDB_CODE_SUCCESS;
31,796,679✔
152
}
153

154
// clang-format off
155
// data format:
156
// +----------------+------------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+
157
// |SDataCacheEntry |  version         | total length | numOfRows    |     group id     | col1_schema | col2_schema | col3_schema... | column#1 length, column#2 length...| col1 bitmap | col1 data | col2 bitmap | col2 data |
158
// |                |  sizeof(int32_t) |sizeof(int32) | sizeof(int32)| sizeof(uint64_t) | (sizeof(int8_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols        | actual size |           |                         |
159
// +----------------+------------------+--------------+--------------+------------------+--------------------------------------------+------------------------------------+-------------+-----------+-------------+-----------+
160
// The length of bitmap is decided by number of rows of this data block, and the length of each column data is
161
// recorded in the first segment, next to the struct header
162
// clang-format on
163
static int32_t toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) {
867,878,376✔
164
  int32_t numOfCols = 0;
867,878,376✔
165
  SNode*  pNode;
166
  int32_t code = TSDB_CODE_SUCCESS;
867,878,376✔
167

168
  if (pHandle->dynamicSchema) {
867,878,376✔
169
    code = inputSafetyCheckForDynamicSchema(pInput);
31,796,679✔
170
    if (code) {
31,796,679✔
171
      qError("failed to check input data for dynamic schema, code:%d", code);
×
172
      return code;
×
173
    }
174
    numOfCols = taosArrayGetSize(pInput->pData->pDataBlock);
31,796,679✔
175
  } else {
176
    code = inputSafetyCheck(pHandle, pInput);
836,085,698✔
177
    if (code) {
836,046,089✔
178
      qError("failed to check input data, code:%d", code);
×
179
      return code;
×
180
    }
181

182
    FOREACH(pNode, pHandle->pSchema->pSlots) {
2,147,483,647✔
183
      SSlotDescNode* pSlotDesc = (SSlotDescNode*)pNode;
2,147,483,647✔
184
      if (pSlotDesc->output) {
2,147,483,647✔
185
        ++numOfCols;
2,147,483,647✔
186
      }
187
    }
188
  }
189

190
  SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData;
867,786,378✔
191
  pEntry->compressed = 0;
867,788,528✔
192
  pEntry->numOfRows = pInput->pData->info.rows;
867,795,651✔
193
  pEntry->numOfCols = numOfCols;
867,821,886✔
194
  pEntry->dataLen = 0;
867,834,890✔
195
  pEntry->rawLen = 0;
867,793,159✔
196

197
  pBuf->useSize = sizeof(SDataCacheEntry);
867,814,327✔
198

199
  {
200
    // allocate additional 8 bytes to avoid invalid write if compress failed to reduce the size
201
    size_t dataEncodeBufSize = pBuf->allocSize + 8;
867,835,707✔
202
    if ((pBuf->allocSize > tsCompressMsgSize) && (tsCompressMsgSize > 0) && pHandle->pManager->cfg.compress) {
867,824,331✔
203
      if (pHandle->pCompressBuf == NULL) {
×
204
        pHandle->pCompressBuf = taosMemoryMalloc(dataEncodeBufSize);
×
205
        if (NULL == pHandle->pCompressBuf) {
×
206
          QRY_RET(terrno);
×
207
        }
208
        pHandle->bufSize = dataEncodeBufSize;
×
209
      } else {
210
        if (pHandle->bufSize < dataEncodeBufSize) {
×
211
          pHandle->bufSize = dataEncodeBufSize;
×
212
          void* p = taosMemoryRealloc(pHandle->pCompressBuf, pHandle->bufSize);
×
213
          if (p != NULL) {
×
214
            pHandle->pCompressBuf = p;
×
215
          } else {
216
            qError("failed to prepare compress buf:%d, code: %x", pHandle->bufSize, terrno);
×
217
            return terrno;
×
218
          }
219
        }
220
      }
221

222
      int32_t dataLen = blockEncode(pInput->pData, pHandle->pCompressBuf, dataEncodeBufSize, numOfCols);
×
223
      if(dataLen < 0) {
×
224
        qError("failed to encode data block, code: %d", dataLen);
×
225
        return terrno;
×
226
      }
227
      int32_t len =
228
          tsCompressString(pHandle->pCompressBuf, dataLen, 1, pEntry->data, pBuf->allocSize, ONE_STAGE_COMP, NULL, 0);
×
229
      if (len < dataLen) {
×
230
        pEntry->compressed = 1;
×
231
        pEntry->dataLen = len;
×
232
        pEntry->rawLen = dataLen;
×
233
      } else {  // no need to compress data
234
        pEntry->compressed = 0;
×
235
        pEntry->dataLen = dataLen;
×
236
        pEntry->rawLen = dataLen;
×
237
        TAOS_MEMCPY(pEntry->data, pHandle->pCompressBuf, dataLen);
×
238
      }
239
    } else {
240
      pEntry->dataLen =
867,756,857✔
241
          pHandle->dynamicSchema
867,819,830✔
242
              ? blockEncodeInternal(pInput->pData, pEntry->data,  pBuf->allocSize, numOfCols)
31,796,679✔
243
              : blockEncode(pInput->pData, pEntry->data,  pBuf->allocSize, numOfCols);
867,783,426✔
244
      if(pEntry->dataLen < 0) {
867,772,802✔
245
        qError("failed to encode data block, code: %d", pEntry->dataLen);
×
246
        return terrno;
×
247
      }
248
      pEntry->rawLen = pEntry->dataLen;
867,759,705✔
249
    }
250
  }
251

252
  pBuf->useSize += pEntry->dataLen;
867,792,576✔
253

254
  (void)atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen);
867,793,340✔
255
  (void)atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
867,870,741✔
256

257
  return TSDB_CODE_SUCCESS;
867,784,796✔
258
}
259

260
static int32_t allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, SDataDispatchBuf* pBuf) {
867,880,000✔
261
  /*
262
    uint32_t capacity = pDispatcher->pManager->cfg.maxDataBlockNumPerQuery;
263
    if (taosQueueItemSize(pDispatcher->pDataBlocks) > capacity) {
264
      qError("SinkNode queue is full, no capacity, max:%d, current:%d, no capacity", capacity,
265
             taosQueueItemSize(pDispatcher->pDataBlocks));
266
      return false;
267
    }
268
  */
269

270
  pBuf->allocSize = sizeof(SDataCacheEntry) + (pDispatcher->dynamicSchema
1,735,743,265✔
271
                                                   ? blockGetInternalEncodeSize(pInput->pData)
31,796,679✔
272
                                                   : blockGetEncodeSize(pInput->pData));
836,087,993✔
273

274
  pBuf->pData = taosMemoryMalloc(pBuf->allocSize);
867,874,310✔
275
  if (pBuf->pData == NULL) {
867,851,624✔
276
    qError("SinkNode failed to malloc memory, size:%d, code:%x", pBuf->allocSize, terrno);
×
277
    return terrno;
×
278
  }
279

280
  return TSDB_CODE_SUCCESS;
867,873,066✔
281
}
282

283
static int32_t updateStatus(SDataDispatchHandle* pDispatcher) {
1,726,128,069✔
284
  (void)taosThreadMutexLock(&pDispatcher->mutex);
1,726,128,069✔
285
  int32_t blockNums = taosQueueItemSize(pDispatcher->pDataBlocks);
1,726,242,338✔
286
  int32_t status =
1,726,179,046✔
287
      (0 == blockNums ? DS_BUF_EMPTY
288
                      : (blockNums < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL));
1,726,158,957✔
289
  pDispatcher->status = status;
1,726,179,046✔
290
  (void)taosThreadMutexUnlock(&pDispatcher->mutex);
1,726,182,311✔
291
  return status;
1,726,250,503✔
292
}
293

294
static int32_t getStatus(SDataDispatchHandle* pDispatcher) {
×
295
  (void)taosThreadMutexLock(&pDispatcher->mutex);
×
296
  int32_t status = pDispatcher->status;
×
297
  (void)taosThreadMutexUnlock(&pDispatcher->mutex);
×
298
  return status;
×
299
}
300

301
static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) {
867,882,593✔
302
  int32_t              code = 0;
867,882,593✔
303
  SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
867,882,593✔
304
  SDataDispatchBuf*    pBuf = NULL;
867,882,593✔
305

306
  code = taosAllocateQitem(sizeof(SDataDispatchBuf), DEF_QITEM, 0, (void**)&pBuf);
867,895,965✔
307
  if (code) {
867,889,475✔
308
    return code;
×
309
  }
310

311
  code = allocBuf(pDispatcher, pInput, pBuf);
867,889,475✔
312
  if (code) {
867,885,895✔
313
    taosFreeQitem(pBuf);
×
314
    return code;
×
315
  }
316

317
  QRY_ERR_JRET(toDataCacheEntry(pDispatcher, pInput, pBuf));
867,885,895✔
318
  QRY_ERR_JRET(taosWriteQitem(pDispatcher->pDataBlocks, pBuf));
867,784,757✔
319

320
  int32_t status = updateStatus(pDispatcher);
867,848,803✔
321
  *pContinue = (status == DS_BUF_LOW || status == DS_BUF_EMPTY) && !(pDispatcher->flags & DS_FLAG_PROCESS_ONE_BLOCK);
867,896,123✔
322
  return TSDB_CODE_SUCCESS;
867,870,482✔
323

324
_return:
×
325

326
  taosMemoryFreeClear(pBuf->pData);
×
327
  taosFreeQitem(pBuf);
×
328
  return code;
×
329
}
330

331
static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) {
384,738,683✔
332
  SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
384,738,683✔
333
  (void)taosThreadMutexLock(&pDispatcher->mutex);
384,738,683✔
334
  pDispatcher->queryEnd = true;
384,754,965✔
335
  pDispatcher->useconds = useconds;
384,760,179✔
336
  (void)taosThreadMutexUnlock(&pDispatcher->mutex);
384,729,997✔
337
}
384,729,694✔
338

339
static void resetDispatcher(struct SDataSinkHandle* pHandle) {
47,642,331✔
340
  SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
47,642,331✔
341
  (void)taosThreadMutexLock(&pDispatcher->mutex);
47,642,331✔
342
  pDispatcher->queryEnd = false;
47,644,877✔
343
  (void)taosThreadMutexUnlock(&pDispatcher->mutex);
47,642,822✔
344
}
47,643,445✔
345

346
static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRowLen, bool* pQueryEnd) {
1,008,671,226✔
347
  SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
1,008,671,226✔
348
  if (taosQueueEmpty(pDispatcher->pDataBlocks)) {
1,008,671,226✔
349
    *pQueryEnd = pDispatcher->queryEnd;
150,417,919✔
350
    *pLen = 0;
150,417,999✔
351
    return;
150,414,791✔
352
  }
353

354
  SDataDispatchBuf* pBuf = NULL;
858,303,442✔
355
  taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf);
858,307,814✔
356
  if (pBuf != NULL) {
858,352,708✔
357
    TAOS_MEMCPY(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf));
858,350,908✔
358
    taosFreeQitem(pBuf);
858,298,586✔
359
  }
360

361
  SDataCacheEntry* pEntry = (SDataCacheEntry*)pDispatcher->nextOutput.pData;
858,284,047✔
362
  *pLen = pEntry->dataLen;
858,295,089✔
363
  *pRowLen = pEntry->rawLen;
858,331,658✔
364

365
  *pQueryEnd = pDispatcher->queryEnd;
858,333,891✔
366
  qDebug("got data len %" PRId64 ", row num %d in sink", *pLen,
858,327,864✔
367
         ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->numOfRows);
368
}
369

370
static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) {
953,288,067✔
371
  SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
953,288,067✔
372
  if (NULL == pDispatcher->nextOutput.pData) {
953,288,067✔
373
    if (!pDispatcher->queryEnd) {
94,995,943✔
374
      qError("empty res while query not end in data dispatcher");
×
375
      return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
×
376
    }
377

378
    pOutput->useconds = pDispatcher->useconds;
94,995,416✔
379
    pOutput->precision = pDispatcher->pSchema->precision;
94,996,748✔
380
    pOutput->bufStatus = DS_BUF_EMPTY;
94,996,077✔
381
    pOutput->queryEnd = pDispatcher->queryEnd;
94,998,621✔
382
    return TSDB_CODE_SUCCESS;
94,997,864✔
383
  }
384

385
  SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData);
858,278,691✔
386
  TAOS_MEMCPY(pOutput->pData, pEntry->data, pEntry->dataLen);
858,296,927✔
387
  pOutput->numOfRows = pEntry->numOfRows;
858,317,863✔
388
  pOutput->numOfCols = pEntry->numOfCols;
858,317,388✔
389
  pOutput->compressed = pEntry->compressed;
858,290,806✔
390

391
  (void)atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen);
858,310,268✔
392
  (void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen);
858,338,663✔
393

394
  taosMemoryFreeClear(pDispatcher->nextOutput.pData);  // todo persistent
858,341,326✔
395
  pOutput->bufStatus = updateStatus(pDispatcher);
858,320,382✔
396
  
397
  (void)taosThreadMutexLock(&pDispatcher->mutex);
858,356,046✔
398
  pOutput->queryEnd = pDispatcher->queryEnd;
858,283,835✔
399
  pOutput->useconds = pDispatcher->useconds;
858,318,866✔
400
  pOutput->precision = pDispatcher->pSchema->precision;
858,268,936✔
401
  (void)taosThreadMutexUnlock(&pDispatcher->mutex);
858,342,412✔
402

403
  return TSDB_CODE_SUCCESS;
858,354,552✔
404
}
405

406
static int32_t destroyDataSinker(SDataSinkHandle* pHandle) {
389,976,356✔
407
  SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
389,976,356✔
408
  (void)atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDispatcher->cachedSize);
389,976,356✔
409
  taosMemoryFreeClear(pDispatcher->nextOutput.pData);
390,013,905✔
410
  nodesDestroyNode((SNode*)pDispatcher->pSinkNode);
390,015,590✔
411

412
  while (!taosQueueEmpty(pDispatcher->pDataBlocks)) {
399,529,450✔
413
    SDataDispatchBuf* pBuf = NULL;
9,538,435✔
414
    taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf);
9,538,435✔
415
    if (pBuf != NULL) {
9,538,435✔
416
      taosMemoryFreeClear(pBuf->pData);
9,538,435✔
417
      taosFreeQitem(pBuf);
9,537,832✔
418
    }
419
  }
420

421
  taosCloseQueue(pDispatcher->pDataBlocks);
390,039,320✔
422
  taosMemoryFreeClear(pDispatcher->pCompressBuf);
389,924,016✔
423
  pDispatcher->bufSize = 0;
389,951,062✔
424

425
  (void)taosThreadMutexDestroy(&pDispatcher->mutex);
389,888,500✔
426
  taosMemoryFree(pDispatcher->pManager);
389,914,797✔
427
  return TSDB_CODE_SUCCESS;
389,805,061✔
428
}
429

430
static int32_t getCacheSize(struct SDataSinkHandle* pHandle, uint64_t* size) {
×
431
  SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
×
432

433
  *size = atomic_load_64(&pDispatcher->cachedSize);
×
434
  return TSDB_CODE_SUCCESS;
×
435
}
436

437
static int32_t getSinkFlags(struct SDataSinkHandle* pHandle, uint64_t* pFlags) {
453,244,232✔
438
  SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle;
453,244,232✔
439

440
  *pFlags = atomic_load_64(&pDispatcher->flags);
453,244,232✔
441
  return TSDB_CODE_SUCCESS;
453,222,837✔
442
}
443

444
static int32_t blockDescNodeCheck(SDataBlockDescNode* pInputDataBlockDesc)  {
389,786,859✔
445
  if(tsSafetyCheckLevel == TSDB_SAFETY_CHECK_LEVELL_NEVER) {
389,786,859✔
446
    return TSDB_CODE_SUCCESS;
×
447
  }
448

449
  if (pInputDataBlockDesc == NULL) {
389,786,859✔
450
    qError("invalid schema");
×
451
    return TSDB_CODE_QRY_INVALID_INPUT;
×
452
  }
453

454
  SNode*  pNode;
455
  int32_t realOutputRowSize = 0;
389,786,859✔
456
  FOREACH(pNode, pInputDataBlockDesc->pSlots) {
2,058,617,888✔
457
    SSlotDescNode* pSlotDesc = (SSlotDescNode*)pNode;
1,689,624,562✔
458
    if (pSlotDesc->output) {
1,689,624,562✔
459
      realOutputRowSize += pSlotDesc->dataType.bytes;
1,668,726,169✔
460
    } else {
461
      // Slots must be sorted, and slots with 'output' set to true must come first
462
      break;
20,738,378✔
463
    }
464
  }
465
  if (realOutputRowSize !=  pInputDataBlockDesc->outputRowSize) {
389,960,651✔
466
    qError("invalid schema, realOutputRowSize:%d, outputRowSize:%d", realOutputRowSize, pInputDataBlockDesc->outputRowSize);
×
467
    return TSDB_CODE_QRY_INVALID_INPUT;
×
468
  }
469
  return TSDB_CODE_SUCCESS;
389,596,025✔
470
}
471

472
int32_t getOutputColCounts(SDataBlockDescNode* pInputDataBlockDesc) {
389,719,207✔
473
  if (pInputDataBlockDesc == NULL) {
389,719,207✔
474
    qError("invalid schema");
×
475
    return 0;
×
476
  }
477
  SNode*  pNode;
478
  int32_t numOfCols = 0;
389,719,207✔
479
  FOREACH(pNode, pInputDataBlockDesc->pSlots) {
2,058,073,710✔
480
    SSlotDescNode* pSlotDesc = (SSlotDescNode*)pNode;
1,689,194,730✔
481
    if (pSlotDesc->output) {
1,689,194,730✔
482
      ++numOfCols;
1,668,354,503✔
483
    } else {
484
      // Slots must be sorted, and slots with 'output' set to true must come first
485
      break;
20,669,170✔
486
    }
487
  }
488
  return numOfCols;
389,415,792✔
489
}
490

491
int32_t createDataDispatcher(SDataSinkManager* pManager, SDataSinkNode** ppDataSink, DataSinkHandle* pHandle, bool processOneBlock) {
389,772,458✔
492
  int32_t code;
493
  SDataSinkNode* pDataSink = *ppDataSink;
389,772,458✔
494
  code = blockDescNodeCheck(pDataSink->pInputDataBlockDesc);
389,869,512✔
495
  if (code) {
389,533,962✔
496
    qError("failed to check input data block desc, code:%d", code);
×
497
    goto _return;
×
498
  }
499

500
  SDataDispatcherNode* pDispatherNode = (SDataDispatcherNode*)pDataSink;
389,533,962✔
501
  SDataDispatchHandle* dispatcher = taosMemoryCalloc(1, sizeof(SDataDispatchHandle));
389,533,962✔
502
  if (NULL == dispatcher) {
389,369,877✔
503
    goto _return;
×
504
  }
505

506
  dispatcher->sink.fPut = putDataBlock;
389,369,877✔
507
  dispatcher->sink.fEndPut = endPut;
389,387,337✔
508
  dispatcher->sink.fReset = resetDispatcher;
389,507,341✔
509
  dispatcher->sink.fGetLen = getDataLength;
389,553,238✔
510
  dispatcher->sink.fGetData = getDataBlock;
389,589,485✔
511
  dispatcher->sink.fDestroy = destroyDataSinker;
389,460,274✔
512
  dispatcher->sink.fGetCacheSize = getCacheSize;
389,649,115✔
513
  dispatcher->sink.fGetFlags = getSinkFlags;
389,599,262✔
514
  dispatcher->pManager = pManager;
389,459,453✔
515
  pManager = NULL;
389,470,947✔
516
  dispatcher->dynamicSchema = pDispatherNode->dynamicSchema;
389,470,947✔
517
  dispatcher->pSchema = pDataSink->pInputDataBlockDesc;
389,582,023✔
518
  dispatcher->pSinkNode = pDataSink;
389,464,234✔
519
  *ppDataSink = NULL;
389,549,447✔
520
  dispatcher->outPutColCounts = getOutputColCounts(dispatcher->pSchema);
389,590,696✔
521
  dispatcher->status = DS_BUF_EMPTY;
389,379,377✔
522
  dispatcher->queryEnd = false;
389,555,978✔
523
  code = taosOpenQueue(&dispatcher->pDataBlocks);
389,606,454✔
524
  if (code) {
389,689,210✔
525
    terrno = code;
×
526
    goto _return;
×
527
  }
528
  code = taosThreadMutexInit(&dispatcher->mutex, NULL);
389,689,210✔
529
  if (code) {
389,527,196✔
530
    terrno = code;
×
531
    goto _return;
×
532
  }
533

534
  dispatcher->flags = DS_FLAG_USE_MEMPOOL;
389,527,196✔
535
  if (processOneBlock) {
389,559,682✔
536
    dispatcher->flags |= DS_FLAG_PROCESS_ONE_BLOCK;
9,529,797✔
537
  }
538

539
  *pHandle = dispatcher;
389,559,077✔
540
  return TSDB_CODE_SUCCESS;
389,578,234✔
541

542
_return:
×
543

544
  taosMemoryFree(pManager);
×
545
  
546
  if (dispatcher) {
×
547
    dsDestroyDataSinker(dispatcher);
×
548
  }
549

550
  nodesDestroyNode((SNode *)*ppDataSink);
×
551
  *ppDataSink = NULL;
×
552
  
553
  return terrno;
×
554
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc