• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

taosdata / TDengine / #3558

17 Dec 2024 06:05AM UTC coverage: 59.778% (+1.6%) from 58.204%
#3558

push

travis-ci

web-flow
Merge pull request #29179 from taosdata/merge/mainto3.0

merge: form main to 3.0 branch

132787 of 287595 branches covered (46.17%)

Branch coverage included in aggregate %.

104 of 191 new or added lines in 5 files covered. (54.45%)

6085 existing lines in 168 files now uncovered.

209348 of 284746 relevant lines covered (73.52%)

8164844.48 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.48
/include/common/tcommon.h
1
/*
2
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3
 *
4
 * This program is free software: you can use, redistribute, and/or modify
5
 * it under the terms of the GNU Affero General Public License, version 3
6
 * or later ("AGPL"), as published by the Free Software Foundation.
7
 *
8
 * This program is distributed in the hope that it will be useful, but WITHOUT
9
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10
 * FITNESS FOR A PARTICULAR PURPOSE.
11
 *
12
 * You should have received a copy of the GNU Affero General Public License
13
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
14
 */
15

16
#ifndef _TD_COMMON_DEF_H_
17
#define _TD_COMMON_DEF_H_
18

19
#include "tarray.h"
20
#include "tmsg.h"
21
#include "tvariant.h"
22

23
#ifdef __cplusplus
24
extern "C" {
25
#endif
26

27
// clang-format off
28
#define IS_META_MSG(x) ( \
29
     x == TDMT_VND_CREATE_STB     \
30
  || x == TDMT_VND_ALTER_STB      \
31
  || x == TDMT_VND_DROP_STB       \
32
  || x == TDMT_VND_CREATE_TABLE   \
33
  || x == TDMT_VND_ALTER_TABLE    \
34
  || x == TDMT_VND_DROP_TABLE     \
35
  || x == TDMT_VND_DELETE         \
36
)
37
// clang-format on
38

39
typedef bool (*state_key_cmpr_fn)(void* pKey1, void* pKey2);
40

41
typedef struct STableKeyInfo {
42
  uint64_t uid;
43
  uint64_t groupId;
44
} STableKeyInfo;
45

46
typedef struct SWinKey {
47
  uint64_t groupId;
48
  TSKEY    ts;
49
} SWinKey;
50

51
typedef struct SSessionKey {
52
  STimeWindow win;
53
  uint64_t    groupId;
54
} SSessionKey;
55

56
typedef int64_t COUNT_TYPE;
57

58
typedef struct SVersionRange {
59
  int64_t minVer;
60
  int64_t maxVer;
61
} SVersionRange;
62

63
static inline int winKeyCmprImpl(const void* pKey1, const void* pKey2) {
29,663,809✔
64
  SWinKey* pWin1 = (SWinKey*)pKey1;
29,663,809✔
65
  SWinKey* pWin2 = (SWinKey*)pKey2;
29,663,809✔
66

67
  if (pWin1->groupId > pWin2->groupId) {
29,663,809✔
68
    return 1;
6,557,555✔
69
  } else if (pWin1->groupId < pWin2->groupId) {
23,106,254✔
70
    return -1;
16,471,459✔
71
  }
72

73
  if (pWin1->ts > pWin2->ts) {
6,634,795✔
74
    return 1;
2,698,977✔
75
  } else if (pWin1->ts < pWin2->ts) {
3,935,818✔
76
    return -1;
3,768,363✔
77
  }
78

79
  return 0;
167,455✔
80
}
81

82
static inline int winKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) {
76,782✔
83
  return winKeyCmprImpl(pKey1, pKey2);
76,782✔
84
}
85

86
typedef struct {
87
  uint64_t groupId;
88
  TSKEY    ts;
89
  int32_t  exprIdx;
90
} STupleKey;
91

92
static inline int STupleKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) {
3,608✔
93
  STupleKey* pTuple1 = (STupleKey*)pKey1;
3,608✔
94
  STupleKey* pTuple2 = (STupleKey*)pKey2;
3,608✔
95

96
  if (pTuple1->groupId > pTuple2->groupId) {
3,608!
97
    return 1;
×
98
  } else if (pTuple1->groupId < pTuple2->groupId) {
3,608!
99
    return -1;
×
100
  }
101

102
  if (pTuple1->ts > pTuple2->ts) {
3,608✔
103
    return 1;
127✔
104
  } else if (pTuple1->ts < pTuple2->ts) {
3,481✔
105
    return -1;
221✔
106
  }
107

108
  if (pTuple1->exprIdx > pTuple2->exprIdx) {
3,260✔
109
    return 1;
518✔
110
  } else if (pTuple1->exprIdx < pTuple2->exprIdx) {
2,742✔
111
    return -1;
2,242✔
112
  }
113

114
  return 0;
500✔
115
}
116

117
enum {
118
  TMQ_MSG_TYPE__POLL_DATA_RSP = 0,
119
  TMQ_MSG_TYPE__POLL_META_RSP,
120
  TMQ_MSG_TYPE__EP_RSP,
121
  TMQ_MSG_TYPE__POLL_DATA_META_RSP,
122
  TMQ_MSG_TYPE__WALINFO_RSP,
123
  TMQ_MSG_TYPE__POLL_BATCH_META_RSP,
124
};
125

126
enum {
127
  STREAM_INPUT__DATA_SUBMIT = 1,
128
  STREAM_INPUT__DATA_BLOCK,
129
  STREAM_INPUT__MERGED_SUBMIT,
130
  STREAM_INPUT__TQ_SCAN,
131
  STREAM_INPUT__DATA_RETRIEVE,
132
  STREAM_INPUT__GET_RES,
133
  STREAM_INPUT__CHECKPOINT,
134
  STREAM_INPUT__CHECKPOINT_TRIGGER,
135
  STREAM_INPUT__TRANS_STATE,
136
  STREAM_INPUT__REF_DATA_BLOCK,
137
  STREAM_INPUT__DESTROY,
138
};
139

140
typedef enum EStreamType {
141
  STREAM_NORMAL = 1,
142
  STREAM_INVERT,
143
  STREAM_CLEAR,
144
  STREAM_INVALID,
145
  STREAM_GET_ALL,
146
  STREAM_DELETE_RESULT,
147
  STREAM_DELETE_DATA,
148
  STREAM_RETRIEVE,
149
  STREAM_PULL_DATA,
150
  STREAM_PULL_OVER,
151
  STREAM_FILL_OVER,
152
  STREAM_CHECKPOINT,
153
  STREAM_CREATE_CHILD_TABLE,
154
  STREAM_TRANS_STATE,
155
  STREAM_MID_RETRIEVE,
156
  STREAM_PARTITION_DELETE_DATA,
157
  STREAM_GET_RESULT,
158
  STREAM_DROP_CHILD_TABLE,
159
} EStreamType;
160

161
#pragma pack(push, 1)
162
typedef struct SColumnDataAgg {
163
  int16_t colId;
164
  int16_t numOfNull;
165
  int64_t sum;
166
  int64_t max;
167
  int64_t min;
168
} SColumnDataAgg;
169
#pragma pack(pop)
170

171
typedef struct SBlockID {
172
  // The uid of table, from which current data block comes. And it is always 0, if current block is the
173
  // result of calculation.
174
  uint64_t uid;
175

176
  // Block id, acquired and assigned from executor, which created according to the hysical planner. Block id is used
177
  // to mark the stage of exec task.
178
  uint64_t blockId;
179

180
  // Generated by group/partition by [value|tags]. Created and assigned by table-scan operator, group-by operator,
181
  // and partition by operator.
182
  uint64_t groupId;
183
} SBlockID;
184

185
typedef struct SPkInfo {
186
  int8_t  type;
187
  int32_t bytes;
188
  union {
189
    int64_t  val;
190
    uint8_t* pData;
191
  } skey;
192
  union {
193
    int64_t  val;
194
    uint8_t* pData;
195
  } ekey;
196
} SPkInfo;
197

198
typedef struct SDataBlockInfo {
199
  STimeWindow window;
200
  int32_t     rowSize;
201
  int64_t     rows;  // todo hide this attribute
202
  uint32_t    capacity;
203
  SBlockID    id;
204
  int16_t     hasVarCol;
205
  int16_t     dataLoad;  // denote if the data is loaded or not
206
  uint8_t     scanFlag;
207
  bool        blankFill;
208
  SValue      pks[2];
209

210
  // TODO: optimize and remove following
211
  int64_t     version;    // used for stream, and need serialization
212
  int32_t     childId;    // used for stream, do not serialize
213
  EStreamType type;       // used for stream, do not serialize
214
  STimeWindow calWin;     // used for stream, do not serialize
215
  TSKEY       watermark;  // used for stream
216

217
  char parTbName[TSDB_TABLE_NAME_LEN];  // used for stream partition
218
} SDataBlockInfo;
219

220
typedef struct SSDataBlock {
221
  SColumnDataAgg* pBlockAgg;
222
  SArray*         pDataBlock;  // SArray<SColumnInfoData>
223
  SDataBlockInfo  info;
224
} SSDataBlock;
225

226
typedef struct SVarColAttr {
227
  int32_t* offset;    // start position for each entry in the list
228
  uint32_t length;    // used buffer size that contain the valid data
229
  uint32_t allocLen;  // allocated buffer size
230
} SVarColAttr;
231

232
// pBlockAgg->numOfNull == info.rows, all data are null
233
// pBlockAgg->numOfNull == 0, no data are null.
234
typedef struct SColumnInfoData {
235
  char* pData;  // the corresponding block data in memory
236
  union {
237
    char*       nullbitmap;  // bitmap, one bit for each item in the list
238
    SVarColAttr varmeta;
239
  };
240
  SColumnInfo info;        // column info
241
  bool        hasNull;     // if current column data has null value.
242
  bool        reassigned;  // if current column data is reassigned.
243
} SColumnInfoData;
244

245
typedef struct SQueryTableDataCond {
246
  uint64_t     suid;
247
  int32_t      order;  // desc|asc order to iterate the data block
248
  int32_t      numOfCols;
249
  SColumnInfo* colList;
250
  int32_t*     pSlotList;  // the column output destation slot, and it may be null
251
  int32_t      type;       // data block load type:
252
  bool         skipRollup;
253
  STimeWindow  twindows;
254
  STimeWindow  extTwindows[2];
255
  int64_t      startVersion;
256
  int64_t      endVersion;
257
  bool         notLoadData;  // response the actual data, not only the rows in the attribute of info.row of ssdatablock
258
} SQueryTableDataCond;
259

260
int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock);
261
void*   tDecodeDataBlock(const void* buf, SSDataBlock* pBlock);
262

263
void colDataDestroy(SColumnInfoData* pColData);
264

265
//======================================================================================================================
266
// the following structure shared by parser and executor
267
typedef struct SColumn {
268
  union {
269
    uint64_t uid;
270
    int64_t  dataBlockId;
271
  };
272

273
  int16_t colId;
274
  int16_t slotId;
275

276
  char    name[TSDB_COL_NAME_LEN];
277
  int16_t colType;  // column type: normal column, tag, or window column
278
  int16_t type;
279
  int32_t bytes;
280
  uint8_t precision;
281
  uint8_t scale;
282
} SColumn;
283

284
typedef struct STableBlockDistInfo {
285
  uint32_t rowSize;
286
  uint16_t numOfFiles;
287
  uint32_t numOfTables;
288
  uint32_t numOfBlocks;
289
  uint64_t totalSize;
290
  uint64_t totalRows;
291
  int32_t  maxRows;
292
  int32_t  minRows;
293
  int32_t  defMinRows;
294
  int32_t  defMaxRows;
295
  int32_t  firstSeekTimeUs;
296
  uint32_t numOfInmemRows;
297
  uint32_t numOfSttRows;
298
  uint32_t numOfVgroups;
299
  int32_t  blockRowsHisto[20];
300
} STableBlockDistInfo;
301

302
int32_t tSerializeBlockDistInfo(void* buf, int32_t bufLen, const STableBlockDistInfo* pInfo);
303
int32_t tDeserializeBlockDistInfo(void* buf, int32_t bufLen, STableBlockDistInfo* pInfo);
304

305
typedef struct SDBBlockUsageInfo {
306
  uint64_t dataInDiskSize;
307
  uint64_t walInDiskSize;
308
  uint64_t rawDataSize;
309
} SDBBlockUsageInfo;
310

311
int32_t tSerializeBlockDbUsage(void* buf, int32_t bufLen, const SDBBlockUsageInfo* pInfo);
312
int32_t tDeserializeBlockDbUsage(void* buf, int32_t bufLen, SDBBlockUsageInfo* pInfo);
313

314
enum {
315
  FUNC_PARAM_TYPE_VALUE = 0x1,
316
  FUNC_PARAM_TYPE_COLUMN = 0x2,
317
};
318

319
typedef struct SFunctParam {
320
  int32_t  type;
321
  SColumn* pCol;
322
  SVariant param;
323
} SFunctParam;
324

325
// the structure for sql function in select clause
326
typedef struct SResSchame {
327
  int8_t  type;
328
  int32_t slotId;
329
  int32_t bytes;
330
  int32_t precision;
331
  int32_t scale;
332
  char    name[TSDB_COL_NAME_LEN];
333
} SResSchema;
334

335
typedef struct SExprBasicInfo {
336
  SResSchema   resSchema;
337
  int16_t      numOfParams;  // argument value of each function
338
  SFunctParam* pParam;
339
} SExprBasicInfo;
340

341
typedef struct SExprInfo {
342
  struct SExprBasicInfo base;
343
  struct tExprNode*     pExpr;
344
} SExprInfo;
345

346
typedef struct {
347
  const char* key;
348
  size_t      keyLen;
349
  uint8_t     type;
350
  union {
351
    const char* value;
352
    int64_t     i;
353
    uint64_t    u;
354
    double      d;
355
    float       f;
356
  };
357
  size_t length;
358
  bool   keyEscaped;
359
  bool   valueEscaped;
360
} SSmlKv;
361

362
#define QUERY_ASC_FORWARD_STEP  1
363
#define QUERY_DESC_FORWARD_STEP -1
364

365
#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)
366

367
#define SORT_QSORT_T              0x1
368
#define SORT_SPILLED_MERGE_SORT_T 0x2
369
typedef struct SSortExecInfo {
370
  int32_t sortMethod;
371
  int32_t sortBuffer;
372
  int32_t loops;       // loop count
373
  int32_t writeBytes;  // write io bytes
374
  int32_t readBytes;   // read io bytes
375
} SSortExecInfo;
376

377
typedef struct SNonSortExecInfo {
378
  int32_t blkNums;
379
} SNonSortExecInfo;
380

381
typedef struct STUidTagInfo {
382
  char*    name;
383
  uint64_t uid;
384
  void*    pTagVal;
385
} STUidTagInfo;
386

387
// stream special block column
388

389
#define START_TS_COLUMN_INDEX           0
390
#define END_TS_COLUMN_INDEX             1
391
#define UID_COLUMN_INDEX                2
392
#define GROUPID_COLUMN_INDEX            3
393
#define CALCULATE_START_TS_COLUMN_INDEX 4
394
#define CALCULATE_END_TS_COLUMN_INDEX   5
395
#define TABLE_NAME_COLUMN_INDEX         6
396
#define PRIMARY_KEY_COLUMN_INDEX        7
397

398
//steam get result block column
399
#define DATA_TS_COLUMN_INDEX            0
400
#define DATA_VERSION_COLUMN_INDEX       1
401

402
// stream create table block column
403
#define UD_TABLE_NAME_COLUMN_INDEX 0
404
#define UD_GROUPID_COLUMN_INDEX    1
405
#define UD_TAG_COLUMN_INDEX        2
406

407
int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime);
408
int32_t dumpConfToDataBlock(SSDataBlock* pBlock, int32_t startCol);
409

410
#define TSMA_RES_STB_POSTFIX          "_tsma_res_stb_"
411
#define MD5_OUTPUT_LEN                32
412
#define TSMA_RES_STB_EXTRA_COLUMN_NUM 4  // 3 columns: _wstart, _wend, _wduration, 1 tag: tbname
413

414
static inline bool isTsmaResSTb(const char* stbName) {
482,013✔
415
  static bool showTsmaTables = false;
416
  if (showTsmaTables) return false;
482,013!
417
  const char* pos = strstr(stbName, TSMA_RES_STB_POSTFIX);
482,013✔
418
  if (pos && strlen(stbName) == (pos - stbName) + strlen(TSMA_RES_STB_POSTFIX)) {
482,013!
UNCOV
419
    return true;
×
420
  }
421
  return false;
482,013✔
422
}
423

424
#ifdef __cplusplus
425
}
426
#endif
427

428
#endif /*_TD_COMMON_DEF_H_*/
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc