#4885

Committed 15 Dec 2025 03:26AM UTC coverage: 65.258% (+4.6%) from 60.617%

Build # #4885

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

feat(tmq): [TS-6379]remove limition for table operation in tmq  (#33834)

Coverage Stats

872 of 1074 new or added lines in 16 files covered. (81.19%)

659 existing lines in 92 files now uncovered.

177890 of 272597 relevant lines covered (65.26%)

103732965.73 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

89.86

/source/client/src/clientSmlTelnet.c

/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include <ctype.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "clientSml.h"

int32_t is_same_child_table_telnet(const void *a, const void *b) {
  SSmlLineInfo *t1 = (SSmlLineInfo *)a;
  SSmlLineInfo *t2 = (SSmlLineInfo *)b;
  if (t1 == NULL || t2 == NULL || t1->measure == NULL || t2->measure == NULL || t1->tags == NULL || t2->tags == NULL)
    return 1;
  return (((t1->measureLen == t2->measureLen) && memcmp(t1->measure, t2->measure, t1->measureLen) == 0) &&
          ((t1->tagsLen == t2->tagsLen) && memcmp(t1->tags, t2->tags, t1->tagsLen) == 0))
             ? 0
             : 1;
}

int64_t smlParseOpenTsdbTime(SSmlHandle *info, const char *data, int32_t len) {
  uint8_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO;

  if (unlikely(!data)) {
    smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL);
    return -1;
  }
  if (unlikely(len == 1 && data[0] == '0')) {
    return taosGetTimestampNs() / smlFactorNS[toPrecision];
  }
  int8_t fromPrecision = smlGetTsTypeByLen(len);
  if (unlikely(fromPrecision == -1)) {
    smlBuildInvalidDataMsg(&info->msgBuf,
                           "timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data);
    return -1;
  }
  int64_t ts = smlGetTimeValue(data, len, fromPrecision, toPrecision);
  if (unlikely(ts == -1)) {
    smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data);
    return -1;
  }
  return ts;
}

static void smlParseTelnetElement(char **sql, char *sqlEnd, char **data, int32_t *len) {
  while (*sql < sqlEnd) {
    if (unlikely((**sql != SPACE && !(*data)))) {
      *data = *sql;
    } else if (unlikely(**sql == SPACE && *data)) {
      *len = *sql - *data;
      break;
    }
    (*sql)++;
  }
}

static int32_t smlProcessTagTelnet(SSmlHandle *info, char *data, char *sqlEnd){
  SArray *preLineKV = info->preLineTagKV;
  taosArrayClearEx(preLineKV, freeSSmlKv);
  int     cnt = 0;

  const char *sql = data;
  while (sql < sqlEnd) {
    JUMP_SPACE(sql, sqlEnd)
    if (unlikely(*sql == '\0' || *sql == '\n')) break;

    const char *key = sql;
    size_t      keyLen = 0;

    // parse key
    while (sql < sqlEnd) {
      if (unlikely(*sql == SPACE)) {
        smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
        return TSDB_CODE_SML_INVALID_DATA;
      }
      if (unlikely(*sql == EQUAL)) {
        keyLen = sql - key;
        sql++;
        break;
      }
      sql++;
    }

    if (unlikely(IS_INVALID_COL_LEN(keyLen))) {
      smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);
      return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;
    }

    // parse value
    const char *value = sql;
    size_t      valueLen = 0;
    while (sql < sqlEnd) {
      // parse value
      if (unlikely(*sql == SPACE)) {
        break;
      }
      if (unlikely(*sql == EQUAL)) {
        smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);
        return TSDB_CODE_SML_INVALID_DATA;
      }
      sql++;
    }
    valueLen = sql - value;

    if (unlikely(valueLen == 0)) {
      smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);
      return TSDB_CODE_TSC_INVALID_VALUE;
    }

    if (unlikely(valueLen > (TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {
      return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;
    }

    SSmlKv kv = {.key = key,
        .keyLen = keyLen,
        .type = TSDB_DATA_TYPE_NCHAR,
        .value = value,
        .length = valueLen,
        .keyEscaped = false,
        .valueEscaped = false};
    if (taosArrayPush(preLineKV, &kv) == NULL){
      return terrno;
    }
    if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {
      return TSDB_CODE_SML_INVALID_DATA;
    }
    cnt++;
  }
  return TSDB_CODE_SUCCESS;
}

static int32_t smlParseTelnetTags(SSmlHandle *info, char *data, char *sqlEnd, SSmlLineInfo *elements) {
  if (is_same_child_table_telnet(elements, &info->preLine) == 0) {
    elements->measureTag = info->preLine.measureTag;
    return TSDB_CODE_SUCCESS;
  }

  int32_t code = 0;
  int32_t lino = 0;
  if(info->dataFormat){
    SML_CHECK_CODE(smlProcessSuperTable(info, elements));
  }
  SML_CHECK_CODE(smlProcessTagTelnet(info, data, sqlEnd));
  SML_CHECK_CODE(smlJoinMeasureTag(elements));

  code = smlProcessChildTable(info, elements);

END:
  if(info->reRun){
    return TSDB_CODE_SUCCESS;
  }
  RETURN
}

// format: <metric> <timestamp> <value> <tagk_1>=<tagv_1>[ <tagk_n>=<tagv_n>]
int32_t smlParseTelnetString(SSmlHandle *info, char *sql, char *sqlEnd, SSmlLineInfo *elements) {
  if (!sql) return TSDB_CODE_SML_INVALID_DATA;

  // parse metric
  smlParseTelnetElement(&sql, sqlEnd, &elements->measure, &elements->measureLen);
  if (unlikely((!(elements->measure) || IS_INVALID_TABLE_LEN(elements->measureLen)))) {
    smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid measure", sql);
    return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
  }

  // parse timestamp
  smlParseTelnetElement(&sql, sqlEnd, &elements->timestamp, &elements->timestampLen);
  if (unlikely(!elements->timestamp || elements->timestampLen == 0)) {
    smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid timestamp", sql);
    return TSDB_CODE_SML_INVALID_DATA;
  }

  bool needConverTime = false;  // get TS before parse tag(get meta), so need convert time
  if (info->dataFormat && info->currSTableMeta == NULL) {
    needConverTime = true;
  }
  int64_t ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen);
  if (unlikely(ts < 0)) {
    smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet parse timestamp failed", sql);
    return TSDB_CODE_INVALID_TIMESTAMP;
  }

  // parse value
  smlParseTelnetElement(&sql, sqlEnd, &elements->cols, &elements->colsLen);
  if (unlikely(!elements->cols || elements->colsLen == 0)) {
    smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid value", sql);
    return TSDB_CODE_TSC_INVALID_VALUE;
  }

  SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen};
  int ret = smlParseValue(&kv, &info->msgBuf);
  if (ret != TSDB_CODE_SUCCESS) {
    uError("SML:0x%" PRIx64 " %s parse value error:%d.", info->id, __FUNCTION__, ret);
    return TSDB_CODE_TSC_INVALID_VALUE;
  }

  JUMP_SPACE(sql, sqlEnd)

  elements->tags = sql;
  elements->tagsLen = sqlEnd - sql;
  if (unlikely(!elements->tags || elements->tagsLen == 0)) {
    smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid tag value", sql);
    return TSDB_CODE_TSC_INVALID_VALUE;
  }

  ret = smlParseTelnetTags(info, sql, sqlEnd, elements);
  if (unlikely(ret != TSDB_CODE_SUCCESS)) {
    return ret;
  }

  if (unlikely(info->reRun)) {
    return TSDB_CODE_SUCCESS;
  }

  SSmlKv kvTs = {0};
  smlBuildTsKv(&kvTs, ts);
  if (needConverTime && info->currSTableMeta != NULL) {
    kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision);
  }

  if (info->dataFormat){
    ret = smlParseEndTelnetJsonFormat(info, elements, &kvTs, &kv);
  } else {
    ret = smlParseEndTelnetJsonUnFormat(info, elements, &kvTs, &kv);
  }
  info->preLine = *elements;

  return ret;
}

1	/*
2	* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3	*
4	* This program is free software: you can use, redistribute, and/or modify
5	* it under the terms of the GNU Affero General Public License, version 3
6	* or later ("AGPL"), as published by the Free Software Foundation.
7	*
8	* This program is distributed in the hope that it will be useful, but WITHOUT
9	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10	* FITNESS FOR A PARTICULAR PURPOSE.
11	*
12	* You should have received a copy of the GNU Affero General Public License
13	* along with this program. If not, see <http://www.gnu.org/licenses/>.
14	*/
15
16	#include <ctype.h>
17	#include <stdio.h>
18	#include <stdlib.h>
19	#include <string.h>
20
21	#include "clientSml.h"
22
23	int32_t is_same_child_table_telnet(const void a, const void b) {	774,175✔
24	SSmlLineInfo t1 = (SSmlLineInfo )a;	774,175✔
25	SSmlLineInfo t2 = (SSmlLineInfo )b;	774,175✔
26	if (t1 == NULL \|\| t2 == NULL \|\| t1->measure == NULL \|\| t2->measure == NULL \|\| t1->tags == NULL \|\| t2->tags == NULL)	774,175✔
27	return 1;	561,562✔
28	return (((t1->measureLen == t2->measureLen) && memcmp(t1->measure, t2->measure, t1->measureLen) == 0) &&	216,378✔
29	((t1->tagsLen == t2->tagsLen) && memcmp(t1->tags, t2->tags, t1->tagsLen) == 0))	210,346✔
30	? 0
31	: 1;	631,111✔
32	}
33
34	int64_t smlParseOpenTsdbTime(SSmlHandle info, const char data, int32_t len) {	293,543✔
35	uint8_t toPrecision = info->currSTableMeta ? info->currSTableMeta->tableInfo.precision : TSDB_TIME_PRECISION_NANO;	293,543✔
36
37	if (unlikely(!data)) {	293,543✔
38	smlBuildInvalidDataMsg(&info->msgBuf, "timestamp can not be null", NULL);	×
39	return -1;	×
40	}
41	if (unlikely(len == 1 && data[0] == '0')) {	293,543✔
42	return taosGetTimestampNs() / smlFactorNS[toPrecision];	2,532✔
43	}
44	int8_t fromPrecision = smlGetTsTypeByLen(len);	292,277✔
45	if (unlikely(fromPrecision == -1)) {	292,209✔
46	smlBuildInvalidDataMsg(&info->msgBuf,	13,527✔
47	"timestamp precision can only be seconds(10 digits) or milli seconds(13 digits)", data);
48	return -1;	13,527✔
49	}
50	int64_t ts = smlGetTimeValue(data, len, fromPrecision, toPrecision);	278,682✔
51	if (unlikely(ts == -1)) {	278,750✔
UNCOV 52	smlBuildInvalidDataMsg(&info->msgBuf, "invalid timestamp", data);	×
53	return -1;	×
54	}
55	return ts;	278,750✔
56	}
57
58	static void smlParseTelnetElement(char *sql, char sqlEnd, char *data, int32_t len) {	875,161✔
59	while (*sql < sqlEnd) {	9,648,851✔
60	if (unlikely((*sql != SPACE && !(data)))) {	9,640,758✔
61	data = sql;	875,229✔
62	} else if (unlikely(*sql == SPACE && data)) {	8,765,529✔
63	len = sql - *data;	867,068✔
64	break;	867,068✔
65	}
66	(*sql)++;	8,773,690✔
67	}
68	}	875,161✔
69
70	static int32_t smlProcessTagTelnet(SSmlHandle info, char data, char *sqlEnd){	105,494✔
71	SArray *preLineKV = info->preLineTagKV;	105,494✔
72	taosArrayClearEx(preLineKV, freeSSmlKv);	105,494✔
73	int cnt = 0;	105,494✔
74
75	const char *sql = data;	105,494✔
76	while (sql < sqlEnd) {	996,996✔
77	JUMP_SPACE(sql, sqlEnd)	1,709,710✔
78	if (unlikely(sql == '\0' \|\| sql == '\n')) break;	905,357✔
79
80	const char *key = sql;	895,015✔
81	size_t keyLen = 0;	895,015✔
82
83	// parse key
84	while (sql < sqlEnd) {	3,026,571✔
85	if (unlikely(*sql == SPACE)) {	3,026,006✔
86	smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);	1,266✔
87	return TSDB_CODE_SML_INVALID_DATA;	1,266✔
88	}
89	if (unlikely(*sql == EQUAL)) {	3,024,740✔
90	keyLen = sql - key;	893,184✔
91	sql++;	893,184✔
92	break;	893,184✔
93	}
94	sql++;	2,131,556✔
95	}
96
97	if (unlikely(IS_INVALID_COL_LEN(keyLen))) {	893,749✔
98	smlBuildInvalidDataMsg(&info->msgBuf, "invalid key or key is too long than 64", key);	1,266✔
99	return TSDB_CODE_TSC_INVALID_COLUMN_LENGTH;	1,266✔
100	}
101
102	// parse value
103	const char *value = sql;	892,483✔
104	size_t valueLen = 0;	892,483✔
105	while (sql < sqlEnd) {	9,815,927✔
106	// parse value
107	if (unlikely(*sql == SPACE)) {	9,724,322✔
108	break;	800,878✔
109	}
110	if (unlikely(*sql == EQUAL)) {	8,923,444✔
111	smlBuildInvalidDataMsg(&info->msgBuf, "invalid data", sql);	×
112	return TSDB_CODE_SML_INVALID_DATA;	×
113	}
114	sql++;	8,923,444✔
115	}
116	valueLen = sql - value;	892,483✔
117
118	if (unlikely(valueLen == 0)) {	892,483✔
119	smlBuildInvalidDataMsg(&info->msgBuf, "invalid value", value);	×
120	return TSDB_CODE_TSC_INVALID_VALUE;	×
121	}
122
123	if (unlikely(valueLen > (TSDB_MAX_TAGS_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)) {	892,483✔
124	return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN;	×
125	}
126
127	SSmlKv kv = {.key = key,	892,483✔
128	.keyLen = keyLen,
129	.type = TSDB_DATA_TYPE_NCHAR,
130	.value = value,
131	.length = valueLen,
132	.keyEscaped = false,
133	.valueEscaped = false};
134	if (taosArrayPush(preLineKV, &kv) == NULL){	892,551✔
135	return terrno;	×
136	}
137	if (info->dataFormat && !isSmlTagAligned(info, cnt, &kv)) {	892,551✔
138	return TSDB_CODE_SML_INVALID_DATA;	981✔
139	}
140	cnt++;	891,502✔
141	}
142	return TSDB_CODE_SUCCESS;	101,981✔
143	}
144
145	static int32_t smlParseTelnetTags(SSmlHandle info, char data, char sqlEnd, SSmlLineInfo elements) {	268,520✔
146	if (is_same_child_table_telnet(elements, &info->preLine) == 0) {	268,520✔
147	elements->measureTag = info->preLine.measureTag;	70,309✔
148	return TSDB_CODE_SUCCESS;	70,309✔
149	}
150
151	int32_t code = 0;	198,279✔
152	int32_t lino = 0;	198,279✔
153	if(info->dataFormat){	198,279✔
154	SML_CHECK_CODE(smlProcessSuperTable(info, elements));	103,410✔
155	}
156	SML_CHECK_CODE(smlProcessTagTelnet(info, data, sqlEnd));	105,562✔
157	SML_CHECK_CODE(smlJoinMeasureTag(elements));	102,015✔
158
159	code = smlProcessChildTable(info, elements);	101,981✔
160
161	END:	198,245✔
162	if(info->reRun){	198,245✔
163	return TSDB_CODE_SUCCESS;	92,701✔
164	}
165	RETURN	105,544✔
166	}
167
168	// format: <metric> <timestamp> <value> <tagk_1>=<tagv_1>[ <tagk_n>=<tagv_n>]
169	int32_t smlParseTelnetString(SSmlHandle info, char sql, char sqlEnd, SSmlLineInfo elements) {	301,738✔
170	if (!sql) return TSDB_CODE_SML_INVALID_DATA;	301,738✔
171
172	// parse metric
173	smlParseTelnetElement(&sql, sqlEnd, &elements->measure, &elements->measureLen);	301,738✔
174	if (unlikely((!(elements->measure) \|\| IS_INVALID_TABLE_LEN(elements->measureLen)))) {	301,772✔
175	smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid measure", sql);	8,229✔
176	return TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;	8,229✔
177	}
178
179	// parse timestamp
180	smlParseTelnetElement(&sql, sqlEnd, &elements->timestamp, &elements->timestampLen);	293,543✔
181	if (unlikely(!elements->timestamp \|\| elements->timestampLen == 0)) {	293,543✔
182	smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid timestamp", sql);	×
183	return TSDB_CODE_SML_INVALID_DATA;	×
184	}
185
186	bool needConverTime = false; // get TS before parse tag(get meta), so need convert time	293,543✔
187	if (info->dataFormat && info->currSTableMeta == NULL) {	293,543✔
188	needConverTime = true;	127,705✔
189	}
190	int64_t ts = smlParseOpenTsdbTime(info, elements->timestamp, elements->timestampLen);	293,543✔
191	if (unlikely(ts < 0)) {	293,543✔
192	smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet parse timestamp failed", sql);	13,527✔
193	return TSDB_CODE_INVALID_TIMESTAMP;	13,527✔
194	}
195
196	// parse value
197	smlParseTelnetElement(&sql, sqlEnd, &elements->cols, &elements->colsLen);	280,016✔
198	if (unlikely(!elements->cols \|\| elements->colsLen == 0)) {	280,016✔
199	smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid value", sql);	633✔
200	return TSDB_CODE_TSC_INVALID_VALUE;	633✔
201	}
202
203	SSmlKv kv = {.key = VALUE, .keyLen = VALUE_LEN, .value = elements->cols, .length = (size_t)elements->colsLen};	279,383✔
204	int ret = smlParseValue(&kv, &info->msgBuf);	279,383✔
205	if (ret != TSDB_CODE_SUCCESS) {	279,349✔
206	uError("SML:0x%" PRIx64 " %s parse value error:%d.", info->id, __FUNCTION__, ret);	10,761✔
207	return TSDB_CODE_TSC_INVALID_VALUE;	10,761✔
208	}
209
210	JUMP_SPACE(sql, sqlEnd)	553,358✔
211
212	elements->tags = sql;	268,588✔
213	elements->tagsLen = sqlEnd - sql;	268,588✔
214	if (unlikely(!elements->tags \|\| elements->tagsLen == 0)) {	268,588✔
215	smlBuildInvalidDataMsg(&info->msgBuf, "SML telnet invalid tag value", sql);	×
216	return TSDB_CODE_TSC_INVALID_VALUE;	×
217	}
218
219	ret = smlParseTelnetTags(info, sql, sqlEnd, elements);	268,588✔
220	if (unlikely(ret != TSDB_CODE_SUCCESS)) {	268,520✔
221	return ret;	3,563✔
222	}
223
224	if (unlikely(info->reRun)) {	264,957✔
225	return TSDB_CODE_SUCCESS;	92,701✔
226	}
227
228	SSmlKv kvTs = {0};	172,256✔
229	smlBuildTsKv(&kvTs, ts);	172,256✔
230	if (needConverTime && info->currSTableMeta != NULL) {	172,290✔
231	kvTs.i = convertTimePrecision(kvTs.i, TSDB_TIME_PRECISION_NANO, info->currSTableMeta->tableInfo.precision);	8,419✔
232	}
233
234	if (info->dataFormat){	172,290✔
235	ret = smlParseEndTelnetJsonFormat(info, elements, &kvTs, &kv);	63,205✔
236	} else {
237	ret = smlParseEndTelnetJsonUnFormat(info, elements, &kvTs, &kv);	109,085✔
238	}
239	info->preLine = *elements;	172,324✔
240
241	return ret;	172,324✔
242	}

taosdata / TDengine / #4885

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous