#3608

Committed 12 Feb 2025 05:57AM UTC coverage: 63.066% (+1.4%) from 61.715%

Build # #3608

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

Merge pull request #29746 from taosdata/merge/mainto3.02

merge: from main to 3.0 branch

Run Details

140199 of 286257 branches covered (48.98%)

Branch coverage included in aggregate %.

89 of 161 new or added lines in 18 files covered. (55.28%)

3211 existing lines in 190 files now uncovered.

218998 of 283298 relevant lines covered (77.3%)

5949310.66 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

74.52

/source/libs/parser/src/parInsertSml.c

/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "parInsertUtil.h"
#include "parInt.h"
#include "parToken.h"
#include "ttime.h"

int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf,
                     int32_t msgBufLen) {
  SMsgBuf    msg = {.buf = msgBuf, .len = msgBufLen};
  SToken  sToken = {0};
  int      code  = TSDB_CODE_SUCCESS;
  int32_t  lino  = 0;

  NEXT_TOKEN(pTableName, sToken);
  TSDB_CHECK_CONDITION(sToken.n != 0, code, lino, end, TSDB_CODE_TSC_INVALID_OPERATION);
  code = insCreateSName(pName, &sToken, acctId, dbName, &msg);
  TSDB_CHECK_CODE(code, lino, end);
  NEXT_TOKEN(pTableName, sToken);
  TSDB_CHECK_CONDITION(sToken.n <= 0, code, lino, end, TSDB_CODE_TSC_INVALID_OPERATION);

end:
  if (code != 0){
    uError("%s failed at %d since %s", __func__, lino, tstrerror(code));
  }
  return TSDB_CODE_SUCCESS;
}

static int32_t smlBoundColumnData(SArray* cols, SBoundColInfo* pBoundInfo, SSchema* pSchema, bool isTag) {
  int      code = TSDB_CODE_SUCCESS;
  int32_t  lino = 0;
  bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool));
  TSDB_CHECK_NULL(pUseCols, code, lino, end, terrno);
  pBoundInfo->numOfBound = 0;
  int16_t lastColIdx = -1;  // last column found

  for (int i = 0; i < taosArrayGetSize(cols); ++i) {
    SSmlKv*  kv = taosArrayGet(cols, i);
    SToken   sToken = {.n = kv->keyLen, .z = (char*)kv->key};
    col_id_t t = lastColIdx + 1;
    col_id_t index = ((t == 0 && !isTag) ? 0 : insFindCol(&sToken, t, pBoundInfo->numOfCols, pSchema));
    uTrace("SML, index:%d, t:%d, ncols:%d", index, t, pBoundInfo->numOfCols);
    if (index < 0 && t > 0) {
      index = insFindCol(&sToken, 0, t, pSchema);
    }

    TSDB_CHECK_CONDITION(index >= 0, code, lino, end, TSDB_CODE_SML_INVALID_DATA);
    TSDB_CHECK_CONDITION(!pUseCols[index], code, lino, end, TSDB_CODE_SML_INVALID_DATA);

    lastColIdx = index;
    pUseCols[index] = true;
    pBoundInfo->pColIndex[pBoundInfo->numOfBound] = index;
    ++pBoundInfo->numOfBound;
  }

end:
  if (code != 0){
    uError("%s failed at %d since %s", __func__, lino, tstrerror(code));
  }
  taosMemoryFree(pUseCols);
  return code;
}

static int32_t smlMbsToUcs4(const char* mbs, size_t mbsLen, void** result, int32_t* resultLen, int32_t maxLen, void* charsetCxt){
  int      code = TSDB_CODE_SUCCESS;
  void*    pUcs4 = NULL;
  int32_t  lino = 0;
  pUcs4 = taosMemoryCalloc(1, maxLen);
  TSDB_CHECK_NULL(pUcs4, code, lino, end, terrno);
  TSDB_CHECK_CONDITION(taosMbsToUcs4(mbs, mbsLen, (TdUcs4*)pUcs4, maxLen, resultLen, charsetCxt), code, lino, end, terrno);
  *result = pUcs4;
  pUcs4 = NULL;

end:
  if (code != 0){
    uError("%s failed at %d since %s", __func__, lino, tstrerror(code));
  }
  taosMemoryFree(pUcs4);
  return code;
}
/**
 * @brief No json tag for schemaless
 *
 * @param cols
 * @param tags
 * @param pSchema
 * @param ppTag
 * @param msg
 * @return int32_t
 */
static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchema, STag** ppTag, SArray** tagName,
                              SMsgBuf* msg, void* charsetCxt) {
  int      code = TSDB_CODE_SUCCESS;
  int32_t  lino = 0;
  SArray*  pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal));
  TSDB_CHECK_NULL(pTagArray, code, lino, end, terrno);
  *tagName = taosArrayInit(8, TSDB_COL_NAME_LEN);
  TSDB_CHECK_NULL(*tagName, code, lino, end, terrno);

  for (int i = 0; i < tags->numOfBound; ++i) {
    SSchema* pTagSchema = &pSchema[tags->pColIndex[i]];
    SSmlKv*  kv = taosArrayGet(cols, i);
    TSDB_CHECK_NULL(kv, code, lino, end, terrno);
    bool cond = (kv->keyLen == strlen(pTagSchema->name) && memcmp(kv->key, pTagSchema->name, kv->keyLen) == 0 && kv->type == pTagSchema->type);
    TSDB_CHECK_CONDITION(cond, code, lino, end, TSDB_CODE_SML_INVALID_DATA);
    TSDB_CHECK_NULL(taosArrayPush(*tagName, pTagSchema->name), code, lino, end, terrno);
    STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};
    if (pTagSchema->type == TSDB_DATA_TYPE_BINARY || pTagSchema->type == TSDB_DATA_TYPE_VARBINARY ||
        pTagSchema->type == TSDB_DATA_TYPE_GEOMETRY) {
      val.pData = (uint8_t*)kv->value;
      val.nData = kv->length;
    } else if (pTagSchema->type == TSDB_DATA_TYPE_NCHAR) {
      code = smlMbsToUcs4(kv->value, kv->length, (void**)&val.pData, (int32_t*)&val.nData, kv->length * TSDB_NCHAR_SIZE, charsetCxt);
      TSDB_CHECK_CODE(code, lino, end);
    } else {
      (void)memcpy(&val.i64, &(kv->value), kv->length);
    }
    TSDB_CHECK_NULL(taosArrayPush(pTagArray, &val), code, lino, end, terrno);
  }
  code = tTagNew(pTagArray, 1, false, ppTag);

end:
  if (code != 0){
    uError("%s failed at %d since %s", __func__, lino, tstrerror(code));
  }
  for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) {
    STagVal* p = (STagVal*)taosArrayGet(pTagArray, i);
    if (p->type == TSDB_DATA_TYPE_NCHAR) {
      taosMemoryFree(p->pData);
    }
  }
  taosArrayDestroy(pTagArray);
  return code;
}

int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt) {
  int      ret = TSDB_CODE_SUCCESS;
  int32_t  lino = 0;
  SVCreateTbReq* pCreateTbReq = NULL;
  ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid,
                                          sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, cxt, false, false);
  TSDB_CHECK_CODE(ret, lino, end);
  ret = initTableColSubmitData(*cxt);
  TSDB_CHECK_CODE(ret, lino, end);

end:
  if (ret != 0){
    uError("%s failed at %d since %s", __func__, lino, tstrerror(ret));
  }
  return ret;
}

void clearColValArraySml(SArray* pCols) {
  int32_t num = taosArrayGetSize(pCols);
  for (int32_t i = 0; i < num; ++i) {
    SColVal* pCol = taosArrayGet(pCols, i);
    if (TSDB_DATA_TYPE_NCHAR == pCol->value.type || TSDB_DATA_TYPE_GEOMETRY == pCol->value.type ||
        TSDB_DATA_TYPE_VARBINARY == pCol->value.type) {
      taosMemoryFreeClear(pCol->value.pData);
    }
    pCol->flag = CV_FLAG_NONE;
    pCol->value.val = 0;
  }
}

int32_t smlBuildRow(STableDataCxt* pTableCxt) {
  int      ret = TSDB_CODE_SUCCESS;
  int32_t  lino = 0;
  SRow**   pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1);
  TSDB_CHECK_NULL(pRow, ret, lino, end, terrno);
  ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow);
  TSDB_CHECK_CODE(ret, lino, end);
  SRowKey key;
  tRowGetKey(*pRow, &key);
  insCheckTableDataOrder(pTableCxt, &key);
end:
  if (ret != 0){
    uError("%s failed at %d since %s", __func__, lino, tstrerror(ret));
  }
  return ret;
}

int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32_t index, void* charsetCxt) {
  int      ret = TSDB_CODE_SUCCESS;
  int32_t  lino = 0;
  SSchema* pColSchema = schema + index;
  SColVal* pVal = taosArrayGet(pTableCxt->pValues, index);
  TSDB_CHECK_NULL(pVal, ret, lino, end, TSDB_CODE_SUCCESS);
  SSmlKv*  kv = (SSmlKv*)data;
  if (kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 ||
      kv->type != pColSchema->type) {
    ret = TSDB_CODE_SML_INVALID_DATA;
    char* tmp = taosMemoryCalloc(kv->keyLen + 1, 1);
    TSDB_CHECK_NULL(tmp, ret, lino, end, terrno);
    (void)memcpy(tmp, kv->key, kv->keyLen);
    uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)", tmp, tDataTypes[kv->type].name,
          pColSchema->name, tDataTypes[pColSchema->type].name);
    taosMemoryFree(tmp);
    goto end;
  }
  if (kv->type == TSDB_DATA_TYPE_NCHAR) {
    ret = smlMbsToUcs4(kv->value, kv->length, (void**)&pVal->value.pData, &pVal->value.nData, pColSchema->bytes - VARSTR_HEADER_SIZE, charsetCxt);
    TSDB_CHECK_CODE(ret, lino, end);
  } else if (kv->type == TSDB_DATA_TYPE_BINARY) {
    pVal->value.nData = kv->length;
    pVal->value.pData = (uint8_t*)kv->value;
  } else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) {
    pVal->value.nData = kv->length;
    pVal->value.pData = taosMemoryMalloc(kv->length);
    TSDB_CHECK_NULL(pVal->value.pData, ret, lino, end, terrno);

    (void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);
  } else {
    (void)memcpy(&pVal->value.val, &(kv->value), kv->length);
  }
  pVal->flag = CV_FLAG_VALUE;

end:
  if (ret != 0){
    uError("%s failed at %d since %s", __func__, lino, tstrerror(ret));
  }
  return ret;
}

int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols,
                    STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
                    char* msgBuf, int32_t msgBufLen, void* charsetCxt) {
  int32_t lino = 0;
  int32_t ret = 0;
  SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};

  SSchema*       pTagsSchema = getTableTagSchema(pTableMeta);
  SBoundColInfo  bindTags = {0};
  SVCreateTbReq* pCreateTblReq = NULL;
  SArray*        tagName = NULL;

  ret = insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags);
  TSDB_CHECK_CODE(ret, lino, end);

  ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true);
  TSDB_CHECK_CODE(ret, lino, end);

  STag* pTag = NULL;
  ret = smlBuildTagRow(tags, &bindTags, pTagsSchema, &pTag, &tagName, &pBuf, charsetCxt);
  TSDB_CHECK_CODE(ret, lino, end);

  pCreateTblReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq));
  TSDB_CHECK_NULL(pCreateTblReq, ret, lino, end, terrno);

  ret = insBuildCreateTbReq(pCreateTblReq, tableName, pTag, pTableMeta->suid, NULL, tagName,
                            pTableMeta->tableInfo.numOfTags, ttl);
  TSDB_CHECK_CODE(ret, lino, end);

  pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1);
  TSDB_CHECK_NULL(pCreateTblReq->ctb.stbName, ret, lino, end, terrno);

  (void)memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen);

  if (dataFormat) {
    STableDataCxt** pTableCxt = (STableDataCxt**)taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj,
                                                             &pTableMeta->uid, sizeof(pTableMeta->uid));
    TSDB_CHECK_NULL(pTableCxt, ret, lino, end, TSDB_CODE_TSC_INVALID_OPERATION);
    (*pTableCxt)->pData->flags |= SUBMIT_REQ_AUTO_CREATE_TABLE;
    (*pTableCxt)->pData->pCreateTbReq = pCreateTblReq;
    (*pTableCxt)->pMeta->uid = pTableMeta->uid;
    (*pTableCxt)->pMeta->vgId = pTableMeta->vgId;
    pCreateTblReq = NULL;
    goto end;
  }

  STableDataCxt* pTableCxt = NULL;
  ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid,
                           sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false, false);
  TSDB_CHECK_CODE(ret, lino, end);

  SSchema* pSchema = getTableColumnSchema(pTableMeta);
  ret = smlBoundColumnData(colsSchema, &pTableCxt->boundColsInfo, pSchema, false);
  TSDB_CHECK_CODE(ret, lino, end);

  ret = initTableColSubmitData(pTableCxt);
  TSDB_CHECK_CODE(ret, lino, end);

  int32_t rowNum = taosArrayGetSize(cols);
  TSDB_CHECK_CONDITION(rowNum > 0, ret, lino, end, TSDB_CODE_TSC_INVALID_OPERATION);

  for (int32_t r = 0; r < rowNum; ++r) {
    void* rowData = taosArrayGetP(cols, r);
    TSDB_CHECK_NULL(rowData, ret, lino, end, terrno);

    // 1. set the parsed value from sql string
    for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) {
      SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]];
      SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]);
      TSDB_CHECK_NULL(pVal, ret, lino, end, terrno);
      void**   p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name));
      if (p == NULL) {
        continue;
      }
      SSmlKv* kv = *(SSmlKv**)p;
      TSDB_CHECK_CONDITION(kv->type == pColSchema->type, ret, lino, end, TSDB_CODE_TSC_INVALID_OPERATION);

      if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) {
        kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision);
      }
      if (kv->type == TSDB_DATA_TYPE_NCHAR) {
        ret = smlMbsToUcs4(kv->value, kv->length, (void**)&pVal->value.pData, (int32_t*)&pVal->value.nData, pColSchema->bytes - VARSTR_HEADER_SIZE, charsetCxt);
        TSDB_CHECK_CODE(ret, lino, end);
      } else if (kv->type == TSDB_DATA_TYPE_BINARY) {
        pVal->value.nData = kv->length;
        pVal->value.pData = (uint8_t*)kv->value;
      } else if (kv->type == TSDB_DATA_TYPE_GEOMETRY || kv->type == TSDB_DATA_TYPE_VARBINARY) {
        pVal->value.nData = kv->length;
        pVal->value.pData = taosMemoryMalloc(kv->length);
        TSDB_CHECK_NULL(pVal->value.pData, ret, lino, end, terrno);
        (void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);
      } else {
        (void)memcpy(&pVal->value.val, &(kv->value), kv->length);
      }
      pVal->flag = CV_FLAG_VALUE;
    }

    SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1);
    TSDB_CHECK_NULL(pRow, ret, lino, end, terrno);
    ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow);
    TSDB_CHECK_CODE(ret, lino, end);
    SRowKey key = {0};
    tRowGetKey(*pRow, &key);
    insCheckTableDataOrder(pTableCxt, &key);
    clearColValArraySml(pTableCxt->pValues);
  }

end:
  if (ret != 0){
    uError("%s failed at %d since %s", __func__, lino, tstrerror(ret));
    ret = buildInvalidOperationMsg(&pBuf, tstrerror(ret));
  }
  insDestroyBoundColInfo(&bindTags);
  tdDestroySVCreateTbReq(pCreateTblReq);
  taosMemoryFree(pCreateTblReq);
  taosArrayDestroy(tagName);
  return ret;
}

int32_t smlInitHandle(SQuery** query) {
  int32_t lino = 0;
  int32_t code = 0;
  SQuery* pQuery = NULL;
  SVnodeModifyOpStmt* stmt = NULL;
  TSDB_CHECK_NULL(query, code, lino, end, TSDB_CODE_INVALID_PARA);

  *query = NULL;
  code = nodesMakeNode(QUERY_NODE_QUERY, (SNode**)&pQuery);
  TSDB_CHECK_CODE(code, lino, end);
  pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;
  pQuery->haveResultSet = false;
  pQuery->msgType = TDMT_VND_SUBMIT;
  code = nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT, (SNode**)&stmt);
  TSDB_CHECK_CODE(code, lino, end);
  stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
  TSDB_CHECK_NULL(stmt->pTableBlockHashObj, code, lino, end, terrno);
  stmt->freeHashFunc = insDestroyTableDataCxtHashMap;
  stmt->freeArrayFunc = insDestroyVgroupDataCxtList;

  pQuery->pRoot = (SNode*)stmt;
  *query = pQuery;
  return code;

end:
  if (code != 0) {
    uError("%s failed at %d since %s", __func__, lino, tstrerror(code));
  }
  nodesDestroyNode((SNode*)stmt);
  qDestroyQuery(pQuery);
  return code;
}

int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) {
  int32_t lino = 0;
  int32_t code = 0;

  SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)(handle)->pRoot;
  code = insMergeTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks, true);
  TSDB_CHECK_CODE(code, lino, end);
  code = insBuildVgDataBlocks(pVgHash, pStmt->pVgDataBlocks, &pStmt->pDataBlocks, false);
  TSDB_CHECK_CODE(code, lino, end);

end:
  if (code != 0) {
    uError("%s failed at %d since %s", __func__, lino, tstrerror(code));
  }
  return code;
}

1	/*
2	* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3	*
4	* This program is free software: you can use, redistribute, and/or modify
5	* it under the terms of the GNU Affero General Public License, version 3
6	* or later ("AGPL"), as published by the Free Software Foundation.
7	*
8	* This program is distributed in the hope that it will be useful, but WITHOUT
9	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10	* FITNESS FOR A PARTICULAR PURPOSE.
11	*
12	* You should have received a copy of the GNU Affero General Public License
13	* along with this program. If not, see <http://www.gnu.org/licenses/>.
14	*/
15
16	#include "parInsertUtil.h"
17	#include "parInt.h"
18	#include "parToken.h"
19	#include "ttime.h"
20
21	int32_t qCreateSName(SName* pName, const char* pTableName, int32_t acctId, char* dbName, char* msgBuf,	93,297✔
22	int32_t msgBufLen) {
23	SMsgBuf msg = {.buf = msgBuf, .len = msgBufLen};	93,297✔
24	SToken sToken = {0};	93,297✔
25	int code = TSDB_CODE_SUCCESS;	93,297✔
26	int32_t lino = 0;	93,297✔
27
28	NEXT_TOKEN(pTableName, sToken);	93,297✔
29	TSDB_CHECK_CONDITION(sToken.n != 0, code, lino, end, TSDB_CODE_TSC_INVALID_OPERATION);	93,311!
30	code = insCreateSName(pName, &sToken, acctId, dbName, &msg);	93,311✔
31	TSDB_CHECK_CODE(code, lino, end);	93,305!
32	NEXT_TOKEN(pTableName, sToken);	93,305✔
33	TSDB_CHECK_CONDITION(sToken.n <= 0, code, lino, end, TSDB_CODE_TSC_INVALID_OPERATION);	93,307!
34
35	end:	93,307✔
36	if (code != 0){	93,307!
UNCOV 37	uError("%s failed at %d since %s", __func__, lino, tstrerror(code));	×
38	}
39	return TSDB_CODE_SUCCESS;	93,305✔
40	}
41
42	static int32_t smlBoundColumnData(SArray* cols, SBoundColInfo* pBoundInfo, SSchema* pSchema, bool isTag) {	3,320✔
43	int code = TSDB_CODE_SUCCESS;	3,320✔
44	int32_t lino = 0;	3,320✔
45	bool* pUseCols = taosMemoryCalloc(pBoundInfo->numOfCols, sizeof(bool));	3,320!
46	TSDB_CHECK_NULL(pUseCols, code, lino, end, terrno);	3,324!
47	pBoundInfo->numOfBound = 0;	3,324✔
48	int16_t lastColIdx = -1; // last column found	3,324✔
49
50	for (int i = 0; i < taosArrayGetSize(cols); ++i) {	28,385✔
51	SSmlKv* kv = taosArrayGet(cols, i);	25,058✔
52	SToken sToken = {.n = kv->keyLen, .z = (char*)kv->key};	25,061✔
53	col_id_t t = lastColIdx + 1;	25,061✔
54	col_id_t index = ((t == 0 && !isTag) ? 0 : insFindCol(&sToken, t, pBoundInfo->numOfCols, pSchema));	25,061✔
55	uTrace("SML, index:%d, t:%d, ncols:%d", index, t, pBoundInfo->numOfCols);	25,060!
56	if (index < 0 && t > 0) {	25,060!
57	index = insFindCol(&sToken, 0, t, pSchema);	62✔
58	}
59
60	TSDB_CHECK_CONDITION(index >= 0, code, lino, end, TSDB_CODE_SML_INVALID_DATA);	25,061!
61	TSDB_CHECK_CONDITION(!pUseCols[index], code, lino, end, TSDB_CODE_SML_INVALID_DATA);	25,061!
62
63	lastColIdx = index;	25,061✔
64	pUseCols[index] = true;	25,061✔
65	pBoundInfo->pColIndex[pBoundInfo->numOfBound] = index;	25,061✔
66	++pBoundInfo->numOfBound;	25,061✔
67	}
68
69	end:	3,322✔
70	if (code != 0){	3,322!
71	uError("%s failed at %d since %s", __func__, lino, tstrerror(code));	×
72	}
73	taosMemoryFree(pUseCols);	3,322!
74	return code;	3,325✔
75	}
76
77	static int32_t smlMbsToUcs4(const char* mbs, size_t mbsLen, void** result, int32_t* resultLen, int32_t maxLen, void* charsetCxt){	10,936✔
78	int code = TSDB_CODE_SUCCESS;	10,936✔
79	void* pUcs4 = NULL;	10,936✔
80	int32_t lino = 0;	10,936✔
81	pUcs4 = taosMemoryCalloc(1, maxLen);	10,936!
82	TSDB_CHECK_NULL(pUcs4, code, lino, end, terrno);	10,938!
83	TSDB_CHECK_CONDITION(taosMbsToUcs4(mbs, mbsLen, (TdUcs4*)pUcs4, maxLen, resultLen, charsetCxt), code, lino, end, terrno);	10,938!
84	*result = pUcs4;	10,939✔
85	pUcs4 = NULL;	10,939✔
86
87	end:	10,939✔
88	if (code != 0){	10,939!
89	uError("%s failed at %d since %s", __func__, lino, tstrerror(code));	×
90	}
91	taosMemoryFree(pUcs4);	10,939!
92	return code;	10,938✔
93	}
94	/**
95	* @brief No json tag for schemaless
96	*
97	* @param cols
98	* @param tags
99	* @param pSchema
100	* @param ppTag
101	* @param msg
102	* @return int32_t
103	*/
104	static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchema, STag ppTag, SArray tagName,	1,963✔
105	SMsgBuf* msg, void* charsetCxt) {
106	int code = TSDB_CODE_SUCCESS;	1,963✔
107	int32_t lino = 0;	1,963✔
108	SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal));	1,963✔
109	TSDB_CHECK_NULL(pTagArray, code, lino, end, terrno);	1,966!
110	*tagName = taosArrayInit(8, TSDB_COL_NAME_LEN);	1,966✔
111	TSDB_CHECK_NULL(*tagName, code, lino, end, terrno);	1,966!
112
113	for (int i = 0; i < tags->numOfBound; ++i) {	16,301✔
114	SSchema* pTagSchema = &pSchema[tags->pColIndex[i]];	14,333✔
115	SSmlKv* kv = taosArrayGet(cols, i);	14,333✔
116	TSDB_CHECK_NULL(kv, code, lino, end, terrno);	14,332!
117	bool cond = (kv->keyLen == strlen(pTagSchema->name) && memcmp(kv->key, pTagSchema->name, kv->keyLen) == 0 && kv->type == pTagSchema->type);	14,334!
118	TSDB_CHECK_CONDITION(cond, code, lino, end, TSDB_CODE_SML_INVALID_DATA);	14,334!
119	TSDB_CHECK_NULL(taosArrayPush(*tagName, pTagSchema->name), code, lino, end, terrno);	28,668!
120	STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type};	14,334✔
121	if (pTagSchema->type == TSDB_DATA_TYPE_BINARY \|\| pTagSchema->type == TSDB_DATA_TYPE_VARBINARY \|\|	14,334!
122	pTagSchema->type == TSDB_DATA_TYPE_GEOMETRY) {	13,168!
123	val.pData = (uint8_t*)kv->value;	1,164✔
124	val.nData = kv->length;	1,164✔
125	} else if (pTagSchema->type == TSDB_DATA_TYPE_NCHAR) {	13,170✔
126	code = smlMbsToUcs4(kv->value, kv->length, (void*)&val.pData, (int32_t)&val.nData, kv->length * TSDB_NCHAR_SIZE, charsetCxt);	9,402✔
127	TSDB_CHECK_CODE(code, lino, end);	9,402!
128	} else {
129	(void)memcpy(&val.i64, &(kv->value), kv->length);	3,768✔
130	}
131	TSDB_CHECK_NULL(taosArrayPush(pTagArray, &val), code, lino, end, terrno);	14,335!
132	}
133	code = tTagNew(pTagArray, 1, false, ppTag);	1,968✔
134
135	end:	1,966✔
136	if (code != 0){	1,966!
137	uError("%s failed at %d since %s", __func__, lino, tstrerror(code));	×
138	}
139	for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) {	16,288✔
140	STagVal* p = (STagVal*)taosArrayGet(pTagArray, i);	14,329✔
141	if (p->type == TSDB_DATA_TYPE_NCHAR) {	14,328✔
142	taosMemoryFree(p->pData);	9,403!
143	}
144	}
145	taosArrayDestroy(pTagArray);	1,963✔
146	return code;	1,966✔
147	}
148
149	int32_t smlInitTableDataCtx(SQuery* query, STableMeta* pTableMeta, STableDataCxt** cxt) {	725✔
150	int ret = TSDB_CODE_SUCCESS;	725✔
151	int32_t lino = 0;	725✔
152	SVCreateTbReq* pCreateTbReq = NULL;	725✔
153	ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid,	725✔
154	sizeof(pTableMeta->uid), pTableMeta, &pCreateTbReq, cxt, false, false);
155	TSDB_CHECK_CODE(ret, lino, end);	728!
156	ret = initTableColSubmitData(*cxt);	728✔
157	TSDB_CHECK_CODE(ret, lino, end);	726!
158
159	end:	726✔
160	if (ret != 0){	726!
161	uError("%s failed at %d since %s", __func__, lino, tstrerror(ret));	×
162	}
163	return ret;	726✔
164	}
165
166	void clearColValArraySml(SArray* pCols) {	101,835✔
167	int32_t num = taosArrayGetSize(pCols);	101,835✔
168	for (int32_t i = 0; i < num; ++i) {	461,652✔
169	SColVal* pCol = taosArrayGet(pCols, i);	364,228✔
170	if (TSDB_DATA_TYPE_NCHAR == pCol->value.type \|\| TSDB_DATA_TYPE_GEOMETRY == pCol->value.type \|\|	360,402!
171	TSDB_DATA_TYPE_VARBINARY == pCol->value.type) {	358,943✔
172	taosMemoryFreeClear(pCol->value.pData);	1,477!
173	}
174	pCol->flag = CV_FLAG_NONE;	360,403✔
175	pCol->value.val = 0;	360,403✔
176	}
177	}	97,424✔
178
179	int32_t smlBuildRow(STableDataCxt* pTableCxt) {	5,068✔
180	int ret = TSDB_CODE_SUCCESS;	5,068✔
181	int32_t lino = 0;	5,068✔
182	SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1);	5,068✔
183	TSDB_CHECK_NULL(pRow, ret, lino, end, terrno);	5,068!
184	ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow);	5,068✔
185	TSDB_CHECK_CODE(ret, lino, end);	5,078!
186	SRowKey key;
187	tRowGetKey(*pRow, &key);	5,078!
188	insCheckTableDataOrder(pTableCxt, &key);	5,078✔
189	end:	5,076✔
190	if (ret != 0){	5,076!
191	uError("%s failed at %d since %s", __func__, lino, tstrerror(ret));	×
192	}
193	return ret;	5,078✔
194	}
195
196	int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32_t index, void* charsetCxt) {	10,252✔
197	int ret = TSDB_CODE_SUCCESS;	10,252✔
198	int32_t lino = 0;	10,252✔
199	SSchema* pColSchema = schema + index;	10,252✔
200	SColVal* pVal = taosArrayGet(pTableCxt->pValues, index);	10,252✔
201	TSDB_CHECK_NULL(pVal, ret, lino, end, TSDB_CODE_SUCCESS);	10,252!
202	SSmlKv* kv = (SSmlKv*)data;	10,252✔
203	if (kv->keyLen != strlen(pColSchema->name) \|\| memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 \|\|	10,252✔
204	kv->type != pColSchema->type) {	10,243!
205	ret = TSDB_CODE_SML_INVALID_DATA;	9✔
206	char* tmp = taosMemoryCalloc(kv->keyLen + 1, 1);	9!
207	TSDB_CHECK_NULL(tmp, ret, lino, end, terrno);	12!
208	(void)memcpy(tmp, kv->key, kv->keyLen);	12✔
209	uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)", tmp, tDataTypes[kv->type].name,	12!
210	pColSchema->name, tDataTypes[pColSchema->type].name);
211	taosMemoryFree(tmp);	12!
212	goto end;	12✔
213	}
214	if (kv->type == TSDB_DATA_TYPE_NCHAR) {	10,243✔
215	ret = smlMbsToUcs4(kv->value, kv->length, (void**)&pVal->value.pData, &pVal->value.nData, pColSchema->bytes - VARSTR_HEADER_SIZE, charsetCxt);	239✔
216	TSDB_CHECK_CODE(ret, lino, end);	240!
217	} else if (kv->type == TSDB_DATA_TYPE_BINARY) {	10,004✔
218	pVal->value.nData = kv->length;	730✔
219	pVal->value.pData = (uint8_t*)kv->value;	730✔
220	} else if (kv->type == TSDB_DATA_TYPE_GEOMETRY \|\| kv->type == TSDB_DATA_TYPE_VARBINARY) {	9,274!
221	pVal->value.nData = kv->length;	×
222	pVal->value.pData = taosMemoryMalloc(kv->length);	×
223	TSDB_CHECK_NULL(pVal->value.pData, ret, lino, end, terrno);	×
224
225	(void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);	×
226	} else {
227	(void)memcpy(&pVal->value.val, &(kv->value), kv->length);	9,278✔
228	}
229	pVal->flag = CV_FLAG_VALUE;	10,248✔
230
231	end:	10,260✔
232	if (ret != 0){	10,260✔
233	uError("%s failed at %d since %s", __func__, lino, tstrerror(ret));	12!
234	}
235	return ret;	10,254✔
236	}
237
238	int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSchema, SArray* cols,	1,961✔
239	STableMeta* pTableMeta, char* tableName, const char* sTableName, int32_t sTableNameLen, int32_t ttl,
240	char* msgBuf, int32_t msgBufLen, void* charsetCxt) {
241	int32_t lino = 0;	1,961✔
242	int32_t ret = 0;	1,961✔
243	SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen};	1,961✔
244
245	SSchema* pTagsSchema = getTableTagSchema(pTableMeta);	1,961✔
246	SBoundColInfo bindTags = {0};	1,961✔
247	SVCreateTbReq* pCreateTblReq = NULL;	1,961✔
248	SArray* tagName = NULL;	1,961✔
249
250	ret = insInitBoundColsInfo(getNumOfTags(pTableMeta), &bindTags);	1,961✔
251	TSDB_CHECK_CODE(ret, lino, end);	1,966!
252
253	ret = smlBoundColumnData(tags, &bindTags, pTagsSchema, true);	1,966✔
254	TSDB_CHECK_CODE(ret, lino, end);	1,965!
255
256	STag* pTag = NULL;	1,965✔
257	ret = smlBuildTagRow(tags, &bindTags, pTagsSchema, &pTag, &tagName, &pBuf, charsetCxt);	1,965✔
258	TSDB_CHECK_CODE(ret, lino, end);	1,966!
259
260	pCreateTblReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq));	1,966!
261	TSDB_CHECK_NULL(pCreateTblReq, ret, lino, end, terrno);	1,966!
262
263	ret = insBuildCreateTbReq(pCreateTblReq, tableName, pTag, pTableMeta->suid, NULL, tagName,	1,966✔
264	pTableMeta->tableInfo.numOfTags, ttl);	1,966✔
265	TSDB_CHECK_CODE(ret, lino, end);	1,965!
266
267	pCreateTblReq->ctb.stbName = taosMemoryCalloc(1, sTableNameLen + 1);	1,965!
268	TSDB_CHECK_NULL(pCreateTblReq->ctb.stbName, ret, lino, end, terrno);	1,964!
269
270	(void)memcpy(pCreateTblReq->ctb.stbName, sTableName, sTableNameLen);	1,964✔
271
272	if (dataFormat) {	1,964✔
273	STableDataCxt pTableCxt = (STableDataCxt)taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj,	606✔
274	&pTableMeta->uid, sizeof(pTableMeta->uid));	606✔
275	TSDB_CHECK_NULL(pTableCxt, ret, lino, end, TSDB_CODE_TSC_INVALID_OPERATION);	605!
276	(*pTableCxt)->pData->flags \|= SUBMIT_REQ_AUTO_CREATE_TABLE;	605✔
277	(*pTableCxt)->pData->pCreateTbReq = pCreateTblReq;	605✔
278	(*pTableCxt)->pMeta->uid = pTableMeta->uid;	605✔
279	(*pTableCxt)->pMeta->vgId = pTableMeta->vgId;	605✔
280	pCreateTblReq = NULL;	605✔
281	goto end;	605✔
282	}
283
284	STableDataCxt* pTableCxt = NULL;	1,358✔
285	ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid,	1,358✔
286	sizeof(pTableMeta->uid), pTableMeta, &pCreateTblReq, &pTableCxt, false, false);
287	TSDB_CHECK_CODE(ret, lino, end);	1,360!
288
289	SSchema* pSchema = getTableColumnSchema(pTableMeta);	1,360✔
290	ret = smlBoundColumnData(colsSchema, &pTableCxt->boundColsInfo, pSchema, false);	1,360✔
291	TSDB_CHECK_CODE(ret, lino, end);	1,360!
292
293	ret = initTableColSubmitData(pTableCxt);	1,360✔
294	TSDB_CHECK_CODE(ret, lino, end);	1,360!
295
296	int32_t rowNum = taosArrayGetSize(cols);	1,360✔
297	TSDB_CHECK_CONDITION(rowNum > 0, ret, lino, end, TSDB_CODE_TSC_INVALID_OPERATION);	1,360!
298
299	for (int32_t r = 0; r < rowNum; ++r) {	87,214✔
300	void* rowData = taosArrayGetP(cols, r);	85,854✔
301	TSDB_CHECK_NULL(rowData, ret, lino, end, terrno);	86,071!
302
303	// 1. set the parsed value from sql string
304	for (int c = 0; c < pTableCxt->boundColsInfo.numOfBound; ++c) {	405,710✔
305	SSchema* pColSchema = &pSchema[pTableCxt->boundColsInfo.pColIndex[c]];	311,248✔
306	SColVal* pVal = taosArrayGet(pTableCxt->pValues, pTableCxt->boundColsInfo.pColIndex[c]);	311,248✔
307	TSDB_CHECK_NULL(pVal, ret, lino, end, terrno);	304,704!
308	void** p = taosHashGet(rowData, pColSchema->name, strlen(pColSchema->name));	304,704✔
309	if (p == NULL) {	321,431✔
310	continue;	276✔
311	}
312	SSmlKv* kv = (SSmlKv*)p;	321,155✔
313	TSDB_CHECK_CONDITION(kv->type == pColSchema->type, ret, lino, end, TSDB_CODE_TSC_INVALID_OPERATION);	321,155!
314
315	if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) {	321,155✔
316	kv->i = convertTimePrecision(kv->i, TSDB_TIME_PRECISION_NANO, pTableMeta->tableInfo.precision);	90,189✔
317	}
318	if (kv->type == TSDB_DATA_TYPE_NCHAR) {	321,312✔
319	ret = smlMbsToUcs4(kv->value, kv->length, (void*)&pVal->value.pData, (int32_t)&pVal->value.nData, pColSchema->bytes - VARSTR_HEADER_SIZE, charsetCxt);	1,296✔
320	TSDB_CHECK_CODE(ret, lino, end);	1,296!
321	} else if (kv->type == TSDB_DATA_TYPE_BINARY) {	320,016✔
322	pVal->value.nData = kv->length;	1,614✔
323	pVal->value.pData = (uint8_t*)kv->value;	1,614✔
324	} else if (kv->type == TSDB_DATA_TYPE_GEOMETRY \|\| kv->type == TSDB_DATA_TYPE_VARBINARY) {	318,402!
325	pVal->value.nData = kv->length;	2,010✔
326	pVal->value.pData = taosMemoryMalloc(kv->length);	2,010!
327	TSDB_CHECK_NULL(pVal->value.pData, ret, lino, end, terrno);	48!
328	(void)memcpy(pVal->value.pData, (uint8_t*)kv->value, kv->length);	48✔
329	} else {
330	(void)memcpy(&pVal->value.val, &(kv->value), kv->length);	316,392✔
331	}
332	pVal->flag = CV_FLAG_VALUE;	319,350✔
333	}
334
335	SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1);	94,462✔
336	TSDB_CHECK_NULL(pRow, ret, lino, end, terrno);	91,981!
337	ret = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow);	91,981✔
338	TSDB_CHECK_CODE(ret, lino, end);	97,270!
339	SRowKey key = {0};	97,270✔
340	tRowGetKey(*pRow, &key);	97,270!
341	insCheckTableDataOrder(pTableCxt, &key);	97,270✔
342	clearColValArraySml(pTableCxt->pValues);	97,378✔
343	}
344
345	end:	1,360✔
346	if (ret != 0){	1,965!
347	uError("%s failed at %d since %s", __func__, lino, tstrerror(ret));	×
348	ret = buildInvalidOperationMsg(&pBuf, tstrerror(ret));	×
349	}
350	insDestroyBoundColInfo(&bindTags);	1,965✔
351	tdDestroySVCreateTbReq(pCreateTblReq);	1,964✔
352	taosMemoryFree(pCreateTblReq);	1,964!
353	taosArrayDestroy(tagName);	1,965✔
354	return ret;	1,965✔
355	}
356
357	int32_t smlInitHandle(SQuery** query) {	1,264✔
358	int32_t lino = 0;	1,264✔
359	int32_t code = 0;	1,264✔
360	SQuery* pQuery = NULL;	1,264✔
361	SVnodeModifyOpStmt* stmt = NULL;	1,264✔
362	TSDB_CHECK_NULL(query, code, lino, end, TSDB_CODE_INVALID_PARA);	1,264✔
363
364	*query = NULL;	1,263✔
365	code = nodesMakeNode(QUERY_NODE_QUERY, (SNode**)&pQuery);	1,263✔
366	TSDB_CHECK_CODE(code, lino, end);	1,269!
367	pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE;	1,269✔
368	pQuery->haveResultSet = false;	1,269✔
369	pQuery->msgType = TDMT_VND_SUBMIT;	1,269✔
370	code = nodesMakeNode(QUERY_NODE_VNODE_MODIFY_STMT, (SNode**)&stmt);	1,269✔
371	TSDB_CHECK_CODE(code, lino, end);	1,269!
372	stmt->pTableBlockHashObj = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);	1,269✔
373	TSDB_CHECK_NULL(stmt->pTableBlockHashObj, code, lino, end, terrno);	1,269!
374	stmt->freeHashFunc = insDestroyTableDataCxtHashMap;	1,269✔
375	stmt->freeArrayFunc = insDestroyVgroupDataCxtList;	1,269✔
376
377	pQuery->pRoot = (SNode*)stmt;	1,269✔
378	*query = pQuery;	1,269✔
379	return code;	1,269✔
380
381	end:	1✔
382	if (code != 0) {	1!
383	uError("%s failed at %d since %s", __func__, lino, tstrerror(code));	1!
384	}
385	nodesDestroyNode((SNode*)stmt);	1✔
386	qDestroyQuery(pQuery);	1✔
387	return code;	1✔
388	}
389
390	int32_t smlBuildOutput(SQuery* handle, SHashObj* pVgHash) {	1,128✔
391	int32_t lino = 0;	1,128✔
392	int32_t code = 0;	1,128✔
393
394	SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)(handle)->pRoot;	1,128✔
395	code = insMergeTableDataCxt(pStmt->pTableBlockHashObj, &pStmt->pVgDataBlocks, true);	1,128✔
396	TSDB_CHECK_CODE(code, lino, end);	1,131!
397	code = insBuildVgDataBlocks(pVgHash, pStmt->pVgDataBlocks, &pStmt->pDataBlocks, false);	1,131✔
398	TSDB_CHECK_CODE(code, lino, end);	1,129!
399
400	end:	1,129✔
401	if (code != 0) {	1,129!
402	uError("%s failed at %d since %s", __func__, lino, tstrerror(code));	×
403	}
404	return code;	1,130✔
405	}

taosdata / TDengine / #3608

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous