#3653

Committed 14 Mar 2025 08:10AM UTC coverage: 22.565% (-41.0%) from 63.596%

Build # #3653

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

feat(keep): support keep on super table level. (#30097)

* Feat: support use keep while create super table.

* Test(keep): add test for create super table with keep option.

* Feat(keep): Add tmsg for create keep.

* Feat(keep): support alter table option keep.

* Fix(keep): Add baisc test for alter table option.

* Fix(keep): memory leek.

* Feat(keep): add keep to metaEntry&metaCache and fix earliestTs with stn keep.

* Test(keep): add some cases for select with stb keep.

* Fix: fix ci core while alter stb.

* Feat(keep): delete expired data in super table level.

* Feat: remove get stb keep while query.

* Fix : build error.

* Revert "Fix : build error."

This reverts commit 0ed66e4e8.

* Revert "Feat(keep): delete expired data in super table level."

This reverts commit 36330f6b4.

* Fix : build errors.

* Feat : support restart taosd.

* Fix : alter table comment problems.

* Test : add tests for super table keep.

* Fix: change sdb stb reserve size.

* Test: add more tests.

* Feat: Disable normal tables and sub tables from setting the keep parameter

* Fix: add more checks to avoid unknown address.

* Docs: Add docs for stable keep.

* Fix: some review changes.

* Fix: review errors.

Run Details

49248 of 302527 branches covered (16.28%)

Branch coverage included in aggregate %.

53 of 99 new or added lines in 12 files covered. (53.54%)

155872 existing lines in 443 files now uncovered.

87359 of 302857 relevant lines covered (28.84%)

570004.22 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

15.38

/source/dnode/mgmt/mgmt_qnode/src/qmWorker.c

/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http:www.gnu.org/licenses/>.
 */

#define _DEFAULT_SOURCE
#include "qmInt.h"

static inline void qmSendRsp(SRpcMsg *pMsg, int32_t code) {
  SRpcMsg rsp = {
      .code = code,
      .pCont = pMsg->info.rsp,
      .contLen = pMsg->info.rspLen,
      .info = pMsg->info,
  };
  tmsgSendRsp(&rsp);
}

static void qmProcessQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) {
  SQnodeMgmt *pMgmt = pInfo->ahandle;
  dTrace("msg:%p, get from qnode queue", pMsg);

  int32_t code = qndProcessQueryMsg(pMgmt->pQnode, pInfo, pMsg);
  if (IsReq(pMsg) && code != TSDB_CODE_ACTION_IN_PROGRESS) {
    if (code != 0 && terrno != 0) code = terrno;
    qmSendRsp(pMsg, code);
  }

  dTrace("msg:%p, is freed, code:0x%x", pMsg, code);
  rpcFreeCont(pMsg->pCont);
  taosFreeQitem(pMsg);
}

static int32_t qmPutNodeMsgToWorker(SSingleWorker *pWorker, SRpcMsg *pMsg) {
  dTrace("msg:%p, put into worker %s, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType));
  return taosWriteQitem(pWorker->queue, pMsg);
}

int32_t qmPutNodeMsgToQueryQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg) {
  int32_t code = qndPreprocessQueryMsg(pMgmt->pQnode, pMsg);
  if (code) return code;
  return qmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg);
}

int32_t qmPutNodeMsgToFetchQueue(SQnodeMgmt *pMgmt, SRpcMsg *pMsg) {
  return qmPutNodeMsgToWorker(&pMgmt->fetchWorker, pMsg);
}

int32_t qmPutRpcMsgToQueue(SQnodeMgmt *pMgmt, EQueueType qtype, SRpcMsg *pRpc) {
  int32_t  code;
  SRpcMsg *pMsg;

  code = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen, (void **)&pMsg);
  if (code) return code;
  memcpy(pMsg, pRpc, sizeof(SRpcMsg));
  pRpc->pCont = NULL;

  switch (qtype) {
    case QUERY_QUEUE:
      dTrace("msg:%p, is created and will put into qnode-query queue, len:%d", pMsg, pRpc->contLen);
      code = taosWriteQitem(pMgmt->queryWorker.queue, pMsg);
      return code;
    case READ_QUEUE:
    case STATUS_QUEUE:
    case FETCH_QUEUE:
      dTrace("msg:%p, is created and will put into qnode-fetch queue, len:%d", pMsg, pRpc->contLen);
      code = taosWriteQitem(pMgmt->fetchWorker.queue, pMsg);
      return code;
    default:
      terrno = TSDB_CODE_INVALID_PARA;
      rpcFreeCont(pMsg->pCont);
      taosFreeQitem(pMsg);
      return terrno;
  }
}

int32_t qmGetQueueSize(SQnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {
  int32_t size = -1;

  switch (qtype) {
    case QUERY_QUEUE:
      size = taosQueueItemSize(pMgmt->queryWorker.queue);
      break;
    case FETCH_QUEUE:
      size = taosQueueItemSize(pMgmt->fetchWorker.queue);
      break;
    default:
      break;
  }

  return size;
}

int32_t qmStartWorker(SQnodeMgmt *pMgmt) {
  int32_t code = 0;

  SSingleWorkerCfg queryCfg = {
      .min = tsNumOfQnodeQueryThreads,
      .max = tsNumOfQnodeQueryThreads,
      .name = "qnode-query",
      .fp = (FItem)qmProcessQueue,
      .param = pMgmt,
      .poolType = QUERY_AUTO_QWORKER_POOL,
  };

  if ((code = tSingleWorkerInit(&pMgmt->queryWorker, &queryCfg)) != 0) {
    dError("failed to start qnode-query worker since %s", tstrerror(code));
    return code;
  }

  tsNumOfQueryThreads += tsNumOfQnodeQueryThreads;

  SSingleWorkerCfg fetchCfg = {
      .min = tsNumOfQnodeFetchThreads,
      .max = tsNumOfQnodeFetchThreads,
      .name = "qnode-fetch",
      .fp = (FItem)qmProcessQueue,
      .param = pMgmt,
  };

  if ((code = tSingleWorkerInit(&pMgmt->fetchWorker, &fetchCfg)) != 0) {
    dError("failed to start qnode-fetch worker since %s", tstrerror(code));
    return code;
  }

  dDebug("qnode workers are initialized");
  return code;
}

void qmStopWorker(SQnodeMgmt *pMgmt) {
  tSingleWorkerCleanup(&pMgmt->queryWorker);
  tSingleWorkerCleanup(&pMgmt->fetchWorker);
  dDebug("qnode workers are closed");
}

1	/*
2	* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3	*
4	* This program is free software: you can use, redistribute, and/or modify
5	* it under the terms of the GNU Affero General Public License, version 3
6	* or later ("AGPL"), as published by the Free Software Foundation.
7	*
8	* This program is distributed in the hope that it will be useful, but WITHOUT
9	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10	* FITNESS FOR A PARTICULAR PURPOSE.
11	*
12	* You should have received a copy of the GNU Affero General Public License
13	* along with this program. If not, see <http:www.gnu.org/licenses/>.
14	*/
15
16	#define _DEFAULT_SOURCE
17	#include "qmInt.h"
18
19	static inline void qmSendRsp(SRpcMsg *pMsg, int32_t code) {	×
20	SRpcMsg rsp = {	×
21	.code = code,
22	.pCont = pMsg->info.rsp,	×
23	.contLen = pMsg->info.rspLen,	×
24	.info = pMsg->info,
25	};
26	tmsgSendRsp(&rsp);	×
27	}	×
28
UNCOV 29	static void qmProcessQueue(SQueueInfo pInfo, SRpcMsg pMsg) {	×
UNCOV 30	SQnodeMgmt *pMgmt = pInfo->ahandle;	×
UNCOV 31	dTrace("msg:%p, get from qnode queue", pMsg);	×
32
UNCOV 33	int32_t code = qndProcessQueryMsg(pMgmt->pQnode, pInfo, pMsg);	×
UNCOV 34	if (IsReq(pMsg) && code != TSDB_CODE_ACTION_IN_PROGRESS) {	×
35	if (code != 0 && terrno != 0) code = terrno;	×
36	qmSendRsp(pMsg, code);	×
37	}
38
UNCOV 39	dTrace("msg:%p, is freed, code:0x%x", pMsg, code);	×
UNCOV 40	rpcFreeCont(pMsg->pCont);	×
UNCOV 41	taosFreeQitem(pMsg);	×
UNCOV 42	}	×
43
UNCOV 44	static int32_t qmPutNodeMsgToWorker(SSingleWorker pWorker, SRpcMsg pMsg) {	×
UNCOV 45	dTrace("msg:%p, put into worker %s, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType));	×
UNCOV 46	return taosWriteQitem(pWorker->queue, pMsg);	×
47	}
48
UNCOV 49	int32_t qmPutNodeMsgToQueryQueue(SQnodeMgmt pMgmt, SRpcMsg pMsg) {	×
UNCOV 50	int32_t code = qndPreprocessQueryMsg(pMgmt->pQnode, pMsg);	×
UNCOV 51	if (code) return code;	×
UNCOV 52	return qmPutNodeMsgToWorker(&pMgmt->queryWorker, pMsg);	×
53	}
54
UNCOV 55	int32_t qmPutNodeMsgToFetchQueue(SQnodeMgmt pMgmt, SRpcMsg pMsg) {	×
UNCOV 56	return qmPutNodeMsgToWorker(&pMgmt->fetchWorker, pMsg);	×
57	}
58
UNCOV 59	int32_t qmPutRpcMsgToQueue(SQnodeMgmt pMgmt, EQueueType qtype, SRpcMsg pRpc) {	×
60	int32_t code;
61	SRpcMsg *pMsg;
62
UNCOV 63	code = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM, pRpc->contLen, (void **)&pMsg);	×
UNCOV 64	if (code) return code;	×
UNCOV 65	memcpy(pMsg, pRpc, sizeof(SRpcMsg));	×
UNCOV 66	pRpc->pCont = NULL;	×
67
UNCOV 68	switch (qtype) {	×
UNCOV 69	case QUERY_QUEUE:	×
UNCOV 70	dTrace("msg:%p, is created and will put into qnode-query queue, len:%d", pMsg, pRpc->contLen);	×
UNCOV 71	code = taosWriteQitem(pMgmt->queryWorker.queue, pMsg);	×
UNCOV 72	return code;	×
73	case READ_QUEUE:	×
74	case STATUS_QUEUE:
75	case FETCH_QUEUE:
76	dTrace("msg:%p, is created and will put into qnode-fetch queue, len:%d", pMsg, pRpc->contLen);	×
77	code = taosWriteQitem(pMgmt->fetchWorker.queue, pMsg);	×
78	return code;	×
79	default:	×
80	terrno = TSDB_CODE_INVALID_PARA;	×
81	rpcFreeCont(pMsg->pCont);	×
82	taosFreeQitem(pMsg);	×
83	return terrno;	×
84	}
85	}
86
UNCOV 87	int32_t qmGetQueueSize(SQnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) {	×
UNCOV 88	int32_t size = -1;	×
89
UNCOV 90	switch (qtype) {	×
UNCOV 91	case QUERY_QUEUE:	×
UNCOV 92	size = taosQueueItemSize(pMgmt->queryWorker.queue);	×
UNCOV 93	break;	×
UNCOV 94	case FETCH_QUEUE:	×
UNCOV 95	size = taosQueueItemSize(pMgmt->fetchWorker.queue);	×
UNCOV 96	break;	×
97	default:	×
98	break;	×
99	}
100
UNCOV 101	return size;	×
102	}
103
104	int32_t qmStartWorker(SQnodeMgmt *pMgmt) {	2✔
105	int32_t code = 0;	2✔
106
107	SSingleWorkerCfg queryCfg = {	2✔
108	.min = tsNumOfQnodeQueryThreads,
109	.max = tsNumOfQnodeQueryThreads,
110	.name = "qnode-query",
111	.fp = (FItem)qmProcessQueue,
112	.param = pMgmt,
113	.poolType = QUERY_AUTO_QWORKER_POOL,
114	};
115
116	if ((code = tSingleWorkerInit(&pMgmt->queryWorker, &queryCfg)) != 0) {	2!
117	dError("failed to start qnode-query worker since %s", tstrerror(code));	×
118	return code;	×
119	}
120
121	tsNumOfQueryThreads += tsNumOfQnodeQueryThreads;	2✔
122
123	SSingleWorkerCfg fetchCfg = {	2✔
124	.min = tsNumOfQnodeFetchThreads,
125	.max = tsNumOfQnodeFetchThreads,
126	.name = "qnode-fetch",
127	.fp = (FItem)qmProcessQueue,
128	.param = pMgmt,
129	};
130
131	if ((code = tSingleWorkerInit(&pMgmt->fetchWorker, &fetchCfg)) != 0) {	2!
132	dError("failed to start qnode-fetch worker since %s", tstrerror(code));	×
133	return code;	×
134	}
135
136	dDebug("qnode workers are initialized");	2!
137	return code;	2✔
138	}
139
140	void qmStopWorker(SQnodeMgmt *pMgmt) {	2✔
141	tSingleWorkerCleanup(&pMgmt->queryWorker);	2✔
142	tSingleWorkerCleanup(&pMgmt->fetchWorker);	2✔
143	dDebug("qnode workers are closed");	2!
144	}	2✔

taosdata / TDengine / #3653

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous