#3655

Committed 14 Mar 2025 08:10AM UTC coverage: 59.532% (+22.6%) from 36.951%

Build # #3655

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

feat(keep): support keep on super table level. (#30097)

* Feat: support use keep while create super table.

* Test(keep): add test for create super table with keep option.

* Feat(keep): Add tmsg for create keep.

* Feat(keep): support alter table option keep.

* Fix(keep): Add baisc test for alter table option.

* Fix(keep): memory leek.

* Feat(keep): add keep to metaEntry&metaCache and fix earliestTs with stn keep.

* Test(keep): add some cases for select with stb keep.

* Fix: fix ci core while alter stb.

* Feat(keep): delete expired data in super table level.

* Feat: remove get stb keep while query.

* Fix : build error.

* Revert "Fix : build error."

This reverts commit 0ed66e4e8.

* Revert "Feat(keep): delete expired data in super table level."

This reverts commit 36330f6b4.

* Fix : build errors.

* Feat : support restart taosd.

* Fix : alter table comment problems.

* Test : add tests for super table keep.

* Fix: change sdb stb reserve size.

* Test: add more tests.

* Feat: Disable normal tables and sub tables from setting the keep parameter

* Fix: add more checks to avoid unknown address.

* Docs: Add docs for stable keep.

* Fix: some review changes.

* Fix: review errors.

Run Details

139898 of 302527 branches covered (46.24%)

Branch coverage included in aggregate %.

71 of 99 new or added lines in 12 files covered. (71.72%)

2746 existing lines in 57 files now uncovered.

220499 of 302857 relevant lines covered (72.81%)

5509252.7 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

68.42

/source/dnode/vnode/src/tsdb/tsdbOpen.c

/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "meta.h"
#include "tsdb.h"
#include "tsdbFS2.h"

extern int32_t tsdbOpenCompMonitor(STsdb *tsdb);
extern void    tsdbCloseCompMonitor(STsdb *tsdb);

void tsdbSetKeepCfg(STsdb *pTsdb, STsdbCfg *pCfg) {
  STsdbKeepCfg *pKeepCfg = &pTsdb->keepCfg;
  pKeepCfg->precision = pCfg->precision;
  pKeepCfg->days = pCfg->days;
  pKeepCfg->keep0 = pCfg->keep0;
  pKeepCfg->keep1 = pCfg->keep1;
  pKeepCfg->keep2 = pCfg->keep2;
  pKeepCfg->keepTimeOffset = pCfg->keepTimeOffset;
}

int64_t tsdbGetEarliestTs(STsdb *pTsdb) {
  STsdbKeepCfg *pCfg = &pTsdb->keepCfg;

  int64_t now = taosGetTimestamp(pCfg->precision);
  int64_t ts = now - (tsTickPerMin[pCfg->precision] * pCfg->keep2) + 1;  // needs to add one tick
  return ts;
}

int32_t tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKeepCfg, int8_t rollback, bool force) {
  STsdb  *pTsdb = NULL;
  int     slen = 0;
  int32_t code;
  int32_t lino;

  *ppTsdb = NULL;
  slen = strlen(pVnode->path) + strlen(dir) + 2;

  // create handle
  pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(*pTsdb) + slen);
  if (pTsdb == NULL) {
    TAOS_CHECK_RETURN(terrno);
  }

  pTsdb->path = (char *)&pTsdb[1];
  snprintf(pTsdb->path, TD_PATH_MAX, "%s%s%s", pVnode->path, TD_DIRSEP, dir);
  // taosRealPath(pTsdb->path, NULL, slen);
  pTsdb->pVnode = pVnode;
  (void)taosThreadMutexInit(&pTsdb->mutex, NULL);
  if (!pKeepCfg) {
    tsdbSetKeepCfg(pTsdb, &pVnode->config.tsdbCfg);
  } else {
    memcpy(&pTsdb->keepCfg, pKeepCfg, sizeof(STsdbKeepCfg));
  }

  // create dir
  if (pVnode->pTfs) {
    code = tfsMkdir(pVnode->pTfs, pTsdb->path);
    TSDB_CHECK_CODE(code, lino, _exit);
  } else {
    code = taosMkDir(pTsdb->path);
    TSDB_CHECK_CODE(code, lino, _exit);
  }

  // open tsdb
  code = tsdbOpenFS(pTsdb, &pTsdb->pFS, rollback);
  TSDB_CHECK_CODE(code, lino, _exit);

  if (pTsdb->pFS->fsstate == TSDB_FS_STATE_INCOMPLETE && force == false) {
    TAOS_CHECK_GOTO(TSDB_CODE_NEED_RETRY, &lino, _exit);
  }

  code = tsdbOpenCache(pTsdb);
  TSDB_CHECK_CODE(code, lino, _exit);

#ifdef TD_ENTERPRISE
  TAOS_CHECK_GOTO(tsdbOpenCompMonitor(pTsdb), &lino, _exit);
#endif

_exit:
  if (code) {
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, lino, tstrerror(code));
    tsdbCloseFS(&pTsdb->pFS);
    (void)taosThreadMutexDestroy(&pTsdb->mutex);
    taosMemoryFree(pTsdb);
  } else {
    tsdbDebug("vgId:%d, tsdb is opened at %s, days:%d, keep:%d,%d,%d, keepTimeoffset:%d", TD_VID(pVnode), pTsdb->path,
              pTsdb->keepCfg.days, pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2,
              pTsdb->keepCfg.keepTimeOffset);
    *ppTsdb = pTsdb;
  }
  return code;
}

void tsdbClose(STsdb **pTsdb) {
  if (*pTsdb) {
    STsdb *pdb = *pTsdb;
    tsdbDebug("vgId:%d, tsdb is close at %s, days:%d, keep:%d,%d,%d, keepTimeOffset:%d", TD_VID(pdb->pVnode), pdb->path,
              pdb->keepCfg.days, pdb->keepCfg.keep0, pdb->keepCfg.keep1, pdb->keepCfg.keep2,
              pdb->keepCfg.keepTimeOffset);
    (void)taosThreadMutexLock(&(*pTsdb)->mutex);
    tsdbMemTableDestroy((*pTsdb)->mem, true);
    (*pTsdb)->mem = NULL;
    (void)taosThreadMutexUnlock(&(*pTsdb)->mutex);

    tsdbCloseFS(&(*pTsdb)->pFS);
    tsdbCloseCache(*pTsdb);
#ifdef TD_ENTERPRISE
    tsdbCloseCompMonitor(*pTsdb);
#endif
    (void)taosThreadMutexDestroy(&(*pTsdb)->mutex);
    taosMemoryFreeClear(*pTsdb);
  }
  return;
}

1	/*
2	* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3	*
4	* This program is free software: you can use, redistribute, and/or modify
5	* it under the terms of the GNU Affero General Public License, version 3
6	* or later ("AGPL"), as published by the Free Software Foundation.
7	*
8	* This program is distributed in the hope that it will be useful, but WITHOUT
9	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10	* FITNESS FOR A PARTICULAR PURPOSE.
11	*
12	* You should have received a copy of the GNU Affero General Public License
13	* along with this program. If not, see <http://www.gnu.org/licenses/>.
14	*/
15
16	#include "meta.h"
17	#include "tsdb.h"
18	#include "tsdbFS2.h"
19
20	extern int32_t tsdbOpenCompMonitor(STsdb *tsdb);
21	extern void tsdbCloseCompMonitor(STsdb *tsdb);
22
23	void tsdbSetKeepCfg(STsdb pTsdb, STsdbCfg pCfg) {	10,792✔
24	STsdbKeepCfg *pKeepCfg = &pTsdb->keepCfg;	10,792✔
25	pKeepCfg->precision = pCfg->precision;	10,792✔
26	pKeepCfg->days = pCfg->days;	10,792✔
27	pKeepCfg->keep0 = pCfg->keep0;	10,792✔
28	pKeepCfg->keep1 = pCfg->keep1;	10,792✔
29	pKeepCfg->keep2 = pCfg->keep2;	10,792✔
30	pKeepCfg->keepTimeOffset = pCfg->keepTimeOffset;	10,792✔
31	}	10,792✔
32
33	int64_t tsdbGetEarliestTs(STsdb *pTsdb) {	1,171,390✔
34	STsdbKeepCfg *pCfg = &pTsdb->keepCfg;	1,171,390✔
35
36	int64_t now = taosGetTimestamp(pCfg->precision);	1,171,390✔
37	int64_t ts = now - (tsTickPerMin[pCfg->precision] * pCfg->keep2) + 1; // needs to add one tick	1,172,801✔
38	return ts;	1,172,801✔
39	}
40
41	int32_t tsdbOpen(SVnode pVnode, STsdb ppTsdb, const char dir, STsdbKeepCfg *pKeepCfg, int8_t rollback, bool force) {	10,779✔
42	STsdb *pTsdb = NULL;	10,779✔
43	int slen = 0;	10,779✔
44	int32_t code;
45	int32_t lino;
46
47	*ppTsdb = NULL;	10,779✔
48	slen = strlen(pVnode->path) + strlen(dir) + 2;	10,779✔
49
50	// create handle
51	pTsdb = (STsdb )taosMemoryCalloc(1, sizeof(pTsdb) + slen);	10,779!
52	if (pTsdb == NULL) {	10,795!
UNCOV 53	TAOS_CHECK_RETURN(terrno);	×
54	}
55
56	pTsdb->path = (char *)&pTsdb[1];	10,795✔
57	snprintf(pTsdb->path, TD_PATH_MAX, "%s%s%s", pVnode->path, TD_DIRSEP, dir);	10,795✔
58	// taosRealPath(pTsdb->path, NULL, slen);
59	pTsdb->pVnode = pVnode;	10,795✔
60	(void)taosThreadMutexInit(&pTsdb->mutex, NULL);	10,795✔
61	if (!pKeepCfg) {	10,798✔
62	tsdbSetKeepCfg(pTsdb, &pVnode->config.tsdbCfg);	10,747✔
63	} else {
64	memcpy(&pTsdb->keepCfg, pKeepCfg, sizeof(STsdbKeepCfg));	51✔
65	}
66
67	// create dir
68	if (pVnode->pTfs) {	10,795!
69	code = tfsMkdir(pVnode->pTfs, pTsdb->path);	10,795✔
70	TSDB_CHECK_CODE(code, lino, _exit);	10,793!
71	} else {
UNCOV 72	code = taosMkDir(pTsdb->path);	×
73	TSDB_CHECK_CODE(code, lino, _exit);	×
74	}
75
76	// open tsdb
77	code = tsdbOpenFS(pTsdb, &pTsdb->pFS, rollback);	10,793✔
78	TSDB_CHECK_CODE(code, lino, _exit);	10,799!
79
80	if (pTsdb->pFS->fsstate == TSDB_FS_STATE_INCOMPLETE && force == false) {	10,799!
UNCOV 81	TAOS_CHECK_GOTO(TSDB_CODE_NEED_RETRY, &lino, _exit);	×
82	}
83
84	code = tsdbOpenCache(pTsdb);	10,799✔
85	TSDB_CHECK_CODE(code, lino, _exit);	10,794!
86
87	#ifdef TD_ENTERPRISE
88	TAOS_CHECK_GOTO(tsdbOpenCompMonitor(pTsdb), &lino, _exit);	10,794!
89	#endif
90
91	_exit:	10,789✔
92	if (code) {	10,789!
UNCOV 93	tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pVnode), __func__, __FILE__, lino, tstrerror(code));	×
94	tsdbCloseFS(&pTsdb->pFS);	×
95	(void)taosThreadMutexDestroy(&pTsdb->mutex);	×
96	taosMemoryFree(pTsdb);	×
97	} else {
98	tsdbDebug("vgId:%d, tsdb is opened at %s, days:%d, keep:%d,%d,%d, keepTimeoffset:%d", TD_VID(pVnode), pTsdb->path,	10,789✔
99	pTsdb->keepCfg.days, pTsdb->keepCfg.keep0, pTsdb->keepCfg.keep1, pTsdb->keepCfg.keep2,
100	pTsdb->keepCfg.keepTimeOffset);
101	*ppTsdb = pTsdb;	10,793✔
102	}
103	return code;	10,793✔
104	}
105
106	void tsdbClose(STsdb **pTsdb) {	10,799✔
107	if (*pTsdb) {	10,799!
108	STsdb pdb = pTsdb;	10,799✔
109	tsdbDebug("vgId:%d, tsdb is close at %s, days:%d, keep:%d,%d,%d, keepTimeOffset:%d", TD_VID(pdb->pVnode), pdb->path,	10,799✔
110	pdb->keepCfg.days, pdb->keepCfg.keep0, pdb->keepCfg.keep1, pdb->keepCfg.keep2,
111	pdb->keepCfg.keepTimeOffset);
112	(void)taosThreadMutexLock(&(*pTsdb)->mutex);	10,799✔
113	tsdbMemTableDestroy((*pTsdb)->mem, true);	10,799✔
114	(*pTsdb)->mem = NULL;	10,798✔
115	(void)taosThreadMutexUnlock(&(*pTsdb)->mutex);	10,798✔
116
117	tsdbCloseFS(&(*pTsdb)->pFS);	10,798✔
118	tsdbCloseCache(*pTsdb);	10,799✔
119	#ifdef TD_ENTERPRISE
120	tsdbCloseCompMonitor(*pTsdb);	10,796✔
121	#endif
122	(void)taosThreadMutexDestroy(&(*pTsdb)->mutex);	10,792✔
123	taosMemoryFreeClear(*pTsdb);	10,794!
124	}
125	return;	10,790✔
126	}

taosdata / TDengine / #3655

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous