#3653

Committed 14 Mar 2025 08:10AM UTC coverage: 22.565% (-41.0%) from 63.596%

Build # #3653

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

feat(keep): support keep on super table level. (#30097)

* Feat: support use keep while create super table.

* Test(keep): add test for create super table with keep option.

* Feat(keep): Add tmsg for create keep.

* Feat(keep): support alter table option keep.

* Fix(keep): Add baisc test for alter table option.

* Fix(keep): memory leek.

* Feat(keep): add keep to metaEntry&metaCache and fix earliestTs with stn keep.

* Test(keep): add some cases for select with stb keep.

* Fix: fix ci core while alter stb.

* Feat(keep): delete expired data in super table level.

* Feat: remove get stb keep while query.

* Fix : build error.

* Revert "Fix : build error."

This reverts commit 0ed66e4e8.

* Revert "Feat(keep): delete expired data in super table level."

This reverts commit 36330f6b4.

* Fix : build errors.

* Feat : support restart taosd.

* Fix : alter table comment problems.

* Test : add tests for super table keep.

* Fix: change sdb stb reserve size.

* Test: add more tests.

* Feat: Disable normal tables and sub tables from setting the keep parameter

* Fix: add more checks to avoid unknown address.

* Docs: Add docs for stable keep.

* Fix: some review changes.

* Fix: review errors.

Run Details

49248 of 302527 branches covered (16.28%)

Branch coverage included in aggregate %.

53 of 99 new or added lines in 12 files covered. (53.54%)

155872 existing lines in 443 files now uncovered.

87359 of 302857 relevant lines covered (28.84%)

570004.22 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/source/dnode/vnode/src/tsdb/tsdbWrite.c

/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "cos.h"
#include "tsdb.h"

/**
 * @brief max key by precision
 *  approximately calculation:
 *  ms: 3600*1000*8765*1000         // 1970 + 1000 years
 *  us: 3600*1000000*8765*1000      // 1970 + 1000 years
 *  ns: 3600*1000000000*8765*292    // 1970 + 292 years
 */
int64_t tsMaxKeyByPrecision[] = {31556995200000L, 31556995200000000L, 9214646400000000000L};

// static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg);

int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2 *pRsp) {
  int32_t code;
  int32_t arrSize = 0;
  int32_t affectedrows = 0;
  int32_t numOfRows = 0;

  if (pTsdb->mem == NULL) {
    TAOS_RETURN(TSDB_CODE_INTERNAL_ERROR);
  }

  arrSize = taosArrayGetSize(pMsg->aSubmitTbData);

  // scan and convert
  if ((code = tsdbScanAndConvertSubmitMsg(pTsdb, pMsg)) < 0) {
    if (code != TSDB_CODE_TDB_TABLE_RECONFIGURE) {
      tsdbError("vgId:%d, failed to insert data since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
    }
    return code;
  }

  // loop to insert
  for (int32_t i = 0; i < arrSize; ++i) {
    TAOS_CHECK_RETURN(tsdbInsertTableData(pTsdb, version, taosArrayGet(pMsg->aSubmitTbData, i), &affectedrows));
  }

  if (pRsp != NULL) {
    // pRsp->affectedRows = affectedrows;
    // pRsp->numOfRows = numOfRows;
  }

  return 0;
}

static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, TSKEY rowKey, TSKEY minKey, TSKEY maxKey,
                                          TSKEY now) {
  if (rowKey < minKey || rowKey > maxKey) {
    tsdbError("vgId:%d, table uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64
              " maxKey %" PRId64 " row key %" PRId64,
              TD_VID(pTsdb->pVnode), uid, now, minKey, maxKey, rowKey);
    return TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE;
  }

  return 0;
}

int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) {
  STsdbKeepCfg *pCfg = &pTsdb->keepCfg;
  TSKEY         now = taosGetTimestamp(pCfg->precision);
  TSKEY         minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2;
  TSKEY         maxKey = tsMaxKeyByPrecision[pCfg->precision];
  int32_t       size = taosArrayGetSize(pMsg->aSubmitTbData);

  for (int32_t i = 0; i < size; ++i) {
    SSubmitTbData *pData = TARRAY_GET_ELEM(pMsg->aSubmitTbData, i);
    if (pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {
      uint64_t  nColData = TARRAY_SIZE(pData->aCol);
      SColData *aColData = (SColData *)TARRAY_DATA(pData->aCol);
      if (nColData > 0) {
        int32_t nRows = aColData[0].nVal;
        TSKEY  *aKey = (TSKEY *)aColData[0].pData;
        for (int32_t r = 0; r < nRows; ++r) {
          TAOS_CHECK_RETURN(tsdbCheckRowRange(pTsdb, pData->uid, aKey[r], minKey, maxKey, now));
        }
      }
    } else {
      int32_t nRows = taosArrayGetSize(pData->aRowP);
      for (int32_t r = 0; r < nRows; ++r) {
        SRow *pRow = (SRow *)taosArrayGetP(pData->aRowP, r);
        TAOS_CHECK_RETURN(tsdbCheckRowRange(pTsdb, pData->uid, pRow->ts, minKey, maxKey, now));
      }
    }
  }

  return 0;
}

1	/*
2	* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3	*
4	* This program is free software: you can use, redistribute, and/or modify
5	* it under the terms of the GNU Affero General Public License, version 3
6	* or later ("AGPL"), as published by the Free Software Foundation.
7	*
8	* This program is distributed in the hope that it will be useful, but WITHOUT
9	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10	* FITNESS FOR A PARTICULAR PURPOSE.
11	*
12	* You should have received a copy of the GNU Affero General Public License
13	* along with this program. If not, see <http://www.gnu.org/licenses/>.
14	*/
15
16	#include "cos.h"
17	#include "tsdb.h"
18
19	/**
20	* @brief max key by precision
21	* approximately calculation:
22	* ms: 360010008765*1000 // 1970 + 1000 years
23	* us: 360010000008765*1000 // 1970 + 1000 years
24	* ns: 360010000000008765*292 // 1970 + 292 years
25	*/
26	int64_t tsMaxKeyByPrecision[] = {31556995200000L, 31556995200000000L, 9214646400000000000L};
27
28	// static int tsdbScanAndConvertSubmitMsg(STsdb pTsdb, SSubmitReq pMsg);
29
UNCOV 30	int tsdbInsertData(STsdb pTsdb, int64_t version, SSubmitReq2 pMsg, SSubmitRsp2 *pRsp) {	×
31	int32_t code;
UNCOV 32	int32_t arrSize = 0;	×
UNCOV 33	int32_t affectedrows = 0;	×
UNCOV 34	int32_t numOfRows = 0;	×
35
UNCOV 36	if (pTsdb->mem == NULL) {	×
37	TAOS_RETURN(TSDB_CODE_INTERNAL_ERROR);	×
38	}
39
UNCOV 40	arrSize = taosArrayGetSize(pMsg->aSubmitTbData);	×
41
42	// scan and convert
UNCOV 43	if ((code = tsdbScanAndConvertSubmitMsg(pTsdb, pMsg)) < 0) {	×
44	if (code != TSDB_CODE_TDB_TABLE_RECONFIGURE) {	×
45	tsdbError("vgId:%d, failed to insert data since %s", TD_VID(pTsdb->pVnode), tstrerror(code));	×
46	}
47	return code;	×
48	}
49
50	// loop to insert
UNCOV 51	for (int32_t i = 0; i < arrSize; ++i) {	×
UNCOV 52	TAOS_CHECK_RETURN(tsdbInsertTableData(pTsdb, version, taosArrayGet(pMsg->aSubmitTbData, i), &affectedrows));	×
53	}
54
55	if (pRsp != NULL) {
56	// pRsp->affectedRows = affectedrows;
57	// pRsp->numOfRows = numOfRows;
58	}
59
UNCOV 60	return 0;	×
61	}
62
63	static FORCE_INLINE int tsdbCheckRowRange(STsdb *pTsdb, tb_uid_t uid, TSKEY rowKey, TSKEY minKey, TSKEY maxKey,
64	TSKEY now) {
UNCOV 65	if (rowKey < minKey \|\| rowKey > maxKey) {	×
66	tsdbError("vgId:%d, table uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64	×
67	" maxKey %" PRId64 " row key %" PRId64,
68	TD_VID(pTsdb->pVnode), uid, now, minKey, maxKey, rowKey);
69	return TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE;	×
70	}
71
UNCOV 72	return 0;	×
73	}
74
UNCOV 75	int tsdbScanAndConvertSubmitMsg(STsdb pTsdb, SSubmitReq2 pMsg) {	×
UNCOV 76	STsdbKeepCfg *pCfg = &pTsdb->keepCfg;	×
UNCOV 77	TSKEY now = taosGetTimestamp(pCfg->precision);	×
UNCOV 78	TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2;	×
UNCOV 79	TSKEY maxKey = tsMaxKeyByPrecision[pCfg->precision];	×
UNCOV 80	int32_t size = taosArrayGetSize(pMsg->aSubmitTbData);	×
81
UNCOV 82	for (int32_t i = 0; i < size; ++i) {	×
UNCOV 83	SSubmitTbData *pData = TARRAY_GET_ELEM(pMsg->aSubmitTbData, i);	×
UNCOV 84	if (pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) {	×
85	uint64_t nColData = TARRAY_SIZE(pData->aCol);	×
86	SColData aColData = (SColData )TARRAY_DATA(pData->aCol);	×
87	if (nColData > 0) {	×
88	int32_t nRows = aColData[0].nVal;	×
89	TSKEY aKey = (TSKEY )aColData[0].pData;	×
90	for (int32_t r = 0; r < nRows; ++r) {	×
91	TAOS_CHECK_RETURN(tsdbCheckRowRange(pTsdb, pData->uid, aKey[r], minKey, maxKey, now));	×
92	}
93	}
94	} else {
UNCOV 95	int32_t nRows = taosArrayGetSize(pData->aRowP);	×
UNCOV 96	for (int32_t r = 0; r < nRows; ++r) {	×
UNCOV 97	SRow pRow = (SRow )taosArrayGetP(pData->aRowP, r);	×
UNCOV 98	TAOS_CHECK_RETURN(tsdbCheckRowRange(pTsdb, pData->uid, pRow->ts, minKey, maxKey, now));	×
99	}
100	}
101	}
102
UNCOV 103	return 0;	×
104	}

taosdata / TDengine / #3653

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous