#3562

Committed 20 Dec 2024 09:57AM UTC coverage: 26.655% (-32.2%) from 58.812%

Build # #3562

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

Merge pull request #29229 from taosdata/enh/TS-5749-3.0

enh: seperate tsdb async tasks to different thread pools

Run Details

21498 of 109421 branches covered (19.65%)

Branch coverage included in aggregate %.

66 of 96 new or added lines in 7 files covered. (68.75%)

39441 existing lines in 157 files now uncovered.

35007 of 102566 relevant lines covered (34.13%)

53922.97 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

18.23

/source/dnode/vnode/src/sma/smaOpen.c

/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "sma.h"
#include "tsdb.h"

static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t precision, int32_t duration,
                           int32_t *days);
static int32_t smaSetKeepCfg(SVnode *pVnode, STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type);
static int32_t rsmaRestore(SSma *pSma);

#define SMA_SET_KEEP_CFG(v, l)                                                                          \
  do {                                                                                                  \
    SRetention *r = &pCfg->retentions[l];                                                               \
    int64_t     keep = -1;                                                                              \
    TAOS_CHECK_EXIT(convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE, &keep)); \
    pKeepCfg->keep2 = (int32_t)keep;                                                                    \
    pKeepCfg->keep0 = pKeepCfg->keep2;                                                                  \
    pKeepCfg->keep1 = pKeepCfg->keep2;                                                                  \
    TAOS_CHECK_EXIT(smaEvalDays(v, pCfg->retentions, l, pCfg->precision, pCfg->days, &pKeepCfg->days)); \
    pKeepCfg->keepTimeOffset = 0;                                                                       \
  } while (0)

#define SMA_OPEN_RSMA_IMPL(v, l, force)                                                                    \
  do {                                                                                                     \
    SRetention *r = (SRetention *)VND_RETENTIONS(v) + l;                                                   \
    if (!RETENTION_VALID(l, r)) {                                                                          \
      if (l == 0) {                                                                                        \
        TAOS_CHECK_EXIT(TSDB_CODE_INVALID_PARA);                                                           \
      }                                                                                                    \
      break;                                                                                               \
    }                                                                                                      \
    TAOS_CHECK_EXIT(smaSetKeepCfg(v, &keepCfg, pCfg, TSDB_TYPE_RSMA_L##l));                                \
    TAOS_CHECK_EXIT(tsdbOpen(v, &SMA_RSMA_TSDB##l(pSma), VNODE_RSMA##l##_DIR, &keepCfg, rollback, force)); \
  } while (0)

/**
 * @brief Evaluate days(duration) for rsma level 1/2/3.
 *  1) level 1: duration from "create database"
 *  2) level 2/3: duration * (freq/freqL1)
 * @param pVnode
 * @param r
 * @param level
 * @param precision
 * @param duration
 * @param days
 * @return int32_t
 */
static int32_t smaEvalDays(SVnode *pVnode, SRetention *r, int8_t level, int8_t precision, int32_t duration,
                           int32_t *days) {
  int32_t code = 0;
  int32_t lino = 0;

  int64_t freqDuration = -1;
  int64_t keepDuration = -1;
  TAOS_CHECK_EXIT(
      convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->freq, precision, TIME_UNIT_MINUTE, &freqDuration));
  TAOS_CHECK_EXIT(
      convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->keep, precision, TIME_UNIT_MINUTE, &keepDuration));
  *days = duration;  // min

  if (*days < freqDuration) {
    *days = freqDuration;
  }

  if (*days > keepDuration) {
    *days = keepDuration;
  }

  if (level < TSDB_RETENTION_L1 || level > TSDB_RETENTION_L2) {
    goto _exit;
  }

  TAOS_CHECK_EXIT(convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE, &freqDuration));
  TAOS_CHECK_EXIT(convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE, &keepDuration));

  int32_t nFreqTimes = (r + level)->freq / (60 * 1000);  // use 60s for freq of 1st level
  *days *= (nFreqTimes > 1 ? nFreqTimes : 1);

  if (*days < freqDuration) {
    *days = freqDuration;
  }

  int32_t maxKeepDuration = TMIN(keepDuration, TSDB_MAX_DURATION_PER_FILE);
  if (*days > maxKeepDuration) {
    *days = maxKeepDuration;
  }

_exit:
  if (code) {
    smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code));
  } else {
    smaInfo("vgId:%d, evaluated duration for level %d is %d, raw val:%d", TD_VID(pVnode), level + 1, *days, duration);
  }
  TAOS_RETURN(code);
}

int smaSetKeepCfg(SVnode *pVnode, STsdbKeepCfg *pKeepCfg, STsdbCfg *pCfg, int type) {
  int32_t code = 0;
  int32_t lino = 0;
  pKeepCfg->precision = pCfg->precision;
  switch (type) {
    case TSDB_TYPE_RSMA_L0:
      SMA_SET_KEEP_CFG(pVnode, 0);
      break;
    case TSDB_TYPE_RSMA_L1:
      SMA_SET_KEEP_CFG(pVnode, 1);
      break;
    case TSDB_TYPE_RSMA_L2:
      SMA_SET_KEEP_CFG(pVnode, 2);
      break;
    default:
      code = TSDB_CODE_APP_ERROR;
      break;
  }
_exit:
  if (code) {
    smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code));
  }
  TAOS_RETURN(code);
}

int32_t smaOpen(SVnode *pVnode, int8_t rollback, bool force) {
  int32_t   code = 0;
  int32_t   lino = 0;
  STsdbCfg *pCfg = &pVnode->config.tsdbCfg;

  SSma *pSma = taosMemoryCalloc(1, sizeof(SSma));
  if (!pSma) {
    TAOS_CHECK_EXIT(terrno);
  }

  pVnode->pSma = pSma;

  pSma->pVnode = pVnode;
  (void)taosThreadMutexInit(&pSma->mutex, NULL);
  pSma->locked = false;

  if (VND_IS_RSMA(pVnode)) {
    STsdbKeepCfg keepCfg = {0};
    for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) {
      if (i == TSDB_RETENTION_L0) {
        SMA_OPEN_RSMA_IMPL(pVnode, 0, force);
      } else if (i == TSDB_RETENTION_L1) {
        SMA_OPEN_RSMA_IMPL(pVnode, 1, force);
      } else if (i == TSDB_RETENTION_L2) {
        SMA_OPEN_RSMA_IMPL(pVnode, 2, force);
      }
    }

    // restore the rsma
    TAOS_CHECK_EXIT(tdRSmaRestore(pSma, RSMA_RESTORE_REBOOT, pVnode->state.committed, rollback));
  }

_exit:
  if (code) {
    smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code));
  }
  TAOS_RETURN(code);
}

void smaClose(SSma *pSma) {
  if (pSma) {
    TAOS_UNUSED(smaPreClose(pSma));
    (void)taosThreadMutexDestroy(&pSma->mutex);
    SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma));
    SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma));
    if SMA_RSMA_TSDB0 (pSma) tsdbClose(&SMA_RSMA_TSDB0(pSma));
    if SMA_RSMA_TSDB1 (pSma) tsdbClose(&SMA_RSMA_TSDB1(pSma));
    if SMA_RSMA_TSDB2 (pSma) tsdbClose(&SMA_RSMA_TSDB2(pSma));
    taosMemoryFreeClear(pSma);
  }
  return;
}

/**
 * @brief rsma env restore
 *
 * @param pSma
 * @param type
 * @param committedVer
 * @return int32_t
 */
int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer, int8_t rollback) {
  if (!VND_IS_RSMA(pSma->pVnode)) {
    TAOS_RETURN(TSDB_CODE_RSMA_INVALID_ENV);
  }

  return tdRSmaProcessRestoreImpl(pSma, type, committedVer, rollback);
}

1	/*
2	* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3	*
4	* This program is free software: you can use, redistribute, and/or modify
5	* it under the terms of the GNU Affero General Public License, version 3
6	* or later ("AGPL"), as published by the Free Software Foundation.
7	*
8	* This program is distributed in the hope that it will be useful, but WITHOUT
9	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10	* FITNESS FOR A PARTICULAR PURPOSE.
11	*
12	* You should have received a copy of the GNU Affero General Public License
13	* along with this program. If not, see <http://www.gnu.org/licenses/>.
14	*/
15
16	#include "sma.h"
17	#include "tsdb.h"
18
19	static int32_t smaEvalDays(SVnode pVnode, SRetention r, int8_t level, int8_t precision, int32_t duration,
20	int32_t *days);
21	static int32_t smaSetKeepCfg(SVnode pVnode, STsdbKeepCfg pKeepCfg, STsdbCfg *pCfg, int type);
22	static int32_t rsmaRestore(SSma *pSma);
23
24	#define SMA_SET_KEEP_CFG(v, l) \
25	do { \
26	SRetention *r = &pCfg->retentions[l]; \
27	int64_t keep = -1; \
28	TAOS_CHECK_EXIT(convertTimeFromPrecisionToUnit(r->keep, pCfg->precision, TIME_UNIT_MINUTE, &keep)); \
29	pKeepCfg->keep2 = (int32_t)keep; \
30	pKeepCfg->keep0 = pKeepCfg->keep2; \
31	pKeepCfg->keep1 = pKeepCfg->keep2; \
32	TAOS_CHECK_EXIT(smaEvalDays(v, pCfg->retentions, l, pCfg->precision, pCfg->days, &pKeepCfg->days)); \
33	pKeepCfg->keepTimeOffset = 0; \
34	} while (0)
35
36	#define SMA_OPEN_RSMA_IMPL(v, l, force) \
37	do { \
38	SRetention r = (SRetention )VND_RETENTIONS(v) + l; \
39	if (!RETENTION_VALID(l, r)) { \
40	if (l == 0) { \
41	TAOS_CHECK_EXIT(TSDB_CODE_INVALID_PARA); \
42	} \
43	break; \
44	} \
45	TAOS_CHECK_EXIT(smaSetKeepCfg(v, &keepCfg, pCfg, TSDB_TYPE_RSMA_L##l)); \
46	TAOS_CHECK_EXIT(tsdbOpen(v, &SMA_RSMA_TSDB##l(pSma), VNODE_RSMA##l##_DIR, &keepCfg, rollback, force)); \
47	} while (0)
48
49	/**
50	* @brief Evaluate days(duration) for rsma level 1/2/3.
51	* 1) level 1: duration from "create database"
52	* 2) level 2/3: duration * (freq/freqL1)
53	* @param pVnode
54	* @param r
55	* @param level
56	* @param precision
57	* @param duration
58	* @param days
59	* @return int32_t
60	*/
UNCOV 61	static int32_t smaEvalDays(SVnode pVnode, SRetention r, int8_t level, int8_t precision, int32_t duration,	×
62	int32_t *days) {
UNCOV 63	int32_t code = 0;	×
UNCOV 64	int32_t lino = 0;	×
65
UNCOV 66	int64_t freqDuration = -1;	×
UNCOV 67	int64_t keepDuration = -1;	×
UNCOV 68	TAOS_CHECK_EXIT(	×
69	convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->freq, precision, TIME_UNIT_MINUTE, &freqDuration));
UNCOV 70	TAOS_CHECK_EXIT(	×
71	convertTimeFromPrecisionToUnit((r + TSDB_RETENTION_L0)->keep, precision, TIME_UNIT_MINUTE, &keepDuration));
UNCOV 72	*days = duration; // min	×
73
UNCOV 74	if (*days < freqDuration) {	×
75	*days = freqDuration;	×
76	}
77
UNCOV 78	if (*days > keepDuration) {	×
UNCOV 79	*days = keepDuration;	×
80	}
81
UNCOV 82	if (level < TSDB_RETENTION_L1 \|\| level > TSDB_RETENTION_L2) {	×
UNCOV 83	goto _exit;	×
84	}
85
UNCOV 86	TAOS_CHECK_EXIT(convertTimeFromPrecisionToUnit((r + level)->freq, precision, TIME_UNIT_MINUTE, &freqDuration));	×
UNCOV 87	TAOS_CHECK_EXIT(convertTimeFromPrecisionToUnit((r + level)->keep, precision, TIME_UNIT_MINUTE, &keepDuration));	×
88
UNCOV 89	int32_t nFreqTimes = (r + level)->freq / (60 * 1000); // use 60s for freq of 1st level	×
UNCOV 90	days = (nFreqTimes > 1 ? nFreqTimes : 1);	×
91
UNCOV 92	if (*days < freqDuration) {	×
93	*days = freqDuration;	×
94	}
95
UNCOV 96	int32_t maxKeepDuration = TMIN(keepDuration, TSDB_MAX_DURATION_PER_FILE);	×
UNCOV 97	if (*days > maxKeepDuration) {	×
98	*days = maxKeepDuration;	×
99	}
100
UNCOV 101	_exit:	×
UNCOV 102	if (code) {	×
103	smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code));	×
104	} else {
UNCOV 105	smaInfo("vgId:%d, evaluated duration for level %d is %d, raw val:%d", TD_VID(pVnode), level + 1, *days, duration);	×
106	}
UNCOV 107	TAOS_RETURN(code);	×
108	}
109
UNCOV 110	int smaSetKeepCfg(SVnode pVnode, STsdbKeepCfg pKeepCfg, STsdbCfg *pCfg, int type) {	×
UNCOV 111	int32_t code = 0;	×
UNCOV 112	int32_t lino = 0;	×
UNCOV 113	pKeepCfg->precision = pCfg->precision;	×
UNCOV 114	switch (type) {	×
UNCOV 115	case TSDB_TYPE_RSMA_L0:	×
UNCOV 116	SMA_SET_KEEP_CFG(pVnode, 0);	×
UNCOV 117	break;	×
UNCOV 118	case TSDB_TYPE_RSMA_L1:	×
UNCOV 119	SMA_SET_KEEP_CFG(pVnode, 1);	×
UNCOV 120	break;	×
UNCOV 121	case TSDB_TYPE_RSMA_L2:	×
UNCOV 122	SMA_SET_KEEP_CFG(pVnode, 2);	×
UNCOV 123	break;	×
124	default:	×
125	code = TSDB_CODE_APP_ERROR;	×
126	break;	×
127	}
UNCOV 128	_exit:	×
UNCOV 129	if (code) {	×
130	smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code));	×
131	}
UNCOV 132	TAOS_RETURN(code);	×
133	}
134
135	int32_t smaOpen(SVnode *pVnode, int8_t rollback, bool force) {	553✔
136	int32_t code = 0;	553✔
137	int32_t lino = 0;	553✔
138	STsdbCfg *pCfg = &pVnode->config.tsdbCfg;	553✔
139
140	SSma *pSma = taosMemoryCalloc(1, sizeof(SSma));	553!
141	if (!pSma) {	553!
142	TAOS_CHECK_EXIT(terrno);	×
143	}
144
145	pVnode->pSma = pSma;	553✔
146
147	pSma->pVnode = pVnode;	553✔
148	(void)taosThreadMutexInit(&pSma->mutex, NULL);	553✔
149	pSma->locked = false;	553✔
150
151	if (VND_IS_RSMA(pVnode)) {	553!
UNCOV 152	STsdbKeepCfg keepCfg = {0};	×
UNCOV 153	for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) {	×
UNCOV 154	if (i == TSDB_RETENTION_L0) {	×
UNCOV 155	SMA_OPEN_RSMA_IMPL(pVnode, 0, force);	×
UNCOV 156	} else if (i == TSDB_RETENTION_L1) {	×
UNCOV 157	SMA_OPEN_RSMA_IMPL(pVnode, 1, force);	×
UNCOV 158	} else if (i == TSDB_RETENTION_L2) {	×
UNCOV 159	SMA_OPEN_RSMA_IMPL(pVnode, 2, force);	×
160	}
161	}
162
163	// restore the rsma
UNCOV 164	TAOS_CHECK_EXIT(tdRSmaRestore(pSma, RSMA_RESTORE_REBOOT, pVnode->state.committed, rollback));	×
165	}
166
167	_exit:	553✔
168	if (code) {	553!
169	smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code));	×
170	}
171	TAOS_RETURN(code);	553✔
172	}
173
174	void smaClose(SSma *pSma) {	553✔
175	if (pSma) {	553!
176	TAOS_UNUSED(smaPreClose(pSma));	553✔
177	(void)taosThreadMutexDestroy(&pSma->mutex);	553✔
178	SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma));	553✔
179	SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma));	553✔
180	if SMA_RSMA_TSDB0 (pSma) tsdbClose(&SMA_RSMA_TSDB0(pSma));	553!
181	if SMA_RSMA_TSDB1 (pSma) tsdbClose(&SMA_RSMA_TSDB1(pSma));	553!
182	if SMA_RSMA_TSDB2 (pSma) tsdbClose(&SMA_RSMA_TSDB2(pSma));	553!
183	taosMemoryFreeClear(pSma);	553!
184	}
185	return;	553✔
186	}
187
188	/**
189	* @brief rsma env restore
190	*
191	* @param pSma
192	* @param type
193	* @param committedVer
194	* @return int32_t
195	*/
UNCOV 196	int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer, int8_t rollback) {	×
UNCOV 197	if (!VND_IS_RSMA(pSma->pVnode)) {	×
198	TAOS_RETURN(TSDB_CODE_RSMA_INVALID_ENV);	×
199	}
200
UNCOV 201	return tdRSmaProcessRestoreImpl(pSma, type, committedVer, rollback);	×
202	}

taosdata / TDengine / #3562

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous