#4791

Committed 13 Oct 2025 06:50AM UTC coverage: 57.628% (-0.8%) from 58.476%

Build # #4791

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

Merge pull request #33213 from taosdata/fix/huoh/timemoe_model_directory

fix: fix tdgpt timemoe model directory

Run Details

136628 of 303332 branches covered (45.04%)

Branch coverage included in aggregate %.

208121 of 294900 relevant lines covered (70.57%)

4250784.02 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

57.25

/source/dnode/vnode/src/tsdb/tsdbRetention.c

/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "tss.h"
#include "tsdb.h"
#include "tsdbFS2.h"
#include "tsdbFSet2.h"
#include "vnd.h"
#include "tsdbInt.h"

extern int32_t tsdbAsyncCompact(STsdb *tsdb, const STimeWindow *tw, ETsdbOpType type);

// tsdbRetentionMonitor.c
extern int32_t tsdbAddRetentionMonitorTask(STsdb *tsdb, int32_t fid, SVATaskID *taskId, int64_t fileSize);
extern void    tsdbRemoveRetentionMonitorTask(STsdb *tsdb, SVATaskID *taskId);

static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) {
  STFileOp op = {
      .optype = TSDB_FOP_REMOVE,
      .fid = fobj->f->fid,
      .of = fobj->f[0],
  };

  return TARRAY2_APPEND(&rtner->fopArr, op);
}

static int32_t tsdbCopyFileWithLimitedSpeed(TdFilePtr from, TdFilePtr to, int64_t size, uint32_t limitMB) {
  int64_t interval = 1000;  // 1s
  int64_t limit = limitMB ? limitMB * 1024 * 1024 : INT64_MAX;
  int64_t offset = 0;
  int64_t remain = size;

  while (remain > 0) {
    int64_t n;
    int64_t last = taosGetTimestampMs();
    if ((n = taosFSendFile(to, from, &offset, TMIN(limit, remain))) < 0) {
      TAOS_CHECK_RETURN(terrno);
    }

    remain -= n;

    if (remain > 0) {
      int64_t elapsed = taosGetTimestampMs() - last;
      if (elapsed < interval) {
        taosMsleep(interval - elapsed);
      }
    }
  }

  return 0;
}

static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFile *to) {
  int32_t   code = 0;
  int32_t   lino = 0;
  TdFilePtr fdFrom = NULL, fdTo = NULL;
  char      fname_from[TSDB_FILENAME_LEN];
  char      fname_to[TSDB_FILENAME_LEN];

  tsdbTFileLastChunkName(rtner->tsdb, from->f, fname_from);
  tsdbTFileLastChunkName(rtner->tsdb, to, fname_to);

  fdFrom = taosOpenFile(fname_from, TD_FILE_READ);
  if (fdFrom == NULL) {
    TAOS_CHECK_GOTO(terrno, &lino, _exit);
  }

  tsdbInfo("vgId: %d, open tofile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname_to, from->f->size);

  fdTo = taosOpenFile(fname_to, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
  if (fdTo == NULL) {
    TAOS_CHECK_GOTO(terrno, &lino, _exit);
  }

  SVnodeCfg *pCfg = &rtner->tsdb->pVnode->config;
  int64_t    chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->ssChunkSize;
  int64_t    lc_size = tsdbLogicToFileSize(to->size, rtner->szPage) - chunksize * (to->lcn - 1);

  if (taosFSendFile(fdTo, fdFrom, 0, lc_size) < 0) {
    TAOS_CHECK_GOTO(terrno, &lino, _exit);
  }

_exit:
  if (code) {
    tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,
              tstrerror(code));
  }
  if (taosCloseFile(&fdFrom) != 0) {
    tsdbError("vgId:%d, failed to close file %s", TD_VID(rtner->tsdb->pVnode), fname_from);
  }
  if (taosCloseFile(&fdTo) != 0) {
    tsdbError("vgId:%d, failed to close file %s", TD_VID(rtner->tsdb->pVnode), fname_to);
  }
  return code;
}

static int32_t tsdbDoCopyFile(SRTNer *rtner, const STFileObj *from, const STFile *to) {
  int32_t code = 0;
  int32_t lino = 0;

  char      fname[TSDB_FILENAME_LEN];
  TdFilePtr fdFrom = NULL;
  TdFilePtr fdTo = NULL;

  tsdbTFileName(rtner->tsdb, to, fname);

  fdFrom = taosOpenFile(from->fname, TD_FILE_READ);
  if (fdFrom == NULL) {
    TAOS_CHECK_GOTO(terrno, &lino, _exit);
  }

  tsdbInfo("vgId: %d, open tofile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, from->f->size);

  fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
  if (fdTo == NULL) {
    TAOS_CHECK_GOTO(terrno, &lino, _exit);
  }
  TSDB_CHECK_CODE(code, lino, _exit);

  TAOS_CHECK_GOTO(tsdbCopyFileWithLimitedSpeed(fdFrom, fdTo, tsdbLogicToFileSize(from->f->size, rtner->szPage),
                                               tsRetentionSpeedLimitMB),
                  &lino, _exit);

_exit:
  if (code) {
    tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,
              tstrerror(code));
  }
  if (taosCloseFile(&fdFrom) != 0) {
    tsdbTrace("vgId:%d, failed to close file", TD_VID(rtner->tsdb->pVnode));
  }
  if (taosCloseFile(&fdTo) != 0) {
    tsdbTrace("vgId:%d, failed to close file", TD_VID(rtner->tsdb->pVnode));
  }
  return code;
}

static int32_t tsdbDoMigrateFileObj(SRTNer *rtner, const STFileObj *fobj, const SDiskID *did) {
  int32_t  code = 0;
  int32_t  lino = 0;
  STFileOp op = {0};
  int32_t  lcn = fobj->f->lcn;

  // remove old
  op = (STFileOp){
      .optype = TSDB_FOP_REMOVE,
      .fid = fobj->f->fid,
      .of = fobj->f[0],
  };

  TAOS_CHECK_GOTO(TARRAY2_APPEND(&rtner->fopArr, op), &lino, _exit);

  // create new
  op = (STFileOp){
      .optype = TSDB_FOP_CREATE,
      .fid = fobj->f->fid,
      .nf =
          {
              .type = fobj->f->type,
              .did = did[0],
              .fid = fobj->f->fid,
              .minVer = fobj->f->minVer,
              .maxVer = fobj->f->maxVer,
              .mid = fobj->f->mid,
              .cid = fobj->f->cid,
              .size = fobj->f->size,
              .lcn = lcn,
              .stt[0] =
                  {
                      .level = fobj->f->stt[0].level,
                  },
          },
  };

  TAOS_CHECK_GOTO(TARRAY2_APPEND(&rtner->fopArr, op), &lino, _exit);

  // do copy the file

  if (lcn < 1) {
    TAOS_CHECK_GOTO(tsdbDoCopyFile(rtner, fobj, &op.nf), &lino, _exit);
  } else {
    TAOS_CHECK_GOTO(tsdbDoCopyFileLC(rtner, fobj, &op.nf), &lino, _exit);
  }

_exit:
  if (code) {
    tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,
              tstrerror(code));
  }
  return code;
}


static int32_t tsdbDoRetentionEnd(SRTNer *rtner, EFEditT etype) {
  int32_t code = 0;
  int32_t lino = 0;

  if (TARRAY2_SIZE(&rtner->fopArr) > 0) {
    TAOS_CHECK_GOTO(tsdbFSEditBegin(rtner->tsdb->pFS, &rtner->fopArr, etype), &lino, _exit);

    (void)taosThreadMutexLock(&rtner->tsdb->mutex);

    code = tsdbFSEditCommit(rtner->tsdb->pFS);
    if (code) {
      (void)taosThreadMutexUnlock(&rtner->tsdb->mutex);
      TSDB_CHECK_CODE(code, lino, _exit);
    }

    (void)taosThreadMutexUnlock(&rtner->tsdb->mutex);

    TARRAY2_DESTROY(&rtner->fopArr, NULL);
  }

_exit:
  if (code) {
    tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,
              tstrerror(code));
  } else {
    tsdbDebug("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__);
  }
  return code;
}

static int32_t tsdbRemoveOrMoveFileObject(SRTNer *rtner, int32_t expLevel, STFileObj *fobj) {
  int32_t code = 0;
  int32_t lino = 0;

  if (fobj == NULL) {
    return code;
  }

  if (expLevel < 0) {
    // remove the file
    code = tsdbDoRemoveFileObject(rtner, fobj);
    TSDB_CHECK_CODE(code, lino, _exit);
  } else if (expLevel > fobj->f->did.level) {
    // Try to move the file to a new level
    for (; expLevel > fobj->f->did.level; expLevel--) {
      SDiskID diskId = {0};

      code = tsdbAllocateDiskAtLevel(rtner->tsdb, expLevel, tsdbFTypeLabel(fobj->f->type), &diskId);
      if (code) {
        tsdbTrace("vgId:%d, cannot allocate disk for file %s, level:%d, reason:%s, skip!", TD_VID(rtner->tsdb->pVnode),
                  fobj->fname, expLevel, tstrerror(code));
        code = 0;
        continue;
      } else {
        tsdbInfo("vgId:%d start to migrate file %s from level %d to %d, size:%" PRId64, TD_VID(rtner->tsdb->pVnode),
                 fobj->fname, fobj->f->did.level, diskId.level, fobj->f->size);

        code = tsdbDoMigrateFileObj(rtner, fobj, &diskId);
        TSDB_CHECK_CODE(code, lino, _exit);

        tsdbInfo("vgId:%d end to migrate file %s from level %d to %d, size:%" PRId64, TD_VID(rtner->tsdb->pVnode),
                 fobj->fname, fobj->f->did.level, diskId.level, fobj->f->size);
        break;
      }
    }
  }

_exit:
  if (code) {
    tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,
              tstrerror(code));
  }
  return code;
}

static int32_t tsdbDoRetention(SRTNer *rtner) {
  int32_t    code = 0;
  int32_t    lino = 0;
  STFileObj *fobj = NULL;
  STFileSet *fset = rtner->fset;

  // handle data file sets
  int32_t expLevel = tsdbFidLevel(fset->fid, &rtner->tsdb->keepCfg, rtner->tw.ekey);
  for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ++ftype) {
    code = tsdbRemoveOrMoveFileObject(rtner, expLevel, fset->farr[ftype]);
    TSDB_CHECK_CODE(code, lino, _exit);
  }

  // handle stt file
  SSttLvl *lvl;
  TARRAY2_FOREACH(fset->lvlArr, lvl) {
    TARRAY2_FOREACH(lvl->fobjArr, fobj) {
      code = tsdbRemoveOrMoveFileObject(rtner, expLevel, fobj);
      TSDB_CHECK_CODE(code, lino, _exit);
    }
  }

_exit:
  if (code) {
    tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,
              tstrerror(code));
  }
  return code;
}

void tsdbRetentionCancel(void *arg) { taosMemoryFree(arg); }

static bool tsdbFSetNeedRetention(STFileSet *fset, int32_t expLevel) {
  if (expLevel < 0) {
    return false;
  }

  STFileObj *fobj = NULL;
  for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ++ftype) {
    fobj = fset->farr[ftype];
    if (fobj && (expLevel > fset->farr[ftype]->f->did.level)) {
      return true;
    }
  }

  // handle stt file
  SSttLvl *lvl = NULL;
  TARRAY2_FOREACH(fset->lvlArr, lvl) {
    TARRAY2_FOREACH(lvl->fobjArr, fobj) {
      if (fobj && (expLevel > fobj->f->did.level)) {
        return true;
      }
    }
  }

  return false;
}
#ifdef TD_ENTERPRISE
static bool tsdbShouldRollup(STsdb *tsdb, SRTNer *rtner, SRtnArg *rtnArg) {
  SVnode    *pVnode = tsdb->pVnode;
  STFileSet *fset = rtner->fset;

  if (!VND_IS_RSMA(pVnode)) {
    return false;
  }

  int32_t expLevel = tsdbFidLevel(fset->fid, &tsdb->keepCfg, rtner->tw.ekey);
  if (expLevel <= 0) {
    return false;
  }

  if (rtnArg->optrType == TSDB_OPTR_ROLLUP) {
    return true;
  } else if (rtnArg->optrType == TSDB_OPTR_NORMAL) {
    return tsdbFSetNeedRetention(fset, expLevel);
  }
  return false;
}
#endif
int32_t tsdbRetention(void *arg) {
  int32_t code = 0;
  int32_t lino = 0;

  SRtnArg   *rtnArg = (SRtnArg *)arg;
  STsdb     *pTsdb = rtnArg->tsdb;
  SVnode    *pVnode = pTsdb->pVnode;
  STFileSet *fset = NULL;
  SRTNer     rtner = {
          .tsdb = pTsdb,
          .szPage = pVnode->config.tsdbPageSize,
          .tw = rtnArg->tw,
          .lastCommit = rtnArg->lastCommit,
          .cid = tsdbFSAllocEid(pTsdb->pFS),
          .nodeId = rtnArg->nodeId,
  };

  // begin task
  (void)taosThreadMutexLock(&pTsdb->mutex);

  // check if background task is disabled
  if (pTsdb->bgTaskDisabled) {
    tsdbInfo("vgId:%d, background task is disabled, skip retention", TD_VID(pTsdb->pVnode));
    (void)taosThreadMutexUnlock(&pTsdb->mutex);
    return 0;
  }

  // set flag and copy
  tsdbBeginTaskOnFileSet(pTsdb, rtnArg->fid, EVA_TASK_RETENTION, &fset);
  if (fset && (code = tsdbTFileSetInitCopy(pTsdb, fset, &rtner.fset))) {
    (void)taosThreadMutexUnlock(&pTsdb->mutex);
    TSDB_CHECK_CODE(code, lino, _exit);
  }

  (void)taosThreadMutexUnlock(&pTsdb->mutex);

  // do retention
  if (rtner.fset) {
    EFEditT etype = TSDB_FEDIT_RETENTION;
    if (rtnArg->optrType == TSDB_OPTR_SSMIGRATE) {
      etype = TSDB_FEDIT_SSMIGRATE;
      TAOS_CHECK_GOTO(tsdbDoSsMigrate(&rtner), &lino, _exit);
#ifdef TD_ENTERPRISE
    } else if (tsdbShouldRollup(pTsdb, &rtner, rtnArg)) {
      etype = TSDB_FEDIT_ROLLUP;
      TAOS_CHECK_GOTO(tsdbDoRollup(&rtner), &lino, _exit);
#endif
    } else if (rtnArg->optrType == TSDB_OPTR_NORMAL) {
      TAOS_CHECK_GOTO(tsdbDoRetention(&rtner), &lino, _exit);
    } else {
      goto _exit;
    }

    TAOS_CHECK_GOTO(tsdbDoRetentionEnd(&rtner, etype), &lino, _exit);
  }

_exit:
  if (rtner.fset) {
    (void)taosThreadMutexLock(&pTsdb->mutex);
    tsdbFinishTaskOnFileSet(pTsdb, rtnArg->fid, EVA_TASK_RETENTION);
    (void)taosThreadMutexUnlock(&pTsdb->mutex);
  }

  // clear resources
  tsdbTFileSetClear(&rtner.fset);
  TARRAY2_DESTROY(&rtner.fopArr, NULL);
  (void)tsdbRemoveRetentionMonitorTask(pTsdb, &rtnArg->taskid);
  taosMemoryFree(arg);
  if (code) {
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
  }
  return code;
}

static bool tsdbInRetentionTimeRange(STsdb *tsdb, int32_t fid, STimeWindow tw, int8_t optrType) {
  if (optrType == TSDB_OPTR_ROLLUP) {
    TSKEY  minKey, maxKey;
    int8_t precision = tsdb->keepCfg.precision;
    if (precision < TSDB_TIME_PRECISION_MILLI || precision > TSDB_TIME_PRECISION_NANO) {
      tsdbError("vgId:%d, failed to check retention time range since invalid precision %" PRIi8, TD_VID(tsdb->pVnode), precision);
      return false;
    }
    tsdbFidKeyRange(fid, tsdb->keepCfg.days, precision, &minKey, &maxKey);

    if ((tw.ekey != INT64_MAX) && ((double)tw.ekey * (double)tsSecTimes[precision] < (double)minKey)) {
      return false;
    }
    if ((tw.skey != INT64_MIN) && ((double)tw.skey * (double)tsSecTimes[precision] > (double)maxKey)) {
      return false;
    }
    return true;
  }
  return true;
}

static int32_t tsdbAsyncRetentionImpl(STsdb *tsdb, STimeWindow tw, int8_t optrType, int8_t triggerType) {
  int32_t code = 0;
  int32_t lino = 0;

  // check if background task is disabled
  if (tsdb->bgTaskDisabled) {
    tsdbInfo("vgId:%d, background task is disabled, skip retention", TD_VID(tsdb->pVnode));
    return 0;
  }

  STFileSet *fset;
  TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {
    if (!tsdbInRetentionTimeRange(tsdb, fset->fid, tw, optrType)) {
      continue;
    }
    SRtnArg *arg = taosMemoryMalloc(sizeof(*arg));
    if (arg == NULL) {
      TAOS_CHECK_GOTO(terrno, &lino, _exit);
    }

    arg->tsdb = tsdb;
    arg->tw = tw;
    arg->fid = fset->fid;
    arg->nodeId = 0;
    arg->optrType = optrType;
    arg->triggerType = triggerType;
    arg->lastCommit = fset->lastCommit;

    code = vnodeAsync(RETENTION_TASK_ASYNC, EVA_PRIORITY_LOW, tsdbRetention, tsdbRetentionCancel, arg,
                      &fset->retentionTask);
    if (code) {
      taosMemoryFree(arg);
      TSDB_CHECK_CODE(code, lino, _exit);
    } else {
      arg->taskid = fset->retentionTask;
      int64_t fileSize = tsdbTFileSetGetDataSize(fset);
      TAOS_UNUSED(tsdbAddRetentionMonitorTask(tsdb, fset->fid, &arg->taskid, fileSize));
    }
  }
_exit:
  if (code) {
    tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
  }
  return code;
}

int32_t tsdbAsyncRetention(STsdb *tsdb, STimeWindow tw, int8_t optrType, int8_t triggerType) {
  int32_t code = 0;
  (void)taosThreadMutexLock(&tsdb->mutex);
  code = tsdbAsyncRetentionImpl(tsdb, tw, optrType, triggerType);
  (void)taosThreadMutexUnlock(&tsdb->mutex);
  return code;
}

1	/*
2	* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3	*
4	* This program is free software: you can use, redistribute, and/or modify
5	* it under the terms of the GNU Affero General Public License, version 3
6	* or later ("AGPL"), as published by the Free Software Foundation.
7	*
8	* This program is distributed in the hope that it will be useful, but WITHOUT
9	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10	* FITNESS FOR A PARTICULAR PURPOSE.
11	*
12	* You should have received a copy of the GNU Affero General Public License
13	* along with this program. If not, see <http://www.gnu.org/licenses/>.
14	*/
15
16	#include "tss.h"
17	#include "tsdb.h"
18	#include "tsdbFS2.h"
19	#include "tsdbFSet2.h"
20	#include "vnd.h"
21	#include "tsdbInt.h"
22
23	extern int32_t tsdbAsyncCompact(STsdb tsdb, const STimeWindow tw, ETsdbOpType type);
24
25	// tsdbRetentionMonitor.c
26	extern int32_t tsdbAddRetentionMonitorTask(STsdb tsdb, int32_t fid, SVATaskID taskId, int64_t fileSize);
27	extern void tsdbRemoveRetentionMonitorTask(STsdb tsdb, SVATaskID taskId);
28
29	static int32_t tsdbDoRemoveFileObject(SRTNer rtner, const STFileObj fobj) {	×
30	STFileOp op = {	×
31	.optype = TSDB_FOP_REMOVE,
32	.fid = fobj->f->fid,	×
33	.of = fobj->f[0],
34	};
35
36	return TARRAY2_APPEND(&rtner->fopArr, op);	×
37	}
38
39	static int32_t tsdbCopyFileWithLimitedSpeed(TdFilePtr from, TdFilePtr to, int64_t size, uint32_t limitMB) {	159✔
40	int64_t interval = 1000; // 1s	159✔
41	int64_t limit = limitMB ? limitMB * 1024 * 1024 : INT64_MAX;	159!
42	int64_t offset = 0;	159✔
43	int64_t remain = size;	159✔
44
45	while (remain > 0) {	318✔
46	int64_t n;
47	int64_t last = taosGetTimestampMs();	159✔
48	if ((n = taosFSendFile(to, from, &offset, TMIN(limit, remain))) < 0) {	159!
49	TAOS_CHECK_RETURN(terrno);	×
50	}
51
52	remain -= n;	159✔
53
54	if (remain > 0) {	159!
55	int64_t elapsed = taosGetTimestampMs() - last;	×
56	if (elapsed < interval) {	×
57	taosMsleep(interval - elapsed);	×
58	}
59	}
60	}
61
62	return 0;	159✔
63	}
64
65	static int32_t tsdbDoCopyFileLC(SRTNer rtner, const STFileObj from, const STFile *to) {	×
66	int32_t code = 0;	×
67	int32_t lino = 0;	×
68	TdFilePtr fdFrom = NULL, fdTo = NULL;	×
69	char fname_from[TSDB_FILENAME_LEN];
70	char fname_to[TSDB_FILENAME_LEN];
71
72	tsdbTFileLastChunkName(rtner->tsdb, from->f, fname_from);	×
73	tsdbTFileLastChunkName(rtner->tsdb, to, fname_to);	×
74
75	fdFrom = taosOpenFile(fname_from, TD_FILE_READ);	×
76	if (fdFrom == NULL) {	×
77	TAOS_CHECK_GOTO(terrno, &lino, _exit);	×
78	}
79
80	tsdbInfo("vgId: %d, open tofile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname_to, from->f->size);	×
81
82	fdTo = taosOpenFile(fname_to, TD_FILE_WRITE \| TD_FILE_CREATE \| TD_FILE_TRUNC);	×
83	if (fdTo == NULL) {	×
84	TAOS_CHECK_GOTO(terrno, &lino, _exit);	×
85	}
86
87	SVnodeCfg *pCfg = &rtner->tsdb->pVnode->config;	×
88	int64_t chunksize = (int64_t)pCfg->tsdbPageSize * pCfg->ssChunkSize;	×
89	int64_t lc_size = tsdbLogicToFileSize(to->size, rtner->szPage) - chunksize * (to->lcn - 1);	×
90
91	if (taosFSendFile(fdTo, fdFrom, 0, lc_size) < 0) {	×
92	TAOS_CHECK_GOTO(terrno, &lino, _exit);	×
93	}
94
95	_exit:	×
96	if (code) {	×
97	tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,	×
98	tstrerror(code));
99	}
100	if (taosCloseFile(&fdFrom) != 0) {	×
101	tsdbError("vgId:%d, failed to close file %s", TD_VID(rtner->tsdb->pVnode), fname_from);	×
102	}
103	if (taosCloseFile(&fdTo) != 0) {	×
104	tsdbError("vgId:%d, failed to close file %s", TD_VID(rtner->tsdb->pVnode), fname_to);	×
105	}
106	return code;	×
107	}
108
109	static int32_t tsdbDoCopyFile(SRTNer rtner, const STFileObj from, const STFile *to) {	159✔
110	int32_t code = 0;	159✔
111	int32_t lino = 0;	159✔
112
113	char fname[TSDB_FILENAME_LEN];
114	TdFilePtr fdFrom = NULL;	159✔
115	TdFilePtr fdTo = NULL;	159✔
116
117	tsdbTFileName(rtner->tsdb, to, fname);	159✔
118
119	fdFrom = taosOpenFile(from->fname, TD_FILE_READ);	159✔
120	if (fdFrom == NULL) {	159!
121	TAOS_CHECK_GOTO(terrno, &lino, _exit);	×
122	}
123
124	tsdbInfo("vgId: %d, open tofile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, from->f->size);	159!
125
126	fdTo = taosOpenFile(fname, TD_FILE_WRITE \| TD_FILE_CREATE \| TD_FILE_TRUNC);	159✔
127	if (fdTo == NULL) {	159!
128	TAOS_CHECK_GOTO(terrno, &lino, _exit);	×
129	}
130	TSDB_CHECK_CODE(code, lino, _exit);	159!
131
132	TAOS_CHECK_GOTO(tsdbCopyFileWithLimitedSpeed(fdFrom, fdTo, tsdbLogicToFileSize(from->f->size, rtner->szPage),	318!
133	tsRetentionSpeedLimitMB),
134	&lino, _exit);
135
136	_exit:	159✔
137	if (code) {	159!
138	tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,	×
139	tstrerror(code));
140	}
141	if (taosCloseFile(&fdFrom) != 0) {	159!
142	tsdbTrace("vgId:%d, failed to close file", TD_VID(rtner->tsdb->pVnode));	×
143	}
144	if (taosCloseFile(&fdTo) != 0) {	159!
145	tsdbTrace("vgId:%d, failed to close file", TD_VID(rtner->tsdb->pVnode));	×
146	}
147	return code;	159✔
148	}
149
150	static int32_t tsdbDoMigrateFileObj(SRTNer rtner, const STFileObj fobj, const SDiskID *did) {	159✔
151	int32_t code = 0;	159✔
152	int32_t lino = 0;	159✔
153	STFileOp op = {0};	159✔
154	int32_t lcn = fobj->f->lcn;	159✔
155
156	// remove old
157	op = (STFileOp){	159✔
158	.optype = TSDB_FOP_REMOVE,
159	.fid = fobj->f->fid,	159✔
160	.of = fobj->f[0],	159✔
161	};
162
163	TAOS_CHECK_GOTO(TARRAY2_APPEND(&rtner->fopArr, op), &lino, _exit);	318!
164
165	// create new
166	op = (STFileOp){	159✔
167	.optype = TSDB_FOP_CREATE,
168	.fid = fobj->f->fid,	159✔
169	.nf =
170	{
171	.type = fobj->f->type,	159✔
172	.did = did[0],	159✔
173	.fid = fobj->f->fid,	159✔
174	.minVer = fobj->f->minVer,	159✔
175	.maxVer = fobj->f->maxVer,	159✔
176	.mid = fobj->f->mid,	159✔
177	.cid = fobj->f->cid,	159✔
178	.size = fobj->f->size,	159✔
179	.lcn = lcn,
180	.stt[0] =
181	{
182	.level = fobj->f->stt[0].level,	159✔
183	},
184	},
185	};
186
187	TAOS_CHECK_GOTO(TARRAY2_APPEND(&rtner->fopArr, op), &lino, _exit);	318!
188
189	// do copy the file
190
191	if (lcn < 1) {	159!
192	TAOS_CHECK_GOTO(tsdbDoCopyFile(rtner, fobj, &op.nf), &lino, _exit);	159!
193	} else {
194	TAOS_CHECK_GOTO(tsdbDoCopyFileLC(rtner, fobj, &op.nf), &lino, _exit);	×
195	}
196
197	_exit:	×
198	if (code) {	159!
199	tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,	×
200	tstrerror(code));
201	}
202	return code;	159✔
203	}
204
205
206	static int32_t tsdbDoRetentionEnd(SRTNer *rtner, EFEditT etype) {	254✔
207	int32_t code = 0;	254✔
208	int32_t lino = 0;	254✔
209
210	if (TARRAY2_SIZE(&rtner->fopArr) > 0) {	254✔
211	TAOS_CHECK_GOTO(tsdbFSEditBegin(rtner->tsdb->pFS, &rtner->fopArr, etype), &lino, _exit);	69!
212
213	(void)taosThreadMutexLock(&rtner->tsdb->mutex);	69✔
214
215	code = tsdbFSEditCommit(rtner->tsdb->pFS);	69✔
216	if (code) {	69!
217	(void)taosThreadMutexUnlock(&rtner->tsdb->mutex);	×
218	TSDB_CHECK_CODE(code, lino, _exit);	×
219	}
220
221	(void)taosThreadMutexUnlock(&rtner->tsdb->mutex);	69✔
222
223	TARRAY2_DESTROY(&rtner->fopArr, NULL);	69!
224	}
225
226	_exit:	185✔
227	if (code) {	254!
228	tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,	×
229	tstrerror(code));
230	} else {
231	tsdbDebug("vid:%d, cid:%" PRId64 ", %s done", TD_VID(rtner->tsdb->pVnode), rtner->cid, __func__);	254✔
232	}
233	return code;	254✔
234	}
235
236	static int32_t tsdbRemoveOrMoveFileObject(SRTNer rtner, int32_t expLevel, STFileObj fobj) {	1,018✔
237	int32_t code = 0;	1,018✔
238	int32_t lino = 0;	1,018✔
239
240	if (fobj == NULL) {	1,018✔
241	return code;	462✔
242	}
243
244	if (expLevel < 0) {	556!
245	// remove the file
246	code = tsdbDoRemoveFileObject(rtner, fobj);	×
247	TSDB_CHECK_CODE(code, lino, _exit);	×
248	} else if (expLevel > fobj->f->did.level) {	556✔
249	// Try to move the file to a new level
250	for (; expLevel > fobj->f->did.level; expLevel--) {	159!
251	SDiskID diskId = {0};	159✔
252
253	code = tsdbAllocateDiskAtLevel(rtner->tsdb, expLevel, tsdbFTypeLabel(fobj->f->type), &diskId);	159✔
254	if (code) {	159!
255	tsdbTrace("vgId:%d, cannot allocate disk for file %s, level:%d, reason:%s, skip!", TD_VID(rtner->tsdb->pVnode),	×
256	fobj->fname, expLevel, tstrerror(code));
257	code = 0;	×
258	continue;	×
259	} else {
260	tsdbInfo("vgId:%d start to migrate file %s from level %d to %d, size:%" PRId64, TD_VID(rtner->tsdb->pVnode),	159!
261	fobj->fname, fobj->f->did.level, diskId.level, fobj->f->size);
262
263	code = tsdbDoMigrateFileObj(rtner, fobj, &diskId);	159✔
264	TSDB_CHECK_CODE(code, lino, _exit);	159!
265
266	tsdbInfo("vgId:%d end to migrate file %s from level %d to %d, size:%" PRId64, TD_VID(rtner->tsdb->pVnode),	159!
267	fobj->fname, fobj->f->did.level, diskId.level, fobj->f->size);
268	break;	159✔
269	}
270	}
271	}
272
273	_exit:	397✔
274	if (code) {	556!
275	tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,	×
276	tstrerror(code));
277	}
278	return code;	556✔
279	}
280
281	static int32_t tsdbDoRetention(SRTNer *rtner) {	228✔
282	int32_t code = 0;	228✔
283	int32_t lino = 0;	228✔
284	STFileObj *fobj = NULL;	228✔
285	STFileSet *fset = rtner->fset;	228✔
286
287	// handle data file sets
288	int32_t expLevel = tsdbFidLevel(fset->fid, &rtner->tsdb->keepCfg, rtner->tw.ekey);	228✔
289	for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ++ftype) {	1,140✔
290	code = tsdbRemoveOrMoveFileObject(rtner, expLevel, fset->farr[ftype]);	912✔
291	TSDB_CHECK_CODE(code, lino, _exit);	912!
292	}
293
294	// handle stt file
295	SSttLvl *lvl;
296	TARRAY2_FOREACH(fset->lvlArr, lvl) {	306✔
297	TARRAY2_FOREACH(lvl->fobjArr, fobj) {	184✔
298	code = tsdbRemoveOrMoveFileObject(rtner, expLevel, fobj);	106✔
299	TSDB_CHECK_CODE(code, lino, _exit);	106!
300	}
301	}
302
303	_exit:	228✔
304	if (code) {	228!
305	tsdbError("vgId:%d, %s failed at %s:%d since %s", TD_VID(rtner->tsdb->pVnode), __func__, __FILE__, lino,	×
306	tstrerror(code));
307	}
308	return code;	228✔
309	}
310
311	void tsdbRetentionCancel(void *arg) { taosMemoryFree(arg); }	×
312
313	static bool tsdbFSetNeedRetention(STFileSet *fset, int32_t expLevel) {	6✔
314	if (expLevel < 0) {	6!
315	return false;	×
316	}
317
318	STFileObj *fobj = NULL;	6✔
319	for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX; ++ftype) {	30✔
320	fobj = fset->farr[ftype];	24✔
321	if (fobj && (expLevel > fset->farr[ftype]->f->did.level)) {	24!
322	return true;	×
323	}
324	}
325
326	// handle stt file
327	SSttLvl *lvl = NULL;	6✔
328	TARRAY2_FOREACH(fset->lvlArr, lvl) {	8✔
329	TARRAY2_FOREACH(lvl->fobjArr, fobj) {	8✔
330	if (fobj && (expLevel > fobj->f->did.level)) {	6!
331	return true;	4✔
332	}
333	}
334	}
335
336	return false;	2✔
337	}
338	#ifdef TD_ENTERPRISE
339	static bool tsdbShouldRollup(STsdb tsdb, SRTNer rtner, SRtnArg *rtnArg) {	254✔
340	SVnode *pVnode = tsdb->pVnode;	254✔
341	STFileSet *fset = rtner->fset;	254✔
342
343	if (!VND_IS_RSMA(pVnode)) {	254✔
344	return false;	226✔
345	}
346
347	int32_t expLevel = tsdbFidLevel(fset->fid, &tsdb->keepCfg, rtner->tw.ekey);	28✔
348	if (expLevel <= 0) {	28!
349	return false;	×
350	}
351
352	if (rtnArg->optrType == TSDB_OPTR_ROLLUP) {	28✔
353	return true;	22✔
354	} else if (rtnArg->optrType == TSDB_OPTR_NORMAL) {	6!
355	return tsdbFSetNeedRetention(fset, expLevel);	6✔
356	}
357	return false;	×
358	}
359	#endif
360	int32_t tsdbRetention(void *arg) {	254✔
361	int32_t code = 0;	254✔
362	int32_t lino = 0;	254✔
363
364	SRtnArg rtnArg = (SRtnArg )arg;	254✔
365	STsdb *pTsdb = rtnArg->tsdb;	254✔
366	SVnode *pVnode = pTsdb->pVnode;	254✔
367	STFileSet *fset = NULL;	254✔
368	SRTNer rtner = {	508✔
369	.tsdb = pTsdb,
370	.szPage = pVnode->config.tsdbPageSize,	254✔
371	.tw = rtnArg->tw,
372	.lastCommit = rtnArg->lastCommit,	254✔
373	.cid = tsdbFSAllocEid(pTsdb->pFS),	254✔
374	.nodeId = rtnArg->nodeId,	254✔
375	};
376
377	// begin task
378	(void)taosThreadMutexLock(&pTsdb->mutex);	254✔
379
380	// check if background task is disabled
381	if (pTsdb->bgTaskDisabled) {	254!
382	tsdbInfo("vgId:%d, background task is disabled, skip retention", TD_VID(pTsdb->pVnode));	×
383	(void)taosThreadMutexUnlock(&pTsdb->mutex);	×
384	return 0;	×
385	}
386
387	// set flag and copy
388	tsdbBeginTaskOnFileSet(pTsdb, rtnArg->fid, EVA_TASK_RETENTION, &fset);	254✔
389	if (fset && (code = tsdbTFileSetInitCopy(pTsdb, fset, &rtner.fset))) {	254!
390	(void)taosThreadMutexUnlock(&pTsdb->mutex);	×
391	TSDB_CHECK_CODE(code, lino, _exit);	×
392	}
393
394	(void)taosThreadMutexUnlock(&pTsdb->mutex);	254✔
395
396	// do retention
397	if (rtner.fset) {	254!
398	EFEditT etype = TSDB_FEDIT_RETENTION;	254✔
399	if (rtnArg->optrType == TSDB_OPTR_SSMIGRATE) {	254!
400	etype = TSDB_FEDIT_SSMIGRATE;	×
401	TAOS_CHECK_GOTO(tsdbDoSsMigrate(&rtner), &lino, _exit);	×
402	#ifdef TD_ENTERPRISE
403	} else if (tsdbShouldRollup(pTsdb, &rtner, rtnArg)) {	254✔
404	etype = TSDB_FEDIT_ROLLUP;	26✔
405	TAOS_CHECK_GOTO(tsdbDoRollup(&rtner), &lino, _exit);	26!
406	#endif
407	} else if (rtnArg->optrType == TSDB_OPTR_NORMAL) {	228!
408	TAOS_CHECK_GOTO(tsdbDoRetention(&rtner), &lino, _exit);	228!
409	} else {
410	goto _exit;	×
411	}
412
413	TAOS_CHECK_GOTO(tsdbDoRetentionEnd(&rtner, etype), &lino, _exit);	254!
414	}
415
416	_exit:	254✔
417	if (rtner.fset) {	254!
418	(void)taosThreadMutexLock(&pTsdb->mutex);	254✔
419	tsdbFinishTaskOnFileSet(pTsdb, rtnArg->fid, EVA_TASK_RETENTION);	254✔
420	(void)taosThreadMutexUnlock(&pTsdb->mutex);	254✔
421	}
422
423	// clear resources
424	tsdbTFileSetClear(&rtner.fset);	254✔
425	TARRAY2_DESTROY(&rtner.fopArr, NULL);	254!
426	(void)tsdbRemoveRetentionMonitorTask(pTsdb, &rtnArg->taskid);	254✔
427	taosMemoryFree(arg);	254!
428	if (code) {	254!
429	tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(pTsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));	×
430	}
431	return code;	254✔
432	}
433
434	static bool tsdbInRetentionTimeRange(STsdb *tsdb, int32_t fid, STimeWindow tw, int8_t optrType) {	256✔
435	if (optrType == TSDB_OPTR_ROLLUP) {	256✔
436	TSKEY minKey, maxKey;
437	int8_t precision = tsdb->keepCfg.precision;	24✔
438	if (precision < TSDB_TIME_PRECISION_MILLI \|\| precision > TSDB_TIME_PRECISION_NANO) {	24!
439	tsdbError("vgId:%d, failed to check retention time range since invalid precision %" PRIi8, TD_VID(tsdb->pVnode), precision);	×
440	return false;	×
441	}
442	tsdbFidKeyRange(fid, tsdb->keepCfg.days, precision, &minKey, &maxKey);	24✔
443
444	if ((tw.ekey != INT64_MAX) && ((double)tw.ekey * (double)tsSecTimes[precision] < (double)minKey)) {	24!
445	return false;	×
446	}
447	if ((tw.skey != INT64_MIN) && ((double)tw.skey * (double)tsSecTimes[precision] > (double)maxKey)) {	24✔
448	return false;	2✔
449	}
450	return true;	22✔
451	}
452	return true;	232✔
453	}
454
455	static int32_t tsdbAsyncRetentionImpl(STsdb *tsdb, STimeWindow tw, int8_t optrType, int8_t triggerType) {	68✔
456	int32_t code = 0;	68✔
457	int32_t lino = 0;	68✔
458
459	// check if background task is disabled
460	if (tsdb->bgTaskDisabled) {	68!
461	tsdbInfo("vgId:%d, background task is disabled, skip retention", TD_VID(tsdb->pVnode));	×
462	return 0;	×
463	}
464
465	STFileSet *fset;
466	TARRAY2_FOREACH(tsdb->pFS->fSetArr, fset) {	324✔
467	if (!tsdbInRetentionTimeRange(tsdb, fset->fid, tw, optrType)) {	256✔
468	continue;	2✔
469	}
470	SRtnArg arg = taosMemoryMalloc(sizeof(arg));	254!
471	if (arg == NULL) {	254!
472	TAOS_CHECK_GOTO(terrno, &lino, _exit);	×
473	}
474
475	arg->tsdb = tsdb;	254✔
476	arg->tw = tw;	254✔
477	arg->fid = fset->fid;	254✔
478	arg->nodeId = 0;	254✔
479	arg->optrType = optrType;	254✔
480	arg->triggerType = triggerType;	254✔
481	arg->lastCommit = fset->lastCommit;	254✔
482
483	code = vnodeAsync(RETENTION_TASK_ASYNC, EVA_PRIORITY_LOW, tsdbRetention, tsdbRetentionCancel, arg,	254✔
484	&fset->retentionTask);
485	if (code) {	252!
486	taosMemoryFree(arg);	×
487	TSDB_CHECK_CODE(code, lino, _exit);	×
488	} else {
489	arg->taskid = fset->retentionTask;	252✔
490	int64_t fileSize = tsdbTFileSetGetDataSize(fset);	252✔
491	TAOS_UNUSED(tsdbAddRetentionMonitorTask(tsdb, fset->fid, &arg->taskid, fileSize));	253✔
492	}
493	}
494	_exit:	68✔
495	if (code) {	68!
496	tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));	×
497	}
498	return code;	68✔
499	}
500
501	int32_t tsdbAsyncRetention(STsdb *tsdb, STimeWindow tw, int8_t optrType, int8_t triggerType) {	66✔
502	int32_t code = 0;	66✔
503	(void)taosThreadMutexLock(&tsdb->mutex);	66✔
504	code = tsdbAsyncRetentionImpl(tsdb, tw, optrType, triggerType);	68✔
505	(void)taosThreadMutexUnlock(&tsdb->mutex);	68✔
506	return code;	68✔
507	}

taosdata / TDengine / #4791

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous