#5074

Committed 17 May 2026 01:15AM UTC coverage: 73.309% (-0.008%) from 73.317%

Build # #5074

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

feat (TDgpt): Dynamic Model Synchronization Enhancements (#35344)

* refactor: do some internal refactor.

* fix: fix multiprocess sync issue.

* feat: add dynamic anomaly detection and forecasting services

* fix: log error message for undeploying model in exception handling

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* fix: handle undeploy when model exists only on disk

Agent-Logs-Url: https://github.com/taosdata/TDengine/sessions/286aafa0-c3ce-4c27-b803-2707571e9dc1

Co-authored-by: hjxilinx <8252296+hjxilinx@users.noreply.github.com>

* fix: guard dynamic registry concurrent access

Agent-Logs-Url: https://github.com/taosdata/TDengine/sessions/5e4db858-6458-40f4-ac28-d1b1b7f97c18

Co-authored-by: hjxilinx <8252296+hjxilinx@users.noreply.github.com>

* fix: tighten service list locking scope

Agent-Logs-Url: https://github.com/taosdata/TDengine/sessions/5e4db858-6458-40f4-ac28-d1b1b7f97c18

Co-authored-by: hjxilinx <8252296+hjxilinx@users.noreply.github.com>

* fix: restore prophet support and update tests per review feedback

Agent-Logs-Url: https://github.com/taosdata/TDengine/sessions/92298ae1-7da6-4d07-b20e-101c7cd0b26b

Co-authored-by: hjxilinx <8252296+hjxilinx@users.noreply.github.com>

* fix: improve test name and move copy inside lock scope

Agent-Logs-Url: https://github.com/taosdata/TDengine/sessions/92298ae1-7da6-4d07-b20e-101c7cd0b26b

Co-authored-by: hjxilinx <8252296+hjxilinx@users.noreply.github.com>

* Potential fix for pull request finding

Co-au... (continued)

Coverage Stats

281356 of 383795 relevant lines covered (73.31%)

138311536.58 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

73.26

/source/dnode/vnode/src/tsdb/tsdbFile2.c

/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "tsdbFile2.h"
#include "vnd.h"

// to_json
static int32_t head_to_json(const STFile *file, cJSON *json);
static int32_t data_to_json(const STFile *file, cJSON *json);
static int32_t sma_to_json(const STFile *file, cJSON *json);
static int32_t tomb_to_json(const STFile *file, cJSON *json);
static int32_t stt_to_json(const STFile *file, cJSON *json);

// from_json
static int32_t head_from_json(const cJSON *json, STFile *file);
static int32_t data_from_json(const cJSON *json, STFile *file);
static int32_t sma_from_json(const cJSON *json, STFile *file);
static int32_t tomb_from_json(const cJSON *json, STFile *file);
static int32_t stt_from_json(const cJSON *json, STFile *file);

static const struct {
  const char *suffix;
  int32_t (*to_json)(const STFile *file, cJSON *json);
  int32_t (*from_json)(const cJSON *json, STFile *file);
} g_tfile_info[] = {
    [TSDB_FTYPE_HEAD] = {"head", head_to_json, head_from_json},
    [TSDB_FTYPE_DATA] = {"data", data_to_json, data_from_json},
    [TSDB_FTYPE_SMA] = {"sma", sma_to_json, sma_from_json},
    [TSDB_FTYPE_TOMB] = {"tomb", tomb_to_json, tomb_from_json},
    [TSDB_FTYPE_STT] = {"stt", stt_to_json, stt_from_json},
};

void tsdbRemoveFile(const char *fname) {
  int32_t code = taosRemoveFile(fname);
  if (code) {
    tsdbError("failed to remove file:%s, code:%d, error:%s", fname, code, tstrerror(code));
  } else {
    tsdbInfo("file:%s is removed", fname);
  }
}

static int32_t tfile_to_json(const STFile *file, cJSON *json) {
  /* did.level */
  if (cJSON_AddNumberToObject(json, "did.level", file->did.level) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  /* did.id */
  if (cJSON_AddNumberToObject(json, "did.id", file->did.id) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  /* lcn - last chunk number */
  if (cJSON_AddNumberToObject(json, "lcn", file->lcn) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  /* fid */
  if (cJSON_AddNumberToObject(json, "fid", file->fid) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  /* mid - migration id */
  if (cJSON_AddNumberToObject(json, "mid", file->mid) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  /* cid */
  if (cJSON_AddNumberToObject(json, "cid", file->cid) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  /* size */
  if (cJSON_AddNumberToObject(json, "size", file->size) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  if (file->minVer <= file->maxVer) {
    /* minVer */
    if (cJSON_AddNumberToObject(json, "minVer", file->minVer) == NULL) {
      return TSDB_CODE_OUT_OF_MEMORY;
    }

    /* maxVer */
    if (cJSON_AddNumberToObject(json, "maxVer", file->maxVer) == NULL) {
      return TSDB_CODE_OUT_OF_MEMORY;
    }
  }
  return 0;
}

static int32_t tfile_from_json(const cJSON *json, STFile *file) {
  const cJSON *item;

  /* did.level */
  item = cJSON_GetObjectItem(json, "did.level");
  if (cJSON_IsNumber(item)) {
    file->did.level = item->valuedouble;
  } else {
    return TSDB_CODE_FILE_CORRUPTED;
  }

  /* did.id */
  item = cJSON_GetObjectItem(json, "did.id");
  if (cJSON_IsNumber(item)) {
    file->did.id = item->valuedouble;
  } else {
    return TSDB_CODE_FILE_CORRUPTED;
  }

  /* lcn */
  item = cJSON_GetObjectItem(json, "lcn");
  if (cJSON_IsNumber(item)) {
    file->lcn = item->valuedouble;
  } else {
    // return TSDB_CODE_FILE_CORRUPTED;
  }

  /* fid */
  item = cJSON_GetObjectItem(json, "fid");
  if (cJSON_IsNumber(item)) {
    file->fid = item->valuedouble;
  } else {
    return TSDB_CODE_FILE_CORRUPTED;
  }

  /* mid - migration id */
  item = cJSON_GetObjectItem(json, "mid");
  if (cJSON_IsNumber(item)) {
    file->mid = item->valuedouble;
  } else {
    file->mid = 0;
  }

  /* cid */
  item = cJSON_GetObjectItem(json, "cid");
  if (cJSON_IsNumber(item)) {
    file->cid = item->valuedouble;
  } else {
    return TSDB_CODE_FILE_CORRUPTED;
  }

  /* size */
  item = cJSON_GetObjectItem(json, "size");
  if (cJSON_IsNumber(item)) {
    file->size = item->valuedouble;
  } else {
    return TSDB_CODE_FILE_CORRUPTED;
  }

  /* minVer */
  file->minVer = VERSION_MAX;
  item = cJSON_GetObjectItem(json, "minVer");
  if (cJSON_IsNumber(item)) {
    file->minVer = item->valuedouble;
  }

  /* maxVer */
  file->maxVer = VERSION_MIN;
  item = cJSON_GetObjectItem(json, "maxVer");
  if (cJSON_IsNumber(item)) {
    file->maxVer = item->valuedouble;
  }
  return 0;
}

static int32_t head_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); }
static int32_t data_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); }
static int32_t sma_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); }
static int32_t tomb_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); }
static int32_t stt_to_json(const STFile *file, cJSON *json) {
  TAOS_CHECK_RETURN(tfile_to_json(file, json));

  /* lvl */
  if (cJSON_AddNumberToObject(json, "level", file->stt->level) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  return 0;
}

static int32_t head_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); }
static int32_t data_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); }
static int32_t sma_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); }
static int32_t tomb_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); }
static int32_t stt_from_json(const cJSON *json, STFile *file) {
  TAOS_CHECK_RETURN(tfile_from_json(json, file));

  const cJSON *item;

  /* lvl */
  item = cJSON_GetObjectItem(json, "level");
  if (cJSON_IsNumber(item)) {
    file->stt->level = item->valuedouble;
  } else {
    return TSDB_CODE_FILE_CORRUPTED;
  }

  return 0;
}

int32_t tsdbTFileToJson(const STFile *file, cJSON *json) {
  if (file->type == TSDB_FTYPE_STT) {
    return g_tfile_info[file->type].to_json(file, json);
  } else {
    cJSON *item = cJSON_AddObjectToObject(json, g_tfile_info[file->type].suffix);
    if (item == NULL) {
      return TSDB_CODE_OUT_OF_MEMORY;
    }
    return g_tfile_info[file->type].to_json(file, item);
  }
}

int32_t tsdbJsonToTFile(const cJSON *json, tsdb_ftype_t ftype, STFile *f) {
  f[0] = (STFile){.type = ftype};

  if (ftype == TSDB_FTYPE_STT) {
    TAOS_CHECK_RETURN(g_tfile_info[ftype].from_json(json, f));
  } else {
    const cJSON *item = cJSON_GetObjectItem(json, g_tfile_info[ftype].suffix);
    if (cJSON_IsObject(item)) {
      TAOS_CHECK_RETURN(g_tfile_info[ftype].from_json(item, f));
    } else {
      return TSDB_CODE_NOT_FOUND;
    }
  }

  return 0;
}

int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) {
  fobj[0] = taosMemoryMalloc(sizeof(*fobj[0]));
  if (!fobj[0]) {
    return terrno;
  }

  (void)taosThreadMutexInit(&fobj[0]->mutex, NULL);
  fobj[0]->f[0] = f[0];
  fobj[0]->state = TSDB_FSTATE_LIVE;
  fobj[0]->ref = 1;
  tsdbTFileName(pTsdb, f, fobj[0]->fname);
  // fobj[0]->nlevel = tfsGetLevel(pTsdb->pVnode->pTfs);
  fobj[0]->nlevel = vnodeNodeId(pTsdb->pVnode);
  return 0;
}

int32_t tsdbTFileObjRef(STFileObj *fobj) {
  int32_t nRef;
  (void)taosThreadMutexLock(&fobj->mutex);

  if (fobj->ref <= 0 || fobj->state != TSDB_FSTATE_LIVE) {
    tsdbError("file %s, fobj:%p ref:%d", fobj->fname, fobj, fobj->ref);
    (void)taosThreadMutexUnlock(&fobj->mutex);
    return TSDB_CODE_FAILED;
  }

  nRef = ++fobj->ref;
  (void)taosThreadMutexUnlock(&fobj->mutex);
  tsdbTrace("ref file %s, fobj:%p ref:%d", fobj->fname, fobj, nRef);
  return 0;
}

int32_t tsdbTFileObjUnref(STFileObj *fobj) {
  (void)taosThreadMutexLock(&fobj->mutex);
  int32_t nRef = --fobj->ref;
  (void)taosThreadMutexUnlock(&fobj->mutex);

  if (nRef < 0) {
    tsdbError("file %s, fobj:%p ref:%d", fobj->fname, fobj, nRef);
    return TSDB_CODE_FAILED;
  }

  tsdbTrace("unref file %s, fobj:%p ref:%d", fobj->fname, fobj, nRef);
  if (nRef == 0) {
    if (fobj->state == TSDB_FSTATE_DEAD) {
      tsdbRemoveFile(fobj->fname);
    }
    (void)taosThreadMutexDestroy(&fobj->mutex);
    taosMemoryFree(fobj);
  }

  return 0;
}

static void tsdbTFileObjRemoveLC(STFileObj *fobj) {
  if (fobj->f->type != TSDB_FTYPE_DATA || fobj->f->lcn < 1) {
    tsdbRemoveFile(fobj->fname);
    return;
  }

#ifdef USE_SHARED_STORAGE
  // remove local last chunk file
  char lc_path[TSDB_FILENAME_LEN];
  tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN);

  char *dot = strrchr(lc_path, '.');
  if (!dot) {
    tsdbError("unexpected path: %s", lc_path);
    return;
  }
  snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", fobj->f->lcn);

  tsdbRemoveFile(lc_path);
#endif
}

int32_t tsdbTFileObjRemove(STFileObj *fobj) {
  (void)taosThreadMutexLock(&fobj->mutex);
  if (fobj->state != TSDB_FSTATE_LIVE || fobj->ref <= 0) {
    tsdbError("file %s, fobj:%p ref:%d", fobj->fname, fobj, fobj->ref);
    (void)taosThreadMutexUnlock(&fobj->mutex);
    return TSDB_CODE_FAILED;
  }
  fobj->state = TSDB_FSTATE_DEAD;
  int32_t nRef = --fobj->ref;
  (void)taosThreadMutexUnlock(&fobj->mutex);
  tsdbTrace("remove unref file %s, fobj:%p ref:%d", fobj->fname, fobj, nRef);
  if (nRef == 0) {
    tsdbTFileObjRemoveLC(fobj);
    (void)taosThreadMutexDestroy(&fobj->mutex);
    taosMemoryFree(fobj);
  }
  return 0;
}

int32_t tsdbTFileObjRemoveUpdateLC(STFileObj *fobj) {
  (void)taosThreadMutexLock(&fobj->mutex);

  if (fobj->state != TSDB_FSTATE_LIVE || fobj->ref <= 0) {
    (void)taosThreadMutexUnlock(&fobj->mutex);
    tsdbError("file %s, fobj:%p ref:%d", fobj->fname, fobj, fobj->ref);
    return TSDB_CODE_FAILED;
  }

  fobj->state = TSDB_FSTATE_DEAD;
  int32_t nRef = --fobj->ref;
  (void)taosThreadMutexUnlock(&fobj->mutex);
  tsdbTrace("remove unref file %s, fobj:%p ref:%d", fobj->fname, fobj, nRef);
  if (nRef == 0) {
    tsdbTFileObjRemoveLC(fobj);
    (void)taosThreadMutexDestroy(&fobj->mutex);
    taosMemoryFree(fobj);
  }
  return 0;
}

void tsdbTFileName(STsdb *pTsdb, const STFile *f, char fname[]) {
  SVnode *pVnode = pTsdb->pVnode;
  STfs   *pTfs = TSDB_TFS(pTsdb->pVnode);

  if (pTfs) {
    if (pVnode->mounted) {
      // NOTE: the case 'if (f->mid != 0)' is not handled at present, this may be
      //       needed in the future.
      snprintf(fname,                                              //
               TSDB_FILENAME_LEN,                                  //
               "%s%svnode%svnode%d%s%s%sv%df%dver%" PRId64 ".%s",  //
               tfsGetDiskPath(pTfs, f->did),                       //
               TD_DIRSEP,                                          //
               TD_DIRSEP,                                          //
               TSDB_VID(pVnode),                                   //
               TD_DIRSEP,                                          //
               pTsdb->name,                                        //
               TD_DIRSEP,                                          //
               TSDB_VID(pVnode),                                   //
               f->fid,                                             //
               f->cid,                                             //
               g_tfile_info[f->type].suffix);
    } else {
      if (f->mid == 0) {
        snprintf(fname,                              //
                TSDB_FILENAME_LEN,                  //
                "%s%s%s%sv%df%dver%" PRId64 ".%s",  //
                tfsGetDiskPath(pTfs, f->did),       //
                TD_DIRSEP,                          //
                pTsdb->path,                        //
                TD_DIRSEP,                          //
                TD_VID(pVnode),                     //
                f->fid,                             //
                f->cid,                             //
                g_tfile_info[f->type].suffix);
      } else {
        snprintf(fname,                          //
                TSDB_FILENAME_LEN,                  //
                "%s%s%s%sv%df%dver%" PRId64 ".m%d.%s",  //
                tfsGetDiskPath(pTfs, f->did),       //
                TD_DIRSEP,                          //
                pTsdb->path,                        //
                TD_DIRSEP,                          //
                TD_VID(pVnode),                     //
                f->fid,                             //
                f->cid,                             //
                f->mid,                             //
                g_tfile_info[f->type].suffix);
      }
    }
  } else {
     if (f->mid == 0) {
      snprintf(fname,                          //
              TSDB_FILENAME_LEN,              //
              "%s%sv%df%dver%" PRId64 ".%s",  //
              pTsdb->path,                    //
              TD_DIRSEP,                      //
              TD_VID(pVnode),                 //
              f->fid,                         //
              f->cid,                         //
              g_tfile_info[f->type].suffix);
    } else {
      snprintf(fname,                          //
              TSDB_FILENAME_LEN,              //
              "%s%sv%df%dver%" PRId64 ".m%d.%s",  //
              pTsdb->path,                    //
              TD_DIRSEP,                      //
              TD_VID(pVnode),                 //
              f->fid,                         //
              f->cid,                         //
              f->mid,                         //
              g_tfile_info[f->type].suffix);
    }
  }
}

void tsdbTFileLastChunkName(STsdb *pTsdb, const STFile *f, char fname[]) {
  SVnode *pVnode = pTsdb->pVnode;
  STfs   *pTfs = TSDB_TFS(pTsdb->pVnode);

  // NOTE: the case 'if (pVnode->mounted)' is not handled at present, this may be needed
  //       in the future.

  if (f->mid == 0) {
    if (pTfs) {
      snprintf(fname,                                 //
              TSDB_FILENAME_LEN,                     //
              "%s%s%s%sv%df%dver%" PRId64 ".%d.%s",  //
              tfsGetDiskPath(pTfs, f->did),          //
              TD_DIRSEP,                             //
              pTsdb->path,                           //
              TD_DIRSEP,                             //
              TD_VID(pVnode),                        //
              f->fid,                                //
              f->cid,                                //
              f->lcn,                                //
              g_tfile_info[f->type].suffix);
    } else {
      snprintf(fname,                             //
              TSDB_FILENAME_LEN,                 //
              "%s%sv%df%dver%" PRId64 ".%d.%s",  //
              pTsdb->path,                       //
              TD_DIRSEP,                         //
              TD_VID(pVnode),                    //
              f->fid,                            //
              f->cid,                            //
              f->lcn,                            //
              g_tfile_info[f->type].suffix);
    }
  } else {
    if (pTfs) {
      snprintf(fname,                                 //
              TSDB_FILENAME_LEN,                     //
              "%s%s%s%sv%df%dver%" PRId64 ".m%d.%d.%s",  //
              tfsGetDiskPath(pTfs, f->did),          //
              TD_DIRSEP,                             //
              pTsdb->path,                           //
              TD_DIRSEP,                             //
              TD_VID(pVnode),                        //
              f->fid,                                //
              f->cid,                                //
              f->mid,                                //
              f->lcn,                                //
              g_tfile_info[f->type].suffix);
    } else {
      snprintf(fname,                             //
              TSDB_FILENAME_LEN,                 //
              "%s%sv%df%dver%" PRId64 ".m%d.%d.%s",  //
              pTsdb->path,                       //
              TD_DIRSEP,                         //
              TD_VID(pVnode),                    //
              f->fid,                            //
              f->cid,                            //
              f->mid,                            //
              f->lcn,                            //
              g_tfile_info[f->type].suffix);
    }
  }
}

bool tsdbIsSameTFile(const STFile *f1, const STFile *f2) {
  if (f1->type != f2->type) return false;
  if (f1->did.level != f2->did.level) return false;
  if (f1->did.id != f2->did.id) return false;
  if (f1->fid != f2->fid) return false;
  if (f1->cid != f2->cid) return false;
  if (f1->lcn != f2->lcn) return false;
  if (f1->mid != f2->mid) return false;
  return true;
}

bool tsdbIsTFileChanged(const STFile *f1, const STFile *f2) {
  if (f1->size != f2->size) return true;
  // if (f1->type == TSDB_FTYPE_STT && f1->stt->nseg != f2->stt->nseg) return true;
  return false;
}

int32_t tsdbTFileObjCmpr(const STFileObj **fobj1, const STFileObj **fobj2) {
  if (fobj1[0]->f->cid < fobj2[0]->f->cid) {
    return -1;
  } else if (fobj1[0]->f->cid > fobj2[0]->f->cid) {
    return 1;
  } else {
    return 0;
  }
}

const char *tsdbFTypeLabel(tsdb_ftype_t ftype) { return g_tfile_info[ftype].suffix; }

1	/*
2	* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3	*
4	* This program is free software: you can use, redistribute, and/or modify
5	* it under the terms of the GNU Affero General Public License, version 3
6	* or later ("AGPL"), as published by the Free Software Foundation.
7	*
8	* This program is distributed in the hope that it will be useful, but WITHOUT
9	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10	* FITNESS FOR A PARTICULAR PURPOSE.
11	*
12	* You should have received a copy of the GNU Affero General Public License
13	* along with this program. If not, see <http://www.gnu.org/licenses/>.
14	*/
15
16	#include "tsdbFile2.h"
17	#include "vnd.h"
18
19	// to_json
20	static int32_t head_to_json(const STFile file, cJSON json);
21	static int32_t data_to_json(const STFile file, cJSON json);
22	static int32_t sma_to_json(const STFile file, cJSON json);
23	static int32_t tomb_to_json(const STFile file, cJSON json);
24	static int32_t stt_to_json(const STFile file, cJSON json);
25
26	// from_json
27	static int32_t head_from_json(const cJSON json, STFile file);
28	static int32_t data_from_json(const cJSON json, STFile file);
29	static int32_t sma_from_json(const cJSON json, STFile file);
30	static int32_t tomb_from_json(const cJSON json, STFile file);
31	static int32_t stt_from_json(const cJSON json, STFile file);
32
33	static const struct {
34	const char *suffix;
35	int32_t (to_json)(const STFile file, cJSON *json);
36	int32_t (from_json)(const cJSON json, STFile *file);
37	} g_tfile_info[] = {
38	[TSDB_FTYPE_HEAD] = {"head", head_to_json, head_from_json},
39	[TSDB_FTYPE_DATA] = {"data", data_to_json, data_from_json},
40	[TSDB_FTYPE_SMA] = {"sma", sma_to_json, sma_from_json},
41	[TSDB_FTYPE_TOMB] = {"tomb", tomb_to_json, tomb_from_json},
42	[TSDB_FTYPE_STT] = {"stt", stt_to_json, stt_from_json},
43	};
44
45	void tsdbRemoveFile(const char *fname) {	3,973,898✔
46	int32_t code = taosRemoveFile(fname);	3,973,898✔
47	if (code) {	3,973,898✔
48	tsdbError("failed to remove file:%s, code:%d, error:%s", fname, code, tstrerror(code));	606✔
49	} else {
50	tsdbInfo("file:%s is removed", fname);	3,973,292✔
51	}
52	}	3,973,898✔
53
54	static int32_t tfile_to_json(const STFile file, cJSON json) {	34,174,984✔
55	/* did.level */
56	if (cJSON_AddNumberToObject(json, "did.level", file->did.level) == NULL) {	34,174,984✔
57	return TSDB_CODE_OUT_OF_MEMORY;	×
58	}
59
60	/* did.id */
61	if (cJSON_AddNumberToObject(json, "did.id", file->did.id) == NULL) {	34,172,787✔
62	return TSDB_CODE_OUT_OF_MEMORY;	×
63	}
64
65	/* lcn - last chunk number */
66	if (cJSON_AddNumberToObject(json, "lcn", file->lcn) == NULL) {	34,173,843✔
67	return TSDB_CODE_OUT_OF_MEMORY;	×
68	}
69
70	/* fid */
71	if (cJSON_AddNumberToObject(json, "fid", file->fid) == NULL) {	34,174,375✔
72	return TSDB_CODE_OUT_OF_MEMORY;	×
73	}
74
75	/* mid - migration id */
76	if (cJSON_AddNumberToObject(json, "mid", file->mid) == NULL) {	34,173,899✔
77	return TSDB_CODE_OUT_OF_MEMORY;	×
78	}
79
80	/* cid */
81	if (cJSON_AddNumberToObject(json, "cid", file->cid) == NULL) {	34,174,570✔
82	return TSDB_CODE_OUT_OF_MEMORY;	×
83	}
84
85	/* size */
86	if (cJSON_AddNumberToObject(json, "size", file->size) == NULL) {	34,172,604✔
87	return TSDB_CODE_OUT_OF_MEMORY;	×
88	}
89
90	if (file->minVer <= file->maxVer) {	34,173,640✔
91	/* minVer */
92	if (cJSON_AddNumberToObject(json, "minVer", file->minVer) == NULL) {	34,172,664✔
93	return TSDB_CODE_OUT_OF_MEMORY;	×
94	}
95
96	/* maxVer */
97	if (cJSON_AddNumberToObject(json, "maxVer", file->maxVer) == NULL) {	34,174,093✔
98	return TSDB_CODE_OUT_OF_MEMORY;	×
99	}
100	}
101	return 0;	34,173,557✔
102	}
103
104	static int32_t tfile_from_json(const cJSON json, STFile file) {	1,482,984✔
105	const cJSON *item;
106
107	/* did.level */
108	item = cJSON_GetObjectItem(json, "did.level");	1,482,984✔
109	if (cJSON_IsNumber(item)) {	1,490,552✔
110	file->did.level = item->valuedouble;	1,490,555✔
111	} else {
112	return TSDB_CODE_FILE_CORRUPTED;	×
113	}
114
115	/* did.id */
116	item = cJSON_GetObjectItem(json, "did.id");	1,488,590✔
117	if (cJSON_IsNumber(item)) {	1,490,491✔
118	file->did.id = item->valuedouble;	1,490,550✔
119	} else {
120	return TSDB_CODE_FILE_CORRUPTED;	×
121	}
122
123	/* lcn */
124	item = cJSON_GetObjectItem(json, "lcn");	1,488,409✔
125	if (cJSON_IsNumber(item)) {	1,489,480✔
126	file->lcn = item->valuedouble;	1,490,553✔
127	} else {
128	// return TSDB_CODE_FILE_CORRUPTED;
129	}
130
131	/* fid */
132	item = cJSON_GetObjectItem(json, "fid");	1,487,515✔
133	if (cJSON_IsNumber(item)) {	1,487,775✔
134	file->fid = item->valuedouble;	1,489,332✔
135	} else {
136	return TSDB_CODE_FILE_CORRUPTED;	×
137	}
138
139	/* mid - migration id */
140	item = cJSON_GetObjectItem(json, "mid");	1,486,499✔
141	if (cJSON_IsNumber(item)) {	1,489,103✔
142	file->mid = item->valuedouble;	1,488,633✔
143	} else {
144	file->mid = 0;	×
145	}
146
147	/* cid */
148	item = cJSON_GetObjectItem(json, "cid");	1,486,744✔
149	if (cJSON_IsNumber(item)) {	1,488,742✔
150	file->cid = item->valuedouble;	1,489,868✔
151	} else {
152	return TSDB_CODE_FILE_CORRUPTED;	×
153	}
154
155	/* size */
156	item = cJSON_GetObjectItem(json, "size");	1,487,441✔
157	if (cJSON_IsNumber(item)) {	1,489,789✔
158	file->size = item->valuedouble;	1,490,586✔
159	} else {
160	return TSDB_CODE_FILE_CORRUPTED;	×
161	}
162
163	/* minVer */
164	file->minVer = VERSION_MAX;	1,486,206✔
165	item = cJSON_GetObjectItem(json, "minVer");	1,486,668✔
166	if (cJSON_IsNumber(item)) {	1,487,059✔
167	file->minVer = item->valuedouble;	1,489,625✔
168	}
169
170	/* maxVer */
171	file->maxVer = VERSION_MIN;	1,483,434✔
172	item = cJSON_GetObjectItem(json, "maxVer");	1,486,131✔
173	if (cJSON_IsNumber(item)) {	1,486,871✔
174	file->maxVer = item->valuedouble;	1,489,227✔
175	}
176	return 0;	1,486,611✔
177	}
178
179	static int32_t head_to_json(const STFile file, cJSON json) { return tfile_to_json(file, json); }	1,850,458✔
180	static int32_t data_to_json(const STFile file, cJSON json) { return tfile_to_json(file, json); }	1,850,458✔
181	static int32_t sma_to_json(const STFile file, cJSON json) { return tfile_to_json(file, json); }	1,850,458✔
182	static int32_t tomb_to_json(const STFile file, cJSON json) { return tfile_to_json(file, json); }	1,528,199✔
183	static int32_t stt_to_json(const STFile file, cJSON json) {	27,075,379✔
184	TAOS_CHECK_RETURN(tfile_to_json(file, json));	27,075,379✔
185
186	/* lvl */
187	if (cJSON_AddNumberToObject(json, "level", file->stt->level) == NULL) {	27,094,460✔
188	return TSDB_CODE_OUT_OF_MEMORY;	×
189	}
190
191	return 0;	27,093,572✔
192	}
193
194	static int32_t head_from_json(const cJSON json, STFile file) { return tfile_from_json(json, file); }	219,564✔
195	static int32_t data_from_json(const cJSON json, STFile file) { return tfile_from_json(json, file); }	219,783✔
196	static int32_t sma_from_json(const cJSON json, STFile file) { return tfile_from_json(json, file); }	219,783✔
197	static int32_t tomb_from_json(const cJSON json, STFile file) { return tfile_from_json(json, file); }	1,611✔
198	static int32_t stt_from_json(const cJSON json, STFile file) {	824,795✔
199	TAOS_CHECK_RETURN(tfile_from_json(json, file));	824,795✔
200
201	const cJSON *item;
202
203	/* lvl */
204	item = cJSON_GetObjectItem(json, "level");	825,849✔
205	if (cJSON_IsNumber(item)) {	826,693✔
206	file->stt->level = item->valuedouble;	827,397✔
207	} else {
208	return TSDB_CODE_FILE_CORRUPTED;	×
209	}
210
211	return 0;	826,067✔
212	}
213
214	int32_t tsdbTFileToJson(const STFile file, cJSON json) {	34,173,402✔
215	if (file->type == TSDB_FTYPE_STT) {	34,173,402✔
216	return g_tfile_info[file->type].to_json(file, json);	27,096,128✔
217	} else {
218	cJSON *item = cJSON_AddObjectToObject(json, g_tfile_info[file->type].suffix);	7,077,959✔
219	if (item == NULL) {	7,079,573✔
220	return TSDB_CODE_OUT_OF_MEMORY;	×
221	}
222	return g_tfile_info[file->type].to_json(file, item);	7,079,573✔
223	}
224	}
225
226	int32_t tsdbJsonToTFile(const cJSON json, tsdb_ftype_t ftype, STFile f) {	4,200,517✔
227	f[0] = (STFile){.type = ftype};	4,200,517✔
228
229	if (ftype == TSDB_FTYPE_STT) {	4,209,460✔
230	TAOS_CHECK_RETURN(g_tfile_info[ftype].from_json(json, f));	827,640✔
231	} else {
232	const cJSON *item = cJSON_GetObjectItem(json, g_tfile_info[ftype].suffix);	3,381,820✔
233	if (cJSON_IsObject(item)) {	3,383,306✔
234	TAOS_CHECK_RETURN(g_tfile_info[ftype].from_json(item, f));	660,868✔
235	} else {
236	return TSDB_CODE_NOT_FOUND;	2,719,825✔
237	}
238	}
239
240	return 0;	1,486,386✔
241	}
242
243	int32_t tsdbTFileObjInit(STsdb pTsdb, const STFile f, STFileObj **fobj) {	76,878,758✔
244	fobj[0] = taosMemoryMalloc(sizeof(*fobj[0]));	76,878,758✔
245	if (!fobj[0]) {	76,867,546✔
246	return terrno;	×
247	}
248
249	(void)taosThreadMutexInit(&fobj[0]->mutex, NULL);	76,868,403✔
250	fobj[0]->f[0] = f[0];	76,879,940✔
251	fobj[0]->state = TSDB_FSTATE_LIVE;	76,880,924✔
252	fobj[0]->ref = 1;	76,887,635✔
253	tsdbTFileName(pTsdb, f, fobj[0]->fname);	76,887,473✔
254	// fobj[0]->nlevel = tfsGetLevel(pTsdb->pVnode->pTfs);
255	fobj[0]->nlevel = vnodeNodeId(pTsdb->pVnode);	76,874,328✔
256	return 0;	76,884,700✔
257	}
258
259	int32_t tsdbTFileObjRef(STFileObj *fobj) {	523,063,072✔
260	int32_t nRef;
261	(void)taosThreadMutexLock(&fobj->mutex);	523,063,072✔
262
263	if (fobj->ref <= 0 \|\| fobj->state != TSDB_FSTATE_LIVE) {	524,142,005✔
264	tsdbError("file %s, fobj:%p ref:%d", fobj->fname, fobj, fobj->ref);	243,609✔
265	(void)taosThreadMutexUnlock(&fobj->mutex);	243,609✔
266	return TSDB_CODE_FAILED;	×
267	}
268
269	nRef = ++fobj->ref;	523,933,956✔
270	(void)taosThreadMutexUnlock(&fobj->mutex);	523,983,303✔
271	tsdbTrace("ref file %s, fobj:%p ref:%d", fobj->fname, fobj, nRef);	524,000,941✔
272	return 0;	523,736,531✔
273	}
274
275	int32_t tsdbTFileObjUnref(STFileObj *fobj) {	597,382,930✔
276	(void)taosThreadMutexLock(&fobj->mutex);	597,382,930✔
277	int32_t nRef = --fobj->ref;	597,662,187✔
278	(void)taosThreadMutexUnlock(&fobj->mutex);	597,678,511✔
279
280	if (nRef < 0) {	597,558,792✔
281	tsdbError("file %s, fobj:%p ref:%d", fobj->fname, fobj, nRef);	×
282	return TSDB_CODE_FAILED;	×
283	}
284
285	tsdbTrace("unref file %s, fobj:%p ref:%d", fobj->fname, fobj, nRef);	597,558,792✔
286	if (nRef == 0) {	597,609,616✔
287	if (fobj->state == TSDB_FSTATE_DEAD) {	73,074,635✔
288	tsdbRemoveFile(fobj->fname);	156,642✔
289	}
290	(void)taosThreadMutexDestroy(&fobj->mutex);	73,078,128✔
291	taosMemoryFree(fobj);	73,073,621✔
292	}
293
294	return 0;	597,582,861✔
295	}
296
297	static void tsdbTFileObjRemoveLC(STFileObj *fobj) {	3,817,256✔
298	if (fobj->f->type != TSDB_FTYPE_DATA \|\| fobj->f->lcn < 1) {	3,817,256✔
299	tsdbRemoveFile(fobj->fname);	3,817,256✔
300	return;	3,817,256✔
301	}
302
303	#ifdef USE_SHARED_STORAGE
304	// remove local last chunk file
305	char lc_path[TSDB_FILENAME_LEN];	×
306	tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN);	×
307
308	char *dot = strrchr(lc_path, '.');	×
309	if (!dot) {	×
310	tsdbError("unexpected path: %s", lc_path);	×
311	return;	×
312	}
313	snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", fobj->f->lcn);	×
314
315	tsdbRemoveFile(lc_path);	×
316	#endif
317	}
318
319	int32_t tsdbTFileObjRemove(STFileObj *fobj) {	3,911,370✔
320	(void)taosThreadMutexLock(&fobj->mutex);	3,911,370✔
321	if (fobj->state != TSDB_FSTATE_LIVE \|\| fobj->ref <= 0) {	3,911,370✔
322	tsdbError("file %s, fobj:%p ref:%d", fobj->fname, fobj, fobj->ref);	×
323	(void)taosThreadMutexUnlock(&fobj->mutex);	×
324	return TSDB_CODE_FAILED;	×
325	}
326	fobj->state = TSDB_FSTATE_DEAD;	3,911,370✔
327	int32_t nRef = --fobj->ref;	3,911,370✔
328	(void)taosThreadMutexUnlock(&fobj->mutex);	3,911,370✔
329	tsdbTrace("remove unref file %s, fobj:%p ref:%d", fobj->fname, fobj, nRef);	3,911,370✔
330	if (nRef == 0) {	3,911,370✔
331	tsdbTFileObjRemoveLC(fobj);	3,754,728✔
332	(void)taosThreadMutexDestroy(&fobj->mutex);	3,754,728✔
333	taosMemoryFree(fobj);	3,754,728✔
334	}
335	return 0;	3,911,370✔
336	}
337
338	int32_t tsdbTFileObjRemoveUpdateLC(STFileObj *fobj) {	62,528✔
339	(void)taosThreadMutexLock(&fobj->mutex);	62,528✔
340
341	if (fobj->state != TSDB_FSTATE_LIVE \|\| fobj->ref <= 0) {	62,528✔
342	(void)taosThreadMutexUnlock(&fobj->mutex);	×
343	tsdbError("file %s, fobj:%p ref:%d", fobj->fname, fobj, fobj->ref);	×
344	return TSDB_CODE_FAILED;	×
345	}
346
347	fobj->state = TSDB_FSTATE_DEAD;	62,528✔
348	int32_t nRef = --fobj->ref;	62,528✔
349	(void)taosThreadMutexUnlock(&fobj->mutex);	62,528✔
350	tsdbTrace("remove unref file %s, fobj:%p ref:%d", fobj->fname, fobj, nRef);	62,528✔
351	if (nRef == 0) {	62,528✔
352	tsdbTFileObjRemoveLC(fobj);	62,528✔
353	(void)taosThreadMutexDestroy(&fobj->mutex);	62,528✔
354	taosMemoryFree(fobj);	62,528✔
355	}
356	return 0;	62,528✔
357	}
358
359	void tsdbTFileName(STsdb pTsdb, const STFile f, char fname[]) {	101,867,469✔
360	SVnode *pVnode = pTsdb->pVnode;	101,867,469✔
361	STfs *pTfs = TSDB_TFS(pTsdb->pVnode);	101,981,308✔
362
363	if (pTfs) {	101,989,640✔
364	if (pVnode->mounted) {	101,989,640✔
365	// NOTE: the case 'if (f->mid != 0)' is not handled at present, this may be
366	// needed in the future.
367	snprintf(fname, //	13,540✔
368	TSDB_FILENAME_LEN, //
369	"%s%svnode%svnode%d%s%s%sv%df%dver%" PRId64 ".%s", //
370	tfsGetDiskPath(pTfs, f->did), //
371	TD_DIRSEP, //
372	TD_DIRSEP, //
373	TSDB_VID(pVnode), //	6,770✔
374	TD_DIRSEP, //
375	pTsdb->name, //	6,770✔
376	TD_DIRSEP, //
377	TSDB_VID(pVnode), //	6,770✔
378	f->fid, //	6,770✔
379	f->cid, //	6,770✔
380	g_tfile_info[f->type].suffix);	6,770✔
381	} else {
382	if (f->mid == 0) {	101,933,434✔
383	snprintf(fname, //	101,975,015✔
384	TSDB_FILENAME_LEN, //
385	"%s%s%s%sv%df%dver%" PRId64 ".%s", //
386	tfsGetDiskPath(pTfs, f->did), //
387	TD_DIRSEP, //
388	pTsdb->path, //
389	TD_DIRSEP, //
390	TD_VID(pVnode), //
391	f->fid, //	101,955,121✔
392	f->cid, //	101,947,608✔
393	g_tfile_info[f->type].suffix);	101,944,606✔
394	} else {
395	snprintf(fname, //	×
396	TSDB_FILENAME_LEN, //
397	"%s%s%s%sv%df%dver%" PRId64 ".m%d.%s", //
398	tfsGetDiskPath(pTfs, f->did), //
399	TD_DIRSEP, //
400	pTsdb->path, //
401	TD_DIRSEP, //
402	TD_VID(pVnode), //
403	f->fid, //	×
404	f->cid, //	×
405	f->mid, //	×
406	g_tfile_info[f->type].suffix);	×
407	}
408	}
409	} else {
410	if (f->mid == 0) {	×
411	snprintf(fname, //	×
412	TSDB_FILENAME_LEN, //
413	"%s%sv%df%dver%" PRId64 ".%s", //
414	pTsdb->path, //
415	TD_DIRSEP, //
416	TD_VID(pVnode), //
417	f->fid, //	×
418	f->cid, //	×
419	g_tfile_info[f->type].suffix);	×
420	} else {
421	snprintf(fname, //	×
422	TSDB_FILENAME_LEN, //
423	"%s%sv%df%dver%" PRId64 ".m%d.%s", //
424	pTsdb->path, //
425	TD_DIRSEP, //
426	TD_VID(pVnode), //
427	f->fid, //	×
428	f->cid, //	×
429	f->mid, //	×
430	g_tfile_info[f->type].suffix);	×
431	}
432	}
433	}	101,971,303✔
434
435	void tsdbTFileLastChunkName(STsdb pTsdb, const STFile f, char fname[]) {	1,688✔
436	SVnode *pVnode = pTsdb->pVnode;	1,688✔
437	STfs *pTfs = TSDB_TFS(pTsdb->pVnode);	1,688✔
438
439	// NOTE: the case 'if (pVnode->mounted)' is not handled at present, this may be needed
440	// in the future.
441
442	if (f->mid == 0) {	1,688✔
443	if (pTfs) {	1,688✔
444	snprintf(fname, //	1,688✔
445	TSDB_FILENAME_LEN, //
446	"%s%s%s%sv%df%dver%" PRId64 ".%d.%s", //
447	tfsGetDiskPath(pTfs, f->did), //
448	TD_DIRSEP, //
449	pTsdb->path, //
450	TD_DIRSEP, //
451	TD_VID(pVnode), //
452	f->fid, //	1,688✔
453	f->cid, //	1,688✔
454	f->lcn, //	1,688✔
455	g_tfile_info[f->type].suffix);	1,688✔
456	} else {
457	snprintf(fname, //	×
458	TSDB_FILENAME_LEN, //
459	"%s%sv%df%dver%" PRId64 ".%d.%s", //
460	pTsdb->path, //
461	TD_DIRSEP, //
462	TD_VID(pVnode), //
463	f->fid, //	×
464	f->cid, //	×
465	f->lcn, //	×
466	g_tfile_info[f->type].suffix);	×
467	}
468	} else {
469	if (pTfs) {	×
470	snprintf(fname, //	×
471	TSDB_FILENAME_LEN, //
472	"%s%s%s%sv%df%dver%" PRId64 ".m%d.%d.%s", //
473	tfsGetDiskPath(pTfs, f->did), //
474	TD_DIRSEP, //
475	pTsdb->path, //
476	TD_DIRSEP, //
477	TD_VID(pVnode), //
478	f->fid, //	×
479	f->cid, //	×
480	f->mid, //	×
481	f->lcn, //	×
482	g_tfile_info[f->type].suffix);	×
483	} else {
484	snprintf(fname, //	×
485	TSDB_FILENAME_LEN, //
486	"%s%sv%df%dver%" PRId64 ".m%d.%d.%s", //
487	pTsdb->path, //
488	TD_DIRSEP, //
489	TD_VID(pVnode), //
490	f->fid, //	×
491	f->cid, //	×
492	f->mid, //	×
493	f->lcn, //	×
494	g_tfile_info[f->type].suffix);	×
495	}
496	}
497	}	1,688✔
498
499	bool tsdbIsSameTFile(const STFile f1, const STFile f2) {	11,715,282✔
500	if (f1->type != f2->type) return false;	11,715,282✔
501	if (f1->did.level != f2->did.level) return false;	11,715,934✔
502	if (f1->did.id != f2->did.id) return false;	11,633,595✔
503	if (f1->fid != f2->fid) return false;	11,620,174✔
504	if (f1->cid != f2->cid) return false;	11,620,907✔
505	if (f1->lcn != f2->lcn) return false;	11,131,323✔
506	if (f1->mid != f2->mid) return false;	11,129,635✔
507	return true;	11,129,635✔
508	}
509
510	bool tsdbIsTFileChanged(const STFile f1, const STFile f2) {	11,129,635✔
511	if (f1->size != f2->size) return true;	11,129,635✔
512	// if (f1->type == TSDB_FTYPE_STT && f1->stt->nseg != f2->stt->nseg) return true;
513	return false;	10,477,378✔
514	}
515
516	int32_t tsdbTFileObjCmpr(const STFileObj fobj1, const STFileObj fobj2) {	4,685,165✔
517	if (fobj1[0]->f->cid < fobj2[0]->f->cid) {	4,685,165✔
518	return -1;	47,211✔
519	} else if (fobj1[0]->f->cid > fobj2[0]->f->cid) {	4,637,302✔
520	return 1;	1,247,950✔
521	} else {
522	return 0;	3,389,352✔
523	}
524	}
525
526	const char *tsdbFTypeLabel(tsdb_ftype_t ftype) { return g_tfile_info[ftype].suffix; }	24,090,123✔

taosdata / TDengine / #5074

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous