#4113

Committed 17 May 2025 06:43AM UTC coverage: 62.054% (-0.8%) from 62.857%

Build # #4113

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

Merge pull request #31115 from taosdata/merge/mainto3.0

merge: from main to 3.0 branch

Run Details

154737 of 318088 branches covered (48.65%)

Branch coverage included in aggregate %.

175 of 225 new or added lines in 20 files covered. (77.78%)

5853 existing lines in 216 files now uncovered.

239453 of 317147 relevant lines covered (75.5%)

15121865.73 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

58.03

/source/dnode/vnode/src/tsdb/tsdbFile2.c

/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "tsdbFile2.h"
#include "tcs.h"
#include "vnd.h"

// to_json
static int32_t head_to_json(const STFile *file, cJSON *json);
static int32_t data_to_json(const STFile *file, cJSON *json);
static int32_t sma_to_json(const STFile *file, cJSON *json);
static int32_t tomb_to_json(const STFile *file, cJSON *json);
static int32_t stt_to_json(const STFile *file, cJSON *json);

// from_json
static int32_t head_from_json(const cJSON *json, STFile *file);
static int32_t data_from_json(const cJSON *json, STFile *file);
static int32_t sma_from_json(const cJSON *json, STFile *file);
static int32_t tomb_from_json(const cJSON *json, STFile *file);
static int32_t stt_from_json(const cJSON *json, STFile *file);

static const struct {
  const char *suffix;
  int32_t (*to_json)(const STFile *file, cJSON *json);
  int32_t (*from_json)(const cJSON *json, STFile *file);
} g_tfile_info[] = {
    [TSDB_FTYPE_HEAD] = {"head", head_to_json, head_from_json},
    [TSDB_FTYPE_DATA] = {"data", data_to_json, data_from_json},
    [TSDB_FTYPE_SMA] = {"sma", sma_to_json, sma_from_json},
    [TSDB_FTYPE_TOMB] = {"tomb", tomb_to_json, tomb_from_json},
    [TSDB_FTYPE_STT] = {"stt", stt_to_json, stt_from_json},
};

void tsdbRemoveFile(const char *fname) {
  int32_t code = taosRemoveFile(fname);
  if (code) {
    tsdbError("failed to remove file:%s, code:%d, error:%s", fname, code, tstrerror(code));
  } else {
    tsdbInfo("file:%s is removed", fname);
  }
}

static int32_t tfile_to_json(const STFile *file, cJSON *json) {
  /* did.level */
  if (cJSON_AddNumberToObject(json, "did.level", file->did.level) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  /* did.id */
  if (cJSON_AddNumberToObject(json, "did.id", file->did.id) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  /* lcn - last chunk number */
  if (cJSON_AddNumberToObject(json, "lcn", file->lcn) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  /* fid */
  if (cJSON_AddNumberToObject(json, "fid", file->fid) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  /* cid */
  if (cJSON_AddNumberToObject(json, "cid", file->cid) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  /* size */
  if (cJSON_AddNumberToObject(json, "size", file->size) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  if (file->minVer <= file->maxVer) {
    /* minVer */
    if (cJSON_AddNumberToObject(json, "minVer", file->minVer) == NULL) {
      return TSDB_CODE_OUT_OF_MEMORY;
    }

    /* maxVer */
    if (cJSON_AddNumberToObject(json, "maxVer", file->maxVer) == NULL) {
      return TSDB_CODE_OUT_OF_MEMORY;
    }
  }
  return 0;
}

static int32_t tfile_from_json(const cJSON *json, STFile *file) {
  const cJSON *item;

  /* did.level */
  item = cJSON_GetObjectItem(json, "did.level");
  if (cJSON_IsNumber(item)) {
    file->did.level = item->valuedouble;
  } else {
    return TSDB_CODE_FILE_CORRUPTED;
  }

  /* did.id */
  item = cJSON_GetObjectItem(json, "did.id");
  if (cJSON_IsNumber(item)) {
    file->did.id = item->valuedouble;
  } else {
    return TSDB_CODE_FILE_CORRUPTED;
  }

  /* lcn */
  item = cJSON_GetObjectItem(json, "lcn");
  if (cJSON_IsNumber(item)) {
    file->lcn = item->valuedouble;
  } else {
    // return TSDB_CODE_FILE_CORRUPTED;
  }

  /* fid */
  item = cJSON_GetObjectItem(json, "fid");
  if (cJSON_IsNumber(item)) {
    file->fid = item->valuedouble;
  } else {
    return TSDB_CODE_FILE_CORRUPTED;
  }

  /* cid */
  item = cJSON_GetObjectItem(json, "cid");
  if (cJSON_IsNumber(item)) {
    file->cid = item->valuedouble;
  } else {
    return TSDB_CODE_FILE_CORRUPTED;
  }

  /* size */
  item = cJSON_GetObjectItem(json, "size");
  if (cJSON_IsNumber(item)) {
    file->size = item->valuedouble;
  } else {
    return TSDB_CODE_FILE_CORRUPTED;
  }

  /* minVer */
  file->minVer = VERSION_MAX;
  item = cJSON_GetObjectItem(json, "minVer");
  if (cJSON_IsNumber(item)) {
    file->minVer = item->valuedouble;
  }

  /* maxVer */
  file->maxVer = VERSION_MIN;
  item = cJSON_GetObjectItem(json, "maxVer");
  if (cJSON_IsNumber(item)) {
    file->maxVer = item->valuedouble;
  }
  return 0;
}

static int32_t head_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); }
static int32_t data_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); }
static int32_t sma_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); }
static int32_t tomb_to_json(const STFile *file, cJSON *json) { return tfile_to_json(file, json); }
static int32_t stt_to_json(const STFile *file, cJSON *json) {
  TAOS_CHECK_RETURN(tfile_to_json(file, json));

  /* lvl */
  if (cJSON_AddNumberToObject(json, "level", file->stt->level) == NULL) {
    return TSDB_CODE_OUT_OF_MEMORY;
  }

  return 0;
}

static int32_t head_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); }
static int32_t data_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); }
static int32_t sma_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); }
static int32_t tomb_from_json(const cJSON *json, STFile *file) { return tfile_from_json(json, file); }
static int32_t stt_from_json(const cJSON *json, STFile *file) {
  TAOS_CHECK_RETURN(tfile_from_json(json, file));

  const cJSON *item;

  /* lvl */
  item = cJSON_GetObjectItem(json, "level");
  if (cJSON_IsNumber(item)) {
    file->stt->level = item->valuedouble;
  } else {
    return TSDB_CODE_FILE_CORRUPTED;
  }

  return 0;
}

int32_t tsdbTFileToJson(const STFile *file, cJSON *json) {
  if (file->type == TSDB_FTYPE_STT) {
    return g_tfile_info[file->type].to_json(file, json);
  } else {
    cJSON *item = cJSON_AddObjectToObject(json, g_tfile_info[file->type].suffix);
    if (item == NULL) {
      return TSDB_CODE_OUT_OF_MEMORY;
    }
    return g_tfile_info[file->type].to_json(file, item);
  }
}

int32_t tsdbJsonToTFile(const cJSON *json, tsdb_ftype_t ftype, STFile *f) {
  f[0] = (STFile){.type = ftype};

  if (ftype == TSDB_FTYPE_STT) {
    TAOS_CHECK_RETURN(g_tfile_info[ftype].from_json(json, f));
  } else {
    const cJSON *item = cJSON_GetObjectItem(json, g_tfile_info[ftype].suffix);
    if (cJSON_IsObject(item)) {
      TAOS_CHECK_RETURN(g_tfile_info[ftype].from_json(item, f));
    } else {
      return TSDB_CODE_NOT_FOUND;
    }
  }

  return 0;
}

int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) {
  fobj[0] = taosMemoryMalloc(sizeof(*fobj[0]));
  if (!fobj[0]) {
    return terrno;
  }

  (void)taosThreadMutexInit(&fobj[0]->mutex, NULL);
  fobj[0]->f[0] = f[0];
  fobj[0]->state = TSDB_FSTATE_LIVE;
  fobj[0]->ref = 1;
  tsdbTFileName(pTsdb, f, fobj[0]->fname);
  // fobj[0]->nlevel = tfsGetLevel(pTsdb->pVnode->pTfs);
  fobj[0]->nlevel = vnodeNodeId(pTsdb->pVnode);
  return 0;
}

int32_t tsdbTFileObjRef(STFileObj *fobj) {
  int32_t nRef;
  (void)taosThreadMutexLock(&fobj->mutex);

  if (fobj->ref <= 0 || fobj->state != TSDB_FSTATE_LIVE) {
    tsdbError("file %s, fobj:%p ref:%d", fobj->fname, fobj, fobj->ref);
    (void)taosThreadMutexUnlock(&fobj->mutex);
    return TSDB_CODE_FAILED;
  }

  nRef = ++fobj->ref;
  (void)taosThreadMutexUnlock(&fobj->mutex);
  tsdbTrace("ref file %s, fobj:%p ref:%d", fobj->fname, fobj, nRef);
  return 0;
}

int32_t tsdbTFileObjUnref(STFileObj *fobj) {
  (void)taosThreadMutexLock(&fobj->mutex);
  int32_t nRef = --fobj->ref;
  (void)taosThreadMutexUnlock(&fobj->mutex);

  if (nRef < 0) {
    tsdbError("file %s, fobj:%p ref:%d", fobj->fname, fobj, nRef);
    return TSDB_CODE_FAILED;
  }

  tsdbTrace("unref file %s, fobj:%p ref:%d", fobj->fname, fobj, nRef);
  if (nRef == 0) {
    if (fobj->state == TSDB_FSTATE_DEAD) {
      tsdbRemoveFile(fobj->fname);
    }
    taosMemoryFree(fobj);
  }

  return 0;
}

static void tsdbTFileObjRemoveLC(STFileObj *fobj, bool remove_all) {
  if (fobj->f->type != TSDB_FTYPE_DATA || fobj->f->lcn < 1) {
    tsdbRemoveFile(fobj->fname);
    return;
  }
#ifdef USE_S3
  if (!remove_all) {
    // remove local last chunk file
    char lc_path[TSDB_FILENAME_LEN];
    tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN);

    char *dot = strrchr(lc_path, '.');
    if (!dot) {
      tsdbError("unexpected path: %s", lc_path);
      return;
    }
    snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", fobj->f->lcn);

    tsdbRemoveFile(lc_path);

  } else {
    // delete by data file prefix
    char lc_path[TSDB_FILENAME_LEN];
    tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN);

    char   *object_name = taosDirEntryBaseName(lc_path);
    int32_t node_id = fobj->nlevel;
    char    object_name_prefix[TSDB_FILENAME_LEN];
    snprintf(object_name_prefix, TSDB_FQDN_LEN, "%d/%s", node_id, object_name);

    char *dot = strrchr(object_name_prefix, '.');
    if (!dot) {
      tsdbError("unexpected path: %s", object_name_prefix);
      return;
    }
    *(dot + 1) = 0;

    tcsDeleteObjectsByPrefix(object_name_prefix);

    // remove local last chunk file
    dot = strrchr(lc_path, '.');
    if (!dot) {
      tsdbError("unexpected path: %s", lc_path);
      return;
    }
    snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", fobj->f->lcn);

    tsdbRemoveFile(lc_path);
  }
#endif
}

int32_t tsdbTFileObjRemove(STFileObj *fobj) {
  (void)taosThreadMutexLock(&fobj->mutex);
  if (fobj->state != TSDB_FSTATE_LIVE || fobj->ref <= 0) {
    tsdbError("file %s, fobj:%p ref:%d", fobj->fname, fobj, fobj->ref);
    (void)taosThreadMutexUnlock(&fobj->mutex);
    return TSDB_CODE_FAILED;
  }
  fobj->state = TSDB_FSTATE_DEAD;
  int32_t nRef = --fobj->ref;
  (void)taosThreadMutexUnlock(&fobj->mutex);
  tsdbTrace("remove unref file %s, fobj:%p ref:%d", fobj->fname, fobj, nRef);
  if (nRef == 0) {
    tsdbTFileObjRemoveLC(fobj, true);
    taosMemoryFree(fobj);
  }
  return 0;
}

int32_t tsdbTFileObjRemoveUpdateLC(STFileObj *fobj) {
  (void)taosThreadMutexLock(&fobj->mutex);

  if (fobj->state != TSDB_FSTATE_LIVE || fobj->ref <= 0) {
    (void)taosThreadMutexUnlock(&fobj->mutex);
    tsdbError("file %s, fobj:%p ref:%d", fobj->fname, fobj, fobj->ref);
    return TSDB_CODE_FAILED;
  }

  fobj->state = TSDB_FSTATE_DEAD;
  int32_t nRef = --fobj->ref;
  (void)taosThreadMutexUnlock(&fobj->mutex);
  tsdbTrace("remove unref file %s, fobj:%p ref:%d", fobj->fname, fobj, nRef);
  if (nRef == 0) {
    tsdbTFileObjRemoveLC(fobj, false);
    taosMemoryFree(fobj);
  }
  return 0;
}

void tsdbTFileName(STsdb *pTsdb, const STFile *f, char fname[]) {
  SVnode *pVnode = pTsdb->pVnode;
  STfs   *pTfs = pVnode->pTfs;

  if (pTfs) {
    snprintf(fname,                              //
             TSDB_FILENAME_LEN,                  //
             "%s%s%s%sv%df%dver%" PRId64 ".%s",  //
             tfsGetDiskPath(pTfs, f->did),       //
             TD_DIRSEP,                          //
             pTsdb->path,                        //
             TD_DIRSEP,                          //
             TD_VID(pVnode),                     //
             f->fid,                             //
             f->cid,                             //
             g_tfile_info[f->type].suffix);
  } else {
    snprintf(fname,                          //
             TSDB_FILENAME_LEN,              //
             "%s%sv%df%dver%" PRId64 ".%s",  //
             pTsdb->path,                    //
             TD_DIRSEP,                      //
             TD_VID(pVnode),                 //
             f->fid,                         //
             f->cid,                         //
             g_tfile_info[f->type].suffix);
  }
}

void tsdbTFileLastChunkName(STsdb *pTsdb, const STFile *f, char fname[]) {
  SVnode *pVnode = pTsdb->pVnode;
  STfs   *pTfs = pVnode->pTfs;

  if (pTfs) {
    snprintf(fname,                                 //
             TSDB_FILENAME_LEN,                     //
             "%s%s%s%sv%df%dver%" PRId64 ".%d.%s",  //
             tfsGetDiskPath(pTfs, f->did),          //
             TD_DIRSEP,                             //
             pTsdb->path,                           //
             TD_DIRSEP,                             //
             TD_VID(pVnode),                        //
             f->fid,                                //
             f->cid,                                //
             f->lcn,                                //
             g_tfile_info[f->type].suffix);
  } else {
    snprintf(fname,                             //
             TSDB_FILENAME_LEN,                 //
             "%s%sv%df%dver%" PRId64 ".%d.%s",  //
             pTsdb->path,                       //
             TD_DIRSEP,                         //
             TD_VID(pVnode),                    //
             f->fid,                            //
             f->cid,                            //
             f->lcn,                            //
             g_tfile_info[f->type].suffix);
  }
}

bool tsdbIsSameTFile(const STFile *f1, const STFile *f2) {
  if (f1->type != f2->type) return false;
  if (f1->did.level != f2->did.level) return false;
  if (f1->did.id != f2->did.id) return false;
  if (f1->fid != f2->fid) return false;
  if (f1->cid != f2->cid) return false;
  if (f1->lcn != f2->lcn) return false;
  return true;
}

bool tsdbIsTFileChanged(const STFile *f1, const STFile *f2) {
  if (f1->size != f2->size) return true;
  // if (f1->type == TSDB_FTYPE_STT && f1->stt->nseg != f2->stt->nseg) return true;
  return false;
}

int32_t tsdbTFileObjCmpr(const STFileObj **fobj1, const STFileObj **fobj2) {
  if (fobj1[0]->f->cid < fobj2[0]->f->cid) {
    return -1;
  } else if (fobj1[0]->f->cid > fobj2[0]->f->cid) {
    return 1;
  } else {
    return 0;
  }
}

const char *tsdbFTypeLabel(tsdb_ftype_t ftype) { return g_tfile_info[ftype].suffix; }

1	/*
2	* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3	*
4	* This program is free software: you can use, redistribute, and/or modify
5	* it under the terms of the GNU Affero General Public License, version 3
6	* or later ("AGPL"), as published by the Free Software Foundation.
7	*
8	* This program is distributed in the hope that it will be useful, but WITHOUT
9	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10	* FITNESS FOR A PARTICULAR PURPOSE.
11	*
12	* You should have received a copy of the GNU Affero General Public License
13	* along with this program. If not, see <http://www.gnu.org/licenses/>.
14	*/
15
16	#include "tsdbFile2.h"
17	#include "tcs.h"
18	#include "vnd.h"
19
20	// to_json
21	static int32_t head_to_json(const STFile file, cJSON json);
22	static int32_t data_to_json(const STFile file, cJSON json);
23	static int32_t sma_to_json(const STFile file, cJSON json);
24	static int32_t tomb_to_json(const STFile file, cJSON json);
25	static int32_t stt_to_json(const STFile file, cJSON json);
26
27	// from_json
28	static int32_t head_from_json(const cJSON json, STFile file);
29	static int32_t data_from_json(const cJSON json, STFile file);
30	static int32_t sma_from_json(const cJSON json, STFile file);
31	static int32_t tomb_from_json(const cJSON json, STFile file);
32	static int32_t stt_from_json(const cJSON json, STFile file);
33
34	static const struct {
35	const char *suffix;
36	int32_t (to_json)(const STFile file, cJSON *json);
37	int32_t (from_json)(const cJSON json, STFile *file);
38	} g_tfile_info[] = {
39	[TSDB_FTYPE_HEAD] = {"head", head_to_json, head_from_json},
40	[TSDB_FTYPE_DATA] = {"data", data_to_json, data_from_json},
41	[TSDB_FTYPE_SMA] = {"sma", sma_to_json, sma_from_json},
42	[TSDB_FTYPE_TOMB] = {"tomb", tomb_to_json, tomb_from_json},
43	[TSDB_FTYPE_STT] = {"stt", stt_to_json, stt_from_json},
44	};
45
46	void tsdbRemoveFile(const char *fname) {	11,480✔
47	int32_t code = taosRemoveFile(fname);	11,480✔
48	if (code) {	11,480!
49	tsdbError("failed to remove file:%s, code:%d, error:%s", fname, code, tstrerror(code));	×
50	} else {
51	tsdbInfo("file:%s is removed", fname);	11,480!
52	}
53	}	11,480✔
54
55	static int32_t tfile_to_json(const STFile file, cJSON json) {	412,726✔
56	/* did.level */
57	if (cJSON_AddNumberToObject(json, "did.level", file->did.level) == NULL) {	412,726!
58	return TSDB_CODE_OUT_OF_MEMORY;	×
59	}
60
61	/* did.id */
62	if (cJSON_AddNumberToObject(json, "did.id", file->did.id) == NULL) {	412,766!
63	return TSDB_CODE_OUT_OF_MEMORY;	×
64	}
65
66	/* lcn - last chunk number */
67	if (cJSON_AddNumberToObject(json, "lcn", file->lcn) == NULL) {	412,761!
68	return TSDB_CODE_OUT_OF_MEMORY;	×
69	}
70
71	/* fid */
72	if (cJSON_AddNumberToObject(json, "fid", file->fid) == NULL) {	412,770!
73	return TSDB_CODE_OUT_OF_MEMORY;	×
74	}
75
76	/* cid */
77	if (cJSON_AddNumberToObject(json, "cid", file->cid) == NULL) {	412,763!
78	return TSDB_CODE_OUT_OF_MEMORY;	×
79	}
80
81	/* size */
82	if (cJSON_AddNumberToObject(json, "size", file->size) == NULL) {	412,735!
83	return TSDB_CODE_OUT_OF_MEMORY;	×
84	}
85
86	if (file->minVer <= file->maxVer) {	412,754!
87	/* minVer */
88	if (cJSON_AddNumberToObject(json, "minVer", file->minVer) == NULL) {	412,754!
89	return TSDB_CODE_OUT_OF_MEMORY;	×
90	}
91
92	/* maxVer */
93	if (cJSON_AddNumberToObject(json, "maxVer", file->maxVer) == NULL) {	412,752!
94	return TSDB_CODE_OUT_OF_MEMORY;	×
95	}
96	}
97	return 0;	412,694✔
98	}
99
100	static int32_t tfile_from_json(const cJSON json, STFile file) {	2,699✔
101	const cJSON *item;
102
103	/* did.level */
104	item = cJSON_GetObjectItem(json, "did.level");	2,699✔
105	if (cJSON_IsNumber(item)) {	2,698!
106	file->did.level = item->valuedouble;	2,699✔
107	} else {
108	return TSDB_CODE_FILE_CORRUPTED;	×
109	}
110
111	/* did.id */
112	item = cJSON_GetObjectItem(json, "did.id");	2,699✔
113	if (cJSON_IsNumber(item)) {	2,698!
114	file->did.id = item->valuedouble;	2,699✔
115	} else {
116	return TSDB_CODE_FILE_CORRUPTED;	×
117	}
118
119	/* lcn */
120	item = cJSON_GetObjectItem(json, "lcn");	2,699✔
121	if (cJSON_IsNumber(item)) {	2,698!
122	file->lcn = item->valuedouble;	2,700✔
123	} else {
124	// return TSDB_CODE_FILE_CORRUPTED;
125	}
126
127	/* fid */
128	item = cJSON_GetObjectItem(json, "fid");	2,700✔
129	if (cJSON_IsNumber(item)) {	2,697!
130	file->fid = item->valuedouble;	2,698✔
131	} else {
132	return TSDB_CODE_FILE_CORRUPTED;	×
133	}
134
135	/* cid */
136	item = cJSON_GetObjectItem(json, "cid");	2,698✔
137	if (cJSON_IsNumber(item)) {	2,697!
138	file->cid = item->valuedouble;	2,701✔
139	} else {
140	return TSDB_CODE_FILE_CORRUPTED;	×
141	}
142
143	/* size */
144	item = cJSON_GetObjectItem(json, "size");	2,701✔
145	if (cJSON_IsNumber(item)) {	2,700!
146	file->size = item->valuedouble;	2,700✔
147	} else {
148	return TSDB_CODE_FILE_CORRUPTED;	×
149	}
150
151	/* minVer */
152	file->minVer = VERSION_MAX;	2,700✔
153	item = cJSON_GetObjectItem(json, "minVer");	2,700✔
154	if (cJSON_IsNumber(item)) {	2,701✔
155	file->minVer = item->valuedouble;	2,700✔
156	}
157
158	/* maxVer */
159	file->maxVer = VERSION_MIN;	2,701✔
160	item = cJSON_GetObjectItem(json, "maxVer");	2,701✔
161	if (cJSON_IsNumber(item)) {	2,703!
162	file->maxVer = item->valuedouble;	2,702✔
163	}
164	return 0;	2,702✔
165	}
166
167	static int32_t head_to_json(const STFile file, cJSON json) { return tfile_to_json(file, json); }	9,969✔
168	static int32_t data_to_json(const STFile file, cJSON json) { return tfile_to_json(file, json); }	9,969✔
169	static int32_t sma_to_json(const STFile file, cJSON json) { return tfile_to_json(file, json); }	9,969✔
170	static int32_t tomb_to_json(const STFile file, cJSON json) { return tfile_to_json(file, json); }	7,905✔
171	static int32_t stt_to_json(const STFile file, cJSON json) {	374,915✔
172	TAOS_CHECK_RETURN(tfile_to_json(file, json));	374,915!
173
174	/* lvl */
175	if (cJSON_AddNumberToObject(json, "level", file->stt->level) == NULL) {	374,866!
176	return TSDB_CODE_OUT_OF_MEMORY;	×
177	}
178
179	return 0;	374,943✔
180	}
181
182	static int32_t head_from_json(const cJSON json, STFile file) { return tfile_from_json(json, file); }	272✔
183	static int32_t data_from_json(const cJSON json, STFile file) { return tfile_from_json(json, file); }	272✔
184	static int32_t sma_from_json(const cJSON json, STFile file) { return tfile_from_json(json, file); }	272✔
185	static int32_t tomb_from_json(const cJSON json, STFile file) { return tfile_from_json(json, file); }	20✔
186	static int32_t stt_from_json(const cJSON json, STFile file) {	1,865✔
187	TAOS_CHECK_RETURN(tfile_from_json(json, file));	1,865!
188
189	const cJSON *item;
190
191	/* lvl */
192	item = cJSON_GetObjectItem(json, "level");	1,866✔
193	if (cJSON_IsNumber(item)) {	1,865!
194	file->stt->level = item->valuedouble;	1,867✔
195	} else {
196	return TSDB_CODE_FILE_CORRUPTED;	×
197	}
198
199	return 0;	1,867✔
200	}
201
202	int32_t tsdbTFileToJson(const STFile file, cJSON json) {	412,727✔
203	if (file->type == TSDB_FTYPE_STT) {	412,727✔
204	return g_tfile_info[file->type].to_json(file, json);	374,915✔
205	} else {
206	cJSON *item = cJSON_AddObjectToObject(json, g_tfile_info[file->type].suffix);	37,812✔
207	if (item == NULL) {	37,812!
208	return TSDB_CODE_OUT_OF_MEMORY;	×
209	}
210	return g_tfile_info[file->type].to_json(file, item);	37,812✔
211	}
212	}
213
214	int32_t tsdbJsonToTFile(const cJSON json, tsdb_ftype_t ftype, STFile f) {	9,107✔
215	f[0] = (STFile){.type = ftype};	9,107✔
216
217	if (ftype == TSDB_FTYPE_STT) {	9,107✔
218	TAOS_CHECK_RETURN(g_tfile_info[ftype].from_json(json, f));	1,865!
219	} else {
220	const cJSON *item = cJSON_GetObjectItem(json, g_tfile_info[ftype].suffix);	7,242✔
221	if (cJSON_IsObject(item)) {	7,326✔
222	TAOS_CHECK_RETURN(g_tfile_info[ftype].from_json(item, f));	835!
223	} else {
224	return TSDB_CODE_NOT_FOUND;	6,494✔
225	}
226	}
227
228	return 0;	2,703✔
229	}
230
231	int32_t tsdbTFileObjInit(STsdb pTsdb, const STFile f, STFileObj **fobj) {	816,519✔
232	fobj[0] = taosMemoryMalloc(sizeof(*fobj[0]));	816,519!
233	if (!fobj[0]) {	816,814!
234	return terrno;	×
235	}
236
237	(void)taosThreadMutexInit(&fobj[0]->mutex, NULL);	816,814✔
238	fobj[0]->f[0] = f[0];	816,689✔
239	fobj[0]->state = TSDB_FSTATE_LIVE;	816,689✔
240	fobj[0]->ref = 1;	816,689✔
241	tsdbTFileName(pTsdb, f, fobj[0]->fname);	816,689✔
242	// fobj[0]->nlevel = tfsGetLevel(pTsdb->pVnode->pTfs);
243	fobj[0]->nlevel = vnodeNodeId(pTsdb->pVnode);	816,887✔
244	return 0;	816,865✔
245	}
246
247	int32_t tsdbTFileObjRef(STFileObj *fobj) {	10,517,834✔
248	int32_t nRef;
249	(void)taosThreadMutexLock(&fobj->mutex);	10,517,834✔
250
251	if (fobj->ref <= 0 \|\| fobj->state != TSDB_FSTATE_LIVE) {	10,529,258!
252	tsdbError("file %s, fobj:%p ref:%d", fobj->fname, fobj, fobj->ref);	×
253	(void)taosThreadMutexUnlock(&fobj->mutex);	×
254	return TSDB_CODE_FAILED;	×
255	}
256
257	nRef = ++fobj->ref;	10,529,556✔
258	(void)taosThreadMutexUnlock(&fobj->mutex);	10,529,556✔
259	tsdbTrace("ref file %s, fobj:%p ref:%d", fobj->fname, fobj, nRef);	10,530,591✔
260	return 0;	10,529,763✔
261	}
262
263	int32_t tsdbTFileObjUnref(STFileObj *fobj) {	11,337,956✔
264	(void)taosThreadMutexLock(&fobj->mutex);	11,337,956✔
265	int32_t nRef = --fobj->ref;	11,341,252✔
266	(void)taosThreadMutexUnlock(&fobj->mutex);	11,341,252✔
267
268	if (nRef < 0) {	11,342,103!
269	tsdbError("file %s, fobj:%p ref:%d", fobj->fname, fobj, nRef);	×
270	return TSDB_CODE_FAILED;	×
271	}
272
273	tsdbTrace("unref file %s, fobj:%p ref:%d", fobj->fname, fobj, nRef);	11,342,103✔
274	if (nRef == 0) {	11,342,103✔
275	if (fobj->state == TSDB_FSTATE_DEAD) {	805,927✔
276	tsdbRemoveFile(fobj->fname);	705✔
277	}
278	taosMemoryFree(fobj);	805,927!
279	}
280
281	return 0;	11,341,363✔
282	}
283
284	static void tsdbTFileObjRemoveLC(STFileObj *fobj, bool remove_all) {	10,775✔
285	if (fobj->f->type != TSDB_FTYPE_DATA \|\| fobj->f->lcn < 1) {	10,775!
286	tsdbRemoveFile(fobj->fname);	10,775✔
287	return;	10,775✔
288	}
289	#ifdef USE_S3
290	if (!remove_all) {	×
291	// remove local last chunk file
292	char lc_path[TSDB_FILENAME_LEN];
293	tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN);	×
294
295	char *dot = strrchr(lc_path, '.');	×
296	if (!dot) {	×
297	tsdbError("unexpected path: %s", lc_path);	×
298	return;	×
299	}
300	snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", fobj->f->lcn);	×
301
302	tsdbRemoveFile(lc_path);	×
303
304	} else {
305	// delete by data file prefix
306	char lc_path[TSDB_FILENAME_LEN];
307	tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN);	×
308
309	char *object_name = taosDirEntryBaseName(lc_path);	×
310	int32_t node_id = fobj->nlevel;	×
311	char object_name_prefix[TSDB_FILENAME_LEN];
312	snprintf(object_name_prefix, TSDB_FQDN_LEN, "%d/%s", node_id, object_name);	×
313
314	char *dot = strrchr(object_name_prefix, '.');	×
315	if (!dot) {	×
316	tsdbError("unexpected path: %s", object_name_prefix);	×
317	return;	×
318	}
319	*(dot + 1) = 0;	×
320
321	tcsDeleteObjectsByPrefix(object_name_prefix);	×
322
323	// remove local last chunk file
324	dot = strrchr(lc_path, '.');	×
325	if (!dot) {	×
326	tsdbError("unexpected path: %s", lc_path);	×
327	return;	×
328	}
329	snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", fobj->f->lcn);	×
330
331	tsdbRemoveFile(lc_path);	×
332	}
333	#endif
334	}
335
336	int32_t tsdbTFileObjRemove(STFileObj *fobj) {	11,480✔
337	(void)taosThreadMutexLock(&fobj->mutex);	11,480✔
338	if (fobj->state != TSDB_FSTATE_LIVE \|\| fobj->ref <= 0) {	11,480!
339	tsdbError("file %s, fobj:%p ref:%d", fobj->fname, fobj, fobj->ref);	×
340	(void)taosThreadMutexUnlock(&fobj->mutex);	×
341	return TSDB_CODE_FAILED;	×
342	}
343	fobj->state = TSDB_FSTATE_DEAD;	11,480✔
344	int32_t nRef = --fobj->ref;	11,480✔
345	(void)taosThreadMutexUnlock(&fobj->mutex);	11,480✔
346	tsdbTrace("remove unref file %s, fobj:%p ref:%d", fobj->fname, fobj, nRef);	11,480✔
347	if (nRef == 0) {	11,480✔
348	tsdbTFileObjRemoveLC(fobj, true);	10,775✔
349	taosMemoryFree(fobj);	10,775!
350	}
351	return 0;	11,480✔
352	}
353
UNCOV 354	int32_t tsdbTFileObjRemoveUpdateLC(STFileObj *fobj) {	×
UNCOV 355	(void)taosThreadMutexLock(&fobj->mutex);	×
356
UNCOV 357	if (fobj->state != TSDB_FSTATE_LIVE \|\| fobj->ref <= 0) {	×
358	(void)taosThreadMutexUnlock(&fobj->mutex);	×
359	tsdbError("file %s, fobj:%p ref:%d", fobj->fname, fobj, fobj->ref);	×
360	return TSDB_CODE_FAILED;	×
361	}
362
UNCOV 363	fobj->state = TSDB_FSTATE_DEAD;	×
UNCOV 364	int32_t nRef = --fobj->ref;	×
UNCOV 365	(void)taosThreadMutexUnlock(&fobj->mutex);	×
UNCOV 366	tsdbTrace("remove unref file %s, fobj:%p ref:%d", fobj->fname, fobj, nRef);	×
UNCOV 367	if (nRef == 0) {	×
UNCOV 368	tsdbTFileObjRemoveLC(fobj, false);	×
UNCOV 369	taosMemoryFree(fobj);	×
370	}
UNCOV 371	return 0;	×
372	}
373
374	void tsdbTFileName(STsdb pTsdb, const STFile f, char fname[]) {	1,173,047✔
375	SVnode *pVnode = pTsdb->pVnode;	1,173,047✔
376	STfs *pTfs = pVnode->pTfs;	1,173,047✔
377
378	if (pTfs) {	1,173,047!
379	snprintf(fname, //	1,173,119✔
380	TSDB_FILENAME_LEN, //
381	"%s%s%s%sv%df%dver%" PRId64 ".%s", //
382	tfsGetDiskPath(pTfs, f->did), //
383	TD_DIRSEP, //
384	pTsdb->path, //
385	TD_DIRSEP, //
386	TD_VID(pVnode), //
387	f->fid, //	1,173,113✔
388	f->cid, //	1,173,113✔
389	g_tfile_info[f->type].suffix);	1,173,113✔
390	} else {
391	snprintf(fname, //	×
392	TSDB_FILENAME_LEN, //
393	"%s%sv%df%dver%" PRId64 ".%s", //
394	pTsdb->path, //
395	TD_DIRSEP, //
396	TD_VID(pVnode), //
397	f->fid, //	×
398	f->cid, //	×
399	g_tfile_info[f->type].suffix);	×
400	}
401	}	1,173,053✔
402
403	void tsdbTFileLastChunkName(STsdb pTsdb, const STFile f, char fname[]) {	×
404	SVnode *pVnode = pTsdb->pVnode;	×
405	STfs *pTfs = pVnode->pTfs;	×
406
407	if (pTfs) {	×
408	snprintf(fname, //	×
409	TSDB_FILENAME_LEN, //
410	"%s%s%s%sv%df%dver%" PRId64 ".%d.%s", //
411	tfsGetDiskPath(pTfs, f->did), //
412	TD_DIRSEP, //
413	pTsdb->path, //
414	TD_DIRSEP, //
415	TD_VID(pVnode), //
416	f->fid, //	×
417	f->cid, //	×
418	f->lcn, //	×
419	g_tfile_info[f->type].suffix);	×
420	} else {
421	snprintf(fname, //	×
422	TSDB_FILENAME_LEN, //
423	"%s%sv%df%dver%" PRId64 ".%d.%s", //
424	pTsdb->path, //
425	TD_DIRSEP, //
426	TD_VID(pVnode), //
427	f->fid, //	×
428	f->cid, //	×
429	f->lcn, //	×
430	g_tfile_info[f->type].suffix);	×
431	}
432	}	×
433
434	bool tsdbIsSameTFile(const STFile f1, const STFile f2) {	60,404✔
435	if (f1->type != f2->type) return false;	60,404!
436	if (f1->did.level != f2->did.level) return false;	60,404✔
437	if (f1->did.id != f2->did.id) return false;	60,386✔
438	if (f1->fid != f2->fid) return false;	60,353!
439	if (f1->cid != f2->cid) return false;	60,353✔
440	if (f1->lcn != f2->lcn) return false;	59,442!
441	return true;	59,442✔
442	}
443
444	bool tsdbIsTFileChanged(const STFile f1, const STFile f2) {	59,442✔
445	if (f1->size != f2->size) return true;	59,442✔
446	// if (f1->type == TSDB_FTYPE_STT && f1->stt->nseg != f2->stt->nseg) return true;
447	return false;	59,017✔
448	}
449
450	int32_t tsdbTFileObjCmpr(const STFileObj fobj1, const STFileObj fobj2) {	15,007✔
451	if (fobj1[0]->f->cid < fobj2[0]->f->cid) {	15,007✔
452	return -1;	140✔
453	} else if (fobj1[0]->f->cid > fobj2[0]->f->cid) {	14,867✔
454	return 1;	4,331✔
455	} else {
456	return 0;	10,536✔
457	}
458	}
459
460	const char *tsdbFTypeLabel(tsdb_ftype_t ftype) { return g_tfile_info[ftype].suffix; }	357,260✔

taosdata / TDengine / #4113

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous