#4720

Committed 08 Sep 2025 08:43AM UTC coverage: 58.139% (-0.6%) from 58.762%

Build # #4720

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

Merge pull request #32881 from taosdata/enh/add-new-windows-ci

fix(ci): update workflow reference to use new Windows CI YAML

Run Details

133181 of 292179 branches covered (45.58%)

Branch coverage included in aggregate %.

201691 of 283811 relevant lines covered (71.07%)

5442780.71 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

59.68

/source/libs/tdb/src/db/tdbDb.c

/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "tdbInt.h"

int32_t tdbOpen(const char *dbname, int32_t szPage, int32_t pages, TDB **ppDb, int8_t rollback,
                int32_t encryptAlgorithm, char *encryptKey) {
  TDB *pDb;
  int  dsize;
  int  zsize;
  int  tsize;
  u8  *pPtr;
  int  ret;

  *ppDb = NULL;

  dsize = strlen(dbname);
  zsize = sizeof(*pDb) + dsize * 2 + strlen(TDB_JOURNAL_NAME) + 3;

  pPtr = (uint8_t *)tdbOsCalloc(1, zsize);
  if (pPtr == NULL) {
    return terrno;
  }

  pDb = (TDB *)pPtr;
  pPtr += sizeof(*pDb);
  // pDb->rootDir
  pDb->dbName = pPtr;
  memcpy(pDb->dbName, dbname, dsize);
  pDb->dbName[dsize] = '\0';
  pPtr = pPtr + dsize + 1;
  // pDb->jfname
  pDb->jnName = pPtr;
  memcpy(pDb->jnName, dbname, dsize);
  pDb->jnName[dsize] = '/';
  memcpy(pDb->jnName + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME));
  pDb->jnName[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0';

  pDb->jfd = -1;

  pDb->encryptAlgorithm = encryptAlgorithm;
  if (encryptKey != NULL) {
    tstrncpy(pDb->encryptKey, encryptKey, ENCRYPT_KEY_LEN + 1);
  }

  ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache));
  if (ret < 0) {
    return ret;
  }

  pDb->nPgrHash = 8;
  tsize = sizeof(SPager *) * pDb->nPgrHash;
  pDb->pgrHash = tdbOsMalloc(tsize);
  if (pDb->pgrHash == NULL) {
    return terrno;
  }
  memset(pDb->pgrHash, 0, tsize);

  ret = taosMulModeMkDir(dbname, 0755, false);
  if (ret < 0) {
    return terrno;
  }

#ifdef USE_MAINDB
  // open main db
  ret = tdbTbOpen(TDB_MAINDB_NAME, -1, sizeof(SBtInfo), NULL, pDb, &pDb->pMainDb, rollback);
  if (ret < 0) {
    return ret;
  }

  ret = tdbTbOpen(TDB_FREEDB_NAME, sizeof(SPgno), 0, NULL, pDb, &pDb->pFreeDb, rollback);
  if (ret < 0) {
    return ret;
  }

  // if the underlying data structure is a btree, it will do re-balance when there are too many pages,
  // the re-balance process include allocate and deallocate of free pages, this is ok for other tables,
  // but not for the free page table, because it is for free pages management, allocate/deallocate of
  // free pages during re-balance result in data corruption. so convert it to a linked list based stack
  // to avoid re-balance.
  int tdbTbBtreeToStack(TTB *pTb);
  ret = tdbTbBtreeToStack(pDb->pFreeDb);
  if (ret < 0) {
    return ret;
  }
#endif

  *ppDb = pDb;
  return 0;
}

void tdbClose(TDB *pDb) {
  SPager *pPager;

  if (pDb) {
#ifdef USE_MAINDB
    if (pDb->pMainDb) tdbTbClose(pDb->pMainDb);
    if (pDb->pFreeDb) tdbTbClose(pDb->pFreeDb);
#endif

    for (pPager = pDb->pgrList; pPager; pPager = pDb->pgrList) {
      pDb->pgrList = pPager->pNext;
      tdbPagerClose(pPager);
    }

    tdbPCacheClose(pDb->pCache);
    tdbOsFree(pDb->pgrHash);
    tdbOsFree(pDb);
  }

  return;
}

int32_t tdbAlter(TDB *pDb, int pages) { return tdbPCacheAlter(pDb->pCache, pages); }

int32_t tdbBegin(TDB *pDb, TXN **ppTxn, void *(*xMalloc)(void *, size_t), void (*xFree)(void *, void *), void *xArg,
                 int flags) {
  SPager *pPager;
  int     ret;
  int64_t txnId = ++pDb->txnId;
  if (txnId == INT64_MAX) {
    pDb->txnId = 0;
  }

  TXN *pTxn = tdbOsCalloc(1, sizeof(*pTxn));
  if (!pTxn) {
    return terrno;
  }

  ret = tdbTxnOpen(pTxn, txnId, xMalloc, xFree, xArg, flags);
  if (ret < 0) {
    tdbOsFree(pTxn);
    return ret;
  }

  for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
    ret = tdbPagerBegin(pPager, pTxn);
    if (ret < 0) {
      tdbError("failed to begin pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
               pTxn->txnId);
      tdbTxnClose(pTxn);
      return ret;
    }
  }

  *ppTxn = pTxn;

  return 0;
}

int32_t tdbCommit(TDB *pDb, TXN *pTxn) {
  SPager *pPager;
  int     ret;

  for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
    ret = tdbPagerCommit(pPager, pTxn);
    if (ret < 0) {
      tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
               pTxn->txnId);
      return ret;
    }
  }

  return 0;
}

int32_t tdbPostCommit(TDB *pDb, TXN *pTxn) {
  SPager *pPager;
  int     ret;

  for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
    ret = tdbPagerPostCommit(pPager, pTxn);
    if (ret < 0) {
      tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
               pTxn->txnId);
      return ret;
    }
  }

  tdbTxnClose(pTxn);

  return 0;
}

int32_t tdbPrepareAsyncCommit(TDB *pDb, TXN *pTxn) {
  SPager *pPager;
  int     ret;

  for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
    ret = tdbPagerPrepareAsyncCommit(pPager, pTxn);
    if (ret < 0) {
      tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
               pTxn->txnId);
      return ret;
    }
  }

  return 0;
}

void tdbAbort(TDB *pDb, TXN *pTxn) {
  SPager *pPager;
  int     ret;

  for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {
    ret = tdbPagerAbort(pPager, pTxn);
    if (ret < 0) {
      tdbError("failed to abort pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,
               pTxn->txnId);
    }
  }

  tdbTxnClose(pTxn);

  return;
}

SPager *tdbEnvGetPager(TDB *pDb, const char *fname) {
  u32      hash;
  SPager **ppPager;

  hash = tdbCstringHash(fname);
  ppPager = &pDb->pgrHash[hash % pDb->nPgrHash];
  tdbTrace("tdbttl getPager1: pager:%p, index:%d, name:%s", *ppPager, hash % pDb->nPgrHash, fname);
  for (; *ppPager && (strcmp(fname, (*ppPager)->dbFileName) != 0); ppPager = &((*ppPager)->pHashNext)) {
  }
  tdbTrace("tdbttl getPager2: pager:%p, index:%d, name:%s", *ppPager, hash % pDb->nPgrHash, fname);
  return *ppPager;
}

void tdbEnvAddPager(TDB *pDb, SPager *pPager) {
  u32      hash;
  SPager **ppPager;

  // rehash if neccessary
  if (pDb->nPager + 1 > pDb->nPgrHash) {
    // TODO
  }

  // add to list
  pPager->pNext = pDb->pgrList;
  pDb->pgrList = pPager;

  // add to hash
  hash = tdbCstringHash(pPager->dbFileName);
  ppPager = &pDb->pgrHash[hash % pDb->nPgrHash];
  tdbTrace("tdbttl addPager1: pager:%p, index:%d, name:%s", *ppPager, hash % pDb->nPgrHash, pPager->dbFileName);
  pPager->pHashNext = *ppPager;
  *ppPager = pPager;

  tdbTrace("tdbttl addPager2: pager:%p, index:%d, name:%s", *ppPager, hash % pDb->nPgrHash, pPager->dbFileName);

  // increase the counter
  pDb->nPager++;
}

void tdbEnvRemovePager(TDB *pDb, SPager *pPager) {
  u32      hash;
  SPager **ppPager;

  // remove from the list
  for (ppPager = &pDb->pgrList; *ppPager && (*ppPager != pPager); ppPager = &((*ppPager)->pNext)) {
  }
  if (*ppPager != pPager) {
    tdbError("tdb/db: invalid pPager: %p, *ppPager: %p", pPager, *ppPager);
    return;
  }
  *ppPager = pPager->pNext;

  // remove from hash
  hash = tdbCstringHash(pPager->dbFileName);
  ppPager = &pDb->pgrHash[hash % pDb->nPgrHash];
  for (; *ppPager && *ppPager != pPager; ppPager = &((*ppPager)->pHashNext)) {
  }
  if (*ppPager != pPager) {
    tdbError("tdb/db: invalid pPager: %p, *ppPager: %p", pPager, *ppPager);
    return;
  }
  *ppPager = pPager->pNext;

  // decrease the counter
  pDb->nPager--;

  // rehash if necessary
  if (pDb->nPgrHash > 8 && pDb->nPager < pDb->nPgrHash / 2) {
    // TODO
  }
}

1	/*
2	* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3	*
4	* This program is free software: you can use, redistribute, and/or modify
5	* it under the terms of the GNU Affero General Public License, version 3
6	* or later ("AGPL"), as published by the Free Software Foundation.
7	*
8	* This program is distributed in the hope that it will be useful, but WITHOUT
9	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10	* FITNESS FOR A PARTICULAR PURPOSE.
11	*
12	* You should have received a copy of the GNU Affero General Public License
13	* along with this program. If not, see <http://www.gnu.org/licenses/>.
14	*/
15
16	#include "tdbInt.h"
17
18	int32_t tdbOpen(const char dbname, int32_t szPage, int32_t pages, TDB *ppDb, int8_t rollback,	24,764✔
19	int32_t encryptAlgorithm, char *encryptKey) {
20	TDB *pDb;
21	int dsize;
22	int zsize;
23	int tsize;
24	u8 *pPtr;
25	int ret;
26
27	*ppDb = NULL;	24,764✔
28
29	dsize = strlen(dbname);	24,764✔
30	zsize = sizeof(pDb) + dsize 2 + strlen(TDB_JOURNAL_NAME) + 3;	24,764✔
31
32	pPtr = (uint8_t *)tdbOsCalloc(1, zsize);	24,764!
33	if (pPtr == NULL) {	24,831!
34	return terrno;	×
35	}
36
37	pDb = (TDB *)pPtr;	24,831✔
38	pPtr += sizeof(*pDb);	24,831✔
39	// pDb->rootDir
40	pDb->dbName = pPtr;	24,831✔
41	memcpy(pDb->dbName, dbname, dsize);	24,831✔
42	pDb->dbName[dsize] = '\0';	24,831✔
43	pPtr = pPtr + dsize + 1;	24,831✔
44	// pDb->jfname
45	pDb->jnName = pPtr;	24,831✔
46	memcpy(pDb->jnName, dbname, dsize);	24,831✔
47	pDb->jnName[dsize] = '/';	24,831✔
48	memcpy(pDb->jnName + dsize + 1, TDB_JOURNAL_NAME, strlen(TDB_JOURNAL_NAME));	24,831✔
49	pDb->jnName[dsize + 1 + strlen(TDB_JOURNAL_NAME)] = '\0';	24,831✔
50
51	pDb->jfd = -1;	24,831✔
52
53	pDb->encryptAlgorithm = encryptAlgorithm;	24,831✔
54	if (encryptKey != NULL) {	24,831✔
55	tstrncpy(pDb->encryptKey, encryptKey, ENCRYPT_KEY_LEN + 1);	12,407✔
56	}
57
58	ret = tdbPCacheOpen(szPage, pages, &(pDb->pCache));	24,831✔
59	if (ret < 0) {	24,843!
60	return ret;	×
61	}
62
63	pDb->nPgrHash = 8;	24,843✔
64	tsize = sizeof(SPager ) pDb->nPgrHash;	24,843✔
65	pDb->pgrHash = tdbOsMalloc(tsize);	24,843!
66	if (pDb->pgrHash == NULL) {	24,843!
67	return terrno;	×
68	}
69	memset(pDb->pgrHash, 0, tsize);	24,843✔
70
71	ret = taosMulModeMkDir(dbname, 0755, false);	24,843✔
72	if (ret < 0) {	24,843!
73	return terrno;	×
74	}
75
76	#ifdef USE_MAINDB
77	// open main db
78	ret = tdbTbOpen(TDB_MAINDB_NAME, -1, sizeof(SBtInfo), NULL, pDb, &pDb->pMainDb, rollback);	24,843✔
79	if (ret < 0) {	24,839!
80	return ret;	×
81	}
82
83	ret = tdbTbOpen(TDB_FREEDB_NAME, sizeof(SPgno), 0, NULL, pDb, &pDb->pFreeDb, rollback);	24,839✔
84	if (ret < 0) {	24,846!
85	return ret;	×
86	}
87
88	// if the underlying data structure is a btree, it will do re-balance when there are too many pages,
89	// the re-balance process include allocate and deallocate of free pages, this is ok for other tables,
90	// but not for the free page table, because it is for free pages management, allocate/deallocate of
91	// free pages during re-balance result in data corruption. so convert it to a linked list based stack
92	// to avoid re-balance.
93	int tdbTbBtreeToStack(TTB *pTb);
94	ret = tdbTbBtreeToStack(pDb->pFreeDb);	24,846✔
95	if (ret < 0) {	24,846!
96	return ret;	×
97	}
98	#endif
99
100	*ppDb = pDb;	24,846✔
101	return 0;	24,846✔
102	}
103
104	void tdbClose(TDB *pDb) {	24,849✔
105	SPager *pPager;
106
107	if (pDb) {	24,849!
108	#ifdef USE_MAINDB
109	if (pDb->pMainDb) tdbTbClose(pDb->pMainDb);	24,849!
110	if (pDb->pFreeDb) tdbTbClose(pDb->pFreeDb);	24,849!
111	#endif
112
113	for (pPager = pDb->pgrList; pPager; pPager = pDb->pgrList) {	49,699✔
114	pDb->pgrList = pPager->pNext;	24,850✔
115	tdbPagerClose(pPager);	24,850✔
116	}
117
118	tdbPCacheClose(pDb->pCache);	24,849✔
119	tdbOsFree(pDb->pgrHash);	24,849!
120	tdbOsFree(pDb);	24,849!
121	}
122
123	return;	24,849✔
124	}
125
126	int32_t tdbAlter(TDB *pDb, int pages) { return tdbPCacheAlter(pDb->pCache, pages); }	6✔
127
128	int32_t tdbBegin(TDB pDb, TXN ppTxn, void (xMalloc)(void , size_t), void (xFree)(void , void ), void xArg,	249,583✔
129	int flags) {
130	SPager *pPager;
131	int ret;
132	int64_t txnId = ++pDb->txnId;	249,583✔
133	if (txnId == INT64_MAX) {	249,583!
134	pDb->txnId = 0;	×
135	}
136
137	TXN pTxn = tdbOsCalloc(1, sizeof(pTxn));	249,583!
138	if (!pTxn) {	249,598!
139	return terrno;	×
140	}
141
142	ret = tdbTxnOpen(pTxn, txnId, xMalloc, xFree, xArg, flags);	249,598✔
143	if (ret < 0) {	249,582✔
144	tdbOsFree(pTxn);	2!
145	return ret;	×
146	}
147
148	for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {	499,096✔
149	ret = tdbPagerBegin(pPager, pTxn);	249,563✔
150	if (ret < 0) {	249,510!
151	tdbError("failed to begin pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,	×
152	pTxn->txnId);
153	tdbTxnClose(pTxn);	×
154	return ret;	×
155	}
156	}
157
158	*ppTxn = pTxn;	249,533✔
159
160	return 0;	249,533✔
161	}
162
163	int32_t tdbCommit(TDB pDb, TXN pTxn) {	271,178✔
164	SPager *pPager;
165	int ret;
166
167	for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {	542,486✔
168	ret = tdbPagerCommit(pPager, pTxn);	271,211✔
169	if (ret < 0) {	271,284!
170	tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,	×
171	pTxn->txnId);
172	return ret;	×
173	}
174	}
175
176	return 0;	271,275✔
177	}
178
179	int32_t tdbPostCommit(TDB pDb, TXN pTxn) {	236,361✔
180	SPager *pPager;
181	int ret;
182
183	for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {	472,738✔
184	ret = tdbPagerPostCommit(pPager, pTxn);	236,363✔
185	if (ret < 0) {	236,338!
186	tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,	×
187	pTxn->txnId);
188	return ret;	×
189	}
190	}
191
192	tdbTxnClose(pTxn);	236,375✔
193
194	return 0;	236,345✔
195	}
196
197	int32_t tdbPrepareAsyncCommit(TDB pDb, TXN pTxn) {	×
198	SPager *pPager;
199	int ret;
200
201	for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {	×
202	ret = tdbPagerPrepareAsyncCommit(pPager, pTxn);	×
203	if (ret < 0) {	×
204	tdbError("failed to commit pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,	×
205	pTxn->txnId);
206	return ret;	×
207	}
208	}
209
210	return 0;	×
211	}
212
213	void tdbAbort(TDB pDb, TXN pTxn) {	13,230✔
214	SPager *pPager;
215	int ret;
216
217	for (pPager = pDb->pgrList; pPager; pPager = pPager->pNext) {	26,464✔
218	ret = tdbPagerAbort(pPager, pTxn);	13,229✔
219	if (ret < 0) {	13,234!
220	tdbError("failed to abort pager since %s. dbName:%s, txnId:%" PRId64, tstrerror(terrno), pDb->dbName,	×
221	pTxn->txnId);
222	}
223	}
224
225	tdbTxnClose(pTxn);	13,235✔
226
227	return;	13,234✔
228	}
229
230	SPager tdbEnvGetPager(TDB pDb, const char *fname) {	248,254✔
231	u32 hash;
232	SPager **ppPager;
233
234	hash = tdbCstringHash(fname);	248,254✔
235	ppPager = &pDb->pgrHash[hash % pDb->nPgrHash];	248,324✔
236	tdbTrace("tdbttl getPager1: pager:%p, index:%d, name:%s", *ppPager, hash % pDb->nPgrHash, fname);	248,324✔
237	for (; ppPager && (strcmp(fname, (ppPager)->dbFileName) != 0); ppPager = &((*ppPager)->pHashNext)) {	248,331!
238	}
239	tdbTrace("tdbttl getPager2: pager:%p, index:%d, name:%s", *ppPager, hash % pDb->nPgrHash, fname);	248,331✔
240	return *ppPager;	248,338✔
241	}
242
243	void tdbEnvAddPager(TDB pDb, SPager pPager) {	24,822✔
244	u32 hash;
245	SPager **ppPager;
246
247	// rehash if neccessary
248	if (pDb->nPager + 1 > pDb->nPgrHash) {	24,822✔
249	// TODO
250	}
251
252	// add to list
253	pPager->pNext = pDb->pgrList;	24,822✔
254	pDb->pgrList = pPager;	24,822✔
255
256	// add to hash
257	hash = tdbCstringHash(pPager->dbFileName);	24,822✔
258	ppPager = &pDb->pgrHash[hash % pDb->nPgrHash];	24,850✔
259	tdbTrace("tdbttl addPager1: pager:%p, index:%d, name:%s", *ppPager, hash % pDb->nPgrHash, pPager->dbFileName);	24,850✔
260	pPager->pHashNext = *ppPager;	24,850✔
261	*ppPager = pPager;	24,850✔
262
263	tdbTrace("tdbttl addPager2: pager:%p, index:%d, name:%s", *ppPager, hash % pDb->nPgrHash, pPager->dbFileName);	24,850✔
264
265	// increase the counter
266	pDb->nPager++;	24,850✔
267	}	24,850✔
268
269	void tdbEnvRemovePager(TDB pDb, SPager pPager) {	×
270	u32 hash;
271	SPager **ppPager;
272
273	// remove from the list
274	for (ppPager = &pDb->pgrList; ppPager && (ppPager != pPager); ppPager = &((*ppPager)->pNext)) {	×
275	}
276	if (*ppPager != pPager) {	×
277	tdbError("tdb/db: invalid pPager: %p, ppPager: %p", pPager, ppPager);	×
278	return;	×
279	}
280	*ppPager = pPager->pNext;	×
281
282	// remove from hash
283	hash = tdbCstringHash(pPager->dbFileName);	×
284	ppPager = &pDb->pgrHash[hash % pDb->nPgrHash];	×
285	for (; ppPager && ppPager != pPager; ppPager = &((*ppPager)->pHashNext)) {	×
286	}
287	if (*ppPager != pPager) {	×
288	tdbError("tdb/db: invalid pPager: %p, ppPager: %p", pPager, ppPager);	×
289	return;	×
290	}
291	*ppPager = pPager->pNext;	×
292
293	// decrease the counter
294	pDb->nPager--;	×
295
296	// rehash if necessary
297	if (pDb->nPgrHash > 8 && pDb->nPager < pDb->nPgrHash / 2) {	×
298	// TODO
299	}
300	}

taosdata / TDengine / #4720

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous