#4473

Committed 08 Jul 2025 09:38AM UTC coverage: 62.922% (+0.7%) from 62.22%

Build # #4473

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

Merge pull request #31712 from taosdata/merge/mainto3.0

merge: from main to 3.0 branch

Run Details

158525 of 321496 branches covered (49.31%)

Branch coverage included in aggregate %.

56 of 60 new or added lines in 13 files covered. (93.33%)

1333 existing lines in 67 files now uncovered.

245526 of 320647 relevant lines covered (76.57%)

17689640.25 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

82.93

/source/libs/tdb/src/inc/tdbInt.h

/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#ifndef _TD_TDB_INTERNAL_H_
#define _TD_TDB_INTERNAL_H_

#include "tdb.h"
#include "tutil.h"

#include "tdef.h"
#include "tlog.h"
#include "trbtree.h"

#ifdef __cplusplus
extern "C" {
#endif

typedef int8_t   i8;
typedef int16_t  i16;
typedef int32_t  i32;
typedef int64_t  i64;
typedef uint8_t  u8;
typedef uint16_t u16;
typedef uint32_t u32;
typedef uint64_t u64;

// SPgno
typedef u32 SPgno;

#include "tdbOs.h"
#include "tdbUtil.h"

// p must be u8 *
#define TDB_GET_U24(p) ((p)[0] * 65536 + *(u16 *)((p) + 1))
#define TDB_PUT_U24(p, v)       \
  do {                          \
    int tv = (v);               \
    (p)[1] = tv & 0xff;         \
    (p)[2] = (tv >> 8) & 0xff;  \
    (p)[0] = (tv >> 16) & 0xff; \
  } while (0)

// fileid
#define TDB_FILE_ID_LEN 24

// SPgid
typedef struct {
  uint8_t fileid[TDB_FILE_ID_LEN];
  SPgno   pgno;
} SPgid;

// pgsz_t
#define TDB_MIN_PGSIZE       512       // 512B
#define TDB_MAX_PGSIZE       16777216  // 16M
#define TDB_DEFAULT_PGSIZE   4096
#define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE))

// dbname
#define TDB_MAX_DBNAME_LEN 24

#define TDB_VARIANT_LEN ((int)-1)

#define TDB_JOURNAL_NAME "tdb.journal"

#define TDB_FILENAME_LEN 128

#define BTREE_MAX_DEPTH 20

#define TDB_FLAG_IS(flags, flag)     ((flags) == (flag))
#define TDB_FLAG_HAS(flags, flag)    (((flags) & (flag)) != 0)
#define TDB_FLAG_NO(flags, flag)     ((flags) & (flag) == 0)
#define TDB_FLAG_ADD(flags, flag)    ((flags) | (flag))
#define TDB_FLAG_REMOVE(flags, flag) ((flags) & (~(flag)))

typedef struct SPager  SPager;
typedef struct SPCache SPCache;
typedef struct SPage   SPage;

// transaction

#define TDB_TXN_IS_WRITE(PTXN)            ((PTXN)->flags & TDB_TXN_WRITE)
#define TDB_TXN_IS_READ(PTXN)             (!TDB_TXN_IS_WRITE(PTXN))
#define TDB_TXN_IS_READ_UNCOMMITTED(PTXN) ((PTXN)->flags & TDB_TXN_READ_UNCOMMITTED)

// tdbEnv.c ====================================
void    tdbEnvAddPager(TDB *pEnv, SPager *pPager);
void    tdbEnvRemovePager(TDB *pEnv, SPager *pPager);
SPager *tdbEnvGetPager(TDB *pEnv, const char *fname);

// tdbBtree.c ====================================
typedef struct SBTree SBTree;
typedef struct SBTC   SBTC;
typedef struct SBtInfo {
  SPgno root;
  int   nLevel;
  int   nData;
} SBtInfo;

#define TDB_CELLD_F_NIL 0x0
#define TDB_CELLD_F_KEY 0x1
#define TDB_CELLD_F_VAL 0x2

#define TDB_CELLDECODER_SET_FREE_NIL(pCellDecoder) ((pCellDecoder)->freeKV = TDB_CELLD_F_NIL)
#define TDB_CELLDECODER_CLZ_FREE_KEY(pCellDecoder) ((pCellDecoder)->freeKV &= ~TDB_CELLD_F_KEY)
#define TDB_CELLDECODER_CLZ_FREE_VAL(pCellDecoder) ((pCellDecoder)->freeKV &= ~TDB_CELLD_F_VAL)
#define TDB_CELLDECODER_SET_FREE_KEY(pCellDecoder) ((pCellDecoder)->freeKV |= TDB_CELLD_F_KEY)
#define TDB_CELLDECODER_SET_FREE_VAL(pCellDecoder) ((pCellDecoder)->freeKV |= TDB_CELLD_F_VAL)

#define TDB_CELLDECODER_FREE_KEY(pCellDecoder) ((pCellDecoder)->freeKV & TDB_CELLD_F_KEY)
#define TDB_CELLDECODER_FREE_VAL(pCellDecoder) ((pCellDecoder)->freeKV & TDB_CELLD_F_VAL)

typedef struct {
  int     kLen;
  u8     *pKey;
  int     vLen;
  u8     *pVal;
  SPgno   pgno;
  u8      freeKV;
} SCellDecoder;

struct SBTC {
  SBTree      *pBt;
  i8           iPage;
  SPage       *pPage;
  int          idx;
  int          idxStack[BTREE_MAX_DEPTH + 1];
  SPage       *pgStack[BTREE_MAX_DEPTH + 1];
  SCellDecoder coder;
  TXN         *pTxn;
  i8           freeTxn;
};

// SBTree
int  tdbBtreeOpen(int keyLen, int valLen, SPager *pFile, char const *tbname, SPgno pgno, tdb_cmpr_fn_t kcmpr, TDB *pEnv,
                  SBTree **ppBt);
void tdbBtreeClose(SBTree *pBt);
int  tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn);
int  tdbBtreeDelete(SBTree *pBt, const void *pKey, int kLen, TXN *pTxn);
// int tdbBtreeUpsert(SBTree *pBt, const void *pKey, int nKey, const void *pData, int nData, TXN *pTxn);
int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen);
int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen);
int tdbFreeOvflPage(SPgno pgno, int nSize, TXN *pTxn, SBTree *pBt);

// these 3 functions are only for free page management, they are using the b-tree as a stack.
// they should only be called on the b-tree of the free page table, never call them for other
// purpose.
int tdbBtreeToStack(SBTree *pBt);
int tdbBtreePushFreePage(SBTree *pBt, SPage *pPage, TXN *pTxn);
int tdbBtreePopFreePage(SBTree *pBt, SPgno* pgno, TXN *pTxn);

typedef struct {
  u8      flags;
  SBTree *pBt;
} SBtreeInitPageArg;

int tdbBtreeInitPage(SPage *pPage, void *arg, int init);

// SBTC
int  tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn);
void tdbBtcClose(SBTC *pBtc);
int  tdbBtcIsValid(SBTC *pBtc);
int  tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst);
int  tdbBtcMoveToFirst(SBTC *pBtc);
int  tdbBtcMoveToLast(SBTC *pBtc);
int  tdbBtcMoveToNext(SBTC *pBtc);
int  tdbBtcMoveToPrev(SBTC *pBtc);
int  tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen);
int  tdbBtreePrev(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen);
int  tdbBtcGet(SBTC *pBtc, const void **ppKey, int *kLen, const void **ppVal, int *vLen);
int  tdbBtcDelete(SBTC *pBtc);
int  tdbBtcUpsert(SBTC *pBtc, const void *pKey, int kLen, const void *pData, int nData, int insert);

// tdbPager.c ====================================

int  tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager);
void tdbPagerClose(SPager *pPager);
int  tdbPagerOpenDB(SPager *pPager, SPgno *ppgno, bool toCreate, SBTree *pBt);
int  tdbPagerWrite(SPager *pPager, SPage *pPage);
int  tdbPagerBegin(SPager *pPager, TXN *pTxn);
int  tdbPagerCommit(SPager *pPager, TXN *pTxn);
int  tdbPagerPostCommit(SPager *pPager, TXN *pTxn);
int  tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn);
int  tdbPagerAbort(SPager *pPager, TXN *pTxn);
int  tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initPage)(SPage *, void *, int), void *arg,
                       TXN *pTxn);
void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn);
int tdbPagerFetchFreePage(SPager *pPager, SPgno pgno, SPage **ppPage, TXN *pTxn);
int  tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn);
// int  tdbPagerAllocPage(SPager *pPager, SPgno *ppgno);
int tdbPagerRestoreJournals(SPager *pPager);
int tdbPagerRollback(SPager *pPager);

// tdbPCache.c ====================================
#define TDB_PCACHE_PAGE    \
  u8           isAnchor;   \
  u8           isLocal;    \
  u8           isDirty;    \
  u8           isFree;     \
  volatile i32 nRef;       \
  i32          id;         \
  SPage       *pFreeNext;  \
  SPage       *pHashNext;  \
  SPage       *pLruNext;   \
  SPage       *pLruPrev;   \
  SPage       *pDirtyNext; \
  SPager      *pPager;     \
  SPgid        pgid;

// For page ref

int    tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache);
void   tdbPCacheClose(SPCache *pCache);
int    tdbPCacheAlter(SPCache *pCache, int32_t nPage);
SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn, bool force, bool* loaded);
void   tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn);
void   tdbPCacheMarkFree(SPCache *pCache, SPage *pPage);
void   tdbPCacheInvalidatePage(SPCache *pCache, SPager *pPager, SPgno pgno);
int    tdbPCacheGetPageSize(SPCache *pCache);

// tdbPage.c ====================================
typedef u8 SCell;

// PAGE APIS implemented
typedef struct {
  int szOffset;
  int szPageHdr;
  int szFreeCell;
  // cell number
  int (*getCellNum)(SPage *);
  void (*setCellNum)(SPage *, int);
  // cell content offset
  int (*getCellBody)(SPage *);
  void (*setCellBody)(SPage *, int);
  // first free cell offset (0 means no free cells)
  int (*getCellFree)(SPage *);
  void (*setCellFree)(SPage *, int);
  // total free bytes
  int (*getFreeBytes)(SPage *);
  void (*setFreeBytes)(SPage *, int);
  // cell offset at idx
  int (*getCellOffset)(SPage *, int);
  void (*setCellOffset)(SPage *, int, int);
  // free cell info
  void (*getFreeCellInfo)(SCell *pCell, int *szCell, int *nxOffset);
  void (*setFreeCellInfo)(SCell *pCell, int szCell, int nxOffset);
} SPageMethods;

#pragma pack(push, 1)

// Page footer
typedef struct {
  u8 cksm[4];
} SPageFtr;
#pragma pack(pop)

struct SPage {
  SRBTreeNode    node;  // must be the first field for pageCmpFn to work
  tdb_spinlock_t lock;
  int            pageSize;
  u8            *pData;
  SPageMethods  *pPageMethods;
  // Fields below used by pager and am
  u8       *pPageHdr;
  u8       *pCellIdx;
  u8       *pFreeStart;
  u8       *pFreeEnd;
  SPageFtr *pPageFtr;
  int       nOverflow;
  SCell    *apOvfl[4];
  int       aiOvfl[4];
  int       kLen;  // key length of the page, -1 for unknown
  int       vLen;  // value length of the page, -1 for unknown
  int       maxLocal;
  int       minLocal;
  int (*xCellSize)(const SPage *, SCell *, int *);
  // Fields used by SPCache
  TDB_PCACHE_PAGE
};

static inline i32 tdbRefPage(SPage *pPage) {
  i32 nRef = atomic_add_fetch_32(&((pPage)->nRef), 1);
  // tdbTrace("ref page %p/%d, nRef %d", pPage, pPage->id, nRef);
  return nRef;
}

static inline i32 tdbUnrefPage(SPage *pPage) {
  i32 nRef = atomic_sub_fetch_32(&((pPage)->nRef), 1);
  // tdbTrace("unref page %p/%d, nRef %d", pPage, pPage->id, nRef);
  return nRef;
}

#define tdbGetPageRef(pPage) atomic_load_32(&((pPage)->nRef))

// For page lock
#define P_LOCK_SUCC 0
#define P_LOCK_BUSY 1
#define P_LOCK_FAIL -1

static inline int tdbTryLockPage(tdb_spinlock_t *pLock) {
  int ret = tdbSpinlockTrylock(pLock);
  if (ret == 0) {
    return P_LOCK_SUCC;
  } else if (ret == EBUSY) {
    return P_LOCK_BUSY;
  } else {
    return P_LOCK_FAIL;
  }
}

#define TDB_INIT_PAGE_LOCK(pPage)    tdbSpinlockInit(&((pPage)->lock), 0)
#define TDB_DESTROY_PAGE_LOCK(pPage) tdbSpinlockDestroy(&((pPage)->lock))
#define TDB_LOCK_PAGE(pPage)         tdbSpinlockLock(&((pPage)->lock))
#define TDB_UNLOCK_PAGE(pPage)       tdbSpinlockUnlock(&((pPage)->lock))
#define TDB_TRY_LOCK_PAGE(pPage)     tdbTryLockPage(&((pPage)->lock))

// APIs
#define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage))
#define TDB_PAGE_USABLE_SIZE(pPage) ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx)
#define TDB_PAGE_FREE_SIZE(pPage)   (*(pPage)->pPageMethods->getFreeBytes)(pPage)
#define TDB_PAGE_PGNO(pPage)        ((pPage)->pgid.pgno)
#define TDB_BYTES_CELL_TAKEN(pPage, pCell) \
  ((*(pPage)->xCellSize)(pPage, pCell, NULL) + (pPage)->pPageMethods->szOffset)
#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset)

int     tdbPageCreate(int pageSize, SPage **ppPage, void *(*xMalloc)(void *, size_t), void *arg);
void    tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg);
void    tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int *));
void    tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int *));
int     tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl);
int     tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt);
int     tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell, TXN *pTxn, SBTree *pBt);
int32_t tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl);
int     tdbPageCapacity(int pageSize, int amHdrSize);

static inline SCell *tdbPageGetCell(SPage *pPage, int idx) {
  SCell *pCell;
  int    iOvfl;
  int    lidx;

  if (idx < 0 || idx >= TDB_PAGE_TOTAL_CELLS(pPage)) {
    terrno = TSDB_CODE_INVALID_PARA;
    return NULL;
  }

  iOvfl = 0;
  for (; iOvfl < pPage->nOverflow; iOvfl++) {
    if (pPage->aiOvfl[iOvfl] == idx) {
      pCell = pPage->apOvfl[iOvfl];
      return pCell;
    } else if (pPage->aiOvfl[iOvfl] > idx) {
      break;
    }
  }

  lidx = idx - iOvfl;
  pCell = pPage->pData + pPage->pPageMethods->getCellOffset(pPage, lidx);

  return pCell;
}

#define USE_MAINDB

#ifdef USE_MAINDB
#define TDB_MAINDB_NAME "main.tdb"
#define TDB_FREEDB_NAME "_free.db"
#endif

struct STDB {
  char    *dbName;
  char    *jnName;
  int      jfd;
  SPCache *pCache;
  SPager  *pgrList;
  int      nPager;
  int      nPgrHash;
  SPager **pgrHash;
#ifdef USE_MAINDB
  TTB *pMainDb;
  TTB *pFreeDb;
#endif
  int64_t txnId;
  int32_t encryptAlgorithm;
  char    encryptKey[ENCRYPT_KEY_LEN + 1];
};

struct SPager {
  char    *dbFileName;
  char    *jFileName;
  int      pageSize;
  uint8_t  fid[TDB_FILE_ID_LEN];
  tdb_fd_t fd;
  SPCache *pCache;
  SPgno    dbFileSize;
  SPgno    dbOrigSize;
  // SPage   *pDirty;
  SRBTree rbt;
  // u8        inTran;
  TXN    *pActiveTxn;
  SPager *pNext;      // used by TDB
  SPager *pHashNext;  // used by TDB
#ifdef USE_MAINDB
  TDB *pEnv;
#endif
};

#ifdef __cplusplus
}
#endif

#endif /*_TD_TDB_INTERNAL_H_*/

1	/*
2	* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3	*
4	* This program is free software: you can use, redistribute, and/or modify
5	* it under the terms of the GNU Affero General Public License, version 3
6	* or later ("AGPL"), as published by the Free Software Foundation.
7	*
8	* This program is distributed in the hope that it will be useful, but WITHOUT
9	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10	* FITNESS FOR A PARTICULAR PURPOSE.
11	*
12	* You should have received a copy of the GNU Affero General Public License
13	* along with this program. If not, see <http://www.gnu.org/licenses/>.
14	*/
15
16	#ifndef _TD_TDB_INTERNAL_H_
17	#define _TD_TDB_INTERNAL_H_
18
19	#include "tdb.h"
20	#include "tutil.h"
21
22	#include "tdef.h"
23	#include "tlog.h"
24	#include "trbtree.h"
25
26	#ifdef __cplusplus
27	extern "C" {
28	#endif
29
30	typedef int8_t i8;
31	typedef int16_t i16;
32	typedef int32_t i32;
33	typedef int64_t i64;
34	typedef uint8_t u8;
35	typedef uint16_t u16;
36	typedef uint32_t u32;
37	typedef uint64_t u64;
38
39	// SPgno
40	typedef u32 SPgno;
41
42	#include "tdbOs.h"
43	#include "tdbUtil.h"
44
45	// p must be u8 *
46	#define TDB_GET_U24(p) ((p)[0] * 65536 + (u16 )((p) + 1))
47	#define TDB_PUT_U24(p, v) \
48	do { \
49	int tv = (v); \
50	(p)[1] = tv & 0xff; \
51	(p)[2] = (tv >> 8) & 0xff; \
52	(p)[0] = (tv >> 16) & 0xff; \
53	} while (0)
54
55	// fileid
56	#define TDB_FILE_ID_LEN 24
57
58	// SPgid
59	typedef struct {
60	uint8_t fileid[TDB_FILE_ID_LEN];
61	SPgno pgno;
62	} SPgid;
63
64	// pgsz_t
65	#define TDB_MIN_PGSIZE 512 // 512B
66	#define TDB_MAX_PGSIZE 16777216 // 16M
67	#define TDB_DEFAULT_PGSIZE 4096
68	#define TDB_IS_PGSIZE_VLD(s) (((s) >= TDB_MIN_PGSIZE) && ((s) <= TDB_MAX_PGSIZE))
69
70	// dbname
71	#define TDB_MAX_DBNAME_LEN 24
72
73	#define TDB_VARIANT_LEN ((int)-1)
74
75	#define TDB_JOURNAL_NAME "tdb.journal"
76
77	#define TDB_FILENAME_LEN 128
78
79	#define BTREE_MAX_DEPTH 20
80
81	#define TDB_FLAG_IS(flags, flag) ((flags) == (flag))
82	#define TDB_FLAG_HAS(flags, flag) (((flags) & (flag)) != 0)
83	#define TDB_FLAG_NO(flags, flag) ((flags) & (flag) == 0)
84	#define TDB_FLAG_ADD(flags, flag) ((flags) \| (flag))
85	#define TDB_FLAG_REMOVE(flags, flag) ((flags) & (~(flag)))
86
87	typedef struct SPager SPager;
88	typedef struct SPCache SPCache;
89	typedef struct SPage SPage;
90
91	// transaction
92
93	#define TDB_TXN_IS_WRITE(PTXN) ((PTXN)->flags & TDB_TXN_WRITE)
94	#define TDB_TXN_IS_READ(PTXN) (!TDB_TXN_IS_WRITE(PTXN))
95	#define TDB_TXN_IS_READ_UNCOMMITTED(PTXN) ((PTXN)->flags & TDB_TXN_READ_UNCOMMITTED)
96
97	// tdbEnv.c ====================================
98	void tdbEnvAddPager(TDB pEnv, SPager pPager);
99	void tdbEnvRemovePager(TDB pEnv, SPager pPager);
100	SPager tdbEnvGetPager(TDB pEnv, const char *fname);
101
102	// tdbBtree.c ====================================
103	typedef struct SBTree SBTree;
104	typedef struct SBTC SBTC;
105	typedef struct SBtInfo {
106	SPgno root;
107	int nLevel;
108	int nData;
109	} SBtInfo;
110
111	#define TDB_CELLD_F_NIL 0x0
112	#define TDB_CELLD_F_KEY 0x1
113	#define TDB_CELLD_F_VAL 0x2
114
115	#define TDB_CELLDECODER_SET_FREE_NIL(pCellDecoder) ((pCellDecoder)->freeKV = TDB_CELLD_F_NIL)
116	#define TDB_CELLDECODER_CLZ_FREE_KEY(pCellDecoder) ((pCellDecoder)->freeKV &= ~TDB_CELLD_F_KEY)
117	#define TDB_CELLDECODER_CLZ_FREE_VAL(pCellDecoder) ((pCellDecoder)->freeKV &= ~TDB_CELLD_F_VAL)
118	#define TDB_CELLDECODER_SET_FREE_KEY(pCellDecoder) ((pCellDecoder)->freeKV \|= TDB_CELLD_F_KEY)
119	#define TDB_CELLDECODER_SET_FREE_VAL(pCellDecoder) ((pCellDecoder)->freeKV \|= TDB_CELLD_F_VAL)
120
121	#define TDB_CELLDECODER_FREE_KEY(pCellDecoder) ((pCellDecoder)->freeKV & TDB_CELLD_F_KEY)
122	#define TDB_CELLDECODER_FREE_VAL(pCellDecoder) ((pCellDecoder)->freeKV & TDB_CELLD_F_VAL)
123
124	typedef struct {
125	int kLen;
126	u8 *pKey;
127	int vLen;
128	u8 *pVal;
129	SPgno pgno;
130	u8 freeKV;
131	} SCellDecoder;
132
133	struct SBTC {
134	SBTree *pBt;
135	i8 iPage;
136	SPage *pPage;
137	int idx;
138	int idxStack[BTREE_MAX_DEPTH + 1];
139	SPage *pgStack[BTREE_MAX_DEPTH + 1];
140	SCellDecoder coder;
141	TXN *pTxn;
142	i8 freeTxn;
143	};
144
145	// SBTree
146	int tdbBtreeOpen(int keyLen, int valLen, SPager pFile, char const tbname, SPgno pgno, tdb_cmpr_fn_t kcmpr, TDB *pEnv,
147	SBTree **ppBt);
148	void tdbBtreeClose(SBTree *pBt);
149	int tdbBtreeInsert(SBTree pBt, const void pKey, int kLen, const void pVal, int vLen, TXN pTxn);
150	int tdbBtreeDelete(SBTree pBt, const void pKey, int kLen, TXN *pTxn);
151	// int tdbBtreeUpsert(SBTree pBt, const void pKey, int nKey, const void pData, int nData, TXN pTxn);
152	int tdbBtreeGet(SBTree pBt, const void pKey, int kLen, void *ppVal, int vLen);
153	int tdbBtreePGet(SBTree pBt, const void pKey, int kLen, void *ppKey, int pkLen, void *ppVal, int vLen);
154	int tdbFreeOvflPage(SPgno pgno, int nSize, TXN pTxn, SBTree pBt);
155
156	// these 3 functions are only for free page management, they are using the b-tree as a stack.
157	// they should only be called on the b-tree of the free page table, never call them for other
158	// purpose.
159	int tdbBtreeToStack(SBTree *pBt);
160	int tdbBtreePushFreePage(SBTree pBt, SPage pPage, TXN *pTxn);
161	int tdbBtreePopFreePage(SBTree pBt, SPgno pgno, TXN *pTxn);
162
163	typedef struct {
164	u8 flags;
165	SBTree *pBt;
166	} SBtreeInitPageArg;
167
168	int tdbBtreeInitPage(SPage pPage, void arg, int init);
169
170	// SBTC
171	int tdbBtcOpen(SBTC pBtc, SBTree pBt, TXN *pTxn);
172	void tdbBtcClose(SBTC *pBtc);
173	int tdbBtcIsValid(SBTC *pBtc);
174	int tdbBtcMoveTo(SBTC pBtc, const void pKey, int kLen, int *pCRst);
175	int tdbBtcMoveToFirst(SBTC *pBtc);
176	int tdbBtcMoveToLast(SBTC *pBtc);
177	int tdbBtcMoveToNext(SBTC *pBtc);
178	int tdbBtcMoveToPrev(SBTC *pBtc);
179	int tdbBtreeNext(SBTC pBtc, void ppKey, int kLen, void *ppVal, int vLen);
180	int tdbBtreePrev(SBTC pBtc, void ppKey, int kLen, void *ppVal, int vLen);
181	int tdbBtcGet(SBTC pBtc, const void ppKey, int kLen, const void *ppVal, int vLen);
182	int tdbBtcDelete(SBTC *pBtc);
183	int tdbBtcUpsert(SBTC pBtc, const void pKey, int kLen, const void *pData, int nData, int insert);
184
185	// tdbPager.c ====================================
186
187	int tdbPagerOpen(SPCache pCache, const char fileName, SPager **ppPager);
188	void tdbPagerClose(SPager *pPager);
189	int tdbPagerOpenDB(SPager pPager, SPgno ppgno, bool toCreate, SBTree *pBt);
190	int tdbPagerWrite(SPager pPager, SPage pPage);
191	int tdbPagerBegin(SPager pPager, TXN pTxn);
192	int tdbPagerCommit(SPager pPager, TXN pTxn);
193	int tdbPagerPostCommit(SPager pPager, TXN pTxn);
194	int tdbPagerPrepareAsyncCommit(SPager pPager, TXN pTxn);
195	int tdbPagerAbort(SPager pPager, TXN pTxn);
196	int tdbPagerFetchPage(SPager pPager, SPgno ppgno, SPage *ppPage, int (initPage)(SPage , void , int), void *arg,
197	TXN *pTxn);
198	void tdbPagerReturnPage(SPager pPager, SPage pPage, TXN *pTxn);
199	int tdbPagerFetchFreePage(SPager pPager, SPgno pgno, SPage ppPage, TXN pTxn);
200	int tdbPagerInsertFreePage(SPager pPager, SPage pPage, TXN *pTxn);
201	// int tdbPagerAllocPage(SPager pPager, SPgno ppgno);
202	int tdbPagerRestoreJournals(SPager *pPager);
203	int tdbPagerRollback(SPager *pPager);
204
205	// tdbPCache.c ====================================
206	#define TDB_PCACHE_PAGE \
207	u8 isAnchor; \
208	u8 isLocal; \
209	u8 isDirty; \
210	u8 isFree; \
211	volatile i32 nRef; \
212	i32 id; \
213	SPage *pFreeNext; \
214	SPage *pHashNext; \
215	SPage *pLruNext; \
216	SPage *pLruPrev; \
217	SPage *pDirtyNext; \
218	SPager *pPager; \
219	SPgid pgid;
220
221	// For page ref
222
223	int tdbPCacheOpen(int pageSize, int cacheSize, SPCache **ppCache);
224	void tdbPCacheClose(SPCache *pCache);
225	int tdbPCacheAlter(SPCache *pCache, int32_t nPage);
226	SPage tdbPCacheFetch(SPCache pCache, const SPgid pPgid, TXN pTxn, bool force, bool* loaded);
227	void tdbPCacheRelease(SPCache pCache, SPage pPage, TXN *pTxn);
228	void tdbPCacheMarkFree(SPCache pCache, SPage pPage);
229	void tdbPCacheInvalidatePage(SPCache pCache, SPager pPager, SPgno pgno);
230	int tdbPCacheGetPageSize(SPCache *pCache);
231
232	// tdbPage.c ====================================
233	typedef u8 SCell;
234
235	// PAGE APIS implemented
236	typedef struct {
237	int szOffset;
238	int szPageHdr;
239	int szFreeCell;
240	// cell number
241	int (getCellNum)(SPage );
242	void (setCellNum)(SPage , int);
243	// cell content offset
244	int (getCellBody)(SPage );
245	void (setCellBody)(SPage , int);
246	// first free cell offset (0 means no free cells)
247	int (getCellFree)(SPage );
248	void (setCellFree)(SPage , int);
249	// total free bytes
250	int (getFreeBytes)(SPage );
251	void (setFreeBytes)(SPage , int);
252	// cell offset at idx
253	int (getCellOffset)(SPage , int);
254	void (setCellOffset)(SPage , int, int);
255	// free cell info
256	void (getFreeCellInfo)(SCell pCell, int szCell, int nxOffset);
257	void (setFreeCellInfo)(SCell pCell, int szCell, int nxOffset);
258	} SPageMethods;
259
260	#pragma pack(push, 1)
261
262	// Page footer
263	typedef struct {
264	u8 cksm[4];
265	} SPageFtr;
266	#pragma pack(pop)
267
268	struct SPage {
269	SRBTreeNode node; // must be the first field for pageCmpFn to work
270	tdb_spinlock_t lock;
271	int pageSize;
272	u8 *pData;
273	SPageMethods *pPageMethods;
274	// Fields below used by pager and am
275	u8 *pPageHdr;
276	u8 *pCellIdx;
277	u8 *pFreeStart;
278	u8 *pFreeEnd;
279	SPageFtr *pPageFtr;
280	int nOverflow;
281	SCell *apOvfl[4];
282	int aiOvfl[4];
283	int kLen; // key length of the page, -1 for unknown
284	int vLen; // value length of the page, -1 for unknown
285	int maxLocal;
286	int minLocal;
287	int (xCellSize)(const SPage , SCell , int );
288	// Fields used by SPCache
289	TDB_PCACHE_PAGE
290	};
291
292	static inline i32 tdbRefPage(SPage *pPage) {	119,342,168✔
293	i32 nRef = atomic_add_fetch_32(&((pPage)->nRef), 1);	119,342,168✔
294	// tdbTrace("ref page %p/%d, nRef %d", pPage, pPage->id, nRef);
295	return nRef;	119,623,938✔
296	}
297
298	static inline i32 tdbUnrefPage(SPage *pPage) {	119,563,810✔
299	i32 nRef = atomic_sub_fetch_32(&((pPage)->nRef), 1);	119,563,810✔
300	// tdbTrace("unref page %p/%d, nRef %d", pPage, pPage->id, nRef);
301	return nRef;	119,653,973✔
302	}
303
304	#define tdbGetPageRef(pPage) atomic_load_32(&((pPage)->nRef))
305
306	// For page lock
307	#define P_LOCK_SUCC 0
308	#define P_LOCK_BUSY 1
309	#define P_LOCK_FAIL -1
310
311	static inline int tdbTryLockPage(tdb_spinlock_t *pLock) {	1,717,936✔
312	int ret = tdbSpinlockTrylock(pLock);	1,717,936✔
313	if (ret == 0) {	1,718,164!
314	return P_LOCK_SUCC;	1,718,177✔
315	} else if (ret == EBUSY) {	×
316	return P_LOCK_BUSY;	7✔
317	} else {
318	return P_LOCK_FAIL;	×
319	}
320	}
321
322	#define TDB_INIT_PAGE_LOCK(pPage) tdbSpinlockInit(&((pPage)->lock), 0)
323	#define TDB_DESTROY_PAGE_LOCK(pPage) tdbSpinlockDestroy(&((pPage)->lock))
324	#define TDB_LOCK_PAGE(pPage) tdbSpinlockLock(&((pPage)->lock))
325	#define TDB_UNLOCK_PAGE(pPage) tdbSpinlockUnlock(&((pPage)->lock))
326	#define TDB_TRY_LOCK_PAGE(pPage) tdbTryLockPage(&((pPage)->lock))
327
328	// APIs
329	#define TDB_PAGE_TOTAL_CELLS(pPage) ((pPage)->nOverflow + (pPage)->pPageMethods->getCellNum(pPage))
330	#define TDB_PAGE_USABLE_SIZE(pPage) ((u8 *)(pPage)->pPageFtr - (pPage)->pCellIdx)
331	#define TDB_PAGE_FREE_SIZE(pPage) (*(pPage)->pPageMethods->getFreeBytes)(pPage)
332	#define TDB_PAGE_PGNO(pPage) ((pPage)->pgid.pgno)
333	#define TDB_BYTES_CELL_TAKEN(pPage, pCell) \
334	((*(pPage)->xCellSize)(pPage, pCell, NULL) + (pPage)->pPageMethods->szOffset)
335	#define TDB_PAGE_OFFSET_SIZE(pPage) ((pPage)->pPageMethods->szOffset)
336
337	int tdbPageCreate(int pageSize, SPage *ppPage, void (xMalloc)(void , size_t), void *arg);
338	void tdbPageDestroy(SPage pPage, void (xFree)(void arg, void ptr), void *arg);
339	void tdbPageZero(SPage pPage, u8 szAmHdr, int (xCellSize)(const SPage , SCell , int *));
340	void tdbPageInit(SPage pPage, u8 szAmHdr, int (xCellSize)(const SPage , SCell , int *));
341	int tdbPageInsertCell(SPage pPage, int idx, SCell pCell, int szCell, u8 asOvfl);
342	int tdbPageDropCell(SPage pPage, int idx, TXN pTxn, SBTree *pBt);
343	int tdbPageUpdateCell(SPage pPage, int idx, SCell pCell, int szCell, TXN pTxn, SBTree pBt);
344	int32_t tdbPageCopy(SPage pFromPage, SPage pToPage, int deepCopyOvfl);
345	int tdbPageCapacity(int pageSize, int amHdrSize);
346
347	static inline SCell tdbPageGetCell(SPage pPage, int idx) {	553,268,180✔
348	SCell *pCell;
349	int iOvfl;
350	int lidx;
351
352	if (idx < 0 \|\| idx >= TDB_PAGE_TOTAL_CELLS(pPage)) {	553,268,180!
UNCOV 353	terrno = TSDB_CODE_INVALID_PARA;	×
354	return NULL;	×
355	}
356
357	iOvfl = 0;	552,162,107✔
358	for (; iOvfl < pPage->nOverflow; iOvfl++) {	557,104,312✔
359	if (pPage->aiOvfl[iOvfl] == idx) {	24,287,333✔
360	pCell = pPage->apOvfl[iOvfl];	392,336✔
361	return pCell;	392,336✔
362	} else if (pPage->aiOvfl[iOvfl] > idx) {	23,894,997✔
363	break;	18,952,792✔
364	}
365	}
366
367	lidx = idx - iOvfl;	551,769,771✔
368	pCell = pPage->pData + pPage->pPageMethods->getCellOffset(pPage, lidx);	551,769,771✔
369
370	return pCell;	550,310,981✔
371	}
372
373	#define USE_MAINDB
374
375	#ifdef USE_MAINDB
376	#define TDB_MAINDB_NAME "main.tdb"
377	#define TDB_FREEDB_NAME "_free.db"
378	#endif
379
380	struct STDB {
381	char *dbName;
382	char *jnName;
383	int jfd;
384	SPCache *pCache;
385	SPager *pgrList;
386	int nPager;
387	int nPgrHash;
388	SPager **pgrHash;
389	#ifdef USE_MAINDB
390	TTB *pMainDb;
391	TTB *pFreeDb;
392	#endif
393	int64_t txnId;
394	int32_t encryptAlgorithm;
395	char encryptKey[ENCRYPT_KEY_LEN + 1];
396	};
397
398	struct SPager {
399	char *dbFileName;
400	char *jFileName;
401	int pageSize;
402	uint8_t fid[TDB_FILE_ID_LEN];
403	tdb_fd_t fd;
404	SPCache *pCache;
405	SPgno dbFileSize;
406	SPgno dbOrigSize;
407	// SPage *pDirty;
408	SRBTree rbt;
409	// u8 inTran;
410	TXN *pActiveTxn;
411	SPager *pNext; // used by TDB
412	SPager *pHashNext; // used by TDB
413	#ifdef USE_MAINDB
414	TDB *pEnv;
415	#endif
416	};
417
418	#ifdef __cplusplus
419	}
420	#endif
421
422	#endif /_TD_TDB_INTERNAL_H_/

taosdata / TDengine / #4473

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous