#3561

Committed 19 Dec 2024 03:15AM UTC coverage: 58.812% (-1.3%) from 60.124%

Build # #3561

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

Merge pull request #29213 from taosdata/merge/mainto3.0

merge: from main to 3.0 branch

Run Details

130770 of 287658 branches covered (45.46%)

Branch coverage included in aggregate %.

32 of 78 new or added lines in 4 files covered. (41.03%)

7347 existing lines in 166 files now uncovered.

205356 of 283866 relevant lines covered (72.34%)

7187865.64 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

52.41

/source/dnode/vnode/src/tq/tqSnapshot.c

/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "meta.h"
#include "tdbInt.h"
#include "tq.h"

// STqSnapReader ========================================
struct STqSnapReader {
  STQ*    pTq;
  int64_t sver;
  int64_t ever;
  TBC*    pCur;
  int8_t  type;
};

int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, int8_t type, STqSnapReader** ppReader) {
  if (pTq == NULL || ppReader == NULL) {
    return TSDB_CODE_INVALID_MSG;
  }
  int32_t        code = 0;
  STqSnapReader* pReader = NULL;

  // alloc
  pReader = (STqSnapReader*)taosMemoryCalloc(1, sizeof(STqSnapReader));
  if (pReader == NULL) {
    code = terrno;
    goto _err;
  }
  pReader->pTq = pTq;
  pReader->sver = sver;
  pReader->ever = ever;
  pReader->type = type;

  // impl
  TTB* pTb = NULL;
  if (type == SNAP_DATA_TQ_CHECKINFO) {
    pTb = pTq->pCheckStore;
  } else if (type == SNAP_DATA_TQ_HANDLE) {
    pTb = pTq->pExecStore;
  } else if (type == SNAP_DATA_TQ_OFFSET) {
    pTb = pTq->pOffsetStore;
  } else {
    code = TSDB_CODE_INVALID_MSG;
    goto _err;
  }
  code = tdbTbcOpen(pTb, &pReader->pCur, NULL);
  if (code) {
    taosMemoryFree(pReader);
    goto _err;
  }

  code = tdbTbcMoveToFirst(pReader->pCur);
  if (code) {
    taosMemoryFree(pReader);
    goto _err;
  }

  tqInfo("vgId:%d, vnode snapshot tq reader opened", TD_VID(pTq->pVnode));

  *ppReader = pReader;
  return code;

_err:
  tqError("vgId:%d, vnode snapshot tq reader open failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
  *ppReader = NULL;
  return code;
}

void tqSnapReaderClose(STqSnapReader** ppReader) {
  if (ppReader == NULL || *ppReader == NULL) {
    return;
  }
  tdbTbcClose((*ppReader)->pCur);
  taosMemoryFree(*ppReader);
  *ppReader = NULL;
}

int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) {
  if (pReader == NULL || ppData == NULL) {
    return TSDB_CODE_INVALID_MSG;
  }
  int32_t code = 0;
  void*   pKey = NULL;
  void*   pVal = NULL;
  int32_t kLen = 0;
  int32_t vLen = 0;

  if (tdbTbcNext(pReader->pCur, &pKey, &kLen, &pVal, &vLen)) {
    goto _exit;
  }

  *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen);
  if (*ppData == NULL) {
    code = TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY);
    goto _err;
  }

  SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData);
  pHdr->type = pReader->type;
  pHdr->size = vLen;
  (void)memcpy(pHdr->data, pVal, vLen);

_exit:
  tdbFree(pKey);
  tdbFree(pVal);
  tqInfo("vgId:%d, vnode snapshot tq read data, vLen:%d", TD_VID(pReader->pTq->pVnode), vLen);
  return code;

_err:
  tdbFree(pKey);
  tdbFree(pVal);
  tqError("vgId:%d, vnode snapshot tq read data failed since %s", TD_VID(pReader->pTq->pVnode), tstrerror(code));
  return code;
}

// STqSnapWriter ========================================
struct STqSnapWriter {
  STQ*    pTq;
  int64_t sver;
  int64_t ever;
  TXN*    txn;
};

int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** ppWriter) {
  if (pTq == NULL || ppWriter == NULL) {
    return TSDB_CODE_INVALID_MSG;
  }
  int32_t        code = 0;
  STqSnapWriter* pWriter = NULL;

  // alloc
  pWriter = (STqSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter));
  if (pWriter == NULL) {
    code = TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY);
    ;
    goto _err;
  }
  pWriter->pTq = pTq;
  pWriter->sver = sver;
  pWriter->ever = ever;

  code = tdbBegin(pTq->pMetaDB, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0);
  if (code < 0) {
    taosMemoryFree(pWriter);
    goto _err;
  }

  *ppWriter = pWriter;
  return code;

_err:
  tqError("vgId:%d, tq snapshot writer open failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
  *ppWriter = NULL;
  return code;
}

int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) {
  if (ppWriter == NULL || *ppWriter == NULL) {
    return TSDB_CODE_INVALID_MSG;
  }
  int32_t        code = 0;
  STqSnapWriter* pWriter = *ppWriter;
  STQ*           pTq = pWriter->pTq;

  if (rollback) {
    tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn);
  } else {
    code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn);
    if (code) goto _err;
    code = tdbPostCommit(pWriter->pTq->pMetaDB, pWriter->txn);
    if (code) goto _err;
  }

  taosMemoryFree(pWriter);
  *ppWriter = NULL;

  return code;

_err:
  tqError("vgId:%d, tq snapshot writer close failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
  return code;
}

int32_t tqSnapHandleWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
  if (pWriter == NULL || pData == NULL || nData < sizeof(SSnapDataHdr)) {
    return TSDB_CODE_INVALID_MSG;
  }
  int32_t   code = 0;
  STQ*      pTq = pWriter->pTq;
  SDecoder  decoder = {0};
  SDecoder* pDecoder = &decoder;
  STqHandle handle = {0};

  tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
  code = tDecodeSTqHandle(pDecoder, &handle);
  if (code) goto end;
  taosWLockLatch(&pTq->lock);
  code = tqMetaSaveInfo(pTq, pTq->pExecStore, handle.subKey, strlen(handle.subKey), pData + sizeof(SSnapDataHdr),
                        nData - sizeof(SSnapDataHdr));
  taosWUnLockLatch(&pTq->lock);

end:
  tDecoderClear(pDecoder);
  tqDestroyTqHandle(&handle);
  tqInfo("vgId:%d, vnode snapshot tq write result:%d", TD_VID(pTq->pVnode), code);
  return code;
}

int32_t tqSnapCheckInfoWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
  if (pWriter == NULL || pData == NULL || nData < sizeof(SSnapDataHdr)) {
    return TSDB_CODE_INVALID_MSG;
  }
  int32_t      code = 0;
  STQ*         pTq = pWriter->pTq;
  STqCheckInfo info = {0};
  code = tqMetaDecodeCheckInfo(&info, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
  if (code != 0) {
    goto _err;
  }

  code = tqMetaSaveInfo(pTq, pTq->pCheckStore, &info.topic, strlen(info.topic), pData + sizeof(SSnapDataHdr),
                        nData - sizeof(SSnapDataHdr));
  tDeleteSTqCheckInfo(&info);
  if (code) goto _err;

  return code;

_err:
  tqError("vgId:%d, vnode check info tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
  return code;
}

int32_t tqSnapOffsetWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
  if (pWriter == NULL || pData == NULL || nData < sizeof(SSnapDataHdr)) {
    return TSDB_CODE_INVALID_MSG;
  }
  int32_t code = 0;
  STQ*    pTq = pWriter->pTq;

  STqOffset info = {0};
  code = tqMetaDecodeOffsetInfo(&info, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
  if (code != 0) {
    goto _err;
  }

  code = tqMetaSaveInfo(pTq, pTq->pOffsetStore, info.subKey, strlen(info.subKey), pData + sizeof(SSnapDataHdr),
                        nData - sizeof(SSnapDataHdr));
  tDeleteSTqOffset(&info);
  if (code) goto _err;

  return code;

_err:
  tqError("vgId:%d, vnode check info tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
  return code;
}

1	/*
2	* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3	*
4	* This program is free software: you can use, redistribute, and/or modify
5	* it under the terms of the GNU Affero General Public License, version 3
6	* or later ("AGPL"), as published by the Free Software Foundation.
7	*
8	* This program is distributed in the hope that it will be useful, but WITHOUT
9	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10	* FITNESS FOR A PARTICULAR PURPOSE.
11	*
12	* You should have received a copy of the GNU Affero General Public License
13	* along with this program. If not, see <http://www.gnu.org/licenses/>.
14	*/
15
16	#include "meta.h"
17	#include "tdbInt.h"
18	#include "tq.h"
19
20	// STqSnapReader ========================================
21	struct STqSnapReader {
22	STQ* pTq;
23	int64_t sver;
24	int64_t ever;
25	TBC* pCur;
26	int8_t type;
27	};
28
29	int32_t tqSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, int8_t type, STqSnapReader** ppReader) {	201✔
30	if (pTq == NULL \|\| ppReader == NULL) {	201!
UNCOV 31	return TSDB_CODE_INVALID_MSG;	×
32	}
33	int32_t code = 0;	201✔
34	STqSnapReader* pReader = NULL;	201✔
35
36	// alloc
37	pReader = (STqSnapReader*)taosMemoryCalloc(1, sizeof(STqSnapReader));	201!
38	if (pReader == NULL) {	201!
UNCOV 39	code = terrno;	×
UNCOV 40	goto _err;	×
41	}
42	pReader->pTq = pTq;	201✔
43	pReader->sver = sver;	201✔
44	pReader->ever = ever;	201✔
45	pReader->type = type;	201✔
46
47	// impl
48	TTB* pTb = NULL;	201✔
49	if (type == SNAP_DATA_TQ_CHECKINFO) {	201✔
50	pTb = pTq->pCheckStore;	67✔
51	} else if (type == SNAP_DATA_TQ_HANDLE) {	134✔
52	pTb = pTq->pExecStore;	67✔
53	} else if (type == SNAP_DATA_TQ_OFFSET) {	67!
54	pTb = pTq->pOffsetStore;	67✔
55	} else {
UNCOV 56	code = TSDB_CODE_INVALID_MSG;	×
UNCOV 57	goto _err;	×
58	}
59	code = tdbTbcOpen(pTb, &pReader->pCur, NULL);	201✔
60	if (code) {	201!
UNCOV 61	taosMemoryFree(pReader);	×
UNCOV 62	goto _err;	×
63	}
64
65	code = tdbTbcMoveToFirst(pReader->pCur);	201✔
66	if (code) {	201!
UNCOV 67	taosMemoryFree(pReader);	×
UNCOV 68	goto _err;	×
69	}
70
71	tqInfo("vgId:%d, vnode snapshot tq reader opened", TD_VID(pTq->pVnode));	201!
72
73	*ppReader = pReader;	201✔
74	return code;	201✔
75
76	_err:	×
UNCOV 77	tqError("vgId:%d, vnode snapshot tq reader open failed since %s", TD_VID(pTq->pVnode), tstrerror(code));	×
UNCOV 78	*ppReader = NULL;	×
UNCOV 79	return code;	×
80	}
81
82	void tqSnapReaderClose(STqSnapReader** ppReader) {	201✔
83	if (ppReader == NULL \|\| *ppReader == NULL) {	201!
UNCOV 84	return;	×
85	}
86	tdbTbcClose((*ppReader)->pCur);	201✔
87	taosMemoryFree(*ppReader);	201!
88	*ppReader = NULL;	201✔
89	}
90
91	int32_t tqSnapRead(STqSnapReader* pReader, uint8_t** ppData) {	287✔
92	if (pReader == NULL \|\| ppData == NULL) {	287!
UNCOV 93	return TSDB_CODE_INVALID_MSG;	×
94	}
95	int32_t code = 0;	287✔
96	void* pKey = NULL;	287✔
97	void* pVal = NULL;	287✔
98	int32_t kLen = 0;	287✔
99	int32_t vLen = 0;	287✔
100
101	if (tdbTbcNext(pReader->pCur, &pKey, &kLen, &pVal, &vLen)) {	287✔
102	goto _exit;	201✔
103	}
104
105	*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen);	86!
106	if (*ppData == NULL) {	86!
UNCOV 107	code = TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY);	×
UNCOV 108	goto _err;	×
109	}
110
111	SSnapDataHdr* pHdr = (SSnapDataHdr)(ppData);	86✔
112	pHdr->type = pReader->type;	86✔
113	pHdr->size = vLen;	86✔
114	(void)memcpy(pHdr->data, pVal, vLen);	86✔
115
116	_exit:	287✔
117	tdbFree(pKey);	287✔
118	tdbFree(pVal);	287✔
119	tqInfo("vgId:%d, vnode snapshot tq read data, vLen:%d", TD_VID(pReader->pTq->pVnode), vLen);	287!
120	return code;	287✔
121
UNCOV 122	_err:	×
UNCOV 123	tdbFree(pKey);	×
UNCOV 124	tdbFree(pVal);	×
UNCOV 125	tqError("vgId:%d, vnode snapshot tq read data failed since %s", TD_VID(pReader->pTq->pVnode), tstrerror(code));	×
UNCOV 126	return code;	×
127	}
128
129	// STqSnapWriter ========================================
130	struct STqSnapWriter {
131	STQ* pTq;
132	int64_t sver;
133	int64_t ever;
134	TXN* txn;
135	};
136
137	int32_t tqSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, STqSnapWriter** ppWriter) {	86✔
138	if (pTq == NULL \|\| ppWriter == NULL) {	86!
UNCOV 139	return TSDB_CODE_INVALID_MSG;	×
140	}
141	int32_t code = 0;	86✔
142	STqSnapWriter* pWriter = NULL;	86✔
143
144	// alloc
145	pWriter = (STqSnapWriter)taosMemoryCalloc(1, sizeof(pWriter));	86!
146	if (pWriter == NULL) {	86!
UNCOV 147	code = TAOS_GET_TERRNO(TSDB_CODE_OUT_OF_MEMORY);	×
148	;
UNCOV 149	goto _err;	×
150	}
151	pWriter->pTq = pTq;	86✔
152	pWriter->sver = sver;	86✔
153	pWriter->ever = ever;	86✔
154
155	code = tdbBegin(pTq->pMetaDB, &pWriter->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0);	86✔
156	if (code < 0) {	86!
UNCOV 157	taosMemoryFree(pWriter);	×
UNCOV 158	goto _err;	×
159	}
160
161	*ppWriter = pWriter;	86✔
162	return code;	86✔
163
164	_err:	×
UNCOV 165	tqError("vgId:%d, tq snapshot writer open failed since %s", TD_VID(pTq->pVnode), tstrerror(code));	×
UNCOV 166	*ppWriter = NULL;	×
UNCOV 167	return code;	×
168	}
169
170	int32_t tqSnapWriterClose(STqSnapWriter** ppWriter, int8_t rollback) {	86✔
171	if (ppWriter == NULL \|\| *ppWriter == NULL) {	86!
UNCOV 172	return TSDB_CODE_INVALID_MSG;	×
173	}
174	int32_t code = 0;	86✔
175	STqSnapWriter* pWriter = *ppWriter;	86✔
176	STQ* pTq = pWriter->pTq;	86✔
177
178	if (rollback) {	86!
179	tdbAbort(pWriter->pTq->pMetaDB, pWriter->txn);	×
180	} else {
181	code = tdbCommit(pWriter->pTq->pMetaDB, pWriter->txn);	86✔
182	if (code) goto _err;	86!
183	code = tdbPostCommit(pWriter->pTq->pMetaDB, pWriter->txn);	86✔
184	if (code) goto _err;	86!
185	}
186
187	taosMemoryFree(pWriter);	86!
188	*ppWriter = NULL;	86✔
189
190	return code;	86✔
191
UNCOV 192	_err:	×
UNCOV 193	tqError("vgId:%d, tq snapshot writer close failed since %s", TD_VID(pTq->pVnode), tstrerror(code));	×
UNCOV 194	return code;	×
195	}
196
197	int32_t tqSnapHandleWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {	43✔
198	if (pWriter == NULL \|\| pData == NULL \|\| nData < sizeof(SSnapDataHdr)) {	43!
UNCOV 199	return TSDB_CODE_INVALID_MSG;	×
200	}
201	int32_t code = 0;	43✔
202	STQ* pTq = pWriter->pTq;	43✔
203	SDecoder decoder = {0};	43✔
204	SDecoder* pDecoder = &decoder;	43✔
205	STqHandle handle = {0};	43✔
206
207	tDecoderInit(pDecoder, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));	43✔
208	code = tDecodeSTqHandle(pDecoder, &handle);	43✔
209	if (code) goto end;	43!
210	taosWLockLatch(&pTq->lock);	43✔
211	code = tqMetaSaveInfo(pTq, pTq->pExecStore, handle.subKey, strlen(handle.subKey), pData + sizeof(SSnapDataHdr),	43✔
212	nData - sizeof(SSnapDataHdr));
213	taosWUnLockLatch(&pTq->lock);	43✔
214
215	end:	43✔
216	tDecoderClear(pDecoder);	43✔
217	tqDestroyTqHandle(&handle);	43✔
218	tqInfo("vgId:%d, vnode snapshot tq write result:%d", TD_VID(pTq->pVnode), code);	43!
219	return code;	43✔
220	}
221
222	int32_t tqSnapCheckInfoWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {	×
UNCOV 223	if (pWriter == NULL \|\| pData == NULL \|\| nData < sizeof(SSnapDataHdr)) {	×
UNCOV 224	return TSDB_CODE_INVALID_MSG;	×
225	}
UNCOV 226	int32_t code = 0;	×
UNCOV 227	STQ* pTq = pWriter->pTq;	×
UNCOV 228	STqCheckInfo info = {0};	×
UNCOV 229	code = tqMetaDecodeCheckInfo(&info, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));	×
UNCOV 230	if (code != 0) {	×
UNCOV 231	goto _err;	×
232	}
233
UNCOV 234	code = tqMetaSaveInfo(pTq, pTq->pCheckStore, &info.topic, strlen(info.topic), pData + sizeof(SSnapDataHdr),	×
235	nData - sizeof(SSnapDataHdr));
UNCOV 236	tDeleteSTqCheckInfo(&info);	×
UNCOV 237	if (code) goto _err;	×
238
UNCOV 239	return code;	×
240
UNCOV 241	_err:	×
242	tqError("vgId:%d, vnode check info tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code));	×
243	return code;	×
244	}
245
246	int32_t tqSnapOffsetWrite(STqSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {	43✔
247	if (pWriter == NULL \|\| pData == NULL \|\| nData < sizeof(SSnapDataHdr)) {	43!
UNCOV 248	return TSDB_CODE_INVALID_MSG;	×
249	}
250	int32_t code = 0;	43✔
251	STQ* pTq = pWriter->pTq;	43✔
252
253	STqOffset info = {0};	43✔
254	code = tqMetaDecodeOffsetInfo(&info, pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));	43✔
255	if (code != 0) {	43!
UNCOV 256	goto _err;	×
257	}
258
259	code = tqMetaSaveInfo(pTq, pTq->pOffsetStore, info.subKey, strlen(info.subKey), pData + sizeof(SSnapDataHdr),	43✔
260	nData - sizeof(SSnapDataHdr));
261	tDeleteSTqOffset(&info);	43✔
262	if (code) goto _err;	43!
263
264	return code;	43✔
265
UNCOV 266	_err:	×
UNCOV 267	tqError("vgId:%d, vnode check info tq write failed since %s", TD_VID(pTq->pVnode), tstrerror(code));	×
UNCOV 268	return code;	×
269	}

taosdata / TDengine / #3561

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous