#3526

Committed 10 Nov 2024 03:50AM UTC coverage: 60.225% (-0.6%) from 60.818%

Build # #3526

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

Merge pull request #28709 from taosdata/main

merge: from main to 3.0 branch

Run Details

117031 of 249004 branches covered (47.0%)

Branch coverage included in aggregate %.

130 of 169 new or added lines in 23 files covered. (76.92%)

4149 existing lines in 176 files now uncovered.

197577 of 273386 relevant lines covered (72.27%)

5840219.36 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

31.43

/source/dnode/vnode/src/tq/tqStreamTaskSnap.c

/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "meta.h"
#include "tdbInt.h"
#include "tq.h"

// STqSnapReader ========================================

typedef struct {
  int8_t type;
  TTB*   tbl;
} STablePair;
struct SStreamTaskReader {
  STQ*    pTq;
  int64_t sver;
  int64_t ever;
  TBC*    pCur;
  SArray* tdbTbList;
  int8_t  pos;
};

int32_t streamTaskSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamTaskReader** ppReader) {
  int32_t            code = 0;
  SStreamTaskReader* pReader = NULL;

  // alloc
  pReader = (SStreamTaskReader*)taosMemoryCalloc(1, sizeof(SStreamTaskReader));
  if (pReader == NULL) {
    TAOS_CHECK_RETURN(terrno);
  }
  pReader->pTq = pTq;
  pReader->sver = sver;
  pReader->ever = ever;
  pReader->tdbTbList = taosArrayInit(4, sizeof(STablePair));
  if (pReader->tdbTbList == NULL) {
    TAOS_CHECK_GOTO(terrno, NULL, _err);
  }

  STablePair pair1 = {.tbl = pTq->pStreamMeta->pTaskDb, .type = SNAP_DATA_STREAM_TASK};
  if (NULL == taosArrayPush(pReader->tdbTbList, &pair1)) {
    TAOS_CHECK_GOTO(terrno, NULL, _err);
  }

  STablePair pair2 = {.tbl = pTq->pStreamMeta->pCheckpointDb, .type = SNAP_DATA_STREAM_TASK_CHECKPOINT};
  if (NULL == taosArrayPush(pReader->tdbTbList, &pair2)) {
    TAOS_CHECK_GOTO(terrno, NULL, _err);
  }

  pReader->pos = 0;

  STablePair* pPair = taosArrayGet(pReader->tdbTbList, pReader->pos);
  code = tdbTbcOpen(pPair->tbl, &pReader->pCur, NULL);
  if (code) {
    tqInfo("vgId:%d, vnode stream-task snapshot reader failed to open, reason: %s", TD_VID(pTq->pVnode),
           tstrerror(code));
    TAOS_CHECK_GOTO(code, NULL, _err);
  }

  code = tdbTbcMoveToFirst(pReader->pCur);
  if (code) {
    tqInfo("vgId:%d, vnode stream-task snapshot reader failed to iterate, reason: %s", TD_VID(pTq->pVnode),
           tstrerror(code));
    TAOS_CHECK_GOTO(code, NULL, _err);
  }

  tqDebug("vgId:%d, vnode stream-task snapshot reader opened", TD_VID(pTq->pVnode));

  *ppReader = pReader;
  return code;

_err:
  tqError("vgId:%d, vnode stream-task snapshot reader open failed since %s", TD_VID(pTq->pVnode), tstrerror(code));
  int32_t ret = streamTaskSnapReaderClose(pReader);
  *ppReader = NULL;
  return code;
}

int32_t streamTaskSnapReaderClose(SStreamTaskReader* pReader) {
  if (pReader == NULL) {
    return 0;
  }

  int32_t code = 0;
  int32_t vgId = TD_VID(pReader->pTq->pVnode);

  taosArrayDestroy(pReader->tdbTbList);
  tdbTbcClose(pReader->pCur);
  tqInfo("vgId:%d, vnode stream-task snapshot reader closed", vgId);

  taosMemoryFree(pReader);
  return code;
}

int32_t streamTaskSnapRead(SStreamTaskReader* pReader, uint8_t** ppData) {
  int32_t     code = 0;
  const void* pKey = NULL;
  void*       pVal = NULL;
  int32_t     kLen = 0;
  int32_t     vLen = 0;
  SDecoder    decoder;
  STqHandle   handle;

  *ppData = NULL;
  int8_t except = 0;
  tqDebug("vgId:%d, vnode stream-task snapshot start read data", TD_VID(pReader->pTq->pVnode));

  STablePair* pPair = taosArrayGet(pReader->tdbTbList, pReader->pos);

NextTbl:
  except = 0;
  for (;;) {
    const void* tVal = NULL;
    int32_t     tLen = 0;
    if (tdbTbcGet(pReader->pCur, &pKey, &kLen, &tVal, &tLen)) {
      except = 1;
      break;
    } else {
      pVal = taosMemoryCalloc(1, tLen);
      if (pVal == NULL) {
        code = terrno;
        goto _err;
      }

      memcpy(pVal, tVal, tLen);
      vLen = tLen;
    }
    TAOS_UNUSED(tdbTbcMoveToNext(pReader->pCur));
    break;
  }
  if (except == 1) {
    if (pReader->pos + 1 < taosArrayGetSize(pReader->tdbTbList)) {
      tdbTbcClose(pReader->pCur);

      pReader->pos += 1;
      pPair = taosArrayGet(pReader->tdbTbList, pReader->pos);
      code = tdbTbcOpen(pPair->tbl, &pReader->pCur, NULL);
      TAOS_UNUSED(tdbTbcMoveToFirst(pReader->pCur));

      goto NextTbl;
    }
  }
  if (pVal == NULL || vLen == 0) {
    *ppData = NULL;
    tqDebug("vgId:%d, vnode stream-task snapshot finished read data", TD_VID(pReader->pTq->pVnode));
    return code;
  }
  *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen);
  if (*ppData == NULL) {
    code = terrno;
    goto _err;
  }

  SSnapDataHdr* pHdr = (SSnapDataHdr*)(*ppData);
  pHdr->type = pPair->type;
  pHdr->size = vLen;
  memcpy(pHdr->data, pVal, vLen);
  taosMemoryFree(pVal);

  tqDebug("vgId:%d, vnode stream-task snapshot read data vLen:%d", TD_VID(pReader->pTq->pVnode), vLen);
  return code;

_err:
  tqError("vgId:%d, vnode stream-task snapshot read data failed since %s", TD_VID(pReader->pTq->pVnode),
          tstrerror(code));
  return code;
}

// STqSnapWriter ========================================
struct SStreamTaskWriter {
  STQ*    pTq;
  int64_t sver;
  int64_t ever;
};

int32_t streamTaskSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamTaskWriter** ppWriter) {
  int32_t            code = 0;
  SStreamTaskWriter* pWriter;

  // alloc
  pWriter = (SStreamTaskWriter*)taosMemoryCalloc(1, sizeof(*pWriter));
  if (pWriter == NULL) {
    TAOS_CHECK_RETURN(terrno);
  }
  pWriter->pTq = pTq;
  pWriter->sver = sver;
  pWriter->ever = ever;

  *ppWriter = pWriter;
  tqDebug("vgId:%d, vnode stream-task snapshot writer opened", TD_VID(pTq->pVnode));
  return code;
}

int32_t streamTaskSnapWriterClose(SStreamTaskWriter* pWriter, int8_t rollback, int8_t loadTask) {
  int32_t code = 0;
  STQ*    pTq = pWriter->pTq;

  streamMetaWLock(pTq->pStreamMeta);
  tqDebug("vgId:%d, vnode stream-task snapshot writer closed", TD_VID(pTq->pVnode));
  if (rollback) {
    TAOS_UNUSED(tdbAbort(pTq->pStreamMeta->db, pTq->pStreamMeta->txn));
  } else {
    code = tdbCommit(pTq->pStreamMeta->db, pTq->pStreamMeta->txn);
    if (code) goto _err;
    code = tdbPostCommit(pTq->pStreamMeta->db, pTq->pStreamMeta->txn);
    if (code) goto _err;
  }

  if ((code = tdbBegin(pTq->pStreamMeta->db, &pTq->pStreamMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0)) < 0) {
    taosMemoryFree(pWriter);
    goto _err;
  }
  streamMetaWUnLock(pTq->pStreamMeta);
  taosMemoryFree(pWriter);

  if (loadTask == 1) {
    streamMetaLoadAllTasks(pTq->pStreamMeta);
  }
  return code;

_err:
  tqError("vgId:%d, vnode stream-task snapshot writer failed to close since %s", TD_VID(pWriter->pTq->pVnode),
          tstrerror(code));
  streamMetaWUnLock(pTq->pStreamMeta);
  return code;
}

int32_t streamTaskSnapWrite(SStreamTaskWriter* pWriter, uint8_t* pData, uint32_t nData) {
  int32_t       code = 0;
  STQ*          pTq = pWriter->pTq;
  SSnapDataHdr* pHdr = (SSnapDataHdr*)pData;
  if (pHdr->type == SNAP_DATA_STREAM_TASK) {
    STaskId taskId = {0};

    SDecoder decoder;
    tDecoderInit(&decoder, (uint8_t*)pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));
    code = tDecodeStreamTaskId(&decoder, &taskId);
    if (code < 0) {
      tDecoderClear(&decoder);
      goto _err;
    }
    tDecoderClear(&decoder);

    int64_t key[2] = {taskId.streamId, taskId.taskId};
    streamMetaWLock(pTq->pStreamMeta);
    if ((code =
             tdbTbUpsert(pTq->pStreamMeta->pTaskDb, key, sizeof(int64_t) << 1, (uint8_t*)pData + sizeof(SSnapDataHdr),
                         nData - sizeof(SSnapDataHdr), pTq->pStreamMeta->txn)) < 0) {
      streamMetaWUnLock(pTq->pStreamMeta);
      return code;
    }
    streamMetaWUnLock(pTq->pStreamMeta);
  } else if (pHdr->type == SNAP_DATA_STREAM_TASK_CHECKPOINT) {
    // do nothing
  }
  tqDebug("vgId:%d, vnode stream-task snapshot write", TD_VID(pTq->pVnode));

  return code;

_err:
  tqError("vgId:%d, vnode stream-task snapshot failed to write since %s", TD_VID(pTq->pVnode), tstrerror(code));
  return code;
}

1	/*
2	* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3	*
4	* This program is free software: you can use, redistribute, and/or modify
5	* it under the terms of the GNU Affero General Public License, version 3
6	* or later ("AGPL"), as published by the Free Software Foundation.
7	*
8	* This program is distributed in the hope that it will be useful, but WITHOUT
9	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10	* FITNESS FOR A PARTICULAR PURPOSE.
11	*
12	* You should have received a copy of the GNU Affero General Public License
13	* along with this program. If not, see <http://www.gnu.org/licenses/>.
14	*/
15
16	#include "meta.h"
17	#include "tdbInt.h"
18	#include "tq.h"
19
20	// STqSnapReader ========================================
21
22	typedef struct {
23	int8_t type;
24	TTB* tbl;
25	} STablePair;
26	struct SStreamTaskReader {
27	STQ* pTq;
28	int64_t sver;
29	int64_t ever;
30	TBC* pCur;
31	SArray* tdbTbList;
32	int8_t pos;
33	};
34
35	int32_t streamTaskSnapReaderOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamTaskReader** ppReader) {	83✔
36	int32_t code = 0;	83✔
37	SStreamTaskReader* pReader = NULL;	83✔
38
39	// alloc
40	pReader = (SStreamTaskReader*)taosMemoryCalloc(1, sizeof(SStreamTaskReader));	83✔
41	if (pReader == NULL) {	83!
42	TAOS_CHECK_RETURN(terrno);	×
43	}
44	pReader->pTq = pTq;	83✔
45	pReader->sver = sver;	83✔
46	pReader->ever = ever;	83✔
47	pReader->tdbTbList = taosArrayInit(4, sizeof(STablePair));	83✔
48	if (pReader->tdbTbList == NULL) {	83!
49	TAOS_CHECK_GOTO(terrno, NULL, _err);	×
50	}
51
52	STablePair pair1 = {.tbl = pTq->pStreamMeta->pTaskDb, .type = SNAP_DATA_STREAM_TASK};	83✔
53	if (NULL == taosArrayPush(pReader->tdbTbList, &pair1)) {	166!
54	TAOS_CHECK_GOTO(terrno, NULL, _err);	×
55	}
56
57	STablePair pair2 = {.tbl = pTq->pStreamMeta->pCheckpointDb, .type = SNAP_DATA_STREAM_TASK_CHECKPOINT};	83✔
58	if (NULL == taosArrayPush(pReader->tdbTbList, &pair2)) {	166!
59	TAOS_CHECK_GOTO(terrno, NULL, _err);	×
60	}
61
62	pReader->pos = 0;	83✔
63
64	STablePair* pPair = taosArrayGet(pReader->tdbTbList, pReader->pos);	83✔
65	code = tdbTbcOpen(pPair->tbl, &pReader->pCur, NULL);	83✔
66	if (code) {	83!
67	tqInfo("vgId:%d, vnode stream-task snapshot reader failed to open, reason: %s", TD_VID(pTq->pVnode),	×
68	tstrerror(code));
69	TAOS_CHECK_GOTO(code, NULL, _err);	×
70	}
71
72	code = tdbTbcMoveToFirst(pReader->pCur);	83✔
73	if (code) {	83!
74	tqInfo("vgId:%d, vnode stream-task snapshot reader failed to iterate, reason: %s", TD_VID(pTq->pVnode),	×
75	tstrerror(code));
76	TAOS_CHECK_GOTO(code, NULL, _err);	×
77	}
78
79	tqDebug("vgId:%d, vnode stream-task snapshot reader opened", TD_VID(pTq->pVnode));	83!
80
81	*ppReader = pReader;	83✔
82	return code;	83✔
83
84	_err:	×
85	tqError("vgId:%d, vnode stream-task snapshot reader open failed since %s", TD_VID(pTq->pVnode), tstrerror(code));	×
86	int32_t ret = streamTaskSnapReaderClose(pReader);	×
87	*ppReader = NULL;	×
88	return code;	×
89	}
90
91	int32_t streamTaskSnapReaderClose(SStreamTaskReader* pReader) {	83✔
92	if (pReader == NULL) {	83!
93	return 0;	×
94	}
95
96	int32_t code = 0;	83✔
97	int32_t vgId = TD_VID(pReader->pTq->pVnode);	83✔
98
99	taosArrayDestroy(pReader->tdbTbList);	83✔
100	tdbTbcClose(pReader->pCur);	83✔
101	tqInfo("vgId:%d, vnode stream-task snapshot reader closed", vgId);	83!
102
103	taosMemoryFree(pReader);	83✔
104	return code;	83✔
105	}
106
107	int32_t streamTaskSnapRead(SStreamTaskReader* pReader, uint8_t** ppData) {	83✔
108	int32_t code = 0;	83✔
109	const void* pKey = NULL;	83✔
110	void* pVal = NULL;	83✔
111	int32_t kLen = 0;	83✔
112	int32_t vLen = 0;	83✔
113	SDecoder decoder;
114	STqHandle handle;
115
116	*ppData = NULL;	83✔
117	int8_t except = 0;	83✔
118	tqDebug("vgId:%d, vnode stream-task snapshot start read data", TD_VID(pReader->pTq->pVnode));	83!
119
120	STablePair* pPair = taosArrayGet(pReader->tdbTbList, pReader->pos);	83✔
121
122	NextTbl:	166✔
123	except = 0;	166✔
124	for (;;) {
125	const void* tVal = NULL;	166✔
126	int32_t tLen = 0;	166✔
127	if (tdbTbcGet(pReader->pCur, &pKey, &kLen, &tVal, &tLen)) {	166!
128	except = 1;	166✔
129	break;	166✔
130	} else {
UNCOV 131	pVal = taosMemoryCalloc(1, tLen);	×
UNCOV 132	if (pVal == NULL) {	×
133	code = terrno;	×
134	goto _err;	×
135	}
136
UNCOV 137	memcpy(pVal, tVal, tLen);	×
UNCOV 138	vLen = tLen;	×
139	}
UNCOV 140	TAOS_UNUSED(tdbTbcMoveToNext(pReader->pCur));	×
UNCOV 141	break;	×
142	}
143	if (except == 1) {	166!
144	if (pReader->pos + 1 < taosArrayGetSize(pReader->tdbTbList)) {	166✔
145	tdbTbcClose(pReader->pCur);	83✔
146
147	pReader->pos += 1;	83✔
148	pPair = taosArrayGet(pReader->tdbTbList, pReader->pos);	83✔
149	code = tdbTbcOpen(pPair->tbl, &pReader->pCur, NULL);	83✔
150	TAOS_UNUSED(tdbTbcMoveToFirst(pReader->pCur));	83✔
151
152	goto NextTbl;	83✔
153	}
154	}
155	if (pVal == NULL \|\| vLen == 0) {	83!
156	*ppData = NULL;	83✔
157	tqDebug("vgId:%d, vnode stream-task snapshot finished read data", TD_VID(pReader->pTq->pVnode));	83!
158	return code;	83✔
159	}
UNCOV 160	*ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + vLen);	×
UNCOV 161	if (*ppData == NULL) {	×
162	code = terrno;	×
163	goto _err;	×
164	}
165
UNCOV 166	SSnapDataHdr* pHdr = (SSnapDataHdr)(ppData);	×
UNCOV 167	pHdr->type = pPair->type;	×
UNCOV 168	pHdr->size = vLen;	×
UNCOV 169	memcpy(pHdr->data, pVal, vLen);	×
UNCOV 170	taosMemoryFree(pVal);	×
171
UNCOV 172	tqDebug("vgId:%d, vnode stream-task snapshot read data vLen:%d", TD_VID(pReader->pTq->pVnode), vLen);	×
UNCOV 173	return code;	×
174
175	_err:	×
176	tqError("vgId:%d, vnode stream-task snapshot read data failed since %s", TD_VID(pReader->pTq->pVnode),	×
177	tstrerror(code));
178	return code;	×
179	}
180
181	// STqSnapWriter ========================================
182	struct SStreamTaskWriter {
183	STQ* pTq;
184	int64_t sver;
185	int64_t ever;
186	};
187
UNCOV 188	int32_t streamTaskSnapWriterOpen(STQ* pTq, int64_t sver, int64_t ever, SStreamTaskWriter** ppWriter) {	×
UNCOV 189	int32_t code = 0;	×
190	SStreamTaskWriter* pWriter;
191
192	// alloc
UNCOV 193	pWriter = (SStreamTaskWriter)taosMemoryCalloc(1, sizeof(pWriter));	×
UNCOV 194	if (pWriter == NULL) {	×
195	TAOS_CHECK_RETURN(terrno);	×
196	}
UNCOV 197	pWriter->pTq = pTq;	×
UNCOV 198	pWriter->sver = sver;	×
UNCOV 199	pWriter->ever = ever;	×
200
UNCOV 201	*ppWriter = pWriter;	×
UNCOV 202	tqDebug("vgId:%d, vnode stream-task snapshot writer opened", TD_VID(pTq->pVnode));	×
UNCOV 203	return code;	×
204	}
205
UNCOV 206	int32_t streamTaskSnapWriterClose(SStreamTaskWriter* pWriter, int8_t rollback, int8_t loadTask) {	×
UNCOV 207	int32_t code = 0;	×
UNCOV 208	STQ* pTq = pWriter->pTq;	×
209
UNCOV 210	streamMetaWLock(pTq->pStreamMeta);	×
UNCOV 211	tqDebug("vgId:%d, vnode stream-task snapshot writer closed", TD_VID(pTq->pVnode));	×
UNCOV 212	if (rollback) {	×
213	TAOS_UNUSED(tdbAbort(pTq->pStreamMeta->db, pTq->pStreamMeta->txn));	×
214	} else {
UNCOV 215	code = tdbCommit(pTq->pStreamMeta->db, pTq->pStreamMeta->txn);	×
UNCOV 216	if (code) goto _err;	×
UNCOV 217	code = tdbPostCommit(pTq->pStreamMeta->db, pTq->pStreamMeta->txn);	×
UNCOV 218	if (code) goto _err;	×
219	}
220
UNCOV 221	if ((code = tdbBegin(pTq->pStreamMeta->db, &pTq->pStreamMeta->txn, tdbDefaultMalloc, tdbDefaultFree, NULL, 0)) < 0) {	×
222	taosMemoryFree(pWriter);	×
223	goto _err;	×
224	}
UNCOV 225	streamMetaWUnLock(pTq->pStreamMeta);	×
UNCOV 226	taosMemoryFree(pWriter);	×
227
UNCOV 228	if (loadTask == 1) {	×
UNCOV 229	streamMetaLoadAllTasks(pTq->pStreamMeta);	×
230	}
UNCOV 231	return code;	×
232
233	_err:	×
234	tqError("vgId:%d, vnode stream-task snapshot writer failed to close since %s", TD_VID(pWriter->pTq->pVnode),	×
235	tstrerror(code));
236	streamMetaWUnLock(pTq->pStreamMeta);	×
237	return code;	×
238	}
239
UNCOV 240	int32_t streamTaskSnapWrite(SStreamTaskWriter* pWriter, uint8_t* pData, uint32_t nData) {	×
UNCOV 241	int32_t code = 0;	×
UNCOV 242	STQ* pTq = pWriter->pTq;	×
UNCOV 243	SSnapDataHdr* pHdr = (SSnapDataHdr*)pData;	×
UNCOV 244	if (pHdr->type == SNAP_DATA_STREAM_TASK) {	×
UNCOV 245	STaskId taskId = {0};	×
246
247	SDecoder decoder;
UNCOV 248	tDecoderInit(&decoder, (uint8_t*)pData + sizeof(SSnapDataHdr), nData - sizeof(SSnapDataHdr));	×
UNCOV 249	code = tDecodeStreamTaskId(&decoder, &taskId);	×
UNCOV 250	if (code < 0) {	×
251	tDecoderClear(&decoder);	×
252	goto _err;	×
253	}
UNCOV 254	tDecoderClear(&decoder);	×
255
UNCOV 256	int64_t key[2] = {taskId.streamId, taskId.taskId};	×
UNCOV 257	streamMetaWLock(pTq->pStreamMeta);	×
UNCOV 258	if ((code =	×
UNCOV 259	tdbTbUpsert(pTq->pStreamMeta->pTaskDb, key, sizeof(int64_t) << 1, (uint8_t*)pData + sizeof(SSnapDataHdr),	×
UNCOV 260	nData - sizeof(SSnapDataHdr), pTq->pStreamMeta->txn)) < 0) {	×
261	streamMetaWUnLock(pTq->pStreamMeta);	×
262	return code;	×
263	}
UNCOV 264	streamMetaWUnLock(pTq->pStreamMeta);	×
265	} else if (pHdr->type == SNAP_DATA_STREAM_TASK_CHECKPOINT) {	×
266	// do nothing
267	}
UNCOV 268	tqDebug("vgId:%d, vnode stream-task snapshot write", TD_VID(pTq->pVnode));	×
269
UNCOV 270	return code;	×
271
272	_err:	×
273	tqError("vgId:%d, vnode stream-task snapshot failed to write since %s", TD_VID(pTq->pVnode), tstrerror(code));	×
274	return code;	×
275	}

taosdata / TDengine / #3526

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous