#4643

Committed 01 Aug 2025 02:04AM UTC coverage: 60.992% (-0.1%) from 61.105%

Build # #4643

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

merge: from 3.3.6 to main branch (#32389)

* enh: grant support for data source ORC (#32378)

* fix: wrong col_id in ins_columns (#32358)

* test: remove un checked case (#32388)

* fix(parser): subquery use last_row can't found the colname (#32353)

* add col_id info in ins_columns

* fix: test case

---------

Co-authored-by: Kaili Xu <klxu@taosdata.com>
Co-authored-by: Tony Zhang <34825804+Tony2h@users.noreply.github.com>
Co-authored-by: hongzhenliu <wluckyjob@gmail.com>
Co-authored-by: Tony Zhang <tonyzhang@taosdata.com>

Coverage Stats

139991 of 291593 branches covered (48.01%)

Branch coverage included in aggregate %.

10 of 10 new or added lines in 3 files covered. (100.0%)

3514 existing lines in 141 files now uncovered.

211118 of 284074 relevant lines covered (74.32%)

17514058.99 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

80.87

/source/libs/sync/src/syncReplication.c

/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#define _DEFAULT_SOURCE
#include "syncReplication.h"
#include "syncIndexMgr.h"
#include "syncPipeline.h"
#include "syncRaftEntry.h"
#include "syncRaftStore.h"
#include "syncUtil.h"

// TLA+ Spec
// AppendEntries(i, j) ==
//    /\ i /= j
//    /\ state[i] = Leader
//    /\ LET prevLogIndex == nextIndex[i][j] - 1
//           prevLogTerm == IF prevLogIndex > 0 THEN
//                              log[i][prevLogIndex].term
//                          ELSE
//                              0
//           \* Send up to 1 entry, constrained by the end of the log.
//           lastEntry == Min({Len(log[i]), nextIndex[i][j]})
//           entries == SubSeq(log[i], nextIndex[i][j], lastEntry)
//       IN Send([mtype          |-> AppendEntriesRequest,
//                mterm          |-> currentTerm[i],
//                mprevLogIndex  |-> prevLogIndex,
//                mprevLogTerm   |-> prevLogTerm,
//                mentries       |-> entries,
//                \* mlog is used as a history variable for the proof.
//                \* It would not exist in a real implementation.
//                mlog           |-> log[i],
//                mcommitIndex   |-> Min({commitIndex[i], lastEntry}),
//                msource        |-> i,
//                mdest          |-> j])
//    /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>

int32_t syncNodeReplicateReset(SSyncNode* pNode, SRaftId* pDestId) {
  SSyncLogBuffer* pBuf = pNode->pLogBuf;
  (void)taosThreadMutexLock(&pBuf->mutex);
  SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(pNode, pDestId);
  syncLogReplReset(pMgr);
  (void)taosThreadMutexUnlock(&pBuf->mutex);

  TAOS_RETURN(TSDB_CODE_SUCCESS);
}

int32_t syncNodeReplicate(SSyncNode* pNode) {
  SSyncLogBuffer* pBuf = pNode->pLogBuf;
  int32_t         ret = 0;
  if ((ret = taosThreadMutexTryLock(&pBuf->mutex)) != 0) {
    sWarn("vgId:%d, failed to get log buffer mutex since %d", pNode->vgId, ret);
    return TSDB_CODE_FAIL_GET_LOCK;
  }
  ret = syncNodeReplicateWithoutLock(pNode);
  (void)taosThreadMutexUnlock(&pBuf->mutex);

  TAOS_RETURN(ret);
}

int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode) {
  if ((pNode->state != TAOS_SYNC_STATE_LEADER && pNode->state != TAOS_SYNC_STATE_ASSIGNED_LEADER) ||
      pNode->raftCfg.cfg.totalReplicaNum == 1) {
    TAOS_RETURN(TSDB_CODE_SUCCESS);
  }
  for (int32_t i = 0; i < pNode->totalReplicaNum; i++) {
    if (syncUtilSameId(&pNode->replicasId[i], &pNode->myRaftId)) {
      continue;
    }
    SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i];
    int32_t          ret = 0;
    if ((ret = syncLogReplStart(pMgr, pNode)) != 0) {
      sWarn("vgId:%d, failed to start log replication to dnode:%d since %s", pNode->vgId, DID(&(pNode->replicasId[i])),
            tstrerror(ret));
    }
  }

  TAOS_RETURN(TSDB_CODE_SUCCESS);
}

int32_t syncNodeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg) {
  SyncAppendEntries* pMsg = pRpcMsg->pCont;
  pMsg->destId = *destRaftId;
  TAOS_CHECK_RETURN(syncNodeSendMsgById(destRaftId, pSyncNode, pRpcMsg));

  int64_t nRef = 0;
  if (pSyncNode != NULL) {
    nRef = atomic_add_fetch_64(&pSyncNode->sendCount, 1);
    if (nRef <= 0) {
      sError("vgId:%d, send count is %" PRId64, pSyncNode->vgId, nRef);
    }
  }

  SSyncLogReplMgr* mgr = syncNodeGetLogReplMgr(pSyncNode, (SRaftId*)destRaftId);
  if (mgr != NULL) {
    nRef = atomic_add_fetch_64(&mgr->sendCount, 1);
    if (nRef <= 0) {
      sError("vgId:%d, send count is %" PRId64, pSyncNode->vgId, nRef);
    }
  }

  TAOS_RETURN(TSDB_CODE_SUCCESS);
}

int32_t syncNodeSendHeartbeat(SSyncNode* pSyncNode, const SRaftId* destId, SRpcMsg* pMsg) {
  SRaftId destIdTmp = *destId;
  TAOS_CHECK_RETURN(syncNodeSendMsgById(destId, pSyncNode, pMsg));

  int64_t tsMs = taosGetTimestampMs();
  syncIndexMgrSetSentTime(pSyncNode->pMatchIndex, &destIdTmp, tsMs);

  return TSDB_CODE_SUCCESS;
}

int32_t syncNodeHeartbeatPeers(SSyncNode* pSyncNode) {
  int64_t ts = taosGetTimestampMs();
  for (int32_t i = 0; i < pSyncNode->peersNum; ++i) {
    SRpcMsg rpcMsg = {0};
    if (syncBuildHeartbeat(&rpcMsg, pSyncNode->vgId) != 0) {
      sError("vgId:%d, build sync-heartbeat error", pSyncNode->vgId);
      continue;
    }

    SyncHeartbeat* pSyncMsg = rpcMsg.pCont;
    pSyncMsg->srcId = pSyncNode->myRaftId;
    pSyncMsg->destId = pSyncNode->peersId[i];
    pSyncMsg->term = raftStoreGetTerm(pSyncNode);
    pSyncMsg->commitIndex = pSyncNode->commitIndex;
    pSyncMsg->minMatchIndex = syncMinMatchIndex(pSyncNode);
    pSyncMsg->privateTerm = 0;
    pSyncMsg->timeStamp = ts;

    // send msg
    TRACE_SET_MSGID(&(rpcMsg.info.traceId), tGenIdPI64());
    TRACE_SET_ROOTID(&(rpcMsg.info.traceId), tGenIdPI64());
    syncLogSendHeartbeat(pSyncNode, pSyncMsg, true, 0, 0, &(rpcMsg.info.traceId));
    int32_t ret = syncNodeSendHeartbeat(pSyncNode, &pSyncMsg->destId, &rpcMsg);
    if (ret != 0) {
      sError("vgId:%d, failed to send sync-heartbeat since %s", pSyncNode->vgId, tstrerror(ret));
    }
  }

  TAOS_RETURN(TSDB_CODE_SUCCESS);
}

1	/*
2	* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3	*
4	* This program is free software: you can use, redistribute, and/or modify
5	* it under the terms of the GNU Affero General Public License, version 3
6	* or later ("AGPL"), as published by the Free Software Foundation.
7	*
8	* This program is distributed in the hope that it will be useful, but WITHOUT
9	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10	* FITNESS FOR A PARTICULAR PURPOSE.
11	*
12	* You should have received a copy of the GNU Affero General Public License
13	* along with this program. If not, see <http://www.gnu.org/licenses/>.
14	*/
15
16	#define _DEFAULT_SOURCE
17	#include "syncReplication.h"
18	#include "syncIndexMgr.h"
19	#include "syncPipeline.h"
20	#include "syncRaftEntry.h"
21	#include "syncRaftStore.h"
22	#include "syncUtil.h"
23
24	// TLA+ Spec
25	// AppendEntries(i, j) ==
26	// /\ i /= j
27	// /\ state[i] = Leader
28	// /\ LET prevLogIndex == nextIndex[i][j] - 1
29	// prevLogTerm == IF prevLogIndex > 0 THEN
30	// log[i][prevLogIndex].term
31	// ELSE
32	// 0
33	// \* Send up to 1 entry, constrained by the end of the log.
34	// lastEntry == Min({Len(log[i]), nextIndex[i][j]})
35	// entries == SubSeq(log[i], nextIndex[i][j], lastEntry)
36	// IN Send([mtype \|-> AppendEntriesRequest,
37	// mterm \|-> currentTerm[i],
38	// mprevLogIndex \|-> prevLogIndex,
39	// mprevLogTerm \|-> prevLogTerm,
40	// mentries \|-> entries,
41	// \* mlog is used as a history variable for the proof.
42	// \* It would not exist in a real implementation.
43	// mlog \|-> log[i],
44	// mcommitIndex \|-> Min({commitIndex[i], lastEntry}),
45	// msource \|-> i,
46	// mdest \|-> j])
47	// /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
48
49	int32_t syncNodeReplicateReset(SSyncNode* pNode, SRaftId* pDestId) {	115✔
50	SSyncLogBuffer* pBuf = pNode->pLogBuf;	115✔
51	(void)taosThreadMutexLock(&pBuf->mutex);	115✔
52	SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(pNode, pDestId);	115✔
53	syncLogReplReset(pMgr);	115✔
54	(void)taosThreadMutexUnlock(&pBuf->mutex);	115✔
55
56	TAOS_RETURN(TSDB_CODE_SUCCESS);	115✔
57	}
58
59	int32_t syncNodeReplicate(SSyncNode* pNode) {	251,929✔
60	SSyncLogBuffer* pBuf = pNode->pLogBuf;	251,929✔
61	int32_t ret = 0;	251,929✔
62	if ((ret = taosThreadMutexTryLock(&pBuf->mutex)) != 0) {	251,929✔
63	sWarn("vgId:%d, failed to get log buffer mutex since %d", pNode->vgId, ret);	80!
64	return TSDB_CODE_FAIL_GET_LOCK;	80✔
65	}
66	ret = syncNodeReplicateWithoutLock(pNode);	252,091✔
67	(void)taosThreadMutexUnlock(&pBuf->mutex);	252,023✔
68
69	TAOS_RETURN(ret);	252,362✔
70	}
71
72	int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode) {	14,035,848✔
73	if ((pNode->state != TAOS_SYNC_STATE_LEADER && pNode->state != TAOS_SYNC_STATE_ASSIGNED_LEADER) \|\|	14,035,848✔
74	pNode->raftCfg.cfg.totalReplicaNum == 1) {	11,149,987✔
75	TAOS_RETURN(TSDB_CODE_SUCCESS);	13,586,824✔
76	}
77	for (int32_t i = 0; i < pNode->totalReplicaNum; i++) {	1,534,690✔
78	if (syncUtilSameId(&pNode->replicasId[i], &pNode->myRaftId)) {	1,085,669✔
79	continue;	449,024✔
80	}
81	SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i];	636,644✔
82	int32_t ret = 0;	636,644✔
83	if ((ret = syncLogReplStart(pMgr, pNode)) != 0) {	636,644✔
84	sWarn("vgId:%d, failed to start log replication to dnode:%d since %s", pNode->vgId, DID(&(pNode->replicasId[i])),	79!
85	tstrerror(ret));
86	}
87	}
88
89	TAOS_RETURN(TSDB_CODE_SUCCESS);	449,021✔
90	}
91
92	int32_t syncNodeSendAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, SRpcMsg* pRpcMsg) {	2,913,210✔
93	SyncAppendEntries* pMsg = pRpcMsg->pCont;	2,913,210✔
94	pMsg->destId = *destRaftId;	2,913,210✔
95	TAOS_CHECK_RETURN(syncNodeSendMsgById(destRaftId, pSyncNode, pRpcMsg));	2,913,210!
96
97	int64_t nRef = 0;	2,913,211✔
98	if (pSyncNode != NULL) {	2,913,211!
99	nRef = atomic_add_fetch_64(&pSyncNode->sendCount, 1);	2,913,211✔
100	if (nRef <= 0) {	2,913,211!
UNCOV 101	sError("vgId:%d, send count is %" PRId64, pSyncNode->vgId, nRef);	×
102	}
103	}
104
105	SSyncLogReplMgr* mgr = syncNodeGetLogReplMgr(pSyncNode, (SRaftId*)destRaftId);	2,913,211✔
106	if (mgr != NULL) {	2,913,210!
107	nRef = atomic_add_fetch_64(&mgr->sendCount, 1);	2,913,210✔
108	if (nRef <= 0) {	2,913,211!
UNCOV 109	sError("vgId:%d, send count is %" PRId64, pSyncNode->vgId, nRef);	×
110	}
111	}
112
113	TAOS_RETURN(TSDB_CODE_SUCCESS);	2,913,211✔
114	}
115
116	int32_t syncNodeSendHeartbeat(SSyncNode* pSyncNode, const SRaftId* destId, SRpcMsg* pMsg) {	65,412✔
117	SRaftId destIdTmp = *destId;	65,412✔
118	TAOS_CHECK_RETURN(syncNodeSendMsgById(destId, pSyncNode, pMsg));	65,412✔
119
120	int64_t tsMs = taosGetTimestampMs();	65,341✔
121	syncIndexMgrSetSentTime(pSyncNode->pMatchIndex, &destIdTmp, tsMs);	65,341✔
122
123	return TSDB_CODE_SUCCESS;	65,341✔
124	}
125
126	int32_t syncNodeHeartbeatPeers(SSyncNode* pSyncNode) {	15,114✔
127	int64_t ts = taosGetTimestampMs();	15,114✔
128	for (int32_t i = 0; i < pSyncNode->peersNum; ++i) {	18,578✔
129	SRpcMsg rpcMsg = {0};	3,464✔
130	if (syncBuildHeartbeat(&rpcMsg, pSyncNode->vgId) != 0) {	3,464!
131	sError("vgId:%d, build sync-heartbeat error", pSyncNode->vgId);	×
UNCOV 132	continue;	×
133	}
134
135	SyncHeartbeat* pSyncMsg = rpcMsg.pCont;	3,464✔
136	pSyncMsg->srcId = pSyncNode->myRaftId;	3,464✔
137	pSyncMsg->destId = pSyncNode->peersId[i];	3,464✔
138	pSyncMsg->term = raftStoreGetTerm(pSyncNode);	3,464✔
139	pSyncMsg->commitIndex = pSyncNode->commitIndex;	3,464✔
140	pSyncMsg->minMatchIndex = syncMinMatchIndex(pSyncNode);	3,464✔
141	pSyncMsg->privateTerm = 0;	3,464✔
142	pSyncMsg->timeStamp = ts;	3,464✔
143
144	// send msg
145	TRACE_SET_MSGID(&(rpcMsg.info.traceId), tGenIdPI64());	3,464✔
146	TRACE_SET_ROOTID(&(rpcMsg.info.traceId), tGenIdPI64());	3,464✔
147	syncLogSendHeartbeat(pSyncNode, pSyncMsg, true, 0, 0, &(rpcMsg.info.traceId));	3,464✔
148	int32_t ret = syncNodeSendHeartbeat(pSyncNode, &pSyncMsg->destId, &rpcMsg);	3,464✔
149	if (ret != 0) {	3,464!
UNCOV 150	sError("vgId:%d, failed to send sync-heartbeat since %s", pSyncNode->vgId, tstrerror(ret));	×
151	}
152	}
153
154	TAOS_RETURN(TSDB_CODE_SUCCESS);	15,114✔
155	}

taosdata / TDengine / #4643

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous