#4803

Committed 17 Oct 2025 04:02AM UTC coverage: 61.147% (-0.05%) from 61.198%

Build # #4803

Build Type

push

travis-ci

Committed by

web-flow

Commit Message

Merge 323e6dde0 into 1afedf5da

Run Details

155455 of 324369 branches covered (47.93%)

Branch coverage included in aggregate %.

1 of 1 new or added line in 1 file covered. (100.0%)

2521 existing lines in 113 files now uncovered.

207702 of 269535 relevant lines covered (77.06%)

126661557.72 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

60.28

/source/libs/sync/src/syncAppendEntries.c

/*
 * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
 *
 * This program is free software: you can use, redistribute, and/or modify
 * it under the terms of the GNU Affero General Public License, version 3
 * or later ("AGPL"), as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#define _DEFAULT_SOURCE
#include "syncAppendEntries.h"
#include "syncPipeline.h"
#include "syncMessage.h"
#include "syncRaftLog.h"
#include "syncRaftStore.h"
#include "syncReplication.h"
#include "syncUtil.h"
#include "syncCommit.h"

// TLA+ Spec
// HandleAppendEntriesRequest(i, j, m) ==
//    LET logOk == \/ m.mprevLogIndex = 0
//                 \/ /\ m.mprevLogIndex > 0
//                    /\ m.mprevLogIndex <= Len(log[i])
//                    /\ m.mprevLogTerm = log[i][m.mprevLogIndex].term
//    IN /\ m.mterm <= currentTerm[i]
//       /\ \/ /\ \* reject request
//                \/ m.mterm < currentTerm[i]
//                \/ /\ m.mterm = currentTerm[i]
//                   /\ state[i] = Follower
//                   /\ \lnot logOk
//             /\ Reply([mtype           |-> AppendEntriesResponse,
//                       mterm           |-> currentTerm[i],
//                       msuccess        |-> FALSE,
//                       mmatchIndex     |-> 0,
//                       msource         |-> i,
//                       mdest           |-> j],
//                       m)
//             /\ UNCHANGED <<serverVars, logVars>>
//          \/ \* return to follower state
//             /\ m.mterm = currentTerm[i]
//             /\ state[i] = Candidate
//             /\ state' = [state EXCEPT ![i] = Follower]
//             /\ UNCHANGED <<currentTerm, votedFor, logVars, messages>>
//          \/ \* accept request
//             /\ m.mterm = currentTerm[i]
//             /\ state[i] = Follower
//             /\ logOk
//             /\ LET index == m.mprevLogIndex + 1
//                IN \/ \* already done with request
//                       /\ \/ m.mentries = << >>
//                          \/ /\ m.mentries /= << >>
//                             /\ Len(log[i]) >= index
//                             /\ log[i][index].term = m.mentries[1].term
//                          \* This could make our commitIndex decrease (for
//                          \* example if we process an old, duplicated request),
//                          \* but that doesn't really affect anything.
//                       /\ commitIndex' = [commitIndex EXCEPT ![i] =
//                                              m.mcommitIndex]
//                       /\ Reply([mtype           |-> AppendEntriesResponse,
//                                 mterm           |-> currentTerm[i],
//                                 msuccess        |-> TRUE,
//                                 mmatchIndex     |-> m.mprevLogIndex +
//                                                     Len(m.mentries),
//                                 msource         |-> i,
//                                 mdest           |-> j],
//                                 m)
//                       /\ UNCHANGED <<serverVars, log>>
//                   \/ \* conflict: remove 1 entry
//                       /\ m.mentries /= << >>
//                       /\ Len(log[i]) >= index
//                       /\ log[i][index].term /= m.mentries[1].term
//                       /\ LET new == [index2 \in 1..(Len(log[i]) - 1) |->
//                                          log[i][index2]]
//                          IN log' = [log EXCEPT ![i] = new]
//                       /\ UNCHANGED <<serverVars, commitIndex, messages>>
//                   \/ \* no conflict: append entry
//                       /\ m.mentries /= << >>
//                       /\ Len(log[i]) = m.mprevLogIndex
//                       /\ log' = [log EXCEPT ![i] =
//                                      Append(log[i], m.mentries[1])]
//                       /\ UNCHANGED <<serverVars, commitIndex, messages>>
//       /\ UNCHANGED <<candidateVars, leaderVars>>
//

int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {
  SyncAppendEntries* pMsg = pRpcMsg->pCont;
  SRpcMsg            rpcRsp = {0};
  bool               accepted = false;
  SSyncRaftEntry*    pEntry = NULL;
  bool               resetElect = false;

  // if already drop replica, do not process
  if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) {
    syncLogRecvAppendEntries(ths, pMsg, "not in my config", &pRpcMsg->info.traceId);
    goto _IGNORE;
  }

  int64_t nRef = atomic_add_fetch_64(&ths->recvCount, 1);
  if (nRef <= 0) {
    sError("vgId:%d, recv count is %" PRId64, ths->vgId, nRef);
  }

  int32_t code = syncBuildAppendEntriesReply(&rpcRsp, ths->vgId);
  if (code != 0) {
    syncLogRecvAppendEntries(ths, pMsg, "build rsp error", &pRpcMsg->info.traceId);
    goto _IGNORE;
  }
  rpcRsp.info.traceId = pRpcMsg->info.traceId;

  SyncAppendEntriesReply* pReply = rpcRsp.pCont;
  // prepare response msg
  pReply->srcId = ths->myRaftId;
  pReply->destId = pMsg->srcId;
  pReply->term = raftStoreGetTerm(ths);
  pReply->success = false;
  pReply->matchIndex = SYNC_INDEX_INVALID;
  pReply->lastSendIndex = pMsg->prevLogIndex + 1;
  pReply->startTime = ths->startTime;

  if (pMsg->term < raftStoreGetTerm(ths)) {
    goto _SEND_RESPONSE;
  }

  if (pMsg->term > raftStoreGetTerm(ths)) {
    pReply->term = pMsg->term;
  }

  if(ths->raftCfg.cfg.nodeInfo[ths->raftCfg.cfg.myIndex].nodeRole != TAOS_SYNC_ROLE_LEARNER){
    syncNodeStepDown(ths, pMsg->term, pMsg->srcId, "appendEntry");
    resetElect = true;
  }

  if (pMsg->dataLen < sizeof(SSyncRaftEntry)) {
    sError("vgId:%d, incomplete append entries received. prev index:%" PRId64 ", term:%" PRId64 ", datalen:%d",
           ths->vgId, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen);
    goto _IGNORE;
  }

  pEntry = syncEntryBuildFromAppendEntries(pMsg);
  if (pEntry == NULL) {
    sError("vgId:%d, failed to get raft entry from append entries since %s", ths->vgId, terrstr());
    goto _IGNORE;
  }

  if (pMsg->prevLogIndex + 1 != pEntry->index || pEntry->term < 0) {
    sError("vgId:%d, invalid previous log index in msg. index:%" PRId64 ",  term:%" PRId64 ", prevLogIndex:%" PRId64
           ", prevLogTerm:%" PRId64,
           ths->vgId, pEntry->index, pEntry->term, pMsg->prevLogIndex, pMsg->prevLogTerm);
    goto _IGNORE;
  }

  sGDebug(&pRpcMsg->info.traceId,
          "vgId:%d, index:%" PRId64 ", recv append entries msg, term:%" PRId64 ", preLogIndex:%" PRId64
          ", prevLogTerm:%" PRId64 " commitIndex:%" PRId64 " entryterm:%" PRId64,
          pMsg->vgId, pMsg->prevLogIndex + 1, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex,
          pEntry->term);

  if (ths->fsmState == SYNC_FSM_STATE_INCOMPLETE) {
    pReply->fsmState = ths->fsmState;
    sWarn("vgId:%d, unable to accept, due to incomplete fsm state. index:%" PRId64, ths->vgId, pEntry->index);
    syncEntryDestroy(pEntry);
    goto _SEND_RESPONSE;
  }

  // accept
  if (syncLogBufferAccept(ths->pLogBuf, ths, pEntry, pMsg->prevLogTerm) < 0) {
    goto _SEND_RESPONSE;
  }
  accepted = true;

_SEND_RESPONSE:
  pEntry = NULL;
  pReply->matchIndex = syncLogBufferProceed(ths->pLogBuf, ths, &pReply->lastMatchTerm, "OnAppn", pRpcMsg);
  bool matched = (pReply->matchIndex >= pReply->lastSendIndex);
  if (accepted && matched) {
    pReply->success = true;
    // update commit index only after matching
    SyncIndex returnIndex = syncNodeUpdateCommitIndex(ths, TMIN(pMsg->commitIndex, pReply->lastSendIndex));
    sGDebug(&rpcRsp.info.traceId, "vgId:%d, index:%" PRId64 ", last commit index:%" PRId64, ths->vgId,
           pMsg->prevLogIndex + 1, returnIndex);
  }

  TRACE_SET_MSGID(&(rpcRsp.info.traceId), tGenIdPI64());
  {
    sGDebug(&rpcRsp.info.traceId,
            "vgId:%d, index:%" PRId64 ", send append entries reply, matchIndex:%" PRId64 " term:%" PRId64
            " lastSendIndex:%" PRId64 " to dest addr:0x%016" PRIx64,
            ths->vgId, pMsg->prevLogIndex + 1, pReply->matchIndex, pReply->term, pReply->lastSendIndex,
            pReply->destId.addr);
  }
  // ack, i.e. send response
  TAOS_CHECK_RETURN(syncNodeSendMsgById(&pReply->destId, ths, &rpcRsp));

  // commit index, i.e. leader notice me
  if (ths->fsmState != SYNC_FSM_STATE_INCOMPLETE &&
      syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex, &pRpcMsg->info.traceId, "sync-append-entries") < 0) {
    sGError(&pRpcMsg->info.traceId, "vgId:%d, failed to commit raft fsm log since %s", ths->vgId, terrstr());
  }

  if (resetElect) syncNodeResetElectTimer(ths);
  return 0;

_IGNORE:
  rpcFreeCont(rpcRsp.pCont);
  syncEntryDestroy(pEntry);
  return 0;
}

1	/*
2	* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
3	*
4	* This program is free software: you can use, redistribute, and/or modify
5	* it under the terms of the GNU Affero General Public License, version 3
6	* or later ("AGPL"), as published by the Free Software Foundation.
7	*
8	* This program is distributed in the hope that it will be useful, but WITHOUT
9	* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10	* FITNESS FOR A PARTICULAR PURPOSE.
11	*
12	* You should have received a copy of the GNU Affero General Public License
13	* along with this program. If not, see <http://www.gnu.org/licenses/>.
14	*/
15
16	#define _DEFAULT_SOURCE
17	#include "syncAppendEntries.h"
18	#include "syncPipeline.h"
19	#include "syncMessage.h"
20	#include "syncRaftLog.h"
21	#include "syncRaftStore.h"
22	#include "syncReplication.h"
23	#include "syncUtil.h"
24	#include "syncCommit.h"
25
26	// TLA+ Spec
27	// HandleAppendEntriesRequest(i, j, m) ==
28	// LET logOk == \/ m.mprevLogIndex = 0
29	// \/ /\ m.mprevLogIndex > 0
30	// /\ m.mprevLogIndex <= Len(log[i])
31	// /\ m.mprevLogTerm = log[i][m.mprevLogIndex].term
32	// IN /\ m.mterm <= currentTerm[i]
33	// /\ \/ /\ \* reject request
34	// \/ m.mterm < currentTerm[i]
35	// \/ /\ m.mterm = currentTerm[i]
36	// /\ state[i] = Follower
37	// /\ \lnot logOk
38	// /\ Reply([mtype \|-> AppendEntriesResponse,
39	// mterm \|-> currentTerm[i],
40	// msuccess \|-> FALSE,
41	// mmatchIndex \|-> 0,
42	// msource \|-> i,
43	// mdest \|-> j],
44	// m)
45	// /\ UNCHANGED <<serverVars, logVars>>
46	// \/ \* return to follower state
47	// /\ m.mterm = currentTerm[i]
48	// /\ state[i] = Candidate
49	// /\ state' = [state EXCEPT ![i] = Follower]
50	// /\ UNCHANGED <<currentTerm, votedFor, logVars, messages>>
51	// \/ \* accept request
52	// /\ m.mterm = currentTerm[i]
53	// /\ state[i] = Follower
54	// /\ logOk
55	// /\ LET index == m.mprevLogIndex + 1
56	// IN \/ \* already done with request
57	// /\ \/ m.mentries = << >>
58	// \/ /\ m.mentries /= << >>
59	// /\ Len(log[i]) >= index
60	// /\ log[i][index].term = m.mentries[1].term
61	// \* This could make our commitIndex decrease (for
62	// \* example if we process an old, duplicated request),
63	// \* but that doesn't really affect anything.
64	// /\ commitIndex' = [commitIndex EXCEPT ![i] =
65	// m.mcommitIndex]
66	// /\ Reply([mtype \|-> AppendEntriesResponse,
67	// mterm \|-> currentTerm[i],
68	// msuccess \|-> TRUE,
69	// mmatchIndex \|-> m.mprevLogIndex +
70	// Len(m.mentries),
71	// msource \|-> i,
72	// mdest \|-> j],
73	// m)
74	// /\ UNCHANGED <<serverVars, log>>
75	// \/ \* conflict: remove 1 entry
76	// /\ m.mentries /= << >>
77	// /\ Len(log[i]) >= index
78	// /\ log[i][index].term /= m.mentries[1].term
79	// /\ LET new == [index2 \in 1..(Len(log[i]) - 1) \|->
80	// log[i][index2]]
81	// IN log' = [log EXCEPT ![i] = new]
82	// /\ UNCHANGED <<serverVars, commitIndex, messages>>
83	// \/ \* no conflict: append entry
84	// /\ m.mentries /= << >>
85	// /\ Len(log[i]) = m.mprevLogIndex
86	// /\ log' = [log EXCEPT ![i] =
87	// Append(log[i], m.mentries[1])]
88	// /\ UNCHANGED <<serverVars, commitIndex, messages>>
89	// /\ UNCHANGED <<candidateVars, leaderVars>>
90	//
91
92	int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) {	31,162,985✔
93	SyncAppendEntries* pMsg = pRpcMsg->pCont;	31,162,985✔
94	SRpcMsg rpcRsp = {0};	31,162,985✔
95	bool accepted = false;	31,162,985✔
96	SSyncRaftEntry* pEntry = NULL;	31,162,985✔
97	bool resetElect = false;	31,162,985✔
98
99	// if already drop replica, do not process
100	if (!syncNodeInRaftGroup(ths, &(pMsg->srcId))) {	31,162,985!
101	syncLogRecvAppendEntries(ths, pMsg, "not in my config", &pRpcMsg->info.traceId);	×
102	goto _IGNORE;	×
103	}
104
105	int64_t nRef = atomic_add_fetch_64(&ths->recvCount, 1);	31,162,985✔
106	if (nRef <= 0) {	31,162,985!
107	sError("vgId:%d, recv count is %" PRId64, ths->vgId, nRef);	×
108	}
109
110	int32_t code = syncBuildAppendEntriesReply(&rpcRsp, ths->vgId);	31,162,985✔
111	if (code != 0) {	31,162,985!
112	syncLogRecvAppendEntries(ths, pMsg, "build rsp error", &pRpcMsg->info.traceId);	×
113	goto _IGNORE;	×
114	}
115	rpcRsp.info.traceId = pRpcMsg->info.traceId;	31,162,985✔
116
117	SyncAppendEntriesReply* pReply = rpcRsp.pCont;	31,162,985✔
118	// prepare response msg
119	pReply->srcId = ths->myRaftId;	31,162,985✔
120	pReply->destId = pMsg->srcId;	31,162,985✔
121	pReply->term = raftStoreGetTerm(ths);	31,162,985✔
122	pReply->success = false;	31,162,985✔
123	pReply->matchIndex = SYNC_INDEX_INVALID;	31,162,985✔
124	pReply->lastSendIndex = pMsg->prevLogIndex + 1;	31,162,985✔
125	pReply->startTime = ths->startTime;	31,162,985✔
126
127	if (pMsg->term < raftStoreGetTerm(ths)) {	31,162,985✔
128	goto _SEND_RESPONSE;	997✔
129	}
130
131	if (pMsg->term > raftStoreGetTerm(ths)) {	31,161,988✔
132	pReply->term = pMsg->term;	291,927✔
133	}
134
135	if(ths->raftCfg.cfg.nodeInfo[ths->raftCfg.cfg.myIndex].nodeRole != TAOS_SYNC_ROLE_LEARNER){	31,161,988✔
136	syncNodeStepDown(ths, pMsg->term, pMsg->srcId, "appendEntry");	26,254,299✔
137	resetElect = true;	26,254,108✔
138	}
139
140	if (pMsg->dataLen < sizeof(SSyncRaftEntry)) {	31,161,797!
141	sError("vgId:%d, incomplete append entries received. prev index:%" PRId64 ", term:%" PRId64 ", datalen:%d",	×
142	ths->vgId, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen);
143	goto _IGNORE;	×
144	}
145
146	pEntry = syncEntryBuildFromAppendEntries(pMsg);	31,161,988✔
147	if (pEntry == NULL) {	31,161,797!
148	sError("vgId:%d, failed to get raft entry from append entries since %s", ths->vgId, terrstr());	×
149	goto _IGNORE;	×
150	}
151
152	if (pMsg->prevLogIndex + 1 != pEntry->index \|\| pEntry->term < 0) {	31,161,797!
UNCOV 153	sError("vgId:%d, invalid previous log index in msg. index:%" PRId64 ", term:%" PRId64 ", prevLogIndex:%" PRId64	×
154	", prevLogTerm:%" PRId64,
155	ths->vgId, pEntry->index, pEntry->term, pMsg->prevLogIndex, pMsg->prevLogTerm);
156	goto _IGNORE;	×
157	}
158
159	sGDebug(&pRpcMsg->info.traceId,	31,161,797!
160	"vgId:%d, index:%" PRId64 ", recv append entries msg, term:%" PRId64 ", preLogIndex:%" PRId64
161	", prevLogTerm:%" PRId64 " commitIndex:%" PRId64 " entryterm:%" PRId64,
162	pMsg->vgId, pMsg->prevLogIndex + 1, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex,
163	pEntry->term);
164
165	if (ths->fsmState == SYNC_FSM_STATE_INCOMPLETE) {	31,161,797!
166	pReply->fsmState = ths->fsmState;	×
167	sWarn("vgId:%d, unable to accept, due to incomplete fsm state. index:%" PRId64, ths->vgId, pEntry->index);	×
168	syncEntryDestroy(pEntry);	×
169	goto _SEND_RESPONSE;	×
170	}
171
172	// accept
173	if (syncLogBufferAccept(ths->pLogBuf, ths, pEntry, pMsg->prevLogTerm) < 0) {	31,161,797✔
174	goto _SEND_RESPONSE;	257,953✔
175	}
176	accepted = true;	30,903,866✔
177
178	_SEND_RESPONSE:	31,162,816✔
179	pEntry = NULL;	31,162,816✔
180	pReply->matchIndex = syncLogBufferProceed(ths->pLogBuf, ths, &pReply->lastMatchTerm, "OnAppn", pRpcMsg);	31,162,816✔
181	bool matched = (pReply->matchIndex >= pReply->lastSendIndex);	31,161,673✔
182	if (accepted && matched) {	31,161,108✔
183	pReply->success = true;	29,094,119✔
184	// update commit index only after matching
185	SyncIndex returnIndex = syncNodeUpdateCommitIndex(ths, TMIN(pMsg->commitIndex, pReply->lastSendIndex));	29,095,456✔
186	sGDebug(&rpcRsp.info.traceId, "vgId:%d, index:%" PRId64 ", last commit index:%" PRId64, ths->vgId,	29,095,681!
187	pMsg->prevLogIndex + 1, returnIndex);
188	}
189
190	TRACE_SET_MSGID(&(rpcRsp.info.traceId), tGenIdPI64());	31,162,670✔
191	{
192	sGDebug(&rpcRsp.info.traceId,	31,160,073!
193	"vgId:%d, index:%" PRId64 ", send append entries reply, matchIndex:%" PRId64 " term:%" PRId64
194	" lastSendIndex:%" PRId64 " to dest addr:0x%016" PRIx64,
195	ths->vgId, pMsg->prevLogIndex + 1, pReply->matchIndex, pReply->term, pReply->lastSendIndex,
196	pReply->destId.addr);
197	}
198	// ack, i.e. send response
199	TAOS_CHECK_RETURN(syncNodeSendMsgById(&pReply->destId, ths, &rpcRsp));	31,160,073!
200
201	// commit index, i.e. leader notice me
202	if (ths->fsmState != SYNC_FSM_STATE_INCOMPLETE &&	62,324,549!
203	syncLogBufferCommit(ths->pLogBuf, ths, ths->commitIndex, &pRpcMsg->info.traceId, "sync-append-entries") < 0) {	31,162,137✔
204	sGError(&pRpcMsg->info.traceId, "vgId:%d, failed to commit raft fsm log since %s", ths->vgId, terrstr());	922!
205	}
206
207	if (resetElect) syncNodeResetElectTimer(ths);	31,161,967✔
208	return 0;	31,162,985✔
209
210	_IGNORE:	×
211	rpcFreeCont(rpcRsp.pCont);	×
212	syncEntryDestroy(pEntry);	×
213	return 0;	×
214	}

taosdata / TDengine / #4803

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous