28425041198

Committed 30 Jun 2026 06:30AM UTC coverage: 90.88% (-0.3%) from 91.186%

Build # 28425041198

Build Type

push

github

Committed by

Hyshmily

Commit Message

refactor: simplify Worker cluster management and failure detection

Signed-off-by: Hyshmily <cxm8607@outlook.com>

Coverage Stats

1250 of 1437 branches covered (86.99%)

Branch coverage included in aggregate %.

10 of 21 new or added lines in 4 files covered. (47.62%)

1 existing line in 1 file now uncovered.

3543 of 3837 relevant lines covered (92.34%)

4.22 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

94.51

/common/src/main/java/io/github/hyshmily/hotkey/sharding/ClusterHealthView.java

/*
 * Copyright 2026 Hyshmily. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.github.hyshmily.hotkey.sharding;

import io.github.hyshmily.hotkey.sync.worker.WorkerHeartbeatMessage;
import io.github.hyshmily.hotkey.sync.worker.WorkerHeartbeatVerifier;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.stream.Collectors;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;

/**
 * Cluster-wide view of Worker health, maintained by consuming periodic
 * {@link WorkerHeartbeatMessage} broadcasts from all Worker nodes.
 *
 * <p>This is the central health authority for the local app instance. It tracks
 * every known Worker by ID with its epoch (restart counter), heartbeat timestamps,
 * readiness flag, decision version high-water mark, and verification failure count.
 *
 * <p><b>Health judgment:</b> The cluster is considered healthy when a strict majority
 * ({@code > knownWorkerCount / 2}) of Workers are alive (ready, not stale, and within
 * the heartbeat timeout window). When the cluster becomes unhealthy, the system may
 * enter graceful degradation mode (see ADR-0009).
 *
 * <p><b>Restart detection:</b> When a Worker's epoch in a new heartbeat exceeds the
 * stored epoch, a new {@link WorkerHealthRecord} is created — all state from the
 * previous incarnation is discarded. This prevents stale health state from surviving
 * a Worker restart.
 *
 * <p><b>Thread safety:</b> All record mutations use {@link ConcurrentHashMap#compute}
 * and {@code computeIfPresent} for atomic per-Worker updates. The
 * {@code lastAnyHeartbeatTime} is a {@code volatile} field safe for
 * concurrent read/write.
 *
 * @see WorkerHeartbeatMessage
 * @see WorkerHeartbeatVerifier
 */
@Slf4j
public class ClusterHealthView {

  /** Per-Worker health records keyed by Worker ID. Thread-safe via {@link ConcurrentHashMap}. */
  private final ConcurrentMap<String, WorkerHealthRecord> records = new ConcurrentHashMap<>();

  @Getter
  private volatile int knownWorkerCount;

  private final long heartbeatTimeoutMs;
  private final int degradeAfterFailures;

  @Setter
  private volatile int minAliveWorkers;

  @Getter
  private volatile long lastAnyHeartbeatTime;

  public ClusterHealthView(int knownWorkerCount, long heartbeatTimeoutMs, int degradeAfterFailures) {
    this.knownWorkerCount = knownWorkerCount;
    this.heartbeatTimeoutMs = heartbeatTimeoutMs;
    this.degradeAfterFailures = degradeAfterFailures;
  }

  /**
   * Updates the expected Worker count for majority-quorum health calculations.
   * <p>Called during initialization (from configuration) and dynamically
   * when the ring is reconciled from observed heartbeats.
   * <p>If the new count is 0, the cluster is always considered unhealthy.
   * If the new count is greater than the previous, the quorum threshold
   * adjusts accordingly.
   *
   * @param count the new expected Worker count; must be {@code >= 0}
   */
  public void setKnownWorkerCount(int count) {
    if (count < 0) {
      log.warn("Invalid knownWorkerCount {}, ignoring update", count);
      return;
    }
    int old = this.knownWorkerCount;
    this.knownWorkerCount = count;
    if (old == 0 && count > 0) {
      log.info("knownWorkerCount updated from 0 to {}; cluster health check now active", count);
    }
  }

  /**
   * Processes an incoming {@link WorkerHeartbeatMessage} from a Worker node and
   * updates the cluster health state accordingly.
   *
   * <p><b>New or restarted Worker:</b> If this is the first heartbeat from a Worker,
   * or if the heartbeat epoch exceeds the stored epoch (indicating a Worker restart),
   * a new {@link WorkerHealthRecord} is created with fresh timestamps and the old
   * decision watermark is discarded.
   *
   * <p><b>Known Worker:</b> Updates the last heartbeat timestamp, decision version
   * watermark (taking the max), load factor, and readiness flag. Clears any stale
   * or restarted flags.
   *
   * <p><b>Cluster recovery:</b> If the cluster was in degraded state and this heartbeat
   * brings the majority back to health, the degraded flag is automatically cleared.
   *
   * @param hb the incoming heartbeat message; must not be null
   */
  public void onHeartbeat(WorkerHeartbeatMessage hb) {
    records.compute(hb.workerId(), (id, existing) -> {
      long now = System.currentTimeMillis();

      if (existing == null || hb.epoch() > existing.epoch) {
        WorkerHealthRecord r = new WorkerHealthRecord();

        r.workerId = hb.workerId();
        r.epoch = hb.epoch();
        r.lastHeartbeatTime = now;
        r.readyToServe = hb.readyToServe();
        return r;
      }

      if (hb.epoch() < existing.epoch) {
        return existing;
      }

      existing.lastHeartbeatTime = now;
      existing.readyToServe = hb.readyToServe();
      existing.stale = false;
      return existing;
    });

    lastAnyHeartbeatTime = System.currentTimeMillis();
  }

  /**
   * Records a successful PONG response from a Worker during active verification
   * ({@link WorkerHeartbeatVerifier#verifySuspectedWorkers}).
   *
   * <p>Resets the Worker's verification failure count to zero and updates its
   * heartbeat timestamp to now, effectively restoring it to the alive set.
   *
   * @param workerId the Worker that responded with a PONG; must not be null
   */
  public void recordPong(String workerId) {
    records.computeIfPresent(workerId, (id, r) -> {
      r.lastHeartbeatTime = System.currentTimeMillis();
      r.verifyFailures = 0;
      return r;
    });
  }

  /**
   * Increments the verification failure count for a Worker and marks it stale
   * if the threshold is reached.
   *
   * <p>When the cumulative failure count reaches {@code degradeAfterFailures},
   * the Worker is marked as stale ({@code stale = true}). Stale Workers are
   * excluded from {@link #isClusterHealthy()} and {@link #getAliveWorkerIds()}
   * — they are effectively considered dead even if they were previously alive.
   *
   * <p>If the Worker ID is not present in the health view (never seen, or already
   * removed), this call is a no-op.
   *
   * @param workerId the Worker that failed active verification; must not be null
   */
  public void markVerificationFailed(String workerId) {
    records.computeIfPresent(workerId, (id, r) -> {
      r.verifyFailures++;
      if (r.verifyFailures >= degradeAfterFailures) {
        r.stale = true;
      }
      return r;
    });
  }

  /**
   * Returns the current verification failure count for a Worker.
   *
   * @param workerId the Worker to query; must not be null
   * @return the failure count, or 0 if the Worker is not tracked
   */
  public int getVerifyFailures(String workerId) {
    WorkerHealthRecord r = records.get(workerId);
    return r != null ? r.verifyFailures : 0;
  }

  /**
   * Removes the health record for a Worker that has been confirmed dead.
   *
   * @param workerId the Worker to remove; must not be null
   */
  public void removeRecord(String workerId) {
    records.remove(workerId);
  }

  /**
   * Returns whether the cluster is currently considered healthy based on majority
   * of known Workers being alive.
   *
   * <p>A Worker is considered alive when all three conditions hold:
   * <ul>
   *   <li>{@code readyToServe == true} (Worker completed initialization)</li>
   *   <li>{@code stale == false} (Worker has not exceeded verification failure threshold)</li>
   *   <li>Time since last heartbeat {@code < heartbeatTimeoutMs}</li>
   * </ul>
   *
   * <p>The cluster is healthy when the count of alive Workers is strictly greater
   * than {@code knownWorkerCount / 2} (simple majority).
   *
   * <p>When {@code knownWorkerCount == 0} (no expected count configured), the
   * cluster is considered healthy if at least one alive Worker is observed via
   * heartbeats. This provides backward compatibility for deployments that rely
   * on dynamic Worker discovery rather than a fixed expected count.
   *
   * @return {@code true} if a strict majority of known Workers are alive and ready;
   *         {@code false} if no Workers are configured or too few are alive
   */
  public boolean isClusterHealthy() {
    int minAlive = minAliveWorkers;

    if (minAlive <= 0) {
      minAlive = knownWorkerCount / 2 + 1;
    }
    long aliveCount = records
      .values()
      .stream()
      .filter(r -> r.isAlive(heartbeatTimeoutMs))
      .count();
    return aliveCount >= minAlive;
  }

  /**
   * Returns the set of Worker IDs that are currently alive (ready and within heartbeat timeout).
   *
   * @return a set of alive Worker IDs, never {@code null}
   */
  public Set<String> getAliveWorkerIds() {
    return records
      .values()
      .stream()
      .filter(r -> r.isAlive(heartbeatTimeoutMs))
      .map(WorkerHealthRecord::getWorkerId)
      .collect(Collectors.toSet());
  }

  /**
   * Returns the set of all Worker IDs that have ever been seen by this health view.
   *
   * @return a set of all known Worker IDs, never {@code null}
   */
  public Set<String> getAllWorkerIds() {
    return records.values().stream().map(WorkerHealthRecord::getWorkerId).collect(Collectors.toSet());
  }

  /**
   * Per-Worker health state tracked within the cluster health view.
   *
   * <p>Each record captures the Worker's current epoch, heartbeat timing,
   * readiness, load, and verification failure state. Records are created
   * on first heartbeat and updated (or replaced on epoch change) via
   * atomic {@code ConcurrentHashMap.compute} operations.
   *
   * <p>A Worker transitions through these states:
   * <ul>
   *   <li><b>Startup:</b> {@code readyToServe = false} until first detection cycle completes</li>
   *   <li><b>Healthy:</b> {@code readyToServe = true, stale = false}, heartbeats arriving within timeout</li>
   *   <li><b>Suspected:</b> Heartbeats timeout; {@link WorkerHeartbeatVerifier} probes actively</li>
   *   <li><b>Stale:</b> Verification failures reach threshold; excluded from health majority</li>
   *   <li><b>Restarted:</b> New epoch detected; old record replaced entirely</li>
   * </ul>
   */
  @Getter
  public static class WorkerHealthRecord {

    public volatile String workerId;
    public volatile long epoch;
    public volatile long lastHeartbeatTime;
    public volatile boolean readyToServe;
    public volatile boolean stale;
    public volatile int verifyFailures;

    /**
     * Returns whether this Worker is currently considered alive for health-majority
     * calculations.
     *
     * <p>All three conditions must hold:
     * <ul>
     *   <li>{@link #readyToServe} is {@code true} — the Worker has completed its
     *       initial detection cycle and is accepting requests</li>
     *   <li>{@link #stale} is {@code false} — the Worker has not exceeded the
     *       configured verification failure threshold</li>
     *   <li>The elapsed wall-clock time since {@link #lastHeartbeatTime} is less
     *       than {@code timeoutMs} — heartbeats are arriving within the expected
     *       interval</li>
     * </ul>
     *
     * @param timeoutMs the heartbeat timeout window in milliseconds; must be positive
     * @return {@code true} if this Worker is ready, not stale, and has sent a heartbeat
     *         within the timeout window
     */
    public boolean isAlive(long timeoutMs) {
      return readyToServe && !stale && System.currentTimeMillis() - lastHeartbeatTime < timeoutMs;
    }
  }
}

1	/*
2	* Copyright 2026 Hyshmily. All Rights Reserved.
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16	package io.github.hyshmily.hotkey.sharding;
17
18	import io.github.hyshmily.hotkey.sync.worker.WorkerHeartbeatMessage;
19	import io.github.hyshmily.hotkey.sync.worker.WorkerHeartbeatVerifier;
20	import java.util.Set;
21	import java.util.concurrent.ConcurrentHashMap;
22	import java.util.concurrent.ConcurrentMap;
23	import java.util.stream.Collectors;
24	import lombok.Getter;
25	import lombok.Setter;
26	import lombok.extern.slf4j.Slf4j;
27
28	/**
29	* Cluster-wide view of Worker health, maintained by consuming periodic
30	* {@link WorkerHeartbeatMessage} broadcasts from all Worker nodes.
31	*
32	* <p>This is the central health authority for the local app instance. It tracks
33	* every known Worker by ID with its epoch (restart counter), heartbeat timestamps,
34	* readiness flag, decision version high-water mark, and verification failure count.
35	*
36	* <p><b>Health judgment:</b> The cluster is considered healthy when a strict majority
37	* ({@code > knownWorkerCount / 2}) of Workers are alive (ready, not stale, and within
38	* the heartbeat timeout window). When the cluster becomes unhealthy, the system may
39	* enter graceful degradation mode (see ADR-0009).
40	*
41	* <p><b>Restart detection:</b> When a Worker's epoch in a new heartbeat exceeds the
42	* stored epoch, a new {@link WorkerHealthRecord} is created — all state from the
43	* previous incarnation is discarded. This prevents stale health state from surviving
44	* a Worker restart.
45	*
46	* <p><b>Thread safety:</b> All record mutations use {@link ConcurrentHashMap#compute}
47	* and {@code computeIfPresent} for atomic per-Worker updates. The
48	* {@code lastAnyHeartbeatTime} is a {@code volatile} field safe for
49	* concurrent read/write.
50	*
51	* @see WorkerHeartbeatMessage
52	* @see WorkerHeartbeatVerifier
53	*/
54	@Slf4j	4✔
55	public class ClusterHealthView {
56
57	/** Per-Worker health records keyed by Worker ID. Thread-safe via {@link ConcurrentHashMap}. */
58	private final ConcurrentMap<String, WorkerHealthRecord> records = new ConcurrentHashMap<>();	5✔
59
60	@Getter
61	private volatile int knownWorkerCount;
62
63	private final long heartbeatTimeoutMs;
64	private final int degradeAfterFailures;
65
66	@Setter
67	private volatile int minAliveWorkers;
68
69	@Getter
70	private volatile long lastAnyHeartbeatTime;
71
72	public ClusterHealthView(int knownWorkerCount, long heartbeatTimeoutMs, int degradeAfterFailures) {	2✔
73	this.knownWorkerCount = knownWorkerCount;	3✔
74	this.heartbeatTimeoutMs = heartbeatTimeoutMs;	3✔
75	this.degradeAfterFailures = degradeAfterFailures;	3✔
76	}	1✔
77
78	/**
79	* Updates the expected Worker count for majority-quorum health calculations.
80	* <p>Called during initialization (from configuration) and dynamically
81	* when the ring is reconciled from observed heartbeats.
82	* <p>If the new count is 0, the cluster is always considered unhealthy.
83	* If the new count is greater than the previous, the quorum threshold
84	* adjusts accordingly.
85	*
86	* @param count the new expected Worker count; must be {@code >= 0}
87	*/
88	public void setKnownWorkerCount(int count) {
89	if (count < 0) {	2✔
90	log.warn("Invalid knownWorkerCount {}, ignoring update", count);	5✔
91	return;	1✔
92	}
93	int old = this.knownWorkerCount;	3✔
94	this.knownWorkerCount = count;	3✔
95	if (old == 0 && count > 0) {	4!
96	log.info("knownWorkerCount updated from 0 to {}; cluster health check now active", count);	5✔
97	}
98	}	1✔
99
100	/**
101	* Processes an incoming {@link WorkerHeartbeatMessage} from a Worker node and
102	* updates the cluster health state accordingly.
103	*
104	* <p><b>New or restarted Worker:</b> If this is the first heartbeat from a Worker,
105	* or if the heartbeat epoch exceeds the stored epoch (indicating a Worker restart),
106	* a new {@link WorkerHealthRecord} is created with fresh timestamps and the old
107	* decision watermark is discarded.
108	*
109	* <p><b>Known Worker:</b> Updates the last heartbeat timestamp, decision version
110	* watermark (taking the max), load factor, and readiness flag. Clears any stale
111	* or restarted flags.
112	*
113	* <p><b>Cluster recovery:</b> If the cluster was in degraded state and this heartbeat
114	* brings the majority back to health, the degraded flag is automatically cleared.
115	*
116	* @param hb the incoming heartbeat message; must not be null
117	*/
118	public void onHeartbeat(WorkerHeartbeatMessage hb) {
119	records.compute(hb.workerId(), (id, existing) -> {	8✔
120	long now = System.currentTimeMillis();	2✔
121
122	if (existing == null \|\| hb.epoch() > existing.epoch) {	8✔
123	WorkerHealthRecord r = new WorkerHealthRecord();	4✔
124
125	r.workerId = hb.workerId();	4✔
126	r.epoch = hb.epoch();	4✔
127	r.lastHeartbeatTime = now;	3✔
128	r.readyToServe = hb.readyToServe();	4✔
129	return r;	2✔
130	}
131
132	if (hb.epoch() < existing.epoch) {	6✔
133	return existing;	2✔
134	}
135
136	existing.lastHeartbeatTime = now;	3✔
137	existing.readyToServe = hb.readyToServe();	4✔
138	existing.stale = false;	3✔
139	return existing;	2✔
140	});
141
142	lastAnyHeartbeatTime = System.currentTimeMillis();	3✔
143	}	1✔
144
145	/**
146	* Records a successful PONG response from a Worker during active verification
147	* ({@link WorkerHeartbeatVerifier#verifySuspectedWorkers}).
148	*
149	* <p>Resets the Worker's verification failure count to zero and updates its
150	* heartbeat timestamp to now, effectively restoring it to the alive set.
151	*
152	* @param workerId the Worker that responded with a PONG; must not be null
153	*/
154	public void recordPong(String workerId) {
155	records.computeIfPresent(workerId, (id, r) -> {	6✔
156	r.lastHeartbeatTime = System.currentTimeMillis();	3✔
157	r.verifyFailures = 0;	3✔
158	return r;	2✔
159	});
160	}	1✔
161
162	/**
163	* Increments the verification failure count for a Worker and marks it stale
164	* if the threshold is reached.
165	*
166	* <p>When the cumulative failure count reaches {@code degradeAfterFailures},
167	* the Worker is marked as stale ({@code stale = true}). Stale Workers are
168	* excluded from {@link #isClusterHealthy()} and {@link #getAliveWorkerIds()}
169	* — they are effectively considered dead even if they were previously alive.
170	*
171	* <p>If the Worker ID is not present in the health view (never seen, or already
172	* removed), this call is a no-op.
173	*
174	* @param workerId the Worker that failed active verification; must not be null
175	*/
176	public void markVerificationFailed(String workerId) {
177	records.computeIfPresent(workerId, (id, r) -> {	7✔
178	r.verifyFailures++;	6✔
179	if (r.verifyFailures >= degradeAfterFailures) {	5✔
180	r.stale = true;	3✔
181	}
182	return r;	2✔
183	});
184	}	1✔
185
186	/**
187	* Returns the current verification failure count for a Worker.
188	*
189	* @param workerId the Worker to query; must not be null
190	* @return the failure count, or 0 if the Worker is not tracked
191	*/
192	public int getVerifyFailures(String workerId) {
193	WorkerHealthRecord r = records.get(workerId);	6✔
194	return r != null ? r.verifyFailures : 0;	6!
195	}
196
197	/**
198	* Removes the health record for a Worker that has been confirmed dead.
199	*
200	* @param workerId the Worker to remove; must not be null
201	*/
202	public void removeRecord(String workerId) {
NEW 203	records.remove(workerId);	×
NEW 204	}	×
205
206	/**
207	* Returns whether the cluster is currently considered healthy based on majority
208	* of known Workers being alive.
209	*
210	* <p>A Worker is considered alive when all three conditions hold:
211	* <ul>
212	* <li>{@code readyToServe == true} (Worker completed initialization)</li>
213	* <li>{@code stale == false} (Worker has not exceeded verification failure threshold)</li>
214	* <li>Time since last heartbeat {@code < heartbeatTimeoutMs}</li>
215	* </ul>
216	*
217	* <p>The cluster is healthy when the count of alive Workers is strictly greater
218	* than {@code knownWorkerCount / 2} (simple majority).
219	*
220	* <p>When {@code knownWorkerCount == 0} (no expected count configured), the
221	* cluster is considered healthy if at least one alive Worker is observed via
222	* heartbeats. This provides backward compatibility for deployments that rely
223	* on dynamic Worker discovery rather than a fixed expected count.
224	*
225	* @return {@code true} if a strict majority of known Workers are alive and ready;
226	* {@code false} if no Workers are configured or too few are alive
227	*/
228	public boolean isClusterHealthy() {
229	int minAlive = minAliveWorkers;	3✔
230
231	if (minAlive <= 0) {	2!
232	minAlive = knownWorkerCount / 2 + 1;	7✔
233	}
234	long aliveCount = records	2✔
235	.values()	1✔
236	.stream()	3✔
237	.filter(r -> r.isAlive(heartbeatTimeoutMs))	6✔
238	.count();	2✔
239	return aliveCount >= minAlive;	9✔
240	}
241
242	/**
243	* Returns the set of Worker IDs that are currently alive (ready and within heartbeat timeout).
244	*
245	* @return a set of alive Worker IDs, never {@code null}
246	*/
247	public Set<String> getAliveWorkerIds() {
248	return records	3✔
249	.values()	1✔
250	.stream()	3✔
251	.filter(r -> r.isAlive(heartbeatTimeoutMs))	7✔
252	.map(WorkerHealthRecord::getWorkerId)	1✔
253	.collect(Collectors.toSet());	3✔
254	}
255
256	/**
257	* Returns the set of all Worker IDs that have ever been seen by this health view.
258	*
259	* @return a set of all known Worker IDs, never {@code null}
260	*/
261	public Set<String> getAllWorkerIds() {
262	return records.values().stream().map(WorkerHealthRecord::getWorkerId).collect(Collectors.toSet());	10✔
263	}
264
265	/**
266	* Per-Worker health state tracked within the cluster health view.
267	*
268	* <p>Each record captures the Worker's current epoch, heartbeat timing,
269	* readiness, load, and verification failure state. Records are created
270	* on first heartbeat and updated (or replaced on epoch change) via
271	* atomic {@code ConcurrentHashMap.compute} operations.
272	*
273	* <p>A Worker transitions through these states:
274	* <ul>
275	* <li><b>Startup:</b> {@code readyToServe = false} until first detection cycle completes</li>
276	* <li><b>Healthy:</b> {@code readyToServe = true, stale = false}, heartbeats arriving within timeout</li>
277	* <li><b>Suspected:</b> Heartbeats timeout; {@link WorkerHeartbeatVerifier} probes actively</li>
278	* <li><b>Stale:</b> Verification failures reach threshold; excluded from health majority</li>
279	* <li><b>Restarted:</b> New epoch detected; old record replaced entirely</li>
280	* </ul>
281	*/
282	@Getter
283	public static class WorkerHealthRecord {	3✔
284
285	public volatile String workerId;
286	public volatile long epoch;
287	public volatile long lastHeartbeatTime;
288	public volatile boolean readyToServe;
289	public volatile boolean stale;
290	public volatile int verifyFailures;
291
292	/**
293	* Returns whether this Worker is currently considered alive for health-majority
294	* calculations.
295	*
296	* <p>All three conditions must hold:
297	* <ul>
298	* <li>{@link #readyToServe} is {@code true} — the Worker has completed its
299	* initial detection cycle and is accepting requests</li>
300	* <li>{@link #stale} is {@code false} — the Worker has not exceeded the
301	* configured verification failure threshold</li>
302	* <li>The elapsed wall-clock time since {@link #lastHeartbeatTime} is less
303	* than {@code timeoutMs} — heartbeats are arriving within the expected
304	* interval</li>
305	* </ul>
306	*
307	* @param timeoutMs the heartbeat timeout window in milliseconds; must be positive
308	* @return {@code true} if this Worker is ready, not stale, and has sent a heartbeat
309	* within the timeout window
310	*/
311	public boolean isAlive(long timeoutMs) {
312	return readyToServe && !stale && System.currentTimeMillis() - lastHeartbeatTime < timeoutMs;	17✔
313	}
314	}
315	}

Hyshmily / hotkey / 28425041198

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous