28216525937

Committed 26 Jun 2026 04:08AM UTC coverage: 91.258% (-1.4%) from 92.707%

Build # 28216525937

Build Type

push

github

Committed by

Hyshmily

Commit Message

feat: add circuit breaker, null-value caching, and heartbeat refinements

- Introduce HotKeyCircuitBreaker with sliding-window failure detection
- Cache null/miss results with configurable TTL (null-value-ttl-seconds)
- Add per-Worker exponential backoff to heartbeat verifier (verify-max-backoff-ms)
- Support min-alive-workers config for cluster health threshold
- Migrate HotKeyEndpoint from Actuator @Endpoint to Spring MVC @RestController
- Add NumberFormatException handling in StateMachineEndpoint

Signed-off-by: Hyshmily <cxm8607@outlook.com>

Coverage Stats

1226 of 1419 branches covered (86.4%)

Branch coverage included in aggregate %.

181 of 229 new or added lines in 12 files covered. (79.04%)

1 existing line in 1 file now uncovered.

3503 of 3763 relevant lines covered (93.09%)

4.22 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

61.29

/common/src/main/java/io/github/hyshmily/hotkey/cache/HotKeyCircuitBreaker.java

/*
 * Copyright 2026 Hyshmily. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package io.github.hyshmily.hotkey.cache;

import io.github.hyshmily.hotkey.autoconfigure.HotKeyProperties;
import lombok.extern.slf4j.Slf4j;

import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.LongAdder;

/**
 * Sliding-window circuit breaker for protecting remote calls from cascading failures.
 *
 * <p>Uses a rolling time window divided into buckets ({@link LongAdder}s) to count
 * successes and failures. When the failure rate exceeds {@code failThreshold} and the
 * total request volume meets {@code requestVolumeThreshold}, the breaker opens.
 * After {@code singleTestIntervalMs} in the open state, a single probe request is
 * allowed through (half-open). If that probe succeeds, the breaker closes; if it
 * fails, the breaker stays open.
 *
 * <p>This class is thread-safe.
 */
@Slf4j
public class HotKeyCircuitBreaker implements AutoCloseable {

  private static final ScheduledExecutorService SCHEDULER = new ScheduledThreadPoolExecutor(
    Runtime.getRuntime().availableProcessors(),
    r -> {
      Thread t = new Thread(r, "hotkey-cb");
      t.setDaemon(true);
      return t;
    }
  );

  private final HotKeyProperties.CircuitBreaker config;
  private final int bucketSize;
  private final LongAdder[] successBuckets;
  private final LongAdder[] failBuckets;
  private volatile int currentIndex;
  private final AtomicBoolean open = new AtomicBoolean(false);
  private volatile long openTimestamp;
  private final AtomicLong lastHalfOpenAttempt = new AtomicLong(0L);
  private final ScheduledFuture<?> slideFuture;

  public HotKeyCircuitBreaker(HotKeyProperties.CircuitBreaker config) {
    this.config = config;
    this.bucketSize = config.getWindowBuckets();
    this.successBuckets = new LongAdder[bucketSize];
    this.failBuckets = new LongAdder[bucketSize];
    for (int i = 0; i < bucketSize; i++) {
      successBuckets[i] = new LongAdder();
      failBuckets[i] = new LongAdder();
    }
    long slideMs = config.getWindowTimeMs() / bucketSize;
    this.slideFuture = SCHEDULER.scheduleAtFixedRate(this::slide, slideMs, slideMs, TimeUnit.MILLISECONDS);
  }

  /**
   * Check whether a request is allowed through.
   * When closed: always allowed. When open: allowed only for half-open probe.
   *
   * @return {@code true} if the request may proceed
   */
  public boolean allowRequest() {
    if (!config.isEnabled()) {
      if (open.get()) {
        open.set(false);
      }
      return true;
    }
    if (!open.get()) {
      return true;
    }

    long now = System.currentTimeMillis();
    long lastTest = lastHalfOpenAttempt.get();
    if (
      now - openTimestamp > config.getSingleTestIntervalMs() &&
      now - lastTest > config.getSingleTestIntervalMs() &&
      lastHalfOpenAttempt.compareAndSet(lastTest, now)
    ) {
      if (config.isLogEnabled()) {
        log.info("CB half-open probe");
      }
      return true;
    }
    return false;
  }

  /** Record a successful call. If the breaker was open, attempt to close it. */
  public void onSuccess() {
    if (!config.isEnabled()) {
      return;
    }
    successBuckets[currentIndex].increment();
    if (open.get() && open.compareAndSet(true, false)) {
      resetAllBuckets();
      if (config.isLogEnabled()) {
        log.info("CB closed");
      }
    }
  }

  /** Record a failed call. */
  public void onFailure() {
    if (config.isEnabled()) {
      failBuckets[currentIndex].increment();
      evaluateThreshold();
    }
  }

  /** Whether the breaker is currently open. */
  public boolean isOpen() {
    return open.get() && config.isEnabled();
  }

  private void slide() {
    int next = (currentIndex + 1) % bucketSize;
    successBuckets[next].reset();
    failBuckets[next].reset();
    currentIndex = next;
    evaluateThreshold();
  }

  private void evaluateThreshold() {
    long totalSuccess = 0, totalFail = 0;

    for (LongAdder a : successBuckets) {
      totalSuccess += a.sum();
    }
    for (LongAdder a : failBuckets) {
      totalFail += a.sum();
    }

    if (totalFail > 0 && totalSuccess + totalFail >= config.getRequestVolumeThreshold()) {
      double rate = (double) totalFail / (totalSuccess + totalFail);

      if (rate > config.getFailThreshold() && open.compareAndSet(false, true)) {
        openTimestamp = System.currentTimeMillis();

        if (config.isLogEnabled()) {
          log.info("CB OPEN (failRate={}, total={})", rate, totalSuccess + totalFail);
        }
      }
    }
  }

  private void resetAllBuckets() {
    for (LongAdder a : successBuckets) {
      a.reset();
    }
    for (LongAdder a : failBuckets) {
      a.reset();
    }
  }

  @Override
  public void close() {
    if (slideFuture != null) {
      slideFuture.cancel(false);
    }
  }
}

1	/*
2	* Copyright 2026 Hyshmily. All Rights Reserved.
3	*
4	* Licensed under the Apache License, Version 2.0 (the "License");
5	* you may not use this file except in compliance with the License.
6	* You may obtain a copy of the License at
7	*
8	* http://www.apache.org/licenses/LICENSE-2.0
9	*
10	* Unless required by applicable law or agreed to in writing, software
11	* distributed under the License is distributed on an "AS IS" BASIS,
12	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	* See the License for the specific language governing permissions and
14	* limitations under the License.
15	*/
16	package io.github.hyshmily.hotkey.cache;
17
18	import io.github.hyshmily.hotkey.autoconfigure.HotKeyProperties;
19	import lombok.extern.slf4j.Slf4j;
20
21	import java.util.concurrent.ScheduledExecutorService;
22	import java.util.concurrent.ScheduledFuture;
23	import java.util.concurrent.ScheduledThreadPoolExecutor;
24	import java.util.concurrent.TimeUnit;
25	import java.util.concurrent.atomic.AtomicBoolean;
26	import java.util.concurrent.atomic.AtomicLong;
27	import java.util.concurrent.atomic.LongAdder;
28
29	/**
30	* Sliding-window circuit breaker for protecting remote calls from cascading failures.
31	*
32	* <p>Uses a rolling time window divided into buckets ({@link LongAdder}s) to count
33	* successes and failures. When the failure rate exceeds {@code failThreshold} and the
34	* total request volume meets {@code requestVolumeThreshold}, the breaker opens.
35	* After {@code singleTestIntervalMs} in the open state, a single probe request is
36	* allowed through (half-open). If that probe succeeds, the breaker closes; if it
37	* fails, the breaker stays open.
38	*
39	* <p>This class is thread-safe.
40	*/
41	@Slf4j	3✔
42	public class HotKeyCircuitBreaker implements AutoCloseable {
43
44	private static final ScheduledExecutorService SCHEDULER = new ScheduledThreadPoolExecutor(	3✔
45	Runtime.getRuntime().availableProcessors(),	5✔
46	r -> {
47	Thread t = new Thread(r, "hotkey-cb");	6✔
48	t.setDaemon(true);	3✔
49	return t;	2✔
50	}
51	);
52
53	private final HotKeyProperties.CircuitBreaker config;
54	private final int bucketSize;
55	private final LongAdder[] successBuckets;
56	private final LongAdder[] failBuckets;
57	private volatile int currentIndex;
58	private final AtomicBoolean open = new AtomicBoolean(false);	6✔
59	private volatile long openTimestamp;
60	private final AtomicLong lastHalfOpenAttempt = new AtomicLong(0L);	6✔
61	private final ScheduledFuture<?> slideFuture;
62
63	public HotKeyCircuitBreaker(HotKeyProperties.CircuitBreaker config) {	2✔
64	this.config = config;	3✔
65	this.bucketSize = config.getWindowBuckets();	4✔
66	this.successBuckets = new LongAdder[bucketSize];	5✔
67	this.failBuckets = new LongAdder[bucketSize];	5✔
68	for (int i = 0; i < bucketSize; i++) {	8✔
69	successBuckets[i] = new LongAdder();	7✔
70	failBuckets[i] = new LongAdder();	7✔
71	}
72	long slideMs = config.getWindowTimeMs() / bucketSize;	7✔
73	this.slideFuture = SCHEDULER.scheduleAtFixedRate(this::slide, slideMs, slideMs, TimeUnit.MILLISECONDS);	9✔
74	}	1✔
75
76	/**
77	* Check whether a request is allowed through.
78	* When closed: always allowed. When open: allowed only for half-open probe.
79	*
80	* @return {@code true} if the request may proceed
81	*/
82	public boolean allowRequest() {
83	if (!config.isEnabled()) {	4✔
84	if (open.get()) {	4!
NEW 85	open.set(false);	×
86	}
87	return true;	2✔
88	}
89	if (!open.get()) {	4!
NEW 90	return true;	×
91	}
92
93	long now = System.currentTimeMillis();	2✔
94	long lastTest = lastHalfOpenAttempt.get();	4✔
95	if (	6✔
96	now - openTimestamp > config.getSingleTestIntervalMs() &&	3!
NEW 97	now - lastTest > config.getSingleTestIntervalMs() &&	×
NEW 98	lastHalfOpenAttempt.compareAndSet(lastTest, now)	×
99	) {
NEW 100	if (config.isLogEnabled()) {	×
NEW 101	log.info("CB half-open probe");	×
102	}
NEW 103	return true;	×
104	}
105	return false;	2✔
106	}
107
108	/** Record a successful call. If the breaker was open, attempt to close it. */
109	public void onSuccess() {
110	if (!config.isEnabled()) {	4!
111	return;	1✔
112	}
NEW 113	successBuckets[currentIndex].increment();	×
NEW 114	if (open.get() && open.compareAndSet(true, false)) {	×
NEW 115	resetAllBuckets();	×
NEW 116	if (config.isLogEnabled()) {	×
NEW 117	log.info("CB closed");	×
118	}
119	}
NEW 120	}	×
121
122	/** Record a failed call. */
123	public void onFailure() {
124	if (config.isEnabled()) {	4✔
125	failBuckets[currentIndex].increment();	6✔
126	evaluateThreshold();	2✔
127	}
128	}	1✔
129
130	/** Whether the breaker is currently open. */
131	public boolean isOpen() {
132	return open.get() && config.isEnabled();	11!
133	}
134
135	private void slide() {
136	int next = (currentIndex + 1) % bucketSize;	8✔
137	successBuckets[next].reset();	5✔
138	failBuckets[next].reset();	5✔
139	currentIndex = next;	3✔
140	evaluateThreshold();	2✔
141	}	1✔
142
143	private void evaluateThreshold() {
144	long totalSuccess = 0, totalFail = 0;	4✔
145
146	for (LongAdder a : successBuckets) {	17✔
147	totalSuccess += a.sum();	5✔
148	}
149	for (LongAdder a : failBuckets) {	17✔
150	totalFail += a.sum();	5✔
151	}
152
153	if (totalFail > 0 && totalSuccess + totalFail >= config.getRequestVolumeThreshold()) {	12!
154	double rate = (double) totalFail / (totalSuccess + totalFail);	8✔
155
156	if (rate > config.getFailThreshold() && open.compareAndSet(false, true)) {	12!
157	openTimestamp = System.currentTimeMillis();	3✔
158
159	if (config.isLogEnabled()) {	4!
160	log.info("CB OPEN (failRate={}, total={})", rate, totalSuccess + totalFail);	9✔
161	}
162	}
163	}
164	}	1✔
165
166	private void resetAllBuckets() {
NEW 167	for (LongAdder a : successBuckets) {	×
NEW 168	a.reset();	×
169	}
NEW 170	for (LongAdder a : failBuckets) {	×
NEW 171	a.reset();	×
172	}
NEW 173	}	×
174
175	@Override
176	public void close() {
NEW 177	if (slideFuture != null) {	×
NEW 178	slideFuture.cancel(false);	×
179	}
NEW 180	}	×
181	}

Hyshmily / hotkey / 28216525937

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous