• Home
  • Features
  • Pricing
  • Docs
  • Announcements
  • Sign In

Hyshmily / hotkey / 27890525626

21 Jun 2026 02:07AM UTC coverage: 93.615% (-0.1%) from 93.742%
27890525626

push

github

Hyshmily
feat: cross-Worker decisionVersion partitioning with nodeId/epoch + expectedWorkerCount

Core changes:
- VersionGuard: 4-param/5-param overloads with nodeId/epoch for per-Worker
  version space isolation
- WorkerMessage: add nodeId/epoch fields, broadcast as AMQP headers
- WorkerListener: propagate nodeId/epoch through DCL into CacheEntry
- CacheEntry: add decisionNodeId/decisionEpoch fields
- WorkerBroadcaster: stamp every HOT/COOL broadcast with nodeId + epoch
- WorkerAutoConfiguration: @Bean workerNodeId(), workerEpochCounter()
- HotKeyConstants: AMQP_HEADER_EPOCH constant

Cluster health:
- HotKeyProperties: expectedWorkerCount (default 0=dynamic)
- ClusterHealthView: majority-quorum (> expectedWorkerCount / 2) health check
- Wire through HotKeyAutoConfiguration, HotKeyAmqpAutoConfiguration,
  HotKeyRedisAutoConfiguration

Production hardening:
- CacheExpireManager: CompletableFuture.orTimeout() with
  RejectedExecutionException catch to prevent stuck refresh markers
- HotKeyStateMachine: updated state transitions
- RingManager: consistent-hash ring refinements
- ReportConsumer: ingest path improvements

Signed-off-by: Hyshmily <cxm8607@outlook.com>

1133 of 1265 branches covered (89.57%)

Branch coverage included in aggregate %.

149 of 160 new or added lines in 15 files covered. (93.13%)

1 existing line in 1 file now uncovered.

3280 of 3449 relevant lines covered (95.1%)

4.31 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.25
/common/src/main/java/io/github/hyshmily/hotkey/autoconfigure/HotKeyAmqpAutoConfiguration.java
1
/*
2
 * Copyright 2026 Hyshmily. All Rights Reserved.
3
 *
4
 * Licensed under the Apache License, Version 2.0 (the "License");
5
 * you may not use this file except in compliance with the License.
6
 * You may obtain a copy of the License at
7
 *
8
 *     http://www.apache.org/licenses/LICENSE-2.0
9
 *
10
 * Unless required by applicable law or agreed to in writing, software
11
 * distributed under the License is distributed on an "AS IS" BASIS,
12
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
 * See the License for the specific language governing permissions and
14
 * limitations under the License.
15
 */
16
package io.github.hyshmily.hotkey.autoconfigure;
17

18
import static io.github.hyshmily.hotkey.constants.HotKeyConstants.ROUTING_KEY_HEARTBEAT;
19

20
import com.github.benmanes.caffeine.cache.Cache;
21
import io.github.hyshmily.hotkey.cache.CacheExpireManager;
22
import io.github.hyshmily.hotkey.reporting.BbrRateLimiter;
23
import io.github.hyshmily.hotkey.reporting.HotKeyReporter;
24
import io.github.hyshmily.hotkey.reporting.ReportPublisher;
25
import io.github.hyshmily.hotkey.rule.RuleMatcher;
26
import io.github.hyshmily.hotkey.sharding.RingManager;
27
import io.github.hyshmily.hotkey.sync.*;
28
import io.github.hyshmily.hotkey.util.InstanceIdGenerator;
29
import io.github.hyshmily.hotkey.util.SystemLoadMonitor;
30
import io.github.hyshmily.hotkey.util.ratelimit.SreRateLimiter;
31
import java.util.concurrent.ScheduledExecutorService;
32
import java.util.function.Function;
33
import org.springframework.amqp.core.*;
34
import org.springframework.amqp.rabbit.connection.ConnectionFactory;
35
import org.springframework.amqp.rabbit.core.RabbitTemplate;
36
import org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer;
37
import org.springframework.amqp.rabbit.listener.api.ChannelAwareMessageListener;
38
import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter;
39
import org.springframework.amqp.support.converter.MessageConverter;
40
import org.springframework.beans.factory.ObjectProvider;
41
import org.springframework.beans.factory.annotation.Qualifier;
42
import org.springframework.boot.autoconfigure.AutoConfiguration;
43
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
44
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
45
import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
46
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
47
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
48
import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
49
import org.springframework.boot.context.properties.EnableConfigurationProperties;
50
import org.springframework.context.annotation.Bean;
51
import org.springframework.context.annotation.Configuration;
52
import org.springframework.data.redis.core.RedisTemplate;
53
import org.springframework.data.redis.core.StringRedisTemplate;
54

55
/**
56
 * Unified AMQP auto-configuration for HotKey messaging: app-to-Worker reporting,
57
 * instance-to-instance cache sync, and Worker decision listening.
58
 *
59
 * <p>Conditionally activates when {@link RabbitTemplate} is on the classpath.
60
 * Sub-groups for cache sync and Worker listener additionally require Redis.
61
 *
62
 * <p><b>Report</b> ({@code hotkey.report.enabled}, default {@code true}):
63
 * app instance aggregates access counts and sends them to the Worker via
64
 * {@link DirectExchange}. No Redis dependency.
65
 *
66
 * <p><b>Cache Sync</b> ({@code hotkey.sync.enabled=true}):
67
 * instance-to-instance INVALIDATE / REFRESH broadcasts via {@link FanoutExchange}.
68
 * Requires Redis for version tracking.
69
 *
70
 * <p><b>Worker Listener</b> ({@code hotkey.worker-listener.enabled=true}):
71
 * receives HOT/COOL decisions from the Worker via {@link FanoutExchange}.
72
 * Requires Redis.
73
 */
74
@AutoConfiguration(after = { RedisAutoConfiguration.class, RabbitAutoConfiguration.class })
75
@ConditionalOnClass(name = "org.springframework.amqp.rabbit.core.RabbitTemplate")
76
@EnableConfigurationProperties({ HotKeyProperties.class, CacheSyncProperties.class, WorkerListenerProperties.class })
77
public class HotKeyAmqpAutoConfiguration {
3✔
78

79
  /**
80
   * Inner configuration for app-to-Worker report routing via DirectExchange.
81
   * Creates the exchange, publisher, ring manager (optional), reporter, and report scheduler.
82
   * Active by default when a {@link RabbitTemplate} bean is present.
83
   */
84
  @Configuration
85
  @ConditionalOnBean(RabbitTemplate.class)
86
  @ConditionalOnProperty(prefix = "hotkey.report", name = "enabled", havingValue = "true", matchIfMissing = true)
87
  static class ReportConfiguration {
3✔
88

89
    /**
90
     * Declare the DirectExchange for report routing (app → Worker).
91
     * Routing keys ({@code report.<appName>.<nodeId>}) ensure each key's
92
     * messages land on the correct worker queue.
93
     *
94
     * @param properties the HotKey configuration properties
95
     * @return a durable, non-auto-delete {@link DirectExchange}
96
     */
97
    @Bean
98
    public DirectExchange hotkeyReportExchange(HotKeyProperties properties) {
99
      return new DirectExchange(properties.getReportExchange(), true, false);
8✔
100
    }
101

102
    /**
103
     * Create the {@link MessageConverter} for serializing report messages to JSON.
104
     * <p>
105
     * Uses Jackson JSON serialization (not Java serialization) for efficiency and cross-version
106
     * compatibility.
107
     *
108
     * @return a new {@link Jackson2JsonMessageConverter} instance
109
     */
110
    @Bean
111
    @ConditionalOnMissingBean(MessageConverter.class)
112
    public MessageConverter reportMessageConverter() {
113
      return new Jackson2JsonMessageConverter();
4✔
114
    }
115

116
    /**
117
     * Create the {@link ReportPublisher} for sending batched access-count reports to the Worker.
118
     *
119
     * @param rabbitTemplate the RabbitMQ template for publishing messages
120
     * @param properties     the HotKey configuration properties
121
     * @return a new {@link ReportPublisher} instance
122
     */
123
    @Bean
124
    @ConditionalOnMissingBean
125
    public ReportPublisher reportPublisher(RabbitTemplate rabbitTemplate, HotKeyProperties properties) {
126
      return new ReportPublisher(rabbitTemplate, properties.getReportExchange(), properties.getAppName());
9✔
127
    }
128

129
    /**
130
     * Create the {@link RingManager} for consistent-hashing report routing.
131
     *
132
     * @param properties the HotKey configuration properties
133
     * @return a new {@link RingManager} instance
134
     */
135
    @Bean
136
    @ConditionalOnMissingBean
137
    public RingManager ringManager(HotKeyProperties properties) {
138
      return new RingManager(properties.getConsistentHashing().getVirtualNodes());
7✔
139
    }
140

141
    /**
142
     * Create the system CPU monitor with EMA smoothing.
143
     * <p>
144
     * Uses the JDK platform MXBean ({@link com.sun.management.OperatingSystemMXBean})
145
     * which is already used by the Worker-side heartbeat producer. The monitor
146
     * starts sampling on creation and stops on context close.
147
     *
148
     * @param properties the HotKey configuration properties
149
     * @return a new {@link SystemLoadMonitor} instance
150
     */
151
    @Bean(initMethod = "start", destroyMethod = "stop")
152
    @ConditionalOnMissingBean
153
    public SystemLoadMonitor hotKeyCpuMonitor(
154
      HotKeyProperties properties,
155
      @Qualifier("hotKeyScheduler") ScheduledExecutorService hotKeyScheduler
156
    ) {
157
      HotKeyProperties.ReporterLimiter cfg = properties.getReporter();
3✔
158
      return new SystemLoadMonitor(hotKeyScheduler, cfg.getCpuPollIntervalMs(), cfg.getCpuDecay());
9✔
159
    }
160

161
    /**
162
     * Create the BBR adaptive rate limiter for the report publisher.
163
     * <p>
164
     * Uses the CPU monitor and the configured BBR parameters. When disabled
165
     * (or when the CPU monitor itself hasn't been fully initialized yet),
166
     * the limiter falls back to a permissive mode.
167
     *
168
     * @param cpuMonitor the system CPU load monitor
169
     * @param properties the HotKey configuration properties
170
     * @return a new {@link BbrRateLimiter} instance
171
     */
172
    @Bean
173
    @ConditionalOnMissingBean
174
    @ConditionalOnProperty(
175
      prefix = "hotkey.local.reporter",
176
      name = "enabled",
177
      havingValue = "true",
178
      matchIfMissing = true
179
    )
180
    public BbrRateLimiter hotKeyBbrRateLimiter(SystemLoadMonitor cpuMonitor, HotKeyProperties properties) {
181
      HotKeyProperties.ReporterLimiter cfg = properties.getReporter();
3✔
182
      return new BbrRateLimiter(
5✔
183
        cpuMonitor,
184
        cfg.getCpuThreshold(),
2✔
185
        cfg.getBbrWindowMs(),
2✔
186
        cfg.getBbrWindowBuckets(),
2✔
187
        cfg.getBbrCooldownMs()
2✔
188
      );
189
    }
190

191
    /**
192
     * Create the {@link HotKeyReporter} that aggregates per-key counts and flushes them
193
     * at the configured interval.
194
     *
195
     * @param reportPublisher       the report publisher for sending batches
196
     * @param properties            the HotKey configuration properties
197
     * @param ringManager           the consistent-hash ring manager
198
     * @param healthViewProvider    optional provider for the cluster health view
199
     * @param bbrRateLimiterProvider optional provider for the BBR rate limiter
200
     * @return a new {@link HotKeyReporter} instance
201
     */
202
    @Bean(initMethod = "start", destroyMethod = "stop")
203
    @ConditionalOnMissingBean
204
    public HotKeyReporter hotKeyReporter(
205
      ReportPublisher reportPublisher,
206
      @Qualifier("hotKeyScheduler") ScheduledExecutorService hotKeyScheduler,
207
      HotKeyProperties properties,
208
      RingManager ringManager,
209
      ObjectProvider<ClusterHealthView> healthViewProvider,
210
      ObjectProvider<BbrRateLimiter> bbrRateLimiterProvider
211
    ) {
212
      HotKeyReporter reporter = new HotKeyReporter(
5✔
213
        reportPublisher,
214
        hotKeyScheduler,
215
        properties.getReportIntervalMs(),
2✔
216
        properties.getAppName(),
2✔
217
        properties.getQueueCapacity(),
2✔
218
        properties.getQueueOfferTimeoutMs(),
2✔
219
        properties.effectiveConsumerCount(),
5✔
220
        ringManager,
221
        healthViewProvider.getIfAvailable(() ->
4✔
NEW
222
          new ClusterHealthView(
×
NEW
223
            properties.getExpectedWorkerCount(),
×
NEW
224
            properties.getHeartbeat().getTimeoutMs(),
×
NEW
225
            properties.getHeartbeat().getDegradeAfterFailures()
×
226
          )
227
        )
228
      );
229
      bbrRateLimiterProvider.ifAvailable(reporter::setBbrRateLimiter);
7✔
230
      return reporter;
2✔
231
    }
232
  }
233

234
  /**
235
   * Inner configuration for instance-to-instance cache synchronization.
236
   * Creates a FanoutExchange, per-instance queue with TTL, binding, publisher,
237
   * Redis loader, sync listener, and a dedicated scheduled executor.
238
   * Requires Redis and {@code hotkey.sync.enabled=true}.
239
   */
240
  @Configuration
241
  @ConditionalOnClass(name = "org.springframework.data.redis.core.RedisTemplate")
242
  @ConditionalOnProperty(prefix = "hotkey.sync", name = "enabled", havingValue = "true")
243
  static class SyncConfiguration {
3✔
244

245
    /**
246
     * Create the FanoutExchange for broadcasting INVALIDATE/REFRESH messages
247
     * to all app instances.
248
     *
249
     * @param properties the cache sync configuration properties
250
     * @return a durable, non-auto-delete {@link FanoutExchange}
251
     */
252
    @Bean
253
    @ConditionalOnClass(name = "org.springframework.amqp.core.FanoutExchange")
254
    public FanoutExchange hotkeySyncExchange(CacheSyncProperties properties) {
255
      return new FanoutExchange(properties.getExchangeName(), true, false);
8✔
256
    }
257

258
    /**
259
     * Create the per-instance sync queue with a 60-second message TTL and 24-hour idle expiry.
260
     *
261
     * @param properties the cache sync configuration properties
262
     * @return a durable {@link Queue} with {@code x-message-ttl} of 60 seconds
263
     *         and {@code x-expires} of 24 hours
264
     */
265
    @Bean
266
    @ConditionalOnClass(name = "org.springframework.amqp.core.Queue")
267
    public Queue hotkeySyncQueue(CacheSyncProperties properties) {
268
      return QueueBuilder.durable(properties.getQueueName())
6✔
269
        .withArgument("x-message-ttl", 60_000)
4✔
270
        .withArgument("x-expires", 86_400_000)
2✔
271
        .build();
1✔
272
    }
273

274
    /**
275
     * Bind the per-instance queue to the sync exchange.
276
     *
277
     * @param hotkeySyncQueue    the per-instance sync queue
278
     * @param hotkeySyncExchange the sync FanoutExchange
279
     * @return a {@link Binding} connecting the queue to the exchange
280
     */
281
    @Bean
282
    public Binding hotkeySyncBinding(Queue hotkeySyncQueue, FanoutExchange hotkeySyncExchange) {
283
      return BindingBuilder.bind(hotkeySyncQueue).to(hotkeySyncExchange);
5✔
284
    }
285

286
    /**
287
     * Create the cache sync publisher for sending INVALIDATE/REFRESH messages.
288
     *
289
     * @param rabbitTemplate the RabbitMQ template for publishing messages
290
     * @param properties     the cache sync configuration properties
291
     * @return a new {@link CacheSyncPublisher} instance
292
     */
293
    @Bean
294
    @ConditionalOnMissingBean
295
    public CacheSyncPublisher cacheSyncPublisher(RabbitTemplate rabbitTemplate, CacheSyncProperties properties) {
296
      return new CacheSyncPublisher(rabbitTemplate, properties);
6✔
297
    }
298

299
    /**
300
     * Default Redis loader used by the sync listener to refresh cache entries via {@code GET}.
301
     *
302
     * @param stringRedisTemplate the String-based Redis template for reading values
303
     * @return a {@link Function} that reads a key from Redis and returns its value
304
     */
305
    @Bean
306
    @ConditionalOnMissingBean(name = "hotKeyRedisLoader")
307
    public Function<String, Object> hotKeyRedisLoader(StringRedisTemplate stringRedisTemplate) {
308
      return key -> stringRedisTemplate.opsForValue().get(key);
3✔
309
    }
310

311
    /**
312
     * Create the sync listener that handles incoming INVALIDATE/REFRESH messages from peers.
313
     *
314
     * @param hotLocalCache       the L1 Caffeine cache
315
     * @param hotKeyRedisLoader   the function for loading values from Redis
316
     * @param properties          the cache sync configuration properties
317
     * @param expireManager       the cache expiry manager
318
     * @param ruleMatcher         the rule matcher for key matching
319
     * @return a new {@link CacheSyncListener} instance
320
     */
321
    @Bean
322
    @ConditionalOnMissingBean
323
    public CacheSyncListener cacheSyncListener(
324
      Cache<String, Object> hotLocalCache,
325
      Function<String, Object> hotKeyRedisLoader,
326
      CacheSyncProperties properties,
327
      @Qualifier("hotKeyScheduler") ScheduledExecutorService hotKeyScheduler,
328
      CacheExpireManager expireManager,
329
      RuleMatcher ruleMatcher
330
    ) {
331
      return new CacheSyncListener(
10✔
332
        hotLocalCache,
333
        hotKeyRedisLoader,
334
        properties,
335
        hotKeyScheduler,
336
        expireManager,
337
        ruleMatcher
338
      );
339
    }
340

341
    /**
342
     * Create the AMQP message listener container that drives the sync listener.
343
     *
344
     * @param connectionFactory  the RabbitMQ connection factory
345
     * @param cacheSyncListener  the sync message handler
346
     * @param properties         the cache sync configuration properties
347
     * @return a configured {@link SimpleMessageListenerContainer}
348
     */
349
    @Bean
350
    @ConditionalOnBean(ConnectionFactory.class)
351
    public SimpleMessageListenerContainer syncListenerContainer(
352
      ConnectionFactory connectionFactory,
353
      CacheSyncListener cacheSyncListener,
354
      CacheSyncProperties properties
355
    ) {
356
      SimpleMessageListenerContainer container = new SimpleMessageListenerContainer(connectionFactory);
5✔
357
      container.setQueueNames(properties.getQueueName());
9✔
358
      container.setAutoStartup(properties.isAutoStartup());
4✔
359
      container.setAcknowledgeMode(AcknowledgeMode.MANUAL);
3✔
360
      container.setConcurrentConsumers(properties.getConcurrentConsumers());
4✔
361
      container.setPrefetchCount(properties.getPrefetchCount());
4✔
362
      container.setMessageListener(
4✔
363
        (ChannelAwareMessageListener) (msg, channel) -> cacheSyncListener.handleSyncMessage(channel, msg)
×
364
      );
365
      return container;
2✔
366
    }
367
  }
368

369
  /**
370
   * Inner configuration for receiving Worker HOT/COOL decisions.
371
   * Creates a FanoutExchange, per-instance queue with TTL, binding, worker listener,
372
   * listener container, and a dedicated scheduled executor.
373
   * Requires Redis and {@code hotkey.worker-listener.enabled=true}.
374
   */
375
  @Configuration
376
  @ConditionalOnClass(name = "org.springframework.data.redis.core.RedisTemplate")
377
  @ConditionalOnBean(RedisTemplate.class)
378
  @ConditionalOnProperty(prefix = "hotkey.worker-listener", name = "enabled", havingValue = "true")
379
  static class WorkerListenerConfiguration {
3✔
380

381
    /**
382
     * Create the FanoutExchange for broadcasting Worker HOT/COOL decisions
383
     * to all app instances.
384
     *
385
     * @param properties the Worker listener configuration properties
386
     * @return a durable, non-auto-delete {@link FanoutExchange}
387
     */
388
    @Bean
389
    @ConditionalOnMissingBean(name = "hotkeyWorkerExchange")
390
    public FanoutExchange hotkeyWorkerExchange(WorkerListenerProperties properties) {
391
      return new FanoutExchange(properties.getExchangeName(), true, false);
8✔
392
    }
393

394
    /**
395
     * Create the per-instance Worker listener queue with a 60-second message TTL and 24-hour idle expiry.
396
     *
397
     * @param properties the Worker listener configuration properties
398
     * @return a durable {@link Queue} with {@code x-message-ttl} of 60 seconds
399
     *         and {@code x-expires} of 24 hours
400
     */
401
    @Bean
402
    @ConditionalOnMissingBean(name = "hotkeyWorkerQueue")
403
    public Queue hotkeyWorkerQueue(WorkerListenerProperties properties) {
404
      return QueueBuilder.durable(properties.getQueueName())
6✔
405
        .withArgument("x-message-ttl", 60_000)
4✔
406
        .withArgument("x-expires", 86_400_000)
2✔
407
        .build();
1✔
408
    }
409

410
    /**
411
     * Bind the per-instance queue to the Worker exchange.
412
     *
413
     * @param hotkeyWorkerQueue    the per-instance Worker listener queue
414
     * @param hotkeyWorkerExchange the Worker FanoutExchange
415
     * @return a {@link Binding} connecting the queue to the exchange
416
     */
417
    @Bean
418
    public Binding hotkeyWorkerBinding(Queue hotkeyWorkerQueue, FanoutExchange hotkeyWorkerExchange) {
419
      return BindingBuilder.bind(hotkeyWorkerQueue).to(hotkeyWorkerExchange);
5✔
420
    }
421

422
    /**
423
     * Create the TopicExchange for Worker heartbeat broadcasts.
424
     *
425
     * @param properties the HotKey configuration properties
426
     * @return a durable, non-auto-delete {@link TopicExchange}
427
     */
428
    @Bean
429
    @ConditionalOnMissingBean(name = "hotkeyHeartbeatExchange")
430
    public TopicExchange hotkeyHeartbeatExchange(HotKeyProperties properties) {
431
      return new TopicExchange(properties.getHeartbeat().getExchangeName(), true, false);
9✔
432
    }
433

434
    /**
435
     * Create the per-instance non-durable heartbeat queue that auto-deletes on disconnect.
436
     *
437
     * @return a non-durable, auto-delete {@link Queue}
438
     */
439
    @Bean
440
    public Queue hotkeyHeartbeatQueue() {
441
      return QueueBuilder.nonDurable("hotkey.heartbeat:" + InstanceIdGenerator.get()).autoDelete().build();
6✔
442
    }
443

444
    /**
445
     * Bind the per-instance heartbeat queue to the heartbeat exchange with routing key {@code heartbeat.*}.
446
     *
447
     * @param hotkeyHeartbeatQueue    the per-instance heartbeat queue
448
     * @param hotkeyHeartbeatExchange the heartbeat TopicExchange
449
     * @return a {@link Binding} connecting the queue to the exchange
450
     */
451
    @Bean
452
    public Binding hotkeyHeartbeatBinding(Queue hotkeyHeartbeatQueue, TopicExchange hotkeyHeartbeatExchange) {
453
      return BindingBuilder.bind(hotkeyHeartbeatQueue).to(hotkeyHeartbeatExchange).with(ROUTING_KEY_HEARTBEAT + "*");
7✔
454
    }
455

456
    /**
457
     * Create the {@link ClusterHealthView} for tracking Worker cluster health.
458
     *
459
     * @param ringManager the consistent-hash ring manager
460
     * @param properties  the HotKey configuration properties
461
     * @return a new {@link ClusterHealthView} instance
462
     */
463
    @Bean
464
    @ConditionalOnMissingBean
465
    public ClusterHealthView clusterHealthView(RingManager ringManager, HotKeyProperties properties) {
466
      ClusterHealthView view = new ClusterHealthView(
3✔
467
        properties.getExpectedWorkerCount(),
2✔
468
        properties.getHeartbeat().getTimeoutMs(),
4✔
469
        properties.getHeartbeat().getDegradeAfterFailures()
4✔
470
      );
471
      ringManager.setOnRingReconciled(aliveCount -> {
5✔
472
        if (properties.getExpectedWorkerCount() <= 0) {
3!
473
          view.setKnownWorkerCount(Math.max(view.getKnownWorkerCount(), aliveCount));
6✔
474
        }
475
      });
1✔
476
      return view;
2✔
477
    }
478

479
    /**
480
     * Create the SRE adaptive rate limiter for WorkerListener HOT-path throttling.
481
     * <p>
482
     * Disabled when {@code hotkey.worker-listener.sre.enabled=false}.
483
     *
484
     * @param properties the Worker listener configuration properties
485
     * @return a new {@link SreRateLimiter} instance
486
     */
487
    @Bean
488
    @ConditionalOnMissingBean
489
    @ConditionalOnProperty(
490
      prefix = "hotkey.worker-listener.sre",
491
      name = "enabled",
492
      havingValue = "true",
493
      matchIfMissing = true
494
    )
495
    public SreRateLimiter hotKeySreRateLimiter(WorkerListenerProperties properties) {
496
      WorkerListenerProperties.Sre sreConfig = properties.getSre();
3✔
497
      return new SreRateLimiter(
4✔
498
        sreConfig.getWindowMs(),
2✔
499
        sreConfig.getBuckets(),
3✔
500
        1.0 / sreConfig.getSuccessThreshold(),
3✔
501
        sreConfig.getMinSamples()
2✔
502
      );
503
    }
504

505
    /**
506
     * Create the listener that processes HOT/COOL decisions broadcast by the Worker.
507
     *
508
     * @param hotLocalCache          the L1 Caffeine cache
509
     * @param hotKeyRedisLoader      the function for loading values from Redis
510
     * @param properties             the Worker listener configuration properties
511
     * @param expireManager          the cache expiry manager
512
     * @param sreRateLimiterProvider optional provider for the SRE rate limiter
513
     * @return a new {@link WorkerListener} instance
514
     */
515
    @Bean
516
    @ConditionalOnMissingBean
517
    public WorkerListener workerListener(
518
      Cache<String, Object> hotLocalCache,
519
      Function<String, Object> hotKeyRedisLoader,
520
      WorkerListenerProperties properties,
521
      @Qualifier("hotKeyScheduler") ScheduledExecutorService hotKeyScheduler,
522
      CacheExpireManager expireManager,
523
      ObjectProvider<SreRateLimiter> sreRateLimiterProvider
524
    ) {
525
      return new WorkerListener(
9✔
526
        hotLocalCache,
527
        hotKeyRedisLoader,
528
        properties,
529
        hotKeyScheduler,
530
        expireManager,
531
        sreRateLimiterProvider.getIfAvailable()
3✔
532
      );
533
    }
534

535
    /**
536
     * Create the AMQP message listener container that processes Worker heartbeat messages.
537
     *
538
     * @param connectionFactory   the RabbitMQ connection factory
539
     * @param healthView          the cluster health view to update on heartbeat reception
540
     * @param hotkeyHeartbeatQueue the heartbeat queue
541
     * @return a configured {@link SimpleMessageListenerContainer}
542
     */
543
    @Bean
544
    @ConditionalOnMissingBean(name = "hotkeyHeartbeatContainer")
545
    public SimpleMessageListenerContainer heartbeatContainer(
546
      ConnectionFactory connectionFactory,
547
      ClusterHealthView healthView,
548
      Queue hotkeyHeartbeatQueue
549
    ) {
550
      SimpleMessageListenerContainer container = new SimpleMessageListenerContainer(connectionFactory);
5✔
551
      container.setQueueNames(hotkeyHeartbeatQueue.getName());
9✔
552
      container.setAcknowledgeMode(AcknowledgeMode.NONE);
3✔
553
      container.setConcurrentConsumers(1);
3✔
554
      container.setPrefetchCount(100);
3✔
555
      container.setMessageListener(msg -> {
4✔
556
        WorkerHeartbeatMessage hb = WorkerHeartbeatMessage.from(msg);
×
557
        if (hb != null) {
×
558
          healthView.onHeartbeat(hb);
×
559
        }
560
      });
×
561
      return container;
2✔
562
    }
563

564
    /**
565
     * Create the AMQP message listener container that processes Worker HOT/COOL decisions
566
     * via the {@link WorkerListener}.
567
     *
568
     * <p>The container uses {@link AcknowledgeMode#MANUAL} because the
569
     * {@link WorkerListener#handleWorkerMessage} performs its own ack/nack
570
     * (ack-before-update pattern, see ADR-0004).
571
     *
572
     * @param connectionFactory the RabbitMQ connection factory
573
     * @param hotkeyWorkerQueue the per-instance Worker listener queue
574
     * @param workerListener    the Worker decision listener
575
     * @param properties        the Worker listener configuration properties
576
     * @return a configured {@link SimpleMessageListenerContainer}
577
     */
578
    @Bean
579
    @ConditionalOnMissingBean(name = "workerListenerContainer")
580
    public SimpleMessageListenerContainer workerListenerContainer(
581
      ConnectionFactory connectionFactory,
582
      Queue hotkeyWorkerQueue,
583
      WorkerListener workerListener,
584
      WorkerListenerProperties properties
585
    ) {
586
      SimpleMessageListenerContainer container = new SimpleMessageListenerContainer(connectionFactory);
5✔
587
      container.setQueueNames(hotkeyWorkerQueue.getName());
9✔
588
      container.setAcknowledgeMode(AcknowledgeMode.MANUAL);
3✔
589
      container.setMessageListener(
4✔
NEW
590
        (ChannelAwareMessageListener) (msg, channel) -> workerListener.handleWorkerMessage(channel, msg)
×
591
      );
592
      container.setConcurrentConsumers(properties.getConcurrentConsumers());
4✔
593
      container.setPrefetchCount(properties.getPrefetchCount());
4✔
594
      container.setAutoStartup(properties.isAutoStartup());
4✔
595
      return container;
2✔
596
    }
597

598
    /**
599
     * Create the {@link WorkerHeartbeatVerifier} that periodically PINGs Workers
600
     * to verify they are alive.
601
     *
602
     * @param rabbitTemplate the RabbitMQ template for sending PING messages
603
     * @param healthView     the cluster health view to update on verification results
604
     * @param properties     the HotKey configuration properties
605
     * @return a new {@link WorkerHeartbeatVerifier} instance
606
     */
607
    @Bean(initMethod = "start", destroyMethod = "stop")
608
    @ConditionalOnMissingBean
609
    public WorkerHeartbeatVerifier workerHeartbeatVerifier(
610
      RabbitTemplate rabbitTemplate,
611
      ClusterHealthView healthView,
612
      HotKeyProperties properties,
613
      @Qualifier("hotKeyScheduler") ScheduledExecutorService hotKeyScheduler
614
    ) {
615
      return new WorkerHeartbeatVerifier(
6✔
616
        rabbitTemplate,
617
        healthView,
618
        properties.getInstanceId(),
2✔
619
        properties.getHeartbeat().getVerifyIntervalMs(),
3✔
620
        properties.getHeartbeat().getPingTimeoutMs(),
3✔
621
        properties.getHeartbeat().getDegradeAfterFailures(),
4✔
622
        hotKeyScheduler
623
      );
624
    }
625
  }
626
}
STATUS · Troubleshooting · Open an Issue · Sales · Support · CAREERS · ENTERPRISE · START FREE · SCHEDULE DEMO
ANNOUNCEMENTS · TWITTER · TOS & SLA · Supported CI Services · What's a CI service? · Automated Testing

© 2026 Coveralls, Inc