#728

Committed 21 Oct 2025 09:16AM UTC coverage: 76.144% (-2.2%) from 78.302%

Build # #728

Build Type

Pull #113

github

Committed by

sceredi

Commit Message

feat: add rewards management to rl controller

Pull Request Pull Request #113: feat: create reinforcement learning controller

Run Details

139 of 233 new or added lines in 22 files covered. (59.66%)

6 existing lines in 3 files now uncovered.

1497 of 1966 relevant lines covered (76.14%)

8.43 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.5

/src/main/scala/io/github/srs/controller/RLControllerModule.scala

package io.github.srs.controller

import cats.effect.unsafe.implicits.global
import io.github.srs.model.ModelModule
import io.github.srs.config.SimulationConfig
import io.github.srs.model.environment.ValidEnvironment
import io.github.srs.utils.random.RNG
import io.github.srs.model.entity.dynamicentity.action.Action
import cats.Id
import io.github.srs.model.Simulation.simulation
import io.github.srs.model.logic.RLLogicsBundle
import io.github.srs.model.entity.dynamicentity.sensor.SensorReadings
import io.github.srs.model.entity.dynamicentity.sensor.Sensor.senseAll
import io.github.srs.view.rendering.EnvironmentRenderer
import io.github.srs.model.entity.dynamicentity.agent.Agent
import io.github.srs.controller.message.DynamicEntityProposal
import cats.effect.IO
import io.github.srs.utils.random.SimpleRNG
import io.github.srs.model.entity.dynamicentity.action.NoAction

object RLControllerModule:

  type Observations = Map[Agent, SensorReadings]
  type Infos = Map[Agent, String]
  type Rewards = Map[Agent, Double]
  type Terminateds = Map[Agent, Boolean]
  type Truncateds = Map[Agent, Boolean]

  /**
   * The resonse after each simulation step.
   */
  case class StepResponse private[RLControllerModule] (
      observations: Observations,
      rewards: Rewards,
      terminateds: Terminateds,
      truncateds: Truncateds,
      infos: Infos,
  )

  /**
   * Controller trait defines the interface for a Reinforcement Learning controller.
   */
  trait Controller[S <: ModelModule.BaseState]:

    /**
     * The type of the image used for rendering the simulation on the RL client.
     */
    type Image = Array[Byte]

    /**
     * The initial state of the controller.
     */
    def initialState: S

    /**
     * The current state of the controller.
     */
    def state: S

    /**
     * Initializes the controller with the given simulation configuration.
     *
     * @param config
     *   the simulation configuration to initialize the controller.
     */
    def init(config: SimulationConfig[ValidEnvironment]): Unit

    /**
     * Resets the controller to its initial state using the provided random number generator.
     *
     * @param rng
     *   the random number generator to use for reproducibility in the next run.
     */
    def reset(rng: RNG): (Observations, Infos)

    /**
     * Performs a simulation step using the provided actions for each agent.
     *
     * @param actions
     *   a map of agents to their corresponding actions to be performed in this step.
     * @return
     *   a response containing the results of the simulation step.
     */
    def step(actions: Map[Agent, Action[IO]]): StepResponse

    /**
     * Renders the current state of the simulation to an image for the RL client.
     *
     * @param width
     *   the width of the rendered image.
     * @param height
     *   the height of the rendered image.
     *
     * @return
     *   an image representing the current state of the simulation.
     */
    def render(width: Int, height: Int): Image
  end Controller

  trait Provider[S <: ModelModule.BaseState]:
    val controller: Controller[S]

  type Requirements[S <: ModelModule.BaseState] = ModelModule.Provider[S]

  trait Component[S <: ModelModule.BaseState]:
    context: Requirements[S] =>

    object Controller:
      def apply()(using bundle: RLLogicsBundle[S]): Controller[S] = new ControllerImpl

      private class ControllerImpl(using bundle: RLLogicsBundle[S]) extends Controller[S]:

        var _initialState: S =
          bundle.stateLogic.createState(SimulationConfig(simulation, ValidEnvironment.empty))

        var _state: S = initialState

        override def initialState: S = _initialState

        override def state: S = _state

        override def init(config: SimulationConfig[ValidEnvironment]): Unit =
          _initialState = bundle.stateLogic.createState(config)
          _state = _initialState

        override def reset(rng: RNG): (Observations, Infos) =
          _state = context.model.update(state)(using _ => bundle.stateLogic.updateState(initialState, rng))
          (state.environment.getObservations, state.environment.getInfos)

        override def step(actions: Map[Agent, Action[IO]]): StepResponse =
          _state = context.model.update(state)(using s => bundle.tickLogic.tick(s, state.dt)).unsafeRunSync()
          val actionsList =
            actions.map { (agent, action) =>
              DynamicEntityProposal(agent.copy(lastAction = Some(action)), action)
            }.toList.sortBy(_.entity.id)
          val prevEnv = state.environment
          _state = context.model
            .update(state)(using
              s => bundle.dynamicEntityActionsLogic.handleDynamicEntityActionsProposals(s, actionsList),
            )
            .unsafeRunSync()
          _state = context.model
            .update(state)(using s => bundle.randomLogic.random(s, SimpleRNG(s.simulationRNG.nextLong._1)))
            .unsafeRunSync()
          StepResponse(
            observations = state.environment.getObservations,
            rewards = state.environment.getRewards(prevEnv),
            terminateds = Map.empty,
            truncateds = Map.empty,
            infos = state.environment.getInfos,
          )

        end step

        override def render(width: Int, height: Int): Image =
          EnvironmentRenderer.renderToPNG(state.environment, width, height)
      end ControllerImpl
    end Controller

    extension (env: ValidEnvironment)

      def getObservations: Observations =
        env.entities.collect { case a: Agent =>
          a -> a.senseAll[Id](env)
        }.toMap

      def getRewards(prev: ValidEnvironment): Rewards =
        env.entities.collect { case a: Agent =>
          a -> a.reward.evaluate(prev, env, a, a.lastAction.getOrElse(NoAction[IO]()))
        }.toMap

      def getInfos: Infos =
        env.entities.collect { case a: Agent =>
          a -> ""
        }.toMap
  end Component

  trait Interface[S <: ModelModule.BaseState] extends Provider[S] with Component[S]:
    self: Requirements[S] =>
end RLControllerModule

1	package io.github.srs.controller
2
3	import cats.effect.unsafe.implicits.global
4	import io.github.srs.model.ModelModule
5	import io.github.srs.config.SimulationConfig
6	import io.github.srs.model.environment.ValidEnvironment
7	import io.github.srs.utils.random.RNG
8	import io.github.srs.model.entity.dynamicentity.action.Action
9	import cats.Id
10	import io.github.srs.model.Simulation.simulation
11	import io.github.srs.model.logic.RLLogicsBundle
12	import io.github.srs.model.entity.dynamicentity.sensor.SensorReadings
13	import io.github.srs.model.entity.dynamicentity.sensor.Sensor.senseAll
14	import io.github.srs.view.rendering.EnvironmentRenderer
15	import io.github.srs.model.entity.dynamicentity.agent.Agent
16	import io.github.srs.controller.message.DynamicEntityProposal
17	import cats.effect.IO
18	import io.github.srs.utils.random.SimpleRNG
19	import io.github.srs.model.entity.dynamicentity.action.NoAction
20
21	object RLControllerModule:
NEW 22		×
23	type Observations = Map[Agent, SensorReadings]
24	type Infos = Map[Agent, String]	5✔
25	type Rewards = Map[Agent, Double]
26	type Terminateds = Map[Agent, Boolean]
27	type Truncateds = Map[Agent, Boolean]
28
29	/**
30	* The resonse after each simulation step.
31	*/
32	case class StepResponse private[RLControllerModule] (
33	observations: Observations,	35✔
NEW 34	rewards: Rewards,	×
NEW 35	terminateds: Terminateds,	×
NEW 36	truncateds: Truncateds,	×
NEW 37	infos: Infos,	×
NEW 38	)	×
39
40	/**
41	* Controller trait defines the interface for a Reinforcement Learning controller.
42	*/
43	trait Controller[S <: ModelModule.BaseState]:
44
45	/**
46	* The type of the image used for rendering the simulation on the RL client.
47	*/
48	type Image = Array[Byte]
49
50	/**
51	* The initial state of the controller.
52	*/
53	def initialState: S
54
55	/**
56	* The current state of the controller.
57	*/
58	def state: S
59
60	/**
61	* Initializes the controller with the given simulation configuration.
62	*
63	* @param config
64	* the simulation configuration to initialize the controller.
65	*/
66	def init(config: SimulationConfig[ValidEnvironment]): Unit
67
68	/**
69	* Resets the controller to its initial state using the provided random number generator.
70	*
71	* @param rng
72	* the random number generator to use for reproducibility in the next run.
73	*/
74	def reset(rng: RNG): (Observations, Infos)
75
76	/**
77	* Performs a simulation step using the provided actions for each agent.
78	*
79	* @param actions
80	* a map of agents to their corresponding actions to be performed in this step.
81	* @return
82	* a response containing the results of the simulation step.
83	*/
84	def step(actions: Map[Agent, Action[IO]]): StepResponse
85
86	/**
87	* Renders the current state of the simulation to an image for the RL client.
88	*
89	* @param width
90	* the width of the rendered image.
91	* @param height
92	* the height of the rendered image.
93	*
94	* @return
95	* an image representing the current state of the simulation.
96	*/
97	def render(width: Int, height: Int): Image
98	end Controller
99
100	trait Provider[S <: ModelModule.BaseState]:
101	val controller: Controller[S]
102
103	type Requirements[S <: ModelModule.BaseState] = ModelModule.Provider[S]
104
105	trait Component[S <: ModelModule.BaseState]:
106	context: Requirements[S] =>	13✔
107
108	object Controller:
109	def apply()(using bundle: RLLogicsBundle[S]): Controller[S] = new ControllerImpl	5✔
110		9✔
111	private class ControllerImpl(using bundle: RLLogicsBundle[S]) extends Controller[S]:
112		8✔
113	var _initialState: S =
114	bundle.stateLogic.createState(SimulationConfig(simulation, ValidEnvironment.empty))	4✔
115		14✔
116	var _state: S = initialState
117		12✔
118	override def initialState: S = _initialState
119		3✔
120	override def state: S = _state
121		3✔
122	override def init(config: SimulationConfig[ValidEnvironment]): Unit =
123	_initialState = bundle.stateLogic.createState(config)
124	_state = _initialState	7✔
125		5✔
126	override def reset(rng: RNG): (Observations, Infos) =
127	_state = context.model.update(state)(using _ => bundle.stateLogic.updateState(initialState, rng))
128	(state.environment.getObservations, state.environment.getInfos)	13✔
129		17✔
130	override def step(actions: Map[Agent, Action[IO]]): StepResponse =
131	_state = context.model.update(state)(using s => bundle.tickLogic.tick(s, state.dt)).unsafeRunSync()
132	val actionsList =	16✔
133	actions.map { (agent, action) =>
134	DynamicEntityProposal(agent.copy(lastAction = Some(action)), action)	5✔
135	}.toList.sortBy(_.entity.id)
136	val prevEnv = state.environment	8✔
137	_state = context.model	4✔
138	.update(state)(using	6✔
139	s => bundle.dynamicEntityActionsLogic.handleDynamicEntityActionsProposals(s, actionsList),	2✔
140	)	4✔
141	.unsafeRunSync()
142	_state = context.model	5✔
143	.update(state)(using s => bundle.randomLogic.random(s, SimpleRNG(s.simulationRNG.nextLong._1)))	6✔
144	.unsafeRunSync()	5✔
145	StepResponse(	5✔
146	observations = state.environment.getObservations,	1✔
147	rewards = state.environment.getRewards(prevEnv),	7✔
148	terminateds = Map.empty,	8✔
149	truncateds = Map.empty,	3✔
150	infos = state.environment.getInfos,	3✔
151	)	9✔
152
153	end step
154
155	override def render(width: Int, height: Int): Image =
156	EnvironmentRenderer.renderToPNG(state.environment, width, height)
157	end ControllerImpl	2✔
158	end Controller
159
160	extension (env: ValidEnvironment)
161
162	def getObservations: Observations =
163	env.entities.collect { case a: Agent =>
164	a -> a.senseAll[Id](env)	18✔
165	}.toMap	29✔
166		4✔
167	def getRewards(prev: ValidEnvironment): Rewards =
168	env.entities.collect { case a: Agent =>
169	a -> a.reward.evaluate(prev, env, a, a.lastAction.getOrElse(NoAction[IO]()))	21✔
170	}.toMap	46✔
171		4✔
172	def getInfos: Infos =
173	env.entities.collect { case a: Agent =>
174	a -> ""	15✔
175	}.toMap	18✔
176	end Component	4✔
177
178	trait Interface[S <: ModelModule.BaseState] extends Provider[S] with Component[S]:
179	self: Requirements[S] =>
180	end RLControllerModule

Scala-Robotics-Simulator / PPS-22-srs / #728

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous