#1023

Committed 07 Jan 2026 11:06AM UTC coverage: 65.786% (-2.4%) from 68.186%

Build # #1023

Build Type

Pull #150

github

Committed by

GiuliaNardicchia

Commit Message

refactor: clean up code

Pull Request Pull Request #150: feat: deep q-learning exploration

Run Details

17 of 136 new or added lines in 8 files covered. (12.5%)

9 existing lines in 5 files now uncovered.

1619 of 2461 relevant lines covered (65.79%)

7.38 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.18

/src/main/scala/io/github/srs/controller/RLControllerModule.scala

package io.github.srs.controller

import cats.Id
import cats.effect.IO
import cats.effect.unsafe.implicits.global
import io.github.srs.config.SimulationConfig
import io.github.srs.controller.message.DynamicEntityProposal
import io.github.srs.logger
import io.github.srs.model.ModelModule
import io.github.srs.model.Simulation.simulation
import io.github.srs.model.entity.dynamicentity.action.{ Action, NoAction }
import io.github.srs.model.entity.dynamicentity.agent.Agent
import io.github.srs.model.entity.dynamicentity.sensor.Sensor.senseAll
import io.github.srs.model.entity.dynamicentity.sensor.SensorReadings
import io.github.srs.model.environment.ValidEnvironment
import io.github.srs.model.logic.RLLogicsBundle
import io.github.srs.utils.random.{ RNG, SimpleRNG }
import io.github.srs.view.rendering.EnvironmentRenderer
import io.github.srs.model.entity.Point2D.*
import io.github.srs.utils.SpatialUtils
import io.github.srs.utils.SpatialUtils.{ discreteCell, nearbyVisitedPositions }

object RLControllerModule:

  final case class AgentObservation(
      sensorReadings: SensorReadings,
      position: (Double, Double),
      orientation: Double,
      visitedPositions: List[Double],
  )

  type Observations = Map[Agent, AgentObservation]
  type Infos = Map[Agent, String]
  type Rewards = Map[Agent, Double]
  type Terminateds = Map[Agent, Boolean]
  type Truncateds = Map[Agent, Boolean]

  /**
   * The resonse after each simulation step.
   */
  case class StepResponse private[RLControllerModule] (
      observations: Observations,
      rewards: Rewards,
      terminateds: Terminateds,
      truncateds: Truncateds,
      infos: Infos,
  )

  /**
   * Controller trait defines the interface for a Reinforcement Learning controller.
   */
  trait Controller[S <: ModelModule.BaseState]:

    /**
     * The type of the image used for rendering the simulation on the RL client.
     */
    type Image = Array[Byte]

    /**
     * The initial state of the controller.
     */
    def initialState: S

    /**
     * The current state of the controller.
     */
    def state: S

    /**
     * Initializes the controller with the given simulation configuration.
     *
     * @param config
     *   the simulation configuration to initialize the controller.
     */
    def init(config: SimulationConfig[ValidEnvironment]): Unit

    /**
     * Resets the controller to its initial state using the provided random number generator.
     *
     * @param rng
     *   the random number generator to use for reproducibility in the next run.
     */
    def reset(rng: RNG): (Observations, Infos)

    /**
     * Performs a simulation step using the provided actions for each agent.
     *
     * @param actions
     *   a map of agents to their corresponding actions to be performed in this step.
     * @return
     *   a response containing the results of the simulation step.
     */
    def step(actions: Map[Agent, Action[IO]]): StepResponse

    /**
     * Renders the current state of the simulation to an image for the RL client.
     *
     * @param width
     *   the width of the rendered image.
     * @param height
     *   the height of the rendered image.
     *
     * @return
     *   an image representing the current state of the simulation.
     */
    def render(width: Int, height: Int): Image
  end Controller

  trait Provider[S <: ModelModule.BaseState]:
    val controller: Controller[S]

  type Requirements[S <: ModelModule.BaseState] = ModelModule.Provider[S]

  trait Component[S <: ModelModule.BaseState]:
    context: Requirements[S] =>

    object Controller:
      def apply()(using bundle: RLLogicsBundle[S]): Controller[S] = new ControllerImpl

      private class ControllerImpl(using bundle: RLLogicsBundle[S]) extends Controller[S]:

        var _initialState: S =
          bundle.stateLogic.createState(SimulationConfig(simulation, ValidEnvironment.empty))

        var _state: S = initialState

        override def initialState: S = _initialState

        override def state: S = _state

        override def init(config: SimulationConfig[ValidEnvironment]): Unit =
          _initialState = bundle.stateLogic.createState(config)
          _state = _initialState
          logger.debug(s"new config, state = \n $state")

        override def reset(rng: RNG): (Observations, Infos) =
          _state = context.model.update(state)(using _ => bundle.stateLogic.updateState(initialState, rng))
          logger.debug("resetting controller")
          (state.environment.getObservations, state.environment.getInfos)

        override def step(actions: Map[Agent, Action[IO]]): StepResponse =
          logger.debug("sending step command to controller")
          logger.debug(s"actions: $actions")
          _state = context.model.update(state)(using s => bundle.tickLogic.tick(s, state.dt)).unsafeRunSync()
          val actionsList =
            actions.map { (agent, action) =>
              DynamicEntityProposal(
                agent.copy(
                  lastAction = Some(action),
                  aliveSteps = agent.aliveSteps + 1,
                  visitedCountPositions = nearbyVisitedPositions(
                    discreteCell(agent.position),
                    agent.visitedCountPositions,
                    state.environment.width,
                    state.environment.height,
                  )._1,
                ),
                action,
              )
            }.toList.sortBy(_.entity.id)
          val prevState = state
          _state = context.model
            .update(state)(using
              s => bundle.dynamicEntityActionsLogic.handleDynamicEntityActionsProposals(s, actionsList),
            )
            .unsafeRunSync()
          _state = context.model
            .update(state)(using s => bundle.randomLogic.random(s, SimpleRNG(s.simulationRNG.nextLong._1)))
            .unsafeRunSync()
          val observations = state.environment.getObservations
          val rewards = state.getRewards(prevState)
          val terminateds = state.getTerminations(prevState)
          val truncateds = state.getTruncations(prevState)
          val infos = state.environment.getInfos
          logger.debug(s"step completed, state = \n $state")
          logger.debug(s"observations: $observations")
          logger.debug(s"rewards: $rewards")
          logger.debug(s"terminateds: $terminateds")
          logger.debug(s"truncateds: $truncateds")
          logger.debug(s"infos: $infos")
          StepResponse(
            observations,
            rewards,
            terminateds,
            truncateds,
            infos,
          )

        end step

        override def render(width: Int, height: Int): Image =
          EnvironmentRenderer.renderToPNG(state.environment, width, height)
      end ControllerImpl
    end Controller

    extension (s: ModelModule.BaseState)

      def getTerminations(prev: ModelModule.BaseState): Terminateds =
        s.environment.entities.collect { case a: Agent =>
          a -> a.termination.evaluate(prev, s, a, a.lastAction.getOrElse(NoAction[IO]()))
        }.toMap

      def getTruncations(prev: ModelModule.BaseState): Truncateds =
        s.environment.entities.collect { case a: Agent =>
          a -> a.truncation.evaluate(prev, s, a, a.lastAction.getOrElse(NoAction[IO]()))
        }.toMap

      def getRewards(prev: ModelModule.BaseState): Rewards =
        s.environment.entities.collect { case a: Agent =>
          a -> a.reward.evaluate(prev, s, a, a.lastAction.getOrElse(NoAction[IO]()))
        }.toMap

    extension (env: ValidEnvironment)

      def getObservations: Observations =
        env.entities.collect { case a: Agent =>
          val sensors = a.senseAll[Id](env)
          val position = (a.position.x, a.position.y)
          val orientation = a.orientation.degrees
          val (_, visitedPositions) =
            nearbyVisitedPositions(discreteCell(position), a.visitedCountPositions, env.width, env.height)

          a -> AgentObservation(sensors, position, orientation, visitedPositions)
        }.toMap

      def getInfos: Infos =
        env.entities.collect { case a: Agent =>
          a -> ""
        }.toMap
  end Component

  trait Interface[S <: ModelModule.BaseState] extends Provider[S] with Component[S]:
    self: Requirements[S] =>
end RLControllerModule

1	package io.github.srs.controller
2
3	import cats.Id
4	import cats.effect.IO
5	import cats.effect.unsafe.implicits.global
6	import io.github.srs.config.SimulationConfig
7	import io.github.srs.controller.message.DynamicEntityProposal
8	import io.github.srs.logger
9	import io.github.srs.model.ModelModule
10	import io.github.srs.model.Simulation.simulation
11	import io.github.srs.model.entity.dynamicentity.action.{ Action, NoAction }
12	import io.github.srs.model.entity.dynamicentity.agent.Agent
13	import io.github.srs.model.entity.dynamicentity.sensor.Sensor.senseAll
14	import io.github.srs.model.entity.dynamicentity.sensor.SensorReadings
15	import io.github.srs.model.environment.ValidEnvironment
16	import io.github.srs.model.logic.RLLogicsBundle
17	import io.github.srs.utils.random.{ RNG, SimpleRNG }
18	import io.github.srs.view.rendering.EnvironmentRenderer
19	import io.github.srs.model.entity.Point2D.*
20	import io.github.srs.utils.SpatialUtils
21	import io.github.srs.utils.SpatialUtils.{ discreteCell, nearbyVisitedPositions }
22
23	object RLControllerModule:
24		×
25	final case class AgentObservation(
26	sensorReadings: SensorReadings,	36✔
27	position: (Double, Double),	×
28	orientation: Double,	×
NEW 29	visitedPositions: List[Double],	×
UNCOV 30	)	×
31
32	type Observations = Map[Agent, AgentObservation]
33	type Infos = Map[Agent, String]
34	type Rewards = Map[Agent, Double]
35	type Terminateds = Map[Agent, Boolean]
36	type Truncateds = Map[Agent, Boolean]
37
38	/**
39	* The resonse after each simulation step.
40	*/
41	case class StepResponse private[RLControllerModule] (
42	observations: Observations,	35✔
43	rewards: Rewards,	×
44	terminateds: Terminateds,	×
45	truncateds: Truncateds,	×
46	infos: Infos,	×
47	)	×
48
49	/**
50	* Controller trait defines the interface for a Reinforcement Learning controller.
51	*/
52	trait Controller[S <: ModelModule.BaseState]:
53
54	/**
55	* The type of the image used for rendering the simulation on the RL client.
56	*/
57	type Image = Array[Byte]
58
59	/**
60	* The initial state of the controller.
61	*/
62	def initialState: S
63
64	/**
65	* The current state of the controller.
66	*/
67	def state: S
68
69	/**
70	* Initializes the controller with the given simulation configuration.
71	*
72	* @param config
73	* the simulation configuration to initialize the controller.
74	*/
75	def init(config: SimulationConfig[ValidEnvironment]): Unit
76
77	/**
78	* Resets the controller to its initial state using the provided random number generator.
79	*
80	* @param rng
81	* the random number generator to use for reproducibility in the next run.
82	*/
83	def reset(rng: RNG): (Observations, Infos)
84
85	/**
86	* Performs a simulation step using the provided actions for each agent.
87	*
88	* @param actions
89	* a map of agents to their corresponding actions to be performed in this step.
90	* @return
91	* a response containing the results of the simulation step.
92	*/
93	def step(actions: Map[Agent, Action[IO]]): StepResponse
94
95	/**
96	* Renders the current state of the simulation to an image for the RL client.
97	*
98	* @param width
99	* the width of the rendered image.
100	* @param height
101	* the height of the rendered image.
102	*
103	* @return
104	* an image representing the current state of the simulation.
105	*/
106	def render(width: Int, height: Int): Image
107	end Controller
108
109	trait Provider[S <: ModelModule.BaseState]:
110	val controller: Controller[S]
111
112	type Requirements[S <: ModelModule.BaseState] = ModelModule.Provider[S]
113
114	trait Component[S <: ModelModule.BaseState]:
115	context: Requirements[S] =>	23✔
116
117	object Controller:
118	def apply()(using bundle: RLLogicsBundle[S]): Controller[S] = new ControllerImpl	5✔
119		9✔
120	private class ControllerImpl(using bundle: RLLogicsBundle[S]) extends Controller[S]:
121		8✔
122	var _initialState: S =
123	bundle.stateLogic.createState(SimulationConfig(simulation, ValidEnvironment.empty))	4✔
124		14✔
125	var _state: S = initialState
126		12✔
127	override def initialState: S = _initialState
128		3✔
129	override def state: S = _state
130		3✔
131	override def init(config: SimulationConfig[ValidEnvironment]): Unit =
132	_initialState = bundle.stateLogic.createState(config)
133	_state = _initialState	7✔
134	logger.debug(s"new config, state = \n $state")	4✔
135		8✔
136	override def reset(rng: RNG): (Observations, Infos) =
137	_state = context.model.update(state)(using _ => bundle.stateLogic.updateState(initialState, rng))
138	logger.debug("resetting controller")	13✔
139	(state.environment.getObservations, state.environment.getInfos)	7✔
140		17✔
141	override def step(actions: Map[Agent, Action[IO]]): StepResponse =
142	logger.debug("sending step command to controller")
143	logger.debug(s"actions: $actions")	7✔
144	_state = context.model.update(state)(using s => bundle.tickLogic.tick(s, state.dt)).unsafeRunSync()	7✔
145	val actionsList =	16✔
146	actions.map { (agent, action) =>
147	DynamicEntityProposal(	6✔
148	agent.copy(
149	lastAction = Some(action),
150	aliveSteps = agent.aliveSteps + 1,
151	visitedCountPositions = nearbyVisitedPositions(
152	discreteCell(agent.position),
153	agent.visitedCountPositions,
154	state.environment.width,
155	state.environment.height,
156	)._1,
157	),
158	action,
159	)
160	}.toList.sortBy(_.entity.id)
161	val prevState = state	8✔
162	_state = context.model	3✔
163	.update(state)(using	6✔
164	s => bundle.dynamicEntityActionsLogic.handleDynamicEntityActionsProposals(s, actionsList),	2✔
165	)	4✔
166	.unsafeRunSync()
167	_state = context.model	5✔
168	.update(state)(using s => bundle.randomLogic.random(s, SimpleRNG(s.simulationRNG.nextLong._1)))	6✔
169	.unsafeRunSync()	5✔
170	val observations = state.environment.getObservations	5✔
171	val rewards = state.getRewards(prevState)	8✔
172	val terminateds = state.getTerminations(prevState)	8✔
173	val truncateds = state.getTruncations(prevState)	8✔
174	val infos = state.environment.getInfos	8✔
175	logger.debug(s"step completed, state = \n $state")	8✔
176	logger.debug(s"observations: $observations")	7✔
177	logger.debug(s"rewards: $rewards")	7✔
178	logger.debug(s"terminateds: $terminateds")	7✔
179	logger.debug(s"truncateds: $truncateds")	7✔
180	logger.debug(s"infos: $infos")	7✔
181	StepResponse(	7✔
182	observations,	1✔
183	rewards,	1✔
184	terminateds,	1✔
185	truncateds,	1✔
186	infos,	1✔
187	)	3✔
188
189	end step
190
191	override def render(width: Int, height: Int): Image =
192	EnvironmentRenderer.renderToPNG(state.environment, width, height)
193	end ControllerImpl	2✔
194	end Controller
195
196	extension (s: ModelModule.BaseState)
197
198	def getTerminations(prev: ModelModule.BaseState): Terminateds =
199	s.environment.entities.collect { case a: Agent =>
200	a -> a.termination.evaluate(prev, s, a, a.lastAction.getOrElse(NoAction[IO]()))	22✔
201	}.toMap	38✔
202		4✔
203	def getTruncations(prev: ModelModule.BaseState): Truncateds =
204	s.environment.entities.collect { case a: Agent =>
205	a -> a.truncation.evaluate(prev, s, a, a.lastAction.getOrElse(NoAction[IO]()))	22✔
206	}.toMap	38✔
207		4✔
208	def getRewards(prev: ModelModule.BaseState): Rewards =
209	s.environment.entities.collect { case a: Agent =>
210	a -> a.reward.evaluate(prev, s, a, a.lastAction.getOrElse(NoAction[IO]()))	22✔
211	}.toMap	38✔
212		4✔
213	extension (env: ValidEnvironment)
214
215	def getObservations: Observations =
216	env.entities.collect { case a: Agent =>
217	val sensors = a.senseAll[Id](env)	18✔
218	val position = (a.position.x, a.position.y)	13✔
219	val orientation = a.orientation.degrees	13✔
220	val (_, visitedPositions) =	4✔
221	nearbyVisitedPositions(discreteCell(position), a.visitedCountPositions, env.width, env.height)	7✔
222		16✔
223	a -> AgentObservation(sensors, position, orientation, visitedPositions)
224	}.toMap	24✔
225		4✔
226	def getInfos: Infos =
227	env.entities.collect { case a: Agent =>
228	a -> ""	15✔
229	}.toMap	18✔
230	end Component	4✔
231
232	trait Interface[S <: ModelModule.BaseState] extends Provider[S] with Component[S]:
233	self: Requirements[S] =>
234	end RLControllerModule

Scala-Robotics-Simulator / PPS-22-srs / #1023

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous