#1004

Committed 16 Dec 2025 03:00PM UTC coverage: 65.785% (-2.4%) from 68.186%

Build # #1004

Build Type

Pull #150

github

Committed by

GiuliaNardicchia

Commit Message

refactor: enhance observation encoding by adding visited positions and create pheromone trail

Pull Request Pull Request #150: feat: deep q-learning exploration

Run Details

15 of 136 new or added lines in 8 files covered. (11.03%)

8 existing lines in 5 files now uncovered.

1617 of 2458 relevant lines covered (65.79%)

7.38 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.01

/src/main/scala/io/github/srs/controller/RLControllerModule.scala

package io.github.srs.controller

import cats.Id
import cats.effect.IO
import cats.effect.unsafe.implicits.global
import io.github.srs.config.SimulationConfig
import io.github.srs.controller.message.DynamicEntityProposal
import io.github.srs.logger
import io.github.srs.model.ModelModule
import io.github.srs.model.Simulation.simulation
import io.github.srs.model.entity.dynamicentity.action.{ Action, NoAction }
import io.github.srs.model.entity.dynamicentity.agent.Agent
import io.github.srs.model.entity.dynamicentity.sensor.Sensor.senseAll
import io.github.srs.model.entity.dynamicentity.sensor.SensorReadings
import io.github.srs.model.environment.ValidEnvironment
import io.github.srs.model.logic.RLLogicsBundle
import io.github.srs.utils.random.{ RNG, SimpleRNG }
import io.github.srs.view.rendering.EnvironmentRenderer
import io.github.srs.model.entity.Point2D.*
import io.github.srs.utils.SpatialUtils
import io.github.srs.utils.SpatialUtils.{ discreteCell, nearbyVisitedPositions }

object RLControllerModule:

  final case class AgentObservation(
      sensorReadings: SensorReadings,
      position: (Double, Double),
      orientation: Double,
      visitedPositions: List[Double],
  )

  type Observations = Map[Agent, AgentObservation]
  type Infos = Map[Agent, String]
  type Rewards = Map[Agent, Double]
  type Terminateds = Map[Agent, Boolean]
  type Truncateds = Map[Agent, Boolean]

  /**
   * The resonse after each simulation step.
   */
  case class StepResponse private[RLControllerModule] (
      observations: Observations,
      rewards: Rewards,
      terminateds: Terminateds,
      truncateds: Truncateds,
      infos: Infos,
  )

  /**
   * Controller trait defines the interface for a Reinforcement Learning controller.
   */
  trait Controller[S <: ModelModule.BaseState]:

    /**
     * The type of the image used for rendering the simulation on the RL client.
     */
    type Image = Array[Byte]

    /**
     * The initial state of the controller.
     */
    def initialState: S

    /**
     * The current state of the controller.
     */
    def state: S

    /**
     * Initializes the controller with the given simulation configuration.
     *
     * @param config
     *   the simulation configuration to initialize the controller.
     */
    def init(config: SimulationConfig[ValidEnvironment]): Unit

    /**
     * Resets the controller to its initial state using the provided random number generator.
     *
     * @param rng
     *   the random number generator to use for reproducibility in the next run.
     */
    def reset(rng: RNG): (Observations, Infos)

    /**
     * Performs a simulation step using the provided actions for each agent.
     *
     * @param actions
     *   a map of agents to their corresponding actions to be performed in this step.
     * @return
     *   a response containing the results of the simulation step.
     */
    def step(actions: Map[Agent, Action[IO]]): StepResponse

    /**
     * Renders the current state of the simulation to an image for the RL client.
     *
     * @param width
     *   the width of the rendered image.
     * @param height
     *   the height of the rendered image.
     *
     * @return
     *   an image representing the current state of the simulation.
     */
    def render(width: Int, height: Int): Image
  end Controller

  trait Provider[S <: ModelModule.BaseState]:
    val controller: Controller[S]

  type Requirements[S <: ModelModule.BaseState] = ModelModule.Provider[S]

  trait Component[S <: ModelModule.BaseState]:
    context: Requirements[S] =>

    object Controller:
      def apply()(using bundle: RLLogicsBundle[S]): Controller[S] = new ControllerImpl

      private class ControllerImpl(using bundle: RLLogicsBundle[S]) extends Controller[S]:

        var _initialState: S =
          bundle.stateLogic.createState(SimulationConfig(simulation, ValidEnvironment.empty))

        var _state: S = initialState

        override def initialState: S = _initialState

        override def state: S = _state

        override def init(config: SimulationConfig[ValidEnvironment]): Unit =
          _initialState = bundle.stateLogic.createState(config)
          _state = _initialState
          logger.debug(s"new config, state = \n $state")

        override def reset(rng: RNG): (Observations, Infos) =
          _state = context.model.update(state)(using _ => bundle.stateLogic.updateState(initialState, rng))
          logger.debug("resetting controller")
          (state.environment.getObservations, state.environment.getInfos)

        override def step(actions: Map[Agent, Action[IO]]): StepResponse =
          logger.debug("sending step command to controller")
          logger.debug(s"actions: $actions")
          _state = context.model.update(state)(using s => bundle.tickLogic.tick(s, state.dt)).unsafeRunSync()
          val actionsList =
            actions.map { (agent, action) =>
              DynamicEntityProposal(
                agent.copy(
                  lastAction = Some(action),
                  aliveSteps = agent.aliveSteps + 1,
                  visitedCountPositions =
                    nearbyVisitedPositions(discreteCell(agent.position), agent.visitedCountPositions)._1,
                ),
                action,
              )
            }.toList.sortBy(_.entity.id)
          val prevState = state
          _state = context.model
            .update(state)(using
              s => bundle.dynamicEntityActionsLogic.handleDynamicEntityActionsProposals(s, actionsList),
            )
            .unsafeRunSync()
          _state = context.model
            .update(state)(using s => bundle.randomLogic.random(s, SimpleRNG(s.simulationRNG.nextLong._1)))
            .unsafeRunSync()
          val observations = state.environment.getObservations
          val rewards = state.getRewards(prevState)
          val terminateds = state.getTerminations(prevState)
          val truncateds = state.getTruncations(prevState)
          val infos = state.environment.getInfos
          logger.debug(s"step completed, state = \n $state")
          logger.debug(s"observations: $observations")
          logger.debug(s"rewards: $rewards")
          logger.debug(s"terminateds: $terminateds")
          logger.debug(s"truncateds: $truncateds")
          logger.debug(s"infos: $infos")
          StepResponse(
            observations,
            rewards,
            terminateds,
            truncateds,
            infos,
          )

        end step

        override def render(width: Int, height: Int): Image =
          EnvironmentRenderer.renderToPNG(state.environment, width, height)
      end ControllerImpl
    end Controller

    extension (s: ModelModule.BaseState)

      def getTerminations(prev: ModelModule.BaseState): Terminateds =
        s.environment.entities.collect { case a: Agent =>
          a -> a.termination.evaluate(prev, s, a, a.lastAction.getOrElse(NoAction[IO]()))
        }.toMap

      def getTruncations(prev: ModelModule.BaseState): Truncateds =
        s.environment.entities.collect { case a: Agent =>
          a -> a.truncation.evaluate(prev, s, a, a.lastAction.getOrElse(NoAction[IO]()))
        }.toMap

      def getRewards(prev: ModelModule.BaseState): Rewards =
        s.environment.entities.collect { case a: Agent =>
          a -> a.reward.evaluate(prev, s, a, a.lastAction.getOrElse(NoAction[IO]()))
        }.toMap

    extension (env: ValidEnvironment)

      def getObservations: Observations =
        env.entities.collect { case a: Agent =>
          val sensors = a.senseAll[Id](env)
          val position = (a.position.x, a.position.y)
          val orientation = a.orientation.degrees
          val (_, visitedPositions) = nearbyVisitedPositions(discreteCell(position), a.visitedCountPositions)

          a -> AgentObservation(sensors, position, orientation, visitedPositions)
        }.toMap

      def getInfos: Infos =
        env.entities.collect { case a: Agent =>
          a -> ""
        }.toMap
  end Component

  trait Interface[S <: ModelModule.BaseState] extends Provider[S] with Component[S]:
    self: Requirements[S] =>
end RLControllerModule

1	package io.github.srs.controller
2
3	import cats.Id
4	import cats.effect.IO
5	import cats.effect.unsafe.implicits.global
6	import io.github.srs.config.SimulationConfig
7	import io.github.srs.controller.message.DynamicEntityProposal
8	import io.github.srs.logger
9	import io.github.srs.model.ModelModule
10	import io.github.srs.model.Simulation.simulation
11	import io.github.srs.model.entity.dynamicentity.action.{ Action, NoAction }
12	import io.github.srs.model.entity.dynamicentity.agent.Agent
13	import io.github.srs.model.entity.dynamicentity.sensor.Sensor.senseAll
14	import io.github.srs.model.entity.dynamicentity.sensor.SensorReadings
15	import io.github.srs.model.environment.ValidEnvironment
16	import io.github.srs.model.logic.RLLogicsBundle
17	import io.github.srs.utils.random.{ RNG, SimpleRNG }
18	import io.github.srs.view.rendering.EnvironmentRenderer
19	import io.github.srs.model.entity.Point2D.*
20	import io.github.srs.utils.SpatialUtils
21	import io.github.srs.utils.SpatialUtils.{ discreteCell, nearbyVisitedPositions }
22
23	object RLControllerModule:
24		×
25	final case class AgentObservation(
26	sensorReadings: SensorReadings,	36✔
27	position: (Double, Double),	×
28	orientation: Double,	×
NEW 29	visitedPositions: List[Double],	×
UNCOV 30	)	×
31
32	type Observations = Map[Agent, AgentObservation]
33	type Infos = Map[Agent, String]
34	type Rewards = Map[Agent, Double]
35	type Terminateds = Map[Agent, Boolean]
36	type Truncateds = Map[Agent, Boolean]
37
38	/**
39	* The resonse after each simulation step.
40	*/
41	case class StepResponse private[RLControllerModule] (
42	observations: Observations,	35✔
43	rewards: Rewards,	×
44	terminateds: Terminateds,	×
45	truncateds: Truncateds,	×
46	infos: Infos,	×
47	)	×
48
49	/**
50	* Controller trait defines the interface for a Reinforcement Learning controller.
51	*/
52	trait Controller[S <: ModelModule.BaseState]:
53
54	/**
55	* The type of the image used for rendering the simulation on the RL client.
56	*/
57	type Image = Array[Byte]
58
59	/**
60	* The initial state of the controller.
61	*/
62	def initialState: S
63
64	/**
65	* The current state of the controller.
66	*/
67	def state: S
68
69	/**
70	* Initializes the controller with the given simulation configuration.
71	*
72	* @param config
73	* the simulation configuration to initialize the controller.
74	*/
75	def init(config: SimulationConfig[ValidEnvironment]): Unit
76
77	/**
78	* Resets the controller to its initial state using the provided random number generator.
79	*
80	* @param rng
81	* the random number generator to use for reproducibility in the next run.
82	*/
83	def reset(rng: RNG): (Observations, Infos)
84
85	/**
86	* Performs a simulation step using the provided actions for each agent.
87	*
88	* @param actions
89	* a map of agents to their corresponding actions to be performed in this step.
90	* @return
91	* a response containing the results of the simulation step.
92	*/
93	def step(actions: Map[Agent, Action[IO]]): StepResponse
94
95	/**
96	* Renders the current state of the simulation to an image for the RL client.
97	*
98	* @param width
99	* the width of the rendered image.
100	* @param height
101	* the height of the rendered image.
102	*
103	* @return
104	* an image representing the current state of the simulation.
105	*/
106	def render(width: Int, height: Int): Image
107	end Controller
108
109	trait Provider[S <: ModelModule.BaseState]:
110	val controller: Controller[S]
111
112	type Requirements[S <: ModelModule.BaseState] = ModelModule.Provider[S]
113
114	trait Component[S <: ModelModule.BaseState]:
115	context: Requirements[S] =>	23✔
116
117	object Controller:
118	def apply()(using bundle: RLLogicsBundle[S]): Controller[S] = new ControllerImpl	5✔
119		9✔
120	private class ControllerImpl(using bundle: RLLogicsBundle[S]) extends Controller[S]:
121		8✔
122	var _initialState: S =
123	bundle.stateLogic.createState(SimulationConfig(simulation, ValidEnvironment.empty))	4✔
124		14✔
125	var _state: S = initialState
126		12✔
127	override def initialState: S = _initialState
128		3✔
129	override def state: S = _state
130		3✔
131	override def init(config: SimulationConfig[ValidEnvironment]): Unit =
132	_initialState = bundle.stateLogic.createState(config)
133	_state = _initialState	7✔
134	logger.debug(s"new config, state = \n $state")	4✔
135		8✔
136	override def reset(rng: RNG): (Observations, Infos) =
137	_state = context.model.update(state)(using _ => bundle.stateLogic.updateState(initialState, rng))
138	logger.debug("resetting controller")	13✔
139	(state.environment.getObservations, state.environment.getInfos)	7✔
140		17✔
141	override def step(actions: Map[Agent, Action[IO]]): StepResponse =
142	logger.debug("sending step command to controller")
143	logger.debug(s"actions: $actions")	7✔
144	_state = context.model.update(state)(using s => bundle.tickLogic.tick(s, state.dt)).unsafeRunSync()	7✔
145	val actionsList =	16✔
146	actions.map { (agent, action) =>
147	DynamicEntityProposal(	5✔
148	agent.copy(
149	lastAction = Some(action),
150	aliveSteps = agent.aliveSteps + 1,
151	visitedCountPositions =
152	nearbyVisitedPositions(discreteCell(agent.position), agent.visitedCountPositions)._1,
153	),
154	action,
155	)
156	}.toList.sortBy(_.entity.id)
157	val prevState = state	8✔
158	_state = context.model	3✔
159	.update(state)(using	6✔
160	s => bundle.dynamicEntityActionsLogic.handleDynamicEntityActionsProposals(s, actionsList),	2✔
161	)	4✔
162	.unsafeRunSync()
163	_state = context.model	5✔
164	.update(state)(using s => bundle.randomLogic.random(s, SimpleRNG(s.simulationRNG.nextLong._1)))	6✔
165	.unsafeRunSync()	5✔
166	val observations = state.environment.getObservations	5✔
167	val rewards = state.getRewards(prevState)	8✔
168	val terminateds = state.getTerminations(prevState)	8✔
169	val truncateds = state.getTruncations(prevState)	8✔
170	val infos = state.environment.getInfos	8✔
171	logger.debug(s"step completed, state = \n $state")	8✔
172	logger.debug(s"observations: $observations")	7✔
173	logger.debug(s"rewards: $rewards")	7✔
174	logger.debug(s"terminateds: $terminateds")	7✔
175	logger.debug(s"truncateds: $truncateds")	7✔
176	logger.debug(s"infos: $infos")	7✔
177	StepResponse(	7✔
178	observations,	1✔
179	rewards,	1✔
180	terminateds,	1✔
181	truncateds,	1✔
182	infos,	1✔
183	)	3✔
184
185	end step
186
187	override def render(width: Int, height: Int): Image =
188	EnvironmentRenderer.renderToPNG(state.environment, width, height)
189	end ControllerImpl	2✔
190	end Controller
191
192	extension (s: ModelModule.BaseState)
193
194	def getTerminations(prev: ModelModule.BaseState): Terminateds =
195	s.environment.entities.collect { case a: Agent =>
196	a -> a.termination.evaluate(prev, s, a, a.lastAction.getOrElse(NoAction[IO]()))	22✔
197	}.toMap	38✔
198		4✔
199	def getTruncations(prev: ModelModule.BaseState): Truncateds =
200	s.environment.entities.collect { case a: Agent =>
201	a -> a.truncation.evaluate(prev, s, a, a.lastAction.getOrElse(NoAction[IO]()))	22✔
202	}.toMap	38✔
203		4✔
204	def getRewards(prev: ModelModule.BaseState): Rewards =
205	s.environment.entities.collect { case a: Agent =>
206	a -> a.reward.evaluate(prev, s, a, a.lastAction.getOrElse(NoAction[IO]()))	22✔
207	}.toMap	38✔
208		4✔
209	extension (env: ValidEnvironment)
210
211	def getObservations: Observations =
212	env.entities.collect { case a: Agent =>
213	val sensors = a.senseAll[Id](env)	18✔
214	val position = (a.position.x, a.position.y)	13✔
215	val orientation = a.orientation.degrees	13✔
216	val (_, visitedPositions) = nearbyVisitedPositions(discreteCell(position), a.visitedCountPositions)	4✔
217		17✔
218	a -> AgentObservation(sensors, position, orientation, visitedPositions)
219	}.toMap	24✔
220		4✔
221	def getInfos: Infos =
222	env.entities.collect { case a: Agent =>
223	a -> ""	15✔
224	}.toMap	18✔
225	end Component	4✔
226
227	trait Interface[S <: ModelModule.BaseState] extends Provider[S] with Component[S]:
228	self: Requirements[S] =>
229	end RLControllerModule

Scala-Robotics-Simulator / PPS-22-srs / #1004

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous