#884

Committed 11 Nov 2025 10:15AM UTC coverage: 67.966% (-3.1%) from 71.024%

Build # #884

Build Type

push

github

Committed by

sceredi

Commit Message

refactor: update exploration notebook for Q-learning

Run Details

1587 of 2335 relevant lines covered (67.97%)

7.68 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

88.0

/src/main/scala/io/github/srs/controller/RLControllerModule.scala

package io.github.srs.controller

import cats.Id
import cats.effect.IO
import cats.effect.unsafe.implicits.global
import io.github.srs.config.SimulationConfig
import io.github.srs.controller.message.DynamicEntityProposal
import io.github.srs.logger
import io.github.srs.model.ModelModule
import io.github.srs.model.Simulation.simulation
import io.github.srs.model.entity.dynamicentity.action.{ Action, NoAction }
import io.github.srs.model.entity.dynamicentity.agent.Agent
import io.github.srs.model.entity.dynamicentity.sensor.Sensor.senseAll
import io.github.srs.model.entity.dynamicentity.sensor.SensorReadings
import io.github.srs.model.environment.ValidEnvironment
import io.github.srs.model.logic.RLLogicsBundle
import io.github.srs.utils.random.{ RNG, SimpleRNG }
import io.github.srs.view.rendering.EnvironmentRenderer
import io.github.srs.model.entity.Point2D.*

object RLControllerModule:

  final case class AgentObservation(
      sensorReadings: SensorReadings,
      position: (Double, Double),
      orientation: Double,
  )

  type Observations = Map[Agent, AgentObservation]
  type Infos = Map[Agent, String]
  type Rewards = Map[Agent, Double]
  type Terminateds = Map[Agent, Boolean]
  type Truncateds = Map[Agent, Boolean]

  /**
   * The resonse after each simulation step.
   */
  case class StepResponse private[RLControllerModule] (
      observations: Observations,
      rewards: Rewards,
      terminateds: Terminateds,
      truncateds: Truncateds,
      infos: Infos,
  )

  /**
   * Controller trait defines the interface for a Reinforcement Learning controller.
   */
  trait Controller[S <: ModelModule.BaseState]:

    /**
     * The type of the image used for rendering the simulation on the RL client.
     */
    type Image = Array[Byte]

    /**
     * The initial state of the controller.
     */
    def initialState: S

    /**
     * The current state of the controller.
     */
    def state: S

    /**
     * Initializes the controller with the given simulation configuration.
     *
     * @param config
     *   the simulation configuration to initialize the controller.
     */
    def init(config: SimulationConfig[ValidEnvironment]): Unit

    /**
     * Resets the controller to its initial state using the provided random number generator.
     *
     * @param rng
     *   the random number generator to use for reproducibility in the next run.
     */
    def reset(rng: RNG): (Observations, Infos)

    /**
     * Performs a simulation step using the provided actions for each agent.
     *
     * @param actions
     *   a map of agents to their corresponding actions to be performed in this step.
     * @return
     *   a response containing the results of the simulation step.
     */
    def step(actions: Map[Agent, Action[IO]]): StepResponse

    /**
     * Renders the current state of the simulation to an image for the RL client.
     *
     * @param width
     *   the width of the rendered image.
     * @param height
     *   the height of the rendered image.
     *
     * @return
     *   an image representing the current state of the simulation.
     */
    def render(width: Int, height: Int): Image
  end Controller

  trait Provider[S <: ModelModule.BaseState]:
    val controller: Controller[S]

  type Requirements[S <: ModelModule.BaseState] = ModelModule.Provider[S]

  trait Component[S <: ModelModule.BaseState]:
    context: Requirements[S] =>

    object Controller:
      def apply()(using bundle: RLLogicsBundle[S]): Controller[S] = new ControllerImpl

      private class ControllerImpl(using bundle: RLLogicsBundle[S]) extends Controller[S]:

        var _initialState: S =
          bundle.stateLogic.createState(SimulationConfig(simulation, ValidEnvironment.empty))

        var _state: S = initialState

        override def initialState: S = _initialState

        override def state: S = _state

        override def init(config: SimulationConfig[ValidEnvironment]): Unit =
          _initialState = bundle.stateLogic.createState(config)
          _state = _initialState
          logger.debug(s"new config, state = \n $state")

        override def reset(rng: RNG): (Observations, Infos) =
          _state = context.model.update(state)(using _ => bundle.stateLogic.updateState(initialState, rng))
          logger.debug("resetting controller")
          (state.environment.getObservations, state.environment.getInfos)

        override def step(actions: Map[Agent, Action[IO]]): StepResponse =
          logger.debug("sending step command to controller")
          logger.debug(s"actions: $actions")
          _state = context.model.update(state)(using s => bundle.tickLogic.tick(s, state.dt)).unsafeRunSync()
          val actionsList =
            actions.map { (agent, action) =>
              DynamicEntityProposal(agent.copy(lastAction = Some(action), aliveSteps = agent.aliveSteps + 1), action)
            }.toList.sortBy(_.entity.id)
          val prevState = state
          _state = context.model
            .update(state)(using
              s => bundle.dynamicEntityActionsLogic.handleDynamicEntityActionsProposals(s, actionsList),
            )
            .unsafeRunSync()
          _state = context.model
            .update(state)(using s => bundle.randomLogic.random(s, SimpleRNG(s.simulationRNG.nextLong._1)))
            .unsafeRunSync()
          val observations = state.environment.getObservations
          val rewards = state.getRewards(prevState)
          val terminateds = state.getTerminations(prevState)
          val truncateds = state.getTruncations(prevState)
          val infos = state.environment.getInfos
          logger.debug(s"step completed, state = \n $state")
          logger.debug(s"observations: $observations")
          logger.debug(s"rewards: $rewards")
          logger.debug(s"terminateds: $terminateds")
          logger.debug(s"truncateds: $truncateds")
          logger.debug(s"infos: $infos")
          StepResponse(
            observations,
            rewards,
            terminateds,
            truncateds,
            infos,
          )

        end step

        override def render(width: Int, height: Int): Image =
          EnvironmentRenderer.renderToPNG(state.environment, width, height)
      end ControllerImpl
    end Controller

    extension (s: ModelModule.BaseState)

      def getTerminations(prev: ModelModule.BaseState): Terminateds =
        s.environment.entities.collect { case a: Agent =>
          a -> a.termination.evaluate(prev, s, a, a.lastAction.getOrElse(NoAction[IO]()))
        }.toMap

      def getTruncations(prev: ModelModule.BaseState): Truncateds =
        s.environment.entities.collect { case a: Agent =>
          a -> a.truncation.evaluate(prev, s, a, a.lastAction.getOrElse(NoAction[IO]()))
        }.toMap

      def getRewards(prev: ModelModule.BaseState): Rewards =
        s.environment.entities.collect { case a: Agent =>
          a -> a.reward.evaluate(prev, s, a, a.lastAction.getOrElse(NoAction[IO]()))
        }.toMap

    extension (env: ValidEnvironment)

      def getObservations: Observations =
        env.entities.collect { case a: Agent =>
          val sensors = a.senseAll[Id](env)
          val position = (a.position.x, a.position.y)
          val orientation = a.orientation.degrees
          a -> AgentObservation(sensors, position, orientation)
        }.toMap

      def getInfos: Infos =
        env.entities.collect { case a: Agent =>
          a -> ""
        }.toMap
  end Component

  trait Interface[S <: ModelModule.BaseState] extends Provider[S] with Component[S]:
    self: Requirements[S] =>
end RLControllerModule

1	package io.github.srs.controller
2
3	import cats.Id
4	import cats.effect.IO
5	import cats.effect.unsafe.implicits.global
6	import io.github.srs.config.SimulationConfig
7	import io.github.srs.controller.message.DynamicEntityProposal
8	import io.github.srs.logger
9	import io.github.srs.model.ModelModule
10	import io.github.srs.model.Simulation.simulation
11	import io.github.srs.model.entity.dynamicentity.action.{ Action, NoAction }
12	import io.github.srs.model.entity.dynamicentity.agent.Agent
13	import io.github.srs.model.entity.dynamicentity.sensor.Sensor.senseAll
14	import io.github.srs.model.entity.dynamicentity.sensor.SensorReadings
15	import io.github.srs.model.environment.ValidEnvironment
16	import io.github.srs.model.logic.RLLogicsBundle
17	import io.github.srs.utils.random.{ RNG, SimpleRNG }
18	import io.github.srs.view.rendering.EnvironmentRenderer
19	import io.github.srs.model.entity.Point2D.*
20
21	object RLControllerModule:
22		×
23	final case class AgentObservation(
24	sensorReadings: SensorReadings,	32✔
25	position: (Double, Double),	×
26	orientation: Double,	×
27	)	×
28
29	type Observations = Map[Agent, AgentObservation]
30	type Infos = Map[Agent, String]
31	type Rewards = Map[Agent, Double]
32	type Terminateds = Map[Agent, Boolean]
33	type Truncateds = Map[Agent, Boolean]
34
35	/**
36	* The resonse after each simulation step.
37	*/
38	case class StepResponse private[RLControllerModule] (
39	observations: Observations,	35✔
40	rewards: Rewards,	×
41	terminateds: Terminateds,	×
42	truncateds: Truncateds,	×
43	infos: Infos,	×
44	)	×
45
46	/**
47	* Controller trait defines the interface for a Reinforcement Learning controller.
48	*/
49	trait Controller[S <: ModelModule.BaseState]:
50
51	/**
52	* The type of the image used for rendering the simulation on the RL client.
53	*/
54	type Image = Array[Byte]
55
56	/**
57	* The initial state of the controller.
58	*/
59	def initialState: S
60
61	/**
62	* The current state of the controller.
63	*/
64	def state: S
65
66	/**
67	* Initializes the controller with the given simulation configuration.
68	*
69	* @param config
70	* the simulation configuration to initialize the controller.
71	*/
72	def init(config: SimulationConfig[ValidEnvironment]): Unit
73
74	/**
75	* Resets the controller to its initial state using the provided random number generator.
76	*
77	* @param rng
78	* the random number generator to use for reproducibility in the next run.
79	*/
80	def reset(rng: RNG): (Observations, Infos)
81
82	/**
83	* Performs a simulation step using the provided actions for each agent.
84	*
85	* @param actions
86	* a map of agents to their corresponding actions to be performed in this step.
87	* @return
88	* a response containing the results of the simulation step.
89	*/
90	def step(actions: Map[Agent, Action[IO]]): StepResponse
91
92	/**
93	* Renders the current state of the simulation to an image for the RL client.
94	*
95	* @param width
96	* the width of the rendered image.
97	* @param height
98	* the height of the rendered image.
99	*
100	* @return
101	* an image representing the current state of the simulation.
102	*/
103	def render(width: Int, height: Int): Image
104	end Controller
105
106	trait Provider[S <: ModelModule.BaseState]:
107	val controller: Controller[S]
108
109	type Requirements[S <: ModelModule.BaseState] = ModelModule.Provider[S]
110
111	trait Component[S <: ModelModule.BaseState]:
112	context: Requirements[S] =>	23✔
113
114	object Controller:
115	def apply()(using bundle: RLLogicsBundle[S]): Controller[S] = new ControllerImpl	5✔
116		9✔
117	private class ControllerImpl(using bundle: RLLogicsBundle[S]) extends Controller[S]:
118		8✔
119	var _initialState: S =
120	bundle.stateLogic.createState(SimulationConfig(simulation, ValidEnvironment.empty))	4✔
121		14✔
122	var _state: S = initialState
123		12✔
124	override def initialState: S = _initialState
125		3✔
126	override def state: S = _state
127		3✔
128	override def init(config: SimulationConfig[ValidEnvironment]): Unit =
129	_initialState = bundle.stateLogic.createState(config)
130	_state = _initialState	7✔
131	logger.debug(s"new config, state = \n $state")	4✔
132		8✔
133	override def reset(rng: RNG): (Observations, Infos) =
134	_state = context.model.update(state)(using _ => bundle.stateLogic.updateState(initialState, rng))
135	logger.debug("resetting controller")	13✔
136	(state.environment.getObservations, state.environment.getInfos)	7✔
137		17✔
138	override def step(actions: Map[Agent, Action[IO]]): StepResponse =
139	logger.debug("sending step command to controller")
140	logger.debug(s"actions: $actions")	7✔
141	_state = context.model.update(state)(using s => bundle.tickLogic.tick(s, state.dt)).unsafeRunSync()	7✔
142	val actionsList =	16✔
143	actions.map { (agent, action) =>
144	DynamicEntityProposal(agent.copy(lastAction = Some(action), aliveSteps = agent.aliveSteps + 1), action)	5✔
145	}.toList.sortBy(_.entity.id)
146	val prevState = state	8✔
147	_state = context.model	3✔
148	.update(state)(using	6✔
149	s => bundle.dynamicEntityActionsLogic.handleDynamicEntityActionsProposals(s, actionsList),	2✔
150	)	4✔
151	.unsafeRunSync()
152	_state = context.model	5✔
153	.update(state)(using s => bundle.randomLogic.random(s, SimpleRNG(s.simulationRNG.nextLong._1)))	6✔
154	.unsafeRunSync()	5✔
155	val observations = state.environment.getObservations	5✔
156	val rewards = state.getRewards(prevState)	8✔
157	val terminateds = state.getTerminations(prevState)	8✔
158	val truncateds = state.getTruncations(prevState)	8✔
159	val infos = state.environment.getInfos	8✔
160	logger.debug(s"step completed, state = \n $state")	8✔
161	logger.debug(s"observations: $observations")	7✔
162	logger.debug(s"rewards: $rewards")	7✔
163	logger.debug(s"terminateds: $terminateds")	7✔
164	logger.debug(s"truncateds: $truncateds")	7✔
165	logger.debug(s"infos: $infos")	7✔
166	StepResponse(	7✔
167	observations,	1✔
168	rewards,	1✔
169	terminateds,	1✔
170	truncateds,	1✔
171	infos,	1✔
172	)	3✔
173
174	end step
175
176	override def render(width: Int, height: Int): Image =
177	EnvironmentRenderer.renderToPNG(state.environment, width, height)
178	end ControllerImpl	2✔
179	end Controller
180
181	extension (s: ModelModule.BaseState)
182
183	def getTerminations(prev: ModelModule.BaseState): Terminateds =
184	s.environment.entities.collect { case a: Agent =>
185	a -> a.termination.evaluate(prev, s, a, a.lastAction.getOrElse(NoAction[IO]()))	22✔
186	}.toMap	38✔
187		4✔
188	def getTruncations(prev: ModelModule.BaseState): Truncateds =
189	s.environment.entities.collect { case a: Agent =>
190	a -> a.truncation.evaluate(prev, s, a, a.lastAction.getOrElse(NoAction[IO]()))	22✔
191	}.toMap	38✔
192		4✔
193	def getRewards(prev: ModelModule.BaseState): Rewards =
194	s.environment.entities.collect { case a: Agent =>
195	a -> a.reward.evaluate(prev, s, a, a.lastAction.getOrElse(NoAction[IO]()))	22✔
196	}.toMap	38✔
197		4✔
198	extension (env: ValidEnvironment)
199
200	def getObservations: Observations =
201	env.entities.collect { case a: Agent =>
202	val sensors = a.senseAll[Id](env)	18✔
203	val position = (a.position.x, a.position.y)	13✔
204	val orientation = a.orientation.degrees	13✔
205	a -> AgentObservation(sensors, position, orientation)	4✔
206	}.toMap	23✔
207		4✔
208	def getInfos: Infos =
209	env.entities.collect { case a: Agent =>
210	a -> ""	15✔
211	}.toMap	18✔
212	end Component	4✔
213
214	trait Interface[S <: ModelModule.BaseState] extends Provider[S] with Component[S]:
215	self: Requirements[S] =>
216	end RLControllerModule

Scala-Robotics-Simulator / PPS-22-srs / #884

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous