6742258790

Committed 01 Nov 2023 05:29PM UTC coverage: 84.035% (+0.8%) from 83.259%

Build # 6742258790

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #529 from SpiNNakerManchester/roc_part

Typing information and some changes to make that easier

Run Details

1552 of 2025 branches covered (0.0%)

Branch coverage included in aggregate %.

1709 of 1844 new or added lines in 95 files covered. (92.68%)

18 existing lines in 12 files now uncovered.

5059 of 5842 relevant lines covered (86.6%)

0.87 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

72.91

/pacman/operations/placer_algorithms/application_placer.py

# Copyright (c) 2021 The University of Manchester
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import logging
import os
from typing import Iterable, List, Optional, Set, Tuple

from spinn_utilities.config_holder import get_config_bool, get_config_str
from spinn_utilities.log import FormatAdapter
from spinn_utilities.ordered_set import OrderedSet
from spinn_utilities.progress_bar import ProgressBar

from spinn_machine import Chip

from pacman.data import PacmanDataView
from pacman.model.placements import Placements, Placement
from pacman.model.graphs import AbstractVirtual
from pacman.model.graphs.machine import MachineVertex
from pacman.model.graphs.application import ApplicationVertex
from pacman.exceptions import (
    PacmanPlaceException, PacmanConfigurationException, PacmanTooBigToPlace)

logger = FormatAdapter(logging.getLogger(__name__))


def place_application_graph(system_placements: Placements) -> Placements:
    """
    Perform placement of an application graph on the machine.

    .. note::
        app_graph must have been partitioned

    :param Placements system_placements:
        The placements of cores doing system tasks. This is what we start from.
    :return: Placements for the application. *Includes the system placements.*
    :rtype: Placements
    """
    # Track the placements and space
    placements = Placements(system_placements)

    plan_n_timesteps = PacmanDataView.get_plan_n_timestep()
    spaces = _Spaces(placements, plan_n_timesteps)

    # Go through the application graph by application vertex
    progress = ProgressBar(
        PacmanDataView.get_n_vertices(), "Placing Vertices")
    for app_vertex in progress.over(PacmanDataView.iterate_vertices()):
        spaces.restore_chips()

        # Try placements from the next chip, but try again if fails
        placed = False
        while not placed:
            chips_attempted = list()
            try:
                same_chip_groups = app_vertex.splitter.get_same_chip_groups()
                if not same_chip_groups:
                    placed = True
                    break

                # Start a new space
                try:
                    next_chip_space, space = spaces.get_next_chip_and_space()
                except PacmanPlaceException as e:
                    raise _place_error(
                        placements, system_placements, e,
                        plan_n_timesteps) from e
                logger.debug(f"Starting placement from {next_chip_space}")

                placements_to_make: List = list()

                # Go through the groups
                last_chip_space: Optional[_ChipWithSpace] = None
                for vertices, sdram in same_chip_groups:
                    vertices_to_place = [
                        vertex
                        for vertex in vertices
                        # No need to place virtual vertices
                        if not isinstance(vertex, AbstractVirtual)
                        and not placements.is_vertex_placed(vertex)]
                    actual_sdram = sdram.get_total_sdram(plan_n_timesteps)
                    n_cores = len(vertices_to_place)

                    # If this group has a fixed location, place it there
                    if _do_fixed_location(vertices_to_place, actual_sdram,
                                          placements, next_chip_space):
                        continue

                    # Try to find a chip with space; this might result in a
                    # _SpaceExceededException
                    while not next_chip_space.is_space(n_cores, actual_sdram):
                        next_chip_space = spaces.get_next_chip_space(
                            space, last_chip_space)
                        last_chip_space = None

                    # If this worked, store placements to be made
                    last_chip_space = next_chip_space
                    chips_attempted.append(next_chip_space.chip)
                    _store_on_chip(
                        placements_to_make, vertices_to_place, actual_sdram,
                        next_chip_space)

                # Now make the placements having confirmed all can be done
                placements.add_placements(placements_to_make)
                placed = True
                logger.debug(f"Used {chips_attempted}")
            except _SpaceExceededException:
                # This might happen while exploring a space; this may not be
                # fatal since the last space might have just been bound by
                # existing placements, and there might be bigger spaces out
                # there to use
                _check_could_fit(app_vertex, vertices_to_place, actual_sdram)
                logger.debug(f"Failed, saving {chips_attempted}")
                spaces.save_chips(chips_attempted)
                chips_attempted.clear()

    if get_config_bool("Reports", "draw_placements"):
        from .draw_placements import draw_placements as dp
        dp(placements, system_placements)

    return placements


def _place_error(
        placements: Placements, system_placements: Placements,
        exception: Exception,
        plan_n_timesteps: Optional[int]) -> PacmanPlaceException:
    """
    :param Placements placements:
    :param Placements system_placements:
    :param PacmanPlaceException exception:
    :param int plan_n_timesteps:
    :rtype: PacmanPlaceException
    """
    unplaceable = list()
    vertex_count = 0
    n_vertices = 0
    for app_vertex in PacmanDataView.iterate_vertices():
        same_chip_groups = app_vertex.splitter.get_same_chip_groups()
        app_vertex_placed = True
        found_placed_cores = False
        for vertices, _sdram in same_chip_groups:
            if isinstance(vertices[0], AbstractVirtual):
                break
            if placements.is_vertex_placed(vertices[0]):
                found_placed_cores = True
            elif found_placed_cores:
                vertex_count += len(vertices)
                n_vertices = len(same_chip_groups)
                app_vertex_placed = False
                break
            else:
                app_vertex_placed = False
                break
        if not app_vertex_placed:
            unplaceable.append(app_vertex)

    report_file = os.path.join(
        PacmanDataView.get_run_dir_path(), "placements_error.txt")
    with open(report_file, 'w', encoding="utf-8") as f:
        f.write(f"Could not place {len(unplaceable)} of "
                f"{PacmanDataView.get_n_vertices()} application vertices.\n")
        f.write(f"    Could not place {vertex_count} of {n_vertices} in the"
                " last app vertex\n\n")
        for x, y in placements.chips_with_placements:
            first = True
            for placement in placements.placements_on_chip(x, y):
                if system_placements.is_vertex_placed(placement.vertex):
                    continue
                if first:
                    f.write(f"Chip ({x}, {y}):\n")
                    first = False
                f.write(f"    Processor {placement.p}:"
                        f" Vertex {placement.vertex}\n")
            if not first:
                f.write("\n")
        f.write("\n")
        f.write("Not placed:\n")
        for app_vertex in unplaceable:
            f.write(f"Vertex: {app_vertex}\n")
            same_chip_groups = app_vertex.splitter.get_same_chip_groups()
            for vertices, sdram in same_chip_groups:
                f.write(f"    Group of {len(vertices)} vertices uses "
                        f"{sdram.get_total_sdram(plan_n_timesteps)} "
                        "bytes of SDRAM:\n")
                for vertex in vertices:
                    f.write(f"        Vertex {vertex}")
                    if placements.is_vertex_placed(vertex):
                        plce = placements.get_placement_of_vertex(vertex)
                        f.write(f" (placed at {plce.x}, {plce.y}, {plce.p})")
                    f.write("\n")

        f.write("\n")
        f.write("Unused chips:\n")
        machine = PacmanDataView.get_machine()
        for x, y in machine.chip_coordinates:
            n_placed = placements.n_placements_on_chip(x, y)
            system_placed = system_placements.n_placements_on_chip(x, y)
            if n_placed - system_placed == 0:
                n_procs = machine[x, y].n_user_processors
                f.write(f"    {x}, {y} ({n_procs - system_placed}"
                        " free cores)\n")

    if get_config_bool("Reports", "draw_placements_on_error"):
        from .draw_placements import draw_placements as dp
        dp(placements, system_placements)

    return PacmanPlaceException(
        f" {exception}."
        f" Report written to {report_file}.")


def _check_could_fit(
        app_vertex: ApplicationVertex, vertices_to_place: List[MachineVertex],
        sdram: int):
    """
    :param ApplicationVertex app_vertex:
    :param list(MachineVertex) vertices_to_place:
    :param int sdram:
    :raises PacmanTooBigToPlace:
    """
    version = PacmanDataView.get_machine_version()
    max_sdram = (
            version.max_sdram_per_chip - PacmanDataView.get_monitor_sdram())
    max_cores = (
            version.max_cores_per_chip - version.n_non_user_cores -
            PacmanDataView.get_monitor_cores())
    n_cores = len(vertices_to_place)
    if sdram <= max_sdram and n_cores <= max_cores:
        # should fit somewhere
        return
    message = (
        f"{app_vertex} will not fit on any possible Chip "
        f"the reason is that {vertices_to_place} ")
    if sdram > max_sdram:
        message += f"requires {sdram} bytes but "
        if sdram > version.max_sdram_per_chip:
            message += f"a Chip only has {version.max_sdram_per_chip} bytes "
        else:
            message += f"after monitors only {max_sdram} bytes are available "
        message += "Lowering max_core_per_chip may resolve this."
        raise PacmanTooBigToPlace(message)
    if n_cores > version.max_cores_per_chip:
        message += " is more vertices than the number of cores on a chip."
        raise PacmanTooBigToPlace(message)
    user_cores = version.max_cores_per_chip - version.n_non_user_cores
    if n_cores > user_cores:
        message += (
            f"is more vertices than the user cores ({user_cores}) "
            "available on a Chip")
    else:
        message += (
            f"is more vertices than the {max_cores} cores available on a "
            f"Chip once {PacmanDataView.get_monitor_cores()} "
            "are reserved for monitors")
    raise PacmanTooBigToPlace(message)


class _SpaceExceededException(Exception):
    pass


def _do_fixed_location(
        vertices: list[MachineVertex], sdram: int, placements: Placements,
        next_chip_space: _ChipWithSpace) -> bool:
    """
    :param list(MachineVertex) vertices:
    :param int sdram:
    :param Placements placements:
    :param _ChipWithSpace next_chip_space:
    :rtype: bool
    :raise PacmanConfigurationException:
    """
    for vertex in vertices:
        loc = vertex.get_fixed_location()
        if loc:
            x, y = loc.x, loc.y
            break
    else:
        return False

    machine = PacmanDataView.get_machine()
    chip = machine.get_chip_at(x, y)
    if chip is None:
        raise PacmanConfigurationException(
            f"Constrained to chip {x, y} but no such chip")
    on_chip = placements.placements_on_chip(x, y)
    cores_used = {p.p for p in on_chip}
    cores = set(p.processor_id for p in chip.processors
                if not p.is_monitor) - cores_used
    next_cores = iter(cores)
    for vertex in vertices:
        next_core = None
        fixed = vertex.get_fixed_location()
        if fixed and fixed.p is not None:
            if fixed.p not in next_cores:
                raise PacmanConfigurationException(
                    f"Core {fixed.p} on {x}, {y} not available to "
                    f"place {vertex} on")
            next_core = fixed.p
        else:
            try:
                next_core = next(next_cores)
            except StopIteration:
                # pylint: disable=raise-missing-from
                raise PacmanConfigurationException(
                    f"No more cores available on {x}, {y}: {on_chip}")
        placements.add_placement(Placement(vertex, x, y, next_core))
        if next_chip_space.x == x and next_chip_space.y == y:
            next_chip_space.cores.remove(next_core)
            next_chip_space.use_sdram(sdram)
    return True


def _store_on_chip(
        placements_to_make: List[Placement], vertices: List[MachineVertex],
        sdram: int, next_chip_space: _ChipWithSpace):
    """
    :param list(Placement) placements_to_make:
    :param list(MachineVertex) vertices:
    :param int sdram:
    :param _ChipWithSpace next_chip_space:
    """
    for vertex in vertices:
        core = next_chip_space.use_next_core()
        placements_to_make.append(Placement(
            vertex, next_chip_space.x, next_chip_space.y, core))
    next_chip_space.use_sdram(sdram)


class _Spaces(object):
    __slots__ = ("__machine", "__chips", "__next_chip", "__used_chips",
                 "__system_placements", "__placements", "__plan_n_timesteps",
                 "__last_chip_space", "__saved_chips", "__restored_chips")

    def __init__(
            self, placements: Placements, plan_n_timesteps: Optional[int]):
        """
        :param Placements placements:
        :param int plan_n_timesteps:
        """
        self.__machine = PacmanDataView.get_machine()
        self.__placements = placements
        self.__plan_n_timesteps = plan_n_timesteps
        self.__chips = iter(self.__chip_order())
        self.__next_chip = next(self.__chips)
        self.__used_chips: Set[Chip] = set()
        self.__last_chip_space: Optional[_ChipWithSpace] = None
        self.__saved_chips: OrderedSet[Chip] = OrderedSet()
        self.__restored_chips: OrderedSet[Chip] = OrderedSet()

    def __chip_order(self):
        """
        :param Machine machine:
        :rtype: iterable(Chip)
        """
        s_x, s_y = get_config_str("Mapping", "placer_start_chip").split(",")
        s_x = int(s_x)
        s_y = int(s_y)

        for x in range(self.__machine.width):
            for y in range(self.__machine.height):
                c_x = (x + s_x) % self.__machine.width
                c_y = (y + s_y) % self.__machine.height
                chip = self.__machine.get_chip_at(c_x, c_y)
                if chip:
                    yield chip

    def __cores_and_sdram(self, chip: Chip) -> Tuple[Set[int], int]:
        """
        :param Chip chip:
        :return cores, sdram
        :rtype: tuple(int, int)
        """
        on_chip = self.__placements.placements_on_chip(chip.x, chip.y)
        cores_used = {p.p for p in on_chip}
        sdram_used = sum(
            p.vertex.sdram_required.get_total_sdram(
                self.__plan_n_timesteps) for p in on_chip)
        return cores_used, sdram_used

    def get_next_chip_and_space(self) -> Tuple[_ChipWithSpace, _Space]:
        """
        :rtype: (_ChipWithSpace, _Space)
        """
        try:
            if self.__last_chip_space is None:
                chip = self.__get_next_chip()
                cores_used, sdram_used = self.__cores_and_sdram(chip)
                self.__last_chip_space = _ChipWithSpace(
                    chip, cores_used, sdram_used)
                self.__used_chips.add(chip)

            # Start a new space by finding all the chips that can be reached
            # from the start chip but have not been used
            return (self.__last_chip_space,
                    _Space(self.__last_chip_space.chip))

        except StopIteration:
            raise PacmanPlaceException(  # pylint: disable=raise-missing-from
                f"No more chips to place on; {self.n_chips_used} of "
                f"{self.__machine.n_chips} used")

    def __get_next_chip(self) -> Chip:
        """
        :rtype: Chip
        :raises: StopIteration
        """
        while self.__restored_chips:
            chip = self.__restored_chips.pop(last=False)
            if chip not in self.__used_chips:
                return chip
        while self.__next_chip in self.__used_chips:
            self.__next_chip = next(self.__chips)
        return self.__next_chip

    def get_next_chip_space(
            self, space: _Space,
            last_chip_space: Optional[_ChipWithSpace]) -> _ChipWithSpace:
        """
        :param _Space space:
        :param _ChipWithSpace last_chip_space:
        :rtype: _ChipWithSpace
        :raises _SpaceExceededException:
        """
        # If we are reporting a used chip, update with reachable chips
        if last_chip_space is not None:
            last_chip = last_chip_space.chip
            space.update(self.__usable_from_chip(last_chip))

        # If no space, error
        if not space:
            self.__last_chip_space = None
            raise _SpaceExceededException(
                "No more chips to place on in this space; "
                f"{self.n_chips_used} of {self.__machine.n_chips} used")
        chip = space.pop()
        self.__used_chips.add(chip)
        self.__restored_chips.discard(chip)
        cores_used, sdram_used = self.__cores_and_sdram(chip)
        self.__last_chip_space = _ChipWithSpace(chip, cores_used, sdram_used)
        return self.__last_chip_space

    @property
    def n_chips_used(self) -> int:
        """
        The number of chips used.

        :rtype: int
        """
        return len(self.__used_chips)

    def __usable_from_chip(self, chip: Chip) -> Iterable[Chip]:
        """
        :param Chip chip:
        :rtype set(Chip)
        """
        for link in chip.router.links:
            target = self.__machine[link.destination_x, link.destination_y]
            if target not in self.__used_chips:
                yield target

    def save_chips(self, chips: Iterable[Chip]):
        """
        :param iterable(Chip) chips:
        """
        self.__saved_chips.update(chips)

    def restore_chips(self) -> None:
        for chip in self.__saved_chips:
            self.__used_chips.remove(chip)
            self.__restored_chips.add(chip)
        self.__saved_chips.clear()


class _Space(object):
    __slots__ = ("__same_board_chips", "__remaining_chips",
                 "__board_x", "__board_y", "__first_chip")

    def __init__(self, chip: Chip):
        self.__board_x = chip.nearest_ethernet_x
        self.__board_y = chip.nearest_ethernet_y
        self.__same_board_chips: OrderedSet[Chip] = OrderedSet()
        self.__remaining_chips: OrderedSet[Chip] = OrderedSet()

    def __len__(self) -> int:
        return len(self.__same_board_chips) + len(self.__remaining_chips)

    def __on_same_board(self, chip: Chip) -> bool:
        return (chip.nearest_ethernet_x == self.__board_x and
                chip.nearest_ethernet_y == self.__board_y)

    def pop(self) -> Chip:
        """
        :rtype: Chip
        :raise: StopIteration
        """
        if self.__same_board_chips:
            return self.__same_board_chips.pop(last=False)
        if self.__remaining_chips:
            next_chip = self.__remaining_chips.pop(last=False)
            self.__board_x = next_chip.nearest_ethernet_x
            self.__board_y = next_chip.nearest_ethernet_y
            to_remove = list()
            for chip in self.__remaining_chips:
                if self.__on_same_board(chip):
                    to_remove.append(chip)
                    self.__same_board_chips.add(chip)
            for chip in to_remove:
                self.__remaining_chips.remove(chip)
            return next_chip
        raise StopIteration

    def update(self, chips: Iterable[Chip]):
        """
        :param iterable(Chip) chips:
        """
        for chip in chips:
            if self.__on_same_board(chip):
                self.__same_board_chips.add(chip)
            else:
                self.__remaining_chips.add(chip)


class _ChipWithSpace(object):
    """
    A chip with space for placement.
    """
    __slots__ = ("chip", "cores", "sdram")

    def __init__(
            self, chip: Chip, used_processors: Set[int], used_sdram: int):
        self.chip = chip
        self.cores = set(p.processor_id for p in chip.processors
                         if not p.is_monitor)
        self.cores -= used_processors
        self.sdram = chip.sdram - used_sdram

    @property
    def x(self) -> int:
        return self.chip.x

    @property
    def y(self) -> int:
        return self.chip.y

    def is_space(self, n_cores: int, sdram: int) -> bool:
        return len(self.cores) >= n_cores and self.sdram >= sdram

    def use_sdram(self, sdram: int):
        self.sdram -= sdram

    def use_next_core(self) -> int:
        core = next(iter(self.cores))
        self.cores.remove(core)
        return core

    def __repr__(self):
        return f"({self.x}, {self.y})"

1	# Copyright (c) 2021 The University of Manchester
2	#
3	# Licensed under the Apache License, Version 2.0 (the "License");
4	# you may not use this file except in compliance with the License.
5	# You may obtain a copy of the License at
6	#
7	# https://www.apache.org/licenses/LICENSE-2.0
8	#
9	# Unless required by applicable law or agreed to in writing, software
10	# distributed under the License is distributed on an "AS IS" BASIS,
11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	# See the License for the specific language governing permissions and
13	# limitations under the License.
14	from __future__ import annotations	1✔
15	import logging	1✔
16	import os	1✔
17	from typing import Iterable, List, Optional, Set, Tuple	1✔
18
19	from spinn_utilities.config_holder import get_config_bool, get_config_str	1✔
20	from spinn_utilities.log import FormatAdapter	1✔
21	from spinn_utilities.ordered_set import OrderedSet	1✔
22	from spinn_utilities.progress_bar import ProgressBar	1✔
23
24	from spinn_machine import Chip	1✔
25
26	from pacman.data import PacmanDataView	1✔
27	from pacman.model.placements import Placements, Placement	1✔
28	from pacman.model.graphs import AbstractVirtual	1✔
29	from pacman.model.graphs.machine import MachineVertex	1✔
30	from pacman.model.graphs.application import ApplicationVertex	1✔
31	from pacman.exceptions import (	1✔
32	PacmanPlaceException, PacmanConfigurationException, PacmanTooBigToPlace)
33
34	logger = FormatAdapter(logging.getLogger(__name__))	1✔
35
36
37	def place_application_graph(system_placements: Placements) -> Placements:	1✔
38	"""
39	Perform placement of an application graph on the machine.
40
41	.. note::
42	app_graph must have been partitioned
43
44	:param Placements system_placements:
45	The placements of cores doing system tasks. This is what we start from.
46	:return: Placements for the application. Includes the system placements.
47	:rtype: Placements
48	"""
49	# Track the placements and space
50	placements = Placements(system_placements)	1✔
51
52	plan_n_timesteps = PacmanDataView.get_plan_n_timestep()	1✔
53	spaces = _Spaces(placements, plan_n_timesteps)	1✔
54
55	# Go through the application graph by application vertex
56	progress = ProgressBar(	1✔
57	PacmanDataView.get_n_vertices(), "Placing Vertices")
58	for app_vertex in progress.over(PacmanDataView.iterate_vertices()):	1✔
59	spaces.restore_chips()	1✔
60
61	# Try placements from the next chip, but try again if fails
62	placed = False	1✔
63	while not placed:	1✔
64	chips_attempted = list()	1✔
65	try:	1✔
66	same_chip_groups = app_vertex.splitter.get_same_chip_groups()	1✔
67	if not same_chip_groups:	1!
UNCOV 68	placed = True	×
69	break	×
70
71	# Start a new space
72	try:	1✔
73	next_chip_space, space = spaces.get_next_chip_and_space()	1✔
74	except PacmanPlaceException as e:	×
NEW 75	raise _place_error(	×
76	placements, system_placements, e,
77	plan_n_timesteps) from e
78	logger.debug(f"Starting placement from {next_chip_space}")	1✔
79
80	placements_to_make: List = list()	1✔
81
82	# Go through the groups
83	last_chip_space: Optional[_ChipWithSpace] = None	1✔
84	for vertices, sdram in same_chip_groups:	1✔
85	vertices_to_place = [	1✔
86	vertex
87	for vertex in vertices
88	# No need to place virtual vertices
89	if not isinstance(vertex, AbstractVirtual)
90	and not placements.is_vertex_placed(vertex)]
91	actual_sdram = sdram.get_total_sdram(plan_n_timesteps)	1✔
92	n_cores = len(vertices_to_place)	1✔
93
94	# If this group has a fixed location, place it there
95	if _do_fixed_location(vertices_to_place, actual_sdram,	1✔
96	placements, next_chip_space):
97	continue	1✔
98
99	# Try to find a chip with space; this might result in a
100	# _SpaceExceededException
101	while not next_chip_space.is_space(n_cores, actual_sdram):	1✔
102	next_chip_space = spaces.get_next_chip_space(	1✔
103	space, last_chip_space)
104	last_chip_space = None	1✔
105
106	# If this worked, store placements to be made
107	last_chip_space = next_chip_space	1✔
108	chips_attempted.append(next_chip_space.chip)	1✔
109	_store_on_chip(	1✔
110	placements_to_make, vertices_to_place, actual_sdram,
111	next_chip_space)
112
113	# Now make the placements having confirmed all can be done
114	placements.add_placements(placements_to_make)	1✔
115	placed = True	1✔
116	logger.debug(f"Used {chips_attempted}")	1✔
117	except _SpaceExceededException:	1✔
118	# This might happen while exploring a space; this may not be
119	# fatal since the last space might have just been bound by
120	# existing placements, and there might be bigger spaces out
121	# there to use
122	_check_could_fit(app_vertex, vertices_to_place, actual_sdram)	1✔
123	logger.debug(f"Failed, saving {chips_attempted}")	1✔
124	spaces.save_chips(chips_attempted)	1✔
125	chips_attempted.clear()	1✔
126
127	if get_config_bool("Reports", "draw_placements"):	1!
NEW 128	from .draw_placements import draw_placements as dp	×
NEW 129	dp(placements, system_placements)	×
130
131	return placements	1✔
132
133
134	def _place_error(	1✔
135	placements: Placements, system_placements: Placements,
136	exception: Exception,
137	plan_n_timesteps: Optional[int]) -> PacmanPlaceException:
138	"""
139	:param Placements placements:
140	:param Placements system_placements:
141	:param PacmanPlaceException exception:
142	:param int plan_n_timesteps:
143	:rtype: PacmanPlaceException
144	"""
145	unplaceable = list()	×
146	vertex_count = 0	×
147	n_vertices = 0	×
148	for app_vertex in PacmanDataView.iterate_vertices():	×
149	same_chip_groups = app_vertex.splitter.get_same_chip_groups()	×
150	app_vertex_placed = True	×
151	found_placed_cores = False	×
152	for vertices, _sdram in same_chip_groups:	×
153	if isinstance(vertices[0], AbstractVirtual):	×
154	break	×
155	if placements.is_vertex_placed(vertices[0]):	×
156	found_placed_cores = True	×
157	elif found_placed_cores:	×
158	vertex_count += len(vertices)	×
159	n_vertices = len(same_chip_groups)	×
160	app_vertex_placed = False	×
161	break	×
162	else:
163	app_vertex_placed = False	×
164	break	×
165	if not app_vertex_placed:	×
166	unplaceable.append(app_vertex)	×
167
168	report_file = os.path.join(	×
169	PacmanDataView.get_run_dir_path(), "placements_error.txt")
170	with open(report_file, 'w', encoding="utf-8") as f:	×
171	f.write(f"Could not place {len(unplaceable)} of "	×
172	f"{PacmanDataView.get_n_vertices()} application vertices.\n")
173	f.write(f" Could not place {vertex_count} of {n_vertices} in the"	×
174	" last app vertex\n\n")
175	for x, y in placements.chips_with_placements:	×
176	first = True	×
177	for placement in placements.placements_on_chip(x, y):	×
178	if system_placements.is_vertex_placed(placement.vertex):	×
179	continue	×
180	if first:	×
181	f.write(f"Chip ({x}, {y}):\n")	×
182	first = False	×
183	f.write(f" Processor {placement.p}:"	×
184	f" Vertex {placement.vertex}\n")
185	if not first:	×
186	f.write("\n")	×
187	f.write("\n")	×
188	f.write("Not placed:\n")	×
189	for app_vertex in unplaceable:	×
190	f.write(f"Vertex: {app_vertex}\n")	×
191	same_chip_groups = app_vertex.splitter.get_same_chip_groups()	×
192	for vertices, sdram in same_chip_groups:	×
193	f.write(f" Group of {len(vertices)} vertices uses "	×
194	f"{sdram.get_total_sdram(plan_n_timesteps)} "
195	"bytes of SDRAM:\n")
196	for vertex in vertices:	×
197	f.write(f" Vertex {vertex}")	×
198	if placements.is_vertex_placed(vertex):	×
199	plce = placements.get_placement_of_vertex(vertex)	×
200	f.write(f" (placed at {plce.x}, {plce.y}, {plce.p})")	×
201	f.write("\n")	×
202
203	f.write("\n")	×
204	f.write("Unused chips:\n")	×
NEW 205	machine = PacmanDataView.get_machine()	×
206	for x, y in machine.chip_coordinates:	×
207	n_placed = placements.n_placements_on_chip(x, y)	×
208	system_placed = system_placements.n_placements_on_chip(x, y)	×
209	if n_placed - system_placed == 0:	×
NEW 210	n_procs = machine[x, y].n_user_processors	×
211	f.write(f" {x}, {y} ({n_procs - system_placed}"	×
212	" free cores)\n")
213
214	if get_config_bool("Reports", "draw_placements_on_error"):	×
NEW 215	from .draw_placements import draw_placements as dp	×
NEW 216	dp(placements, system_placements)	×
217
NEW 218	return PacmanPlaceException(	×
219	f" {exception}."
220	f" Report written to {report_file}.")
221
222
223	def _check_could_fit(	1✔
224	app_vertex: ApplicationVertex, vertices_to_place: List[MachineVertex],
225	sdram: int):
226	"""
227	:param ApplicationVertex app_vertex:
228	:param list(MachineVertex) vertices_to_place:
229	:param int sdram:
230	:raises PacmanTooBigToPlace:
231	"""
232	version = PacmanDataView.get_machine_version()	1✔
233	max_sdram = (	1✔
234	version.max_sdram_per_chip - PacmanDataView.get_monitor_sdram())
235	max_cores = (	1✔
236	version.max_cores_per_chip - version.n_non_user_cores -
237	PacmanDataView.get_monitor_cores())
238	n_cores = len(vertices_to_place)	1✔
239	if sdram <= max_sdram and n_cores <= max_cores:	1✔
240	# should fit somewhere
241	return	1✔
242	message = (	1✔
243	f"{app_vertex} will not fit on any possible Chip "
244	f"the reason is that {vertices_to_place} ")
245	if sdram > max_sdram:	1✔
246	message += f"requires {sdram} bytes but "	1✔
247	if sdram > version.max_sdram_per_chip:	1✔
248	message += f"a Chip only has {version.max_sdram_per_chip} bytes "	1✔
249	else:
250	message += f"after monitors only {max_sdram} bytes are available "	1✔
251	message += "Lowering max_core_per_chip may resolve this."	1✔
252	raise PacmanTooBigToPlace(message)	1✔
253	if n_cores > version.max_cores_per_chip:	1✔
254	message += " is more vertices than the number of cores on a chip."	1✔
255	raise PacmanTooBigToPlace(message)	1✔
256	user_cores = version.max_cores_per_chip - version.n_non_user_cores	1✔
257	if n_cores > user_cores:	1✔
258	message += (	1✔
259	f"is more vertices than the user cores ({user_cores}) "
260	"available on a Chip")
261	else:
262	message += (	1✔
263	f"is more vertices than the {max_cores} cores available on a "
264	f"Chip once {PacmanDataView.get_monitor_cores()} "
265	"are reserved for monitors")
266	raise PacmanTooBigToPlace(message)	1✔
267
268
269	class _SpaceExceededException(Exception):	1✔
270	pass	1✔
271
272
273	def _do_fixed_location(	1✔
274	vertices: list[MachineVertex], sdram: int, placements: Placements,
275	next_chip_space: _ChipWithSpace) -> bool:
276	"""
277	:param list(MachineVertex) vertices:
278	:param int sdram:
279	:param Placements placements:
280	:param _ChipWithSpace next_chip_space:
281	:rtype: bool
282	:raise PacmanConfigurationException:
283	"""
284	for vertex in vertices:	1✔
285	loc = vertex.get_fixed_location()	1✔
286	if loc:	1✔
287	x, y = loc.x, loc.y	1✔
288	break	1✔
289	else:
290	return False	1✔
291
292	machine = PacmanDataView.get_machine()	1✔
293	chip = machine.get_chip_at(x, y)	1✔
294	if chip is None:	1!
295	raise PacmanConfigurationException(	×
296	f"Constrained to chip {x, y} but no such chip")
297	on_chip = placements.placements_on_chip(x, y)	1✔
298	cores_used = {p.p for p in on_chip}	1✔
299	cores = set(p.processor_id for p in chip.processors	1✔
300	if not p.is_monitor) - cores_used
301	next_cores = iter(cores)	1✔
302	for vertex in vertices:	1✔
303	next_core = None	1✔
304	fixed = vertex.get_fixed_location()	1✔
305	if fixed and fixed.p is not None:	1!
NEW 306	if fixed.p not in next_cores:	×
NEW 307	raise PacmanConfigurationException(	×
308	f"Core {fixed.p} on {x}, {y} not available to "
309	f"place {vertex} on")
NEW 310	next_core = fixed.p	×
311	else:
312	try:	1✔
313	next_core = next(next_cores)	1✔
314	except StopIteration:	1✔
315	# pylint: disable=raise-missing-from
316	raise PacmanConfigurationException(	1✔
317	f"No more cores available on {x}, {y}: {on_chip}")
318	placements.add_placement(Placement(vertex, x, y, next_core))	1✔
319	if next_chip_space.x == x and next_chip_space.y == y:	1✔
320	next_chip_space.cores.remove(next_core)	1✔
321	next_chip_space.use_sdram(sdram)	1✔
322	return True	1✔
323
324
325	def _store_on_chip(	1✔
326	placements_to_make: List[Placement], vertices: List[MachineVertex],
327	sdram: int, next_chip_space: _ChipWithSpace):
328	"""
329	:param list(Placement) placements_to_make:
330	:param list(MachineVertex) vertices:
331	:param int sdram:
332	:param _ChipWithSpace next_chip_space:
333	"""
334	for vertex in vertices:	1✔
335	core = next_chip_space.use_next_core()	1✔
336	placements_to_make.append(Placement(	1✔
337	vertex, next_chip_space.x, next_chip_space.y, core))
338	next_chip_space.use_sdram(sdram)	1✔
339
340
341	class _Spaces(object):	1✔
342	__slots__ = ("__machine", "__chips", "__next_chip", "__used_chips",	1✔
343	"__system_placements", "__placements", "__plan_n_timesteps",
344	"__last_chip_space", "__saved_chips", "__restored_chips")
345
346	def __init__(	1✔
347	self, placements: Placements, plan_n_timesteps: Optional[int]):
348	"""
349	:param Placements placements:
350	:param int plan_n_timesteps:
351	"""
352	self.__machine = PacmanDataView.get_machine()	1✔
353	self.__placements = placements	1✔
354	self.__plan_n_timesteps = plan_n_timesteps	1✔
355	self.__chips = iter(self.__chip_order())	1✔
356	self.__next_chip = next(self.__chips)	1✔
357	self.__used_chips: Set[Chip] = set()	1✔
358	self.__last_chip_space: Optional[_ChipWithSpace] = None	1✔
359	self.__saved_chips: OrderedSet[Chip] = OrderedSet()	1✔
360	self.__restored_chips: OrderedSet[Chip] = OrderedSet()	1✔
361
362	def __chip_order(self):	1✔
363	"""
364	:param Machine machine:
365	:rtype: iterable(Chip)
366	"""
367	s_x, s_y = get_config_str("Mapping", "placer_start_chip").split(",")	1✔
368	s_x = int(s_x)	1✔
369	s_y = int(s_y)	1✔
370
371	for x in range(self.__machine.width):	1!
372	for y in range(self.__machine.height):	1✔
373	c_x = (x + s_x) % self.__machine.width	1✔
374	c_y = (y + s_y) % self.__machine.height	1✔
375	chip = self.__machine.get_chip_at(c_x, c_y)	1✔
376	if chip:	1✔
377	yield chip	1✔
378
379	def __cores_and_sdram(self, chip: Chip) -> Tuple[Set[int], int]:	1✔
380	"""
381	:param Chip chip:
382	:return cores, sdram
383	:rtype: tuple(int, int)
384	"""
385	on_chip = self.__placements.placements_on_chip(chip.x, chip.y)	1✔
386	cores_used = {p.p for p in on_chip}	1✔
387	sdram_used = sum(	1✔
388	p.vertex.sdram_required.get_total_sdram(
389	self.__plan_n_timesteps) for p in on_chip)
390	return cores_used, sdram_used	1✔
391
392	def get_next_chip_and_space(self) -> Tuple[_ChipWithSpace, _Space]:	1✔
393	"""
394	:rtype: (_ChipWithSpace, _Space)
395	"""
396	try:	1✔
397	if self.__last_chip_space is None:	1✔
398	chip = self.__get_next_chip()	1✔
399	cores_used, sdram_used = self.__cores_and_sdram(chip)	1✔
400	self.__last_chip_space = _ChipWithSpace(	1✔
401	chip, cores_used, sdram_used)
402	self.__used_chips.add(chip)	1✔
403
404	# Start a new space by finding all the chips that can be reached
405	# from the start chip but have not been used
406	return (self.__last_chip_space,	1✔
407	_Space(self.__last_chip_space.chip))
408
409	except StopIteration:	×
410	raise PacmanPlaceException( # pylint: disable=raise-missing-from	×
411	f"No more chips to place on; {self.n_chips_used} of "
412	f"{self.__machine.n_chips} used")
413
414	def __get_next_chip(self) -> Chip:	1✔
415	"""
416	:rtype: Chip
417	:raises: StopIteration
418	"""
419	while self.__restored_chips:	1✔
420	chip = self.__restored_chips.pop(last=False)	1✔
421	if chip not in self.__used_chips:	1!
422	return chip	1✔
423	while self.__next_chip in self.__used_chips:	1✔
424	self.__next_chip = next(self.__chips)	1✔
425	return self.__next_chip	1✔
426
427	def get_next_chip_space(	1✔
428	self, space: _Space,
429	last_chip_space: Optional[_ChipWithSpace]) -> _ChipWithSpace:
430	"""
431	:param _Space space:
432	:param _ChipWithSpace last_chip_space:
433	:rtype: _ChipWithSpace
434	:raises _SpaceExceededException:
435	"""
436	# If we are reporting a used chip, update with reachable chips
437	if last_chip_space is not None:	1✔
438	last_chip = last_chip_space.chip	1✔
439	space.update(self.__usable_from_chip(last_chip))	1✔
440
441	# If no space, error
442	if not space:	1✔
443	self.__last_chip_space = None	1✔
444	raise _SpaceExceededException(	1✔
445	"No more chips to place on in this space; "
446	f"{self.n_chips_used} of {self.__machine.n_chips} used")
447	chip = space.pop()	1✔
448	self.__used_chips.add(chip)	1✔
449	self.__restored_chips.discard(chip)	1✔
450	cores_used, sdram_used = self.__cores_and_sdram(chip)	1✔
451	self.__last_chip_space = _ChipWithSpace(chip, cores_used, sdram_used)	1✔
452	return self.__last_chip_space	1✔
453
454	@property	1✔
455	def n_chips_used(self) -> int:	1✔
456	"""
457	The number of chips used.
458
459	:rtype: int
460	"""
461	return len(self.__used_chips)	1✔
462
463	def __usable_from_chip(self, chip: Chip) -> Iterable[Chip]:	1✔
464	"""
465	:param Chip chip:
466	:rtype set(Chip)
467	"""
468	for link in chip.router.links:	1✔
469	target = self.__machine[link.destination_x, link.destination_y]	1✔
470	if target not in self.__used_chips:	1✔
471	yield target	1✔
472
473	def save_chips(self, chips: Iterable[Chip]):	1✔
474	"""
475	:param iterable(Chip) chips:
476	"""
477	self.__saved_chips.update(chips)	1✔
478
479	def restore_chips(self) -> None:	1✔
480	for chip in self.__saved_chips:	1✔
481	self.__used_chips.remove(chip)	1✔
482	self.__restored_chips.add(chip)	1✔
483	self.__saved_chips.clear()	1✔
484
485
486	class _Space(object):	1✔
487	__slots__ = ("__same_board_chips", "__remaining_chips",	1✔
488	"__board_x", "__board_y", "__first_chip")
489
490	def __init__(self, chip: Chip):	1✔
491	self.__board_x = chip.nearest_ethernet_x	1✔
492	self.__board_y = chip.nearest_ethernet_y	1✔
493	self.__same_board_chips: OrderedSet[Chip] = OrderedSet()	1✔
494	self.__remaining_chips: OrderedSet[Chip] = OrderedSet()	1✔
495
496	def __len__(self) -> int:	1✔
497	return len(self.__same_board_chips) + len(self.__remaining_chips)	1✔
498
499	def __on_same_board(self, chip: Chip) -> bool:	1✔
500	return (chip.nearest_ethernet_x == self.__board_x and	1✔
501	chip.nearest_ethernet_y == self.__board_y)
502
503	def pop(self) -> Chip:	1✔
504	"""
505	:rtype: Chip
506	:raise: StopIteration
507	"""
508	if self.__same_board_chips:	1✔
509	return self.__same_board_chips.pop(last=False)	1✔
510	if self.__remaining_chips:	1!
511	next_chip = self.__remaining_chips.pop(last=False)	1✔
512	self.__board_x = next_chip.nearest_ethernet_x	1✔
513	self.__board_y = next_chip.nearest_ethernet_y	1✔
514	to_remove = list()	1✔
515	for chip in self.__remaining_chips:	1✔
516	if self.__on_same_board(chip):	1✔
517	to_remove.append(chip)	1✔
518	self.__same_board_chips.add(chip)	1✔
519	for chip in to_remove:	1✔
520	self.__remaining_chips.remove(chip)	1✔
521	return next_chip	1✔
522	raise StopIteration	×
523
524	def update(self, chips: Iterable[Chip]):	1✔
525	"""
526	:param iterable(Chip) chips:
527	"""
528	for chip in chips:	1✔
529	if self.__on_same_board(chip):	1✔
530	self.__same_board_chips.add(chip)	1✔
531	else:
532	self.__remaining_chips.add(chip)	1✔
533
534
535	class _ChipWithSpace(object):	1✔
536	"""
537	A chip with space for placement.
538	"""
539	__slots__ = ("chip", "cores", "sdram")	1✔
540
541	def __init__(	1✔
542	self, chip: Chip, used_processors: Set[int], used_sdram: int):
543	self.chip = chip	1✔
544	self.cores = set(p.processor_id for p in chip.processors	1✔
545	if not p.is_monitor)
546	self.cores -= used_processors	1✔
547	self.sdram = chip.sdram - used_sdram	1✔
548
549	@property	1✔
550	def x(self) -> int:	1✔
551	return self.chip.x	1✔
552
553	@property	1✔
554	def y(self) -> int:	1✔
555	return self.chip.y	1✔
556
557	def is_space(self, n_cores: int, sdram: int) -> bool:	1✔
558	return len(self.cores) >= n_cores and self.sdram >= sdram	1✔
559
560	def use_sdram(self, sdram: int):	1✔
561	self.sdram -= sdram	1✔
562
563	def use_next_core(self) -> int:	1✔
564	core = next(iter(self.cores))	1✔
565	self.cores.remove(core)	1✔
566	return core	1✔
567
568	def __repr__(self):	1✔
569	return f"({self.x}, {self.y})"	1✔

SpiNNakerManchester / PACMAN / 6742258790

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous