8785183552

Committed 22 Apr 2024 01:46PM UTC coverage: 60.178% (-0.007%) from 60.185%

Build # 8785183552

Build Type

Pull #397

github

Committed by

web-flow

Commit Message

Merge branch 'master' into v_to_p_map

Pull Request Pull Request #397: move V to p map from Chip to DataView

Run Details

2229 of 3507 branches covered (63.56%)

Branch coverage included in aggregate %.

1 of 2 new or added lines in 2 files covered. (50.0%)

58 existing lines in 1 file now uncovered.

4789 of 8155 relevant lines covered (58.72%)

0.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

17.17

/spinnman/processes/get_machine_process.py

# Copyright (c) 2015 The University of Manchester
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from collections import defaultdict
from contextlib import suppress
import logging
import functools
from os.path import join
from typing import Dict, List, Optional, Set, Tuple, cast

from spinn_utilities.config_holder import (
    get_config_bool, get_config_int_or_none, get_config_str_or_none)
from spinn_utilities.data import UtilsDataView
from spinn_utilities.log import FormatAdapter
from spinn_utilities.progress_bar import ProgressBar
from spinn_utilities.typing.coords import XY

from spinn_machine import (Router, Chip, Link, Machine)
from spinn_machine.ignores import IgnoreChip, IgnoreCore, IgnoreLink
from spinn_machine.machine_factory import machine_repair

from spinnman.constants import (
    ROUTER_REGISTER_P2P_ADDRESS, SYSTEM_VARIABLE_BASE_ADDRESS)
from spinnman.data import SpiNNManDataView
from spinnman.exceptions import SpinnmanUnexpectedResponseCodeException
from spinnman.messages.scp.abstract_messages import AbstractSCPRequest
from spinnman.messages.scp.impl import ReadMemory, ReadLink, GetChipInfo
from spinnman.messages.scp.impl.get_chip_info_response import (
    GetChipInfoResponse)
from spinnman.messages.scp.impl.read_memory import Response
from spinnman.messages.spinnaker_boot import (
    SystemVariableDefinition)
from spinnman.model import ChipSummaryInfo, P2PTable
from spinnman.model.enums import CPUState

from .abstract_multi_connection_process import AbstractMultiConnectionProcess
from .abstract_multi_connection_process_connection_selector import \
    ConnectionSelector

logger = FormatAdapter(logging.getLogger(__name__))

REPORT_FILE = "Ignores_report.rpt"

P_TO_V = SystemVariableDefinition.physical_to_virtual_core_map
V_TO_P = SystemVariableDefinition.virtual_to_physical_core_map
P_TO_V_ADDR = SYSTEM_VARIABLE_BASE_ADDRESS + P_TO_V.offset
_P_TO_V_SIZE = cast(int, P_TO_V.array_size)
P_MAPS_SIZE = _P_TO_V_SIZE + cast(int, V_TO_P.array_size)


class GetMachineProcess(AbstractMultiConnectionProcess):
    """
    A process for getting the machine details over a set of connections.
    """
    __slots__ = (
        "_chip_info",
        # Used if there are any ignores with IP addresses
        # Holds a mapping from IP to board root (x,y)
        "_ethernets",
        # Holds a map from x,y to a set of virtual cores to ignores
        "_ignore_cores_map",
        "_p2p_column_data",
        # Holds a mapping from (x,y) to a mapping of physical to virtual core
        "_virtual_to_physical_map",
        # Holds a mapping from (x,y) to a mapping of virtual to physical core
        "_physical_to_virtual_map",
        # Progress bar to fill in as details are received
        "_progress")

    def __init__(self, connection_selector: ConnectionSelector):
        """
        :param ConnectionSelector connection_selector:
        """
        super().__init__(connection_selector)

        self._ignore_cores_map: Dict[XY, Set[int]] = defaultdict(set)

        self._p2p_column_data: List[Tuple[bytes, int]] = list()

        # A dictionary of (x, y) -> ChipInfo
        self._chip_info: Dict[XY, ChipSummaryInfo] = dict()

        # Set to None meaning not computed yet
        self._ethernets: Optional[Dict[str, XY]] = None

        # Maps between virtual and physical cores
        self._virtual_to_physical_map: Dict[XY, bytes] = dict()
        self._physical_to_virtual_map: Dict[XY, bytes] = dict()
        self._progress: Optional[ProgressBar] = None

    def _make_chip(self, chip_info: ChipSummaryInfo, machine: Machine) -> Chip:
        """
        Creates a chip from a ChipSummaryInfo structure.

        :param ChipSummaryInfo chip_info:
            The ChipSummaryInfo structure to create the chip from
        :return: The created chip
        :rtype: ~spinn_machine.Chip
        """
        # Keep track of progress
        self._progress = None
        # Create the down cores set if any
        n_cores = \
            SpiNNManDataView.get_machine_version().max_cores_per_chip
        n_cores = min(chip_info.n_cores, n_cores)
        core_states = chip_info.core_states
        down_cores = self._ignore_cores_map.get(
            (chip_info.x, chip_info.y), None)
        for i in range(1, n_cores):
            if core_states[i] != CPUState.IDLE:
                self._report_ignore(
                    "Not using core {}, {}, {} in state {}",
                    chip_info.x, chip_info.y, i,  core_states[i])
                if down_cores is None:
                    down_cores = set()
                down_cores.add(i)

        # Create the router
        router = self._make_router(chip_info, machine)

        # Create the chip's SDRAM object
        sdram_size = chip_info.largest_free_sdram_block
        max_sdram_size = get_config_int_or_none(
            "Machine", "max_sdram_allowed_per_chip")
        if (max_sdram_size is not None and
                sdram_size > max_sdram_size):
            sdram_size = max_sdram_size

        # Create the chip
        return Chip(
            x=chip_info.x, y=chip_info.y, n_processors=n_cores,
            router=router, sdram=sdram_size,
            ip_address=chip_info.ethernet_ip_address,
            nearest_ethernet_x=chip_info.nearest_ethernet_x,
            nearest_ethernet_y=chip_info.nearest_ethernet_y,
            down_cores=down_cores, parent_link=chip_info.parent_link)

    def _make_router(
            self, chip_info: ChipSummaryInfo, machine: Machine) -> Router:
        """
        :param ChipSummaryInfo chip_info:
        :param ~spinn_machine.Machine machine:
        :rtype: ~spinn_machine.Router
        """
        links = list()
        for link in chip_info.working_links:
            dest_xy = machine.xy_over_link(chip_info.x, chip_info.y, link)
            if dest_xy in self._chip_info:
                links.append(Link(
                    chip_info.x, chip_info.y, link, dest_xy[0], dest_xy[1]))
            else:
                logger.warning(
                    "Link {},{}:{} points to {} but that is not included ",
                    chip_info.x, chip_info.y, link, dest_xy)

        return Router(
            links=links,
            n_available_multicast_entries=(
                chip_info.n_free_multicast_routing_entries))

    def __receive_p2p_data(
            self, column: int, scp_read_response: Response):
        """
        :param int column:
        :param Response scp_read_response:
        """
        self._p2p_column_data[column] = (
            scp_read_response.data, scp_read_response.offset)

    def _receive_chip_info(
            self, scp_read_chip_info_response: GetChipInfoResponse):
        """
        :param GetChipInfoResponse scp_read_chip_info_response:
        """
        chip_info = scp_read_chip_info_response.chip_info
        self._chip_info[chip_info.x, chip_info.y] = chip_info
        if self._progress is not None:
            self._progress.update()

    def _receive_p_maps(
            self, x: int, y: int, scp_read_response: Response):
        """
        Receive the physical-to-virtual and virtual-to-physical maps.

        :param Response scp_read_response:
        """
        data = scp_read_response.data
        off = scp_read_response.offset
        self._physical_to_virtual_map[x, y] = data[off:_P_TO_V_SIZE + off]
        off += _P_TO_V_SIZE
        self._virtual_to_physical_map[x, y] = data[off:]

    def _receive_error(
            self, request: AbstractSCPRequest, exception, tb, connection):
        """
        :param AbstractSCPRequest request:
        :param Exception exception:
        """
        # If we get an ReadLink with a
        # SpinnmanUnexpectedResponseCodeException, this is a failed link
        # and so can be ignored
        if isinstance(request, ReadLink):
            if isinstance(exception, SpinnmanUnexpectedResponseCodeException):
                return
        super()._receive_error(request, exception, tb, connection)

    def get_machine_details(
            self, boot_x: int, boot_y: int,
            width: int, height: int) -> Machine:
        """
        :param int boot_x:
        :param int boot_y:
        :param int width:
        :param int height:
        :rtype: ~spinn_machine.Machine
        """
        # Get the P2P table - 8 entries are packed into each 32-bit word
        p2p_column_bytes = P2PTable.get_n_column_bytes(height)
        blank = (b'', 0)
        self._p2p_column_data = [blank] * width
        with self._collect_responses():
            for column in range(width):
                offset = P2PTable.get_column_offset(column)
                self._send_request(
                    ReadMemory(
                        coordinates=(boot_x, boot_y, 0),
                        base_address=(ROUTER_REGISTER_P2P_ADDRESS + offset),
                        size=p2p_column_bytes),
                    functools.partial(self.__receive_p2p_data, column))
        p2p_table = P2PTable(width, height, self._p2p_column_data)

        # Get the chip information for each chip
        self._progress = ProgressBar(
            p2p_table.n_routes,
            f"Reading details from {p2p_table.n_routes} chips")
        with suppress(Exception), self._collect_responses():
            # Ignore errors, as any error here just means that a chip
            # is down that wasn't marked as down
            for (x, y) in p2p_table.iterchips():
                self._send_request(GetChipInfo(x, y), self._receive_chip_info)
                self._send_request(
                    ReadMemory((x, y, 0), P_TO_V_ADDR, P_MAPS_SIZE),
                    functools.partial(self._receive_p_maps, x, y))
        self._progress.end()
        SpiNNManDataView.set_v_to_p_map(self._virtual_to_physical_map)

        # Warn about unexpected missing chips
        for (x, y) in p2p_table.iterchips():
            if (x, y) not in self._chip_info:
                logger.warning(
                    "Chip {}, {} was expected but didn't reply", x, y)

        version = SpiNNManDataView.get_machine_version()
        machine = version.create_machine(width, height)
        self._preprocess_ignore_chips(machine)
        self._process_ignore_links(machine)
        self._preprocess_ignore_cores(machine)

        return self._fill_machine(machine)

    def _fill_machine(self, machine: Machine) -> Machine:
        """
        :param ~spinn_machine.Machine machine:
        :rtype: ~spinn_machine.Machine
        """
        for chip_info in sorted(
                self._chip_info.values(), key=lambda chip: (chip.x, chip.y)):
            if (chip_info.ethernet_ip_address is not None and
                    (chip_info.x != chip_info.nearest_ethernet_x
                     or chip_info.y != chip_info.nearest_ethernet_y)):
                if get_config_bool("Machine", "ignore_bad_ethernets"):
                    logger.warning(
                        "Chip {}:{} claimed it has ip address: {}. "
                        "This ip will not be used.",
                        chip_info.x, chip_info.y,
                        chip_info.ethernet_ip_address)
                    chip_info.clear_ethernet_ip_address()
                else:
                    logger.warning(
                        "Not using chip {}:{} as it has an unexpected "
                        "ip address: {}", chip_info.x, chip_info.y,
                        chip_info.ethernet_ip_address)
                    continue

            # If the above has not continued, add the chip
            machine.add_chip(self._make_chip(chip_info, machine))

        machine.validate()
        return machine_repair(machine)

    # Stuff below here is purely for dealing with ignores

    def _process_ignore_links(self, machine: Machine):
        """
        Processes the collection of ignore links to remove then from chip info.

        Converts any local (x, y, IP address) to global (x, y)

        Discards any ignores which are known to have no
        affect based on the already read chip_info

        Also removes any inverse links

        Logs all actions except for ignores with unused IP addresses

        :param ~spinn_machine.Machine machine:
            An empty machine to handle wrap-arounds
        """
        for ignore in IgnoreLink.parse_string(
                get_config_str_or_none("Machine", "down_links")):
            global_xy = self._ignores_local_to_global(
                ignore.x, ignore.y, ignore.ip_address, machine)
            if global_xy is None:
                continue
            chip_info = self._chip_info.get(global_xy, None)
            if chip_info is None:
                self._report_ignore(
                    "Discarding ignore link on chip {} as it is not/ no longer"
                    " in info", global_xy)
                continue
            link = ignore.link
            if link in chip_info.working_links:
                chip_info.working_links.remove(link)
                self._report_ignore(
                    "On chip {} ignoring link:{}", global_xy, link)
                # ignore the inverse link too
                inv_xy = machine.xy_over_link(global_xy[0], global_xy[1], link)
                if inv_xy in self._chip_info:
                    inv_chip_info = self._chip_info[inv_xy]
                    inv_link = (link + 3) % 6
                    if inv_link in inv_chip_info.working_links:
                        inv_chip_info.working_links.remove(inv_link)
                        self._report_ignore(
                            "On chip {} ignoring link {} as it is the inverse "
                            "of link {} on chip {}", inv_xy, inv_link, link,
                            global_xy)
            else:
                self._report_ignore(
                    "Discarding ignore link {} on chip {} as it is not/"
                    "no longer in info", link, global_xy)

    def _preprocess_ignore_cores(self, machine: Machine):
        """
        Converts the collection of ignore cores into a map of ignore by x,y.

        Converts any local (x, y, IP address) to global (x, y)

        Discards (with log messages) any ignores which are known to have no
        affect based on the already read chip_info

        Converts any physical  cores to virtual ones.
        Core numbers <= 0 are assumed to be 0 - physical_id

        :param ~spinn_machine.Machine machine:
            An empty machine to handle wrap-arounds
        """
        # Convert by IP to global
        for ignore in IgnoreCore.parse_string(
                get_config_str_or_none("Machine", "down_cores")):
            global_xy = self._ignores_local_to_global(
                ignore.x, ignore.y, ignore.ip_address, machine)
            if global_xy is None:
                continue
            p = self._get_virtual_p(global_xy, ignore.p)
            if p is not None:
                self._ignore_cores_map[global_xy].add(p)

    def _preprocess_ignore_chips(self, machine: Machine):
        """
        Processes the collection of ignore chips and discards their chip info.

        Converts any local (x, y IP address) to global (x, y)

        Discards any ignores which are known to have no
        affect based on the already read chip_info

        Logs all actions except for ignores with unused IP addresses

        :param ~spinn_machine.Machine machine:
            An empty machine to handle wrap-arounds
        """
        for ignore in IgnoreChip.parse_string(
                get_config_str_or_none("Machine", "down_chips")):
            # Convert by IP to global
            global_xy = self._ignores_local_to_global(
                ignore.x, ignore.y, ignore.ip_address, machine)
            if global_xy is None:
                continue  # Never on this machine
            chip_info = self._chip_info.pop(global_xy, None)
            if chip_info is None:
                continue  # Already ignored maybe by a dead chip list
            self._report_ignore("Chip {} will be ignored", global_xy)
            for link in chip_info.working_links:
                # ignore the inverse link
                inv_xy = machine.xy_over_link(global_xy[0], global_xy[1], link)
                if inv_xy in self._chip_info:
                    inv_chip_info = self._chip_info[inv_xy]
                    inv_link = (link + 3) % 6
                    if inv_link in inv_chip_info.working_links:
                        inv_chip_info.working_links.remove(inv_link)
                        self._report_ignore(
                            "On chip {} ignoring link {} as it points to "
                            "ignored chip chip {}",
                            inv_xy, inv_link, global_xy)

    def _ignores_local_to_global(
            self, local_x: int, local_y: int, ip_address: Optional[str],
            machine: Machine) -> Optional[XY]:
        """
        :param int local_x:
        :param int local_y:
        :param str ip_address:
        :param ~spinn_machine.Machine machine:
        :rtype: tuple(int,int)
        """
        if ip_address is None:
            global_xy = (local_x, local_y)
        else:
            ethernet = self._ethernet_by_ipaddress(ip_address)
            if ethernet is None:
                self._report_ignore(
                    "Ignore with ip:{} will be discarded as no board with "
                    "that ip in this machine", ip_address)
                return None
            global_xy = machine.get_global_xy(
                local_x, local_y, ethernet[0], ethernet[1])
            self._report_ignore(
                "Ignores for local x:{} y:{} and ip:{} map to global {} with "
                "root {}", local_x, local_y, ip_address, global_xy,
                self._chip_info[(0, 0)].ethernet_ip_address)

        if global_xy in self._chip_info:
            return global_xy
        else:
            self._report_ignore(
                "Ignore for global chip {} will be discarded as no such chip "
                "in this machine", global_xy)
            return None

    def _ethernet_by_ipaddress(self, ip_address: str) -> Optional[XY]:
        """
        :param str ip_address:
        :rtype: tuple(int,int)
        """
        if self._ethernets is None:
            self._ethernets = dict()
            for chip_info in self._chip_info.values():
                if chip_info.ethernet_ip_address is not None:
                    self._ethernets[chip_info.ethernet_ip_address] = \
                        (chip_info.x, chip_info.y)
        return self._ethernets.get(ip_address, None)

    def _get_virtual_p(self, xy: XY, p: int) -> Optional[int]:
        """
        :param tuple(int,int) xy:
        :param int p:
        :rtype: int
        """
        if p > 0:
            self._report_ignore("On chip {} ignoring core {}", xy, p)
            return p
        virtual_map = self._physical_to_virtual_map[xy]
        physical = 0 - p
        if physical >= len(virtual_map) or virtual_map[physical] == 0xFF:
            self._report_ignore(
                "On chip {} physical core {} was not used "
                "so ignore is being discarded.", xy, physical)
            return None
        virtual_p = virtual_map[physical]
        if virtual_p == 0:
            self._report_ignore(
                "On chip {} physical core {} was used as the monitor "
                "so will NOT be ignored", xy, physical)
            return None
        self._report_ignore(
            "On chip {} ignoring core {} as it maps to physical "
            "core {}", xy, virtual_p, physical)
        return virtual_p

    def _report_ignore(self, message: str, *args):
        """
        Writes the ignore message by either creating or appending the report.

        The implementation choice to reopen the file every time is not the
        fastest but is the cleanest and safest for code that in default
        conditions is never run.

        :param str message:
        """
        full_message = message.format(*args) + "\n"
        report_file = join(UtilsDataView.get_run_dir_path(), REPORT_FILE)
        with open(report_file, "a", encoding="utf-8") as r_file:
            r_file.write(full_message)

1	# Copyright (c) 2015 The University of Manchester
2	#
3	# Licensed under the Apache License, Version 2.0 (the "License");
4	# you may not use this file except in compliance with the License.
5	# You may obtain a copy of the License at
6	#
7	# https://www.apache.org/licenses/LICENSE-2.0
8	#
9	# Unless required by applicable law or agreed to in writing, software
10	# distributed under the License is distributed on an "AS IS" BASIS,
11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	# See the License for the specific language governing permissions and
13	# limitations under the License.
14
15	from collections import defaultdict	1✔
16	from contextlib import suppress	1✔
17	import logging	1✔
18	import functools	1✔
19	from os.path import join	1✔
20	from typing import Dict, List, Optional, Set, Tuple, cast	1✔
21
22	from spinn_utilities.config_holder import (	1✔
23	get_config_bool, get_config_int_or_none, get_config_str_or_none)
24	from spinn_utilities.data import UtilsDataView	1✔
25	from spinn_utilities.log import FormatAdapter	1✔
26	from spinn_utilities.progress_bar import ProgressBar	1✔
27	from spinn_utilities.typing.coords import XY	1✔
28
29	from spinn_machine import (Router, Chip, Link, Machine)	1✔
30	from spinn_machine.ignores import IgnoreChip, IgnoreCore, IgnoreLink	1✔
31	from spinn_machine.machine_factory import machine_repair	1✔
32
33	from spinnman.constants import (	1✔
34	ROUTER_REGISTER_P2P_ADDRESS, SYSTEM_VARIABLE_BASE_ADDRESS)
35	from spinnman.data import SpiNNManDataView	1✔
36	from spinnman.exceptions import SpinnmanUnexpectedResponseCodeException	1✔
37	from spinnman.messages.scp.abstract_messages import AbstractSCPRequest	1✔
38	from spinnman.messages.scp.impl import ReadMemory, ReadLink, GetChipInfo	1✔
39	from spinnman.messages.scp.impl.get_chip_info_response import (	1✔
40	GetChipInfoResponse)
41	from spinnman.messages.scp.impl.read_memory import Response	1✔
42	from spinnman.messages.spinnaker_boot import (	1✔
43	SystemVariableDefinition)
44	from spinnman.model import ChipSummaryInfo, P2PTable	1✔
45	from spinnman.model.enums import CPUState	1✔
46
47	from .abstract_multi_connection_process import AbstractMultiConnectionProcess	1✔
48	from .abstract_multi_connection_process_connection_selector import \	1✔
49	ConnectionSelector
50
51	logger = FormatAdapter(logging.getLogger(__name__))	1✔
52
53	REPORT_FILE = "Ignores_report.rpt"	1✔
54
55	P_TO_V = SystemVariableDefinition.physical_to_virtual_core_map	1✔
56	V_TO_P = SystemVariableDefinition.virtual_to_physical_core_map	1✔
57	P_TO_V_ADDR = SYSTEM_VARIABLE_BASE_ADDRESS + P_TO_V.offset	1✔
58	_P_TO_V_SIZE = cast(int, P_TO_V.array_size)	1✔
59	P_MAPS_SIZE = _P_TO_V_SIZE + cast(int, V_TO_P.array_size)	1✔
60
61
62	class GetMachineProcess(AbstractMultiConnectionProcess):	1✔
63	"""
64	A process for getting the machine details over a set of connections.
65	"""
66	__slots__ = (	1✔
67	"_chip_info",
68	# Used if there are any ignores with IP addresses
69	# Holds a mapping from IP to board root (x,y)
70	"_ethernets",
71	# Holds a map from x,y to a set of virtual cores to ignores
72	"_ignore_cores_map",
73	"_p2p_column_data",
74	# Holds a mapping from (x,y) to a mapping of physical to virtual core
75	"_virtual_to_physical_map",
76	# Holds a mapping from (x,y) to a mapping of virtual to physical core
77	"_physical_to_virtual_map",
78	# Progress bar to fill in as details are received
79	"_progress")
80
81	def __init__(self, connection_selector: ConnectionSelector):	1✔
82	"""
83	:param ConnectionSelector connection_selector:
84	"""
85	super().__init__(connection_selector)	×
86
87	self._ignore_cores_map: Dict[XY, Set[int]] = defaultdict(set)	×
88
89	self._p2p_column_data: List[Tuple[bytes, int]] = list()	×
90
91	# A dictionary of (x, y) -> ChipInfo
92	self._chip_info: Dict[XY, ChipSummaryInfo] = dict()	×
93
94	# Set to None meaning not computed yet
95	self._ethernets: Optional[Dict[str, XY]] = None	×
96
97	# Maps between virtual and physical cores
98	self._virtual_to_physical_map: Dict[XY, bytes] = dict()	×
99	self._physical_to_virtual_map: Dict[XY, bytes] = dict()	×
100	self._progress: Optional[ProgressBar] = None	×
101
102	def _make_chip(self, chip_info: ChipSummaryInfo, machine: Machine) -> Chip:	1✔
103	"""
104	Creates a chip from a ChipSummaryInfo structure.
105
106	:param ChipSummaryInfo chip_info:
107	The ChipSummaryInfo structure to create the chip from
108	:return: The created chip
109	:rtype: ~spinn_machine.Chip
110	"""
111	# Keep track of progress
112	self._progress = None	×
113	# Create the down cores set if any
114	n_cores = \	×
115	SpiNNManDataView.get_machine_version().max_cores_per_chip
116	n_cores = min(chip_info.n_cores, n_cores)	×
117	core_states = chip_info.core_states	×
118	down_cores = self._ignore_cores_map.get(	×
119	(chip_info.x, chip_info.y), None)
120	for i in range(1, n_cores):	×
121	if core_states[i] != CPUState.IDLE:	×
122	self._report_ignore(	×
123	"Not using core {}, {}, {} in state {}",
124	chip_info.x, chip_info.y, i, core_states[i])
125	if down_cores is None:	×
126	down_cores = set()	×
127	down_cores.add(i)	×
128
129	# Create the router
130	router = self._make_router(chip_info, machine)	×
131
132	# Create the chip's SDRAM object
133	sdram_size = chip_info.largest_free_sdram_block	×
134	max_sdram_size = get_config_int_or_none(	×
135	"Machine", "max_sdram_allowed_per_chip")
136	if (max_sdram_size is not None and	×
137	sdram_size > max_sdram_size):
138	sdram_size = max_sdram_size	×
139
140	# Create the chip
141	return Chip(	×
142	x=chip_info.x, y=chip_info.y, n_processors=n_cores,
143	router=router, sdram=sdram_size,
144	ip_address=chip_info.ethernet_ip_address,
145	nearest_ethernet_x=chip_info.nearest_ethernet_x,
146	nearest_ethernet_y=chip_info.nearest_ethernet_y,
147	down_cores=down_cores, parent_link=chip_info.parent_link)
148
149	def _make_router(	1✔
150	self, chip_info: ChipSummaryInfo, machine: Machine) -> Router:
151	"""
152	:param ChipSummaryInfo chip_info:
153	:param ~spinn_machine.Machine machine:
154	:rtype: ~spinn_machine.Router
155	"""
UNCOV 156	links = list()	×
UNCOV 157	for link in chip_info.working_links:	×
158	dest_xy = machine.xy_over_link(chip_info.x, chip_info.y, link)	×
159	if dest_xy in self._chip_info:	×
160	links.append(Link(	×
161	chip_info.x, chip_info.y, link, dest_xy[0], dest_xy[1]))
162	else:
UNCOV 163	logger.warning(	×
164	"Link {},{}:{} points to {} but that is not included ",
165	chip_info.x, chip_info.y, link, dest_xy)
166
UNCOV 167	return Router(	×
168	links=links,
169	n_available_multicast_entries=(
170	chip_info.n_free_multicast_routing_entries))
171
172	def __receive_p2p_data(	1✔
173	self, column: int, scp_read_response: Response):
174	"""
175	:param int column:
176	:param Response scp_read_response:
177	"""
UNCOV 178	self._p2p_column_data[column] = (	×
179	scp_read_response.data, scp_read_response.offset)
180
181	def _receive_chip_info(	1✔
182	self, scp_read_chip_info_response: GetChipInfoResponse):
183	"""
184	:param GetChipInfoResponse scp_read_chip_info_response:
185	"""
UNCOV 186	chip_info = scp_read_chip_info_response.chip_info	×
UNCOV 187	self._chip_info[chip_info.x, chip_info.y] = chip_info	×
188	if self._progress is not None:	×
189	self._progress.update()	×
190
191	def _receive_p_maps(	1✔
192	self, x: int, y: int, scp_read_response: Response):
193	"""
194	Receive the physical-to-virtual and virtual-to-physical maps.
195
196	:param Response scp_read_response:
197	"""
UNCOV 198	data = scp_read_response.data	×
UNCOV 199	off = scp_read_response.offset	×
200	self._physical_to_virtual_map[x, y] = data[off:_P_TO_V_SIZE + off]	×
201	off += _P_TO_V_SIZE	×
202	self._virtual_to_physical_map[x, y] = data[off:]	×
203
204	def _receive_error(	1✔
205	self, request: AbstractSCPRequest, exception, tb, connection):
206	"""
207	:param AbstractSCPRequest request:
208	:param Exception exception:
209	"""
210	# If we get an ReadLink with a
211	# SpinnmanUnexpectedResponseCodeException, this is a failed link
212	# and so can be ignored
UNCOV 213	if isinstance(request, ReadLink):	×
UNCOV 214	if isinstance(exception, SpinnmanUnexpectedResponseCodeException):	×
215	return	×
216	super()._receive_error(request, exception, tb, connection)	×
217
218	def get_machine_details(	1✔
219	self, boot_x: int, boot_y: int,
220	width: int, height: int) -> Machine:
221	"""
222	:param int boot_x:
223	:param int boot_y:
224	:param int width:
225	:param int height:
226	:rtype: ~spinn_machine.Machine
227	"""
228	# Get the P2P table - 8 entries are packed into each 32-bit word
UNCOV 229	p2p_column_bytes = P2PTable.get_n_column_bytes(height)	×
UNCOV 230	blank = (b'', 0)	×
231	self._p2p_column_data = [blank] * width	×
232	with self._collect_responses():	×
233	for column in range(width):	×
234	offset = P2PTable.get_column_offset(column)	×
235	self._send_request(	×
236	ReadMemory(
237	coordinates=(boot_x, boot_y, 0),
238	base_address=(ROUTER_REGISTER_P2P_ADDRESS + offset),
239	size=p2p_column_bytes),
240	functools.partial(self.__receive_p2p_data, column))
UNCOV 241	p2p_table = P2PTable(width, height, self._p2p_column_data)	×
242
243	# Get the chip information for each chip
UNCOV 244	self._progress = ProgressBar(	×
245	p2p_table.n_routes,
246	f"Reading details from {p2p_table.n_routes} chips")
UNCOV 247	with suppress(Exception), self._collect_responses():	×
248	# Ignore errors, as any error here just means that a chip
249	# is down that wasn't marked as down
UNCOV 250	for (x, y) in p2p_table.iterchips():	×
UNCOV 251	self._send_request(GetChipInfo(x, y), self._receive_chip_info)	×
252	self._send_request(	×
253	ReadMemory((x, y, 0), P_TO_V_ADDR, P_MAPS_SIZE),
254	functools.partial(self._receive_p_maps, x, y))
UNCOV 255	self._progress.end()	×
NEW UNCOV 256	SpiNNManDataView.set_v_to_p_map(self._virtual_to_physical_map)	×
257
258	# Warn about unexpected missing chips
UNCOV 259	for (x, y) in p2p_table.iterchips():	×
260	if (x, y) not in self._chip_info:	×
261	logger.warning(	×
262	"Chip {}, {} was expected but didn't reply", x, y)
263
UNCOV 264	version = SpiNNManDataView.get_machine_version()	×
265	machine = version.create_machine(width, height)	×
266	self._preprocess_ignore_chips(machine)	×
267	self._process_ignore_links(machine)	×
268	self._preprocess_ignore_cores(machine)	×
269
UNCOV 270	return self._fill_machine(machine)	×
271
272	def _fill_machine(self, machine: Machine) -> Machine:	1✔
273	"""
274	:param ~spinn_machine.Machine machine:
275	:rtype: ~spinn_machine.Machine
276	"""
UNCOV 277	for chip_info in sorted(	×
278	self._chip_info.values(), key=lambda chip: (chip.x, chip.y)):
UNCOV 279	if (chip_info.ethernet_ip_address is not None and	×
280	(chip_info.x != chip_info.nearest_ethernet_x
281	or chip_info.y != chip_info.nearest_ethernet_y)):
UNCOV 282	if get_config_bool("Machine", "ignore_bad_ethernets"):	×
283	logger.warning(	×
284	"Chip {}:{} claimed it has ip address: {}. "
285	"This ip will not be used.",
286	chip_info.x, chip_info.y,
287	chip_info.ethernet_ip_address)
UNCOV 288	chip_info.clear_ethernet_ip_address()	×
289	else:
UNCOV 290	logger.warning(	×
291	"Not using chip {}:{} as it has an unexpected "
292	"ip address: {}", chip_info.x, chip_info.y,
293	chip_info.ethernet_ip_address)
UNCOV 294	continue	×
295
296	# If the above has not continued, add the chip
UNCOV 297	machine.add_chip(self._make_chip(chip_info, machine))	×
298
UNCOV 299	machine.validate()	×
300	return machine_repair(machine)	×
301
302	# Stuff below here is purely for dealing with ignores
303
304	def _process_ignore_links(self, machine: Machine):	1✔
305	"""
306	Processes the collection of ignore links to remove then from chip info.
307
308	Converts any local (x, y, IP address) to global (x, y)
309
310	Discards any ignores which are known to have no
311	affect based on the already read chip_info
312
313	Also removes any inverse links
314
315	Logs all actions except for ignores with unused IP addresses
316
317	:param ~spinn_machine.Machine machine:
318	An empty machine to handle wrap-arounds
319	"""
UNCOV 320	for ignore in IgnoreLink.parse_string(	×
321	get_config_str_or_none("Machine", "down_links")):
UNCOV 322	global_xy = self._ignores_local_to_global(	×
323	ignore.x, ignore.y, ignore.ip_address, machine)
UNCOV 324	if global_xy is None:	×
325	continue	×
326	chip_info = self._chip_info.get(global_xy, None)	×
327	if chip_info is None:	×
328	self._report_ignore(	×
329	"Discarding ignore link on chip {} as it is not/ no longer"
330	" in info", global_xy)
UNCOV 331	continue	×
332	link = ignore.link	×
333	if link in chip_info.working_links:	×
334	chip_info.working_links.remove(link)	×
335	self._report_ignore(	×
336	"On chip {} ignoring link:{}", global_xy, link)
337	# ignore the inverse link too
UNCOV 338	inv_xy = machine.xy_over_link(global_xy[0], global_xy[1], link)	×
339	if inv_xy in self._chip_info:	×
340	inv_chip_info = self._chip_info[inv_xy]	×
341	inv_link = (link + 3) % 6	×
342	if inv_link in inv_chip_info.working_links:	×
343	inv_chip_info.working_links.remove(inv_link)	×
344	self._report_ignore(	×
345	"On chip {} ignoring link {} as it is the inverse "
346	"of link {} on chip {}", inv_xy, inv_link, link,
347	global_xy)
348	else:
UNCOV 349	self._report_ignore(	×
350	"Discarding ignore link {} on chip {} as it is not/"
351	"no longer in info", link, global_xy)
352
353	def _preprocess_ignore_cores(self, machine: Machine):	1✔
354	"""
355	Converts the collection of ignore cores into a map of ignore by x,y.
356
357	Converts any local (x, y, IP address) to global (x, y)
358
359	Discards (with log messages) any ignores which are known to have no
360	affect based on the already read chip_info
361
362	Converts any physical cores to virtual ones.
363	Core numbers <= 0 are assumed to be 0 - physical_id
364
365	:param ~spinn_machine.Machine machine:
366	An empty machine to handle wrap-arounds
367	"""
368	# Convert by IP to global
UNCOV 369	for ignore in IgnoreCore.parse_string(	×
370	get_config_str_or_none("Machine", "down_cores")):
UNCOV 371	global_xy = self._ignores_local_to_global(	×
372	ignore.x, ignore.y, ignore.ip_address, machine)
UNCOV 373	if global_xy is None:	×
374	continue	×
375	p = self._get_virtual_p(global_xy, ignore.p)	×
376	if p is not None:	×
377	self._ignore_cores_map[global_xy].add(p)	×
378
379	def _preprocess_ignore_chips(self, machine: Machine):	1✔
380	"""
381	Processes the collection of ignore chips and discards their chip info.
382
383	Converts any local (x, y IP address) to global (x, y)
384
385	Discards any ignores which are known to have no
386	affect based on the already read chip_info
387
388	Logs all actions except for ignores with unused IP addresses
389
390	:param ~spinn_machine.Machine machine:
391	An empty machine to handle wrap-arounds
392	"""
UNCOV 393	for ignore in IgnoreChip.parse_string(	×
394	get_config_str_or_none("Machine", "down_chips")):
395	# Convert by IP to global
UNCOV 396	global_xy = self._ignores_local_to_global(	×
397	ignore.x, ignore.y, ignore.ip_address, machine)
UNCOV 398	if global_xy is None:	×
399	continue # Never on this machine	×
400	chip_info = self._chip_info.pop(global_xy, None)	×
401	if chip_info is None:	×
402	continue # Already ignored maybe by a dead chip list	×
403	self._report_ignore("Chip {} will be ignored", global_xy)	×
404	for link in chip_info.working_links:	×
405	# ignore the inverse link
UNCOV 406	inv_xy = machine.xy_over_link(global_xy[0], global_xy[1], link)	×
407	if inv_xy in self._chip_info:	×
408	inv_chip_info = self._chip_info[inv_xy]	×
409	inv_link = (link + 3) % 6	×
410	if inv_link in inv_chip_info.working_links:	×
411	inv_chip_info.working_links.remove(inv_link)	×
412	self._report_ignore(	×
413	"On chip {} ignoring link {} as it points to "
414	"ignored chip chip {}",
415	inv_xy, inv_link, global_xy)
416
417	def _ignores_local_to_global(	1✔
418	self, local_x: int, local_y: int, ip_address: Optional[str],
419	machine: Machine) -> Optional[XY]:
420	"""
421	:param int local_x:
422	:param int local_y:
423	:param str ip_address:
424	:param ~spinn_machine.Machine machine:
425	:rtype: tuple(int,int)
426	"""
UNCOV 427	if ip_address is None:	×
428	global_xy = (local_x, local_y)	×
429	else:
UNCOV 430	ethernet = self._ethernet_by_ipaddress(ip_address)	×
431	if ethernet is None:	×
432	self._report_ignore(	×
433	"Ignore with ip:{} will be discarded as no board with "
434	"that ip in this machine", ip_address)
UNCOV 435	return None	×
436	global_xy = machine.get_global_xy(	×
437	local_x, local_y, ethernet[0], ethernet[1])
UNCOV 438	self._report_ignore(	×
439	"Ignores for local x:{} y:{} and ip:{} map to global {} with "
440	"root {}", local_x, local_y, ip_address, global_xy,
441	self._chip_info[(0, 0)].ethernet_ip_address)
442
UNCOV 443	if global_xy in self._chip_info:	×
444	return global_xy	×
445	else:
UNCOV 446	self._report_ignore(	×
447	"Ignore for global chip {} will be discarded as no such chip "
448	"in this machine", global_xy)
UNCOV 449	return None	×
450
451	def _ethernet_by_ipaddress(self, ip_address: str) -> Optional[XY]:	1✔
452	"""
453	:param str ip_address:
454	:rtype: tuple(int,int)
455	"""
UNCOV 456	if self._ethernets is None:	×
457	self._ethernets = dict()	×
458	for chip_info in self._chip_info.values():	×
459	if chip_info.ethernet_ip_address is not None:	×
460	self._ethernets[chip_info.ethernet_ip_address] = \	×
461	(chip_info.x, chip_info.y)
UNCOV 462	return self._ethernets.get(ip_address, None)	×
463
464	def _get_virtual_p(self, xy: XY, p: int) -> Optional[int]:	1✔
465	"""
466	:param tuple(int,int) xy:
467	:param int p:
468	:rtype: int
469	"""
UNCOV 470	if p > 0:	×
471	self._report_ignore("On chip {} ignoring core {}", xy, p)	×
472	return p	×
473	virtual_map = self._physical_to_virtual_map[xy]	×
474	physical = 0 - p	×
475	if physical >= len(virtual_map) or virtual_map[physical] == 0xFF:	×
476	self._report_ignore(	×
477	"On chip {} physical core {} was not used "
478	"so ignore is being discarded.", xy, physical)
UNCOV 479	return None	×
480	virtual_p = virtual_map[physical]	×
481	if virtual_p == 0:	×
482	self._report_ignore(	×
483	"On chip {} physical core {} was used as the monitor "
484	"so will NOT be ignored", xy, physical)
UNCOV 485	return None	×
486	self._report_ignore(	×
487	"On chip {} ignoring core {} as it maps to physical "
488	"core {}", xy, virtual_p, physical)
UNCOV 489	return virtual_p	×
490
491	def _report_ignore(self, message: str, *args):	1✔
492	"""
493	Writes the ignore message by either creating or appending the report.
494
495	The implementation choice to reopen the file every time is not the
496	fastest but is the cleanest and safest for code that in default
497	conditions is never run.
498
499	:param str message:
500	"""
UNCOV 501	full_message = message.format(*args) + "\n"	×
502	report_file = join(UtilsDataView.get_run_dir_path(), REPORT_FILE)	×
503	with open(report_file, "a", encoding="utf-8") as r_file:	×
504	r_file.write(full_message)	×

SpiNNakerManchester / SpiNNMan / 8785183552

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous