6574804013

Committed 19 Oct 2023 12:47PM UTC coverage: 51.937% (+1.2%) from 50.777%

Build # 6574804013

Build Type

Pull #327

github

Committed by

Christian-B

Commit Message

typing changes

Pull Request Pull Request #327: Type Annotations and Checking

Run Details

105 of 1288 branches covered (0.0%)

Branch coverage included in aggregate %.

2375 of 2375 new or added lines in 180 files covered. (100.0%)

4775 of 8108 relevant lines covered (58.89%)

0.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

17.59

/spinnman/processes/get_machine_process.py

# Copyright (c) 2015 The University of Manchester
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from collections import defaultdict
from contextlib import suppress
import logging
import functools
from os.path import join
from typing import Dict, List, Optional, Set, Tuple, cast
from spinn_utilities.config_holder import (
    get_config_bool, get_config_int_or_none, get_config_str_or_none)
from spinn_utilities.data import UtilsDataView
from spinn_utilities.log import FormatAdapter
from spinn_utilities.typing.coords import XY
from spinn_machine import (Router, Chip, Link, Machine)
from spinn_machine.ignores import IgnoreChip, IgnoreCore, IgnoreLink
from spinn_machine.machine_factory import machine_repair
from spinnman.constants import (
    ROUTER_REGISTER_P2P_ADDRESS, SYSTEM_VARIABLE_BASE_ADDRESS)
from spinnman.data import SpiNNManDataView
from spinnman.messages.spinnaker_boot import (
    SystemVariableDefinition)
from spinnman.exceptions import SpinnmanUnexpectedResponseCodeException
from spinnman.messages.scp.abstract_messages import AbstractSCPRequest
from spinnman.messages.scp.impl import ReadMemory, ReadLink, GetChipInfo
from spinnman.messages.scp.impl.get_chip_info_response import (
    GetChipInfoResponse)
from spinnman.messages.scp.impl.read_memory import Response
from spinnman.model import ChipSummaryInfo, P2PTable
from spinnman.model.enums import CPUState
from .abstract_multi_connection_process import AbstractMultiConnectionProcess
from .abstract_multi_connection_process_connection_selector import \
    ConnectionSelector
from spinn_utilities.progress_bar import ProgressBar

logger = FormatAdapter(logging.getLogger(__name__))

REPORT_FILE = "Ignores_report.rpt"

P_TO_V = SystemVariableDefinition.physical_to_virtual_core_map
V_TO_P = SystemVariableDefinition.virtual_to_physical_core_map
P_TO_V_ADDR = SYSTEM_VARIABLE_BASE_ADDRESS + P_TO_V.offset
_P_TO_V_SIZE = cast(int, P_TO_V.array_size)
P_MAPS_SIZE = _P_TO_V_SIZE + cast(int, V_TO_P.array_size)


class GetMachineProcess(AbstractMultiConnectionProcess):
    """
    A process for getting the machine details over a set of connections.
    """
    __slots__ = (
        "_chip_info",
        # Used if there are any ignores with ip addresses
        # Holds a mapping from ip to board root (x,y)
        "_ethernets",
        # Holds a map from x,y to a set of virtual cores to ignores
        "_ignore_cores_map",
        "_p2p_column_data",
        # Holds a mapping from (x,y) to a mapping of physical to virtual core
        "_virtual_to_physical_map",
        # Holds a mapping from (x,y) to a mapping of virtual to physical core
        "_physical_to_virtual_map",
        # Progress bar to fill in as details are received
        "_progress")

    def __init__(self, connection_selector: ConnectionSelector):
        """
        :param ConnectionSelector connection_selector:
        """
        super().__init__(connection_selector)

        self._ignore_cores_map: Dict[XY, Set[int]] = defaultdict(set)

        self._p2p_column_data: List[Tuple[bytes, int]] = list()

        # A dictionary of (x, y) -> ChipInfo
        self._chip_info: Dict[XY, ChipSummaryInfo] = dict()

        # Set ethernets to None meaning not computed yet
        self._ethernets: Optional[Dict[str, XY]] = None

        # Maps between virtual and physical cores
        self._virtual_to_physical_map: Dict[XY, bytes] = dict()
        self._physical_to_virtual_map: Dict[XY, bytes] = dict()
        self._progress: Optional[ProgressBar] = None

    def _make_chip(self, chip_info: ChipSummaryInfo, machine: Machine) -> Chip:
        """
        Creates a chip from a ChipSummaryInfo structure.

        :param ChipSummaryInfo chip_info:
            The ChipSummaryInfo structure to create the chip from
        :return: The created chip
        :rtype: ~spinn_machine.Chip
        """
        # Keep track of progress
        self._progress = None
        # Create the down cores set if any
        n_cores = \
            SpiNNManDataView.get_machine_version().max_cores_per_chip
        n_cores = min(chip_info.n_cores, n_cores)
        core_states = chip_info.core_states
        down_cores = self._ignore_cores_map.get(
            (chip_info.x, chip_info.y), None)
        for i in range(1, n_cores):
            if core_states[i] != CPUState.IDLE:
                self._report_ignore(
                    "Not using core {}, {}, {} in state {}",
                    chip_info.x, chip_info.y, i,  core_states[i])
                if down_cores is None:
                    down_cores = set()
                down_cores.add(i)

        # Create the router
        router = self._make_router(chip_info, machine)

        # Create the chip's SDRAM object
        sdram_size = chip_info.largest_free_sdram_block
        max_sdram_size = get_config_int_or_none(
            "Machine", "max_sdram_allowed_per_chip")
        if (max_sdram_size is not None and
                sdram_size > max_sdram_size):
            sdram_size = max_sdram_size

        # Create the chip
        return Chip(
            x=chip_info.x, y=chip_info.y, n_processors=n_cores,
            router=router, sdram=sdram_size,
            ip_address=chip_info.ethernet_ip_address,
            nearest_ethernet_x=chip_info.nearest_ethernet_x,
            nearest_ethernet_y=chip_info.nearest_ethernet_y,
            down_cores=down_cores, parent_link=chip_info.parent_link,
            v_to_p_map=self._virtual_to_physical_map.get(
                (chip_info.x, chip_info.y)))

    def _make_router(
            self, chip_info: ChipSummaryInfo, machine: Machine) -> Router:
        """
        :param ChipSummaryInfo chip_info:
        :param ~spinn_machine.Machine machine:
        :rtype: ~spinn_machine.Router
        """
        links = list()
        for link in chip_info.working_links:
            dest_xy = machine.xy_over_link(chip_info.x, chip_info.y, link)
            if dest_xy in self._chip_info:
                links.append(Link(
                    chip_info.x, chip_info.y, link, dest_xy[0], dest_xy[1]))
            else:
                logger.warning(
                    "Link {},{}:{} points to {} but that is not included ",
                    chip_info.x, chip_info.y, link, dest_xy)

        return Router(
            links=links,
            n_available_multicast_entries=(
                chip_info.n_free_multicast_routing_entries))

    def __receive_p2p_data(
            self, column: int, scp_read_response: Response):
        """
        :param int column:
        :param Response scp_read_response:
        """
        self._p2p_column_data[column] = (
            scp_read_response.data, scp_read_response.offset)

    def _receive_chip_info(
            self, scp_read_chip_info_response: GetChipInfoResponse):
        """
        :param GetChipInfoResponse scp_read_chip_info_response:
        """
        chip_info = scp_read_chip_info_response.chip_info
        self._chip_info[chip_info.x, chip_info.y] = chip_info
        if self._progress is not None:
            self._progress.update()

    def _receive_p_maps(
            self, x: int, y: int, scp_read_response: Response):
        """
        Receive the physical-to-virtual and virtual-to-physical maps.

        :param Response scp_read_response:
        """
        data = scp_read_response.data
        off = scp_read_response.offset
        self._physical_to_virtual_map[x, y] = data[off:_P_TO_V_SIZE + off]
        off += _P_TO_V_SIZE
        self._virtual_to_physical_map[x, y] = data[off:]

    def _receive_error(
            self, request: AbstractSCPRequest, exception, tb, connection):
        """
        :param AbstractSCPRequest request:
        :param Exception exception:
        """
        # If we get an ReadLink with a
        # SpinnmanUnexpectedResponseCodeException, this is a failed link
        # and so can be ignored
        if isinstance(request, ReadLink):
            if isinstance(exception, SpinnmanUnexpectedResponseCodeException):
                return
        super()._receive_error(request, exception, tb, connection)

    def get_machine_details(
            self, boot_x: int, boot_y: int,
            width: int, height: int) -> Machine:
        """
        :param int boot_x:
        :param int boot_y:
        :param int width:
        :param int height:
        :rtype: ~spinn_machine.Machine
        """
        # Get the P2P table - 8 entries are packed into each 32-bit word
        p2p_column_bytes = P2PTable.get_n_column_bytes(height)
        blank = (b'', 0)
        self._p2p_column_data = [blank] * width
        with self._collect_responses():
            for column in range(width):
                offset = P2PTable.get_column_offset(column)
                self._send_request(
                    ReadMemory(
                        coordinates=(boot_x, boot_y, 0),
                        base_address=(ROUTER_REGISTER_P2P_ADDRESS + offset),
                        size=p2p_column_bytes),
                    functools.partial(self.__receive_p2p_data, column))
        p2p_table = P2PTable(width, height, self._p2p_column_data)

        # Get the chip information for each chip
        self._progress = ProgressBar(
            p2p_table.n_routes,
            f"Reading details from {p2p_table.n_routes} chips")
        with suppress(Exception), self._collect_responses():
            # Ignore errors, as any error here just means that a chip
            # is down that wasn't marked as down
            for (x, y) in p2p_table.iterchips():
                self._send_request(GetChipInfo(x, y), self._receive_chip_info)
                self._send_request(
                    ReadMemory((x, y, 0), P_TO_V_ADDR, P_MAPS_SIZE),
                    functools.partial(self._receive_p_maps, x, y))
        self._progress.end()

        # Warn about unexpected missing chips
        for (x, y) in p2p_table.iterchips():
            if (x, y) not in self._chip_info:
                logger.warning(
                    "Chip {}, {} was expected but didn't reply", x, y)

        version = SpiNNManDataView.get_machine_version()
        machine = version.create_machine(width, height)
        self._preprocess_ignore_chips(machine)
        self._process_ignore_links(machine)
        self._preprocess_ignore_cores(machine)

        return self._fill_machine(machine)

    def _fill_machine(self, machine: Machine) -> Machine:
        """
        :param ~spinn_machine.Machine machine:
        :rtype: ~spinn_machine.Machine
        """
        for chip_info in sorted(
                self._chip_info.values(), key=lambda chip: (chip.x, chip.y)):
            if (chip_info.ethernet_ip_address is not None and
                    (chip_info.x != chip_info.nearest_ethernet_x
                     or chip_info.y != chip_info.nearest_ethernet_y)):
                if get_config_bool("Machine", "ignore_bad_ethernets"):
                    logger.warning(
                        "Chip {}:{} claimed it has ip address: {}. "
                        "This ip will not be used.",
                        chip_info.x, chip_info.y,
                        chip_info.ethernet_ip_address)
                    chip_info.clear_ethernet_ip_address()
                else:
                    logger.warning(
                        "Not using chip {}:{} as it has an unexpected "
                        "ip address: {}", chip_info.x, chip_info.y,
                        chip_info.ethernet_ip_address)
                    continue

            # If the above has not continued, add the chip
            machine.add_chip(self._make_chip(chip_info, machine))

        machine.validate()
        return machine_repair(machine)

    # Stuff below here is purely for dealing with ignores

    def _process_ignore_links(self, machine: Machine):
        """
        Processes the collection of ignore links to remove then from chip info.

        Converts any local (x, y, IP address) to global (x, y)

        Discards any ignores which are known to have no
        affect based on the already read chip_info

        Also removes any inverse links

        Logs all actions except for ignores with unused IP addresses

        :param ~spinn_machine.Machine machine:
            An empty machine to handle wrap-arounds
        """
        for ignore in IgnoreLink.parse_string(
                get_config_str_or_none("Machine", "down_links")):
            global_xy = self._ignores_local_to_global(
                ignore.x, ignore.y, ignore.ip_address, machine)
            if global_xy is None:
                continue
            chip_info = self._chip_info.get(global_xy, None)
            if chip_info is None:
                self._report_ignore(
                    "Discarding ignore link on chip {} as it is not/ no longer"
                    " in info", global_xy)
                continue
            link = ignore.link
            if link in chip_info.working_links:
                chip_info.working_links.remove(link)
                self._report_ignore(
                    "On chip {} ignoring link:{}", global_xy, link)
                # ignore the inverse link too
                inv_xy = machine.xy_over_link(global_xy[0], global_xy[1], link)
                if inv_xy in self._chip_info:
                    inv_chip_info = self._chip_info[inv_xy]
                    inv_link = (link + 3) % 6
                    if inv_link in inv_chip_info.working_links:
                        inv_chip_info.working_links.remove(inv_link)
                        self._report_ignore(
                            "On chip {} ignoring link {} as it is the inverse "
                            "of link {} on chip {}", inv_xy, inv_link, link,
                            global_xy)
            else:
                self._report_ignore(
                    "Discarding ignore link {} on chip {} as it is not/"
                    "no longer in info", link, global_xy)

    def _preprocess_ignore_cores(self, machine: Machine):
        """
        Converts the collection of ignore cores into a map of ignore by x,y.

        Converts any local (x, y, IP address) to global (x, y)

        Discards (with log messages) any ignores which are known to have no
        affect based on the already read chip_info

        Converts any physical  cores to virtual ones.
        Core numbers <= 0 are assumed to be 0 - physical_id

        :param ~spinn_machine.Machine machine:
            An empty machine to handle wrap-arounds
        """
        # Convert by ip to global
        for ignore in IgnoreCore.parse_string(
                get_config_str_or_none("Machine", "down_cores")):
            global_xy = self._ignores_local_to_global(
                ignore.x, ignore.y, ignore.ip_address, machine)
            if global_xy is None:
                continue
            p = self._get_virtual_p(global_xy, ignore.p)
            if p is not None:
                self._ignore_cores_map[global_xy].add(p)

    def _preprocess_ignore_chips(self, machine: Machine):
        """
        Processes the collection of ignore chips and discards their chip info.

        Converts any local (x, y IP address) to global (x, y)

        Discards any ignores which are known to have no
        affect based on the already read chip_info

        Logs all actions except for ignores with unused IP addresses

        :param ~spinn_machine.Machine machine:
            An empty machine to handle wrap-arounds
        """
        for ignore in IgnoreChip.parse_string(
                get_config_str_or_none("Machine", "down_chips")):
            # Convert by ip to global
            global_xy = self._ignores_local_to_global(
                ignore.x, ignore.y, ignore.ip_address, machine)
            if global_xy is None:
                continue  # Never on this machine
            chip_info = self._chip_info.pop(global_xy, None)
            if chip_info is None:
                continue  # Already ignored maybe by a dead chip list
            self._report_ignore("Chip {} will be ignored", global_xy)
            for link in chip_info.working_links:
                # ignore the inverse link
                inv_xy = machine.xy_over_link(global_xy[0], global_xy[1], link)
                if inv_xy in self._chip_info:
                    inv_chip_info = self._chip_info[inv_xy]
                    inv_link = (link + 3) % 6
                    if inv_link in inv_chip_info.working_links:
                        inv_chip_info.working_links.remove(inv_link)
                        self._report_ignore(
                            "On chip {} ignoring link {} as it points to "
                            "ignored chip chip {}",
                            inv_xy, inv_link, global_xy)

    def _ignores_local_to_global(
            self, local_x: int, local_y: int, ip_address: Optional[str],
            machine: Machine) -> Optional[XY]:
        """
        :param int local_x:
        :param int local_y:
        :param str ip_address:
        :param ~spinn_machine.Machine machine:
        :rtype: tuple(int,int)
        """
        if ip_address is None:
            global_xy = (local_x, local_y)
        else:
            ethernet = self._ethernet_by_ipaddress(ip_address)
            if ethernet is None:
                self._report_ignore(
                    "Ignore with ip:{} will be discarded as no board with "
                    "that ip in this machine", ip_address)
                return None
            global_xy = machine.get_global_xy(
                local_x, local_y, ethernet[0], ethernet[1])
            self._report_ignore(
                "Ignores for local x:{} y:{} and ip:{} map to global {} with "
                "root {}", local_x, local_y, ip_address, global_xy,
                self._chip_info[(0, 0)].ethernet_ip_address)

        if global_xy in self._chip_info:
            return global_xy
        else:
            self._report_ignore(
                "Ignore for global chip {} will be discarded as no such chip "
                "in this machine", global_xy)
            return None

    def _ethernet_by_ipaddress(self, ip_address: str) -> Optional[XY]:
        """
        :param str ip_address:
        :rtype: tuple(int,int)
        """
        if self._ethernets is None:
            self._ethernets = dict()
            for chip_info in self._chip_info.values():
                if chip_info.ethernet_ip_address is not None:
                    self._ethernets[chip_info.ethernet_ip_address] = \
                        (chip_info.x, chip_info.y)
        return self._ethernets.get(ip_address, None)

    def _get_virtual_p(self, xy: XY, p: int) -> Optional[int]:
        """
        :param tuple(int,int) xy:
        :param int p:
        :rtype: int
        """
        if p > 0:
            self._report_ignore("On chip {} ignoring core {}", xy, p)
            return p
        virtual_map = self._physical_to_virtual_map[xy]
        physical = 0 - p
        if physical >= len(virtual_map) or virtual_map[physical] == 0xFF:
            self._report_ignore(
                "On chip {} physical core {} was not used "
                "so ignore is being discarded.", xy, physical)
            return None
        virtual_p = virtual_map[physical]
        if virtual_p == 0:
            self._report_ignore(
                "On chip {} physical core {} was used as the monitor "
                "so will NOT be ignored", xy, physical)
            return None
        self._report_ignore(
            "On chip {} ignoring core {} as it maps to physical "
            "core {}", xy, virtual_p, physical)
        return virtual_p

    def _report_ignore(self, message: str, *args):
        """
        Writes the ignore message by either creating or appending the report.

        The implementation choice to reopen the file every time is not the
        fastest but is the cleanest and safest for code that in default
        conditions is never run.

        :param str message:
        """
        full_message = message.format(*args) + "\n"
        report_file = join(UtilsDataView.get_run_dir_path(), REPORT_FILE)
        with open(report_file, "a", encoding="utf-8") as r_file:
            r_file.write(full_message)

1	# Copyright (c) 2015 The University of Manchester
2	#
3	# Licensed under the Apache License, Version 2.0 (the "License");
4	# you may not use this file except in compliance with the License.
5	# You may obtain a copy of the License at
6	#
7	# https://www.apache.org/licenses/LICENSE-2.0
8	#
9	# Unless required by applicable law or agreed to in writing, software
10	# distributed under the License is distributed on an "AS IS" BASIS,
11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	# See the License for the specific language governing permissions and
13	# limitations under the License.
14
15	from collections import defaultdict	1✔
16	from contextlib import suppress	1✔
17	import logging	1✔
18	import functools	1✔
19	from os.path import join	1✔
20	from typing import Dict, List, Optional, Set, Tuple, cast	1✔
21	from spinn_utilities.config_holder import (	1✔
22	get_config_bool, get_config_int_or_none, get_config_str_or_none)
23	from spinn_utilities.data import UtilsDataView	1✔
24	from spinn_utilities.log import FormatAdapter	1✔
25	from spinn_utilities.typing.coords import XY	1✔
26	from spinn_machine import (Router, Chip, Link, Machine)	1✔
27	from spinn_machine.ignores import IgnoreChip, IgnoreCore, IgnoreLink	1✔
28	from spinn_machine.machine_factory import machine_repair	1✔
29	from spinnman.constants import (	1✔
30	ROUTER_REGISTER_P2P_ADDRESS, SYSTEM_VARIABLE_BASE_ADDRESS)
31	from spinnman.data import SpiNNManDataView	1✔
32	from spinnman.messages.spinnaker_boot import (	1✔
33	SystemVariableDefinition)
34	from spinnman.exceptions import SpinnmanUnexpectedResponseCodeException	1✔
35	from spinnman.messages.scp.abstract_messages import AbstractSCPRequest	1✔
36	from spinnman.messages.scp.impl import ReadMemory, ReadLink, GetChipInfo	1✔
37	from spinnman.messages.scp.impl.get_chip_info_response import (	1✔
38	GetChipInfoResponse)
39	from spinnman.messages.scp.impl.read_memory import Response	1✔
40	from spinnman.model import ChipSummaryInfo, P2PTable	1✔
41	from spinnman.model.enums import CPUState	1✔
42	from .abstract_multi_connection_process import AbstractMultiConnectionProcess	1✔
43	from .abstract_multi_connection_process_connection_selector import \	1✔
44	ConnectionSelector
45	from spinn_utilities.progress_bar import ProgressBar	1✔
46
47	logger = FormatAdapter(logging.getLogger(__name__))	1✔
48
49	REPORT_FILE = "Ignores_report.rpt"	1✔
50
51	P_TO_V = SystemVariableDefinition.physical_to_virtual_core_map	1✔
52	V_TO_P = SystemVariableDefinition.virtual_to_physical_core_map	1✔
53	P_TO_V_ADDR = SYSTEM_VARIABLE_BASE_ADDRESS + P_TO_V.offset	1✔
54	_P_TO_V_SIZE = cast(int, P_TO_V.array_size)	1✔
55	P_MAPS_SIZE = _P_TO_V_SIZE + cast(int, V_TO_P.array_size)	1✔
56
57
58	class GetMachineProcess(AbstractMultiConnectionProcess):	1✔
59	"""
60	A process for getting the machine details over a set of connections.
61	"""
62	__slots__ = (	1✔
63	"_chip_info",
64	# Used if there are any ignores with ip addresses
65	# Holds a mapping from ip to board root (x,y)
66	"_ethernets",
67	# Holds a map from x,y to a set of virtual cores to ignores
68	"_ignore_cores_map",
69	"_p2p_column_data",
70	# Holds a mapping from (x,y) to a mapping of physical to virtual core
71	"_virtual_to_physical_map",
72	# Holds a mapping from (x,y) to a mapping of virtual to physical core
73	"_physical_to_virtual_map",
74	# Progress bar to fill in as details are received
75	"_progress")
76
77	def __init__(self, connection_selector: ConnectionSelector):	1✔
78	"""
79	:param ConnectionSelector connection_selector:
80	"""
81	super().__init__(connection_selector)	×
82
83	self._ignore_cores_map: Dict[XY, Set[int]] = defaultdict(set)	×
84
85	self._p2p_column_data: List[Tuple[bytes, int]] = list()	×
86
87	# A dictionary of (x, y) -> ChipInfo
88	self._chip_info: Dict[XY, ChipSummaryInfo] = dict()	×
89
90	# Set ethernets to None meaning not computed yet
91	self._ethernets: Optional[Dict[str, XY]] = None	×
92
93	# Maps between virtual and physical cores
94	self._virtual_to_physical_map: Dict[XY, bytes] = dict()	×
95	self._physical_to_virtual_map: Dict[XY, bytes] = dict()	×
96	self._progress: Optional[ProgressBar] = None	×
97
98	def _make_chip(self, chip_info: ChipSummaryInfo, machine: Machine) -> Chip:	1✔
99	"""
100	Creates a chip from a ChipSummaryInfo structure.
101
102	:param ChipSummaryInfo chip_info:
103	The ChipSummaryInfo structure to create the chip from
104	:return: The created chip
105	:rtype: ~spinn_machine.Chip
106	"""
107	# Keep track of progress
108	self._progress = None	×
109	# Create the down cores set if any
110	n_cores = \	×
111	SpiNNManDataView.get_machine_version().max_cores_per_chip
112	n_cores = min(chip_info.n_cores, n_cores)	×
113	core_states = chip_info.core_states	×
114	down_cores = self._ignore_cores_map.get(	×
115	(chip_info.x, chip_info.y), None)
116	for i in range(1, n_cores):	×
117	if core_states[i] != CPUState.IDLE:	×
118	self._report_ignore(	×
119	"Not using core {}, {}, {} in state {}",
120	chip_info.x, chip_info.y, i, core_states[i])
121	if down_cores is None:	×
122	down_cores = set()	×
123	down_cores.add(i)	×
124
125	# Create the router
126	router = self._make_router(chip_info, machine)	×
127
128	# Create the chip's SDRAM object
129	sdram_size = chip_info.largest_free_sdram_block	×
130	max_sdram_size = get_config_int_or_none(	×
131	"Machine", "max_sdram_allowed_per_chip")
132	if (max_sdram_size is not None and	×
133	sdram_size > max_sdram_size):
134	sdram_size = max_sdram_size	×
135
136	# Create the chip
137	return Chip(	×
138	x=chip_info.x, y=chip_info.y, n_processors=n_cores,
139	router=router, sdram=sdram_size,
140	ip_address=chip_info.ethernet_ip_address,
141	nearest_ethernet_x=chip_info.nearest_ethernet_x,
142	nearest_ethernet_y=chip_info.nearest_ethernet_y,
143	down_cores=down_cores, parent_link=chip_info.parent_link,
144	v_to_p_map=self._virtual_to_physical_map.get(
145	(chip_info.x, chip_info.y)))
146
147	def _make_router(	1✔
148	self, chip_info: ChipSummaryInfo, machine: Machine) -> Router:
149	"""
150	:param ChipSummaryInfo chip_info:
151	:param ~spinn_machine.Machine machine:
152	:rtype: ~spinn_machine.Router
153	"""
154	links = list()	×
155	for link in chip_info.working_links:	×
156	dest_xy = machine.xy_over_link(chip_info.x, chip_info.y, link)	×
157	if dest_xy in self._chip_info:	×
158	links.append(Link(	×
159	chip_info.x, chip_info.y, link, dest_xy[0], dest_xy[1]))
160	else:
161	logger.warning(	×
162	"Link {},{}:{} points to {} but that is not included ",
163	chip_info.x, chip_info.y, link, dest_xy)
164
165	return Router(	×
166	links=links,
167	n_available_multicast_entries=(
168	chip_info.n_free_multicast_routing_entries))
169
170	def __receive_p2p_data(	1✔
171	self, column: int, scp_read_response: Response):
172	"""
173	:param int column:
174	:param Response scp_read_response:
175	"""
176	self._p2p_column_data[column] = (	×
177	scp_read_response.data, scp_read_response.offset)
178
179	def _receive_chip_info(	1✔
180	self, scp_read_chip_info_response: GetChipInfoResponse):
181	"""
182	:param GetChipInfoResponse scp_read_chip_info_response:
183	"""
184	chip_info = scp_read_chip_info_response.chip_info	×
185	self._chip_info[chip_info.x, chip_info.y] = chip_info	×
186	if self._progress is not None:	×
187	self._progress.update()	×
188
189	def _receive_p_maps(	1✔
190	self, x: int, y: int, scp_read_response: Response):
191	"""
192	Receive the physical-to-virtual and virtual-to-physical maps.
193
194	:param Response scp_read_response:
195	"""
196	data = scp_read_response.data	×
197	off = scp_read_response.offset	×
198	self._physical_to_virtual_map[x, y] = data[off:_P_TO_V_SIZE + off]	×
199	off += _P_TO_V_SIZE	×
200	self._virtual_to_physical_map[x, y] = data[off:]	×
201
202	def _receive_error(	1✔
203	self, request: AbstractSCPRequest, exception, tb, connection):
204	"""
205	:param AbstractSCPRequest request:
206	:param Exception exception:
207	"""
208	# If we get an ReadLink with a
209	# SpinnmanUnexpectedResponseCodeException, this is a failed link
210	# and so can be ignored
211	if isinstance(request, ReadLink):	×
212	if isinstance(exception, SpinnmanUnexpectedResponseCodeException):	×
213	return	×
214	super()._receive_error(request, exception, tb, connection)	×
215
216	def get_machine_details(	1✔
217	self, boot_x: int, boot_y: int,
218	width: int, height: int) -> Machine:
219	"""
220	:param int boot_x:
221	:param int boot_y:
222	:param int width:
223	:param int height:
224	:rtype: ~spinn_machine.Machine
225	"""
226	# Get the P2P table - 8 entries are packed into each 32-bit word
227	p2p_column_bytes = P2PTable.get_n_column_bytes(height)	×
228	blank = (b'', 0)	×
229	self._p2p_column_data = [blank] * width	×
230	with self._collect_responses():	×
231	for column in range(width):	×
232	offset = P2PTable.get_column_offset(column)	×
233	self._send_request(	×
234	ReadMemory(
235	coordinates=(boot_x, boot_y, 0),
236	base_address=(ROUTER_REGISTER_P2P_ADDRESS + offset),
237	size=p2p_column_bytes),
238	functools.partial(self.__receive_p2p_data, column))
239	p2p_table = P2PTable(width, height, self._p2p_column_data)	×
240
241	# Get the chip information for each chip
242	self._progress = ProgressBar(	×
243	p2p_table.n_routes,
244	f"Reading details from {p2p_table.n_routes} chips")
245	with suppress(Exception), self._collect_responses():	×
246	# Ignore errors, as any error here just means that a chip
247	# is down that wasn't marked as down
248	for (x, y) in p2p_table.iterchips():	×
249	self._send_request(GetChipInfo(x, y), self._receive_chip_info)	×
250	self._send_request(	×
251	ReadMemory((x, y, 0), P_TO_V_ADDR, P_MAPS_SIZE),
252	functools.partial(self._receive_p_maps, x, y))
253	self._progress.end()	×
254
255	# Warn about unexpected missing chips
256	for (x, y) in p2p_table.iterchips():	×
257	if (x, y) not in self._chip_info:	×
258	logger.warning(	×
259	"Chip {}, {} was expected but didn't reply", x, y)
260
261	version = SpiNNManDataView.get_machine_version()	×
262	machine = version.create_machine(width, height)	×
263	self._preprocess_ignore_chips(machine)	×
264	self._process_ignore_links(machine)	×
265	self._preprocess_ignore_cores(machine)	×
266
267	return self._fill_machine(machine)	×
268
269	def _fill_machine(self, machine: Machine) -> Machine:	1✔
270	"""
271	:param ~spinn_machine.Machine machine:
272	:rtype: ~spinn_machine.Machine
273	"""
274	for chip_info in sorted(	×
275	self._chip_info.values(), key=lambda chip: (chip.x, chip.y)):
276	if (chip_info.ethernet_ip_address is not None and	×
277	(chip_info.x != chip_info.nearest_ethernet_x
278	or chip_info.y != chip_info.nearest_ethernet_y)):
279	if get_config_bool("Machine", "ignore_bad_ethernets"):	×
280	logger.warning(	×
281	"Chip {}:{} claimed it has ip address: {}. "
282	"This ip will not be used.",
283	chip_info.x, chip_info.y,
284	chip_info.ethernet_ip_address)
285	chip_info.clear_ethernet_ip_address()	×
286	else:
287	logger.warning(	×
288	"Not using chip {}:{} as it has an unexpected "
289	"ip address: {}", chip_info.x, chip_info.y,
290	chip_info.ethernet_ip_address)
291	continue	×
292
293	# If the above has not continued, add the chip
294	machine.add_chip(self._make_chip(chip_info, machine))	×
295
296	machine.validate()	×
297	return machine_repair(machine)	×
298
299	# Stuff below here is purely for dealing with ignores
300
301	def _process_ignore_links(self, machine: Machine):	1✔
302	"""
303	Processes the collection of ignore links to remove then from chip info.
304
305	Converts any local (x, y, IP address) to global (x, y)
306
307	Discards any ignores which are known to have no
308	affect based on the already read chip_info
309
310	Also removes any inverse links
311
312	Logs all actions except for ignores with unused IP addresses
313
314	:param ~spinn_machine.Machine machine:
315	An empty machine to handle wrap-arounds
316	"""
317	for ignore in IgnoreLink.parse_string(	×
318	get_config_str_or_none("Machine", "down_links")):
319	global_xy = self._ignores_local_to_global(	×
320	ignore.x, ignore.y, ignore.ip_address, machine)
321	if global_xy is None:	×
322	continue	×
323	chip_info = self._chip_info.get(global_xy, None)	×
324	if chip_info is None:	×
325	self._report_ignore(	×
326	"Discarding ignore link on chip {} as it is not/ no longer"
327	" in info", global_xy)
328	continue	×
329	link = ignore.link	×
330	if link in chip_info.working_links:	×
331	chip_info.working_links.remove(link)	×
332	self._report_ignore(	×
333	"On chip {} ignoring link:{}", global_xy, link)
334	# ignore the inverse link too
335	inv_xy = machine.xy_over_link(global_xy[0], global_xy[1], link)	×
336	if inv_xy in self._chip_info:	×
337	inv_chip_info = self._chip_info[inv_xy]	×
338	inv_link = (link + 3) % 6	×
339	if inv_link in inv_chip_info.working_links:	×
340	inv_chip_info.working_links.remove(inv_link)	×
341	self._report_ignore(	×
342	"On chip {} ignoring link {} as it is the inverse "
343	"of link {} on chip {}", inv_xy, inv_link, link,
344	global_xy)
345	else:
346	self._report_ignore(	×
347	"Discarding ignore link {} on chip {} as it is not/"
348	"no longer in info", link, global_xy)
349
350	def _preprocess_ignore_cores(self, machine: Machine):	1✔
351	"""
352	Converts the collection of ignore cores into a map of ignore by x,y.
353
354	Converts any local (x, y, IP address) to global (x, y)
355
356	Discards (with log messages) any ignores which are known to have no
357	affect based on the already read chip_info
358
359	Converts any physical cores to virtual ones.
360	Core numbers <= 0 are assumed to be 0 - physical_id
361
362	:param ~spinn_machine.Machine machine:
363	An empty machine to handle wrap-arounds
364	"""
365	# Convert by ip to global
366	for ignore in IgnoreCore.parse_string(	×
367	get_config_str_or_none("Machine", "down_cores")):
368	global_xy = self._ignores_local_to_global(	×
369	ignore.x, ignore.y, ignore.ip_address, machine)
370	if global_xy is None:	×
371	continue	×
372	p = self._get_virtual_p(global_xy, ignore.p)	×
373	if p is not None:	×
374	self._ignore_cores_map[global_xy].add(p)	×
375
376	def _preprocess_ignore_chips(self, machine: Machine):	1✔
377	"""
378	Processes the collection of ignore chips and discards their chip info.
379
380	Converts any local (x, y IP address) to global (x, y)
381
382	Discards any ignores which are known to have no
383	affect based on the already read chip_info
384
385	Logs all actions except for ignores with unused IP addresses
386
387	:param ~spinn_machine.Machine machine:
388	An empty machine to handle wrap-arounds
389	"""
390	for ignore in IgnoreChip.parse_string(	×
391	get_config_str_or_none("Machine", "down_chips")):
392	# Convert by ip to global
393	global_xy = self._ignores_local_to_global(	×
394	ignore.x, ignore.y, ignore.ip_address, machine)
395	if global_xy is None:	×
396	continue # Never on this machine	×
397	chip_info = self._chip_info.pop(global_xy, None)	×
398	if chip_info is None:	×
399	continue # Already ignored maybe by a dead chip list	×
400	self._report_ignore("Chip {} will be ignored", global_xy)	×
401	for link in chip_info.working_links:	×
402	# ignore the inverse link
403	inv_xy = machine.xy_over_link(global_xy[0], global_xy[1], link)	×
404	if inv_xy in self._chip_info:	×
405	inv_chip_info = self._chip_info[inv_xy]	×
406	inv_link = (link + 3) % 6	×
407	if inv_link in inv_chip_info.working_links:	×
408	inv_chip_info.working_links.remove(inv_link)	×
409	self._report_ignore(	×
410	"On chip {} ignoring link {} as it points to "
411	"ignored chip chip {}",
412	inv_xy, inv_link, global_xy)
413
414	def _ignores_local_to_global(	1✔
415	self, local_x: int, local_y: int, ip_address: Optional[str],
416	machine: Machine) -> Optional[XY]:
417	"""
418	:param int local_x:
419	:param int local_y:
420	:param str ip_address:
421	:param ~spinn_machine.Machine machine:
422	:rtype: tuple(int,int)
423	"""
424	if ip_address is None:	×
425	global_xy = (local_x, local_y)	×
426	else:
427	ethernet = self._ethernet_by_ipaddress(ip_address)	×
428	if ethernet is None:	×
429	self._report_ignore(	×
430	"Ignore with ip:{} will be discarded as no board with "
431	"that ip in this machine", ip_address)
432	return None	×
433	global_xy = machine.get_global_xy(	×
434	local_x, local_y, ethernet[0], ethernet[1])
435	self._report_ignore(	×
436	"Ignores for local x:{} y:{} and ip:{} map to global {} with "
437	"root {}", local_x, local_y, ip_address, global_xy,
438	self._chip_info[(0, 0)].ethernet_ip_address)
439
440	if global_xy in self._chip_info:	×
441	return global_xy	×
442	else:
443	self._report_ignore(	×
444	"Ignore for global chip {} will be discarded as no such chip "
445	"in this machine", global_xy)
446	return None	×
447
448	def _ethernet_by_ipaddress(self, ip_address: str) -> Optional[XY]:	1✔
449	"""
450	:param str ip_address:
451	:rtype: tuple(int,int)
452	"""
453	if self._ethernets is None:	×
454	self._ethernets = dict()	×
455	for chip_info in self._chip_info.values():	×
456	if chip_info.ethernet_ip_address is not None:	×
457	self._ethernets[chip_info.ethernet_ip_address] = \	×
458	(chip_info.x, chip_info.y)
459	return self._ethernets.get(ip_address, None)	×
460
461	def _get_virtual_p(self, xy: XY, p: int) -> Optional[int]:	1✔
462	"""
463	:param tuple(int,int) xy:
464	:param int p:
465	:rtype: int
466	"""
467	if p > 0:	×
468	self._report_ignore("On chip {} ignoring core {}", xy, p)	×
469	return p	×
470	virtual_map = self._physical_to_virtual_map[xy]	×
471	physical = 0 - p	×
472	if physical >= len(virtual_map) or virtual_map[physical] == 0xFF:	×
473	self._report_ignore(	×
474	"On chip {} physical core {} was not used "
475	"so ignore is being discarded.", xy, physical)
476	return None	×
477	virtual_p = virtual_map[physical]	×
478	if virtual_p == 0:	×
479	self._report_ignore(	×
480	"On chip {} physical core {} was used as the monitor "
481	"so will NOT be ignored", xy, physical)
482	return None	×
483	self._report_ignore(	×
484	"On chip {} ignoring core {} as it maps to physical "
485	"core {}", xy, virtual_p, physical)
486	return virtual_p	×
487
488	def _report_ignore(self, message: str, *args):	1✔
489	"""
490	Writes the ignore message by either creating or appending the report.
491
492	The implementation choice to reopen the file every time is not the
493	fastest but is the cleanest and safest for code that in default
494	conditions is never run.
495
496	:param str message:
497	"""
498	full_message = message.format(*args) + "\n"	×
499	report_file = join(UtilsDataView.get_run_dir_path(), REPORT_FILE)	×
500	with open(report_file, "a", encoding="utf-8") as r_file:	×
501	r_file.write(full_message)	×

SpiNNakerManchester / SpiNNMan / 6574804013

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous