8143402062

Committed 04 Mar 2024 04:13PM UTC coverage: 47.969% (-0.02%) from 47.992%

Build # 8143402062

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #1157 from SpiNNakerManchester/pylint_default

Pylint default

Run Details

2050 of 4800 branches covered (42.71%)

Branch coverage included in aggregate %.

63 of 110 new or added lines in 37 files covered. (57.27%)

26 existing lines in 18 files now uncovered.

5484 of 10906 relevant lines covered (50.28%)

0.5 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

19.05

/spinn_front_end_common/interface/interface_functions/graph_data_specification_writer.py

# Copyright (c) 2016 The University of Manchester
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from collections import defaultdict
import logging
import os
from typing import Iterable, List, Sequence, Optional

from spinn_utilities.progress_bar import ProgressBar
from spinn_utilities.log import FormatAdapter
from pacman.model.graphs import AbstractVertex
from pacman.model.graphs.machine import MachineVertex
from pacman.model.resources import MultiRegionSDRAM, ConstantSDRAM
from pacman.model.placements import Placement
from spinn_front_end_common.abstract_models import (
    AbstractRewritesDataSpecification, AbstractGeneratesDataSpecification)
from spinn_front_end_common.data import FecDataView
from spinn_front_end_common.utilities.constants import APP_PTR_TABLE_BYTE_SIZE
from spinn_front_end_common.utilities.exceptions import (
    ConfigurationException, DataSpecException)
from spinn_front_end_common.interface.ds import (
    DataSpecificationGenerator, DsSqlliteDatabase)
from spinn_front_end_common.utilities.utility_calls import get_report_writer

logger = FormatAdapter(logging.getLogger(__name__))


def graph_data_specification_writer(placement_order=None):
    """
    :param list(~pacman.model.placements.Placement) placement_order:
        the optional order in which placements should be examined
    :return: Path to DSG targets database
    :rtype: str
    :raises ConfigurationException:
        If the DSG asks to use more SDRAM than is available.
    """
    return _GraphDataSpecificationWriter().run(placement_order)


class _GraphDataSpecificationWriter(object):
    """
    Executes the data specification generation step.
    """

    __slots__ = (
        # Dict of SDRAM usage by chip coordinates
        "_sdram_usage",
        # Dict of list of vertices by chip coordinates
        "_vertices_by_chip")

    def __init__(self):
        self._sdram_usage = defaultdict(lambda: 0)
        self._vertices_by_chip = defaultdict(list)

    def run(self, placement_order: Optional[Sequence[Placement]] = None):
        """
        :param list(~pacman.model.placements.Placement) placement_order:
            the optional order in which placements should be examined
        :return: Path to DSG targets database
        :rtype: str
        :raises ConfigurationException:
            If the DSG asks to use more SDRAM than is available.
        """
        # iterate though vertices and call generate_data_spec for each
        # vertex
        path = os.path.join(FecDataView.get_run_dir_path(),
                            f"ds{FecDataView.get_reset_str()}.sqlite3")
        with DsSqlliteDatabase(path) as ds_db:
            ds_db.write_session_credentials_to_db()
            ds_db. set_app_id()

            placements: Iterable[Placement]
            if placement_order is None:
                placements = FecDataView.iterate_placemements()
                n_placements = FecDataView.get_n_placements()
            else:
                placements = placement_order
                n_placements = len(placement_order)

            progress = ProgressBar(n_placements,
                                   "Generating data specifications")
            vertices_to_reset: List[AbstractRewritesDataSpecification] = list()

            for placement in progress.over(placements):
                # Try to generate the data spec for the placement
                vertex = placement.vertex
                generated = self.__generate_data_spec_for_vertices(
                    placement, vertex, ds_db)

                if generated and isinstance(
                        vertex, AbstractRewritesDataSpecification):
                    vertices_to_reset.append(vertex)

                # If the spec wasn't generated directly, and there is an
                # application vertex, try with that
                if not generated and vertex.app_vertex is not None:
                    generated = self.__generate_data_spec_for_vertices(
                        placement, vertex.app_vertex, ds_db)
                    if generated and isinstance(
                            vertex.app_vertex,
                            AbstractRewritesDataSpecification):
                        vertices_to_reset.append(vertex.app_vertex)

            # Ensure that the vertices know their regions have been reloaded
            for rewriter in vertices_to_reset:
                rewriter.set_reload_required(False)

            self._run_check_queries(ds_db)

        return path

    def __generate_data_spec_for_vertices(
            self, placement: Placement, vertex: AbstractVertex,
            ds_db: DsSqlliteDatabase) -> bool:
        """
        :param ~.Placement placement: placement of machine graph to cores
        :param ~.AbstractVertex vertex: the specific vertex to write DSG for.
        :param DsSqlliteDatabase ds_db:
        :return: True if the vertex was data spec-able, False otherwise
        :rtype: bool
        :raises ConfigurationException: if things don't fit
        """
        # if the vertex can generate a DSG, call it
        if not isinstance(vertex, AbstractGeneratesDataSpecification):
            return False

        x = placement.x
        y = placement.y
        p = placement.p

        report_writer = get_report_writer(x, y, p)
        spec = DataSpecificationGenerator(
            x, y, p, vertex, ds_db, report_writer)

        # generate the DSG file
        vertex.generate_data_specification(spec, placement)

        # Check the memory usage
        total_size = ds_db.get_total_regions_size(x, y, p)
        region_size = APP_PTR_TABLE_BYTE_SIZE + total_size
        total_est_size = 0

        # Check per-region memory usage if possible
        if isinstance(vertex, MachineVertex):
            sdram = vertex.sdram_required
            if isinstance(sdram, MultiRegionSDRAM):
                region_sizes = ds_db.get_region_sizes(x, y, p)
                for i, size in region_sizes.items():
                    est_size = sdram.regions.get(i, ConstantSDRAM(0))
                    est_size = est_size.get_total_sdram(
                        FecDataView.get_max_run_time_steps())
                    total_est_size += est_size
                    if size > est_size:
                        logger.warning(
                            "Region {} of vertex {} is bigger than "
                            "expected: {} estimated vs. {} actual",
                            i, vertex.label, est_size, size)

        self._vertices_by_chip[x, y].append(vertex)
        self._sdram_usage[x, y] += total_size
        if (self._sdram_usage[x, y] <=
                FecDataView().get_chip_at(x, y).sdram):
            return True

        # creating the error message which contains the memory usage of
        # what each core within the chip uses and its original estimate.
        # pylint: disable=consider-using-f-string
        memory_usage = "\n".join(
            "    {}: {} (total={}, estimated={})".format(
                vert, region_size, total_est_size,
                vert.sdram_required.get_total_sdram(
                    FecDataView.get_max_run_time_steps()))
            for vert in self._vertices_by_chip[x, y])

        raise ConfigurationException(
            f"Too much SDRAM has been used on {x}, {y}.  Vertices and"
            f" their usage on that chip is as follows:\n{memory_usage}")

    def _run_check_queries(self, ds_db: DsSqlliteDatabase):
        msg = ""
        for x, y, p, region, reference, lbl in ds_db.get_unlinked_references():
            if lbl is None:
                label = ""
            else:
                label = f"({lbl})"
            msg += f"core {x}:{y}:{p} has a broken reference " \
                f"{reference}{label} from region {region} "

        for x, y, p, region in ds_db.get_double_region():
            msg += f"core {x}:{y}:{p} {region} " \
                "has both a region reserve and a reference "

        if msg != "":
            raise DataSpecException(msg)

1	# Copyright (c) 2016 The University of Manchester
2	#
3	# Licensed under the Apache License, Version 2.0 (the "License");
4	# you may not use this file except in compliance with the License.
5	# You may obtain a copy of the License at
6	#
7	# https://www.apache.org/licenses/LICENSE-2.0
8	#
9	# Unless required by applicable law or agreed to in writing, software
10	# distributed under the License is distributed on an "AS IS" BASIS,
11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	# See the License for the specific language governing permissions and
13	# limitations under the License.
14
15	from collections import defaultdict	1✔
16	import logging	1✔
17	import os	1✔
18	from typing import Iterable, List, Sequence, Optional	1✔
19
20	from spinn_utilities.progress_bar import ProgressBar	1✔
21	from spinn_utilities.log import FormatAdapter	1✔
22	from pacman.model.graphs import AbstractVertex	1✔
23	from pacman.model.graphs.machine import MachineVertex	1✔
24	from pacman.model.resources import MultiRegionSDRAM, ConstantSDRAM	1✔
25	from pacman.model.placements import Placement	1✔
26	from spinn_front_end_common.abstract_models import (	1✔
27	AbstractRewritesDataSpecification, AbstractGeneratesDataSpecification)
28	from spinn_front_end_common.data import FecDataView	1✔
29	from spinn_front_end_common.utilities.constants import APP_PTR_TABLE_BYTE_SIZE	1✔
30	from spinn_front_end_common.utilities.exceptions import (	1✔
31	ConfigurationException, DataSpecException)
32	from spinn_front_end_common.interface.ds import (	1✔
33	DataSpecificationGenerator, DsSqlliteDatabase)
34	from spinn_front_end_common.utilities.utility_calls import get_report_writer	1✔
35
36	logger = FormatAdapter(logging.getLogger(__name__))	1✔
37
38
39	def graph_data_specification_writer(placement_order=None):	1✔
40	"""
41	:param list(~pacman.model.placements.Placement) placement_order:
42	the optional order in which placements should be examined
43	:return: Path to DSG targets database
44	:rtype: str
45	:raises ConfigurationException:
46	If the DSG asks to use more SDRAM than is available.
47	"""
48	return _GraphDataSpecificationWriter().run(placement_order)	×
49
50
51	class _GraphDataSpecificationWriter(object):	1✔
52	"""
53	Executes the data specification generation step.
54	"""
55
56	__slots__ = (	1✔
57	# Dict of SDRAM usage by chip coordinates
58	"_sdram_usage",
59	# Dict of list of vertices by chip coordinates
60	"_vertices_by_chip")
61
62	def __init__(self):	1✔
63	self._sdram_usage = defaultdict(lambda: 0)	×
64	self._vertices_by_chip = defaultdict(list)	×
65
66	def run(self, placement_order: Optional[Sequence[Placement]] = None):	1✔
67	"""
68	:param list(~pacman.model.placements.Placement) placement_order:
69	the optional order in which placements should be examined
70	:return: Path to DSG targets database
71	:rtype: str
72	:raises ConfigurationException:
73	If the DSG asks to use more SDRAM than is available.
74	"""
75	# iterate though vertices and call generate_data_spec for each
76	# vertex
77	path = os.path.join(FecDataView.get_run_dir_path(),	×
78	f"ds{FecDataView.get_reset_str()}.sqlite3")
79	with DsSqlliteDatabase(path) as ds_db:	×
80	ds_db.write_session_credentials_to_db()	×
81	ds_db. set_app_id()	×
82
83	placements: Iterable[Placement]
84	if placement_order is None:	×
85	placements = FecDataView.iterate_placemements()	×
86	n_placements = FecDataView.get_n_placements()	×
87	else:
88	placements = placement_order	×
89	n_placements = len(placement_order)	×
90
91	progress = ProgressBar(n_placements,	×
92	"Generating data specifications")
93	vertices_to_reset: List[AbstractRewritesDataSpecification] = list()	×
94
95	for placement in progress.over(placements):	×
96	# Try to generate the data spec for the placement
97	vertex = placement.vertex	×
98	generated = self.__generate_data_spec_for_vertices(	×
99	placement, vertex, ds_db)
100
101	if generated and isinstance(	×
102	vertex, AbstractRewritesDataSpecification):
103	vertices_to_reset.append(vertex)	×
104
105	# If the spec wasn't generated directly, and there is an
106	# application vertex, try with that
107	if not generated and vertex.app_vertex is not None:	×
108	generated = self.__generate_data_spec_for_vertices(	×
109	placement, vertex.app_vertex, ds_db)
110	if generated and isinstance(	×
111	vertex.app_vertex,
112	AbstractRewritesDataSpecification):
113	vertices_to_reset.append(vertex.app_vertex)	×
114
115	# Ensure that the vertices know their regions have been reloaded
116	for rewriter in vertices_to_reset:	×
117	rewriter.set_reload_required(False)	×
118
119	self._run_check_queries(ds_db)	×
120
121	return path	×
122
123	def __generate_data_spec_for_vertices(	1✔
124	self, placement: Placement, vertex: AbstractVertex,
125	ds_db: DsSqlliteDatabase) -> bool:
126	"""
127	:param ~.Placement placement: placement of machine graph to cores
128	:param ~.AbstractVertex vertex: the specific vertex to write DSG for.
129	:param DsSqlliteDatabase ds_db:
130	:return: True if the vertex was data spec-able, False otherwise
131	:rtype: bool
132	:raises ConfigurationException: if things don't fit
133	"""
134	# if the vertex can generate a DSG, call it
135	if not isinstance(vertex, AbstractGeneratesDataSpecification):	×
136	return False	×
137
138	x = placement.x	×
139	y = placement.y	×
140	p = placement.p	×
141
142	report_writer = get_report_writer(x, y, p)	×
143	spec = DataSpecificationGenerator(	×
144	x, y, p, vertex, ds_db, report_writer)
145
146	# generate the DSG file
147	vertex.generate_data_specification(spec, placement)	×
148
149	# Check the memory usage
150	total_size = ds_db.get_total_regions_size(x, y, p)	×
151	region_size = APP_PTR_TABLE_BYTE_SIZE + total_size	×
152	total_est_size = 0	×
153
154	# Check per-region memory usage if possible
155	if isinstance(vertex, MachineVertex):	×
156	sdram = vertex.sdram_required	×
157	if isinstance(sdram, MultiRegionSDRAM):	×
158	region_sizes = ds_db.get_region_sizes(x, y, p)	×
159	for i, size in region_sizes.items():	×
160	est_size = sdram.regions.get(i, ConstantSDRAM(0))	×
161	est_size = est_size.get_total_sdram(	×
162	FecDataView.get_max_run_time_steps())
163	total_est_size += est_size	×
164	if size > est_size:	×
165	logger.warning(	×
166	"Region {} of vertex {} is bigger than "
167	"expected: {} estimated vs. {} actual",
168	i, vertex.label, est_size, size)
169
170	self._vertices_by_chip[x, y].append(vertex)	×
171	self._sdram_usage[x, y] += total_size	×
172	if (self._sdram_usage[x, y] <=	×
173	FecDataView().get_chip_at(x, y).sdram):
174	return True	×
175
176	# creating the error message which contains the memory usage of
177	# what each core within the chip uses and its original estimate.
178	# pylint: disable=consider-using-f-string
UNCOV 179	memory_usage = "\n".join(	×
180	" {}: {} (total={}, estimated={})".format(
181	vert, region_size, total_est_size,
182	vert.sdram_required.get_total_sdram(
183	FecDataView.get_max_run_time_steps()))
184	for vert in self._vertices_by_chip[x, y])
185
186	raise ConfigurationException(	×
187	f"Too much SDRAM has been used on {x}, {y}. Vertices and"
188	f" their usage on that chip is as follows:\n{memory_usage}")
189
190	def _run_check_queries(self, ds_db: DsSqlliteDatabase):	1✔
191	msg = ""	×
192	for x, y, p, region, reference, lbl in ds_db.get_unlinked_references():	×
193	if lbl is None:	×
194	label = ""	×
195	else:
196	label = f"({lbl})"	×
197	msg += f"core {x}:{y}:{p} has a broken reference " \	×
198	f"{reference}{label} from region {region} "
199
200	for x, y, p, region in ds_db.get_double_region():	×
201	msg += f"core {x}:{y}:{p} {region} " \	×
202	"has both a region reserve and a reference "
203
204	if msg != "":	×
205	raise DataSpecException(msg)	×

SpiNNakerManchester / SpiNNFrontEndCommon / 8143402062

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous