8906184429

Committed 01 May 2024 06:19AM UTC coverage: 47.241% (-0.06%) from 47.305%

Build # 8906184429

Build Type

Pull #1181

github

Committed by

Christian-B

Commit Message

add comment

Pull Request Pull Request #1181: Compressor error

Run Details

1761 of 4471 branches covered (39.39%)

Branch coverage included in aggregate %.

9 of 32 new or added lines in 4 files covered. (28.13%)

1 existing line in 1 file now uncovered.

5507 of 10914 relevant lines covered (50.46%)

0.5 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

50.27

/spinn_front_end_common/interface/config_handler.py

# Copyright (c) 2017 The University of Manchester
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from configparser import NoOptionError
import logging
import os
import shutil
import traceback
from typing import Optional, Type
from spinn_utilities.log import FormatAdapter
from spinn_utilities.config_holder import (
    config_options, has_config_option, load_config, get_config_bool,
    get_config_int, get_config_str, get_config_str_list, set_config)
from spinn_front_end_common.interface.interface_functions.\
    insert_chip_power_monitors_to_graphs import sample_chip_power_monitor
from spinn_front_end_common.interface.interface_functions.\
    insert_extra_monitor_vertices_to_graphs import (
        sample_monitor_vertex, sample_speedup_vertex)
from spinn_front_end_common.interface.provenance import LogStoreDB
from spinn_front_end_common.data.fec_data_writer import FecDataWriter
from spinn_front_end_common.utilities.exceptions import ConfigurationException

logger = FormatAdapter(logging.getLogger(__name__))

APP_DIRNAME = 'application_generated_data_files'
FINISHED_FILENAME = "finished"
ERRORED_FILENAME = "errored"
REPORTS_DIRNAME = "reports"
TIMESTAMP_FILENAME = "time_stamp"
WARNING_LOGS_FILENAME = "warning_logs.txt"

# options names are all lower without _ inside config
_DEBUG_ENABLE_OPTS = frozenset([
    "reportsenabled",
    "cleariobufduringrun", "extractiobuf"])
_DEBUG_MAPPING_OPTS = frozenset([
    "routertablecompressasfaraspossible", "runcompressionchecker"])
_REPORT_DISABLE_OPTS = frozenset([
    "cleariobufduringrun", "extractiobuf"])


class ConfigHandler(object):
    """
    Superclass of AbstractSpinnakerBase that handles function only
    dependent of the configuration and the order its methods are called.
    """

    __slots__ = (
        # The writer and therefore view of the global data
        "_data_writer", )

    def __init__(self, data_writer_cls: Optional[Type[FecDataWriter]] = None):
        """
        :param FecDataWriter data_writer:
            The Global data writer object
        """
        load_config()

        if data_writer_cls:
            self._data_writer = data_writer_cls.setup()
        else:
            self._data_writer = FecDataWriter.setup()
        logger.set_log_store(LogStoreDB())

        # set up machine targeted data
        self._debug_configs()
        self._previous_handler()
        self._reserve_system_vertices()

    def _debug_configs(self) -> None:
        """
        Adjusts and checks the configuration based on mode and
        `reports_enabled`.

        :raises ConfigurationException:
        """
        if get_config_str("Mode", "mode") == "Debug":
            for option in config_options("Reports"):
                # options names are all lower without _ inside config
                if option in _DEBUG_ENABLE_OPTS or option[:5] == "write":
                    if not get_config_bool("Reports", option):
                        set_config("Reports", option, "True")
                        logger.info("As mode == \"Debug\", [Reports] {} "
                                    "has been set to True", option)
            for option in config_options("Mapping"):
                # options names are all lower without _ inside config
                if option in _DEBUG_MAPPING_OPTS:
                    if not get_config_bool("Mapping", option):
                        set_config("Mapping", option, "True")
                        logger.info("As mode == \"Debug\", [Mapping] {} "
                                    "has been set to True", option)
        elif not get_config_bool("Reports", "reportsEnabled"):
            for option in config_options("Reports"):
                # options names are all lower without _ inside config
                if option in _REPORT_DISABLE_OPTS or option[:5] == "write":
                    if not get_config_bool("Reports", option):
                        set_config("Reports", option, "False")
                        logger.info(
                            "As reportsEnabled == \"False\", [Reports] {} "
                            "has been set to False", option)
        if get_config_bool("Machine", "virtual_board"):
            # TODO handle in the execute methods
            if get_config_bool("Reports", "write_energy_report"):
                set_config("Reports", "write_energy_report", "False")
                logger.info("[Reports]write_energy_report has been set to "
                            "False as using virtual boards")

    def _previous_handler(self) -> None:
        self._error_on_previous("loading_algorithms")
        self._error_on_previous("application_to_machine_graph_algorithms")
        self._error_on_previous("machine_graph_to_machine_algorithms")
        self._error_on_previous("machine_graph_to_virtual_machine_algorithms")
        self._replaced_cfg("Reports",
                           "write_routing_table_reports", "write_uncompressed")
        self._replaced_cfg("Reports",
                           "write_routing_tables_from_machine_reports",
                           "write_compressed, write_compression_comparison,"
                           " and write_compression_summary")
        self._replaced_cfg("Reports",
                           "write_routing_compression_checker_report",
                           "run_compression_checker")

    def _error_on_previous(self, option) -> None:
        try:
            get_config_str_list("Mapping", option)
        except NoOptionError:
            # GOOD!
            return
        raise ConfigurationException(
            f"cfg setting {option} is no longer supported! "
            "See https://spinnakermanchester.github.io/common_pages/"
            "Algorithms.html.")

    def _replaced_cfg(self, section: str, previous: str, new: str):
        if has_config_option(section, previous):
            if get_config_bool(section, previous):
                raise ConfigurationException(
                    f"cfg setting [{section}] {previous} "
                    f"is no longer supported! Use {new} instead")
            else:
                logger.warning(f"cfg setting [{section}] {previous} "
                               f"is no longer supported! Use {new} instead")

    def _reserve_system_vertices(self):
        """
        Reserves the sizes for the system vertices
        """
        if get_config_bool("Reports", "write_energy_report"):
            self._data_writer.add_sample_monitor_vertex(
                sample_chip_power_monitor(), True)
        if (get_config_bool("Machine", "enable_advanced_monitor_support")
                or get_config_bool("Machine", "enable_reinjection")):
            self._data_writer.add_sample_monitor_vertex(
                sample_monitor_vertex(), True)
            self._data_writer.add_sample_monitor_vertex(
                sample_speedup_vertex(), False)

    def _adjust_config(self, runtime: Optional[float]):
        """
        Adjust and checks the configuration based on runtime

        :param runtime:
        :type runtime: int or bool
        :raises ConfigurationException:
        """
        if runtime is None:
            if get_config_bool("Reports", "write_energy_report"):
                set_config("Reports", "write_energy_report", "False")
                logger.info("[Reports]write_energy_report has been set to "
                            "False as runtime is set to forever")

    def _remove_excess_folders(
            self, max_kept: int, starting_directory: str,
            remove_errored_folders: Optional[bool]):
        try:
            files_in_report_folder = os.listdir(starting_directory)

            # while there's more than the valid max, remove the oldest one
            if len(files_in_report_folder) > max_kept:
                # sort files into time frame
                files_in_report_folder.sort(
                    key=lambda temp_file: os.path.getmtime(
                        os.path.join(starting_directory, temp_file)))

                # remove only the number of files required, and only if they
                # have the finished flag file created
                num_files_to_remove = len(files_in_report_folder) - max_kept
                files_removed = 0
                files_not_closed = 0
                for current_oldest_file in files_in_report_folder:
                    finished_flag = os.path.join(os.path.join(
                        starting_directory, current_oldest_file),
                        FINISHED_FILENAME)
                    errored_flag = os.path.join(os.path.join(
                        starting_directory, current_oldest_file),
                        ERRORED_FILENAME)
                    finished_flag_exists = os.path.exists(finished_flag)
                    errored_flag_exists = os.path.exists(errored_flag)
                    if finished_flag_exists and (
                            not errored_flag_exists or remove_errored_folders):
                        shutil.rmtree(os.path.join(
                            starting_directory, current_oldest_file),
                            ignore_errors=True)
                        files_removed += 1
                    else:
                        files_not_closed += 1
                    if files_removed + files_not_closed >= num_files_to_remove:
                        break
                if files_not_closed > max_kept // 4:
                    logger.warning(
                        "{} has {} old reports that have not been closed",
                        starting_directory, files_not_closed)
        except IOError:
            # This might happen if there is an open file, or more than one
            # process in the same folder, but we shouldn't die because of it
            pass

    def _set_up_report_specifics(self) -> None:
        # clear and clean out folders considered not useful any more
        report_dir_path = self._data_writer.get_report_dir_path()
        if os.listdir(report_dir_path):
            self._remove_excess_folders(
                get_config_int("Reports", "max_reports_kept"),
                report_dir_path,
                get_config_bool("Reports", "remove_errored_folders"))

        # store timestamp in latest/time_stamp for provenance reasons
        timestamp_dir_path = self._data_writer.get_timestamp_dir_path()
        time_of_run_file_name = os.path.join(
            timestamp_dir_path, TIMESTAMP_FILENAME)
        _, timestamp = os.path.split(timestamp_dir_path)
        with open(time_of_run_file_name, "w", encoding="utf-8") as f:
            f.writelines(timestamp)
            f.write("\n")
            f.write("Traceback of setup call:\n")
            traceback.print_stack(file=f)

    def __write_marker_file(self, file_name: str):
        app_file_name = os.path.join(
            self._data_writer.get_timestamp_dir_path(), file_name)
        with open(app_file_name, "w", encoding="utf-8") as f:
            # TODO What should this file contain?
            f.writelines("file_name")

    def write_finished_file(self) -> None:
        """
        Write a finished file that allows file removal to only remove
        folders that are finished.
        """
        self.__write_marker_file(FINISHED_FILENAME)

    def write_errored_file(self) -> None:
        """
        Writes an ``errored`` file that allows file removal to only remove
        folders that have errors if requested to do so
        """
        self.__write_marker_file(ERRORED_FILENAME)

1	# Copyright (c) 2017 The University of Manchester
2	#
3	# Licensed under the Apache License, Version 2.0 (the "License");
4	# you may not use this file except in compliance with the License.
5	# You may obtain a copy of the License at
6	#
7	# https://www.apache.org/licenses/LICENSE-2.0
8	#
9	# Unless required by applicable law or agreed to in writing, software
10	# distributed under the License is distributed on an "AS IS" BASIS,
11	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12	# See the License for the specific language governing permissions and
13	# limitations under the License.
14
15	from configparser import NoOptionError	1✔
16	import logging	1✔
17	import os	1✔
18	import shutil	1✔
19	import traceback	1✔
20	from typing import Optional, Type	1✔
21	from spinn_utilities.log import FormatAdapter	1✔
22	from spinn_utilities.config_holder import (	1✔
23	config_options, has_config_option, load_config, get_config_bool,
24	get_config_int, get_config_str, get_config_str_list, set_config)
25	from spinn_front_end_common.interface.interface_functions.\	1✔
26	insert_chip_power_monitors_to_graphs import sample_chip_power_monitor
27	from spinn_front_end_common.interface.interface_functions.\	1✔
28	insert_extra_monitor_vertices_to_graphs import (
29	sample_monitor_vertex, sample_speedup_vertex)
30	from spinn_front_end_common.interface.provenance import LogStoreDB	1✔
31	from spinn_front_end_common.data.fec_data_writer import FecDataWriter	1✔
32	from spinn_front_end_common.utilities.exceptions import ConfigurationException	1✔
33
34	logger = FormatAdapter(logging.getLogger(__name__))	1✔
35
36	APP_DIRNAME = 'application_generated_data_files'	1✔
37	FINISHED_FILENAME = "finished"	1✔
38	ERRORED_FILENAME = "errored"	1✔
39	REPORTS_DIRNAME = "reports"	1✔
40	TIMESTAMP_FILENAME = "time_stamp"	1✔
41	WARNING_LOGS_FILENAME = "warning_logs.txt"	1✔
42
43	# options names are all lower without _ inside config
44	_DEBUG_ENABLE_OPTS = frozenset([	1✔
45	"reportsenabled",
46	"cleariobufduringrun", "extractiobuf"])
47	_DEBUG_MAPPING_OPTS = frozenset([	1✔
48	"routertablecompressasfaraspossible", "runcompressionchecker"])
49	_REPORT_DISABLE_OPTS = frozenset([	1✔
50	"cleariobufduringrun", "extractiobuf"])
51
52
53	class ConfigHandler(object):	1✔
54	"""
55	Superclass of AbstractSpinnakerBase that handles function only
56	dependent of the configuration and the order its methods are called.
57	"""
58
59	__slots__ = (	1✔
60	# The writer and therefore view of the global data
61	"_data_writer", )
62
63	def __init__(self, data_writer_cls: Optional[Type[FecDataWriter]] = None):	1✔
64	"""
65	:param FecDataWriter data_writer:
66	The Global data writer object
67	"""
68	load_config()	1✔
69
70	if data_writer_cls:	1!
71	self._data_writer = data_writer_cls.setup()	×
72	else:
73	self._data_writer = FecDataWriter.setup()	1✔
74	logger.set_log_store(LogStoreDB())	1✔
75
76	# set up machine targeted data
77	self._debug_configs()	1✔
78	self._previous_handler()	1✔
79	self._reserve_system_vertices()	1✔
80
81	def _debug_configs(self) -> None:	1✔
82	"""
83	Adjusts and checks the configuration based on mode and
84	`reports_enabled`.
85
86	:raises ConfigurationException:
87	"""
88	if get_config_str("Mode", "mode") == "Debug":	1!
89	for option in config_options("Reports"):	×
90	# options names are all lower without _ inside config
91	if option in _DEBUG_ENABLE_OPTS or option[:5] == "write":	×
92	if not get_config_bool("Reports", option):	×
93	set_config("Reports", option, "True")	×
94	logger.info("As mode == \"Debug\", [Reports] {} "	×
95	"has been set to True", option)
NEW 96	for option in config_options("Mapping"):	×
97	# options names are all lower without _ inside config
NEW 98	if option in _DEBUG_MAPPING_OPTS:	×
NEW 99	if not get_config_bool("Mapping", option):	×
NEW 100	set_config("Mapping", option, "True")	×
NEW 101	logger.info("As mode == \"Debug\", [Mapping] {} "	×
102	"has been set to True", option)
103	elif not get_config_bool("Reports", "reportsEnabled"):	1!
104	for option in config_options("Reports"):	×
105	# options names are all lower without _ inside config
106	if option in _REPORT_DISABLE_OPTS or option[:5] == "write":	×
107	if not get_config_bool("Reports", option):	×
108	set_config("Reports", option, "False")	×
109	logger.info(	×
110	"As reportsEnabled == \"False\", [Reports] {} "
111	"has been set to False", option)
112	if get_config_bool("Machine", "virtual_board"):	1!
113	# TODO handle in the execute methods
114	if get_config_bool("Reports", "write_energy_report"):	×
115	set_config("Reports", "write_energy_report", "False")	×
116	logger.info("[Reports]write_energy_report has been set to "	×
117	"False as using virtual boards")
118
119	def _previous_handler(self) -> None:	1✔
120	self._error_on_previous("loading_algorithms")	1✔
121	self._error_on_previous("application_to_machine_graph_algorithms")	1✔
122	self._error_on_previous("machine_graph_to_machine_algorithms")	1✔
123	self._error_on_previous("machine_graph_to_virtual_machine_algorithms")	1✔
124	self._replaced_cfg("Reports",	1✔
125	"write_routing_table_reports", "write_uncompressed")
126	self._replaced_cfg("Reports",	1✔
127	"write_routing_tables_from_machine_reports",
128	"write_compressed, write_compression_comparison,"
129	" and write_compression_summary")
130	self._replaced_cfg("Reports",	1✔
131	"write_routing_compression_checker_report",
132	"run_compression_checker")
133
134	def _error_on_previous(self, option) -> None:	1✔
135	try:	1✔
136	get_config_str_list("Mapping", option)	1✔
137	except NoOptionError:	1✔
138	# GOOD!
139	return	1✔
140	raise ConfigurationException(	×
141	f"cfg setting {option} is no longer supported! "
142	"See https://spinnakermanchester.github.io/common_pages/"
143	"Algorithms.html.")
144
145	def _replaced_cfg(self, section: str, previous: str, new: str):	1✔
146	if has_config_option(section, previous):	1!
NEW 147	if get_config_bool(section, previous):	×
NEW 148	raise ConfigurationException(	×
149	f"cfg setting [{section}] {previous} "
150	f"is no longer supported! Use {new} instead")
151	else:
NEW 152	logger.warning(f"cfg setting [{section}] {previous} "	×
153	f"is no longer supported! Use {new} instead")
154
155	def _reserve_system_vertices(self):	1✔
156	"""
157	Reserves the sizes for the system vertices
158	"""
159	if get_config_bool("Reports", "write_energy_report"):	1!
160	self._data_writer.add_sample_monitor_vertex(	×
161	sample_chip_power_monitor(), True)
162	if (get_config_bool("Machine", "enable_advanced_monitor_support")	1!
163	or get_config_bool("Machine", "enable_reinjection")):
164	self._data_writer.add_sample_monitor_vertex(	1✔
165	sample_monitor_vertex(), True)
166	self._data_writer.add_sample_monitor_vertex(	1✔
167	sample_speedup_vertex(), False)
168
169	def _adjust_config(self, runtime: Optional[float]):	1✔
170	"""
171	Adjust and checks the configuration based on runtime
172
173	:param runtime:
174	:type runtime: int or bool
175	:raises ConfigurationException:
176	"""
177	if runtime is None:	×
178	if get_config_bool("Reports", "write_energy_report"):	×
179	set_config("Reports", "write_energy_report", "False")	×
180	logger.info("[Reports]write_energy_report has been set to "	×
181	"False as runtime is set to forever")
182
183	def _remove_excess_folders(	1✔
184	self, max_kept: int, starting_directory: str,
185	remove_errored_folders: Optional[bool]):
186	try:	1✔
187	files_in_report_folder = os.listdir(starting_directory)	1✔
188
189	# while there's more than the valid max, remove the oldest one
190	if len(files_in_report_folder) > max_kept:	1!
191	# sort files into time frame
192	files_in_report_folder.sort(	×
193	key=lambda temp_file: os.path.getmtime(
194	os.path.join(starting_directory, temp_file)))
195
196	# remove only the number of files required, and only if they
197	# have the finished flag file created
198	num_files_to_remove = len(files_in_report_folder) - max_kept	×
199	files_removed = 0	×
200	files_not_closed = 0	×
201	for current_oldest_file in files_in_report_folder:	×
202	finished_flag = os.path.join(os.path.join(	×
203	starting_directory, current_oldest_file),
204	FINISHED_FILENAME)
205	errored_flag = os.path.join(os.path.join(	×
206	starting_directory, current_oldest_file),
207	ERRORED_FILENAME)
208	finished_flag_exists = os.path.exists(finished_flag)	×
209	errored_flag_exists = os.path.exists(errored_flag)	×
210	if finished_flag_exists and (	×
211	not errored_flag_exists or remove_errored_folders):
212	shutil.rmtree(os.path.join(	×
213	starting_directory, current_oldest_file),
214	ignore_errors=True)
215	files_removed += 1	×
216	else:
217	files_not_closed += 1	×
218	if files_removed + files_not_closed >= num_files_to_remove:	×
219	break	×
220	if files_not_closed > max_kept // 4:	×
221	logger.warning(	×
222	"{} has {} old reports that have not been closed",
223	starting_directory, files_not_closed)
224	except IOError:	×
225	# This might happen if there is an open file, or more than one
226	# process in the same folder, but we shouldn't die because of it
227	pass	×
228
229	def _set_up_report_specifics(self) -> None:	1✔
230	# clear and clean out folders considered not useful any more
231	report_dir_path = self._data_writer.get_report_dir_path()	1✔
232	if os.listdir(report_dir_path):	1!
233	self._remove_excess_folders(	1✔
234	get_config_int("Reports", "max_reports_kept"),
235	report_dir_path,
236	get_config_bool("Reports", "remove_errored_folders"))
237
238	# store timestamp in latest/time_stamp for provenance reasons
239	timestamp_dir_path = self._data_writer.get_timestamp_dir_path()	1✔
240	time_of_run_file_name = os.path.join(	1✔
241	timestamp_dir_path, TIMESTAMP_FILENAME)
242	_, timestamp = os.path.split(timestamp_dir_path)	1✔
243	with open(time_of_run_file_name, "w", encoding="utf-8") as f:	1✔
244	f.writelines(timestamp)	1✔
245	f.write("\n")	1✔
246	f.write("Traceback of setup call:\n")	1✔
247	traceback.print_stack(file=f)	1✔
248
249	def __write_marker_file(self, file_name: str):	1✔
250	app_file_name = os.path.join(	1✔
251	self._data_writer.get_timestamp_dir_path(), file_name)
252	with open(app_file_name, "w", encoding="utf-8") as f:	1✔
253	# TODO What should this file contain?
254	f.writelines("file_name")	1✔
255
256	def write_finished_file(self) -> None:	1✔
257	"""
258	Write a finished file that allows file removal to only remove
259	folders that are finished.
260	"""
261	self.__write_marker_file(FINISHED_FILENAME)	1✔
262
263	def write_errored_file(self) -> None:	1✔
264	"""
265	Writes an ``errored`` file that allows file removal to only remove
266	folders that have errors if requested to do so
267	"""
268	self.__write_marker_file(ERRORED_FILENAME)	×

SpiNNakerManchester / SpiNNFrontEndCommon / 8906184429

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous