1632

pending completion

Build # 1632

Build Type

Pull #527

travis-ci

Committed by

web-flow

Commit Message

<a href="https://github.com/fitbenchmarking/fitbenchmarking/commit/<a class=hub.com/fitbenchmarking/fitbenchmarking/commit/<a class="double-link" href="https://git"><a class=hub.com/fitbenchmarking/fitbenchmarking/commit/<a class="double-link" href="https://git"><a class=hub.com/fitbenchmarking/fitbenchmarking/commit/<a class="double-link" href="https://git"><a class=hub.com/fitbenchmarking/fitbenchmarking/commit/<a class="double-link" href="https://git"><a class=hub.com/fitbenchmarking/fitbenchmarking/commit/5f6483d9cb63096a9283dd41fac0056776f39609">5f6483d9c">&lt;a href=&quot;https://github.com/fitbenchmarking/fitbenchmarking/commit/</a><a class="double-link" href="https://github.com/fitbenchmarking/fitbenchmarking/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/fitbenchmarking/fitbenchmarking/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/fitbenchmarking/fitbenchmarking/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/fitbenchmarking/fitbenchmarking/commit/5f6483d9cb63096a9283dd41fac0056776f39609">5f6483d9c</a><a href="https://github.com/fitbenchmarking/fitbenchmarking/commit/5f6483d9cb63096a9283dd41fac0056776f39609">&lt;a href=&quot;https://github.com/fitbenchmarking/fitbenchmarking/commit/5f6483d9cb63096a9283dd41fac0056776f39609&quot;&gt;&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/fitbenchmarking/fitbenchmarking/commit/&amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/fitbenchmarking/fitbenchmarking/commit/&amp;amp;lt;a class=&amp;amp;quot;double-link&amp;amp;quot; href=&amp;amp;quot;https://git&amp;quot;&amp;gt;&amp;amp;lt;a class=&amp;lt;/a&amp;gt;hub.com/fitbenchmarking/fitbenchmarking/commit/5f6483d9cb63096a9283dd41fac0056776f39609&amp;quot;&amp;gt;5f6483d9c&amp;lt;/a&amp;gt;&amp;lt;a href=&amp;quot;https://github.com/fitbenchmarking/fitbenchmarking/commit/5f6483d9cb63096a9283dd41fac0056776f39609&amp;quot;&amp;gt;&amp;amp;lt;a href=&amp;amp;quot;https://github.com/fitbenchmarking/fitbenchmarking/commit/5f6483d9cb63096a9283dd41fac00567&amp;lt;/a&amp;gt;76f39609&amp;quot;&amp;gt;&amp;</a>amp;quot;&amp;gt;Merge &amp;lt;/a&amp;gt;&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://github.com/fitbenchmarking/fitbenchmarking/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot;... (continued)

Pull Request Pull Request #527: Algorithm type option

Run Details

65 of 65 new or added lines in 15 files covered. (100.0%)

1902 of 2255 relevant lines covered (84.35%)

0.84 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

12.5

/fitbenchmarking/core/fitbenchmark_one_problem.py

"""
Fit benchmark one problem functions.
"""

from __future__ import absolute_import, division, print_function

import timeit
import warnings

import numpy as np

from fitbenchmarking.controllers.controller_factory import ControllerFactory
from fitbenchmarking.utils import fitbm_result, output_grabber
from fitbenchmarking.utils.exceptions import UnknownMinimizerError
from fitbenchmarking.utils.log import get_logger

LOGGER = get_logger()


def fitbm_one_prob(problem, options):
    """
    Sets up the controller for a particular problem and fits the models
    provided in the problem object.

    :param problem: a problem object containing information used in fitting
    :type problem: FittingProblem
    :param options: all the information specified by the user
    :type options: fitbenchmarking.utils.options.Options

    :return: list of all results, failed problem names and dictionary of
             unselected minimizers based on algorithm_type
    :rtype: tuple(list of fibenchmarking.utils.fitbm_result.FittingResult,
                  list of failed problem names,
                  dictionary of minimizers)
    """
    grabbed_output = output_grabber.OutputGrabber(options)
    results = []

    software = options.software
    if not isinstance(software, list):
        software = [software]

    name = problem.name
    num_start_vals = len(problem.starting_values)
    problem_fails = []
    unselected_minimzers = {}
    for i in range(num_start_vals):
        LOGGER.info("    Starting value: {0}/{1}".format(i + 1,
                                                         num_start_vals))

        if num_start_vals > 1:
            problem.name = name + ', Start {}'.format(i + 1)
        software_results = []
        for s in software:
            LOGGER.info("        Software: %s", s.upper())
            try:
                minimizers = options.minimizers[s]
            except KeyError:
                raise UnknownMinimizerError(
                    'No minimizer given for software: {}'.format(s))
            with grabbed_output:
                controller_cls = ControllerFactory.create_controller(
                    software=s)
                controller = controller_cls(problem=problem)

            controller.parameter_set = i
            problem_result, minimizer_failed = benchmark(controller=controller,
                                                         minimizers=minimizers,
                                                         options=options)
            unselected_minimzers[s] = minimizer_failed
            software_results.extend(problem_result)

        # Checks to see if all of the minimizers raise and exception and
        # records the problems name for that case
        software_check = [np.isinf(v.chi_sq) for v in software_results]
        if all(software_check):
            software_results = []
            problem_fails.append(problem.name)
        results.extend(software_results)
    # Reset problem.name
    problem.name = name
    return results, problem_fails, unselected_minimzers


def benchmark(controller, minimizers, options):
    """
    Fit benchmark one problem, with one function definition and all
    the selected minimizers, using the chosen fitting software.

    :param controller: The software controller for the fitting
    :type controller: Object derived from BaseSoftwareController
    :param minimizers: array of minimizers used in fitting
    :type minimizers: list
    :param options: all the information specified by the user
    :type options: fitbenchmarking.utils.options.Options

    :return: list of all results and dictionary of unselected minimizers
             based on algorithm_type
    :rtype: tuple(list of fibenchmarking.utils.fitbm_result.FittingResult,
                  dictionary of minimizers)
    """
    grabbed_output = output_grabber.OutputGrabber(options)
    problem = controller.problem
    jac = controller.problem.jac

    results_problem = []
    minimizer_failed = []

    num_runs = options.num_runs
    algorithm_type = options.algorithm_type

    for minimizer in minimizers:
        minimizer_check = True
        LOGGER.info("            Minimizer: %s", minimizer)

        controller.minimizer = minimizer
        try:
            with grabbed_output:
                controller.validate_minimizer(minimizer, algorithm_type)
                # Calls timeit repeat with repeat = num_runs and number = 1
                runtime_list = \
                    timeit.Timer(setup=controller.prepare,
                                 stmt=controller.fit).repeat(num_runs, 1)
                runtime = sum(runtime_list) / num_runs
                controller.cleanup()
        # Catching all exceptions as this means runtime cannot be calculated
        # pylint: disable=broad-except
        except Exception as excp:
            if isinstance(excp, UnknownMinimizerError):
                minimizer_failed.append(minimizer)
                minimizer_check = False
            LOGGER.warn(str(excp))

            runtime = np.inf
            controller.flag = 3
            controller.final_params = None if not problem.multifit \
                else [None] * len(controller.data_x)
            chi_sq = np.inf if not problem.multifit \
                else [np.inf] * len(controller.data_x)

        controller.check_attributes()

        if controller.flag <= 2:
            ratio = np.max(runtime_list) / np.min(runtime_list)
            tol = 4
            if ratio > tol:
                warnings.warn('The ratio of the max time to the min is {0}'
                              ' which is  larger than the tolerance of {1},'
                              ' which may indicate that caching has occurred'
                              ' in the timing results'.format(ratio, tol))
            chi_sq = controller.eval_chisq(params=controller.final_params,
                                           x=controller.data_x,
                                           y=controller.data_y,
                                           e=controller.data_e)

        result_args = {'options': options,
                       'problem': problem,
                       'jac': jac,
                       'chi_sq': chi_sq,
                       'runtime': runtime,
                       'minimizer': minimizer,
                       'initial_params': controller.initial_params,
                       'params': controller.final_params,
                       'error_flag': controller.flag,
                       'name': problem.name}
        if minimizer_check:
            if problem.multifit:
                # Multi fit (will raise TypeError if these are not iterable)
                for i in range(len(chi_sq)):
                    result_args.update({'dataset_id': i,
                                        'name': '{}, Dataset {}'.format(
                                            problem.name, (i + 1))})
                    individual_result = \
                        fitbm_result.FittingResult(**result_args)
                    results_problem.append(individual_result)
            else:
                # Normal fitting
                individual_result = fitbm_result.FittingResult(**result_args)
                results_problem.append(individual_result)
    return results_problem, minimizer_failed

1	"""
2	Fit benchmark one problem functions.
3	"""
4
5	from __future__ import absolute_import, division, print_function	1✔
6
7	import timeit	1✔
8	import warnings	1✔
9
10	import numpy as np	1✔
11
12	from fitbenchmarking.controllers.controller_factory import ControllerFactory	1✔
13	from fitbenchmarking.utils import fitbm_result, output_grabber	1✔
14	from fitbenchmarking.utils.exceptions import UnknownMinimizerError	1✔
15	from fitbenchmarking.utils.log import get_logger	1✔
16
17	LOGGER = get_logger()	1✔
18
19
20	def fitbm_one_prob(problem, options):	1✔
21	"""
22	Sets up the controller for a particular problem and fits the models
23	provided in the problem object.
24
25	:param problem: a problem object containing information used in fitting
26	:type problem: FittingProblem
27	:param options: all the information specified by the user
28	:type options: fitbenchmarking.utils.options.Options
29
30	:return: list of all results, failed problem names and dictionary of
31	unselected minimizers based on algorithm_type
32	:rtype: tuple(list of fibenchmarking.utils.fitbm_result.FittingResult,
33	list of failed problem names,
34	dictionary of minimizers)
35	"""
36	grabbed_output = output_grabber.OutputGrabber(options)	×
37	results = []	×
38
39	software = options.software	×
40	if not isinstance(software, list):	×
41	software = [software]	×
42
43	name = problem.name	×
44	num_start_vals = len(problem.starting_values)	×
45	problem_fails = []	×
46	unselected_minimzers = {}	×
47	for i in range(num_start_vals):	×
48	LOGGER.info(" Starting value: {0}/{1}".format(i + 1,	×
49	num_start_vals))
50
51	if num_start_vals > 1:	×
52	problem.name = name + ', Start {}'.format(i + 1)	×
53	software_results = []	×
54	for s in software:	×
55	LOGGER.info(" Software: %s", s.upper())	×
56	try:	×
57	minimizers = options.minimizers[s]	×
58	except KeyError:	×
59	raise UnknownMinimizerError(	×
60	'No minimizer given for software: {}'.format(s))
61	with grabbed_output:	×
62	controller_cls = ControllerFactory.create_controller(	×
63	software=s)
64	controller = controller_cls(problem=problem)	×
65
66	controller.parameter_set = i	×
67	problem_result, minimizer_failed = benchmark(controller=controller,	×
68	minimizers=minimizers,
69	options=options)
70	unselected_minimzers[s] = minimizer_failed	×
71	software_results.extend(problem_result)	×
72
73	# Checks to see if all of the minimizers raise and exception and
74	# records the problems name for that case
75	software_check = [np.isinf(v.chi_sq) for v in software_results]	×
76	if all(software_check):	×
77	software_results = []	×
78	problem_fails.append(problem.name)	×
79	results.extend(software_results)	×
80	# Reset problem.name
81	problem.name = name	×
82	return results, problem_fails, unselected_minimzers	×
83
84
85	def benchmark(controller, minimizers, options):	1✔
86	"""
87	Fit benchmark one problem, with one function definition and all
88	the selected minimizers, using the chosen fitting software.
89
90	:param controller: The software controller for the fitting
91	:type controller: Object derived from BaseSoftwareController
92	:param minimizers: array of minimizers used in fitting
93	:type minimizers: list
94	:param options: all the information specified by the user
95	:type options: fitbenchmarking.utils.options.Options
96
97	:return: list of all results and dictionary of unselected minimizers
98	based on algorithm_type
99	:rtype: tuple(list of fibenchmarking.utils.fitbm_result.FittingResult,
100	dictionary of minimizers)
101	"""
102	grabbed_output = output_grabber.OutputGrabber(options)	×
103	problem = controller.problem	×
104	jac = controller.problem.jac	×
105
106	results_problem = []	×
107	minimizer_failed = []	×
108
109	num_runs = options.num_runs	×
110	algorithm_type = options.algorithm_type	×
111
112	for minimizer in minimizers:	×
113	minimizer_check = True	×
114	LOGGER.info(" Minimizer: %s", minimizer)	×
115
116	controller.minimizer = minimizer	×
117	try:	×
118	with grabbed_output:	×
119	controller.validate_minimizer(minimizer, algorithm_type)	×
120	# Calls timeit repeat with repeat = num_runs and number = 1
121	runtime_list = \	×
122	timeit.Timer(setup=controller.prepare,
123	stmt=controller.fit).repeat(num_runs, 1)
124	runtime = sum(runtime_list) / num_runs	×
125	controller.cleanup()	×
126	# Catching all exceptions as this means runtime cannot be calculated
127	# pylint: disable=broad-except
128	except Exception as excp:	×
129	if isinstance(excp, UnknownMinimizerError):	×
130	minimizer_failed.append(minimizer)	×
131	minimizer_check = False	×
132	LOGGER.warn(str(excp))	×
133
134	runtime = np.inf	×
135	controller.flag = 3	×
136	controller.final_params = None if not problem.multifit \	×
137	else [None] * len(controller.data_x)
138	chi_sq = np.inf if not problem.multifit \	×
139	else [np.inf] * len(controller.data_x)
140
141	controller.check_attributes()	×
142
143	if controller.flag <= 2:	×
144	ratio = np.max(runtime_list) / np.min(runtime_list)	×
145	tol = 4	×
146	if ratio > tol:	×
147	warnings.warn('The ratio of the max time to the min is {0}'	×
148	' which is larger than the tolerance of {1},'
149	' which may indicate that caching has occurred'
150	' in the timing results'.format(ratio, tol))
151	chi_sq = controller.eval_chisq(params=controller.final_params,	×
152	x=controller.data_x,
153	y=controller.data_y,
154	e=controller.data_e)
155
156	result_args = {'options': options,	×
157	'problem': problem,
158	'jac': jac,
159	'chi_sq': chi_sq,
160	'runtime': runtime,
161	'minimizer': minimizer,
162	'initial_params': controller.initial_params,
163	'params': controller.final_params,
164	'error_flag': controller.flag,
165	'name': problem.name}
166	if minimizer_check:	×
167	if problem.multifit:	×
168	# Multi fit (will raise TypeError if these are not iterable)
169	for i in range(len(chi_sq)):	×
170	result_args.update({'dataset_id': i,	×
171	'name': '{}, Dataset {}'.format(
172	problem.name, (i + 1))})
173	individual_result = \	×
174	fitbm_result.FittingResult(**result_args)
175	results_problem.append(individual_result)	×
176	else:
177	# Normal fitting
178	individual_result = fitbm_result.FittingResult(**result_args)	×
179	results_problem.append(individual_result)	×
180	return results_problem, minimizer_failed	×

fitbenchmarking / fitbenchmarking / 1632

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous