#6556

pending completion

Build # #6556

Build Type

push

coveralls-python

Committed by

web-flow

Commit Message

Bump future from 0.18.2 to 0.18.3 in /requirements (#3792)

Bumps [future](https://github.com/PythonCharmers/python-future) from 0.18.2 to 0.18.3.
- [Release notes](https://github.com/PythonCharmers/python-future/releases)
- [Changelog](https://github.com/PythonCharmers/python-future/blob/master/docs/changelog.rst)
- [Commits](https://github.com/PythonCharmers/python-future/compare/v0.18.2...v0.18.3)

---
updated-dependencies:
- dependency-name: future
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

Run Details

15695 of 22613 relevant lines covered (69.41%)

0.69 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

17.31

/seed/analysis_pipelines/better/helpers.py

import logging
import pathlib
from collections import namedtuple

from django.core.files.base import File as BaseFile

from seed.analysis_pipelines.better.buildingsync import (
    _parse_analysis_property_view_id
)
from seed.analysis_pipelines.pipeline import (
    AnalysisPipelineException,
    StopAnalysisTaskChain
)
from seed.models import (
    AnalysisMessage,
    AnalysisOutputFile,
    AnalysisPropertyView
)

logger = logging.getLogger(__name__)


class BuildingAnalysis:
    """Used to track AnalysisPropertyViews and BETTER Building and Analysis IDs"""

    def __init__(self, analysis_property_view_id, better_building_id, better_analysis_id):
        self.analysis_property_view_id = analysis_property_view_id
        self.better_building_id = better_building_id
        self.better_analysis_id = better_analysis_id


class BETTERPipelineContext:
    """Datastructure to avoid multiple pass-through variables"""

    def __init__(self, analysis, progress_data, better_client):
        """
        :param analysis: Analysis
        :param progress_data, ProgressData
        :param better_client: BETTERClient
        """
        self.analysis = analysis
        self.progress_data = progress_data
        self.client = better_client


# Used to define json paths to parse from analysis results, which are linked to an extra data column
# column_name: Column.column_name (also the extra_data key)
# column_display_name: Column.display_name
# unit_multiplier: applied to result before being saved to extra_data
# json_path: naive json path -- dot separated keys into the parsed analysis results dict
ExtraDataColumnPath = namedtuple('ExtraDataColumnPath', ['column_name', 'column_display_name', 'unit_multiplier', 'json_path'])


def _check_errors(errors, what_failed_desc, context, analysis_property_view_id=None, fail_on_error=False):
    """Creates error messages for the analysis if any are found.

    :param analysis: Analysis
    :param progress_data: ProgressData
    :param errors: list[str], list of debug error messages
    :param what_failed_desc: str, description of what was happening when failure occurred
        e.g., what were you trying to do
    :param analysis_property_view_id: int, optional, if provided, the error messages will be linked
        to this property view
    :param fail_on_error: bool, optional, if True and errors were found, this fails the pipeline
        and stops the celery task chain
    """
    if not errors:
        return

    for error in errors:
        AnalysisMessage.log_and_create(
            logger=logger,
            type_=AnalysisMessage.ERROR,
            analysis_id=context.analysis.id,
            analysis_property_view_id=analysis_property_view_id,
            user_message='Unexpected error from BETTER service. Please confirm your Organization\'s API token is correct and try again or contact the SEED administrators.',
            debug_message=f'{what_failed_desc}: {error}',
        )

    if fail_on_error:
        # avoid circular import
        from seed.analysis_pipelines.better.pipeline import BETTERPipeline

        pipeline = BETTERPipeline(context.analysis.id)
        pipeline.fail(what_failed_desc, logger, progress_data_key=context.progress_data.key)
        # stop the task chain
        raise StopAnalysisTaskChain(what_failed_desc)


def _run_better_portfolio_analysis(better_portfolio_id, better_building_analyses, analysis_config, context):
    """Create and run an analysis for a BETTER portfolio. Updates all BuildingAnalysis
    objects in better_building_analyses to store their individual building analysis IDs.

    :param better_portfolio_id: int
    :param better_building_analyses: list[BuildingAnalysis]
    :param analysis_config: dict, config for the analysis API
    :param analysis: Analysis
    :param progress_data: ProgressData
    :returns: int, better_analysis_id, ID of the analysis which was created and run
    """
    better_analysis_id, errors = context.client.create_portfolio_analysis(
        better_portfolio_id,
        analysis_config,
    )
    _check_errors(
        errors,
        'Failed to create BETTER portfolio analysis',
        context,
        fail_on_error=True
    )

    errors = context.client.run_portfolio_analysis(
        better_portfolio_id,
        better_analysis_id
    )
    if errors:
        _check_errors(
            errors,
            'Failed to generate BETTER portfolio analysis',
            context,
            fail_on_error=True,
        )

    # find and store all individual building analysis IDs for the portfolio
    # so we can fetch and save those individual analysis results later
    better_portfolio_analysis, errors = context.client.get_portfolio_analysis(better_portfolio_id, better_analysis_id)
    _check_errors(
        errors,
        'Failed to get BETTER portfolio analysis as JSON',
        context,
        fail_on_error=True
    )

    api_building_analytics = better_portfolio_analysis.get('building_analytics_set', [])
    for api_building_analysis in api_building_analytics:
        # find the corresponding BuildingAnalysis
        building_analysis = next(
            (ba for ba in better_building_analyses if ba.better_building_id == api_building_analysis['building_id']),
            None
        )
        building_analysis.better_analysis_id = api_building_analysis['id']

    return better_analysis_id


def _store_better_portfolio_analysis_results(better_analysis_id, better_building_analyses, context):
    """Stores results for portfolio analysis. Analysis should be completed before calling.

    :param better_analysis_id: int
    :param better_building_analyses: list[BuildingAnalysis]
    :param analysis: Analysis
    :param progress_data: ProgressData
    """
    results_dir, errors = context.client.get_portfolio_analysis_standalone_html(better_analysis_id)
    _check_errors(
        errors,
        'Failed to get BETTER portfolio analysis standalone HTML',
        context,
        fail_on_error=True
    )
    for result_file_path in pathlib.Path(results_dir.name).iterdir():
        with open(result_file_path, 'r') as f:
            if result_file_path.suffix != '.html':
                raise AnalysisPipelineException(
                    f'Received unhandled file type from BETTER: {result_file_path.name}'
                )

            content_type = AnalysisOutputFile.HTML
            file_ = BaseFile(f)
            analysis_output_file = AnalysisOutputFile(
                content_type=content_type,
            )
            padded_id = f'{context.analysis.id:06d}'
            analysis_output_file.file.save(f'better_portfolio_output_{padded_id}_{result_file_path.name}', file_)
            analysis_output_file.clean()
            analysis_output_file.save()
            # Since this is a portfolio analysis, add the result to all properties
            analysis_output_file.analysis_property_views.set([b.analysis_property_view_id for b in better_building_analyses])


def _run_better_building_analyses(better_building_analyses, analysis_config, context):
    """Runs building analysis for each building. Updates the BuildingAnalysis objects
    in better_building_analyses with the IDs of BETTER analyses created.

    :param better_building_analyses: list[BuildingAnalysis]
    :param analysis_config: dict, dictionary of required BETTER API body
    :param analysis: Analysis
    :param progress_data: ProgressData
    """
    for building_analysis in better_building_analyses:
        better_building_id = building_analysis.better_building_id
        analysis_property_view_id = building_analysis.analysis_property_view_id

        better_analysis_id, errors = context.client.create_and_run_building_analysis(
            better_building_id,
            analysis_config
        )
        if errors:
            _check_errors(
                errors,
                'Failed to run BETTER building analysis',
                context,
                analysis_property_view_id=analysis_property_view_id,
                fail_on_error=False,
            )
            # continue to next building
            continue

        # save the analysis ID so we can fetch and store the analysis results later
        building_analysis.better_analysis_id = better_analysis_id


def _store_better_building_analysis_results(better_building_analyses, context):
    """Stores results for building analysis. Analysis should be completed before calling.

    Specifically, it stores each building's standalone HTML file and links it to
    the analysis property view.
    It also stores each building's JSON analysis results in the analysis property view.

    :param better_building_analyses: list[BuildingAnalysis]
    :param analysis: Analysis
    :param progress_data: ProgressData
    """
    for building_analysis in better_building_analyses:
        better_building_id = building_analysis.better_building_id
        better_analysis_id = building_analysis.better_analysis_id
        analysis_property_view_id = building_analysis.analysis_property_view_id

        #
        # Store the standalone HTML
        #
        results_dir, errors = context.client.get_building_analysis_standalone_html(better_analysis_id)
        if errors:
            _check_errors(
                errors,
                'Failed to get BETTER building analysis standalone HTML',
                context,
                analysis_property_view_id=analysis_property_view_id,
                fail_on_error=False,
            )
            # continue to next building analysis
            continue

        for result_file_path in pathlib.Path(results_dir.name).iterdir():
            with open(result_file_path, 'r') as f:
                if result_file_path.suffix == '.html':
                    content_type = AnalysisOutputFile.HTML
                    file_ = BaseFile(f)
                else:
                    raise AnalysisPipelineException(
                        f'Received unhandled file type from better: {result_file_path.name}')

                analysis_output_file = AnalysisOutputFile(
                    content_type=content_type,
                )
                padded_id = f'{analysis_property_view_id:06d}'
                analysis_output_file.file.save(f'better_output_{padded_id}_{result_file_path.name}', file_)
                analysis_output_file.clean()
                analysis_output_file.save()
                analysis_output_file.analysis_property_views.set([analysis_property_view_id])

        #
        # Store the JSON results into the AnalysisPropertyView
        #
        results_dict, errors = context.client.get_building_analysis(better_building_id, better_analysis_id)
        if errors:
            _check_errors(
                errors,
                'Failed to get BETTER building analysis results',
                context,
                analysis_property_view_id=analysis_property_view_id,
                fail_on_error=False,
            )
            # continue to next building analysis
            continue

        analysis_property_view = AnalysisPropertyView.objects.get(id=analysis_property_view_id)
        analysis_property_view.parsed_results = results_dict
        analysis_property_view.save()


def _create_better_buildings(better_portfolio_id, context):
    """Create a BETTER building

    :param analysis: Analysis
    :param better_portfolio_id: int | None
    :return: list[BuildingAnalysis]
    """
    better_building_analyses = []
    for input_file in context.analysis.input_files.all():
        analysis_property_view_id = _parse_analysis_property_view_id(input_file.file.path)
        better_building_id, errors = context.client.create_building(input_file.file.path, better_portfolio_id)
        if errors:
            _check_errors(
                errors,
                f'Failed to create building for analysis property view {analysis_property_view_id}',
                context,
                analysis_property_view_id,
                fail_on_error=False
            )
            # go to next building
            continue

        better_building_analyses.append(
            BuildingAnalysis(
                analysis_property_view_id,
                better_building_id,
                None
            )
        )
        logger.info(f'Created BETTER building ({better_building_id}) for AnalysisPropertyView ({analysis_property_view_id})')

    return better_building_analyses

1	import logging	1✔
2	import pathlib	1✔
3	from collections import namedtuple	1✔
4
5	from django.core.files.base import File as BaseFile
6
7	from seed.analysis_pipelines.better.buildingsync import (	1✔
8	_parse_analysis_property_view_id
9	)
10	from seed.analysis_pipelines.pipeline import (	1✔
11	AnalysisPipelineException,
12	StopAnalysisTaskChain
13	)
14	from seed.models import (	1✔
15	AnalysisMessage,
16	AnalysisOutputFile,
17	AnalysisPropertyView
18	)
19
20	logger = logging.getLogger(__name__)	1✔
21
22
23	class BuildingAnalysis:	1✔
24	"""Used to track AnalysisPropertyViews and BETTER Building and Analysis IDs"""
25
26	def __init__(self, analysis_property_view_id, better_building_id, better_analysis_id):	1✔
27	self.analysis_property_view_id = analysis_property_view_id	×
28	self.better_building_id = better_building_id	×
29	self.better_analysis_id = better_analysis_id	×
30
31
32	class BETTERPipelineContext:	1✔
33	"""Datastructure to avoid multiple pass-through variables"""
34
35	def __init__(self, analysis, progress_data, better_client):	1✔
36	"""
37	:param analysis: Analysis
38	:param progress_data, ProgressData
39	:param better_client: BETTERClient
40	"""
41	self.analysis = analysis	×
42	self.progress_data = progress_data	×
43	self.client = better_client	×
44
45
46	# Used to define json paths to parse from analysis results, which are linked to an extra data column
47	# column_name: Column.column_name (also the extra_data key)
48	# column_display_name: Column.display_name
49	# unit_multiplier: applied to result before being saved to extra_data
50	# json_path: naive json path -- dot separated keys into the parsed analysis results dict
51	ExtraDataColumnPath = namedtuple('ExtraDataColumnPath', ['column_name', 'column_display_name', 'unit_multiplier', 'json_path'])	1✔
52
53
54	def _check_errors(errors, what_failed_desc, context, analysis_property_view_id=None, fail_on_error=False):	1✔
55	"""Creates error messages for the analysis if any are found.
56
57	:param analysis: Analysis
58	:param progress_data: ProgressData
59	:param errors: list[str], list of debug error messages
60	:param what_failed_desc: str, description of what was happening when failure occurred
61	e.g., what were you trying to do
62	:param analysis_property_view_id: int, optional, if provided, the error messages will be linked
63	to this property view
64	:param fail_on_error: bool, optional, if True and errors were found, this fails the pipeline
65	and stops the celery task chain
66	"""
67	if not errors:	×
68	return	×
69
70	for error in errors:	×
71	AnalysisMessage.log_and_create(	×
72	logger=logger,
73	type_=AnalysisMessage.ERROR,
74	analysis_id=context.analysis.id,
75	analysis_property_view_id=analysis_property_view_id,
76	user_message='Unexpected error from BETTER service. Please confirm your Organization\'s API token is correct and try again or contact the SEED administrators.',
77	debug_message=f'{what_failed_desc}: {error}',
78	)
79
80	if fail_on_error:	×
81	# avoid circular import
82	from seed.analysis_pipelines.better.pipeline import BETTERPipeline	×
83
84	pipeline = BETTERPipeline(context.analysis.id)	×
85	pipeline.fail(what_failed_desc, logger, progress_data_key=context.progress_data.key)	×
86	# stop the task chain
87	raise StopAnalysisTaskChain(what_failed_desc)	×
88
89
90	def _run_better_portfolio_analysis(better_portfolio_id, better_building_analyses, analysis_config, context):	1✔
91	"""Create and run an analysis for a BETTER portfolio. Updates all BuildingAnalysis
92	objects in better_building_analyses to store their individual building analysis IDs.
93
94	:param better_portfolio_id: int
95	:param better_building_analyses: list[BuildingAnalysis]
96	:param analysis_config: dict, config for the analysis API
97	:param analysis: Analysis
98	:param progress_data: ProgressData
99	:returns: int, better_analysis_id, ID of the analysis which was created and run
100	"""
101	better_analysis_id, errors = context.client.create_portfolio_analysis(	×
102	better_portfolio_id,
103	analysis_config,
104	)
105	_check_errors(	×
106	errors,
107	'Failed to create BETTER portfolio analysis',
108	context,
109	fail_on_error=True
110	)
111
112	errors = context.client.run_portfolio_analysis(	×
113	better_portfolio_id,
114	better_analysis_id
115	)
116	if errors:	×
117	_check_errors(	×
118	errors,
119	'Failed to generate BETTER portfolio analysis',
120	context,
121	fail_on_error=True,
122	)
123
124	# find and store all individual building analysis IDs for the portfolio
125	# so we can fetch and save those individual analysis results later
126	better_portfolio_analysis, errors = context.client.get_portfolio_analysis(better_portfolio_id, better_analysis_id)	×
127	_check_errors(	×
128	errors,
129	'Failed to get BETTER portfolio analysis as JSON',
130	context,
131	fail_on_error=True
132	)
133
134	api_building_analytics = better_portfolio_analysis.get('building_analytics_set', [])	×
135	for api_building_analysis in api_building_analytics:	×
136	# find the corresponding BuildingAnalysis
137	building_analysis = next(	×
138	(ba for ba in better_building_analyses if ba.better_building_id == api_building_analysis['building_id']),
139	None
140	)
141	building_analysis.better_analysis_id = api_building_analysis['id']	×
142
143	return better_analysis_id	×
144
145
146	def _store_better_portfolio_analysis_results(better_analysis_id, better_building_analyses, context):	1✔
147	"""Stores results for portfolio analysis. Analysis should be completed before calling.
148
149	:param better_analysis_id: int
150	:param better_building_analyses: list[BuildingAnalysis]
151	:param analysis: Analysis
152	:param progress_data: ProgressData
153	"""
154	results_dir, errors = context.client.get_portfolio_analysis_standalone_html(better_analysis_id)	×
155	_check_errors(	×
156	errors,
157	'Failed to get BETTER portfolio analysis standalone HTML',
158	context,
159	fail_on_error=True
160	)
161	for result_file_path in pathlib.Path(results_dir.name).iterdir():	×
162	with open(result_file_path, 'r') as f:	×
163	if result_file_path.suffix != '.html':	×
164	raise AnalysisPipelineException(	×
165	f'Received unhandled file type from BETTER: {result_file_path.name}'
166	)
167
168	content_type = AnalysisOutputFile.HTML	×
169	file_ = BaseFile(f)	×
170	analysis_output_file = AnalysisOutputFile(	×
171	content_type=content_type,
172	)
173	padded_id = f'{context.analysis.id:06d}'	×
174	analysis_output_file.file.save(f'better_portfolio_output_{padded_id}_{result_file_path.name}', file_)	×
175	analysis_output_file.clean()	×
176	analysis_output_file.save()	×
177	# Since this is a portfolio analysis, add the result to all properties
178	analysis_output_file.analysis_property_views.set([b.analysis_property_view_id for b in better_building_analyses])	×
179
180
181	def _run_better_building_analyses(better_building_analyses, analysis_config, context):	1✔
182	"""Runs building analysis for each building. Updates the BuildingAnalysis objects
183	in better_building_analyses with the IDs of BETTER analyses created.
184
185	:param better_building_analyses: list[BuildingAnalysis]
186	:param analysis_config: dict, dictionary of required BETTER API body
187	:param analysis: Analysis
188	:param progress_data: ProgressData
189	"""
190	for building_analysis in better_building_analyses:	×
191	better_building_id = building_analysis.better_building_id	×
192	analysis_property_view_id = building_analysis.analysis_property_view_id	×
193
194	better_analysis_id, errors = context.client.create_and_run_building_analysis(	×
195	better_building_id,
196	analysis_config
197	)
198	if errors:	×
199	_check_errors(	×
200	errors,
201	'Failed to run BETTER building analysis',
202	context,
203	analysis_property_view_id=analysis_property_view_id,
204	fail_on_error=False,
205	)
206	# continue to next building
207	continue	×
208
209	# save the analysis ID so we can fetch and store the analysis results later
210	building_analysis.better_analysis_id = better_analysis_id	×
211
212
213	def _store_better_building_analysis_results(better_building_analyses, context):	1✔
214	"""Stores results for building analysis. Analysis should be completed before calling.
215
216	Specifically, it stores each building's standalone HTML file and links it to
217	the analysis property view.
218	It also stores each building's JSON analysis results in the analysis property view.
219
220	:param better_building_analyses: list[BuildingAnalysis]
221	:param analysis: Analysis
222	:param progress_data: ProgressData
223	"""
224	for building_analysis in better_building_analyses:	×
225	better_building_id = building_analysis.better_building_id	×
226	better_analysis_id = building_analysis.better_analysis_id	×
227	analysis_property_view_id = building_analysis.analysis_property_view_id	×
228
229	#
230	# Store the standalone HTML
231	#
232	results_dir, errors = context.client.get_building_analysis_standalone_html(better_analysis_id)	×
233	if errors:	×
234	_check_errors(	×
235	errors,
236	'Failed to get BETTER building analysis standalone HTML',
237	context,
238	analysis_property_view_id=analysis_property_view_id,
239	fail_on_error=False,
240	)
241	# continue to next building analysis
242	continue	×
243
244	for result_file_path in pathlib.Path(results_dir.name).iterdir():	×
245	with open(result_file_path, 'r') as f:	×
246	if result_file_path.suffix == '.html':	×
247	content_type = AnalysisOutputFile.HTML	×
248	file_ = BaseFile(f)	×
249	else:
250	raise AnalysisPipelineException(	×
251	f'Received unhandled file type from better: {result_file_path.name}')
252
253	analysis_output_file = AnalysisOutputFile(	×
254	content_type=content_type,
255	)
256	padded_id = f'{analysis_property_view_id:06d}'	×
257	analysis_output_file.file.save(f'better_output_{padded_id}_{result_file_path.name}', file_)	×
258	analysis_output_file.clean()	×
259	analysis_output_file.save()	×
260	analysis_output_file.analysis_property_views.set([analysis_property_view_id])	×
261
262	#
263	# Store the JSON results into the AnalysisPropertyView
264	#
265	results_dict, errors = context.client.get_building_analysis(better_building_id, better_analysis_id)	×
266	if errors:	×
267	_check_errors(	×
268	errors,
269	'Failed to get BETTER building analysis results',
270	context,
271	analysis_property_view_id=analysis_property_view_id,
272	fail_on_error=False,
273	)
274	# continue to next building analysis
275	continue	×
276
277	analysis_property_view = AnalysisPropertyView.objects.get(id=analysis_property_view_id)	×
278	analysis_property_view.parsed_results = results_dict	×
279	analysis_property_view.save()	×
280
281
282	def _create_better_buildings(better_portfolio_id, context):	1✔
283	"""Create a BETTER building
284
285	:param analysis: Analysis
286	:param better_portfolio_id: int \| None
287	:return: list[BuildingAnalysis]
288	"""
289	better_building_analyses = []	×
290	for input_file in context.analysis.input_files.all():	×
291	analysis_property_view_id = _parse_analysis_property_view_id(input_file.file.path)	×
292	better_building_id, errors = context.client.create_building(input_file.file.path, better_portfolio_id)	×
293	if errors:	×
294	_check_errors(	×
295	errors,
296	f'Failed to create building for analysis property view {analysis_property_view_id}',
297	context,
298	analysis_property_view_id,
299	fail_on_error=False
300	)
301	# go to next building
302	continue	×
303
304	better_building_analyses.append(	×
305	BuildingAnalysis(
306	analysis_property_view_id,
307	better_building_id,
308	None
309	)
310	)
311	logger.info(f'Created BETTER building ({better_building_id}) for AnalysisPropertyView ({analysis_property_view_id})')	×
312
313	return better_building_analyses	×

SEED-platform / seed / #6556

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous