#6522

pending completion

Build # #6522

Build Type

push

coveralls-python

Committed by

web-flow

Commit Message

Update derived column migration to prevent conflicting column names and prevent duplicate column names (#3728)

* update derived column migration to create unique names for derived columns

* prevent derived column names that are duplicated with column names

* disable create/save on error

* update and add test

Run Details

15680 of 22591 relevant lines covered (69.41%)

0.69 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

27.43

/seed/models/compliance_metrics.py

# encoding: utf-8
"""
:copyright (c) 2014 - 2022, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved.
"""


from django.db import models
from django.http import QueryDict

from seed.lib.superperms.orgs.models import Organization
from seed.models.columns import Column
from seed.models.cycles import Cycle
from seed.models.filter_group import FilterGroup
from seed.utils.properties import properties_across_cycles_with_filters


class ComplianceMetric(models.Model):

    TARGET_NONE = 0
    TARGET_GT_ACTUAL = 1  # example: GHG, Site EUI
    TARGET_LT_ACTUAL = 2  # example: EnergyStar Score
    METRIC_TYPES = (
        (TARGET_NONE, ''),
        (TARGET_GT_ACTUAL, 'Target > Actual for Compliance'),
        (TARGET_LT_ACTUAL, 'Target < Actual for Compliance'),
    )

    organization = models.ForeignKey(Organization, on_delete=models.CASCADE, related_name='compliance_metrics', blank=True, null=True)
    name = models.CharField(max_length=255)
    start = models.DateTimeField(null=True, blank=True)  # only care about year, but adding as a DateTime
    end = models.DateTimeField(null=True, blank=True)  # only care about year, but adding as a DateTime
    created = models.DateTimeField(auto_now_add=True)
    # TODO: could these be derived columns?
    actual_energy_column = models.ForeignKey(Column, related_name="actual_energy_column", null=True, on_delete=models.CASCADE)
    target_energy_column = models.ForeignKey(Column, related_name="target_energy_column", null=True, on_delete=models.CASCADE)
    energy_metric_type = models.IntegerField(choices=METRIC_TYPES, blank=True, null=True)
    actual_emission_column = models.ForeignKey(Column, related_name="actual_emission_column", null=True, on_delete=models.CASCADE)
    target_emission_column = models.ForeignKey(Column, related_name="target_emission_column", null=True, on_delete=models.CASCADE)
    emission_metric_type = models.IntegerField(choices=METRIC_TYPES, blank=True, null=True)
    filter_group = models.ForeignKey(FilterGroup, related_name="filter_group", null=True, on_delete=models.CASCADE)
    x_axis_columns = models.ManyToManyField(Column, related_name="x_axis_columns", blank=True)

    def __str__(self):
        return 'Program Metric - %s' % self.name

    def evaluate(self):
        response = {
            'meta': {
                'organization': self.organization.id,
                'compliance_metric': self.id,
            },
            'name': self.name,
            'graph_data': {
                'labels': [],
                'datasets': []
            },
            'cycles': []
        }

        query_dict = QueryDict(mutable=True)
        if self.filter_group and self.filter_group.query_dict:
            query_dict.update(self.filter_group.query_dict)
        # print(f"query dict: {query_dict}")

        # grab cycles within start and end dates
        cycles = Cycle.objects.filter(organization_id=self.organization.id, start__lte=self.end, end__gte=self.start).order_by('start')
        cycle_ids = cycles.values_list('pk', flat=True)
        response['graph_data']['labels'] = list(cycles.values_list('name', flat=True))
        response['cycles'] = list(cycles.values('id', 'name'))

        # get properties (no filter)
        # property_response = properties_across_cycles(self.organization_id, -1, cycle_ids)
        # get properties (applies filter group)
        display_field_id = Column.objects.get(table_name="PropertyState", column_name=self.organization.property_display_field, organization=self.organization).id
        # array of columns to return
        column_ids = [
            display_field_id
        ]

        if self.actual_energy_column is not None:
            column_ids.append(self.actual_energy_column.id)
            if self.target_energy_column is not None:
                column_ids.append(self.target_energy_column.id)

        if self.actual_emission_column is not None:
            column_ids.append(self.actual_emission_column.id)
            if self.target_emission_column is not None:
                column_ids.append(self.target_emission_column.id)

        for col in self.x_axis_columns.all():
            column_ids.append(col.id)

        property_response = properties_across_cycles_with_filters(
            self.organization_id,
            cycle_ids,
            query_dict,
            column_ids
        )

        datasets = {'y': {'data': [], 'label': 'compliant'}, 'n': {'data': [], 'label': 'non-compliant'}, 'u': {'data': [], 'label': 'unknown'}}
        results_by_cycles = {}
#        property_datasets = {}
        # figure out what kind of metric it is (energy? emission? combo? bool?)
        metric = {'energy_metric': False, 'emission_metric': False, 'energy_bool': False, 'emission_bool': False,
                  'actual_energy_column': None, 'actual_energy_column_name': None, 'target_energy_column': None,
                  'energy_metric_type': self.energy_metric_type, 'actual_emission_column': None, 'actual_emission_column_name': None,
                  'target_emission_column': None, 'emission_metric_type': self.emission_metric_type,
                  'fliter_group': None,
                  'x_axis_columns': list(self.x_axis_columns.all().values('id', 'display_name'))}

        if self.actual_energy_column is not None:
            metric['actual_energy_column'] = self.actual_energy_column.id
            metric['actual_energy_column_name'] = self.actual_energy_column.display_name
            metric['energy_metric'] = True
            if self.target_energy_column is None:
                metric['energy_bool'] = True
            else:
                metric['target_energy_column'] = self.target_energy_column.id

        if self.actual_emission_column is not None:
            metric['emission_metric'] = True
            metric['actual_emission_column'] = self.actual_emission_column.id
            metric['actual_emission_column_name'] = self.actual_emission_column.display_name
            if self.target_emission_column is None:
                metric['emission_bool'] = True
            else:
                metric['target_emission_column'] = self.target_emission_column.id

        for cyc in property_response:

            properties = {}
            cnts = {'y': 0, 'n': 0, 'u': 0}

            for p in property_response[cyc]:

                # initialize
                properties[p['property_view_id']] = None
                # energy metric
                if metric['energy_metric']:
                    properties[p['property_view_id']] = self._calculate_compliance(p, metric['energy_bool'], 'energy')
                # emission metric
                if metric['emission_metric'] and properties[p['property_view_id']] != 'u':
                    temp_val = self._calculate_compliance(p, metric['emission_bool'], 'emission')

                    # reconcile
                    if temp_val == 'u':
                        # unknown stays unknown (missing data)
                        properties[p['property_view_id']] = 'u'
                    elif properties[p['property_view_id']] is None:
                        # only emission metric (not energy metric)
                        properties[p['property_view_id']] = temp_val
                    else:
                        # compliant if both are compliant
                        properties[p['property_view_id']] = temp_val if temp_val == 'n' else properties[p['property_view_id']]

            # count compliant, non-compliant, unknown for each property with data
            for key in cnts:
                cnts[key] = sum(map((key).__eq__, properties.values()))
                # add to dataset
                datasets[key]['data'].append(cnts[key])

            # reshape and save
            results_by_cycles[cyc] = {}
            for key in cnts:
                results_by_cycles[cyc][key] = [i for i in properties if properties[i] == key]

        # save to response
        response['results_by_cycles'] = results_by_cycles
        response['properties_by_cycles'] = property_response
        response['metric'] = metric

        for key in datasets:
            response['graph_data']['datasets'].append(datasets[key])

        return response

    # returns compliant, non-compliant, and unknown counts
    def _calculate_compliance(self, the_property, bool_metric, metric_type):

        actual_col_id = self.actual_energy_column.id if metric_type == 'energy' else self.actual_emission_column.id
        target_col_id = self.target_energy_column.id if metric_type == 'energy' else self.target_emission_column.id
        actual_val = self._get_column_data(the_property, actual_col_id)
        if actual_val is None:
            return 'u'

        if bool_metric:
            return 'y' if actual_val > 0 else 'n'

        target_val = self._get_column_data(the_property, target_col_id)
        if target_val is None:
            return 'u'

        # test metric type
        the_type = self.energy_metric_type if metric_type == 'energy' else self.emission_metric_type
        if the_type == 1:
            differential = target_val - actual_val
        else:
            differential = actual_val - target_val

        return 'y' if differential >= 0 else 'n'

    # retrieves column data by id substring
    def _get_column_data(self, data, substring):
        value = next(v for (k, v) in data.items() if k.endswith('_' + str(substring)))
        return value

    class Meta:
        ordering = ['-created']
        get_latest_by = 'created'

1	# encoding: utf-8
2	"""	1✔
3	:copyright (c) 2014 - 2022, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved.
4	"""
5
6
7	from django.db import models
8	from django.http import QueryDict
9
10	from seed.lib.superperms.orgs.models import Organization	1✔
11	from seed.models.columns import Column	1✔
12	from seed.models.cycles import Cycle	1✔
13	from seed.models.filter_group import FilterGroup	1✔
14	from seed.utils.properties import properties_across_cycles_with_filters	1✔
15
16
17	class ComplianceMetric(models.Model):	1✔
18
19	TARGET_NONE = 0	1✔
20	TARGET_GT_ACTUAL = 1 # example: GHG, Site EUI	1✔
21	TARGET_LT_ACTUAL = 2 # example: EnergyStar Score	1✔
22	METRIC_TYPES = (	1✔
23	(TARGET_NONE, ''),
24	(TARGET_GT_ACTUAL, 'Target > Actual for Compliance'),
25	(TARGET_LT_ACTUAL, 'Target < Actual for Compliance'),
26	)
27
28	organization = models.ForeignKey(Organization, on_delete=models.CASCADE, related_name='compliance_metrics', blank=True, null=True)	1✔
29	name = models.CharField(max_length=255)	1✔
30	start = models.DateTimeField(null=True, blank=True) # only care about year, but adding as a DateTime	1✔
31	end = models.DateTimeField(null=True, blank=True) # only care about year, but adding as a DateTime	1✔
32	created = models.DateTimeField(auto_now_add=True)	1✔
33	# TODO: could these be derived columns?
34	actual_energy_column = models.ForeignKey(Column, related_name="actual_energy_column", null=True, on_delete=models.CASCADE)	1✔
35	target_energy_column = models.ForeignKey(Column, related_name="target_energy_column", null=True, on_delete=models.CASCADE)	1✔
36	energy_metric_type = models.IntegerField(choices=METRIC_TYPES, blank=True, null=True)	1✔
37	actual_emission_column = models.ForeignKey(Column, related_name="actual_emission_column", null=True, on_delete=models.CASCADE)	1✔
38	target_emission_column = models.ForeignKey(Column, related_name="target_emission_column", null=True, on_delete=models.CASCADE)	1✔
39	emission_metric_type = models.IntegerField(choices=METRIC_TYPES, blank=True, null=True)	1✔
40	filter_group = models.ForeignKey(FilterGroup, related_name="filter_group", null=True, on_delete=models.CASCADE)	1✔
41	x_axis_columns = models.ManyToManyField(Column, related_name="x_axis_columns", blank=True)	1✔
42
43	def __str__(self):	1✔
44	return 'Program Metric - %s' % self.name	×
45
46	def evaluate(self):	1✔
47	response = {	×
48	'meta': {
49	'organization': self.organization.id,
50	'compliance_metric': self.id,
51	},
52	'name': self.name,
53	'graph_data': {
54	'labels': [],
55	'datasets': []
56	},
57	'cycles': []
58	}
59
60	query_dict = QueryDict(mutable=True)	×
61	if self.filter_group and self.filter_group.query_dict:	×
62	query_dict.update(self.filter_group.query_dict)	×
63	# print(f"query dict: {query_dict}")
64
65	# grab cycles within start and end dates
66	cycles = Cycle.objects.filter(organization_id=self.organization.id, start__lte=self.end, end__gte=self.start).order_by('start')	×
67	cycle_ids = cycles.values_list('pk', flat=True)	×
68	response['graph_data']['labels'] = list(cycles.values_list('name', flat=True))	×
69	response['cycles'] = list(cycles.values('id', 'name'))	×
70
71	# get properties (no filter)
72	# property_response = properties_across_cycles(self.organization_id, -1, cycle_ids)
73	# get properties (applies filter group)
74	display_field_id = Column.objects.get(table_name="PropertyState", column_name=self.organization.property_display_field, organization=self.organization).id	×
75	# array of columns to return
76	column_ids = [	×
77	display_field_id
78	]
79
80	if self.actual_energy_column is not None:	×
81	column_ids.append(self.actual_energy_column.id)	×
82	if self.target_energy_column is not None:	×
83	column_ids.append(self.target_energy_column.id)	×
84
85	if self.actual_emission_column is not None:	×
86	column_ids.append(self.actual_emission_column.id)	×
87	if self.target_emission_column is not None:	×
88	column_ids.append(self.target_emission_column.id)	×
89
90	for col in self.x_axis_columns.all():	×
91	column_ids.append(col.id)	×
92
93	property_response = properties_across_cycles_with_filters(	×
94	self.organization_id,
95	cycle_ids,
96	query_dict,
97	column_ids
98	)
99
100	datasets = {'y': {'data': [], 'label': 'compliant'}, 'n': {'data': [], 'label': 'non-compliant'}, 'u': {'data': [], 'label': 'unknown'}}	×
101	results_by_cycles = {}	×
102	# property_datasets = {}
103	# figure out what kind of metric it is (energy? emission? combo? bool?)
104	metric = {'energy_metric': False, 'emission_metric': False, 'energy_bool': False, 'emission_bool': False,	×
105	'actual_energy_column': None, 'actual_energy_column_name': None, 'target_energy_column': None,
106	'energy_metric_type': self.energy_metric_type, 'actual_emission_column': None, 'actual_emission_column_name': None,
107	'target_emission_column': None, 'emission_metric_type': self.emission_metric_type,
108	'fliter_group': None,
109	'x_axis_columns': list(self.x_axis_columns.all().values('id', 'display_name'))}
110
111	if self.actual_energy_column is not None:	×
112	metric['actual_energy_column'] = self.actual_energy_column.id	×
113	metric['actual_energy_column_name'] = self.actual_energy_column.display_name	×
114	metric['energy_metric'] = True	×
115	if self.target_energy_column is None:	×
116	metric['energy_bool'] = True	×
117	else:
118	metric['target_energy_column'] = self.target_energy_column.id	×
119
120	if self.actual_emission_column is not None:	×
121	metric['emission_metric'] = True	×
122	metric['actual_emission_column'] = self.actual_emission_column.id	×
123	metric['actual_emission_column_name'] = self.actual_emission_column.display_name	×
124	if self.target_emission_column is None:	×
125	metric['emission_bool'] = True	×
126	else:
127	metric['target_emission_column'] = self.target_emission_column.id	×
128
129	for cyc in property_response:	×
130
131	properties = {}	×
132	cnts = {'y': 0, 'n': 0, 'u': 0}	×
133
134	for p in property_response[cyc]:	×
135
136	# initialize
137	properties[p['property_view_id']] = None	×
138	# energy metric
139	if metric['energy_metric']:	×
140	properties[p['property_view_id']] = self._calculate_compliance(p, metric['energy_bool'], 'energy')	×
141	# emission metric
142	if metric['emission_metric'] and properties[p['property_view_id']] != 'u':	×
143	temp_val = self._calculate_compliance(p, metric['emission_bool'], 'emission')	×
144
145	# reconcile
146	if temp_val == 'u':	×
147	# unknown stays unknown (missing data)
148	properties[p['property_view_id']] = 'u'	×
149	elif properties[p['property_view_id']] is None:	×
150	# only emission metric (not energy metric)
151	properties[p['property_view_id']] = temp_val	×
152	else:
153	# compliant if both are compliant
154	properties[p['property_view_id']] = temp_val if temp_val == 'n' else properties[p['property_view_id']]	×
155
156	# count compliant, non-compliant, unknown for each property with data
157	for key in cnts:	×
158	cnts[key] = sum(map((key).__eq__, properties.values()))	×
159	# add to dataset
160	datasets[key]['data'].append(cnts[key])	×
161
162	# reshape and save
163	results_by_cycles[cyc] = {}	×
164	for key in cnts:	×
165	results_by_cycles[cyc][key] = [i for i in properties if properties[i] == key]	×
166
167	# save to response
168	response['results_by_cycles'] = results_by_cycles	×
169	response['properties_by_cycles'] = property_response	×
170	response['metric'] = metric	×
171
172	for key in datasets:	×
173	response['graph_data']['datasets'].append(datasets[key])	×
174
175	return response	×
176
177	# returns compliant, non-compliant, and unknown counts
178	def _calculate_compliance(self, the_property, bool_metric, metric_type):	1✔
179
180	actual_col_id = self.actual_energy_column.id if metric_type == 'energy' else self.actual_emission_column.id	×
181	target_col_id = self.target_energy_column.id if metric_type == 'energy' else self.target_emission_column.id	×
182	actual_val = self._get_column_data(the_property, actual_col_id)	×
183	if actual_val is None:	×
184	return 'u'	×
185
186	if bool_metric:	×
187	return 'y' if actual_val > 0 else 'n'	×
188
189	target_val = self._get_column_data(the_property, target_col_id)	×
190	if target_val is None:	×
191	return 'u'	×
192
193	# test metric type
194	the_type = self.energy_metric_type if metric_type == 'energy' else self.emission_metric_type	×
195	if the_type == 1:	×
196	differential = target_val - actual_val	×
197	else:
198	differential = actual_val - target_val	×
199
200	return 'y' if differential >= 0 else 'n'	×
201
202	# retrieves column data by id substring
203	def _get_column_data(self, data, substring):	1✔
204	value = next(v for (k, v) in data.items() if k.endswith('_' + str(substring)))	×
205	return value	×
206
207	class Meta:	1✔
208	ordering = ['-created']	1✔
209	get_latest_by = 'created'	1✔

SEED-platform / seed / #6522

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous