9347426112

Committed 03 Jun 2024 09:09AM UTC coverage: 66.823% (+0.01%) from 66.813%

Build Type

github

Committed by Nikolai

Commit Message

Merge main

Signed-off-by: Nikolai <nrozanov@iponweb.net>

Pull Request Pull Request #1072: Asset audit log

Run Details

5906 of 9678 branches covered (61.03%)

Branch coverage included in aggregate %.

59 of 86 new or added lines in 7 files covered. (68.6%)

14 existing lines in 2 files now uncovered.

7458 of 10321 relevant lines covered (72.26%)

5.72 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

44.27

/flexmeasures/cli/data_edit.py

"""
CLI commands for editing data
"""

from __future__ import annotations

from datetime import timedelta

import click
import pandas as pd
from flask import current_app as app
from flask.cli import with_appcontext
import json
from flexmeasures.data.models.user import Account
from flexmeasures.data.schemas.account import AccountIdField
from sqlalchemy import delete

from flexmeasures import Sensor, Asset
from flexmeasures.data import db
from flexmeasures.data.schemas.attributes import validate_special_attributes
from flexmeasures.data.schemas.generic_assets import GenericAssetIdField
from flexmeasures.data.schemas.sensors import SensorIdField
from flexmeasures.data.models.generic_assets import GenericAsset
from flexmeasures.data.models.audit_log import AssetAuditLog
from flexmeasures.data.models.time_series import TimedBelief
from flexmeasures.data.utils import save_to_db
from flexmeasures.cli.utils import MsgStyle, DeprecatedOption, DeprecatedOptionsCommand


@click.group("edit")
def fm_edit_data():
    """FlexMeasures: Edit data."""


@fm_edit_data.command("attribute", cls=DeprecatedOptionsCommand)
@with_appcontext
@click.option(
    "--asset",
    "--asset-id",
    "assets",
    required=False,
    multiple=True,
    type=GenericAssetIdField(),
    cls=DeprecatedOption,
    deprecated=["--asset-id"],
    preferred="--asset",
    help="Add/edit attribute to this asset. Follow up with the asset's ID.",
)
@click.option(
    "--sensor",
    "--sensor-id",
    "sensors",
    required=False,
    multiple=True,
    type=SensorIdField(),
    cls=DeprecatedOption,
    deprecated=["--sensor-id"],
    preferred="--sensor",
    help="Add/edit attribute to this sensor. Follow up with the sensor's ID.",
)
@click.option(
    "--attribute",
    "attribute_key",
    required=True,
    help="Add/edit this attribute. Follow up with the name of the attribute.",
)
@click.option(
    "--float",
    "attribute_float_value",
    required=False,
    type=float,
    help="Set the attribute to this float value.",
)
@click.option(
    "--bool",
    "attribute_bool_value",
    required=False,
    type=bool,
    help="Set the attribute to this bool value.",
)
@click.option(
    "--str",
    "attribute_str_value",
    required=False,
    type=str,
    help="Set the attribute to this string value.",
)
@click.option(
    "--int",
    "attribute_int_value",
    required=False,
    type=int,
    help="Set the attribute to this integer value.",
)
@click.option(
    "--list",
    "attribute_list_value",
    required=False,
    type=str,
    help="Set the attribute to this list value. Pass a string with a JSON-parse-able list representation, e.g. '[1,\"a\"]'.",
)
@click.option(
    "--dict",
    "attribute_dict_value",
    required=False,
    type=str,
    help="Set the attribute to this dict value. Pass a string with a JSON-parse-able dict representation, e.g. '{1:\"a\"}'.",
)
@click.option(
    "--null",
    "attribute_null_value",
    required=False,
    is_flag=True,
    default=False,
    help="Set the attribute to a null value.",
)
def edit_attribute(
    attribute_key: str,
    assets: list[GenericAsset],
    sensors: list[Sensor],
    attribute_null_value: bool,
    attribute_float_value: float | None = None,
    attribute_bool_value: bool | None = None,
    attribute_str_value: str | None = None,
    attribute_int_value: int | None = None,
    attribute_list_value: str | None = None,
    attribute_dict_value: str | None = None,
):
    """Edit (or add) an asset attribute or sensor attribute."""

    if not assets and not sensors:
        raise ValueError("Missing flag: pass at least one --asset-id or --sensor-id.")

    # Parse attribute value
    attribute_value = parse_attribute_value(
        attribute_float_value=attribute_float_value,
        attribute_bool_value=attribute_bool_value,
        attribute_str_value=attribute_str_value,
        attribute_int_value=attribute_int_value,
        attribute_list_value=attribute_list_value,
        attribute_dict_value=attribute_dict_value,
        attribute_null_value=attribute_null_value,
    )

    # Some attributes with special in meaning in FlexMeasures must pass validation
    validate_special_attributes(attribute_key, attribute_value)

    # Set attribute
    for asset in assets:
        AssetAuditLog.add_record_for_attribute_update(
            attribute_key, attribute_value, "asset", asset
        )
        asset.attributes[attribute_key] = attribute_value
        db.session.add(asset)
    for sensor in sensors:
        AssetAuditLog.add_record_for_attribute_update(
            attribute_key, attribute_value, "sensor", sensor
        )
        sensor.attributes[attribute_key] = attribute_value
        db.session.add(sensor)
    db.session.commit()
    click.secho("Successfully edited/added attribute.", **MsgStyle.SUCCESS)


@fm_edit_data.command("resample-data", cls=DeprecatedOptionsCommand)
@with_appcontext
@click.option(
    "--sensor",
    "--sensor-id",
    "sensor_ids",
    multiple=True,
    required=True,
    cls=DeprecatedOption,
    deprecated=["--sensor-id"],
    preferred="--sensor",
    help="Resample data for this sensor. Follow up with the sensor's ID. This argument can be given multiple times.",
)
@click.option(
    "--event-resolution",
    "event_resolution_in_minutes",
    type=int,
    required=True,
    help="New event resolution as an integer number of minutes.",
)
@click.option(
    "--from",
    "start_str",
    required=False,
    help="Resample only data from this datetime onwards. Follow up with a timezone-aware datetime in ISO 6801 format.",
)
@click.option(
    "--until",
    "end_str",
    required=False,
    help="Resample only data until this datetime. Follow up with a timezone-aware datetime in ISO 6801 format.",
)
@click.option(
    "--skip-integrity-check",
    is_flag=True,
    help="Whether to skip checking the resampled time series data for each sensor."
    " By default, an excerpt and the mean value of the original"
    " and resampled data will be shown for manual approval.",
)
def resample_sensor_data(
    sensor_ids: list[int],
    event_resolution_in_minutes: int,
    start_str: str | None = None,
    end_str: str | None = None,
    skip_integrity_check: bool = False,
):
    """Assign a new event resolution to an existing sensor and resample its data accordingly."""
    event_resolution = timedelta(minutes=event_resolution_in_minutes)
    event_starts_after = pd.Timestamp(start_str)  # note that "" or None becomes NaT
    event_ends_before = pd.Timestamp(end_str)
    for sensor_id in sensor_ids:
        sensor = db.session.get(Sensor, sensor_id)
        if sensor.event_resolution == event_resolution:
            click.echo(f"{sensor} already has the desired event resolution.")
            continue
        df_original = sensor.search_beliefs(
            most_recent_beliefs_only=False,
            event_starts_after=event_starts_after,
            event_ends_before=event_ends_before,
        ).sort_values("event_start")
        df_resampled = df_original.resample_events(event_resolution).sort_values(
            "event_start"
        )
        if not skip_integrity_check:
            message = ""
            if sensor.event_resolution < event_resolution:
                message += f"Downsampling {sensor} to {event_resolution} will result in a loss of data. "
            click.confirm(
                message
                + f"Data before:\n{df_original}\nData after:\n{df_resampled}\nMean before: {df_original['event_value'].mean()}\nMean after: {df_resampled['event_value'].mean()}\nContinue?",
                abort=True,
            )

        AssetAuditLog.add_record(
            sensor.generic_asset,
            f"Resampled sensor data for sensor '{sensor.name}': {sensor.id} to {event_resolution} from {sensor.event_resolution}",
        )

        # Update sensor
        sensor.event_resolution = event_resolution
        db.session.add(sensor)

        # Update sensor data
        query = delete(TimedBelief).filter_by(sensor=sensor)
        if not pd.isnull(event_starts_after):
            query = query.filter(TimedBelief.event_start >= event_starts_after)
        if not pd.isnull(event_ends_before):
            query = query.filter(
                TimedBelief.event_start + sensor.event_resolution <= event_ends_before
            )
        db.session.execute(query)
        save_to_db(df_resampled, bulk_save_objects=True)
    db.session.commit()
    click.secho("Successfully resampled sensor data.", **MsgStyle.SUCCESS)


@fm_edit_data.command("transfer-ownership")
@with_appcontext
@click.option(
    "--asset",
    "asset",
    type=GenericAssetIdField(),
    required=True,
    help="Change the ownership of this asset and its children. Follow up with the asset's ID.",
)
@click.option(
    "--new-owner",
    "new_owner",
    type=AccountIdField(),
    required=True,
    help="New owner of the asset and its children.",
)
def transfer_ownership(asset: Asset, new_owner: Account):
    """
    Transfer the ownership of and asset and its children to an account.
    """

    def transfer_ownership_recursive(asset: Asset, account: Account):
        AssetAuditLog.add_record(
            asset,
            f"Transfered ownership for asset '{asset.name}': {asset.id} from '{asset.owner.name}': {asset.owner.id} to '{account.name}': {account.id}",
        )

        asset.owner = account
        for child in asset.child_assets:
            transfer_ownership_recursive(child, account)

    transfer_ownership_recursive(asset, new_owner)
    click.secho(
        f"Success! Asset `{asset}` ownership was transfered to account `{new_owner}`.",
        **MsgStyle.SUCCESS,
    )

    db.session.commit()


app.cli.add_command(fm_edit_data)


def parse_attribute_value(  # noqa: C901
    attribute_null_value: bool,
    attribute_float_value: float | None = None,
    attribute_bool_value: bool | None = None,
    attribute_str_value: str | None = None,
    attribute_int_value: int | None = None,
    attribute_list_value: str | None = None,
    attribute_dict_value: str | None = None,
) -> float | int | bool | str | list | dict | None:
    """Parse attribute value."""
    if not single_true(
        [attribute_null_value]
        + [
            v is not None
            for v in [
                attribute_float_value,
                attribute_bool_value,
                attribute_str_value,
                attribute_int_value,
                attribute_list_value,
                attribute_dict_value,
            ]
        ]
    ):
        raise ValueError("Cannot set multiple values simultaneously.")
    if attribute_null_value:
        return None
    elif attribute_float_value is not None:
        return float(attribute_float_value)
    elif attribute_bool_value is not None:
        return bool(attribute_bool_value)
    elif attribute_int_value is not None:
        return int(attribute_int_value)
    elif attribute_list_value is not None:
        try:
            val = json.loads(attribute_list_value)
        except json.decoder.JSONDecodeError as jde:
            raise ValueError(f"Error parsing list value: {jde}")
        if not isinstance(val, list):
            raise ValueError(f"{val} is not a list.")
        return val
    elif attribute_dict_value is not None:
        try:
            val = json.loads(attribute_dict_value)
        except json.decoder.JSONDecodeError as jde:
            raise ValueError(f"Error parsing dict value: {jde}")
        if not isinstance(val, dict):
            raise ValueError(f"{val} is not a dict.")
        return val
    return attribute_str_value


def single_true(iterable) -> bool:
    i = iter(iterable)
    return any(i) and not any(i)

1	"""	4✔
2	CLI commands for editing data
3	"""
4
5	from __future__ import annotations	8✔
6
7	from datetime import timedelta	8✔
8
9	import click	8✔
10	import pandas as pd	8✔
11	from flask import current_app as app	8✔
12	from flask.cli import with_appcontext	8✔
13	import json	8✔
14	from flexmeasures.data.models.user import Account	8✔
15	from flexmeasures.data.schemas.account import AccountIdField	8✔
16	from sqlalchemy import delete	8✔
17
18	from flexmeasures import Sensor, Asset	8✔
19	from flexmeasures.data import db	8✔
20	from flexmeasures.data.schemas.attributes import validate_special_attributes	8✔
21	from flexmeasures.data.schemas.generic_assets import GenericAssetIdField	8✔
22	from flexmeasures.data.schemas.sensors import SensorIdField	8✔
23	from flexmeasures.data.models.generic_assets import GenericAsset	8✔
24	from flexmeasures.data.models.audit_log import AssetAuditLog	8✔
25	from flexmeasures.data.models.time_series import TimedBelief	8✔
26	from flexmeasures.data.utils import save_to_db	8✔
27	from flexmeasures.cli.utils import MsgStyle, DeprecatedOption, DeprecatedOptionsCommand	8✔
28
29
30	@click.group("edit")	8✔
31	def fm_edit_data():	8✔
32	"""FlexMeasures: Edit data."""
33
34
35	@fm_edit_data.command("attribute", cls=DeprecatedOptionsCommand)	8✔
36	@with_appcontext	8✔
37	@click.option(	8✔
38	"--asset",
39	"--asset-id",
40	"assets",
41	required=False,
42	multiple=True,
43	type=GenericAssetIdField(),
44	cls=DeprecatedOption,
45	deprecated=["--asset-id"],
46	preferred="--asset",
47	help="Add/edit attribute to this asset. Follow up with the asset's ID.",
48	)
49	@click.option(	8✔
50	"--sensor",
51	"--sensor-id",
52	"sensors",
53	required=False,
54	multiple=True,
55	type=SensorIdField(),
56	cls=DeprecatedOption,
57	deprecated=["--sensor-id"],
58	preferred="--sensor",
59	help="Add/edit attribute to this sensor. Follow up with the sensor's ID.",
60	)
61	@click.option(	8✔
62	"--attribute",
63	"attribute_key",
64	required=True,
65	help="Add/edit this attribute. Follow up with the name of the attribute.",
66	)
67	@click.option(	8✔
68	"--float",
69	"attribute_float_value",
70	required=False,
71	type=float,
72	help="Set the attribute to this float value.",
73	)
74	@click.option(	8✔
75	"--bool",
76	"attribute_bool_value",
77	required=False,
78	type=bool,
79	help="Set the attribute to this bool value.",
80	)
81	@click.option(	8✔
82	"--str",
83	"attribute_str_value",
84	required=False,
85	type=str,
86	help="Set the attribute to this string value.",
87	)
88	@click.option(	8✔
89	"--int",
90	"attribute_int_value",
91	required=False,
92	type=int,
93	help="Set the attribute to this integer value.",
94	)
95	@click.option(	8✔
96	"--list",
97	"attribute_list_value",
98	required=False,
99	type=str,
100	help="Set the attribute to this list value. Pass a string with a JSON-parse-able list representation, e.g. '[1,\"a\"]'.",
101	)
102	@click.option(	8✔
103	"--dict",
104	"attribute_dict_value",
105	required=False,
106	type=str,
107	help="Set the attribute to this dict value. Pass a string with a JSON-parse-able dict representation, e.g. '{1:\"a\"}'.",
108	)
109	@click.option(	8✔
110	"--null",
111	"attribute_null_value",
112	required=False,
113	is_flag=True,
114	default=False,
115	help="Set the attribute to a null value.",
116	)
117	def edit_attribute(	8✔
118	attribute_key: str,
119	assets: list[GenericAsset],
120	sensors: list[Sensor],
121	attribute_null_value: bool,
122	attribute_float_value: float \| None = None,
123	attribute_bool_value: bool \| None = None,
124	attribute_str_value: str \| None = None,
125	attribute_int_value: int \| None = None,
126	attribute_list_value: str \| None = None,
127	attribute_dict_value: str \| None = None,
128	):
129	"""Edit (or add) an asset attribute or sensor attribute."""
130
131	if not assets and not sensors:	×
132	raise ValueError("Missing flag: pass at least one --asset-id or --sensor-id.")	×
133
134	# Parse attribute value
135	attribute_value = parse_attribute_value(	×
136	attribute_float_value=attribute_float_value,
137	attribute_bool_value=attribute_bool_value,
138	attribute_str_value=attribute_str_value,
139	attribute_int_value=attribute_int_value,
140	attribute_list_value=attribute_list_value,
141	attribute_dict_value=attribute_dict_value,
142	attribute_null_value=attribute_null_value,
143	)
144
145	# Some attributes with special in meaning in FlexMeasures must pass validation
146	validate_special_attributes(attribute_key, attribute_value)	×
147
148	# Set attribute
149	for asset in assets:	×
NEW 150	AssetAuditLog.add_record_for_attribute_update(	×
151	attribute_key, attribute_value, "asset", asset
152	)
153	asset.attributes[attribute_key] = attribute_value	×
154	db.session.add(asset)	×
155	for sensor in sensors:	×
NEW 156	AssetAuditLog.add_record_for_attribute_update(	×
157	attribute_key, attribute_value, "sensor", sensor
158	)
159	sensor.attributes[attribute_key] = attribute_value	×
160	db.session.add(sensor)	×
161	db.session.commit()	×
162	click.secho("Successfully edited/added attribute.", **MsgStyle.SUCCESS)	×
163
164
165	@fm_edit_data.command("resample-data", cls=DeprecatedOptionsCommand)	8✔
166	@with_appcontext	8✔
167	@click.option(	8✔
168	"--sensor",
169	"--sensor-id",
170	"sensor_ids",
171	multiple=True,
172	required=True,
173	cls=DeprecatedOption,
174	deprecated=["--sensor-id"],
175	preferred="--sensor",
176	help="Resample data for this sensor. Follow up with the sensor's ID. This argument can be given multiple times.",
177	)
178	@click.option(	8✔
179	"--event-resolution",
180	"event_resolution_in_minutes",
181	type=int,
182	required=True,
183	help="New event resolution as an integer number of minutes.",
184	)
185	@click.option(	8✔
186	"--from",
187	"start_str",
188	required=False,
189	help="Resample only data from this datetime onwards. Follow up with a timezone-aware datetime in ISO 6801 format.",
190	)
191	@click.option(	8✔
192	"--until",
193	"end_str",
194	required=False,
195	help="Resample only data until this datetime. Follow up with a timezone-aware datetime in ISO 6801 format.",
196	)
197	@click.option(	8✔
198	"--skip-integrity-check",
199	is_flag=True,
200	help="Whether to skip checking the resampled time series data for each sensor."
201	" By default, an excerpt and the mean value of the original"
202	" and resampled data will be shown for manual approval.",
203	)
204	def resample_sensor_data(	8✔
205	sensor_ids: list[int],
206	event_resolution_in_minutes: int,
207	start_str: str \| None = None,
208	end_str: str \| None = None,
209	skip_integrity_check: bool = False,
210	):
211	"""Assign a new event resolution to an existing sensor and resample its data accordingly."""
212	event_resolution = timedelta(minutes=event_resolution_in_minutes)	×
213	event_starts_after = pd.Timestamp(start_str) # note that "" or None becomes NaT	×
214	event_ends_before = pd.Timestamp(end_str)	×
215	for sensor_id in sensor_ids:	×
216	sensor = db.session.get(Sensor, sensor_id)	×
217	if sensor.event_resolution == event_resolution:	×
218	click.echo(f"{sensor} already has the desired event resolution.")	×
219	continue	×
220	df_original = sensor.search_beliefs(	×
221	most_recent_beliefs_only=False,
222	event_starts_after=event_starts_after,
223	event_ends_before=event_ends_before,
224	).sort_values("event_start")
225	df_resampled = df_original.resample_events(event_resolution).sort_values(	×
226	"event_start"
227	)
228	if not skip_integrity_check:	×
229	message = ""	×
230	if sensor.event_resolution < event_resolution:	×
231	message += f"Downsampling {sensor} to {event_resolution} will result in a loss of data. "	×
232	click.confirm(	×
233	message
234	+ f"Data before:\n{df_original}\nData after:\n{df_resampled}\nMean before: {df_original['event_value'].mean()}\nMean after: {df_resampled['event_value'].mean()}\nContinue?",
235	abort=True,
236	)
237
NEW 238	AssetAuditLog.add_record(	×
239	sensor.generic_asset,
240	f"Resampled sensor data for sensor '{sensor.name}': {sensor.id} to {event_resolution} from {sensor.event_resolution}",
241	)
242
243	# Update sensor
244	sensor.event_resolution = event_resolution	×
245	db.session.add(sensor)	×
246
247	# Update sensor data
248	query = delete(TimedBelief).filter_by(sensor=sensor)	×
249	if not pd.isnull(event_starts_after):	×
250	query = query.filter(TimedBelief.event_start >= event_starts_after)	×
251	if not pd.isnull(event_ends_before):	×
252	query = query.filter(	×
253	TimedBelief.event_start + sensor.event_resolution <= event_ends_before
254	)
255	db.session.execute(query)	×
256	save_to_db(df_resampled, bulk_save_objects=True)	×
257	db.session.commit()	×
258	click.secho("Successfully resampled sensor data.", **MsgStyle.SUCCESS)	×
259
260
261	@fm_edit_data.command("transfer-ownership")	8✔
262	@with_appcontext	8✔
263	@click.option(	8✔
264	"--asset",
265	"asset",
266	type=GenericAssetIdField(),
267	required=True,
268	help="Change the ownership of this asset and its children. Follow up with the asset's ID.",
269	)
270	@click.option(	8✔
271	"--new-owner",
272	"new_owner",
273	type=AccountIdField(),
274	required=True,
275	help="New owner of the asset and its children.",
276	)
277	def transfer_ownership(asset: Asset, new_owner: Account):	8✔
278	"""
279	Transfer the ownership of and asset and its children to an account.
280	"""
281
282	def transfer_ownership_recursive(asset: Asset, account: Account):	8✔
283	AssetAuditLog.add_record(	8✔
284	asset,
285	f"Transfered ownership for asset '{asset.name}': {asset.id} from '{asset.owner.name}': {asset.owner.id} to '{account.name}': {account.id}",
286	)
287
288	asset.owner = account	8✔
289	for child in asset.child_assets:	8✔
290	transfer_ownership_recursive(child, account)	8✔
291
292	transfer_ownership_recursive(asset, new_owner)	8✔
293	click.secho(	8✔
294	f"Success! Asset `{asset}` ownership was transfered to account `{new_owner}`.",
295	**MsgStyle.SUCCESS,
296	)
297
298	db.session.commit()	8✔
299
300
301	app.cli.add_command(fm_edit_data)	8✔
302
303
304	def parse_attribute_value( # noqa: C901	8✔
305	attribute_null_value: bool,
306	attribute_float_value: float \| None = None,
307	attribute_bool_value: bool \| None = None,
308	attribute_str_value: str \| None = None,
309	attribute_int_value: int \| None = None,
310	attribute_list_value: str \| None = None,
311	attribute_dict_value: str \| None = None,
312	) -> float \| int \| bool \| str \| list \| dict \| None:
313	"""Parse attribute value."""
314	if not single_true(	×
315	[attribute_null_value]
316	+ [
317	v is not None
318	for v in [
319	attribute_float_value,
320	attribute_bool_value,
321	attribute_str_value,
322	attribute_int_value,
323	attribute_list_value,
324	attribute_dict_value,
325	]
326	]
327	):
328	raise ValueError("Cannot set multiple values simultaneously.")	×
329	if attribute_null_value:	×
330	return None	×
331	elif attribute_float_value is not None:	×
332	return float(attribute_float_value)	×
333	elif attribute_bool_value is not None:	×
334	return bool(attribute_bool_value)	×
335	elif attribute_int_value is not None:	×
336	return int(attribute_int_value)	×
337	elif attribute_list_value is not None:	×
338	try:	×
339	val = json.loads(attribute_list_value)	×
340	except json.decoder.JSONDecodeError as jde:	×
341	raise ValueError(f"Error parsing list value: {jde}")	×
342	if not isinstance(val, list):	×
343	raise ValueError(f"{val} is not a list.")	×
344	return val	×
345	elif attribute_dict_value is not None:	×
346	try:	×
347	val = json.loads(attribute_dict_value)	×
348	except json.decoder.JSONDecodeError as jde:	×
349	raise ValueError(f"Error parsing dict value: {jde}")	×
350	if not isinstance(val, dict):	×
351	raise ValueError(f"{val} is not a dict.")	×
352	return val	×
353	return attribute_str_value	×
354
355
356	def single_true(iterable) -> bool:	8✔
357	i = iter(iterable)	×
358	return any(i) and not any(i)	×

FlexMeasures / flexmeasures / 9347426112

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous