10393655557

Committed 14 Aug 2024 07:18PM UTC coverage: 95.662% (+0.06%) from 95.603%

Build # 10393655557

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #1101 from oemof/revision/simplified_fake_sequence

Replace _Sequence by _FakeSequence

Run Details

1318 of 1400 branches covered (94.14%)

Branch coverage included in aggregate %.

58 of 60 new or added lines in 4 files covered. (96.67%)

1 existing line in 1 file now uncovered.

2651 of 2749 relevant lines covered (96.44%)

2.89 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.32

/src/oemof/solph/processing.py

# -*- coding: utf-8 -*-

"""Modules for providing a convenient data structure for solph results.

Information about the possible usage is provided within the examples.

SPDX-FileCopyrightText: Uwe Krien <krien@uni-bremen.de>
SPDX-FileCopyrightText: Simon Hilpert
SPDX-FileCopyrightText: Cord Kaldemeyer
SPDX-FileCopyrightText: Stephan Günther
SPDX-FileCopyrightText: henhuy
SPDX-FileCopyrightText: Johannes Kochems
SPDX-FileCopyrightText: Patrik Schönfeldt <patrik.schoenfeldt@dlr.de>

SPDX-License-Identifier: MIT

"""

import sys
from collections import abc
from itertools import groupby

import numpy as np
import pandas as pd
from oemof.network.network import Entity
from pyomo.core.base.piecewise import IndexedPiecewise
from pyomo.core.base.var import Var

from ._plumbing import _FakeSequence
from .helpers import flatten


def get_tuple(x):
    """Get oemof tuple within iterable or create it

    Tuples from Pyomo are of type `(n, n, int)`, `(n, n)` and `(n, int)`.
    For single nodes `n` a tuple with one object `(n,)` is created.
    """
    for i in x:
        if isinstance(i, tuple):
            return i
        elif issubclass(type(i), Entity):
            return (i,)

    # for standalone variables, x is used as identifying tuple
    if isinstance(x, tuple):
        return x


def get_timestep(x):
    """Get the timestep from oemof tuples

    The timestep from tuples `(n, n, int)`, `(n, n)`, `(n, int)` and (n,)
    is fetched as the last element. For time-independent data (scalars)
    zero ist returned.
    """
    if all(issubclass(type(n), Entity) for n in x):
        return 0
    else:
        return x[-1]


def remove_timestep(x):
    """Remove the timestep from oemof tuples

    The timestep is removed from tuples of type `(n, n, int)` and `(n, int)`.
    """
    if all(issubclass(type(n), Entity) for n in x):
        return x
    else:
        return x[:-1]


def create_dataframe(om):
    """Create a result DataFrame with all optimization data

    Results from Pyomo are written into one common pandas.DataFrame where
    separate columns are created for the variable index e.g. for tuples
    of the flows and components or the timesteps.
    """
    # get all pyomo variables including their block
    block_vars = list(
        set([bv.parent_component() for bv in om.component_data_objects(Var)])
    )
    var_dict = {}
    for bv in block_vars:
        # Drop the auxiliary variables introduced by pyomo's Piecewise
        parent_component = bv.parent_block().parent_component()
        if not isinstance(parent_component, IndexedPiecewise):
            try:
                idx_set = getattr(bv, "_index_set")
            except AttributeError:
                # To make it compatible with Pyomo < 6.4.1
                idx_set = getattr(bv, "_index")

            for i in idx_set:
                key = (str(bv).split(".")[0], str(bv).split(".")[-1], i)
                value = bv[i].value
                var_dict[key] = value

    # use this to create a pandas dataframe
    df = pd.DataFrame(list(var_dict.items()), columns=["pyomo_tuple", "value"])
    df["variable_name"] = df["pyomo_tuple"].str[1]

    # adapt the dataframe by separating tuple data into columns depending
    # on which dimension the variable/parameter has (scalar/sequence).
    # columns for the oemof tuple and timestep are created
    df["oemof_tuple"] = df["pyomo_tuple"].map(get_tuple)
    df = df[df["oemof_tuple"].map(lambda x: x is not None)]
    df["timestep"] = df["oemof_tuple"].map(get_timestep)
    df["oemof_tuple"] = df["oemof_tuple"].map(remove_timestep)

    # Use another call of remove timestep to get rid of period not needed
    df.loc[df["variable_name"] == "flow", "oemof_tuple"] = df.loc[
        df["variable_name"] == "flow", "oemof_tuple"
    ].map(remove_timestep)

    # order the data by oemof tuple and timestep
    df = df.sort_values(["oemof_tuple", "timestep"], ascending=[True, True])

    # drop empty decision variables
    df = df.dropna(subset=["value"])

    return df


def divide_scalars_sequences(df_dict, k):
    """Split results into scalars and sequences results

    Parameters
    ----------
    df_dict: dict
        dict of pd.DataFrames, keyed by oemof tuples
    k: tuple
        oemof tuple for results processing
    """
    try:
        condition = df_dict[k][:-1].isnull().any()
        scalars = df_dict[k].loc[:, condition].dropna().iloc[0]
        sequences = df_dict[k].loc[:, ~condition]
        return {"scalars": scalars, "sequences": sequences}
    except IndexError:
        error_message = (
            "Cannot access index on result data. "
            + "Did the optimization terminate"
            + " without errors?"
        )
        raise IndexError(error_message)


def set_result_index(df_dict, k, result_index):
    """Define index for results

    Parameters
    ----------
    df_dict: dict
        dict of pd.DataFrames, keyed by oemof tuples
    k: tuple
        oemof tuple for results processing
    result_index: pd.Index
        Index to use for results
    """
    try:
        df_dict[k].index = result_index
    except ValueError:
        try:
            df_dict[k] = df_dict[k][:-1]
            df_dict[k].index = result_index
        except ValueError as e:
            msg = (
                "\nFlow: {0}-{1}. This could be caused by NaN-values "
                "in your input data."
            )
            raise type(e)(
                str(e) + msg.format(k[0].label, k[1].label)
            ).with_traceback(sys.exc_info()[2])


def set_sequences_index(df, result_index):
    try:
        df.index = result_index
    except ValueError:
        try:
            df = df[:-1]
            df.index = result_index
        except ValueError:
            raise ValueError("Results extraction failed!")


def results(model, remove_last_time_point=False):
    """Create a nested result dictionary from the result DataFrame

    The already rearranged results from Pyomo from the result DataFrame are
    transferred into a nested dictionary of pandas objects.
    The first level key of that dictionary is a node (denoting the respective
    flow or component).

    The second level keys are "sequences" and "scalars" for a *standard model*:

    * A pd.DataFrame holds all results that are time-dependent, i.e. given as
      a sequence and can be indexed with the energy system's timeindex.
    * A pd.Series holds all scalar values which are applicable for timestep 0
      (i.e. investments).

    For a *multi-period model*, the second level key for "sequences" remains
    the same while instead of "scalars", the key "period_scalars" is used:

    * For sequences, see standard model.
    * Instead of a pd.Series, a pd.DataFrame holds scalar values indexed
      by periods. These hold investment-related variables.

    Examples
    --------
    * *Standard model*: `results[idx]['scalars']`
      and flows `results[n, n]['sequences']`.
    * *Multi-period model*: `results[idx]['period_scalars']`
      and flows `results[n, n]['sequences']`.

    Parameters
    ----------
    model : oemof.solph.BaseModel
        A solved oemof.solph model.
    remove_last_time_point : bool
        The last time point of all TIMEPOINT variables is removed to get the
        same length as the TIMESTEP (interval) variables without getting
        nan-values. By default, the last time point is removed if it has not
        been defined by the user in the EnergySystem but inferred. If all
        time points have been defined explicitly by the user the last time
        point will not be removed by default. In that case all interval
        variables will get one row with nan-values to have the same index
        for all variables.
    """
    # Extraction steps that are the same for both model types
    df = create_dataframe(model)

    # create a dict of dataframes keyed by oemof tuples
    df_dict = {
        k if len(k) > 1 else (k[0], None): v[
            ["timestep", "variable_name", "value"]
        ]
        for k, v in df.groupby("oemof_tuple")
    }

    # Define index
    if model.es.timeindex is None:
        result_index = list(range(len(model.es.timeincrement) + 1))
    else:
        result_index = model.es.timeindex

    # create final result dictionary by splitting up the dataframes in the
    # dataframe dict into a series for scalar data and dataframe for sequences
    result = {}

    # Standard model results extraction
    if model.es.periods is None:
        result = _extract_standard_model_result(
            df_dict, result, result_index, remove_last_time_point
        )
        scalars_col = "scalars"

    # Results extraction for a multi-period model
    else:
        period_indexed = ["invest", "total", "old", "old_end", "old_exo"]

        result = _extract_multi_period_model_result(
            model,
            df_dict,
            period_indexed,
            result,
            result_index,
            remove_last_time_point,
        )
        scalars_col = "period_scalars"

    # add dual variables for bus constraints
    if model.dual is not None:
        grouped = groupby(
            sorted(model.BusBlock.balance.iterkeys()), lambda t: t[0]
        )
        for bus, timestep in grouped:
            duals = [
                model.dual[model.BusBlock.balance[bus, t]] for _, t in timestep
            ]
            if model.es.periods is None:
                df = pd.DataFrame({"duals": duals}, index=result_index[:-1])
            # TODO: Align with standard model
            else:
                df = pd.DataFrame({"duals": duals}, index=result_index)
            if (bus, None) not in result.keys():
                result[(bus, None)] = {
                    "sequences": df,
                    scalars_col: pd.Series(dtype=float),
                }
            else:
                result[(bus, None)]["sequences"]["duals"] = duals

    return result


def _extract_standard_model_result(
    df_dict, result, result_index, remove_last_time_point
):
    """Extract and return the results of a standard model

    * Optionally remove last time point or include it elsewise.
    * Set index to timeindex and pivot results such that values are displayed
      for the respective variables. Reindex with the energy system's timeindex.
    * Filter for columns with nan values to retrieve scalar variables. Split
      up the DataFrame into sequences and scalars and return it.

    Parameters
    ----------
    df_dict : dict
        dictionary of results DataFrames
    result : dict
        dictionary to store the results
    result_index : pd.DatetimeIndex
        timeindex to use for the results (derived from EnergySystem)
    remove_last_time_point : bool
        if True, remove the last time point

    Returns
    -------
    result : dict
        dictionary with results stored
    """
    if remove_last_time_point:
        # The values of intervals belong to the time at the beginning of the
        # interval.
        for k in df_dict:
            df_dict[k].set_index("timestep", inplace=True)
            df_dict[k] = df_dict[k].pivot(
                columns="variable_name", values="value"
            )
            set_result_index(df_dict, k, result_index[:-1])
            result[k] = divide_scalars_sequences(df_dict, k)
    else:
        for k in df_dict:
            df_dict[k].set_index("timestep", inplace=True)
            df_dict[k] = df_dict[k].pivot(
                columns="variable_name", values="value"
            )
            # Add empty row with nan at the end of the table by adding 1 to the
            # last value of the numeric index.
            df_dict[k].loc[df_dict[k].index[-1] + 1, :] = np.nan
            set_result_index(df_dict, k, result_index)
            result[k] = divide_scalars_sequences(df_dict, k)

    return result


def _extract_multi_period_model_result(
    model,
    df_dict,
    period_indexed=None,
    result=None,
    result_index=None,
    remove_last_time_point=False,
):
    """Extract and return the results of a multi-period model

    Difference to standard model is in the way, scalar values are extracted
    since they now depend on periods.

    Parameters
    ----------
    model : oemof.solph.models.Model
        The optimization model
    df_dict : dict
        dictionary of results DataFrames
    period_indexed : list
        list of variables that are indexed by periods
    result : dict
        dictionary to store the results
    result_index : pd.DatetimeIndex
        timeindex to use for the results (derived from EnergySystem)
    remove_last_time_point : bool
        if True, remove the last time point

    Returns
    -------
    result : dict
        dictionary with results stored
    """
    for k in df_dict:
        df_dict[k].set_index("timestep", inplace=True)
        df_dict[k] = df_dict[k].pivot(columns="variable_name", values="value")
        # Split data set
        period_cols = [
            col for col in df_dict[k].columns if col in period_indexed
        ]
        # map periods to their start years for displaying period results
        d = {
            key: val + model.es.periods[0].min().year
            for key, val in enumerate(model.es.periods_years)
        }
        period_scalars = df_dict[k].loc[:, period_cols].dropna()
        sequences = df_dict[k].loc[
            :, [col for col in df_dict[k].columns if col not in period_cols]
        ]
        if remove_last_time_point:
            set_sequences_index(sequences, result_index[:-1])
        else:
            set_sequences_index(sequences, result_index)
        if period_scalars.empty:
            period_scalars = pd.DataFrame(index=d.values())
        try:
            period_scalars.rename(index=d, inplace=True)
            period_scalars.index.name = "period"
            result[k] = {
                "period_scalars": period_scalars,
                "sequences": sequences,
            }
        except IndexError:
            error_message = (
                "Some indices seem to be not matching.\n"
                "Cannot properly extract model results."
            )
            raise IndexError(error_message)

    return result


def convert_keys_to_strings(result, keep_none_type=False):
    """
    Convert the dictionary keys to strings.

    All (tuple) keys of the result object e.g. results[(pp1, bus1)] are
    converted into strings that represent the object labels
    e.g. results[('pp1','bus1')].
    """
    if keep_none_type:
        converted = {
            (
                tuple([str(e) if e is not None else None for e in k])
                if isinstance(k, tuple)
                else str(k) if k is not None else None
            ): v
            for k, v in result.items()
        }
    else:
        converted = {
            tuple(map(str, k)) if isinstance(k, tuple) else str(k): v
            for k, v in result.items()
        }
    return converted


def meta_results(om, undefined=False):
    """
    Fetch some metadata from the Solver. Feel free to add more keys.

    Valid keys of the resulting dictionary are: 'objective', 'problem',
    'solver'.

    om : oemof.solph.Model
        A solved Model.
    undefined : bool
        By default (False) only defined keys can be found in the dictionary.
        Set to True to get also the undefined keys.

    Returns
    -------
    dict
    """
    meta_res = {"objective": om.objective()}

    for k1 in ["Problem", "Solver"]:
        k1 = k1.lower()
        meta_res[k1] = {}
        for k2, v2 in om.es.results[k1][0].items():
            try:
                if str(om.es.results[k1][0][k2]) == "<undefined>":
                    if undefined:
                        meta_res[k1][k2] = str(om.es.results[k1][0][k2])
                else:
                    meta_res[k1][k2] = om.es.results[k1][0][k2]
            except TypeError:
                if undefined:
                    msg = "Cannot fetch meta results of type {0}"
                    meta_res[k1][k2] = msg.format(
                        type(om.es.results[k1][0][k2])
                    )

    return meta_res


def __separate_attrs(
    system, exclude_attrs, get_flows=False, exclude_none=True
):
    """
    Create a dictionary with flow scalars and series.

    The dictionary is structured with flows as tuples and nested dictionaries
    holding the scalars and series e.g.
    {(node1, node2): {'scalars': {'attr1': scalar, 'attr2': 'text'},
    'sequences': {'attr1': iterable, 'attr2': iterable}}}

    system:
        A solved oemof.solph.Model or oemof.solph.Energysystem
    exclude_attrs: List[str]
        List of additional attributes which shall be excluded from
        parameter dict
    get_flows: bool
        Whether to include flow values or not
    exclude_none: bool
        If set, scalars and sequences containing None values are excluded

    Returns
    -------
    dict
    """

    def detect_scalars_and_sequences(com):
        scalars = {}
        sequences = {}

        default_exclusions = [
            "__",
            "_",
            "registry",
            "inputs",
            "outputs",
            "Label",
            "input",
            "output",
            "constraint_group",
        ]
        # Must be tuple in order to work with `str.startswith()`:
        exclusions = tuple(default_exclusions + exclude_attrs)
        attrs = [
            i
            for i in dir(com)
            if not (i.startswith(exclusions) or callable(getattr(com, i)))
        ]

        for a in attrs:
            attr_value = getattr(com, a)

            # Iterate trough investment and add scalars and sequences with
            # "investment" prefix to component data:
            if attr_value.__class__.__name__ == "Investment":
                invest_data = detect_scalars_and_sequences(attr_value)
                scalars.update(
                    {
                        "investment_" + str(k): v
                        for k, v in invest_data["scalars"].items()
                    }
                )
                sequences.update(
                    {
                        "investment_" + str(k): v
                        for k, v in invest_data["sequences"].items()
                    }
                )
                continue

            if isinstance(attr_value, str):
                scalars[a] = attr_value
                continue

            # If the label is a tuple it is iterable, therefore it should be
            # converted to a string. Otherwise, it will be a sequence.
            if a == "label":
                attr_value = str(attr_value)

            if isinstance(attr_value, abc.Iterable):
                sequences[a] = attr_value
            elif isinstance(attr_value, _FakeSequence):
                scalars[a] = attr_value.value
            else:
                scalars[a] = attr_value

        sequences = flatten(sequences)

        com_data = {
            "scalars": scalars,
            "sequences": sequences,
        }
        move_undetected_scalars(com_data)
        if exclude_none:
            remove_nones(com_data)

        com_data = {
            "scalars": pd.Series(com_data["scalars"]),
            "sequences": pd.DataFrame(com_data["sequences"]),
        }
        return com_data

    def move_undetected_scalars(com):
        for ckey, value in list(com["sequences"].items()):
            if isinstance(value, str):
                com["scalars"][ckey] = value
                del com["sequences"][ckey]
            elif isinstance(value, _FakeSequence):
                com["scalars"][ckey] = value.value
                del com["sequences"][ckey]
            elif len(value) == 0:
                del com["sequences"][ckey]

    def remove_nones(com):
        for ckey, value in list(com["scalars"].items()):
            if value is None:
                del com["scalars"][ckey]
        for ckey, value in list(com["sequences"].items()):
            if len(value) == 0 or value[0] is None:
                del com["sequences"][ckey]

    # Check if system is es or om:
    if system.__class__.__name__ == "EnergySystem":
        components = system.flows() if get_flows else system.nodes
    else:
        components = system.flows if get_flows else system.es.nodes

    data = {}
    for com_key in components:
        component = components[com_key] if get_flows else com_key
        component_data = detect_scalars_and_sequences(component)
        comkey = com_key if get_flows else (com_key, None)
        data[comkey] = component_data
    return data


def parameter_as_dict(system, exclude_none=True, exclude_attrs=None):
    """
    Create a result dictionary containing node parameters.

    Results are written into a dictionary of pandas objects where
    a Series holds all scalar values and a dataframe all sequences for nodes
    and flows.
    The dictionary is keyed by flows (n, n) and nodes (n, None), e.g.
    `parameter[(n, n)]['sequences']` or `parameter[(n, n)]['scalars']`.

    Parameters
    ----------
    system: energy_system.EnergySystem
        A populated energy system.
    exclude_none: bool
        If True, all scalars and sequences containing None values are excluded
    exclude_attrs: Optional[List[str]]
        Optional list of additional attributes which shall be excluded from
        parameter dict

    Returns
    -------
    dict: Parameters for all nodes and flows
    """

    if exclude_attrs is None:
        exclude_attrs = []

    flow_data = __separate_attrs(
        system, exclude_attrs, get_flows=True, exclude_none=exclude_none
    )
    node_data = __separate_attrs(
        system, exclude_attrs, get_flows=False, exclude_none=exclude_none
    )

    flow_data.update(node_data)
    return flow_data

1	# -- coding: utf-8 --
2
3	"""Modules for providing a convenient data structure for solph results.
4
5	Information about the possible usage is provided within the examples.
6
7	SPDX-FileCopyrightText: Uwe Krien <krien@uni-bremen.de>
8	SPDX-FileCopyrightText: Simon Hilpert
9	SPDX-FileCopyrightText: Cord Kaldemeyer
10	SPDX-FileCopyrightText: Stephan Günther
11	SPDX-FileCopyrightText: henhuy
12	SPDX-FileCopyrightText: Johannes Kochems
13	SPDX-FileCopyrightText: Patrik Schönfeldt <patrik.schoenfeldt@dlr.de>
14
15	SPDX-License-Identifier: MIT
16
17	"""
18
19	import sys	3✔
20	from collections import abc	3✔
21	from itertools import groupby	3✔
22
23	import numpy as np	3✔
24	import pandas as pd	3✔
25	from oemof.network.network import Entity	3✔
26	from pyomo.core.base.piecewise import IndexedPiecewise	3✔
27	from pyomo.core.base.var import Var	3✔
28
29	from ._plumbing import _FakeSequence	3✔
30	from .helpers import flatten	3✔
31
32
33	def get_tuple(x):	3✔
34	"""Get oemof tuple within iterable or create it
35
36	Tuples from Pyomo are of type `(n, n, int)`, `(n, n)` and `(n, int)`.
37	For single nodes `n` a tuple with one object `(n,)` is created.
38	"""
39	for i in x:	3!
40	if isinstance(i, tuple):	3✔
41	return i	3✔
42	elif issubclass(type(i), Entity):	3✔
43	return (i,)	3✔
44
45	# for standalone variables, x is used as identifying tuple
46	if isinstance(x, tuple):	×
47	return x	×
48
49
50	def get_timestep(x):	3✔
51	"""Get the timestep from oemof tuples
52
53	The timestep from tuples `(n, n, int)`, `(n, n)`, `(n, int)` and (n,)
54	is fetched as the last element. For time-independent data (scalars)
55	zero ist returned.
56	"""
57	if all(issubclass(type(n), Entity) for n in x):	3✔
58	return 0	3✔
59	else:
60	return x[-1]	3✔
61
62
63	def remove_timestep(x):	3✔
64	"""Remove the timestep from oemof tuples
65
66	The timestep is removed from tuples of type `(n, n, int)` and `(n, int)`.
67	"""
68	if all(issubclass(type(n), Entity) for n in x):	3✔
69	return x	3✔
70	else:
71	return x[:-1]	3✔
72
73
74	def create_dataframe(om):	3✔
75	"""Create a result DataFrame with all optimization data
76
77	Results from Pyomo are written into one common pandas.DataFrame where
78	separate columns are created for the variable index e.g. for tuples
79	of the flows and components or the timesteps.
80	"""
81	# get all pyomo variables including their block
82	block_vars = list(	3✔
83	set([bv.parent_component() for bv in om.component_data_objects(Var)])
84	)
85	var_dict = {}	3✔
86	for bv in block_vars:	3✔
87	# Drop the auxiliary variables introduced by pyomo's Piecewise
88	parent_component = bv.parent_block().parent_component()	3✔
89	if not isinstance(parent_component, IndexedPiecewise):	3✔
90	try:	3✔
91	idx_set = getattr(bv, "_index_set")	3✔
92	except AttributeError:	×
93	# To make it compatible with Pyomo < 6.4.1
94	idx_set = getattr(bv, "_index")	×
95
96	for i in idx_set:	3✔
97	key = (str(bv).split(".")[0], str(bv).split(".")[-1], i)	3✔
98	value = bv[i].value	3✔
99	var_dict[key] = value	3✔
100
101	# use this to create a pandas dataframe
102	df = pd.DataFrame(list(var_dict.items()), columns=["pyomo_tuple", "value"])	3✔
103	df["variable_name"] = df["pyomo_tuple"].str[1]	3✔
104
105	# adapt the dataframe by separating tuple data into columns depending
106	# on which dimension the variable/parameter has (scalar/sequence).
107	# columns for the oemof tuple and timestep are created
108	df["oemof_tuple"] = df["pyomo_tuple"].map(get_tuple)	3✔
109	df = df[df["oemof_tuple"].map(lambda x: x is not None)]	3✔
110	df["timestep"] = df["oemof_tuple"].map(get_timestep)	3✔
111	df["oemof_tuple"] = df["oemof_tuple"].map(remove_timestep)	3✔
112
113	# Use another call of remove timestep to get rid of period not needed
114	df.loc[df["variable_name"] == "flow", "oemof_tuple"] = df.loc[	3✔
115	df["variable_name"] == "flow", "oemof_tuple"
116	].map(remove_timestep)
117
118	# order the data by oemof tuple and timestep
119	df = df.sort_values(["oemof_tuple", "timestep"], ascending=[True, True])	3✔
120
121	# drop empty decision variables
122	df = df.dropna(subset=["value"])	3✔
123
124	return df	3✔
125
126
127	def divide_scalars_sequences(df_dict, k):	3✔
128	"""Split results into scalars and sequences results
129
130	Parameters
131	----------
132	df_dict: dict
133	dict of pd.DataFrames, keyed by oemof tuples
134	k: tuple
135	oemof tuple for results processing
136	"""
137	try:	3✔
138	condition = df_dict[k][:-1].isnull().any()	3✔
139	scalars = df_dict[k].loc[:, condition].dropna().iloc[0]	3✔
140	sequences = df_dict[k].loc[:, ~condition]	3✔
141	return {"scalars": scalars, "sequences": sequences}	3✔
142	except IndexError:	×
143	error_message = (	×
144	"Cannot access index on result data. "
145	+ "Did the optimization terminate"
146	+ " without errors?"
147	)
148	raise IndexError(error_message)	×
149
150
151	def set_result_index(df_dict, k, result_index):	3✔
152	"""Define index for results
153
154	Parameters
155	----------
156	df_dict: dict
157	dict of pd.DataFrames, keyed by oemof tuples
158	k: tuple
159	oemof tuple for results processing
160	result_index: pd.Index
161	Index to use for results
162	"""
163	try:	3✔
164	df_dict[k].index = result_index	3✔
165	except ValueError:	3✔
166	try:	3✔
167	df_dict[k] = df_dict[k][:-1]	3✔
168	df_dict[k].index = result_index	3✔
169	except ValueError as e:	3✔
170	msg = (	3✔
171	"\nFlow: {0}-{1}. This could be caused by NaN-values "
172	"in your input data."
173	)
174	raise type(e)(	3✔
175	str(e) + msg.format(k[0].label, k[1].label)
176	).with_traceback(sys.exc_info()[2])
177
178
179	def set_sequences_index(df, result_index):	3✔
180	try:	3✔
181	df.index = result_index	3✔
182	except ValueError:	3✔
183	try:	3✔
184	df = df[:-1]	3✔
185	df.index = result_index	3✔
186	except ValueError:	×
187	raise ValueError("Results extraction failed!")	×
188
189
190	def results(model, remove_last_time_point=False):	3✔
191	"""Create a nested result dictionary from the result DataFrame
192
193	The already rearranged results from Pyomo from the result DataFrame are
194	transferred into a nested dictionary of pandas objects.
195	The first level key of that dictionary is a node (denoting the respective
196	flow or component).
197
198	The second level keys are "sequences" and "scalars" for a standard model:
199
200	* A pd.DataFrame holds all results that are time-dependent, i.e. given as
201	a sequence and can be indexed with the energy system's timeindex.
202	* A pd.Series holds all scalar values which are applicable for timestep 0
203	(i.e. investments).
204
205	For a multi-period model, the second level key for "sequences" remains
206	the same while instead of "scalars", the key "period_scalars" is used:
207
208	* For sequences, see standard model.
209	* Instead of a pd.Series, a pd.DataFrame holds scalar values indexed
210	by periods. These hold investment-related variables.
211
212	Examples
213	--------
214	* Standard model: `results[idx]['scalars']`
215	and flows `results[n, n]['sequences']`.
216	* Multi-period model: `results[idx]['period_scalars']`
217	and flows `results[n, n]['sequences']`.
218
219	Parameters
220	----------
221	model : oemof.solph.BaseModel
222	A solved oemof.solph model.
223	remove_last_time_point : bool
224	The last time point of all TIMEPOINT variables is removed to get the
225	same length as the TIMESTEP (interval) variables without getting
226	nan-values. By default, the last time point is removed if it has not
227	been defined by the user in the EnergySystem but inferred. If all
228	time points have been defined explicitly by the user the last time
229	point will not be removed by default. In that case all interval
230	variables will get one row with nan-values to have the same index
231	for all variables.
232	"""
233	# Extraction steps that are the same for both model types
234	df = create_dataframe(model)	3✔
235
236	# create a dict of dataframes keyed by oemof tuples
237	df_dict = {	3✔
238	k if len(k) > 1 else (k[0], None): v[
239	["timestep", "variable_name", "value"]
240	]
241	for k, v in df.groupby("oemof_tuple")
242	}
243
244	# Define index
245	if model.es.timeindex is None:	3✔
246	result_index = list(range(len(model.es.timeincrement) + 1))	3✔
247	else:
248	result_index = model.es.timeindex	3✔
249
250	# create final result dictionary by splitting up the dataframes in the
251	# dataframe dict into a series for scalar data and dataframe for sequences
252	result = {}	3✔
253
254	# Standard model results extraction
255	if model.es.periods is None:	3✔
256	result = _extract_standard_model_result(	3✔
257	df_dict, result, result_index, remove_last_time_point
258	)
259	scalars_col = "scalars"	3✔
260
261	# Results extraction for a multi-period model
262	else:
263	period_indexed = ["invest", "total", "old", "old_end", "old_exo"]	3✔
264
265	result = _extract_multi_period_model_result(	3✔
266	model,
267	df_dict,
268	period_indexed,
269	result,
270	result_index,
271	remove_last_time_point,
272	)
273	scalars_col = "period_scalars"	3✔
274
275	# add dual variables for bus constraints
276	if model.dual is not None:	3✔
277	grouped = groupby(	3✔
278	sorted(model.BusBlock.balance.iterkeys()), lambda t: t[0]
279	)
280	for bus, timestep in grouped:	3✔
281	duals = [	3✔
282	model.dual[model.BusBlock.balance[bus, t]] for _, t in timestep
283	]
284	if model.es.periods is None:	3✔
285	df = pd.DataFrame({"duals": duals}, index=result_index[:-1])	3✔
286	# TODO: Align with standard model
287	else:
288	df = pd.DataFrame({"duals": duals}, index=result_index)	3✔
289	if (bus, None) not in result.keys():	3!
290	result[(bus, None)] = {	3✔
291	"sequences": df,
292	scalars_col: pd.Series(dtype=float),
293	}
294	else:
295	result[(bus, None)]["sequences"]["duals"] = duals	×
296
297	return result	3✔
298
299
300	def _extract_standard_model_result(	3✔
301	df_dict, result, result_index, remove_last_time_point
302	):
303	"""Extract and return the results of a standard model
304
305	* Optionally remove last time point or include it elsewise.
306	* Set index to timeindex and pivot results such that values are displayed
307	for the respective variables. Reindex with the energy system's timeindex.
308	* Filter for columns with nan values to retrieve scalar variables. Split
309	up the DataFrame into sequences and scalars and return it.
310
311	Parameters
312	----------
313	df_dict : dict
314	dictionary of results DataFrames
315	result : dict
316	dictionary to store the results
317	result_index : pd.DatetimeIndex
318	timeindex to use for the results (derived from EnergySystem)
319	remove_last_time_point : bool
320	if True, remove the last time point
321
322	Returns
323	-------
324	result : dict
325	dictionary with results stored
326	"""
327	if remove_last_time_point:	3✔
328	# The values of intervals belong to the time at the beginning of the
329	# interval.
330	for k in df_dict:	3✔
331	df_dict[k].set_index("timestep", inplace=True)	3✔
332	df_dict[k] = df_dict[k].pivot(	3✔
333	columns="variable_name", values="value"
334	)
335	set_result_index(df_dict, k, result_index[:-1])	3✔
336	result[k] = divide_scalars_sequences(df_dict, k)	3✔
337	else:
338	for k in df_dict:	3✔
339	df_dict[k].set_index("timestep", inplace=True)	3✔
340	df_dict[k] = df_dict[k].pivot(	3✔
341	columns="variable_name", values="value"
342	)
343	# Add empty row with nan at the end of the table by adding 1 to the
344	# last value of the numeric index.
345	df_dict[k].loc[df_dict[k].index[-1] + 1, :] = np.nan	3✔
346	set_result_index(df_dict, k, result_index)	3✔
347	result[k] = divide_scalars_sequences(df_dict, k)	3✔
348
349	return result	3✔
350
351
352	def _extract_multi_period_model_result(	3✔
353	model,
354	df_dict,
355	period_indexed=None,
356	result=None,
357	result_index=None,
358	remove_last_time_point=False,
359	):
360	"""Extract and return the results of a multi-period model
361
362	Difference to standard model is in the way, scalar values are extracted
363	since they now depend on periods.
364
365	Parameters
366	----------
367	model : oemof.solph.models.Model
368	The optimization model
369	df_dict : dict
370	dictionary of results DataFrames
371	period_indexed : list
372	list of variables that are indexed by periods
373	result : dict
374	dictionary to store the results
375	result_index : pd.DatetimeIndex
376	timeindex to use for the results (derived from EnergySystem)
377	remove_last_time_point : bool
378	if True, remove the last time point
379
380	Returns
381	-------
382	result : dict
383	dictionary with results stored
384	"""
385	for k in df_dict:	3✔
386	df_dict[k].set_index("timestep", inplace=True)	3✔
387	df_dict[k] = df_dict[k].pivot(columns="variable_name", values="value")	3✔
388	# Split data set
389	period_cols = [	3✔
390	col for col in df_dict[k].columns if col in period_indexed
391	]
392	# map periods to their start years for displaying period results
393	d = {	3✔
394	key: val + model.es.periods[0].min().year
395	for key, val in enumerate(model.es.periods_years)
396	}
397	period_scalars = df_dict[k].loc[:, period_cols].dropna()	3✔
398	sequences = df_dict[k].loc[	3✔
399	:, [col for col in df_dict[k].columns if col not in period_cols]
400	]
401	if remove_last_time_point:	3!
402	set_sequences_index(sequences, result_index[:-1])	×
403	else:
404	set_sequences_index(sequences, result_index)	3✔
405	if period_scalars.empty:	3✔
406	period_scalars = pd.DataFrame(index=d.values())	3✔
407	try:	3✔
408	period_scalars.rename(index=d, inplace=True)	3✔
409	period_scalars.index.name = "period"	3✔
410	result[k] = {	3✔
411	"period_scalars": period_scalars,
412	"sequences": sequences,
413	}
414	except IndexError:	×
415	error_message = (	×
416	"Some indices seem to be not matching.\n"
417	"Cannot properly extract model results."
418	)
419	raise IndexError(error_message)	×
420
421	return result	3✔
422
423
424	def convert_keys_to_strings(result, keep_none_type=False):	3✔
425	"""
426	Convert the dictionary keys to strings.
427
428	All (tuple) keys of the result object e.g. results[(pp1, bus1)] are
429	converted into strings that represent the object labels
430	e.g. results[('pp1','bus1')].
431	"""
432	if keep_none_type:	3✔
433	converted = {	3✔
434	(
435	tuple([str(e) if e is not None else None for e in k])
436	if isinstance(k, tuple)
437	else str(k) if k is not None else None
438	): v
439	for k, v in result.items()
440	}
441	else:
442	converted = {	3✔
443	tuple(map(str, k)) if isinstance(k, tuple) else str(k): v
444	for k, v in result.items()
445	}
446	return converted	3✔
447
448
449	def meta_results(om, undefined=False):	3✔
450	"""
451	Fetch some metadata from the Solver. Feel free to add more keys.
452
453	Valid keys of the resulting dictionary are: 'objective', 'problem',
454	'solver'.
455
456	om : oemof.solph.Model
457	A solved Model.
458	undefined : bool
459	By default (False) only defined keys can be found in the dictionary.
460	Set to True to get also the undefined keys.
461
462	Returns
463	-------
464	dict
465	"""
466	meta_res = {"objective": om.objective()}	3✔
467
468	for k1 in ["Problem", "Solver"]:	3✔
469	k1 = k1.lower()	3✔
470	meta_res[k1] = {}	3✔
471	for k2, v2 in om.es.results[k1][0].items():	3✔
472	try:	3✔
473	if str(om.es.results[k1][0][k2]) == "<undefined>":	3✔
474	if undefined:	3!
475	meta_res[k1][k2] = str(om.es.results[k1][0][k2])	×
476	else:
477	meta_res[k1][k2] = om.es.results[k1][0][k2]	3✔
478	except TypeError:	×
479	if undefined:	×
480	msg = "Cannot fetch meta results of type {0}"	×
481	meta_res[k1][k2] = msg.format(	×
482	type(om.es.results[k1][0][k2])
483	)
484
485	return meta_res	3✔
486
487
488	def __separate_attrs(	3✔
489	system, exclude_attrs, get_flows=False, exclude_none=True
490	):
491	"""
492	Create a dictionary with flow scalars and series.
493
494	The dictionary is structured with flows as tuples and nested dictionaries
495	holding the scalars and series e.g.
496	{(node1, node2): {'scalars': {'attr1': scalar, 'attr2': 'text'},
497	'sequences': {'attr1': iterable, 'attr2': iterable}}}
498
499	system:
500	A solved oemof.solph.Model or oemof.solph.Energysystem
501	exclude_attrs: List[str]
502	List of additional attributes which shall be excluded from
503	parameter dict
504	get_flows: bool
505	Whether to include flow values or not
506	exclude_none: bool
507	If set, scalars and sequences containing None values are excluded
508
509	Returns
510	-------
511	dict
512	"""
513
514	def detect_scalars_and_sequences(com):	3✔
515	scalars = {}	3✔
516	sequences = {}	3✔
517
518	default_exclusions = [	3✔
519	"__",
520	"_",
521	"registry",
522	"inputs",
523	"outputs",
524	"Label",
525	"input",
526	"output",
527	"constraint_group",
528	]
529	# Must be tuple in order to work with `str.startswith()`:
530	exclusions = tuple(default_exclusions + exclude_attrs)	3✔
531	attrs = [	3✔
532	i
533	for i in dir(com)
534	if not (i.startswith(exclusions) or callable(getattr(com, i)))
535	]
536
537	for a in attrs:	3✔
538	attr_value = getattr(com, a)	3✔
539
540	# Iterate trough investment and add scalars and sequences with
541	# "investment" prefix to component data:
542	if attr_value.__class__.__name__ == "Investment":	3✔
543	invest_data = detect_scalars_and_sequences(attr_value)	3✔
544	scalars.update(	3✔
545	{
546	"investment_" + str(k): v
547	for k, v in invest_data["scalars"].items()
548	}
549	)
550	sequences.update(	3✔
551	{
552	"investment_" + str(k): v
553	for k, v in invest_data["sequences"].items()
554	}
555	)
556	continue	3✔
557
558	if isinstance(attr_value, str):	3✔
559	scalars[a] = attr_value	3✔
560	continue	3✔
561
562	# If the label is a tuple it is iterable, therefore it should be
563	# converted to a string. Otherwise, it will be a sequence.
564	if a == "label":	3✔
565	attr_value = str(attr_value)	3✔
566
567	if isinstance(attr_value, abc.Iterable):	3✔
568	sequences[a] = attr_value	3✔
569	elif isinstance(attr_value, _FakeSequence):	3!
NEW 570	scalars[a] = attr_value.value	×
571	else:
572	scalars[a] = attr_value	3✔
573
574	sequences = flatten(sequences)	3✔
575
576	com_data = {	3✔
577	"scalars": scalars,
578	"sequences": sequences,
579	}
580	move_undetected_scalars(com_data)	3✔
581	if exclude_none:	3✔
582	remove_nones(com_data)	3✔
583
584	com_data = {	3✔
585	"scalars": pd.Series(com_data["scalars"]),
586	"sequences": pd.DataFrame(com_data["sequences"]),
587	}
588	return com_data	3✔
589
590	def move_undetected_scalars(com):	3✔
591	for ckey, value in list(com["sequences"].items()):	3✔
592	if isinstance(value, str):	3✔
593	com["scalars"][ckey] = value	3✔
594	del com["sequences"][ckey]	3✔
595	elif isinstance(value, _FakeSequence):	3✔
596	com["scalars"][ckey] = value.value	3✔
597	del com["sequences"][ckey]	3✔
598	elif len(value) == 0:	3!
UNCOV 599	del com["sequences"][ckey]	×
600
601	def remove_nones(com):	3✔
602	for ckey, value in list(com["scalars"].items()):	3✔
603	if value is None:	3✔
604	del com["scalars"][ckey]	3✔
605	for ckey, value in list(com["sequences"].items()):	3✔
606	if len(value) == 0 or value[0] is None:	3!
607	del com["sequences"][ckey]	×
608
609	# Check if system is es or om:
610	if system.__class__.__name__ == "EnergySystem":	3✔
611	components = system.flows() if get_flows else system.nodes	3✔
612	else:
613	components = system.flows if get_flows else system.es.nodes	3✔
614
615	data = {}	3✔
616	for com_key in components:	3✔
617	component = components[com_key] if get_flows else com_key	3✔
618	component_data = detect_scalars_and_sequences(component)	3✔
619	comkey = com_key if get_flows else (com_key, None)	3✔
620	data[comkey] = component_data	3✔
621	return data	3✔
622
623
624	def parameter_as_dict(system, exclude_none=True, exclude_attrs=None):	3✔
625	"""
626	Create a result dictionary containing node parameters.
627
628	Results are written into a dictionary of pandas objects where
629	a Series holds all scalar values and a dataframe all sequences for nodes
630	and flows.
631	The dictionary is keyed by flows (n, n) and nodes (n, None), e.g.
632	`parameter[(n, n)]['sequences']` or `parameter[(n, n)]['scalars']`.
633
634	Parameters
635	----------
636	system: energy_system.EnergySystem
637	A populated energy system.
638	exclude_none: bool
639	If True, all scalars and sequences containing None values are excluded
640	exclude_attrs: Optional[List[str]]
641	Optional list of additional attributes which shall be excluded from
642	parameter dict
643
644	Returns
645	-------
646	dict: Parameters for all nodes and flows
647	"""
648
649	if exclude_attrs is None:	3✔
650	exclude_attrs = []	3✔
651
652	flow_data = __separate_attrs(	3✔
653	system, exclude_attrs, get_flows=True, exclude_none=exclude_none
654	)
655	node_data = __separate_attrs(	3✔
656	system, exclude_attrs, get_flows=False, exclude_none=exclude_none
657	)
658
659	flow_data.update(node_data)	3✔
660	return flow_data	3✔

oemof / oemof-solph / 10393655557

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous