4911681207

Build Type

github

Committed by GitHub

Commit Message

Merge 9186e10d1 into e48989cce

Pull Request Pull Request #231: Liu 355

Run Details

180 of 229 new or added lines in 17 files covered. (78.6%)

26 existing lines in 5 files now uncovered.

15345 of 19059 relevant lines covered (80.51%)

1.65 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

92.11

/daliuge-engine/dlg/data/drops/environmentvar_drop.py

#
#    ICRAR - International Centre for Radio Astronomy Research
#    (c) UWA - The University of Western Australia, 2014
#    Copyright by UWA (in the framework of the ICRAR)
#    All rights reserved
#
#    This library is free software; you can redistribute it and/or
#    modify it under the terms of the GNU Lesser General Public
#    License as published by the Free Software Foundation; either
#    version 2.1 of the License, or (at your option) any later version.
#
#    This library is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    Lesser General Public License for more details.
#
#    You should have received a copy of the GNU Lesser General Public
#    License along with this library; if not, write to the Free Software
#    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
#    MA 02111-1307  USA
#
import abc
import io
import os
import json

from dlg.drop import AbstractDROP, DEFAULT_INTERNAL_PARAMETERS
from dlg.data.io import MemoryIO


class KeyValueDROP:
    @abc.abstractmethod
    def get(self, key):
        """
        Returns the value stored by this drop for the given key. Returns None if not present
        """

    @abc.abstractmethod
    def get_multiple(self, keys: list):
        """
        Returns a list of values stored by this drop. Maintains order returning None for any keys
        not present.
        """

    # TODO: Implement Set(key, value) operations
    @abc.abstractmethod
    def set(self, key, value):
        """
        Should update a value in the key-val store or add a new values if not present.
        """
        # Will be difficult to handle shared memory considerations.
        # For such a job, using a REDIS or other distributed key-val store may be more appropriate.


def _filter_parameters(parameters: dict):
    return {
        key: val
        for key, val in parameters.items()
        if key not in DEFAULT_INTERNAL_PARAMETERS
    }


##
# @brief EnvironmentVariables
# @details A set of environment variables, wholly specified in EAGLE and accessible to all drops.
# @par EAGLE_START
# @param category EnvironmentVariables
# @param tag daliuge
# @param dropclass dropclass/dlg.data.drops.environmentvar_drop.EnvironmentVarDROP/String/ComponentParameter/readwrite//False/False/Drop class
# @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data
# @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
# @par EAGLE_END
class EnvironmentVarDROP(AbstractDROP, KeyValueDROP):
    """
    Drop storing static variables for access by all drops.
    Functions effectively like a globally-available Python dictionary
    """

    def initialize(self, **kwargs):
        """
        Runs through all parameters, putting each into this drop's variable dict
        """
        super(EnvironmentVarDROP, self).initialize(**kwargs)
        self._variables = dict()
        self._variables.update(_filter_parameters(self.parameters))

    def getIO(self):
        return MemoryIO(
            io.BytesIO(json.dumps(self._variables).encode("utf-8"))
        )

    def get(self, key):
        """
        Fetches key from internal store if present.
        If not present, attempts to fetch variable from environment
        """
        value = self._variables.get(key)
        if value is None:
            value = os.environ.get(key)
        return value

    def get_multiple(self, keys: list):
        return_vars = []
        for key in keys:
            return_vars.append(self._variables.get(key))
        return return_vars

    def set(self, key, value):
        raise NotImplementedError(
            "Setting EnvironmentVariables mid-execution is not currently implemented"
        )

    @property
    def dataURL(self) -> str:
        hostname = os.uname()[1]
        return f"config://{hostname}/{os.getpid()}/{id(self._variables)}"

1	#
2	# ICRAR - International Centre for Radio Astronomy Research
3	# (c) UWA - The University of Western Australia, 2014
4	# Copyright by UWA (in the framework of the ICRAR)
5	# All rights reserved
6	#
7	# This library is free software; you can redistribute it and/or
8	# modify it under the terms of the GNU Lesser General Public
9	# License as published by the Free Software Foundation; either
10	# version 2.1 of the License, or (at your option) any later version.
11	#
12	# This library is distributed in the hope that it will be useful,
13	# but WITHOUT ANY WARRANTY; without even the implied warranty of
14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15	# Lesser General Public License for more details.
16	#
17	# You should have received a copy of the GNU Lesser General Public
18	# License along with this library; if not, write to the Free Software
19	# Foundation, Inc., 59 Temple Place, Suite 330, Boston,
20	# MA 02111-1307 USA
21	#
22	import abc	2✔
23	import io	2✔
24	import os	2✔
25	import json	2✔
26
27	from dlg.drop import AbstractDROP, DEFAULT_INTERNAL_PARAMETERS	2✔
28	from dlg.data.io import MemoryIO	2✔
29
30
31	class KeyValueDROP:	2✔
32	@abc.abstractmethod	2✔
33	def get(self, key):	2✔
34	"""
35	Returns the value stored by this drop for the given key. Returns None if not present
36	"""
37
38	@abc.abstractmethod	2✔
39	def get_multiple(self, keys: list):	2✔
40	"""
41	Returns a list of values stored by this drop. Maintains order returning None for any keys
42	not present.
43	"""
44
45	# TODO: Implement Set(key, value) operations
46	@abc.abstractmethod	2✔
47	def set(self, key, value):	2✔
48	"""
49	Should update a value in the key-val store or add a new values if not present.
50	"""
51	# Will be difficult to handle shared memory considerations.
52	# For such a job, using a REDIS or other distributed key-val store may be more appropriate.
53
54
55	def _filter_parameters(parameters: dict):	2✔
56	return {	2✔
57	key: val
58	for key, val in parameters.items()
59	if key not in DEFAULT_INTERNAL_PARAMETERS
60	}
61
62
63	##
64	# @brief EnvironmentVariables
65	# @details A set of environment variables, wholly specified in EAGLE and accessible to all drops.
66	# @par EAGLE_START
67	# @param category EnvironmentVariables
68	# @param tag daliuge
69	# @param dropclass dropclass/dlg.data.drops.environmentvar_drop.EnvironmentVarDROP/String/ComponentParameter/readwrite//False/False/Drop class
70	# @param streaming Streaming/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component streams input and output data
71	# @param persist Persist/False/Boolean/ComponentParameter/readwrite//False/False/Specifies whether this data component contains data that should not be deleted after execution
72	# @param dummy dummy//Object/OutputPort/readwrite//False/False/Dummy output port
73	# @par EAGLE_END
74	class EnvironmentVarDROP(AbstractDROP, KeyValueDROP):	2✔
75	"""
76	Drop storing static variables for access by all drops.
77	Functions effectively like a globally-available Python dictionary
78	"""
79
80	def initialize(self, **kwargs):	2✔
81	"""
82	Runs through all parameters, putting each into this drop's variable dict
83	"""
84	super(EnvironmentVarDROP, self).initialize(**kwargs)	2✔
85	self._variables = dict()	2✔
86	self._variables.update(_filter_parameters(self.parameters))	2✔
87
88	def getIO(self):	2✔
NEW 89	return MemoryIO(	×
90	io.BytesIO(json.dumps(self._variables).encode("utf-8"))
91	)
92
93	def get(self, key):	2✔
94	"""
95	Fetches key from internal store if present.
96	If not present, attempts to fetch variable from environment
97	"""
98	value = self._variables.get(key)	2✔
99	if value is None:	2✔
100	value = os.environ.get(key)	2✔
101	return value	2✔
102
103	def get_multiple(self, keys: list):	2✔
104	return_vars = []	2✔
105	for key in keys:	2✔
106	return_vars.append(self._variables.get(key))	2✔
107	return return_vars	2✔
108
109	def set(self, key, value):	2✔
110	raise NotImplementedError(	2✔
111	"Setting EnvironmentVariables mid-execution is not currently implemented"
112	)
113
114	@property	2✔
115	def dataURL(self) -> str:	2✔
116	hostname = os.uname()[1]	×
117	return f"config://{hostname}/{os.getpid()}/{id(self._variables)}"	×

ICRAR / daliuge / 4911681207

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous