a4c010ba-78be-4818-8e6f-1da08c6af280

Committed 31 Aug 2023 11:59PM UTC coverage: 70.992% (-10.6%) from 81.552%

Build # a4c010ba-78be-4818-8e6f-1da08c6af280

Build Type

push

circle-ci

Committed by

web-flow

Commit Message

Merge branch 'staging' into evadb_staging

Run Details

54 of 54 new or added lines in 3 files covered. (100.0%)

8020 of 11297 relevant lines covered (70.99%)

0.71 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

57.5

/evadb/executor/execution_context.py

# coding=utf-8
# Copyright 2018-2023 EvaDB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import random
from typing import List

from evadb.configuration.configuration_manager import ConfigurationManager
from evadb.constants import NO_GPU
from evadb.utils.generic_utils import get_gpu_count, is_gpu_available


class Context:
    """
    Stores the context information of the executor, i.e.,
    if using spark, name of the application, current spark executors,
    if using horovod: current rank etc.
    """

    def __new__(cls):
        if not hasattr(cls, "_instance"):
            cls._instance = super(Context, cls).__new__(cls)
        return cls._instance

    def __init__(self):
        self._config_manager = ConfigurationManager()
        self._gpus = self._populate_gpu_ids()

    @property
    def gpus(self):
        return self._gpus

    def _populate_gpu_from_config(self) -> List:
        # Populate GPU IDs from yaml config file.
        gpu_conf = self._config_manager.get_value("executor", "gpu_ids")
        available_gpus = [i for i in range(get_gpu_count())]
        return list(set(available_gpus) & set(gpu_conf))

    def _populate_gpu_from_env(self) -> List:
        # Populate GPU IDs from env variable.
        gpu_conf = map(
            lambda x: x.strip(),
            os.environ.get("CUDA_VISIBLE_DEVICES", "").strip().split(","),
        )
        gpu_conf = list(filter(lambda x: x, gpu_conf))
        gpu_conf = [int(gpu_id) for gpu_id in gpu_conf]
        available_gpus = [i for i in range(get_gpu_count())]
        return list(set(available_gpus) & set(gpu_conf))

    def _populate_gpu_ids(self) -> List:
        # Populate available GPU IDs from Torch library. Then, select subset of GPUs
        # from available GPUs based on user configuration.
        if not is_gpu_available():
            return []
        gpus = self._populate_gpu_from_config()
        if len(gpus) == 0:
            gpus = self._populate_gpu_from_env()
        return gpus

    def _select_random_gpu(self) -> str:
        """
        A random GPU selection strategy
        Returns:
            (str): GPU device ID
        """
        return random.choice(self.gpus)

    def gpu_device(self) -> str:
        """
        Selects a GPU on which the task can be executed
        Returns:
             (str): GPU device ID
        """
        if self.gpus:
            # TODO: Should allow choosing GPU based on Spark and Horovod
            return self._select_random_gpu()
        return NO_GPU

1	# coding=utf-8
2	# Copyright 2018-2023 EvaDB
3	#
4	# Licensed under the Apache License, Version 2.0 (the "License");
5	# you may not use this file except in compliance with the License.
6	# You may obtain a copy of the License at
7	#
8	# http://www.apache.org/licenses/LICENSE-2.0
9	#
10	# Unless required by applicable law or agreed to in writing, software
11	# distributed under the License is distributed on an "AS IS" BASIS,
12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	# See the License for the specific language governing permissions and
14	# limitations under the License.
15	import os	1✔
16	import random	1✔
17	from typing import List	1✔
18
19	from evadb.configuration.configuration_manager import ConfigurationManager	1✔
20	from evadb.constants import NO_GPU	1✔
21	from evadb.utils.generic_utils import get_gpu_count, is_gpu_available	1✔
22
23
24	class Context:	1✔
25	"""
26	Stores the context information of the executor, i.e.,
27	if using spark, name of the application, current spark executors,
28	if using horovod: current rank etc.
29	"""
30
31	def __new__(cls):	1✔
32	if not hasattr(cls, "_instance"):	1✔
33	cls._instance = super(Context, cls).__new__(cls)	1✔
34	return cls._instance	1✔
35
36	def __init__(self):	1✔
37	self._config_manager = ConfigurationManager()	1✔
38	self._gpus = self._populate_gpu_ids()	1✔
39
40	@property	1✔
41	def gpus(self):	1✔
42	return self._gpus	×
43
44	def _populate_gpu_from_config(self) -> List:	1✔
45	# Populate GPU IDs from yaml config file.
46	gpu_conf = self._config_manager.get_value("executor", "gpu_ids")	×
47	available_gpus = [i for i in range(get_gpu_count())]	×
48	return list(set(available_gpus) & set(gpu_conf))	×
49
50	def _populate_gpu_from_env(self) -> List:	1✔
51	# Populate GPU IDs from env variable.
52	gpu_conf = map(	×
53	lambda x: x.strip(),
54	os.environ.get("CUDA_VISIBLE_DEVICES", "").strip().split(","),
55	)
56	gpu_conf = list(filter(lambda x: x, gpu_conf))	×
57	gpu_conf = [int(gpu_id) for gpu_id in gpu_conf]	×
58	available_gpus = [i for i in range(get_gpu_count())]	×
59	return list(set(available_gpus) & set(gpu_conf))	×
60
61	def _populate_gpu_ids(self) -> List:	1✔
62	# Populate available GPU IDs from Torch library. Then, select subset of GPUs
63	# from available GPUs based on user configuration.
64	if not is_gpu_available():	1✔
65	return []	1✔
66	gpus = self._populate_gpu_from_config()	×
67	if len(gpus) == 0:	×
68	gpus = self._populate_gpu_from_env()	×
69	return gpus	×
70
71	def _select_random_gpu(self) -> str:	1✔
72	"""
73	A random GPU selection strategy
74	Returns:
75	(str): GPU device ID
76	"""
77	return random.choice(self.gpus)	×
78
79	def gpu_device(self) -> str:	1✔
80	"""
81	Selects a GPU on which the task can be executed
82	Returns:
83	(str): GPU device ID
84	"""
85	if self.gpus:	×
86	# TODO: Should allow choosing GPU based on Spark and Horovod
87	return self._select_random_gpu()	×
88	return NO_GPU	×

georgia-tech-db / eva / a4c010ba-78be-4818-8e6f-1da08c6af280

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous