#841

Committed 18 Oct 2023 09:51PM UTC coverage: 68.616% (-9.8%) from 78.391%

Build # #841

Build Type

push

circle-ci

Committed by

jiashenC

Commit Message

[BUMP]: v0.3.9+dev

Run Details

2 of 2 new or added lines in 1 file covered. (100.0%)

8634 of 12583 relevant lines covered (68.62%)

0.69 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

87.5

/evadb/functions/function_bootstrap_queries.py

# coding=utf-8
# Copyright 2018-2023 EvaDB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from evadb.configuration.constants import EvaDB_INSTALLATION_DIR
from evadb.database import EvaDBDatabase
from evadb.server.command_handler import execute_query_fetch_all

NDARRAY_DIR = "ndarray"
TUTORIALS_DIR = "tutorials"

DummyObjectDetector_function_query = """CREATE FUNCTION IF NOT EXISTS DummyObjectDetector
                  INPUT  (Frame_Array NDARRAY INT8(3, ANYDIM, ANYDIM))
                  OUTPUT (label NDARRAY STR(1))
                  TYPE  Classification
                  IMPL  '{}/../test/util.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

DummyMultiObjectDetector_function_query = """CREATE FUNCTION
                  IF NOT EXISTS  DummyMultiObjectDetector
                  INPUT  (Frame_Array NDARRAY INT8(3, ANYDIM, ANYDIM))
                  OUTPUT (labels NDARRAY STR(2))
                  TYPE  Classification
                  IMPL  '{}/../test/util.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

DummyFeatureExtractor_function_query = """CREATE FUNCTION
                  IF NOT EXISTS DummyFeatureExtractor
                  INPUT (Frame_Array NDARRAY UINT8(3, ANYDIM, ANYDIM))
                  OUTPUT (features NDARRAY FLOAT32(1, ANYDIM))
                  TYPE Classification
                  IMPL '{}/../test/util.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

DummyNoInputFunction_function_query = """CREATE FUNCTION
                  IF NOT EXISTS DummyNoInputFunction
                  IMPL '{}/../test/util.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

DummyLLM_function_query = """CREATE FUNCTION
                  IF NOT EXISTS DummyLLM
                  IMPL '{}/../test/util.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

fuzzy_function_query = """CREATE FUNCTION IF NOT EXISTS FuzzDistance
                    INPUT (Input_Array1 NDARRAY ANYTYPE, Input_Array2 NDARRAY ANYTYPE)
                    OUTPUT (distance FLOAT(32, 7))
                    TYPE NdarrayFunction
                    IMPL "{}/functions/{}/fuzzy_join.py";
        """.format(
    EvaDB_INSTALLATION_DIR, NDARRAY_DIR
)

ArrayCount_function_query = """CREATE FUNCTION
            IF NOT EXISTS  ArrayCount
            INPUT (Input_Array NDARRAY ANYTYPE, Search_Key ANYTYPE)
            OUTPUT (key_count INTEGER)
            TYPE NdarrayFunction
            IMPL "{}/functions/{}/array_count.py";
        """.format(
    EvaDB_INSTALLATION_DIR, NDARRAY_DIR
)

Crop_function_query = """CREATE FUNCTION IF NOT EXISTS Crop
                INPUT  (Frame_Array NDARRAY UINT8(3, ANYDIM, ANYDIM),
                        bboxes NDARRAY FLOAT32(ANYDIM, 4))
                OUTPUT (Cropped_Frame_Array NDARRAY UINT8(3, ANYDIM, ANYDIM))
                TYPE  NdarrayFunction
                IMPL  "{}/functions/{}/crop.py";
        """.format(
    EvaDB_INSTALLATION_DIR, NDARRAY_DIR
)

Open_function_query = """CREATE FUNCTION IF NOT EXISTS Open
                INPUT (img_path TEXT(1000))
                OUTPUT (data NDARRAY UINT8(3, ANYDIM, ANYDIM))
                TYPE NdarrayFunction
                IMPL "{}/functions/{}/open.py";
        """.format(
    EvaDB_INSTALLATION_DIR, NDARRAY_DIR
)

Similarity_function_query = """CREATE FUNCTION IF NOT EXISTS Similarity
                    INPUT (Frame_Array_Open NDARRAY UINT8(3, ANYDIM, ANYDIM),
                           Frame_Array_Base NDARRAY UINT8(3, ANYDIM, ANYDIM),
                           Feature_Extractor_Name TEXT(100))
                    OUTPUT (distance FLOAT(32, 7))
                    TYPE NdarrayFunction
                    IMPL "{}/functions/{}/similarity.py";
        """.format(
    EvaDB_INSTALLATION_DIR, NDARRAY_DIR
)

Unnest_function_query = """CREATE FUNCTION IF NOT EXISTS Unnest
                INPUT  (inp NDARRAY ANYTYPE)
                OUTPUT (out ANYTYPE)
                TYPE  NdarrayFunction
                IMPL  "{}/functions/{}/unnest.py";
        """.format(
    EvaDB_INSTALLATION_DIR, NDARRAY_DIR
)

Fastrcnn_function_query = """CREATE FUNCTION IF NOT EXISTS FastRCNNObjectDetector
      INPUT  (Frame_Array NDARRAY UINT8(3, ANYDIM, ANYDIM))
      OUTPUT (labels NDARRAY STR(ANYDIM), bboxes NDARRAY FLOAT32(ANYDIM, 4),
                scores NDARRAY FLOAT32(ANYDIM))
      TYPE  Classification
      IMPL  '{}/functions/fastrcnn_object_detector.py';
      """.format(
    EvaDB_INSTALLATION_DIR
)

Yolo_function_query = """CREATE FUNCTION IF NOT EXISTS Yolo
      TYPE  ultralytics
      MODEL 'yolov8m.pt';
      """

face_detection_function_query = """CREATE FUNCTION IF NOT EXISTS FaceDetector
                  INPUT  (frame NDARRAY UINT8(3, ANYDIM, ANYDIM))
                  OUTPUT (bboxes NDARRAY FLOAT32(ANYDIM, 4),
                          scores NDARRAY FLOAT32(ANYDIM))
                  TYPE  FaceDetection
                  IMPL  '{}/functions/face_detector.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

Mvit_function_query = """CREATE FUNCTION IF NOT EXISTS MVITActionRecognition
        INPUT  (Frame_Array NDARRAY UINT8(3, 16, 224, 224))
        OUTPUT (labels NDARRAY STR(ANYDIM))
        TYPE  Classification
        IMPL  '{}/functions/mvit_action_recognition.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

Asl_function_query = """CREATE FUNCTION IF NOT EXISTS ASLActionRecognition
        INPUT  (Frame_Array NDARRAY UINT8(3, 16, 224, 224))
        OUTPUT (labels NDARRAY STR(ANYDIM))
        TYPE  Classification
        IMPL  '{}/functions/asl_action_recognition.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

norfair_obj_tracker_query = """CREATE FUNCTION IF NOT EXISTS NorFairTracker
                  IMPL  '{}/functions/trackers/nor_fair.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

Sift_function_query = """CREATE FUNCTION IF NOT EXISTS SiftFeatureExtractor
        IMPL  '{}/functions/sift_feature_extractor.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

Text_feat_function_query = """CREATE FUNCTION IF NOT EXISTS SentenceFeatureExtractor
        IMPL  '{}/functions/sentence_feature_extractor.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

mnistcnn_function_query = """CREATE FUNCTION IF NOT EXISTS MnistImageClassifier
        INPUT  (data NDARRAY (3, 28, 28))
        OUTPUT (label TEXT(2))
        TYPE  Classification
        IMPL  '{}/functions/mnist_image_classifier.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

chatgpt_function_query = """CREATE FUNCTION IF NOT EXISTS ChatGPT
        IMPL '{}/functions/chatgpt.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

yolo8n_query = """CREATE FUNCTION IF NOT EXISTS Yolo
            TYPE  ultralytics
            MODEL 'yolov8n.pt';
        """

stablediffusion_function_query = """CREATE FUNCTION IF NOT EXISTS StableDiffusion
        IMPL '{}/functions/stable_diffusion.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

dalle_function_query = """CREATE FUNCTION IF NOT EXISTS DallE
        IMPL '{}/functions/dalle.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)


def init_builtin_functions(db: EvaDBDatabase, mode: str = "debug") -> None:
    """Load the built-in functions into the system during system bootstrapping.

    The function loads a set of pre-defined function queries based on the `mode` argument.
    In 'debug' mode, the function loads debug functions along with release functions.
    In 'release' mode, only release functions are loaded. In addition, in 'debug' mode,
    the function loads a smaller model to accelerate the test suite time.

    Args:G
        mode (str, optional): The mode for loading functions, either 'debug' or 'release'.
        Defaults to 'debug'.

    """

    # Attempting to import torch module
    # It is necessary to import torch before to avoid encountering a
    # "RuntimeError: random_device could not be read"
    # The suspicion is that importing torch prior to decord resolves this issue
    try:
        import torch  # noqa: F401
    except ImportError:
        pass

    # Enable environment variables
    # Relevant for transformer-based models
    import os

    os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
    os.environ["TOKENIZERS_PARALLELISM"] = "false"

    # list of function queries to load
    queries = [
        mnistcnn_function_query,
        Fastrcnn_function_query,
        ArrayCount_function_query,
        Crop_function_query,
        Open_function_query,
        Similarity_function_query,
        norfair_obj_tracker_query,
        chatgpt_function_query,
        face_detection_function_query,
        # Mvit_function_query,
        Sift_function_query,
        Yolo_function_query,
        stablediffusion_function_query,
        dalle_function_query,
    ]

    # if mode is 'debug', add debug functions
    if mode == "debug":
        queries.extend(
            [
                DummyObjectDetector_function_query,
                DummyMultiObjectDetector_function_query,
                DummyFeatureExtractor_function_query,
                DummyNoInputFunction_function_query,
                DummyLLM_function_query,
            ]
        )

    # execute each query in the list of function queries
    # ignore exceptions during the bootstrapping phase due to missing packages
    for query in queries:
        try:
            execute_query_fetch_all(
                db, query, do_not_print_exceptions=False, do_not_raise_exceptions=True
            )
        except Exception:
            pass

1	# coding=utf-8
2	# Copyright 2018-2023 EvaDB
3	#
4	# Licensed under the Apache License, Version 2.0 (the "License");
5	# you may not use this file except in compliance with the License.
6	# You may obtain a copy of the License at
7	#
8	# http://www.apache.org/licenses/LICENSE-2.0
9	#
10	# Unless required by applicable law or agreed to in writing, software
11	# distributed under the License is distributed on an "AS IS" BASIS,
12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	# See the License for the specific language governing permissions and
14	# limitations under the License.
15
16	from evadb.configuration.constants import EvaDB_INSTALLATION_DIR	1✔
17	from evadb.database import EvaDBDatabase	1✔
18	from evadb.server.command_handler import execute_query_fetch_all	1✔
19
20	NDARRAY_DIR = "ndarray"	1✔
21	TUTORIALS_DIR = "tutorials"	1✔
22
23	DummyObjectDetector_function_query = """CREATE FUNCTION IF NOT EXISTS DummyObjectDetector	1✔
24	INPUT (Frame_Array NDARRAY INT8(3, ANYDIM, ANYDIM))
25	OUTPUT (label NDARRAY STR(1))
26	TYPE Classification
27	IMPL '{}/../test/util.py';
28	""".format(
29	EvaDB_INSTALLATION_DIR
30	)
31
32	DummyMultiObjectDetector_function_query = """CREATE FUNCTION	1✔
33	IF NOT EXISTS DummyMultiObjectDetector
34	INPUT (Frame_Array NDARRAY INT8(3, ANYDIM, ANYDIM))
35	OUTPUT (labels NDARRAY STR(2))
36	TYPE Classification
37	IMPL '{}/../test/util.py';
38	""".format(
39	EvaDB_INSTALLATION_DIR
40	)
41
42	DummyFeatureExtractor_function_query = """CREATE FUNCTION	1✔
43	IF NOT EXISTS DummyFeatureExtractor
44	INPUT (Frame_Array NDARRAY UINT8(3, ANYDIM, ANYDIM))
45	OUTPUT (features NDARRAY FLOAT32(1, ANYDIM))
46	TYPE Classification
47	IMPL '{}/../test/util.py';
48	""".format(
49	EvaDB_INSTALLATION_DIR
50	)
51
52	DummyNoInputFunction_function_query = """CREATE FUNCTION	1✔
53	IF NOT EXISTS DummyNoInputFunction
54	IMPL '{}/../test/util.py';
55	""".format(
56	EvaDB_INSTALLATION_DIR
57	)
58
59	DummyLLM_function_query = """CREATE FUNCTION	1✔
60	IF NOT EXISTS DummyLLM
61	IMPL '{}/../test/util.py';
62	""".format(
63	EvaDB_INSTALLATION_DIR
64	)
65
66	fuzzy_function_query = """CREATE FUNCTION IF NOT EXISTS FuzzDistance	1✔
67	INPUT (Input_Array1 NDARRAY ANYTYPE, Input_Array2 NDARRAY ANYTYPE)
68	OUTPUT (distance FLOAT(32, 7))
69	TYPE NdarrayFunction
70	IMPL "{}/functions/{}/fuzzy_join.py";
71	""".format(
72	EvaDB_INSTALLATION_DIR, NDARRAY_DIR
73	)
74
75	ArrayCount_function_query = """CREATE FUNCTION	1✔
76	IF NOT EXISTS ArrayCount
77	INPUT (Input_Array NDARRAY ANYTYPE, Search_Key ANYTYPE)
78	OUTPUT (key_count INTEGER)
79	TYPE NdarrayFunction
80	IMPL "{}/functions/{}/array_count.py";
81	""".format(
82	EvaDB_INSTALLATION_DIR, NDARRAY_DIR
83	)
84
85	Crop_function_query = """CREATE FUNCTION IF NOT EXISTS Crop	1✔
86	INPUT (Frame_Array NDARRAY UINT8(3, ANYDIM, ANYDIM),
87	bboxes NDARRAY FLOAT32(ANYDIM, 4))
88	OUTPUT (Cropped_Frame_Array NDARRAY UINT8(3, ANYDIM, ANYDIM))
89	TYPE NdarrayFunction
90	IMPL "{}/functions/{}/crop.py";
91	""".format(
92	EvaDB_INSTALLATION_DIR, NDARRAY_DIR
93	)
94
95	Open_function_query = """CREATE FUNCTION IF NOT EXISTS Open	1✔
96	INPUT (img_path TEXT(1000))
97	OUTPUT (data NDARRAY UINT8(3, ANYDIM, ANYDIM))
98	TYPE NdarrayFunction
99	IMPL "{}/functions/{}/open.py";
100	""".format(
101	EvaDB_INSTALLATION_DIR, NDARRAY_DIR
102	)
103
104	Similarity_function_query = """CREATE FUNCTION IF NOT EXISTS Similarity	1✔
105	INPUT (Frame_Array_Open NDARRAY UINT8(3, ANYDIM, ANYDIM),
106	Frame_Array_Base NDARRAY UINT8(3, ANYDIM, ANYDIM),
107	Feature_Extractor_Name TEXT(100))
108	OUTPUT (distance FLOAT(32, 7))
109	TYPE NdarrayFunction
110	IMPL "{}/functions/{}/similarity.py";
111	""".format(
112	EvaDB_INSTALLATION_DIR, NDARRAY_DIR
113	)
114
115	Unnest_function_query = """CREATE FUNCTION IF NOT EXISTS Unnest	1✔
116	INPUT (inp NDARRAY ANYTYPE)
117	OUTPUT (out ANYTYPE)
118	TYPE NdarrayFunction
119	IMPL "{}/functions/{}/unnest.py";
120	""".format(
121	EvaDB_INSTALLATION_DIR, NDARRAY_DIR
122	)
123
124	Fastrcnn_function_query = """CREATE FUNCTION IF NOT EXISTS FastRCNNObjectDetector	1✔
125	INPUT (Frame_Array NDARRAY UINT8(3, ANYDIM, ANYDIM))
126	OUTPUT (labels NDARRAY STR(ANYDIM), bboxes NDARRAY FLOAT32(ANYDIM, 4),
127	scores NDARRAY FLOAT32(ANYDIM))
128	TYPE Classification
129	IMPL '{}/functions/fastrcnn_object_detector.py';
130	""".format(
131	EvaDB_INSTALLATION_DIR
132	)
133
134	Yolo_function_query = """CREATE FUNCTION IF NOT EXISTS Yolo	1✔
135	TYPE ultralytics
136	MODEL 'yolov8m.pt';
137	"""
138
139	face_detection_function_query = """CREATE FUNCTION IF NOT EXISTS FaceDetector	1✔
140	INPUT (frame NDARRAY UINT8(3, ANYDIM, ANYDIM))
141	OUTPUT (bboxes NDARRAY FLOAT32(ANYDIM, 4),
142	scores NDARRAY FLOAT32(ANYDIM))
143	TYPE FaceDetection
144	IMPL '{}/functions/face_detector.py';
145	""".format(
146	EvaDB_INSTALLATION_DIR
147	)
148
149	Mvit_function_query = """CREATE FUNCTION IF NOT EXISTS MVITActionRecognition	1✔
150	INPUT (Frame_Array NDARRAY UINT8(3, 16, 224, 224))
151	OUTPUT (labels NDARRAY STR(ANYDIM))
152	TYPE Classification
153	IMPL '{}/functions/mvit_action_recognition.py';
154	""".format(
155	EvaDB_INSTALLATION_DIR
156	)
157
158	Asl_function_query = """CREATE FUNCTION IF NOT EXISTS ASLActionRecognition	1✔
159	INPUT (Frame_Array NDARRAY UINT8(3, 16, 224, 224))
160	OUTPUT (labels NDARRAY STR(ANYDIM))
161	TYPE Classification
162	IMPL '{}/functions/asl_action_recognition.py';
163	""".format(
164	EvaDB_INSTALLATION_DIR
165	)
166
167	norfair_obj_tracker_query = """CREATE FUNCTION IF NOT EXISTS NorFairTracker	1✔
168	IMPL '{}/functions/trackers/nor_fair.py';
169	""".format(
170	EvaDB_INSTALLATION_DIR
171	)
172
173	Sift_function_query = """CREATE FUNCTION IF NOT EXISTS SiftFeatureExtractor	1✔
174	IMPL '{}/functions/sift_feature_extractor.py';
175	""".format(
176	EvaDB_INSTALLATION_DIR
177	)
178
179	Text_feat_function_query = """CREATE FUNCTION IF NOT EXISTS SentenceFeatureExtractor	1✔
180	IMPL '{}/functions/sentence_feature_extractor.py';
181	""".format(
182	EvaDB_INSTALLATION_DIR
183	)
184
185	mnistcnn_function_query = """CREATE FUNCTION IF NOT EXISTS MnistImageClassifier	1✔
186	INPUT (data NDARRAY (3, 28, 28))
187	OUTPUT (label TEXT(2))
188	TYPE Classification
189	IMPL '{}/functions/mnist_image_classifier.py';
190	""".format(
191	EvaDB_INSTALLATION_DIR
192	)
193
194	chatgpt_function_query = """CREATE FUNCTION IF NOT EXISTS ChatGPT	1✔
195	IMPL '{}/functions/chatgpt.py';
196	""".format(
197	EvaDB_INSTALLATION_DIR
198	)
199
200	yolo8n_query = """CREATE FUNCTION IF NOT EXISTS Yolo	1✔
201	TYPE ultralytics
202	MODEL 'yolov8n.pt';
203	"""
204
205	stablediffusion_function_query = """CREATE FUNCTION IF NOT EXISTS StableDiffusion	1✔
206	IMPL '{}/functions/stable_diffusion.py';
207	""".format(
208	EvaDB_INSTALLATION_DIR
209	)
210
211	dalle_function_query = """CREATE FUNCTION IF NOT EXISTS DallE	×
212	IMPL '{}/functions/dalle.py';
213	""".format(
214	EvaDB_INSTALLATION_DIR
215	)
216
217
218	def init_builtin_functions(db: EvaDBDatabase, mode: str = "debug") -> None:	×
219	"""Load the built-in functions into the system during system bootstrapping.
220
221	The function loads a set of pre-defined function queries based on the `mode` argument.
222	In 'debug' mode, the function loads debug functions along with release functions.
223	In 'release' mode, only release functions are loaded. In addition, in 'debug' mode,
224	the function loads a smaller model to accelerate the test suite time.
225
226	Args:G
227	mode (str, optional): The mode for loading functions, either 'debug' or 'release'.
228	Defaults to 'debug'.
229
230	"""
231
232	# Attempting to import torch module
233	# It is necessary to import torch before to avoid encountering a
234	# "RuntimeError: random_device could not be read"
235	# The suspicion is that importing torch prior to decord resolves this issue
236	try:	×
237	import torch # noqa: F401	1✔
238	except ImportError:
239	pass
240
241	# Enable environment variables
242	# Relevant for transformer-based models
243	import os	1✔
244
245	os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"	1✔
246	os.environ["TOKENIZERS_PARALLELISM"] = "false"	1✔
247
248	# list of function queries to load
249	queries = [	1✔
250	mnistcnn_function_query,
251	Fastrcnn_function_query,
252	ArrayCount_function_query,
253	Crop_function_query,
254	Open_function_query,
255	Similarity_function_query,
256	norfair_obj_tracker_query,
257	chatgpt_function_query,
258	face_detection_function_query,
259	# Mvit_function_query,
260	Sift_function_query,
261	Yolo_function_query,
262	stablediffusion_function_query,
263	dalle_function_query,
264	]
265
266	# if mode is 'debug', add debug functions
267	if mode == "debug":	1✔
268	queries.extend(	1✔
269	[
270	DummyObjectDetector_function_query,
271	DummyMultiObjectDetector_function_query,
272	DummyFeatureExtractor_function_query,
273	DummyNoInputFunction_function_query,
274	DummyLLM_function_query,
275	]
276	)
277
278	# execute each query in the list of function queries
279	# ignore exceptions during the bootstrapping phase due to missing packages
280	for query in queries:	×
281	try:	×
282	execute_query_fetch_all(
283	db, query, do_not_print_exceptions=False, do_not_raise_exceptions=True
284	)
285	except Exception:
286	pass

georgia-tech-db / eva / #841

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous