#763

Committed 06 Sep 2023 03:00AM UTC coverage: 74.923% (+5.1%) from 69.779%

Build # #763

Build Type

push

circle-ci

Committed by

jiashenC

Commit Message

fix lint

Run Details

10 of 10 new or added lines in 1 file covered. (100.0%)

8739 of 11664 relevant lines covered (74.92%)

0.75 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.22

/evadb/functions/function_bootstrap_queries.py

# coding=utf-8
# Copyright 2018-2023 EvaDB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from evadb.configuration.constants import EvaDB_INSTALLATION_DIR
from evadb.database import EvaDBDatabase
from evadb.server.command_handler import execute_query_fetch_all

NDARRAY_DIR = "ndarray"
TUTORIALS_DIR = "tutorials"

DummyObjectDetector_function_query = """CREATE FUNCTION IF NOT EXISTS DummyObjectDetector
                  INPUT  (Frame_Array NDARRAY INT8(3, ANYDIM, ANYDIM))
                  OUTPUT (label NDARRAY STR(1))
                  TYPE  Classification
                  IMPL  '{}/../test/util.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

DummyMultiObjectDetector_function_query = """CREATE FUNCTION
                  IF NOT EXISTS  DummyMultiObjectDetector
                  INPUT  (Frame_Array NDARRAY INT8(3, ANYDIM, ANYDIM))
                  OUTPUT (labels NDARRAY STR(2))
                  TYPE  Classification
                  IMPL  '{}/../test/util.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

DummyFeatureExtractor_function_query = """CREATE FUNCTION
                  IF NOT EXISTS DummyFeatureExtractor
                  INPUT (Frame_Array NDARRAY UINT8(3, ANYDIM, ANYDIM))
                  OUTPUT (features NDARRAY FLOAT32(1, ANYDIM))
                  TYPE Classification
                  IMPL '{}/../test/util.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

fuzzy_function_query = """CREATE FUNCTION IF NOT EXISTS FuzzDistance
                    INPUT (Input_Array1 NDARRAY ANYTYPE, Input_Array2 NDARRAY ANYTYPE)
                    OUTPUT (distance FLOAT(32, 7))
                    TYPE NdarrayFunction
                    IMPL "{}/functions/{}/fuzzy_join.py";
        """.format(
    EvaDB_INSTALLATION_DIR, NDARRAY_DIR
)

ArrayCount_function_query = """CREATE FUNCTION
            IF NOT EXISTS  ArrayCount
            INPUT (Input_Array NDARRAY ANYTYPE, Search_Key ANYTYPE)
            OUTPUT (key_count INTEGER)
            TYPE NdarrayFunction
            IMPL "{}/functions/{}/array_count.py";
        """.format(
    EvaDB_INSTALLATION_DIR, NDARRAY_DIR
)

Crop_function_query = """CREATE FUNCTION IF NOT EXISTS Crop
                INPUT  (Frame_Array NDARRAY UINT8(3, ANYDIM, ANYDIM),
                        bboxes NDARRAY FLOAT32(ANYDIM, 4))
                OUTPUT (Cropped_Frame_Array NDARRAY UINT8(3, ANYDIM, ANYDIM))
                TYPE  NdarrayFunction
                IMPL  "{}/functions/{}/crop.py";
        """.format(
    EvaDB_INSTALLATION_DIR, NDARRAY_DIR
)

Open_function_query = """CREATE FUNCTION IF NOT EXISTS Open
                INPUT (img_path TEXT(1000))
                OUTPUT (data NDARRAY UINT8(3, ANYDIM, ANYDIM))
                TYPE NdarrayFunction
                IMPL "{}/functions/{}/open.py";
        """.format(
    EvaDB_INSTALLATION_DIR, NDARRAY_DIR
)

Similarity_function_query = """CREATE FUNCTION IF NOT EXISTS Similarity
                    INPUT (Frame_Array_Open NDARRAY UINT8(3, ANYDIM, ANYDIM),
                           Frame_Array_Base NDARRAY UINT8(3, ANYDIM, ANYDIM),
                           Feature_Extractor_Name TEXT(100))
                    OUTPUT (distance FLOAT(32, 7))
                    TYPE NdarrayFunction
                    IMPL "{}/functions/{}/similarity.py";
        """.format(
    EvaDB_INSTALLATION_DIR, NDARRAY_DIR
)

Unnest_function_query = """CREATE FUNCTION IF NOT EXISTS Unnest
                INPUT  (inp NDARRAY ANYTYPE)
                OUTPUT (out ANYTYPE)
                TYPE  NdarrayFunction
                IMPL  "{}/functions/{}/unnest.py";
        """.format(
    EvaDB_INSTALLATION_DIR, NDARRAY_DIR
)

Fastrcnn_function_query = """CREATE FUNCTION IF NOT EXISTS FastRCNNObjectDetector
      INPUT  (Frame_Array NDARRAY UINT8(3, ANYDIM, ANYDIM))
      OUTPUT (labels NDARRAY STR(ANYDIM), bboxes NDARRAY FLOAT32(ANYDIM, 4),
                scores NDARRAY FLOAT32(ANYDIM))
      TYPE  Classification
      IMPL  '{}/functions/fastrcnn_object_detector.py';
      """.format(
    EvaDB_INSTALLATION_DIR
)

Yolo_function_query = """CREATE FUNCTION IF NOT EXISTS Yolo
      TYPE  ultralytics
      MODEL 'yolov8m.pt';
      """

face_detection_function_query = """CREATE FUNCTION IF NOT EXISTS FaceDetector
                  INPUT  (frame NDARRAY UINT8(3, ANYDIM, ANYDIM))
                  OUTPUT (bboxes NDARRAY FLOAT32(ANYDIM, 4),
                          scores NDARRAY FLOAT32(ANYDIM))
                  TYPE  FaceDetection
                  IMPL  '{}/functions/face_detector.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

Mvit_function_query = """CREATE FUNCTION IF NOT EXISTS MVITActionRecognition
        INPUT  (Frame_Array NDARRAY UINT8(3, 16, 224, 224))
        OUTPUT (labels NDARRAY STR(ANYDIM))
        TYPE  Classification
        IMPL  '{}/functions/mvit_action_recognition.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

Asl_function_query = """CREATE FUNCTION IF NOT EXISTS ASLActionRecognition
        INPUT  (Frame_Array NDARRAY UINT8(3, 16, 224, 224))
        OUTPUT (labels NDARRAY STR(ANYDIM))
        TYPE  Classification
        IMPL  '{}/functions/asl_action_recognition.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

norfair_obj_tracker_query = """CREATE FUNCTION IF NOT EXISTS NorFairTracker
                  IMPL  '{}/functions/trackers/nor_fair.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

Sift_function_query = """CREATE FUNCTION IF NOT EXISTS SiftFeatureExtractor
        IMPL  '{}/functions/sift_feature_extractor.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

Text_feat_function_query = """CREATE FUNCTION IF NOT EXISTS SentenceFeatureExtractor
        IMPL  '{}/functions/sentence_feature_extractor.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

mnistcnn_function_query = """CREATE FUNCTION IF NOT EXISTS MnistImageClassifier
        INPUT  (data NDARRAY (3, 28, 28))
        OUTPUT (label TEXT(2))
        TYPE  Classification
        IMPL  '{}/functions/mnist_image_classifier.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

chatgpt_function_query = """CREATE FUNCTION IF NOT EXISTS ChatGPT
        IMPL '{}/functions/chatgpt.py';
        """.format(
    EvaDB_INSTALLATION_DIR
)

yolo8n_query = """CREATE FUNCTION IF NOT EXISTS Yolo
            TYPE  ultralytics
            MODEL 'yolov8n.pt';
        """


def init_builtin_functions(db: EvaDBDatabase, mode: str = "debug") -> None:
    """Load the built-in functions into the system during system bootstrapping.

    The function loads a set of pre-defined function queries based on the `mode` argument.
    In 'debug' mode, the function loads debug functions along with release functions.
    In 'release' mode, only release functions are loaded. In addition, in 'debug' mode,
    the function loads a smaller model to accelerate the test suite time.

    Args:
        mode (str, optional): The mode for loading functions, either 'debug' or 'release'.
        Defaults to 'debug'.

    """

    # Attempting to import torch module
    # It is necessary to import torch before to avoid encountering a
    # "RuntimeError: random_device could not be read"
    # The suspicion is that importing torch prior to decord resolves this issue
    try:
        import torch  # noqa: F401
    except ImportError:
        pass

    # Enable environment variables
    # Relevant for transformer-based models
    import os

    os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"
    os.environ["TOKENIZERS_PARALLELISM"] = "false"

    # list of function queries to load
    queries = [
        mnistcnn_function_query,
        Fastrcnn_function_query,
        ArrayCount_function_query,
        Crop_function_query,
        Open_function_query,
        Similarity_function_query,
        norfair_obj_tracker_query,
        chatgpt_function_query,
        face_detection_function_query,
        # Mvit_function_query,
        Sift_function_query,
        Yolo_function_query,
    ]

    # if mode is 'debug', add debug functions
    if mode == "debug":
        queries.extend(
            [
                DummyObjectDetector_function_query,
                DummyMultiObjectDetector_function_query,
                DummyFeatureExtractor_function_query,
            ]
        )

    # execute each query in the list of function queries
    # ignore exceptions during the bootstrapping phase due to missing packages
    for query in queries:
        try:
            execute_query_fetch_all(
                db, query, do_not_print_exceptions=True, do_not_raise_exceptions=True
            )
        except Exception:
            pass

1	# coding=utf-8
2	# Copyright 2018-2023 EvaDB
3	#
4	# Licensed under the Apache License, Version 2.0 (the "License");
5	# you may not use this file except in compliance with the License.
6	# You may obtain a copy of the License at
7	#
8	# http://www.apache.org/licenses/LICENSE-2.0
9	#
10	# Unless required by applicable law or agreed to in writing, software
11	# distributed under the License is distributed on an "AS IS" BASIS,
12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	# See the License for the specific language governing permissions and
14	# limitations under the License.
15
16	from evadb.configuration.constants import EvaDB_INSTALLATION_DIR	1✔
17	from evadb.database import EvaDBDatabase	1✔
18	from evadb.server.command_handler import execute_query_fetch_all	1✔
19
20	NDARRAY_DIR = "ndarray"	1✔
21	TUTORIALS_DIR = "tutorials"	1✔
22
23	DummyObjectDetector_function_query = """CREATE FUNCTION IF NOT EXISTS DummyObjectDetector	1✔
24	INPUT (Frame_Array NDARRAY INT8(3, ANYDIM, ANYDIM))
25	OUTPUT (label NDARRAY STR(1))
26	TYPE Classification
27	IMPL '{}/../test/util.py';
28	""".format(
29	EvaDB_INSTALLATION_DIR
30	)
31
32	DummyMultiObjectDetector_function_query = """CREATE FUNCTION	1✔
33	IF NOT EXISTS DummyMultiObjectDetector
34	INPUT (Frame_Array NDARRAY INT8(3, ANYDIM, ANYDIM))
35	OUTPUT (labels NDARRAY STR(2))
36	TYPE Classification
37	IMPL '{}/../test/util.py';
38	""".format(
39	EvaDB_INSTALLATION_DIR
40	)
41
42	DummyFeatureExtractor_function_query = """CREATE FUNCTION	1✔
43	IF NOT EXISTS DummyFeatureExtractor
44	INPUT (Frame_Array NDARRAY UINT8(3, ANYDIM, ANYDIM))
45	OUTPUT (features NDARRAY FLOAT32(1, ANYDIM))
46	TYPE Classification
47	IMPL '{}/../test/util.py';
48	""".format(
49	EvaDB_INSTALLATION_DIR
50	)
51
52	fuzzy_function_query = """CREATE FUNCTION IF NOT EXISTS FuzzDistance	1✔
53	INPUT (Input_Array1 NDARRAY ANYTYPE, Input_Array2 NDARRAY ANYTYPE)
54	OUTPUT (distance FLOAT(32, 7))
55	TYPE NdarrayFunction
56	IMPL "{}/functions/{}/fuzzy_join.py";
57	""".format(
58	EvaDB_INSTALLATION_DIR, NDARRAY_DIR
59	)
60
61	ArrayCount_function_query = """CREATE FUNCTION	1✔
62	IF NOT EXISTS ArrayCount
63	INPUT (Input_Array NDARRAY ANYTYPE, Search_Key ANYTYPE)
64	OUTPUT (key_count INTEGER)
65	TYPE NdarrayFunction
66	IMPL "{}/functions/{}/array_count.py";
67	""".format(
68	EvaDB_INSTALLATION_DIR, NDARRAY_DIR
69	)
70
71	Crop_function_query = """CREATE FUNCTION IF NOT EXISTS Crop	1✔
72	INPUT (Frame_Array NDARRAY UINT8(3, ANYDIM, ANYDIM),
73	bboxes NDARRAY FLOAT32(ANYDIM, 4))
74	OUTPUT (Cropped_Frame_Array NDARRAY UINT8(3, ANYDIM, ANYDIM))
75	TYPE NdarrayFunction
76	IMPL "{}/functions/{}/crop.py";
77	""".format(
78	EvaDB_INSTALLATION_DIR, NDARRAY_DIR
79	)
80
81	Open_function_query = """CREATE FUNCTION IF NOT EXISTS Open	1✔
82	INPUT (img_path TEXT(1000))
83	OUTPUT (data NDARRAY UINT8(3, ANYDIM, ANYDIM))
84	TYPE NdarrayFunction
85	IMPL "{}/functions/{}/open.py";
86	""".format(
87	EvaDB_INSTALLATION_DIR, NDARRAY_DIR
88	)
89
90	Similarity_function_query = """CREATE FUNCTION IF NOT EXISTS Similarity	1✔
91	INPUT (Frame_Array_Open NDARRAY UINT8(3, ANYDIM, ANYDIM),
92	Frame_Array_Base NDARRAY UINT8(3, ANYDIM, ANYDIM),
93	Feature_Extractor_Name TEXT(100))
94	OUTPUT (distance FLOAT(32, 7))
95	TYPE NdarrayFunction
96	IMPL "{}/functions/{}/similarity.py";
97	""".format(
98	EvaDB_INSTALLATION_DIR, NDARRAY_DIR
99	)
100
101	Unnest_function_query = """CREATE FUNCTION IF NOT EXISTS Unnest	1✔
102	INPUT (inp NDARRAY ANYTYPE)
103	OUTPUT (out ANYTYPE)
104	TYPE NdarrayFunction
105	IMPL "{}/functions/{}/unnest.py";
106	""".format(
107	EvaDB_INSTALLATION_DIR, NDARRAY_DIR
108	)
109
110	Fastrcnn_function_query = """CREATE FUNCTION IF NOT EXISTS FastRCNNObjectDetector	1✔
111	INPUT (Frame_Array NDARRAY UINT8(3, ANYDIM, ANYDIM))
112	OUTPUT (labels NDARRAY STR(ANYDIM), bboxes NDARRAY FLOAT32(ANYDIM, 4),
113	scores NDARRAY FLOAT32(ANYDIM))
114	TYPE Classification
115	IMPL '{}/functions/fastrcnn_object_detector.py';
116	""".format(
117	EvaDB_INSTALLATION_DIR
118	)
119
120	Yolo_function_query = """CREATE FUNCTION IF NOT EXISTS Yolo	1✔
121	TYPE ultralytics
122	MODEL 'yolov8m.pt';
123	"""
124
125	face_detection_function_query = """CREATE FUNCTION IF NOT EXISTS FaceDetector	1✔
126	INPUT (frame NDARRAY UINT8(3, ANYDIM, ANYDIM))
127	OUTPUT (bboxes NDARRAY FLOAT32(ANYDIM, 4),
128	scores NDARRAY FLOAT32(ANYDIM))
129	TYPE FaceDetection
130	IMPL '{}/functions/face_detector.py';
131	""".format(
132	EvaDB_INSTALLATION_DIR
133	)
134
135	Mvit_function_query = """CREATE FUNCTION IF NOT EXISTS MVITActionRecognition	1✔
136	INPUT (Frame_Array NDARRAY UINT8(3, 16, 224, 224))
137	OUTPUT (labels NDARRAY STR(ANYDIM))
138	TYPE Classification
139	IMPL '{}/functions/mvit_action_recognition.py';
140	""".format(
141	EvaDB_INSTALLATION_DIR
142	)
143
144	Asl_function_query = """CREATE FUNCTION IF NOT EXISTS ASLActionRecognition	1✔
145	INPUT (Frame_Array NDARRAY UINT8(3, 16, 224, 224))
146	OUTPUT (labels NDARRAY STR(ANYDIM))
147	TYPE Classification
148	IMPL '{}/functions/asl_action_recognition.py';
149	""".format(
150	EvaDB_INSTALLATION_DIR
151	)
152
153	norfair_obj_tracker_query = """CREATE FUNCTION IF NOT EXISTS NorFairTracker	1✔
154	IMPL '{}/functions/trackers/nor_fair.py';
155	""".format(
156	EvaDB_INSTALLATION_DIR
157	)
158
159	Sift_function_query = """CREATE FUNCTION IF NOT EXISTS SiftFeatureExtractor	1✔
160	IMPL '{}/functions/sift_feature_extractor.py';
161	""".format(
162	EvaDB_INSTALLATION_DIR
163	)
164
165	Text_feat_function_query = """CREATE FUNCTION IF NOT EXISTS SentenceFeatureExtractor	1✔
166	IMPL '{}/functions/sentence_feature_extractor.py';
167	""".format(
168	EvaDB_INSTALLATION_DIR
169	)
170
171	mnistcnn_function_query = """CREATE FUNCTION IF NOT EXISTS MnistImageClassifier	1✔
172	INPUT (data NDARRAY (3, 28, 28))
173	OUTPUT (label TEXT(2))
174	TYPE Classification
175	IMPL '{}/functions/mnist_image_classifier.py';
176	""".format(
177	EvaDB_INSTALLATION_DIR
178	)
179
180	chatgpt_function_query = """CREATE FUNCTION IF NOT EXISTS ChatGPT	1✔
181	IMPL '{}/functions/chatgpt.py';
182	""".format(
183	EvaDB_INSTALLATION_DIR
184	)
185
186	yolo8n_query = """CREATE FUNCTION IF NOT EXISTS Yolo	1✔
187	TYPE ultralytics
188	MODEL 'yolov8n.pt';
189	"""
190
191
192	def init_builtin_functions(db: EvaDBDatabase, mode: str = "debug") -> None:	1✔
193	"""Load the built-in functions into the system during system bootstrapping.
194
195	The function loads a set of pre-defined function queries based on the `mode` argument.
196	In 'debug' mode, the function loads debug functions along with release functions.
197	In 'release' mode, only release functions are loaded. In addition, in 'debug' mode,
198	the function loads a smaller model to accelerate the test suite time.
199
200	Args:
201	mode (str, optional): The mode for loading functions, either 'debug' or 'release'.
202	Defaults to 'debug'.
203
204	"""
205
206	# Attempting to import torch module
207	# It is necessary to import torch before to avoid encountering a
208	# "RuntimeError: random_device could not be read"
209	# The suspicion is that importing torch prior to decord resolves this issue
210	try:	1✔
211	import torch # noqa: F401	1✔
212	except ImportError:
213	pass
214
215	# Enable environment variables
216	# Relevant for transformer-based models
217	import os	1✔
218
219	os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python"	1✔
220	os.environ["TOKENIZERS_PARALLELISM"] = "false"	1✔
221
222	# list of function queries to load
223	queries = [	1✔
224	mnistcnn_function_query,
225	Fastrcnn_function_query,
226	ArrayCount_function_query,
227	Crop_function_query,
228	Open_function_query,
229	Similarity_function_query,
230	norfair_obj_tracker_query,
231	chatgpt_function_query,
232	face_detection_function_query,
233	# Mvit_function_query,
234	Sift_function_query,
235	Yolo_function_query,
236	]
237
238	# if mode is 'debug', add debug functions
239	if mode == "debug":	1✔
240	queries.extend(	×
241	[
242	DummyObjectDetector_function_query,
243	DummyMultiObjectDetector_function_query,
244	DummyFeatureExtractor_function_query,
245	]
246	)
247
248	# execute each query in the list of function queries
249	# ignore exceptions during the bootstrapping phase due to missing packages
250	for query in queries:	1✔
251	try:	1✔
252	execute_query_fetch_all(
253	db, query, do_not_print_exceptions=True, do_not_raise_exceptions=True
254	)
255	except Exception:
256	pass

georgia-tech-db / eva / #763

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous