20a9a0f9-edcc-437c-815d-bcc1a2d22b17

Committed 10 Nov 2023 04:50AM UTC coverage: 66.644% (-10.2%) from 76.812%

Build # 20a9a0f9-edcc-437c-815d-bcc1a2d22b17

Build Type

push

circleci

Committed by

americast

Commit Message

update docs

Run Details

0 of 1 new or added line in 1 file covered. (0.0%)

1354 existing lines in 113 files now uncovered.

8767 of 13155 relevant lines covered (66.64%)

0.67 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

54.43

/evadb/binder/function_expression_binder.py

# coding=utf-8
# Copyright 2018-2023 EvaDB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path

from evadb.binder.binder_utils import (
    BinderError,
    extend_star,
    resolve_alias_table_value_expression,
)
from evadb.binder.statement_binder import StatementBinder
from evadb.catalog.catalog_utils import (
    get_metadata_properties,
    get_video_table_column_definitions,
)
from evadb.configuration.constants import EvaDB_INSTALLATION_DIR
from evadb.executor.execution_context import Context
from evadb.expression.function_expression import FunctionExpression
from evadb.expression.tuple_value_expression import TupleValueExpression
from evadb.parser.types import FunctionType
from evadb.third_party.huggingface.binder import assign_hf_function
from evadb.utils.generic_utils import (
    load_function_class_from_file,
    string_comparison_case_insensitive,
)
from evadb.utils.logging_manager import logger


def bind_func_expr(binder: StatementBinder, node: FunctionExpression):
    # setup the context
    # we read the GPUs from the catalog and populate in the context
    gpus_ids = binder._catalog().get_configuration_catalog_value("gpu_ids")
    node._context = Context(gpus_ids)

    # handle the special case of "extract_object"
    if node.name.upper() == str(FunctionType.EXTRACT_OBJECT):
        handle_bind_extract_object_function(node, binder)
        return

    # Handle Func(*)
    if (
        len(node.children) == 1
        and isinstance(node.children[0], TupleValueExpression)
        and node.children[0].name == "*"
    ):
        node.children = extend_star(binder._binder_context)
    # bind all the children
    for child in node.children:
        binder.bind(child)

    function_obj = binder._catalog().get_function_catalog_entry_by_name(node.name)
    if function_obj is None:
        err_msg = (
            f"Function '{node.name}' does not exist in the catalog. "
            "Please create the function using CREATE FUNCTION command."
        )
        logger.error(err_msg)
        raise BinderError(err_msg)

    if string_comparison_case_insensitive(function_obj.type, "HuggingFace"):
        node.function = assign_hf_function(function_obj)

    elif string_comparison_case_insensitive(function_obj.type, "Ludwig"):
        function_class = load_function_class_from_file(
            function_obj.impl_file_path,
            "GenericLudwigModel",
        )
        function_metadata = get_metadata_properties(function_obj)
        assert "model_path" in function_metadata, "Ludwig models expect 'model_path'."
        node.function = lambda: function_class(
            model_path=function_metadata["model_path"]
        )

    else:
        if function_obj.type == "ultralytics":
            # manually set the impl_path for yolo functions we only handle object
            # detection for now, hopefully this can be generalized
            function_dir = Path(EvaDB_INSTALLATION_DIR) / "functions"
            function_obj.impl_file_path = (
                Path(f"{function_dir}/yolo_object_detector.py").absolute().as_posix()
            )

        # Verify the consistency of the function. If the checksum of the function does not
        # match the one stored in the catalog, an error will be thrown and the user
        # will be asked to register the function again.
        # assert (
        #     get_file_checksum(function_obj.impl_file_path) == function_obj.checksum
        # ), f"""Function file {function_obj.impl_file_path} has been modified from the
        #     registration. Please use DROP FUNCTION to drop it and re-create it # using CREATE FUNCTION."""

        try:
            function_class = load_function_class_from_file(
                function_obj.impl_file_path,
                function_obj.name,
            )
            # certain functions take additional inputs like yolo needs the model_name
            # these arguments are passed by the user as part of metadata
            # we also handle the special case of ChatGPT where we need to send the
            # OpenAPI key as part of the parameter if not provided by the user
            properties = get_metadata_properties(function_obj)
            if string_comparison_case_insensitive(node.name, "CHATGPT"):
                # if the user didn't provide any API_KEY, check if we have one in the catalog
                if "OPENAI_API_KEY" not in properties.keys():
                    openai_key = binder._catalog().get_configuration_catalog_value(
                        "OPENAI_API_KEY"
                    )
                    properties["openai_api_key"] = openai_key

            node.function = lambda: function_class(**properties)
        except Exception as e:
            err_msg = (
                f"{str(e)}. Please verify that the function class name in the "
                "implementation file matches the function name."
            )
            logger.error(err_msg)
            raise BinderError(err_msg)

    node.function_obj = function_obj
    output_objs = binder._catalog().get_function_io_catalog_output_entries(function_obj)
    if node.output:
        for obj in output_objs:
            if obj.name.lower() == node.output:
                node.output_objs = [obj]
        if not node.output_objs:
            err_msg = f"Output {node.output} does not exist for {function_obj.name}."
            logger.error(err_msg)
            raise BinderError(err_msg)
        node.projection_columns = [node.output]
    else:
        node.output_objs = output_objs
        node.projection_columns = [obj.name.lower() for obj in output_objs]

    resolve_alias_table_value_expression(node)


def handle_bind_extract_object_function(
    node: FunctionExpression, binder_context: StatementBinder
):
    """Handles the binding of extract_object function.
        1. Bind the source video data
        2. Create and bind the detector function expression using the provided name.
        3. Create and bind the tracker function expression.
            Its inputs are id, data, output of detector.
        4. Bind the EXTRACT_OBJECT function expression and append the new children.
        5. Handle the alias and populate the outputs of the EXTRACT_OBJECT function
    Args:
        node (FunctionExpression): The function expression representing the extract object operation.
        binder_context (StatementBinder): The binder object used to bind expressions in the statement.
    Raises:
        AssertionError: If the number of children in the `node` is not equal to 3.
    """
    assert (
        len(node.children) == 3
    ), f"Invalid arguments provided to {node}. Example correct usage, (data, Detector, Tracker)"

    # 1. Bind the source video
    video_data = node.children[0]
    binder_context.bind(video_data)

    # 2. Construct the detector
    # convert detector to FunctionExpression before binding
    # eg. YoloV5 -> YoloV5(data)
    detector = FunctionExpression(None, node.children[1].name)
    detector.append_child(video_data.copy())
    binder_context.bind(detector)

    # 3. Construct the tracker
    # convert tracker to FunctionExpression before binding
    # eg. ByteTracker -> ByteTracker(id, data, labels, bboxes, scores)
    tracker = FunctionExpression(None, node.children[2].name)
    # create the video id expression
    columns = get_video_table_column_definitions()
    tracker.append_child(
        TupleValueExpression(name=columns[1].name, table_alias=video_data.table_alias)
    )
    tracker.append_child(video_data.copy())
    binder_context.bind(tracker)
    # append the bound output of detector
    for obj in detector.output_objs:
        col_alias = "{}.{}".format(obj.function_name.lower(), obj.name.lower())
        child = TupleValueExpression(
            obj.name,
            table_alias=obj.function_name.lower(),
            col_object=obj,
            col_alias=col_alias,
        )
        tracker.append_child(child)

    # 4. Bind the EXTRACT_OBJECT expression and append the new children.
    node.children = []
    node.children = [video_data, detector, tracker]

    # 5. assign the outputs of tracker to the output of extract_object
    node.output_objs = tracker.output_objs
    node.projection_columns = [obj.name.lower() for obj in node.output_objs]

    # 5. resolve alias based on the what user provided
    # we assign the alias to tracker as it governs the output of the extract object
    resolve_alias_table_value_expression(node)
    tracker.alias = node.alias

1	# coding=utf-8
2	# Copyright 2018-2023 EvaDB
3	#
4	# Licensed under the Apache License, Version 2.0 (the "License");
5	# you may not use this file except in compliance with the License.
6	# You may obtain a copy of the License at
7	#
8	# http://www.apache.org/licenses/LICENSE-2.0
9	#
10	# Unless required by applicable law or agreed to in writing, software
11	# distributed under the License is distributed on an "AS IS" BASIS,
12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	# See the License for the specific language governing permissions and
14	# limitations under the License.
15	from pathlib import Path	1✔
16
17	from evadb.binder.binder_utils import (	1✔
18	BinderError,
19	extend_star,
20	resolve_alias_table_value_expression,
21	)
22	from evadb.binder.statement_binder import StatementBinder	1✔
23	from evadb.catalog.catalog_utils import (	1✔
24	get_metadata_properties,
25	get_video_table_column_definitions,
26	)
27	from evadb.configuration.constants import EvaDB_INSTALLATION_DIR	1✔
28	from evadb.executor.execution_context import Context	1✔
29	from evadb.expression.function_expression import FunctionExpression	1✔
30	from evadb.expression.tuple_value_expression import TupleValueExpression	1✔
31	from evadb.parser.types import FunctionType	1✔
32	from evadb.third_party.huggingface.binder import assign_hf_function	1✔
33	from evadb.utils.generic_utils import (	1✔
34	load_function_class_from_file,
35	string_comparison_case_insensitive,
36	)
37	from evadb.utils.logging_manager import logger	1✔
38
39
40	def bind_func_expr(binder: StatementBinder, node: FunctionExpression):	1✔
41	# setup the context
42	# we read the GPUs from the catalog and populate in the context
43	gpus_ids = binder._catalog().get_configuration_catalog_value("gpu_ids")	1✔
44	node._context = Context(gpus_ids)	1✔
45
46	# handle the special case of "extract_object"
47	if node.name.upper() == str(FunctionType.EXTRACT_OBJECT):	1✔
48	handle_bind_extract_object_function(node, binder)	×
49	return	×
50
51	# Handle Func(*)
52	if (	1✔
53	len(node.children) == 1
54	and isinstance(node.children[0], TupleValueExpression)
55	and node.children[0].name == "*"
56	):
UNCOV 57	node.children = extend_star(binder._binder_context)	×
58	# bind all the children
59	for child in node.children:	1✔
60	binder.bind(child)	1✔
61
62	function_obj = binder._catalog().get_function_catalog_entry_by_name(node.name)	1✔
63	if function_obj is None:	1✔
UNCOV 64	err_msg = (	×
65	f"Function '{node.name}' does not exist in the catalog. "
66	"Please create the function using CREATE FUNCTION command."
67	)
UNCOV 68	logger.error(err_msg)	×
69	raise BinderError(err_msg)
70
71	if string_comparison_case_insensitive(function_obj.type, "HuggingFace"):	1✔
72	node.function = assign_hf_function(function_obj)	×
73
74	elif string_comparison_case_insensitive(function_obj.type, "Ludwig"):	1✔
75	function_class = load_function_class_from_file(	×
76	function_obj.impl_file_path,
77	"GenericLudwigModel",
78	)
79	function_metadata = get_metadata_properties(function_obj)	×
80	assert "model_path" in function_metadata, "Ludwig models expect 'model_path'."	×
81	node.function = lambda: function_class(	×
82	model_path=function_metadata["model_path"]
83	)
84
85	else:
86	if function_obj.type == "ultralytics":	1✔
87	# manually set the impl_path for yolo functions we only handle object
88	# detection for now, hopefully this can be generalized
89	function_dir = Path(EvaDB_INSTALLATION_DIR) / "functions"	×
90	function_obj.impl_file_path = (	×
91	Path(f"{function_dir}/yolo_object_detector.py").absolute().as_posix()
92	)
93
94	# Verify the consistency of the function. If the checksum of the function does not
95	# match the one stored in the catalog, an error will be thrown and the user
96	# will be asked to register the function again.
97	# assert (
98	# get_file_checksum(function_obj.impl_file_path) == function_obj.checksum
99	# ), f"""Function file {function_obj.impl_file_path} has been modified from the
100	# registration. Please use DROP FUNCTION to drop it and re-create it # using CREATE FUNCTION."""
101
102	try:	1✔
103	function_class = load_function_class_from_file(	1✔
104	function_obj.impl_file_path,
105	function_obj.name,
106	)
107	# certain functions take additional inputs like yolo needs the model_name
108	# these arguments are passed by the user as part of metadata
109	# we also handle the special case of ChatGPT where we need to send the
110	# OpenAPI key as part of the parameter if not provided by the user
111	properties = get_metadata_properties(function_obj)	1✔
112	if string_comparison_case_insensitive(node.name, "CHATGPT"):	1✔
113	# if the user didn't provide any API_KEY, check if we have one in the catalog
114	if "OPENAI_API_KEY" not in properties.keys():	×
115	openai_key = binder._catalog().get_configuration_catalog_value(	×
116	"OPENAI_API_KEY"
117	)
118	properties["openai_api_key"] = openai_key	×
119
120	node.function = lambda: function_class(**properties)	1✔
121	except Exception as e:
122	err_msg = (
123	f"{str(e)}. Please verify that the function class name in the "
124	"implementation file matches the function name."
125	)
126	logger.error(err_msg)
127	raise BinderError(err_msg)
128
129	node.function_obj = function_obj	1✔
130	output_objs = binder._catalog().get_function_io_catalog_output_entries(function_obj)	1✔
131	if node.output:	1✔
132	for obj in output_objs:	1✔
133	if obj.name.lower() == node.output:	1✔
134	node.output_objs = [obj]	1✔
135	if not node.output_objs:	1✔
136	err_msg = f"Output {node.output} does not exist for {function_obj.name}."	1✔
137	logger.error(err_msg)	1✔
138	raise BinderError(err_msg)
139	node.projection_columns = [node.output]	1✔
140	else:
141	node.output_objs = output_objs	1✔
142	node.projection_columns = [obj.name.lower() for obj in output_objs]	1✔
143
144	resolve_alias_table_value_expression(node)	1✔
145
146
147	def handle_bind_extract_object_function(	1✔
148	node: FunctionExpression, binder_context: StatementBinder
149	):
150	"""Handles the binding of extract_object function.
151	1. Bind the source video data
152	2. Create and bind the detector function expression using the provided name.
153	3. Create and bind the tracker function expression.
154	Its inputs are id, data, output of detector.
155	4. Bind the EXTRACT_OBJECT function expression and append the new children.
156	5. Handle the alias and populate the outputs of the EXTRACT_OBJECT function
157	Args:
158	node (FunctionExpression): The function expression representing the extract object operation.
159	binder_context (StatementBinder): The binder object used to bind expressions in the statement.
160	Raises:
161	AssertionError: If the number of children in the `node` is not equal to 3.
162	"""
163	assert (	×
164	len(node.children) == 3
165	), f"Invalid arguments provided to {node}. Example correct usage, (data, Detector, Tracker)"
166
167	# 1. Bind the source video
168	video_data = node.children[0]	×
169	binder_context.bind(video_data)	×
170
171	# 2. Construct the detector
172	# convert detector to FunctionExpression before binding
173	# eg. YoloV5 -> YoloV5(data)
174	detector = FunctionExpression(None, node.children[1].name)	×
175	detector.append_child(video_data.copy())	×
176	binder_context.bind(detector)	×
177
178	# 3. Construct the tracker
179	# convert tracker to FunctionExpression before binding
180	# eg. ByteTracker -> ByteTracker(id, data, labels, bboxes, scores)
181	tracker = FunctionExpression(None, node.children[2].name)	×
182	# create the video id expression
183	columns = get_video_table_column_definitions()	×
184	tracker.append_child(	×
185	TupleValueExpression(name=columns[1].name, table_alias=video_data.table_alias)
186	)
187	tracker.append_child(video_data.copy())	×
188	binder_context.bind(tracker)	×
189	# append the bound output of detector
190	for obj in detector.output_objs:	×
191	col_alias = "{}.{}".format(obj.function_name.lower(), obj.name.lower())	×
192	child = TupleValueExpression(	×
193	obj.name,
194	table_alias=obj.function_name.lower(),
195	col_object=obj,
196	col_alias=col_alias,
197	)
198	tracker.append_child(child)	×
199
200	# 4. Bind the EXTRACT_OBJECT expression and append the new children.
201	node.children = []	×
202	node.children = [video_data, detector, tracker]	×
203
204	# 5. assign the outputs of tracker to the output of extract_object
205	node.output_objs = tracker.output_objs	×
206	node.projection_columns = [obj.name.lower() for obj in node.output_objs]	×
207
208	# 5. resolve alias based on the what user provided
209	# we assign the alias to tracker as it governs the output of the extract object
210	resolve_alias_table_value_expression(node)	×
211	tracker.alias = node.alias	×

georgia-tech-db / eva / 20a9a0f9-edcc-437c-815d-bcc1a2d22b17

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous