#725

pending completion

Build # #725

Build Type

push

circle-ci

Committed by

web-flow

Commit Message

chore: reducing coverage loss (#619)

* adding delete operation

* Adding Insert Statement

* checkpoint

* supporting multiple entries

* implemented for structured data error

* adding parser visitor for delete

* delete executor

* delete plan and rules

* adding delete to plan executor

* change position of LogicalDelete

* logical delimeter

* delete test case

* adding test case

* adding test case

* adding delete testcase

* adding predicate to delete executor

* adding delete to Image storage

* bug fix in delete

* fixing testcase

* adding test case for insert statement

* remove order_by from statement_binder.py

* better variable names, using Batch

* error message for insert

* removing order_by and limit from delete

* remove order_by and limit

* use f-string

* adding to changelog

* removing commit messages

* formatting

* fixing comments

* formatting

* eva insert f32 values

* fix: should delete range

* delete multiple rows

* udf bootstrap

* try to run tests in parallel

* minor fix for ray to work

* ray fixes

---------

Co-authored-by: Aryan-Rajoria <aryanrajoria1003@gmail.com>
Co-authored-by: Gaurav <gaurav21776@gmail.com>

Run Details

236 of 236 new or added lines in 53 files covered. (100.0%)

7003 of 9005 relevant lines covered (77.77%)

0.78 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

28.79

/eva/optimizer/optimizer_utils.py

# coding=utf-8
# Copyright 2018-2022 EVA
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import List, Tuple

from eva.catalog.models.udf_io_catalog import UdfIOCatalogEntry
from eva.catalog.models.udf_metadata_catalog import UdfMetadataCatalogEntry
from eva.expression.abstract_expression import AbstractExpression, ExpressionType
from eva.expression.expression_utils import (
    conjuction_list_to_expression_tree,
    contains_single_column,
    get_columns_in_predicate,
    is_simple_predicate,
    to_conjunction_list,
)
from eva.parser.alias import Alias
from eva.parser.create_statement import ColumnDefinition


def column_definition_to_udf_io(col_list: List[ColumnDefinition], is_input: bool):
    """Create the UdfIOCatalogEntry object fro each column definition provided

    Arguments:
        col_list(List[ColumnDefinition]): parsed input/output definitions
        is_input(bool): true if input else false
    """
    if isinstance(col_list, ColumnDefinition):
        col_list = [col_list]

    result_list = []
    for col in col_list:
        assert col is not None, "Empty column definition while creating udf io"
        result_list.append(
            UdfIOCatalogEntry(
                col.name,
                col.type,
                col.cci.nullable,
                array_type=col.array_type,
                array_dimensions=col.dimension,
                is_input=is_input,
            )
        )
    return result_list


def metadata_definition_to_udf_metadata(metadata_list: List[Tuple[str, str]]):
    """Create the UdfMetadataCatalogEntry object for each metadata definition provided

    Arguments:
        col_list(List[Tuple[str, str]]): parsed metadata definitions
    """
    result_list = []
    for metadata in metadata_list:
        result_list.append(
            UdfMetadataCatalogEntry(
                metadata[0],
                metadata[1],
            )
        )
    return result_list


def extract_equi_join_keys(
    join_predicate: AbstractExpression,
    left_table_aliases: List[Alias],
    right_table_aliases: List[Alias],
) -> Tuple[List[AbstractExpression], List[AbstractExpression]]:
    pred_list = to_conjunction_list(join_predicate)
    left_join_keys = []
    right_join_keys = []
    left_table_alias_strs = [
        left_table_alias.alias_name for left_table_alias in left_table_aliases
    ]
    right_table_alias_strs = [
        right_table_alias.alias_name for right_table_alias in right_table_aliases
    ]

    for pred in pred_list:
        if pred.etype == ExpressionType.COMPARE_EQUAL:
            left_child = pred.children[0]
            right_child = pred.children[1]
            # only extract if both are TupleValueExpression
            if (
                left_child.etype == ExpressionType.TUPLE_VALUE
                and right_child.etype == ExpressionType.TUPLE_VALUE
            ):
                if (
                    left_child.table_alias in left_table_alias_strs
                    and right_child.table_alias in right_table_alias_strs
                ):
                    left_join_keys.append(left_child)
                    right_join_keys.append(right_child)
                elif (
                    left_child.table_alias in right_table_alias_strs
                    and right_child.table_alias in left_table_alias_strs
                ):
                    left_join_keys.append(right_child)
                    right_join_keys.append(left_child)

    return (left_join_keys, right_join_keys)


def extract_pushdown_predicate(
    predicate: AbstractExpression, column_alias: str
) -> Tuple[AbstractExpression, AbstractExpression]:
    """Decompose the predicate into pushdown predicate and remaining predicate

    Args:
        predicate (AbstractExpression): predicate that needs to be decomposed
        column (str): column_alias to extract predicate
    Returns:
        Tuple[AbstractExpression, AbstractExpression]: (pushdown predicate,
        remaining predicate)
    """
    if predicate is None:
        return None, None

    if contains_single_column(predicate, column_alias):
        if is_simple_predicate(predicate):
            return predicate, None

    pushdown_preds = []
    rem_pred = []
    pred_list = to_conjunction_list(predicate)
    for pred in pred_list:
        if contains_single_column(pred, column_alias) and is_simple_predicate(pred):
            pushdown_preds.append(pred)
        else:
            rem_pred.append(pred)

    return (
        conjuction_list_to_expression_tree(pushdown_preds),
        conjuction_list_to_expression_tree(rem_pred),
    )


def extract_pushdown_predicate_for_alias(
    predicate: AbstractExpression, aliases: List[Alias]
):
    """Extract predicate that can be pushed down based on the input aliases.

    Atomic predicates on the table columns that are the subset of the input aliases are
    considered as candidates for pushdown.

    Args:
        predicate (AbstractExpression): input predicate
        aliases (List[str]): aliases for which predicate can be pushed
    """
    if predicate is None:
        return None, None

    pred_list = to_conjunction_list(predicate)
    pushdown_preds = []
    rem_pred = []
    aliases = [alias.alias_name for alias in aliases]
    for pred in pred_list:
        column_aliases = get_columns_in_predicate(pred)
        table_aliases = set([col.split(".")[0] for col in column_aliases])
        if table_aliases.issubset(set(aliases)):
            pushdown_preds.append(pred)
        else:
            rem_pred.append(pred)
    return (
        conjuction_list_to_expression_tree(pushdown_preds),
        conjuction_list_to_expression_tree(rem_pred),
    )

1	# coding=utf-8
2	# Copyright 2018-2022 EVA
3	#
4	# Licensed under the Apache License, Version 2.0 (the "License");
5	# you may not use this file except in compliance with the License.
6	# You may obtain a copy of the License at
7	#
8	# http://www.apache.org/licenses/LICENSE-2.0
9	#
10	# Unless required by applicable law or agreed to in writing, software
11	# distributed under the License is distributed on an "AS IS" BASIS,
12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	# See the License for the specific language governing permissions and
14	# limitations under the License.
15	from typing import List, Tuple	1✔
16
17	from eva.catalog.models.udf_io_catalog import UdfIOCatalogEntry	1✔
18	from eva.catalog.models.udf_metadata_catalog import UdfMetadataCatalogEntry	1✔
19	from eva.expression.abstract_expression import AbstractExpression, ExpressionType	1✔
20	from eva.expression.expression_utils import (	1✔
21	conjuction_list_to_expression_tree,
22	contains_single_column,
23	get_columns_in_predicate,
24	is_simple_predicate,
25	to_conjunction_list,
26	)
27	from eva.parser.alias import Alias	1✔
28	from eva.parser.create_statement import ColumnDefinition	1✔
29
30
31	def column_definition_to_udf_io(col_list: List[ColumnDefinition], is_input: bool):	1✔
32	"""Create the UdfIOCatalogEntry object fro each column definition provided
33
34	Arguments:
35	col_list(List[ColumnDefinition]): parsed input/output definitions
36	is_input(bool): true if input else false
37	"""
38	if isinstance(col_list, ColumnDefinition):	1✔
39	col_list = [col_list]	1✔
40
41	result_list = []	1✔
42	for col in col_list:	1✔
43	assert col is not None, "Empty column definition while creating udf io"	1✔
44	result_list.append(	1✔
45	UdfIOCatalogEntry(
46	col.name,
47	col.type,
48	col.cci.nullable,
49	array_type=col.array_type,
50	array_dimensions=col.dimension,
51	is_input=is_input,
52	)
53	)
54	return result_list	1✔
55
56
57	def metadata_definition_to_udf_metadata(metadata_list: List[Tuple[str, str]]):	1✔
58	"""Create the UdfMetadataCatalogEntry object for each metadata definition provided
59
60	Arguments:
61	col_list(List[Tuple[str, str]]): parsed metadata definitions
62	"""
63	result_list = []	×
64	for metadata in metadata_list:	×
65	result_list.append(	×
66	UdfMetadataCatalogEntry(
67	metadata[0],
68	metadata[1],
69	)
70	)
71	return result_list	×
72
73
74	def extract_equi_join_keys(	1✔
75	join_predicate: AbstractExpression,
76	left_table_aliases: List[Alias],
77	right_table_aliases: List[Alias],
78	) -> Tuple[List[AbstractExpression], List[AbstractExpression]]:
79	pred_list = to_conjunction_list(join_predicate)	×
80	left_join_keys = []	×
81	right_join_keys = []	×
82	left_table_alias_strs = [	×
83	left_table_alias.alias_name for left_table_alias in left_table_aliases
84	]
85	right_table_alias_strs = [	×
86	right_table_alias.alias_name for right_table_alias in right_table_aliases
87	]
88
89	for pred in pred_list:	×
90	if pred.etype == ExpressionType.COMPARE_EQUAL:	×
91	left_child = pred.children[0]	×
92	right_child = pred.children[1]	×
93	# only extract if both are TupleValueExpression
94	if (	×
95	left_child.etype == ExpressionType.TUPLE_VALUE
96	and right_child.etype == ExpressionType.TUPLE_VALUE
97	):
98	if (	×
99	left_child.table_alias in left_table_alias_strs
100	and right_child.table_alias in right_table_alias_strs
101	):
102	left_join_keys.append(left_child)	×
103	right_join_keys.append(right_child)	×
104	elif (	×
105	left_child.table_alias in right_table_alias_strs
106	and right_child.table_alias in left_table_alias_strs
107	):
108	left_join_keys.append(right_child)	×
109	right_join_keys.append(left_child)	×
110
111	return (left_join_keys, right_join_keys)	×
112
113
114	def extract_pushdown_predicate(	1✔
115	predicate: AbstractExpression, column_alias: str
116	) -> Tuple[AbstractExpression, AbstractExpression]:
117	"""Decompose the predicate into pushdown predicate and remaining predicate
118
119	Args:
120	predicate (AbstractExpression): predicate that needs to be decomposed
121	column (str): column_alias to extract predicate
122	Returns:
123	Tuple[AbstractExpression, AbstractExpression]: (pushdown predicate,
124	remaining predicate)
125	"""
126	if predicate is None:	×
127	return None, None	×
128
129	if contains_single_column(predicate, column_alias):	×
130	if is_simple_predicate(predicate):	×
131	return predicate, None	×
132
133	pushdown_preds = []	×
134	rem_pred = []	×
135	pred_list = to_conjunction_list(predicate)	×
136	for pred in pred_list:	×
137	if contains_single_column(pred, column_alias) and is_simple_predicate(pred):	×
138	pushdown_preds.append(pred)	×
139	else:
140	rem_pred.append(pred)	×
141
142	return (	×
143	conjuction_list_to_expression_tree(pushdown_preds),
144	conjuction_list_to_expression_tree(rem_pred),
145	)
146
147
148	def extract_pushdown_predicate_for_alias(	1✔
149	predicate: AbstractExpression, aliases: List[Alias]
150	):
151	"""Extract predicate that can be pushed down based on the input aliases.
152
153	Atomic predicates on the table columns that are the subset of the input aliases are
154	considered as candidates for pushdown.
155
156	Args:
157	predicate (AbstractExpression): input predicate
158	aliases (List[str]): aliases for which predicate can be pushed
159	"""
160	if predicate is None:	×
161	return None, None	×
162
163	pred_list = to_conjunction_list(predicate)	×
164	pushdown_preds = []	×
165	rem_pred = []	×
166	aliases = [alias.alias_name for alias in aliases]	×
167	for pred in pred_list:	×
168	column_aliases = get_columns_in_predicate(pred)	×
169	table_aliases = set([col.split(".")[0] for col in column_aliases])	×
170	if table_aliases.issubset(set(aliases)):	×
171	pushdown_preds.append(pred)	×
172	else:
173	rem_pred.append(pred)	×
174	return (	×
175	conjuction_list_to_expression_tree(pushdown_preds),
176	conjuction_list_to_expression_tree(rem_pred),
177	)

georgia-tech-db / eva / #725

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous