No new info detected.

34ed13e7-9a44-4bbe-9e76-127613a9c9fe

Build # 34ed13e7-9a44-4bbe-9e76-127613a9c9fe

Build Type

circle-ci

Committed by jarulraj

Commit Message

checkpoint

Pull Request Pull Request #551: feat: add support for aggregates and toxicity classification

Run Details

88 of 88 new or added lines in 10 files covered. (100.0%)

8200 of 8727 relevant lines covered (93.96%)

0.94 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.25

/eva/optimizer/optimizer_utils.py

# coding=utf-8
# Copyright 2018-2022 EVA
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import List, Tuple

from eva.catalog.catalog_manager import CatalogManager
from eva.expression.abstract_expression import AbstractExpression, ExpressionType
from eva.expression.expression_utils import (
    conjuction_list_to_expression_tree,
    contains_single_column,
    expression_tree_to_conjunction_list,
    get_columns_in_predicate,
    is_simple_predicate,
)
from eva.parser.alias import Alias
from eva.parser.create_statement import ColumnDefinition
from eva.utils.logging_manager import logger


def column_definition_to_udf_io(col_list: List[ColumnDefinition], is_input: bool):
    """Create the UdfIOCatalog object fro each column definition provided

    Arguments:
        col_list(List[ColumnDefinition]): parsed input/output definitions
        is_input(bool): true if input else false
    """
    if isinstance(col_list, ColumnDefinition):
        col_list = [col_list]

    result_list = []
    for col in col_list:
        if col is None:
            logger.error("Empty column definition while creating udf io")
            result_list.append(col)
        result_list.append(
            CatalogManager().udf_io(
                col.name,
                col.type,
                array_type=col.array_type,
                dimensions=col.dimension,
                is_input=is_input,
            )
        )
    return result_list


def extract_equi_join_keys(
    join_predicate: AbstractExpression,
    left_table_aliases: List[str],
    right_table_aliases: List[str],
) -> Tuple[List[AbstractExpression], List[AbstractExpression]]:

    pred_list = expression_tree_to_conjunction_list(join_predicate)
    left_join_keys = []
    right_join_keys = []
    for pred in pred_list:
        if pred.etype == ExpressionType.COMPARE_EQUAL:
            left_child = pred.children[0]
            right_child = pred.children[1]
            # only extract if both are TupleValueExpression
            if (
                left_child.etype == ExpressionType.TUPLE_VALUE
                and right_child.etype == ExpressionType.TUPLE_VALUE
            ):
                if (
                    left_child.table_alias in left_table_aliases
                    and right_child.table_alias in right_table_aliases
                ):
                    left_join_keys.append(left_child)
                    right_join_keys.append(right_child)
                elif (
                    left_child.table_alias in right_table_aliases
                    and right_child.table_alias in left_table_aliases
                ):
                    left_join_keys.append(right_child)
                    right_join_keys.append(left_child)

    return (left_join_keys, right_join_keys)


def extract_pushdown_predicate(
    predicate: AbstractExpression, column_alias: str
) -> Tuple[AbstractExpression, AbstractExpression]:
    """Decompose the predicate into pushdown predicate and remaining predicate

    Args:
        predicate (AbstractExpression): predicate that needs to be decomposed
        column (str): column_alias to extract predicate
    Returns:
        Tuple[AbstractExpression, AbstractExpression]: (pushdown predicate,
        remaining predicate)
    """
    if predicate is None:
        return None, None

    if contains_single_column(predicate, column_alias):
        if is_simple_predicate(predicate):
            return predicate, None

    pushdown_preds = []
    rem_pred = []
    pred_list = expression_tree_to_conjunction_list(predicate)
    for pred in pred_list:
        if contains_single_column(pred, column_alias) and is_simple_predicate(pred):
            pushdown_preds.append(pred)
        else:
            rem_pred.append(pred)

    return (
        conjuction_list_to_expression_tree(pushdown_preds),
        conjuction_list_to_expression_tree(rem_pred),
    )


def extract_pushdown_predicate_for_alias(
    predicate: AbstractExpression, aliases: List[Alias]
):
    """Extract predicate that can be pushed down based on the input aliases.

    Atomic predicates on the table columns that are the subset of the input aliases are
    considered as candidates for pushdown.

    Args:
        predicate (AbstractExpression): input predicate
        aliases (List[str]): aliases for which predicate can be pushed
    """
    if predicate is None:
        return None, None

    pred_list = expression_tree_to_conjunction_list(predicate)
    pushdown_preds = []
    rem_pred = []
    aliases = [alias.alias_name for alias in aliases]
    for pred in pred_list:
        column_aliases = get_columns_in_predicate(pred)
        table_aliases = set([col.split(".")[0] for col in column_aliases])
        if table_aliases.issubset(set(aliases)):
            pushdown_preds.append(pred)
        else:
            rem_pred.append(pred)
    return (
        conjuction_list_to_expression_tree(pushdown_preds),
        conjuction_list_to_expression_tree(rem_pred),
    )

1	# coding=utf-8
2	# Copyright 2018-2022 EVA
3	#
4	# Licensed under the Apache License, Version 2.0 (the "License");
5	# you may not use this file except in compliance with the License.
6	# You may obtain a copy of the License at
7	#
8	# http://www.apache.org/licenses/LICENSE-2.0
9	#
10	# Unless required by applicable law or agreed to in writing, software
11	# distributed under the License is distributed on an "AS IS" BASIS,
12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	# See the License for the specific language governing permissions and
14	# limitations under the License.
15	from typing import List, Tuple	1✔
16
17	from eva.catalog.catalog_manager import CatalogManager	1✔
18	from eva.expression.abstract_expression import AbstractExpression, ExpressionType	1✔
19	from eva.expression.expression_utils import (	1✔
20	conjuction_list_to_expression_tree,
21	contains_single_column,
22	expression_tree_to_conjunction_list,
23	get_columns_in_predicate,
24	is_simple_predicate,
25	)
26	from eva.parser.alias import Alias	1✔
27	from eva.parser.create_statement import ColumnDefinition	1✔
28	from eva.utils.logging_manager import logger	1✔
29
30
31	def column_definition_to_udf_io(col_list: List[ColumnDefinition], is_input: bool):	1✔
32	"""Create the UdfIOCatalog object fro each column definition provided
33
34	Arguments:
35	col_list(List[ColumnDefinition]): parsed input/output definitions
36	is_input(bool): true if input else false
37	"""
38	if isinstance(col_list, ColumnDefinition):	1✔
39	col_list = [col_list]	1✔
40
41	result_list = []	1✔
42	for col in col_list:	1✔
43	if col is None:	1✔
44	logger.error("Empty column definition while creating udf io")	×
45	result_list.append(col)	×
46	result_list.append(	1✔
47	CatalogManager().udf_io(
48	col.name,
49	col.type,
50	array_type=col.array_type,
51	dimensions=col.dimension,
52	is_input=is_input,
53	)
54	)
55	return result_list	1✔
56
57
58	def extract_equi_join_keys(	1✔
59	join_predicate: AbstractExpression,
60	left_table_aliases: List[str],
61	right_table_aliases: List[str],
62	) -> Tuple[List[AbstractExpression], List[AbstractExpression]]:
63
64	pred_list = expression_tree_to_conjunction_list(join_predicate)	1✔
65	left_join_keys = []	1✔
66	right_join_keys = []	1✔
67	for pred in pred_list:	1✔
68	if pred.etype == ExpressionType.COMPARE_EQUAL:	1✔
69	left_child = pred.children[0]	1✔
70	right_child = pred.children[1]	1✔
71	# only extract if both are TupleValueExpression
72	if (	1✔
73	left_child.etype == ExpressionType.TUPLE_VALUE
74	and right_child.etype == ExpressionType.TUPLE_VALUE
75	):
76	if (	1✔
77	left_child.table_alias in left_table_aliases
78	and right_child.table_alias in right_table_aliases
79	):
80	left_join_keys.append(left_child)	×
81	right_join_keys.append(right_child)	×
82	elif (	1✔
83	left_child.table_alias in right_table_aliases
84	and right_child.table_alias in left_table_aliases
85	):
86	left_join_keys.append(right_child)	×
87	right_join_keys.append(left_child)	×
88
89	return (left_join_keys, right_join_keys)	1✔
90
91
92	def extract_pushdown_predicate(	1✔
93	predicate: AbstractExpression, column_alias: str
94	) -> Tuple[AbstractExpression, AbstractExpression]:
95	"""Decompose the predicate into pushdown predicate and remaining predicate
96
97	Args:
98	predicate (AbstractExpression): predicate that needs to be decomposed
99	column (str): column_alias to extract predicate
100	Returns:
101	Tuple[AbstractExpression, AbstractExpression]: (pushdown predicate,
102	remaining predicate)
103	"""
104	if predicate is None:	1✔
105	return None, None	×
106
107	if contains_single_column(predicate, column_alias):	1✔
108	if is_simple_predicate(predicate):	1✔
109	return predicate, None	1✔
110
111	pushdown_preds = []	1✔
112	rem_pred = []	1✔
113	pred_list = expression_tree_to_conjunction_list(predicate)	1✔
114	for pred in pred_list:	1✔
115	if contains_single_column(pred, column_alias) and is_simple_predicate(pred):	1✔
116	pushdown_preds.append(pred)	×
117	else:
118	rem_pred.append(pred)	1✔
119
120	return (	1✔
121	conjuction_list_to_expression_tree(pushdown_preds),
122	conjuction_list_to_expression_tree(rem_pred),
123	)
124
125
126	def extract_pushdown_predicate_for_alias(	1✔
127	predicate: AbstractExpression, aliases: List[Alias]
128	):
129	"""Extract predicate that can be pushed down based on the input aliases.
130
131	Atomic predicates on the table columns that are the subset of the input aliases are
132	considered as candidates for pushdown.
133
134	Args:
135	predicate (AbstractExpression): input predicate
136	aliases (List[str]): aliases for which predicate can be pushed
137	"""
138	if predicate is None:	1✔
139	return None, None	1✔
140
141	pred_list = expression_tree_to_conjunction_list(predicate)	1✔
142	pushdown_preds = []	1✔
143	rem_pred = []	1✔
144	aliases = [alias.alias_name for alias in aliases]	1✔
145	for pred in pred_list:	1✔
146	column_aliases = get_columns_in_predicate(pred)	1✔
147	table_aliases = set([col.split(".")[0] for col in column_aliases])	1✔
148	if table_aliases.issubset(set(aliases)):	1✔
149	pushdown_preds.append(pred)	1✔
150	else:
151	rem_pred.append(pred)	×
152	return (	1✔
153	conjuction_list_to_expression_tree(pushdown_preds),
154	conjuction_list_to_expression_tree(rem_pred),
155	)

georgia-tech-db / eva / 34ed13e7-9a44-4bbe-9e76-127613a9c9fe

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous