#758

Committed 04 Sep 2023 08:37PM UTC coverage: 0.0% (-78.3%) from 78.333%

Build # #758

Build Type

push

circle-ci

Committed by

hershd23

Commit Message

Increased underline length in at line 75 in text_summarization.rst
	modified:   docs/source/benchmarks/text_summarization.rst

Run Details

0 of 11303 relevant lines covered (0.0%)

0.0 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/evadb/catalog/models/utils.py

# coding=utf-8
# Copyright 2018-2023 EvaDB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import contextlib
import json
from dataclasses import dataclass, field
from typing import List, Tuple

import sqlalchemy
from sqlalchemy.engine import Engine
from sqlalchemy.types import TypeDecorator
from sqlalchemy_utils import create_database, database_exists

from evadb.catalog.catalog_type import (
    ColumnType,
    NdArrayType,
    TableType,
    VectorStoreType,
)
from evadb.catalog.models.base_model import BaseModel
from evadb.catalog.sql_config import CATALOG_TABLES
from evadb.utils.logging_manager import logger


class TextPickleType(TypeDecorator):
    """Used to handle serialization and deserialization to Text
    https://stackoverflow.com/questions/1378325/python-dicts-in-sqlalchemy
    """

    impl = sqlalchemy.String(1024)

    def process_bind_param(self, value, dialect):
        if value is not None:
            value = json.dumps(value)

        return value

    def process_result_value(self, value, dialect):
        if value is not None:
            value = json.loads(value)
        return value


def init_db(engine: Engine):
    """Create database if doesn't exist and create all tables."""
    if not database_exists(engine.url):
        logger.info("Database does not exist, creating database.")
        create_database(engine.url)
    logger.info("Creating tables")
    BaseModel.metadata.create_all(bind=engine)


def truncate_catalog_tables(engine: Engine):
    """Truncate all the catalog tables"""
    # https://stackoverflow.com/questions/4763472/sqlalchemy-clear-database-content-but-dont-drop-the-schema/5003705#5003705 #noqa
    # reflect to refresh the metadata
    BaseModel.metadata.reflect(bind=engine)
    insp = sqlalchemy.inspect(engine)
    if database_exists(engine.url):
        with contextlib.closing(engine.connect()) as con:
            trans = con.begin()
            for table in reversed(BaseModel.metadata.sorted_tables):
                if insp.has_table(table.name):
                    con.execute(table.delete())
            trans.commit()


def drop_all_tables_except_catalog(engine: Engine):
    """drop all the tables except the catalog"""
    # reflect to refresh the metadata
    BaseModel.metadata.reflect(bind=engine)
    insp = sqlalchemy.inspect(engine)
    if database_exists(engine.url):
        with contextlib.closing(engine.connect()) as con:
            trans = con.begin()
            for table in reversed(BaseModel.metadata.sorted_tables):
                if table.name not in CATALOG_TABLES:
                    if insp.has_table(table.name):
                        table.drop(con)
            trans.commit()


#####
# Dataclass equivalents of catalog entries
# This is done to ensure we don't expose the sqlalchemy dependencies beyond catalog
# service. Further, sqlalchemy does not allow sharing of objects across threads.


@dataclass(unsafe_hash=True)
class UdfCacheCatalogEntry:
    """Dataclass representing an entry in the `UdfCatalog`."""

    name: str
    udf_id: int
    cache_path: str
    args: Tuple[str]
    row_id: int = None
    udf_depends: Tuple[int] = field(compare=False, default_factory=tuple)
    col_depends: Tuple[int] = field(compare=False, default_factory=tuple)


@dataclass(unsafe_hash=True)
class ColumnCatalogEntry:
    """Class decouples the ColumnCatalog from the sqlalchemy."""

    name: str
    type: ColumnType
    is_nullable: bool = False
    array_type: NdArrayType = None
    array_dimensions: Tuple[int] = field(default_factory=tuple)
    table_id: int = None
    table_name: str = None
    row_id: int = None
    dep_caches: List[UdfCacheCatalogEntry] = field(compare=False, default_factory=list)


@dataclass(unsafe_hash=True)
class TableCatalogEntry:
    """Dataclass representing an entry in the ColumnCatalog."""

    name: str
    file_url: str
    table_type: TableType
    identifier_column: str = "id"
    columns: List[ColumnCatalogEntry] = field(compare=False, default_factory=list)
    row_id: int = None


@dataclass(unsafe_hash=True)
class UdfMetadataCatalogEntry:
    """Class decouples the `UdfMetadataCatalog` from the sqlalchemy."""

    key: str
    value: str
    udf_id: int = None
    udf_name: str = None
    row_id: int = None

    def display_format(self):
        return f"{self.udf_name} - {self.key}: {self.value}"


@dataclass(unsafe_hash=True)
class UdfIOCatalogEntry:
    """Class decouples the `UdfIOCatalog` from the sqlalchemy."""

    name: str
    type: ColumnType
    is_nullable: bool = False
    array_type: NdArrayType = None
    array_dimensions: Tuple[int] = None
    is_input: bool = True
    udf_id: int = None
    udf_name: str = None
    row_id: int = None

    def display_format(self):
        data_type = self.type.name
        if self.type == ColumnType.NDARRAY:
            data_type = "{} {} {}".format(
                data_type, self.array_type.name, self.array_dimensions
            )

        return {"name": self.name, "data_type": data_type}


@dataclass(unsafe_hash=True)
class UdfCostCatalogEntry:
    """Dataclass representing an entry in the `UdfCostCatalog`."""

    name: str
    cost: float = None
    udf_id: int = None
    row_id: int = None

    def display_format(self):
        return {"udf_id": self.udf_id, "name": self.name, "cost": self.cost}


@dataclass(unsafe_hash=True)
class IndexCatalogEntry:
    """Dataclass representing an entry in the IndexCatalogEntry."""

    name: str
    save_file_path: str
    type: VectorStoreType
    row_id: int = None
    feat_column_id: int = None
    udf_signature: str = None
    feat_column: ColumnCatalogEntry = None


@dataclass(unsafe_hash=True)
class UdfCatalogEntry:
    """Dataclass representing an entry in the `UdfCatalog`.
    This is done to ensure we don't expose the sqlalchemy dependencies beyond catalog service. Further, sqlalchemy does not allow sharing of objects across threads.
    """

    name: str
    impl_file_path: str
    type: str
    checksum: str
    row_id: int = None
    args: List[UdfIOCatalogEntry] = field(compare=False, default_factory=list)
    outputs: List[UdfIOCatalogEntry] = field(compare=False, default_factory=list)
    metadata: List[UdfMetadataCatalogEntry] = field(compare=False, default_factory=list)
    dep_caches: List[UdfIOCatalogEntry] = field(compare=False, default_factory=list)

    def display_format(self):
        def _to_str(col):
            col_display = col.display_format()
            return f"{col_display['name']} {col_display['data_type']}"

        return {
            "name": self.name,
            "inputs": [_to_str(col) for col in self.args],
            "outputs": [_to_str(col) for col in self.outputs],
            "type": self.type,
            "impl": self.impl_file_path,
            "metadata": self.metadata,
        }


@dataclass(unsafe_hash=True)
class DatabaseCatalogEntry:
    """Dataclass representing an entry in the `DatabaseCatalog`.
    This is done to ensure we don't expose the sqlalchemy dependencies beyond catalog service. Further, sqlalchemy does not allow sharing of objects across threads.
    """

    name: str
    engine: str
    params: dict
    row_id: int = None

    def display_format(self):
        return {
            "name": self.name,
            "engine": self.engine,
            "params": self.params,
        }

1	# coding=utf-8
2	# Copyright 2018-2023 EvaDB
3	#
4	# Licensed under the Apache License, Version 2.0 (the "License");
5	# you may not use this file except in compliance with the License.
6	# You may obtain a copy of the License at
7	#
8	# http://www.apache.org/licenses/LICENSE-2.0
9	#
10	# Unless required by applicable law or agreed to in writing, software
11	# distributed under the License is distributed on an "AS IS" BASIS,
12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	# See the License for the specific language governing permissions and
14	# limitations under the License.
15	import contextlib	×
16	import json	×
17	from dataclasses import dataclass, field	×
18	from typing import List, Tuple	×
19
20	import sqlalchemy	×
21	from sqlalchemy.engine import Engine	×
22	from sqlalchemy.types import TypeDecorator	×
23	from sqlalchemy_utils import create_database, database_exists	×
24
25	from evadb.catalog.catalog_type import (	×
26	ColumnType,
27	NdArrayType,
28	TableType,
29	VectorStoreType,
30	)
31	from evadb.catalog.models.base_model import BaseModel	×
32	from evadb.catalog.sql_config import CATALOG_TABLES	×
33	from evadb.utils.logging_manager import logger	×
34
35
36	class TextPickleType(TypeDecorator):	×
37	"""Used to handle serialization and deserialization to Text
38	https://stackoverflow.com/questions/1378325/python-dicts-in-sqlalchemy
39	"""
40
41	impl = sqlalchemy.String(1024)	×
42
43	def process_bind_param(self, value, dialect):	×
44	if value is not None:	×
45	value = json.dumps(value)	×
46
47	return value	×
48
49	def process_result_value(self, value, dialect):	×
50	if value is not None:	×
51	value = json.loads(value)	×
52	return value	×
53
54
55	def init_db(engine: Engine):	×
56	"""Create database if doesn't exist and create all tables."""
57	if not database_exists(engine.url):	×
58	logger.info("Database does not exist, creating database.")	×
59	create_database(engine.url)	×
60	logger.info("Creating tables")	×
61	BaseModel.metadata.create_all(bind=engine)	×
62
63
64	def truncate_catalog_tables(engine: Engine):	×
65	"""Truncate all the catalog tables"""
66	# https://stackoverflow.com/questions/4763472/sqlalchemy-clear-database-content-but-dont-drop-the-schema/5003705#5003705 #noqa
67	# reflect to refresh the metadata
68	BaseModel.metadata.reflect(bind=engine)	×
69	insp = sqlalchemy.inspect(engine)	×
70	if database_exists(engine.url):	×
71	with contextlib.closing(engine.connect()) as con:	×
72	trans = con.begin()	×
73	for table in reversed(BaseModel.metadata.sorted_tables):	×
74	if insp.has_table(table.name):	×
75	con.execute(table.delete())	×
76	trans.commit()	×
77
78
79	def drop_all_tables_except_catalog(engine: Engine):
80	"""drop all the tables except the catalog"""
81	# reflect to refresh the metadata
82	BaseModel.metadata.reflect(bind=engine)
83	insp = sqlalchemy.inspect(engine)
84	if database_exists(engine.url):
85	with contextlib.closing(engine.connect()) as con:
86	trans = con.begin()
87	for table in reversed(BaseModel.metadata.sorted_tables):
88	if table.name not in CATALOG_TABLES:
89	if insp.has_table(table.name):
90	table.drop(con)
91	trans.commit()
92
93
94	#####
95	# Dataclass equivalents of catalog entries
96	# This is done to ensure we don't expose the sqlalchemy dependencies beyond catalog
97	# service. Further, sqlalchemy does not allow sharing of objects across threads.
98
99
100	@dataclass(unsafe_hash=True)	×
101	class UdfCacheCatalogEntry:	×
102	"""Dataclass representing an entry in the `UdfCatalog`."""
103
104	name: str	×
105	udf_id: int	×
106	cache_path: str	×
107	args: Tuple[str]	×
108	row_id: int = None	×
109	udf_depends: Tuple[int] = field(compare=False, default_factory=tuple)	×
110	col_depends: Tuple[int] = field(compare=False, default_factory=tuple)	×
111
112
113	@dataclass(unsafe_hash=True)	×
114	class ColumnCatalogEntry:	×
115	"""Class decouples the ColumnCatalog from the sqlalchemy."""
116
117	name: str	×
118	type: ColumnType	×
119	is_nullable: bool = False	×
120	array_type: NdArrayType = None	×
121	array_dimensions: Tuple[int] = field(default_factory=tuple)	×
122	table_id: int = None	×
123	table_name: str = None	×
124	row_id: int = None	×
125	dep_caches: List[UdfCacheCatalogEntry] = field(compare=False, default_factory=list)	×
126
127
128	@dataclass(unsafe_hash=True)	×
129	class TableCatalogEntry:	×
130	"""Dataclass representing an entry in the ColumnCatalog."""
131
132	name: str	×
133	file_url: str	×
134	table_type: TableType	×
135	identifier_column: str = "id"	×
136	columns: List[ColumnCatalogEntry] = field(compare=False, default_factory=list)	×
137	row_id: int = None	×
138
139
140	@dataclass(unsafe_hash=True)	×
141	class UdfMetadataCatalogEntry:	×
142	"""Class decouples the `UdfMetadataCatalog` from the sqlalchemy."""
143
144	key: str	×
145	value: str	×
146	udf_id: int = None	×
147	udf_name: str = None	×
148	row_id: int = None	×
149
150	def display_format(self):	×
151	return f"{self.udf_name} - {self.key}: {self.value}"	×
152
153
154	@dataclass(unsafe_hash=True)	×
155	class UdfIOCatalogEntry:	×
156	"""Class decouples the `UdfIOCatalog` from the sqlalchemy."""
157
158	name: str	×
159	type: ColumnType	×
160	is_nullable: bool = False	×
161	array_type: NdArrayType = None	×
162	array_dimensions: Tuple[int] = None	×
163	is_input: bool = True	×
164	udf_id: int = None	×
165	udf_name: str = None	×
166	row_id: int = None	×
167
168	def display_format(self):	×
169	data_type = self.type.name	×
170	if self.type == ColumnType.NDARRAY:	×
171	data_type = "{} {} {}".format(	×
172	data_type, self.array_type.name, self.array_dimensions
173	)
174
175	return {"name": self.name, "data_type": data_type}	×
176
177
178	@dataclass(unsafe_hash=True)	×
179	class UdfCostCatalogEntry:	×
180	"""Dataclass representing an entry in the `UdfCostCatalog`."""
181
182	name: str	×
183	cost: float = None	×
184	udf_id: int = None	×
185	row_id: int = None	×
186
187	def display_format(self):	×
188	return {"udf_id": self.udf_id, "name": self.name, "cost": self.cost}	×
189
190
191	@dataclass(unsafe_hash=True)	×
192	class IndexCatalogEntry:	×
193	"""Dataclass representing an entry in the IndexCatalogEntry."""
194
195	name: str	×
196	save_file_path: str	×
197	type: VectorStoreType	×
198	row_id: int = None	×
199	feat_column_id: int = None	×
200	udf_signature: str = None	×
201	feat_column: ColumnCatalogEntry = None	×
202
203
204	@dataclass(unsafe_hash=True)	×
205	class UdfCatalogEntry:	×
206	"""Dataclass representing an entry in the `UdfCatalog`.
207	This is done to ensure we don't expose the sqlalchemy dependencies beyond catalog service. Further, sqlalchemy does not allow sharing of objects across threads.
208	"""
209
210	name: str	×
211	impl_file_path: str	×
212	type: str	×
213	checksum: str	×
214	row_id: int = None	×
215	args: List[UdfIOCatalogEntry] = field(compare=False, default_factory=list)	×
216	outputs: List[UdfIOCatalogEntry] = field(compare=False, default_factory=list)	×
217	metadata: List[UdfMetadataCatalogEntry] = field(compare=False, default_factory=list)	×
218	dep_caches: List[UdfIOCatalogEntry] = field(compare=False, default_factory=list)	×
219
220	def display_format(self):	×
221	def _to_str(col):	×
222	col_display = col.display_format()	×
223	return f"{col_display['name']} {col_display['data_type']}"	×
224
225	return {	×
226	"name": self.name,
227	"inputs": [_to_str(col) for col in self.args],
228	"outputs": [_to_str(col) for col in self.outputs],
229	"type": self.type,
230	"impl": self.impl_file_path,
231	"metadata": self.metadata,
232	}
233
234
235	@dataclass(unsafe_hash=True)	×
236	class DatabaseCatalogEntry:	×
237	"""Dataclass representing an entry in the `DatabaseCatalog`.
238	This is done to ensure we don't expose the sqlalchemy dependencies beyond catalog service. Further, sqlalchemy does not allow sharing of objects across threads.
239	"""
240
241	name: str	×
242	engine: str	×
243	params: dict	×
244	row_id: int = None	×
245
246	def display_format(self):	×
247	return {	×
248	"name": self.name,
249	"engine": self.engine,
250	"params": self.params,
251	}

georgia-tech-db / eva / #758

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous