20a9a0f9-edcc-437c-815d-bcc1a2d22b17

Committed 10 Nov 2023 04:50AM UTC coverage: 66.644% (-10.2%) from 76.812%

Build # 20a9a0f9-edcc-437c-815d-bcc1a2d22b17

Build Type

push

circleci

Committed by

americast

Commit Message

update docs

Run Details

0 of 1 new or added line in 1 file covered. (0.0%)

1354 existing lines in 113 files now uncovered.

8767 of 13155 relevant lines covered (66.64%)

0.67 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

50.0

/evadb/storage/document_storage_engine.py

# coding=utf-8
# Copyright 2018-2023 EvaDB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
from typing import Iterator

from evadb.catalog.models.table_catalog import TableCatalogEntry
from evadb.catalog.sql_config import ROW_NUM_COLUMN, ROW_NUM_MAGIC
from evadb.database import EvaDBDatabase
from evadb.models.storage.batch import Batch
from evadb.readers.document.document_reader import DocumentReader
from evadb.storage.abstract_media_storage_engine import AbstractMediaStorageEngine


class DocumentStorageEngine(AbstractMediaStorageEngine):
    def __init__(self, db: EvaDBDatabase):
        super().__init__(db)

    def read(self, table: TableCatalogEntry, chunk_params: dict) -> Iterator[Batch]:
        for doc_files in self._rdb_handler.read(self._get_metadata_table(table), 12):
            for _, (row_id, file_name, _) in doc_files.iterrows():
                system_file_name = self._xform_file_url_to_file_name(file_name)
                doc_file = Path(table.file_url) / system_file_name
                # setting batch_mem_size = 1, we need fix it
                reader = DocumentReader(
                    str(doc_file), batch_mem_size=1, chunk_params=chunk_params
                )
                for batch in reader.read():
                    batch.frames[table.columns[0].name] = row_id
                    batch.frames[table.columns[1].name] = str(file_name)
                    batch.frames[ROW_NUM_COLUMN] = (
                        row_id * ROW_NUM_MAGIC + batch.frames[ROW_NUM_COLUMN]
                    )
                    yield batch

1	# coding=utf-8
2	# Copyright 2018-2023 EvaDB
3	#
4	# Licensed under the Apache License, Version 2.0 (the "License");
5	# you may not use this file except in compliance with the License.
6	# You may obtain a copy of the License at
7	#
8	# http://www.apache.org/licenses/LICENSE-2.0
9	#
10	# Unless required by applicable law or agreed to in writing, software
11	# distributed under the License is distributed on an "AS IS" BASIS,
12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	# See the License for the specific language governing permissions and
14	# limitations under the License.
15	from pathlib import Path	1✔
16	from typing import Iterator	1✔
17
18	from evadb.catalog.models.table_catalog import TableCatalogEntry	1✔
19	from evadb.catalog.sql_config import ROW_NUM_COLUMN, ROW_NUM_MAGIC	1✔
20	from evadb.database import EvaDBDatabase	1✔
21	from evadb.models.storage.batch import Batch	1✔
22	from evadb.readers.document.document_reader import DocumentReader	1✔
23	from evadb.storage.abstract_media_storage_engine import AbstractMediaStorageEngine	1✔
24
25
26	class DocumentStorageEngine(AbstractMediaStorageEngine):	1✔
27	def __init__(self, db: EvaDBDatabase):	1✔
UNCOV 28	super().__init__(db)	×
29
30	def read(self, table: TableCatalogEntry, chunk_params: dict) -> Iterator[Batch]:	1✔
31	for doc_files in self._rdb_handler.read(self._get_metadata_table(table), 12):	×
32	for _, (row_id, file_name, _) in doc_files.iterrows():	×
33	system_file_name = self._xform_file_url_to_file_name(file_name)	×
34	doc_file = Path(table.file_url) / system_file_name	×
35	# setting batch_mem_size = 1, we need fix it
36	reader = DocumentReader(	×
37	str(doc_file), batch_mem_size=1, chunk_params=chunk_params
38	)
39	for batch in reader.read():	×
40	batch.frames[table.columns[0].name] = row_id	×
41	batch.frames[table.columns[1].name] = str(file_name)	×
42	batch.frames[ROW_NUM_COLUMN] = (	×
43	row_id * ROW_NUM_MAGIC + batch.frames[ROW_NUM_COLUMN]
44	)
45	yield batch	×

georgia-tech-db / eva / 20a9a0f9-edcc-437c-815d-bcc1a2d22b17

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous