e6161546-9e33-42e7-a2b6-f8fbe6aa8255

Committed 08 Sep 2023 02:22AM UTC coverage: 80.449% (-12.5%) from 92.929%

Build # e6161546-9e33-42e7-a2b6-f8fbe6aa8255

Build Type

push

circle-ci

Committed by

jiashenC

Commit Message

fix lint

Run Details

13 of 13 new or added lines in 8 files covered. (100.0%)

9398 of 11682 relevant lines covered (80.45%)

1.45 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

34.78

/evadb/readers/document/document_reader.py

# coding=utf-8
# Copyright 2018-2023 EvaDB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
from typing import Dict, Iterator

from evadb.catalog.sql_config import ROW_NUM_COLUMN
from evadb.readers.abstract_reader import AbstractReader
from evadb.readers.document.registry import (
    _lazy_import_loader,
    _lazy_import_text_splitter,
)


class DocumentReader(AbstractReader):
    def __init__(self, *args, chunk_params, **kwargs):
        super().__init__(*args, **kwargs)
        self._LOADER_MAPPING = _lazy_import_loader()
        self._splitter_class = _lazy_import_text_splitter()

        # https://github.com/hwchase17/langchain/blob/5b6bbf4ab2a33ed0d33ff5d3cb3979a7edc15682/langchain/text_splitter.py#L570
        # by default we use chunk_size 4000 and overlap 200
        self._chunk_size = chunk_params.get("chunk_size", 4000)
        self._chunk_overlap = chunk_params.get("chunk_overlap", 200)

    def _read(self) -> Iterator[Dict]:
        ext = Path(self.file_url).suffix
        assert ext in self._LOADER_MAPPING, f"File Format {ext} not supported"
        loader_class, loader_args = self._LOADER_MAPPING[ext]
        loader = loader_class(self.file_url, **loader_args)

        # todo: implement out own splitter
        langchain_text_splitter = self._splitter_class(
            chunk_size=self._chunk_size, chunk_overlap=self._chunk_overlap
        )

        row_num = 0
        for data in loader.load():
            for chunk_id, row in enumerate(
                langchain_text_splitter.split_documents([data])
            ):
                yield {
                    "chunk_id": chunk_id,
                    "data": row.page_content,
                    ROW_NUM_COLUMN: row_num,
                }
                row_num += 1

1	# coding=utf-8
2	# Copyright 2018-2023 EvaDB
3	#
4	# Licensed under the Apache License, Version 2.0 (the "License");
5	# you may not use this file except in compliance with the License.
6	# You may obtain a copy of the License at
7	#
8	# http://www.apache.org/licenses/LICENSE-2.0
9	#
10	# Unless required by applicable law or agreed to in writing, software
11	# distributed under the License is distributed on an "AS IS" BASIS,
12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	# See the License for the specific language governing permissions and
14	# limitations under the License.
15	from pathlib import Path	2✔
16	from typing import Dict, Iterator	2✔
17
18	from evadb.catalog.sql_config import ROW_NUM_COLUMN	2✔
19	from evadb.readers.abstract_reader import AbstractReader	2✔
20	from evadb.readers.document.registry import (	2✔
21	_lazy_import_loader,
22	_lazy_import_text_splitter,
23	)
24
25
26	class DocumentReader(AbstractReader):	2✔
27	def __init__(self, args, chunk_params, *kwargs):	2✔
28	super().__init__(args, *kwargs)	×
29	self._LOADER_MAPPING = _lazy_import_loader()	×
30	self._splitter_class = _lazy_import_text_splitter()	×
31
32	# https://github.com/hwchase17/langchain/blob/5b6bbf4ab2a33ed0d33ff5d3cb3979a7edc15682/langchain/text_splitter.py#L570
33	# by default we use chunk_size 4000 and overlap 200
34	self._chunk_size = chunk_params.get("chunk_size", 4000)	×
35	self._chunk_overlap = chunk_params.get("chunk_overlap", 200)	×
36
37	def _read(self) -> Iterator[Dict]:	2✔
38	ext = Path(self.file_url).suffix	×
39	assert ext in self._LOADER_MAPPING, f"File Format {ext} not supported"	×
40	loader_class, loader_args = self._LOADER_MAPPING[ext]	×
41	loader = loader_class(self.file_url, **loader_args)	×
42
43	# todo: implement out own splitter
44	langchain_text_splitter = self._splitter_class(	×
45	chunk_size=self._chunk_size, chunk_overlap=self._chunk_overlap
46	)
47
48	row_num = 0	×
49	for data in loader.load():	×
50	for chunk_id, row in enumerate(	×
51	langchain_text_splitter.split_documents([data])
52	):
53	yield {	×
54	"chunk_id": chunk_id,
55	"data": row.page_content,
56	ROW_NUM_COLUMN: row_num,
57	}
58	row_num += 1	×

georgia-tech-db / eva / e6161546-9e33-42e7-a2b6-f8fbe6aa8255

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous