#758

Committed 04 Sep 2023 08:37PM UTC coverage: 0.0% (-78.3%) from 78.333%

Build # #758

Build Type

push

circle-ci

Committed by

hershd23

Commit Message

Increased underline length in at line 75 in text_summarization.rst
	modified:   docs/source/benchmarks/text_summarization.rst

Run Details

0 of 11303 relevant lines covered (0.0%)

0.0 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/evadb/readers/document/document_reader.py

# coding=utf-8
# Copyright 2018-2023 EvaDB
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
from typing import Dict, Iterator

from evadb.readers.abstract_reader import AbstractReader
from evadb.readers.document.registry import (
    _lazy_import_loader,
    _lazy_import_text_splitter,
)


class DocumentReader(AbstractReader):
    def __init__(self, *args, chunk_params, **kwargs):
        super().__init__(*args, **kwargs)
        self._LOADER_MAPPING = _lazy_import_loader()
        self._splitter_class = _lazy_import_text_splitter()

        # https://github.com/hwchase17/langchain/blob/5b6bbf4ab2a33ed0d33ff5d3cb3979a7edc15682/langchain/text_splitter.py#L570
        # by default we use chunk_size 4000 and overlap 200
        self._chunk_size = chunk_params.get("chunk_size", 4000)
        self._chunk_overlap = chunk_params.get("chunk_overlap", 200)

    def _read(self) -> Iterator[Dict]:
        ext = Path(self.file_url).suffix
        assert ext in self._LOADER_MAPPING, f"File Format {ext} not supported"
        loader_class, loader_args = self._LOADER_MAPPING[ext]
        loader = loader_class(self.file_url, **loader_args)

        # todo: implement out own splitter
        langchain_text_splitter = self._splitter_class(
            chunk_size=self._chunk_size, chunk_overlap=self._chunk_overlap
        )

        for data in loader.load():
            for chunk_id, row in enumerate(
                langchain_text_splitter.split_documents([data])
            ):
                yield {"chunk_id": chunk_id, "data": row.page_content}

1	# coding=utf-8
2	# Copyright 2018-2023 EvaDB
3	#
4	# Licensed under the Apache License, Version 2.0 (the "License");
5	# you may not use this file except in compliance with the License.
6	# You may obtain a copy of the License at
7	#
8	# http://www.apache.org/licenses/LICENSE-2.0
9	#
10	# Unless required by applicable law or agreed to in writing, software
11	# distributed under the License is distributed on an "AS IS" BASIS,
12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	# See the License for the specific language governing permissions and
14	# limitations under the License.
15	from pathlib import Path	×
16	from typing import Dict, Iterator	×
17
18	from evadb.readers.abstract_reader import AbstractReader	×
19	from evadb.readers.document.registry import (	×
20	_lazy_import_loader,
21	_lazy_import_text_splitter,
22	)
23
24
25	class DocumentReader(AbstractReader):	×
26	def __init__(self, args, chunk_params, *kwargs):	×
27	super().__init__(args, *kwargs)	×
28	self._LOADER_MAPPING = _lazy_import_loader()	×
29	self._splitter_class = _lazy_import_text_splitter()	×
30
31	# https://github.com/hwchase17/langchain/blob/5b6bbf4ab2a33ed0d33ff5d3cb3979a7edc15682/langchain/text_splitter.py#L570
32	# by default we use chunk_size 4000 and overlap 200
33	self._chunk_size = chunk_params.get("chunk_size", 4000)	×
34	self._chunk_overlap = chunk_params.get("chunk_overlap", 200)	×
35
36	def _read(self) -> Iterator[Dict]:	×
37	ext = Path(self.file_url).suffix	×
38	assert ext in self._LOADER_MAPPING, f"File Format {ext} not supported"	×
39	loader_class, loader_args = self._LOADER_MAPPING[ext]	×
40	loader = loader_class(self.file_url, **loader_args)	×
41
42	# todo: implement out own splitter
43	langchain_text_splitter = self._splitter_class(	×
44	chunk_size=self._chunk_size, chunk_overlap=self._chunk_overlap
45	)
46
47	for data in loader.load():	×
48	for chunk_id, row in enumerate(	×
49	langchain_text_splitter.split_documents([data])
50	):
51	yield {"chunk_id": chunk_id, "data": row.page_content}	×

georgia-tech-db / eva / #758

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous