524684088

Build Type

github-actions

Committed by GitHub

Commit Message

<a href="https://github.com/PyThaiNLP/pythainlp/commit/<a class=hub.com/PyThaiNLP/pythainlp/commit/<a class="double-link" href="https://git"><a class=hub.com/PyThaiNLP/pythainlp/commit/<a class="double-link" href="https://git"><a class=hub.com/PyThaiNLP/pythainlp/commit/0780a3d6e98c7ce3c1fc56c75293d23a55ea50cd">0780a3d6e">&lt;a href=&quot;https://github.com/PyThaiNLP/pythainlp/commit/</a><a class="double-link" href="https://github.com/PyThaiNLP/pythainlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/PyThaiNLP/pythainlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/PyThaiNLP/pythainlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/PyThaiNLP/pythainlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/PyThaiNLP/pythainlp/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://git">&lt;a class=</a>hub.com/PyThaiNLP/pythainlp/commit/0780a3d6e98c7ce3c1fc56c75293d23a55ea50cd">0780a3d6e</a><a href="https://github.com/PyThaiNLP/pythainlp/commit/0780a3d6e98c7ce3c1fc56c75293d23a55ea50cd">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/PyThaiNLP/pythainlp/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/PyThaiNLP/pythainlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/PyThaiNLP/pythainlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/PyThaiNLP/pythainlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/PyThaiNLP/pythainlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/PyThaiNLP/pythainlp/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/PyThaiNLP/pythainlp/commit/0780a3d6e98c7ce3c1fc56c75293d23a55ea50cd&quot;&gt;0780a3d6e&lt;/a&gt;&lt;a href=&quot;https://github.com/PyThaiNLP/pythainlp/commi... (continued)

Pull Request Pull Request #532: Update import

Run Details

20 of 20 new or added lines in 4 files covered. (100.0%)

3330 of 3474 relevant lines covered (95.85%)

0.96 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

97.98

/pythainlp/tokenize/core.py

# -*- coding: utf-8 -*-
"""
Tokenizer generic functions
"""
import re
from typing import Iterable, List, Union

from pythainlp.tokenize import (
    DEFAULT_SENT_TOKENIZE_ENGINE,
    DEFAULT_SUBWORD_TOKENIZE_ENGINE,
    DEFAULT_SYLLABLE_DICT_TRIE,
    DEFAULT_SYLLABLE_TOKENIZE_ENGINE,
    DEFAULT_WORD_DICT_TRIE,
    DEFAULT_WORD_TOKENIZE_ENGINE,
)
from pythainlp.util.trie import Trie, dict_trie


def clause_tokenize(doc: List[str]) -> List[List[str]]:
    """
    Clause tokenizer. (or Clause segmentation)

    Tokenizes running word list into list of clauses (list of strings).
    split by CRF trained on LST20 Corpus.

    :param str doc: word list to be clause
    :return: list of claues
    :rtype: list[list[str]]

    :Example:

        from pythainlp.tokenize import clause_tokenize

        clause_tokenize(["ฉัน","นอน","และ","คุณ","เล่น","มือถือ","ส่วน","น้อง","เขียน","โปรแกรม"])
        [['ฉัน', 'นอน'],
        ['และ', 'คุณ', 'เล่น', 'มือถือ'],
        ['ส่วน', 'น้อง', 'เขียน', 'โปรแกรม']]
    """
    if not doc or not isinstance(doc, str):
        return []

    from pythainlp.tokenize.crfcls import segment

    return segment(doc)


def word_tokenize(
    text: str,
    custom_dict: Trie = None,
    engine: str = DEFAULT_WORD_TOKENIZE_ENGINE,
    keep_whitespace: bool = True,
) -> List[str]:
    """
    Word tokenizer.

    Tokenizes running text into words (list of strings).

    :param str text: text to be tokenized
    :param str engine: name of the tokenizer to be used
    :param pythainlp.util.Trie custom_dict: dictionary trie
    :param bool keep_whitespace: True to keep whitespaces, a common mark
                                 for end of phrase in Thai.
                                 Otherwise, whitespaces are omitted.
    :return: list of words
    :rtype: list[str]
    **Options for engine**
        * *newmm* (default) - dictionary-based, Maximum Matching +
          Thai Character Cluster
        * *newmm-safe* - newmm, with a mechanism to help avoid long
          processing time for text with continuous ambiguous breaking points
        * *longest* - dictionary-based, Longest Matching
        * *icu* - wrapper for ICU (International Components for Unicode,
          using PyICU), dictionary-based
        * *attacut* - wrapper for
          `AttaCut <https://github.com/PyThaiNLP/attacut>`_.,
          learning-based approach
        * *deepcut* - wrapper for
          `DeepCut <https://github.com/rkcosmos/deepcut>`_,
          learning-based approach
        * *nercut* - Dictionary-based maximal matching word segmentation,
          constrained with Thai Character Cluster (TCC) boundaries,
          and combining tokens that are parts of the same named-entity.

    :Note:
        - The parameter **custom_dict** can be provided as an argument \
          only for *newmm*, *longest*, and *attacut* engine.
    :Example:

    Tokenize text with different tokenizer::

        from pythainlp.tokenize import word_tokenize

        text = "โอเคบ่พวกเรารักภาษาบ้านเกิด"

        word_tokenize(text, engine="newmm")
        # output: ['โอเค', 'บ่', 'พวกเรา', 'รัก', 'ภาษา', 'บ้านเกิด']

        word_tokenize(text, engine='attacut')
        # output: ['โอเค', 'บ่', 'พวกเรา', 'รัก', 'ภาษา', 'บ้านเกิด']

    Tokenize text by omiting whitespaces::

        text = "วรรณกรรม ภาพวาด และการแสดงงิ้ว "

        word_tokenize(text, engine="newmm")
        # output:
        # ['วรรณกรรม', ' ', 'ภาพวาด', ' ', 'และ', 'การแสดง', 'งิ้ว', ' ']

        word_tokenize(text, engine="newmm", keep_whitespace=False)
        # output: ['วรรณกรรม', 'ภาพวาด', 'และ', 'การแสดง', 'งิ้ว']

    Tokenize with default and custom dictionary::

        from pythainlp.corpus.common import thai_words
        from pythainlp.tokenize import dict_trie

        text = 'ชินโซ อาเบะ เกิด 21 กันยายน'

        word_tokenize(text, engine="newmm")
        # output:
        # ['ชิน', 'โซ', ' ', 'อา', 'เบะ', ' ',
        #  'เกิด', ' ', '21', ' ', 'กันยายน']

        custom_dict_japanese_name = set(thai_words()
        custom_dict_japanese_name.add('ชินโซ')
        custom_dict_japanese_name.add('อาเบะ')

        trie = dict_trie(dict_source=custom_dict_japanese_name)

        word_tokenize(text, engine="newmm", custom_dict=trie))
        # output:
        # ['ชินโซ', ' ', 'อาเบะ',
        #   ' ', 'เกิด', ' ', '21', ' ', 'กันยายน']
    """
    if not text or not isinstance(text, str):
        return []

    segments = []

    if engine == "newmm" or engine == "onecut":
        from pythainlp.tokenize.newmm import segment

        segments = segment(text, custom_dict)
    elif engine == "newmm-safe":
        from pythainlp.tokenize.newmm import segment

        segments = segment(text, custom_dict, safe_mode=True)
    elif engine == "attacut":
        from pythainlp.tokenize.attacut import segment

        segments = segment(text)
    elif engine == "longest":
        from pythainlp.tokenize.longest import segment

        segments = segment(text, custom_dict)
    elif engine == "mm" or engine == "multi_cut":
        from pythainlp.tokenize.multi_cut import segment

        segments = segment(text, custom_dict)
    elif engine == "deepcut":  # deepcut can optionally use dictionary
        from pythainlp.tokenize.deepcut import segment

        if custom_dict:
            custom_dict = list(custom_dict)
            segments = segment(text, custom_dict)
        else:
            segments = segment(text)
    elif engine == "icu":
        from pythainlp.tokenize.pyicu import segment

        segments = segment(text)
    elif engine == "nercut":
        from .nercut import segment

        segments = segment(text)
    else:
        raise ValueError(
            f"""Tokenizer \"{engine}\" not found.
            It might be a typo; if not, please consult our document."""
        )

    if not keep_whitespace:
        segments = [token.strip(" ") for token in segments if token.strip(" ")]

    return segments


def sent_tokenize(
    text: str,
    engine: str = DEFAULT_SENT_TOKENIZE_ENGINE,
    keep_whitespace: bool = True,
) -> List[str]:
    """
    Sentence tokenizer.

    Tokenizes running text into "sentences"

    :param str text: the text to be tokenized
    :param str engine: choose among *'crfcut'*, *'whitespace'*, \
    *'whitespace+newline'*
    :return: list of splited sentences
    :rtype: list[str]
    **Options for engine**
        * *crfcut* - (default) split by CRF trained on TED dataset
        * *whitespace+newline* - split by whitespaces and newline.
        * *whitespace* - split by whitespaces. Specifiaclly, with \
                         :class:`regex` pattern  ``r" +"``
    :Example:

    Split the text based on *whitespace*::

        from pythainlp.tokenize import sent_tokenize

        sentence_1 = "ฉันไปประชุมเมื่อวันที่ 11 มีนาคม"
        sentence_2 = "ข้าราชการได้รับการหมุนเวียนเป็นระยะ \\
        และได้รับมอบหมายให้ประจำในระดับภูมิภาค"

        sent_tokenize(sentence_1, engine="whitespace")
        # output: ['ฉันไปประชุมเมื่อวันที่', '11', 'มีนาคม']

        sent_tokenize(sentence_2, engine="whitespace")
        # output: ['ข้าราชการได้รับการหมุนเวียนเป็นระยะ',
        #   '\\nและได้รับมอบหมายให้ประจำในระดับภูมิภาค']

    Split the text based on *whitespace* and *newline*::

        sentence_1 = "ฉันไปประชุมเมื่อวันที่ 11 มีนาคม"
        sentence_2 = "ข้าราชการได้รับการหมุนเวียนเป็นระยะ \\
        และได้รับมอบหมายให้ประจำในระดับภูมิภาค"

        sent_tokenize(sentence_1, engine="whitespace+newline")
        # output: ['ฉันไปประชุมเมื่อวันที่', '11', 'มีนาคม']
        sent_tokenize(sentence_2, engine="whitespace+newline")
        # output: ['ข้าราชการได้รับการหมุนเวียนเป็นระยะ',
        '\\nและได้รับมอบหมายให้ประจำในระดับภูมิภาค']

    Split the text using CRF trained on TED dataset::

        sentence_1 = "ฉันไปประชุมเมื่อวันที่ 11 มีนาคม"
        sentence_2 = "ข้าราชการได้รับการหมุนเวียนเป็นระยะ \\
        และเขาได้รับมอบหมายให้ประจำในระดับภูมิภาค"

        sent_tokenize(sentence_1, engine="crfcut")
        # output: ['ฉันไปประชุมเมื่อวันที่ 11 มีนาคม']

        sent_tokenize(sentence_2, engine="crfcut")
        # output: ['ข้าราชการได้รับการหมุนเวียนเป็นระยะ ',
        'และเขาได้รับมอบหมายให้ประจำในระดับภูมิภาค']
    """

    if not text or not isinstance(text, str):
        return []

    segments = []

    if engine == "crfcut":
        from pythainlp.tokenize.crfcut import segment

        segments = segment(text)
    elif engine == "whitespace":
        segments = re.split(r" +", text, re.U)
    elif engine == "whitespace+newline":
        segments = text.split()
    else:
        raise ValueError(
            f"""Tokenizer \"{engine}\" not found.
            It might be a typo; if not, please consult our document."""
        )

    if not keep_whitespace:
        segments = [token.strip(" ") for token in segments if token.strip(" ")]

    return segments


def subword_tokenize(
    text: str,
    engine: str = DEFAULT_SUBWORD_TOKENIZE_ENGINE,
    keep_whitespace: bool = True,
) -> List[str]:
    """
    Subword tokenizer. Can be smaller than syllable.

    Tokenizes text into inseparable units of
    Thai contiguous characters namely
    `Thai Character Clusters (TCCs) \
    <https://www.researchgate.net/publication/2853284_Character_Cluster_Based_Thai_Information_Retrieval>`_
    TCCs are the units based on Thai spelling feature that could not be
    separated any character further such as   'ก็', 'จะ', 'ไม่', and 'ฝา'.
    If the following units are separated, they could not be spelled out.
    This function apply the TCC rules to tokenizes the text into
    the smallest units.

    For example, the word 'ขนมชั้น' would be tokenized
    into 'ข', 'น', 'ม', and 'ชั้น'.

    :param str text: text to be tokenized
    :param str engine: the name subword tokenizer
    :return: list of subwords
    :rtype: list[str]
    **Options for engine**
        * *tcc* (default) -  Thai Character Cluster (Theeramunkong et al. 2000)
        * *etcc* - Enhanced Thai Character Cluster (Inrut et al. 2001)

    :Example:

    Tokenize text into subword based on *tcc*::

        from pythainlp.tokenize import subword_tokenize

        text_1 = "ยุคเริ่มแรกของ ราชวงศ์หมิง"
        text_2 = "ความแปลกแยกและพัฒนาการ"

        subword_tokenize(text_1, engine='tcc')
        # output: ['ยุ', 'ค', 'เริ่ม', 'แร', 'ก',
        #   'ข', 'อ', 'ง', ' ', 'รา', 'ช', 'ว', 'ง',
        #   'ศ', '์', 'ห', 'มิ', 'ง']

        subword_tokenize(text_2, engine='tcc')
        # output: ['ค', 'วา', 'ม', 'แป', 'ล', 'ก', 'แย', 'ก',
        'และ', 'พัฒ','นา', 'กา', 'ร']

    Tokenize text into subword based on *etcc* **(Work In Progress)**::

        text_1 = "ยุคเริ่มแรกของ ราชวงศ์หมิง"
        text_2 = "ความแปลกแยกและพัฒนาการ"

        subword_tokenize(text_1, engine='etcc')
        # output: ['ยุคเริ่มแรกของ ราชวงศ์หมิง']

        subword_tokenize(text_2, engine='etcc')
        # output: ['ความแปลกแยกและ', 'พัฒ', 'นาการ']
    """
    if not text or not isinstance(text, str):
        return []

    if engine == "tcc":
        from pythainlp.tokenize.tcc import segment
    elif engine == "etcc":
        from pythainlp.tokenize.etcc import segment
    else:
        raise ValueError(
            f"""Tokenizer \"{engine}\" not found.
            It might be a typo; if not, please consult our document."""
        )

    segments = segment(text)

    if not keep_whitespace:
        segments = [token.strip(" ") for token in segments if token.strip(" ")]

    return segments


def syllable_tokenize(
    text: str,
    engine: str = DEFAULT_SYLLABLE_TOKENIZE_ENGINE,
    keep_whitespace: bool = True,
) -> List[str]:
    """
    Syllable tokenizer.

    Tokenizes text into syllable (Thai: พยางค์), a unit of
    pronunciation having one vowel sound.  For example, the word 'รถไฟ'
    contains two syallbles including 'รถ', and 'ไฟ'.

    Under the hood, this function uses :func:`pythainlp.tokenize.word_tokenize`
    with *newmm* as a tokenizer. The function tokenize the text with
    the dictionary of Thai words from
    :func:`pythainlp.corpus.common.thai_words`
    and then dictionary of Thai syllable from
    :func:`pythainlp.corpus.common.thai_syllables`.
    As a result, only syllables are obtained.

    :param str text: input string to be tokenized
    :param str engine: name of the syllable tokenizer
    :return: list of syllables where whitespaces in the text **are included**
    :rtype: list[str]
    **Options for engine**
        * *dict* (default) - newmm word tokenizer with a syllable dictionary
        * *ssg* - CRF syllable segmenter for Thai
    :Example::
    ::

        from pythainlp.tokenize import syllable_tokenize

        text = 'รถไฟสมัยใหม่จะใช้กำลังจากหัวรถจักรดีเซล หรือจากไฟฟ้า'
        syllable_tokenize(text)
        ['รถ', 'ไฟ', 'สมัย', 'ใหม่', 'ใช้', 'กำ', 'ลัง', 'จาก', 'หัว',
        'รถ', 'จักร', 'ดี', 'เซล', ' ', 'หรือ', 'จาก', 'ไฟ', 'ฟ้า']
    """

    if not text or not isinstance(text, str):
        return []

    segments = []

    if engine == "dict" or engine == "default":  # use syllable dictionary
        words = word_tokenize(text)
        for word in words:
            segments.extend(
                word_tokenize(
                    text=word, custom_dict=DEFAULT_SYLLABLE_DICT_TRIE
                )
            )
    elif engine == "ssg":
        from pythainlp.tokenize.ssg import segment

        segments = segment(text)
    else:
        raise ValueError(
            f"""Tokenizer \"{engine}\" not found.
            It might be a typo; if not, please consult our document."""
        )

    if not keep_whitespace:
        segments = [token.strip(" ") for token in segments if token.strip(" ")]

    return segments


class Tokenizer:
    """
    Tokenizer class, for a custom tokenizer.

    This class allows users to pre-define custom dictionary along with
    tokenizer and encapsulate them into one single object.
    It is an wrapper for both two functions including
    :func:`pythainlp.tokenize.word_tokenize`,
    and :func:`pythainlp.util.dict_trie`

    :Example:

    Tokenizer object instantiated with :class:`pythainlp.util.Trie`::

        from pythainlp.tokenize import Tokenizer
        from pythainlp.corpus.common import thai_words
        from pythainlp.util import dict_trie

        custom_words_list = set(thai_words())
        custom_words_list.add('อะเฟเซีย')
        custom_words_list.add('Aphasia')
        trie = dict_trie(dict_source=custom_words_list)

        text = "อะเฟเซีย (Aphasia*) เป็นอาการผิดปกติของการพูด"
        _tokenizer = Tokenizer(custom_dict=trie, engine='newmm')
        # output: ['อะเฟเซีย', ' ', '(', 'Aphasia', ')', ' ', 'เป็น', 'อาการ',
        'ผิดปกติ', 'ของ', 'การ', 'พูด']

    Tokenizer object instantiated with a list of words::

        text = "อะเฟเซีย (Aphasia) เป็นอาการผิดปกติของการพูด"
        _tokenizer = Tokenizer(custom_dict=list(thai_words()), engine='newmm')
        _tokenizer.word_tokenize(text)
        # output:
        # ['อะ', 'เฟเซีย', ' ', '(', 'Aphasia', ')', ' ', 'เป็น', 'อาการ',
        #   'ผิดปกติ', 'ของ', 'การ', 'พูด']

    Tokenizer object instantiated with a file path containing list of
    word separated with *newline* and explicitly set a new tokenizer
    after initiation::

        PATH_TO_CUSTOM_DICTIONARY = './custom_dictionary.txtt'

        # write a file
        with open(PATH_TO_CUSTOM_DICTIONARY, 'w', encoding='utf-8') as f:
            f.write('อะเฟเซีย\\nAphasia\\nผิด\\nปกติ')

        text = "อะเฟเซีย (Aphasia) เป็นอาการผิดปกติของการพูด"

        # initate an object from file with `attacut` as tokenizer
        _tokenizer = Tokenizer(custom_dict=PATH_TO_CUSTOM_DICTIONARY, \\
            engine='attacut')

        _tokenizer.word_tokenize(text)
        # output:
        # ['อะเฟเซีย', ' ', '(', 'Aphasia', ')', ' ', 'เป็น', 'อาการ', 'ผิด',
        #   'ปกติ', 'ของ', 'การ', 'พูด']

        # change tokenizer to `newmm`
        _tokenizer.set_tokenizer_engine(engine='newmm')
        _tokenizer.word_tokenize(text)
        # output:
        # ['อะเฟเซีย', ' ', '(', 'Aphasia', ')', ' ', 'เป็นอาการ', 'ผิด',
        #   'ปกติ', 'ของการพูด']
    """

    def __init__(
        self,
        custom_dict: Union[Trie, Iterable[str], str] = None,
        engine: str = "newmm",
        keep_whitespace: bool = True,
    ):
        """
        Initialize tokenizer object.

        :param str custom_dict: a file path, a list of vocaburaies* to be
                    used to create a trie, or an instantiated
                    :class:`pythainlp.util.Trie` object.
        :param str engine: choose between different options of engine to token
                           (i.e.  *newmm*, *longest*, *attacut*)
        :param bool keep_whitespace: True to keep whitespaces, a common mark
                                    for end of phrase in Thai
        """
        self.__trie_dict = None
        if custom_dict:
            self.__trie_dict = dict_trie(custom_dict)
        else:
            self.__trie_dict = DEFAULT_WORD_DICT_TRIE
        self.__engine = engine
        self.__keep_whitespace = keep_whitespace

    def word_tokenize(self, text: str) -> List[str]:
        """
        Main tokenization function.

        :param str text: text to be tokenized
        :return: list of words, tokenized from the text
        :rtype: list[str]
        """
        return word_tokenize(
            text,
            custom_dict=self.__trie_dict,
            engine=self.__engine,
            keep_whitespace=self.__keep_whitespace,
        )

    def set_tokenize_engine(self, engine: str) -> None:
        """
        Set the tokenizer's engine.

        :param str engine: choose between different options of engine to token
                           (i.e. *newmm*, *longest*, *attacut*)
        """
        self.__engine = engine

1	# -- coding: utf-8 --
2	"""	1✔
3	Tokenizer generic functions
4	"""
5	import re	1✔
6	from typing import Iterable, List, Union	1✔
7
8	from pythainlp.tokenize import (	1✔
9	DEFAULT_SENT_TOKENIZE_ENGINE,
10	DEFAULT_SUBWORD_TOKENIZE_ENGINE,
11	DEFAULT_SYLLABLE_DICT_TRIE,
12	DEFAULT_SYLLABLE_TOKENIZE_ENGINE,
13	DEFAULT_WORD_DICT_TRIE,
14	DEFAULT_WORD_TOKENIZE_ENGINE,
15	)
16	from pythainlp.util.trie import Trie, dict_trie	1✔
17
18
19	def clause_tokenize(doc: List[str]) -> List[List[str]]:	1✔
20	"""
21	Clause tokenizer. (or Clause segmentation)
22
23	Tokenizes running word list into list of clauses (list of strings).
24	split by CRF trained on LST20 Corpus.
25
26	:param str doc: word list to be clause
27	:return: list of claues
28	:rtype: list[list[str]]
29
30	:Example:
31
32	from pythainlp.tokenize import clause_tokenize
33
34	clause_tokenize(["ฉัน","นอน","และ","คุณ","เล่น","มือถือ","ส่วน","น้อง","เขียน","โปรแกรม"])
35	[['ฉัน', 'นอน'],
36	['และ', 'คุณ', 'เล่น', 'มือถือ'],
37	['ส่วน', 'น้อง', 'เขียน', 'โปรแกรม']]
38	"""
39	if not doc or not isinstance(doc, str):	1✔
40	return []	1✔
41
42	from pythainlp.tokenize.crfcls import segment	×
43
44	return segment(doc)	×
45
46
47	def word_tokenize(	1✔
48	text: str,
49	custom_dict: Trie = None,
50	engine: str = DEFAULT_WORD_TOKENIZE_ENGINE,
51	keep_whitespace: bool = True,
52	) -> List[str]:
53	"""
54	Word tokenizer.
55
56	Tokenizes running text into words (list of strings).
57
58	:param str text: text to be tokenized
59	:param str engine: name of the tokenizer to be used
60	:param pythainlp.util.Trie custom_dict: dictionary trie
61	:param bool keep_whitespace: True to keep whitespaces, a common mark
62	for end of phrase in Thai.
63	Otherwise, whitespaces are omitted.
64	:return: list of words
65	:rtype: list[str]
66	Options for engine
67	* newmm (default) - dictionary-based, Maximum Matching +
68	Thai Character Cluster
69	* newmm-safe - newmm, with a mechanism to help avoid long
70	processing time for text with continuous ambiguous breaking points
71	* longest - dictionary-based, Longest Matching
72	* icu - wrapper for ICU (International Components for Unicode,
73	using PyICU), dictionary-based
74	* attacut - wrapper for
75	`AttaCut <https://github.com/PyThaiNLP/attacut>`_.,
76	learning-based approach
77	* deepcut - wrapper for
78	`DeepCut <https://github.com/rkcosmos/deepcut>`_,
79	learning-based approach
80	* nercut - Dictionary-based maximal matching word segmentation,
81	constrained with Thai Character Cluster (TCC) boundaries,
82	and combining tokens that are parts of the same named-entity.
83
84	:Note:
85	- The parameter custom_dict can be provided as an argument \
86	only for newmm, longest, and attacut engine.
87	:Example:
88
89	Tokenize text with different tokenizer::
90
91	from pythainlp.tokenize import word_tokenize
92
93	text = "โอเคบ่พวกเรารักภาษาบ้านเกิด"
94
95	word_tokenize(text, engine="newmm")
96	# output: ['โอเค', 'บ่', 'พวกเรา', 'รัก', 'ภาษา', 'บ้านเกิด']
97
98	word_tokenize(text, engine='attacut')
99	# output: ['โอเค', 'บ่', 'พวกเรา', 'รัก', 'ภาษา', 'บ้านเกิด']
100
101	Tokenize text by omiting whitespaces::
102
103	text = "วรรณกรรม ภาพวาด และการแสดงงิ้ว "
104
105	word_tokenize(text, engine="newmm")
106	# output:
107	# ['วรรณกรรม', ' ', 'ภาพวาด', ' ', 'และ', 'การแสดง', 'งิ้ว', ' ']
108
109	word_tokenize(text, engine="newmm", keep_whitespace=False)
110	# output: ['วรรณกรรม', 'ภาพวาด', 'และ', 'การแสดง', 'งิ้ว']
111
112	Tokenize with default and custom dictionary::
113
114	from pythainlp.corpus.common import thai_words
115	from pythainlp.tokenize import dict_trie
116
117	text = 'ชินโซ อาเบะ เกิด 21 กันยายน'
118
119	word_tokenize(text, engine="newmm")
120	# output:
121	# ['ชิน', 'โซ', ' ', 'อา', 'เบะ', ' ',
122	# 'เกิด', ' ', '21', ' ', 'กันยายน']
123
124	custom_dict_japanese_name = set(thai_words()
125	custom_dict_japanese_name.add('ชินโซ')
126	custom_dict_japanese_name.add('อาเบะ')
127
128	trie = dict_trie(dict_source=custom_dict_japanese_name)
129
130	word_tokenize(text, engine="newmm", custom_dict=trie))
131	# output:
132	# ['ชินโซ', ' ', 'อาเบะ',
133	# ' ', 'เกิด', ' ', '21', ' ', 'กันยายน']
134	"""
135	if not text or not isinstance(text, str):	1✔
136	return []	1✔
137
138	segments = []	1✔
139
140	if engine == "newmm" or engine == "onecut":	1✔
141	from pythainlp.tokenize.newmm import segment	1✔
142
143	segments = segment(text, custom_dict)	1✔
144	elif engine == "newmm-safe":	1✔
145	from pythainlp.tokenize.newmm import segment	1✔
146
147	segments = segment(text, custom_dict, safe_mode=True)	1✔
148	elif engine == "attacut":	1✔
149	from pythainlp.tokenize.attacut import segment	1✔
150
151	segments = segment(text)	1✔
152	elif engine == "longest":	1✔
153	from pythainlp.tokenize.longest import segment	1✔
154
155	segments = segment(text, custom_dict)	1✔
156	elif engine == "mm" or engine == "multi_cut":	1✔
157	from pythainlp.tokenize.multi_cut import segment	1✔
158
159	segments = segment(text, custom_dict)	1✔
160	elif engine == "deepcut": # deepcut can optionally use dictionary	1✔
161	from pythainlp.tokenize.deepcut import segment	1✔
162
163	if custom_dict:	1✔
164	custom_dict = list(custom_dict)	1✔
165	segments = segment(text, custom_dict)	1✔
166	else:
167	segments = segment(text)	1✔
168	elif engine == "icu":	1✔
169	from pythainlp.tokenize.pyicu import segment	1✔
170
171	segments = segment(text)	1✔
172	elif engine == "nercut":	1✔
173	from .nercut import segment	1✔
174
175	segments = segment(text)	1✔
176	else:
177	raise ValueError(	1✔
178	f"""Tokenizer \"{engine}\" not found.
179	It might be a typo; if not, please consult our document."""
180	)
181
182	if not keep_whitespace:	1✔
183	segments = [token.strip(" ") for token in segments if token.strip(" ")]	1✔
184
185	return segments	1✔
186
187
188	def sent_tokenize(	1✔
189	text: str,
190	engine: str = DEFAULT_SENT_TOKENIZE_ENGINE,
191	keep_whitespace: bool = True,
192	) -> List[str]:
193	"""
194	Sentence tokenizer.
195
196	Tokenizes running text into "sentences"
197
198	:param str text: the text to be tokenized
199	:param str engine: choose among 'crfcut', 'whitespace', \
200	'whitespace+newline'
201	:return: list of splited sentences
202	:rtype: list[str]
203	Options for engine
204	* crfcut - (default) split by CRF trained on TED dataset
205	* whitespace+newline - split by whitespaces and newline.
206	* whitespace - split by whitespaces. Specifiaclly, with \
207	:class:`regex` pattern ``r" +"``
208	:Example:
209
210	Split the text based on whitespace::
211
212	from pythainlp.tokenize import sent_tokenize
213
214	sentence_1 = "ฉันไปประชุมเมื่อวันที่ 11 มีนาคม"
215	sentence_2 = "ข้าราชการได้รับการหมุนเวียนเป็นระยะ \\
216	และได้รับมอบหมายให้ประจำในระดับภูมิภาค"
217
218	sent_tokenize(sentence_1, engine="whitespace")
219	# output: ['ฉันไปประชุมเมื่อวันที่', '11', 'มีนาคม']
220
221	sent_tokenize(sentence_2, engine="whitespace")
222	# output: ['ข้าราชการได้รับการหมุนเวียนเป็นระยะ',
223	# '\\nและได้รับมอบหมายให้ประจำในระดับภูมิภาค']
224
225	Split the text based on whitespace and newline::
226
227	sentence_1 = "ฉันไปประชุมเมื่อวันที่ 11 มีนาคม"
228	sentence_2 = "ข้าราชการได้รับการหมุนเวียนเป็นระยะ \\
229	และได้รับมอบหมายให้ประจำในระดับภูมิภาค"
230
231	sent_tokenize(sentence_1, engine="whitespace+newline")
232	# output: ['ฉันไปประชุมเมื่อวันที่', '11', 'มีนาคม']
233	sent_tokenize(sentence_2, engine="whitespace+newline")
234	# output: ['ข้าราชการได้รับการหมุนเวียนเป็นระยะ',
235	'\\nและได้รับมอบหมายให้ประจำในระดับภูมิภาค']
236
237	Split the text using CRF trained on TED dataset::
238
239	sentence_1 = "ฉันไปประชุมเมื่อวันที่ 11 มีนาคม"
240	sentence_2 = "ข้าราชการได้รับการหมุนเวียนเป็นระยะ \\
241	และเขาได้รับมอบหมายให้ประจำในระดับภูมิภาค"
242
243	sent_tokenize(sentence_1, engine="crfcut")
244	# output: ['ฉันไปประชุมเมื่อวันที่ 11 มีนาคม']
245
246	sent_tokenize(sentence_2, engine="crfcut")
247	# output: ['ข้าราชการได้รับการหมุนเวียนเป็นระยะ ',
248	'และเขาได้รับมอบหมายให้ประจำในระดับภูมิภาค']
249	"""
250
251	if not text or not isinstance(text, str):	1✔
252	return []	1✔
253
254	segments = []	1✔
255
256	if engine == "crfcut":	1✔
257	from pythainlp.tokenize.crfcut import segment	1✔
258
259	segments = segment(text)	1✔
260	elif engine == "whitespace":	1✔
261	segments = re.split(r" +", text, re.U)	1✔
262	elif engine == "whitespace+newline":	1✔
263	segments = text.split()	1✔
264	else:
265	raise ValueError(	1✔
266	f"""Tokenizer \"{engine}\" not found.
267	It might be a typo; if not, please consult our document."""
268	)
269
270	if not keep_whitespace:	1✔
271	segments = [token.strip(" ") for token in segments if token.strip(" ")]	1✔
272
273	return segments	1✔
274
275
276	def subword_tokenize(	1✔
277	text: str,
278	engine: str = DEFAULT_SUBWORD_TOKENIZE_ENGINE,
279	keep_whitespace: bool = True,
280	) -> List[str]:
281	"""
282	Subword tokenizer. Can be smaller than syllable.
283
284	Tokenizes text into inseparable units of
285	Thai contiguous characters namely
286	`Thai Character Clusters (TCCs) \
287	<https://www.researchgate.net/publication/2853284_Character_Cluster_Based_Thai_Information_Retrieval>`_
288	TCCs are the units based on Thai spelling feature that could not be
289	separated any character further such as 'ก็', 'จะ', 'ไม่', and 'ฝา'.
290	If the following units are separated, they could not be spelled out.
291	This function apply the TCC rules to tokenizes the text into
292	the smallest units.
293
294	For example, the word 'ขนมชั้น' would be tokenized
295	into 'ข', 'น', 'ม', and 'ชั้น'.
296
297	:param str text: text to be tokenized
298	:param str engine: the name subword tokenizer
299	:return: list of subwords
300	:rtype: list[str]
301	Options for engine
302	* tcc (default) - Thai Character Cluster (Theeramunkong et al. 2000)
303	* etcc - Enhanced Thai Character Cluster (Inrut et al. 2001)
304
305	:Example:
306
307	Tokenize text into subword based on tcc::
308
309	from pythainlp.tokenize import subword_tokenize
310
311	text_1 = "ยุคเริ่มแรกของ ราชวงศ์หมิง"
312	text_2 = "ความแปลกแยกและพัฒนาการ"
313
314	subword_tokenize(text_1, engine='tcc')
315	# output: ['ยุ', 'ค', 'เริ่ม', 'แร', 'ก',
316	# 'ข', 'อ', 'ง', ' ', 'รา', 'ช', 'ว', 'ง',
317	# 'ศ', '์', 'ห', 'มิ', 'ง']
318
319	subword_tokenize(text_2, engine='tcc')
320	# output: ['ค', 'วา', 'ม', 'แป', 'ล', 'ก', 'แย', 'ก',
321	'และ', 'พัฒ','นา', 'กา', 'ร']
322
323	Tokenize text into subword based on etcc (Work In Progress)::
324
325	text_1 = "ยุคเริ่มแรกของ ราชวงศ์หมิง"
326	text_2 = "ความแปลกแยกและพัฒนาการ"
327
328	subword_tokenize(text_1, engine='etcc')
329	# output: ['ยุคเริ่มแรกของ ราชวงศ์หมิง']
330
331	subword_tokenize(text_2, engine='etcc')
332	# output: ['ความแปลกแยกและ', 'พัฒ', 'นาการ']
333	"""
334	if not text or not isinstance(text, str):	1✔
335	return []	1✔
336
337	if engine == "tcc":	1✔
338	from pythainlp.tokenize.tcc import segment	1✔
339	elif engine == "etcc":	1✔
340	from pythainlp.tokenize.etcc import segment	1✔
341	else:
342	raise ValueError(	1✔
343	f"""Tokenizer \"{engine}\" not found.
344	It might be a typo; if not, please consult our document."""
345	)
346
347	segments = segment(text)	1✔
348
349	if not keep_whitespace:	1✔
350	segments = [token.strip(" ") for token in segments if token.strip(" ")]	1✔
351
352	return segments	1✔
353
354
355	def syllable_tokenize(	1✔
356	text: str,
357	engine: str = DEFAULT_SYLLABLE_TOKENIZE_ENGINE,
358	keep_whitespace: bool = True,
359	) -> List[str]:
360	"""
361	Syllable tokenizer.
362
363	Tokenizes text into syllable (Thai: พยางค์), a unit of
364	pronunciation having one vowel sound. For example, the word 'รถไฟ'
365	contains two syallbles including 'รถ', and 'ไฟ'.
366
367	Under the hood, this function uses :func:`pythainlp.tokenize.word_tokenize`
368	with newmm as a tokenizer. The function tokenize the text with
369	the dictionary of Thai words from
370	:func:`pythainlp.corpus.common.thai_words`
371	and then dictionary of Thai syllable from
372	:func:`pythainlp.corpus.common.thai_syllables`.
373	As a result, only syllables are obtained.
374
375	:param str text: input string to be tokenized
376	:param str engine: name of the syllable tokenizer
377	:return: list of syllables where whitespaces in the text are included
378	:rtype: list[str]
379	Options for engine
380	* dict (default) - newmm word tokenizer with a syllable dictionary
381	* ssg - CRF syllable segmenter for Thai
382	:Example::
383	::
384
385	from pythainlp.tokenize import syllable_tokenize
386
387	text = 'รถไฟสมัยใหม่จะใช้กำลังจากหัวรถจักรดีเซล หรือจากไฟฟ้า'
388	syllable_tokenize(text)
389	['รถ', 'ไฟ', 'สมัย', 'ใหม่', 'ใช้', 'กำ', 'ลัง', 'จาก', 'หัว',
390	'รถ', 'จักร', 'ดี', 'เซล', ' ', 'หรือ', 'จาก', 'ไฟ', 'ฟ้า']
391	"""
392
393	if not text or not isinstance(text, str):	1✔
394	return []	1✔
395
396	segments = []	1✔
397
398	if engine == "dict" or engine == "default": # use syllable dictionary	1✔
399	words = word_tokenize(text)	1✔
400	for word in words:	1✔
401	segments.extend(	1✔
402	word_tokenize(
403	text=word, custom_dict=DEFAULT_SYLLABLE_DICT_TRIE
404	)
405	)
406	elif engine == "ssg":	1✔
407	from pythainlp.tokenize.ssg import segment	1✔
408
409	segments = segment(text)	1✔
410	else:
411	raise ValueError(	1✔
412	f"""Tokenizer \"{engine}\" not found.
413	It might be a typo; if not, please consult our document."""
414	)
415
416	if not keep_whitespace:	1✔
417	segments = [token.strip(" ") for token in segments if token.strip(" ")]	1✔
418
419	return segments	1✔
420
421
422	class Tokenizer:	1✔
423	"""
424	Tokenizer class, for a custom tokenizer.
425
426	This class allows users to pre-define custom dictionary along with
427	tokenizer and encapsulate them into one single object.
428	It is an wrapper for both two functions including
429	:func:`pythainlp.tokenize.word_tokenize`,
430	and :func:`pythainlp.util.dict_trie`
431
432	:Example:
433
434	Tokenizer object instantiated with :class:`pythainlp.util.Trie`::
435
436	from pythainlp.tokenize import Tokenizer
437	from pythainlp.corpus.common import thai_words
438	from pythainlp.util import dict_trie
439
440	custom_words_list = set(thai_words())
441	custom_words_list.add('อะเฟเซีย')
442	custom_words_list.add('Aphasia')
443	trie = dict_trie(dict_source=custom_words_list)
444
445	text = "อะเฟเซีย (Aphasia*) เป็นอาการผิดปกติของการพูด"
446	_tokenizer = Tokenizer(custom_dict=trie, engine='newmm')
447	# output: ['อะเฟเซีย', ' ', '(', 'Aphasia', ')', ' ', 'เป็น', 'อาการ',
448	'ผิดปกติ', 'ของ', 'การ', 'พูด']
449
450	Tokenizer object instantiated with a list of words::
451
452	text = "อะเฟเซีย (Aphasia) เป็นอาการผิดปกติของการพูด"
453	_tokenizer = Tokenizer(custom_dict=list(thai_words()), engine='newmm')
454	_tokenizer.word_tokenize(text)
455	# output:
456	# ['อะ', 'เฟเซีย', ' ', '(', 'Aphasia', ')', ' ', 'เป็น', 'อาการ',
457	# 'ผิดปกติ', 'ของ', 'การ', 'พูด']
458
459	Tokenizer object instantiated with a file path containing list of
460	word separated with newline and explicitly set a new tokenizer
461	after initiation::
462
463	PATH_TO_CUSTOM_DICTIONARY = './custom_dictionary.txtt'
464
465	# write a file
466	with open(PATH_TO_CUSTOM_DICTIONARY, 'w', encoding='utf-8') as f:
467	f.write('อะเฟเซีย\\nAphasia\\nผิด\\nปกติ')
468
469	text = "อะเฟเซีย (Aphasia) เป็นอาการผิดปกติของการพูด"
470
471	# initate an object from file with `attacut` as tokenizer
472	_tokenizer = Tokenizer(custom_dict=PATH_TO_CUSTOM_DICTIONARY, \\
473	engine='attacut')
474
475	_tokenizer.word_tokenize(text)
476	# output:
477	# ['อะเฟเซีย', ' ', '(', 'Aphasia', ')', ' ', 'เป็น', 'อาการ', 'ผิด',
478	# 'ปกติ', 'ของ', 'การ', 'พูด']
479
480	# change tokenizer to `newmm`
481	_tokenizer.set_tokenizer_engine(engine='newmm')
482	_tokenizer.word_tokenize(text)
483	# output:
484	# ['อะเฟเซีย', ' ', '(', 'Aphasia', ')', ' ', 'เป็นอาการ', 'ผิด',
485	# 'ปกติ', 'ของการพูด']
486	"""
487
488	def __init__(	1✔
489	self,
490	custom_dict: Union[Trie, Iterable[str], str] = None,
491	engine: str = "newmm",
492	keep_whitespace: bool = True,
493	):
494	"""
495	Initialize tokenizer object.
496
497	:param str custom_dict: a file path, a list of vocaburaies* to be
498	used to create a trie, or an instantiated
499	:class:`pythainlp.util.Trie` object.
500	:param str engine: choose between different options of engine to token
501	(i.e. newmm, longest, attacut)
502	:param bool keep_whitespace: True to keep whitespaces, a common mark
503	for end of phrase in Thai
504	"""
505	self.__trie_dict = None	1✔
506	if custom_dict:	1✔
507	self.__trie_dict = dict_trie(custom_dict)	1✔
508	else:
509	self.__trie_dict = DEFAULT_WORD_DICT_TRIE	1✔
510	self.__engine = engine	1✔
511	self.__keep_whitespace = keep_whitespace	1✔
512
513	def word_tokenize(self, text: str) -> List[str]:	1✔
514	"""
515	Main tokenization function.
516
517	:param str text: text to be tokenized
518	:return: list of words, tokenized from the text
519	:rtype: list[str]
520	"""
521	return word_tokenize(	1✔
522	text,
523	custom_dict=self.__trie_dict,
524	engine=self.__engine,
525	keep_whitespace=self.__keep_whitespace,
526	)
527
528	def set_tokenize_engine(self, engine: str) -> None:	1✔
529	"""
530	Set the tokenizer's engine.
531
532	:param str engine: choose between different options of engine to token
533	(i.e. newmm, longest, attacut)
534	"""
535	self.__engine = engine	1✔

PyThaiNLP / pythainlp / 524684088

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous