SamhammerAG / ai-data-preprocessing-queue / 13242820435

coverage: 91.515% (-0.05%) from 91.566%
13242820435

push

github

web-flow 
Merge pull request #13 from SamhammerAG/KIT-4026

KIT-4026 update

151 of 165 relevant lines covered (91.52%)

0.92 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

100.0
/ai_data_preprocessing_queue/Steps/snowball_stemmer.py
1 
import warnings
1✔
2 
from typing import Any
1✔
3 









4



with warnings.catch_warnings():




1✔





5



    warnings.filterwarnings("ignore", category=DeprecationWarning)




1✔





6



    from nltk.stem.snowball import SnowballStemmer




1✔





7












8












9



lang_mapping = {"de": SnowballStemmer("german"), "en": SnowballStemmer("english")}




1✔





10












11












12



def step(item: Any, item_state: dict[str, Any], global_state: dict[str, Any] | None, preprocessor_data: str) -> Any:




1✔





13



    stemmer = lang_mapping.get(item_state["language"], lang_mapping["en"])




1✔





14












15



    stemmed_words = [stemmer.stem(word) for word in item.split()]




1✔





16












17



    return " ".join(stemmed_words)




1✔



















    

  • 

    •