11535225574

Committed 26 Oct 2024 09:54PM UTC coverage: 94.149%. First build

Build # 11535225574

Build Type

Pull #6423

github

Committed by

web-flow

Commit Message

Merge 477a25e35 into 1b4d774f3

Pull Request Pull Request #6423: fix: tools - remove duplicates and support compounding

Run Details

13776 of 15880 branches covered (86.75%)

123 of 127 new or added lines in 10 files covered. (96.85%)

15014 of 15947 relevant lines covered (94.15%)

31412.51 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

86.27

/packages/cspell-tools/src/compiler/createWordsCollection.ts

import { parseDictionary } from 'cspell-trie-lib';

import type { FilePath } from '../config/config.js';
import { createReader } from './Reader.js';
import { DictionaryReader, Reader } from './readers/ReaderOptions.js';
import type { AllowedSplitWordsCollection, ExcludeWordsCollection, WordsCollection } from './WordsCollection.js';
import { defaultAllowedSplitWords, defaultExcludeWordsCollection } from './WordsCollection.js';

class AllowedSplitWordsImpl implements AllowedSplitWordsCollection {
    private collection: WordsCollection;
    readonly size: number;

    constructor(collection: WordsCollection) {
        this.collection = collection;
        this.size = collection.size;
    }

    public has(word: string, caseSensitive: boolean) {
        return !this.size || this.collection.has(word, caseSensitive);
    }
}

export async function createAllowedSplitWordsFromFiles(
    files: FilePath | FilePath[] | undefined,
): Promise<AllowedSplitWordsCollection> {
    if (!files || !files.length) return defaultAllowedSplitWords;

    const collection = await createWordsCollectionFromFiles(files);
    return new AllowedSplitWordsImpl(collection);
}

export function createAllowedSplitWords(words: Iterable<string> | undefined): AllowedSplitWordsCollection {
    if (!words) return defaultAllowedSplitWords;

    return new AllowedSplitWordsImpl(createWordsCollection(words));
}

function buildHasFn(dict: { hasWord: (word: string, caseSensitive: boolean) => boolean }) {
    function has(word: string, caseSensitive: boolean) {
        const r = dict.hasWord(word, true);
        if (r || caseSensitive) return r;
        const lc = word.toLowerCase();
        if (lc == word) return false;
        return dict.hasWord(lc, true);
    }

    return has;
}

async function readFile(filename: string): Promise<Reader> {
    return await createReader(filename, {});
}

function readersToCollection(readers: Reader[]): WordsCollection {
    const dictReaders = readers.filter(isDictionaryReader).map(dictReaderToCollection);
    const nonDictCollection = lineReadersToCollection(readers.filter((a) => !isDictionaryReader(a)));
    const collections = [...dictReaders, nonDictCollection];

    const collection = {
        size: collections.reduce((s, a) => s + a.size, 0),
        has: (word: string, caseSensitive: boolean) => collections.some((a) => a.has(word, caseSensitive)),
    };

    return collection;
}

const cache = new WeakMap<FilePath[], WordsCollection>();

export async function createWordsCollectionFromFiles(files: FilePath | FilePath[]): Promise<WordsCollection> {
    files = Array.isArray(files) ? files : [files];

    const cached = cache.get(files);
    if (cached) return cached;

    const sources = await Promise.all(files.map((file) => readFile(file)));

    const collection = readersToCollection(sources);

    cache.set(files, collection);
    return collection;
}

export function createWordsCollection(words: Iterable<string>): WordsCollection {
    if (words instanceof Set) return words;

    const arrWords = (Array.isArray(words) ? words : [...words])
        .map((a) => a.trim())
        .filter((a) => !!a)
        .filter((a) => !a.startsWith('#'));
    const setOfWords = new Set(arrWords);
    const has = buildHasFn({ hasWord: (word: string) => setOfWords.has(word) });
    return { size: setOfWords.size, has };
}

class ExcludeWordsCollectionImpl implements ExcludeWordsCollection {
    private collection: WordsCollection;
    readonly size: number;

    constructor(collection: WordsCollection) {
        this.collection = collection;
        this.size = collection.size;
    }

    public has(word: string, caseSensitive: boolean) {
        return this.collection.has(word, caseSensitive);
    }
}

export async function createExcludeWordsCollectionFromFiles(
    files: FilePath | FilePath[] | undefined,
): Promise<ExcludeWordsCollection> {
    if (!files || !files.length) return defaultExcludeWordsCollection;

    const collection = await createWordsCollectionFromFiles(files);
    return new ExcludeWordsCollectionImpl(collection);
}

export function createExcludeWordsCollection(words: Iterable<string> | undefined): ExcludeWordsCollection {
    return new ExcludeWordsCollectionImpl(words ? createWordsCollection(words) : new Set());
}

function isDictionaryReader(reader: Reader | DictionaryReader): reader is DictionaryReader {
    return 'hasWord' in reader && !!reader.hasWord;
}

function dictReaderToCollection(reader: DictionaryReader): WordsCollection {
    return { size: reader.size, has: buildHasFn(reader) };
}

function lineReadersToCollection(readers: Reader[]): WordsCollection {
    function* words() {
        for (const reader of readers) {
            yield* reader.lines;
        }
    }

    const dict = parseDictionary(words(), { stripCaseAndAccents: false });

    return { size: dict.size, has: buildHasFn(dict) };
}

1	import { parseDictionary } from 'cspell-trie-lib';
2
3	import type { FilePath } from '../config/config.js';
4	import { createReader } from './Reader.js';
5	import { DictionaryReader, Reader } from './readers/ReaderOptions.js';
6	import type { AllowedSplitWordsCollection, ExcludeWordsCollection, WordsCollection } from './WordsCollection.js';
7	import { defaultAllowedSplitWords, defaultExcludeWordsCollection } from './WordsCollection.js';
8
9	class AllowedSplitWordsImpl implements AllowedSplitWordsCollection {
10	private collection: WordsCollection;
11	readonly size: number;
12
13	constructor(collection: WordsCollection) {
14	this.collection = collection;	32✔
15	this.size = collection.size;	32✔
16	}
17
18	public has(word: string, caseSensitive: boolean) {
19	return !this.size \|\| this.collection.has(word, caseSensitive);	107✔
20	}
21	}
22
23	export async function createAllowedSplitWordsFromFiles(
24	files: FilePath \| FilePath[] \| undefined,
25	): Promise<AllowedSplitWordsCollection> {
26	if (!files \|\| !files.length) return defaultAllowedSplitWords;	57✔
27
28	const collection = await createWordsCollectionFromFiles(files);	5✔
29	return new AllowedSplitWordsImpl(collection);	5✔
30	}
31
32	export function createAllowedSplitWords(words: Iterable<string> \| undefined): AllowedSplitWordsCollection {
33	if (!words) return defaultAllowedSplitWords;	33✔
34
35	return new AllowedSplitWordsImpl(createWordsCollection(words));	27✔
36	}
37
38	function buildHasFn(dict: { hasWord: (word: string, caseSensitive: boolean) => boolean }) {
39	function has(word: string, caseSensitive: boolean) {
40	const r = dict.hasWord(word, true);	126✔
41	if (r \|\| caseSensitive) return r;	126✔
42	const lc = word.toLowerCase();	34✔
43	if (lc == word) return false;	34✔
44	return dict.hasWord(lc, true);	31✔
45	}
46
47	return has;	35✔
48	}
49
50	async function readFile(filename: string): Promise<Reader> {
51	return await createReader(filename, {});	7✔
52	}
53
54	function readersToCollection(readers: Reader[]): WordsCollection {
55	const dictReaders = readers.filter(isDictionaryReader).map(dictReaderToCollection);	7✔
56	const nonDictCollection = lineReadersToCollection(readers.filter((a) => !isDictionaryReader(a)));	7✔
57	const collections = [...dictReaders, nonDictCollection];	7✔
58
59	const collection = {	7✔
60	size: collections.reduce((s, a) => s + a.size, 0),	8✔
61	has: (word: string, caseSensitive: boolean) => collections.some((a) => a.has(word, caseSensitive)),	24✔
62	};
63
64	return collection;	7✔
65	}
66
67	const cache = new WeakMap<FilePath[], WordsCollection>();	7✔
68
69	export async function createWordsCollectionFromFiles(files: FilePath \| FilePath[]): Promise<WordsCollection> {
70	files = Array.isArray(files) ? files : [files];	9!
71
72	const cached = cache.get(files);	9✔
73	if (cached) return cached;	9✔
74
75	const sources = await Promise.all(files.map((file) => readFile(file)));	7✔
76
77	const collection = readersToCollection(sources);	7✔
78
79	cache.set(files, collection);	7✔
80	return collection;	7✔
81	}
82
83	export function createWordsCollection(words: Iterable<string>): WordsCollection {
84	if (words instanceof Set) return words;	27!
85
86	const arrWords = (Array.isArray(words) ? words : [...words])	27!
87	.map((a) => a.trim())	56✔
88	.filter((a) => !!a)	56✔
89	.filter((a) => !a.startsWith('#'));	56✔
90	const setOfWords = new Set(arrWords);	27✔
91	const has = buildHasFn({ hasWord: (word: string) => setOfWords.has(word) });	133✔
92	return { size: setOfWords.size, has };	27✔
93	}
94
95	class ExcludeWordsCollectionImpl implements ExcludeWordsCollection {
96	private collection: WordsCollection;
97	readonly size: number;
98
99	constructor(collection: WordsCollection) {
NEW 100	this.collection = collection;	×
101	this.size = collection.size;	×
102	}
103
104	public has(word: string, caseSensitive: boolean) {
NEW 105	return this.collection.has(word, caseSensitive);	×
106	}
107	}
108
109	export async function createExcludeWordsCollectionFromFiles(
110	files: FilePath \| FilePath[] \| undefined,
111	): Promise<ExcludeWordsCollection> {
112	if (!files \|\| !files.length) return defaultExcludeWordsCollection;	×
113
114	const collection = await createWordsCollectionFromFiles(files);	×
115	return new ExcludeWordsCollectionImpl(collection);	×
116	}
117
118	export function createExcludeWordsCollection(words: Iterable<string> \| undefined): ExcludeWordsCollection {
119	return new ExcludeWordsCollectionImpl(words ? createWordsCollection(words) : new Set());	×
120	}
121
122	function isDictionaryReader(reader: Reader \| DictionaryReader): reader is DictionaryReader {
123	return 'hasWord' in reader && !!reader.hasWord;	14✔
124	}
125
126	function dictReaderToCollection(reader: DictionaryReader): WordsCollection {
127	return { size: reader.size, has: buildHasFn(reader) };	1✔
128	}
129
130	function lineReadersToCollection(readers: Reader[]): WordsCollection {
131	function* words() {
132	for (const reader of readers) {	7✔
133	yield* reader.lines;	6✔
134	}
135	}
136
137	const dict = parseDictionary(words(), { stripCaseAndAccents: false });	7✔
138
139	return { size: dict.size, has: buildHasFn(dict) };	7✔
140	}

streetsidesoftware / cspell / 11535225574

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous