8745810937

Committed 18 Apr 2024 11:05PM UTC coverage: 93.481% (+0.02%) from 93.46%

Build # 8745810937

Build Type

Pull #5502

github

Committed by

web-flow

Commit Message

Merge 53f2b8079 into c515cc91c

Pull Request Pull Request #5502: chore: Add lint rules

Run Details

6439 of 7332 branches covered (87.82%)

126 of 144 new or added lines in 68 files covered. (87.5%)

3 existing lines in 3 files now uncovered.

13192 of 14112 relevant lines covered (93.48%)

23103.11 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/packages/hunspell-reader/src/commandWords.ts

// cSpell:ignore findup
import { createWriteStream, openSync, writeSync } from 'node:fs';

import { Command } from 'commander';
import type { Sequence } from 'gensequence';
import { genSequence } from 'gensequence';

import type { AffWord } from './affDef.js';
import { asAffWord } from './affLegacy.js';
import { IterableHunspellReaderLegacy } from './IterableHunspellReaderLegacy.js';
import { iterableToStream } from './iterableToStream.js';
import { batch, uniqueFilter } from './util.js';

const uniqueHistorySize = 500_000;

let logStream: NodeJS.WritableStream = process.stderr;

export function getCommand(): Command {
    const commander = new Command('words');

    commander
        .arguments('<hunspell_dic_file>')
        .option('-o, --output <file>', 'output file - defaults to stdout')
        .option('-s, --sort', 'sort the list of words')
        .option('-u, --unique', 'make sure the words are unique.')
        .option('-l, --lower_case', 'output in lower case')
        .option('-T, --no-transform', 'Do not apply the prefix and suffix transforms.  Root words only.')
        .option('-x, --infix', 'Return words with prefix / suffix breaks. ex: "un<do>ing"')
        .option('-r, --rules', 'Append rules used to generate word.')
        .option('-p, --progress', 'Show progress.')
        .option('-m, --max_depth <limit>', 'Maximum depth to apply suffix rules.')
        .option('-n, --number <limit>', 'Limit the number of words to output.')
        .option('--forbidden', 'include forbidden words')
        .option('--partial_compounds', 'include words that must be part of a compound word')
        .option('--only_forbidden', 'includes only words that are forbidden')
        .description('Output all the words in the <hunspell.dic> file.')
        .action(action);

    return commander;
}

function notify(message: string, newLine = true) {
    message = message + (newLine ? '\n' : '');
    logStream.write(message, 'utf8');
}

function yesNo(value: boolean) {
    return value ? 'Yes' : 'No';
}

function affWordToInfix(aff: AffWord): AffWord {
    return { ...aff, word: aff.prefix + '<' + aff.base + '>' + aff.suffix };
}

function mapWord(map: (word: string) => string): (aff: AffWord) => AffWord {
    return (aff: AffWord) => ({ ...aff, word: map(aff.word) });
}

function appendRules(aff: AffWord): AffWord {
    return { ...aff, word: aff.word + '\t[' + aff.rulesApplied + ' ]\t' + '(' + aff.dic + ')' };
}

function writeSeqToFile(seq: Sequence<string>, outFile: string | undefined): Promise<void> {
    return new Promise((resolve, reject) => {
        let resolved = false;
        const out = outFile ? createWriteStream(outFile) : process.stdout;
        const bufferedSeq = genSequence(batch(seq, 500)).map((batch) => batch.join(''));
        const dataStream = iterableToStream(bufferedSeq);
        const fileStream = dataStream.pipe(out);
        const endEvents = ['finish', 'close', 'end'];

        function resolvePromise() {
            if (!resolved) {
                resolved = true;
                resolve();
            }
        }
        const endHandler = () => {
            cleanupStreams();
            setTimeout(resolvePromise, 10);
        };
        const errorHandler = (e: Error) => {
            cleanupStreams();
            reject(e);
        };

        listenToStreams();

        function listenToStreams() {
            endEvents.forEach((event) => fileStream.addListener(event, endHandler));
            fileStream.addListener('error', errorHandler);
            dataStream.addListener('end', endHandler);
        }

        function cleanupStreams() {
            endEvents.forEach((event) => fileStream.removeListener(event, endHandler));
            fileStream.removeListener('error', errorHandler);
            dataStream.removeListener('end', endHandler);
        }
    });
}

interface ErrorWithCode {
    code?: string;
}

async function action(hunspellDicFilename: string, options: Options): Promise<void> {
    try {
        await actionPrime(hunspellDicFilename, options);
    } catch (err) {
        const reason = asError(err);
        if (reason?.code === 'EPIPE') {
            console.log(reason);
            return;
        }
        throw err;
    }
}

function asError(err: unknown): ErrorWithCode | undefined {
    return err && typeof err === 'object' ? (err as ErrorWithCode) : undefined;
}

interface Options {
    sort?: boolean;
    unique?: boolean;
    lower_case?: boolean;
    output?: string;
    transform?: boolean;
    infix?: boolean;
    rules?: boolean; // append rules
    progress?: boolean; // show progress
    number?: string; // limit the number to output
    max_depth?: string; // limit the recursive depth to apply suffixes.
    forbidden: boolean; // include forbidden words
    only_forbidden: boolean; // only include forbidden words
    partial_compounds: boolean; // include partial word compounds
}

async function actionPrime(hunspellDicFilename: string, options: Options) {
    const {
        sort = false,
        unique = false,
        output: outputFile,
        lower_case: lowerCase = false,
        transform = true,
        infix = false,
        rules = false,
        progress: showProgress = false,
        max_depth,
        forbidden = false,
        only_forbidden: onlyForbidden = false,
        partial_compounds: partialCompoundsAllowed = false,
    } = options;
    logStream = outputFile ? process.stdout : process.stderr;
    const log = notify;
    log('Write words');
    log(`Sort: ${yesNo(sort)}`);
    log(`Unique: ${yesNo(unique)}`);
    const baseFile = hunspellDicFilename.replace(/\.(dic|aff)$/, '');
    const dicFile = baseFile + '.dic';
    const affFile = baseFile + '.aff';
    log(`Dic file: ${dicFile}`);
    log(`Aff file: ${affFile}`);
    log(`Generating Words...`);
    const reader = await IterableHunspellReaderLegacy.createFromFiles(affFile, dicFile);
    if (max_depth && Number.parseInt(max_depth) >= 0) {
        reader.maxDepth = Number.parseInt(max_depth);
    }
    const transformers: ((aff: AffWord) => AffWord)[] = [];
    const filters: ((aff: AffWord) => boolean)[] = [];
    if (!forbidden && !onlyForbidden) filters.push((aff) => !aff.flags.isForbiddenWord);
    if (onlyForbidden) filters.push((aff) => !!aff.flags.isForbiddenWord);
    if (!partialCompoundsAllowed) filters.push((aff) => !aff.flags.isOnlyAllowedInCompound);
    if (infix) {
        transformers.push(affWordToInfix);
    }
    if (lowerCase) {
        transformers.push(mapWord((a) => a.toLowerCase()));
    }
    if (rules) {
        transformers.push(appendRules);
    }
    transformers.push(mapWord((a) => a.trim()));
    const dicSize = reader.dic.length;
    let current = 0;
    const calcProgress = () => '\r' + current + ' / ' + dicSize;
    const reportProgressRate = 253;
    const callback = showProgress
        ? () => {
              current++;
              !(current % reportProgressRate) && process.stderr.write(calcProgress(), 'utf8');
          }
        : () => {
              /* void */
          };
    const seqWords = transform ? reader.seqAffWords(callback) : reader.seqRootWords().map(asAffWord);
    const filterUnique = unique ? uniqueFilter<string>(uniqueHistorySize) : (_: string) => true;

    const applyTransformers = (aff: AffWord) => transformers.reduce((aff, fn) => fn(aff), aff);
    const applyFilters = (aff: AffWord) => filters.reduce((cur, fn) => cur && fn(aff), true);

    const allWords = seqWords
        .filter(applyFilters)
        .map(applyTransformers)
        .map((a) => a.word)
        .filter((a) => !!a)
        .filter(filterUnique)
        .map((a) => a + '\n');

    const words = options.number ? allWords.take(Number.parseInt(options.number)) : allWords;

    if (sort) {
        log('Sorting...');
        const data = words.toArray().sort().join('');
        const fd = outputFile ? openSync(outputFile, 'w') : 1;
        writeSync(fd, data);
    } else {
        await writeSeqToFile(words, outputFile);
    }
    if (showProgress) {
        console.error(calcProgress());
    }
    log('Done.');
}

1	// cSpell:ignore findup
2	import { createWriteStream, openSync, writeSync } from 'node:fs';
3
4	import { Command } from 'commander';
5	import type { Sequence } from 'gensequence';
6	import { genSequence } from 'gensequence';
7
8	import type { AffWord } from './affDef.js';
9	import { asAffWord } from './affLegacy.js';
10	import { IterableHunspellReaderLegacy } from './IterableHunspellReaderLegacy.js';
11	import { iterableToStream } from './iterableToStream.js';
12	import { batch, uniqueFilter } from './util.js';
13
14	const uniqueHistorySize = 500_000;	×
15
16	let logStream: NodeJS.WritableStream = process.stderr;	×
17
18	export function getCommand(): Command {
19	const commander = new Command('words');	×
20
21	commander	×
22	.arguments('<hunspell_dic_file>')
23	.option('-o, --output <file>', 'output file - defaults to stdout')
24	.option('-s, --sort', 'sort the list of words')
25	.option('-u, --unique', 'make sure the words are unique.')
26	.option('-l, --lower_case', 'output in lower case')
27	.option('-T, --no-transform', 'Do not apply the prefix and suffix transforms. Root words only.')
28	.option('-x, --infix', 'Return words with prefix / suffix breaks. ex: "un<do>ing"')
29	.option('-r, --rules', 'Append rules used to generate word.')
30	.option('-p, --progress', 'Show progress.')
31	.option('-m, --max_depth <limit>', 'Maximum depth to apply suffix rules.')
32	.option('-n, --number <limit>', 'Limit the number of words to output.')
33	.option('--forbidden', 'include forbidden words')
34	.option('--partial_compounds', 'include words that must be part of a compound word')
35	.option('--only_forbidden', 'includes only words that are forbidden')
36	.description('Output all the words in the <hunspell.dic> file.')
37	.action(action);
38
39	return commander;	×
40	}
41
42	function notify(message: string, newLine = true) {	×
43	message = message + (newLine ? '\n' : '');	×
NEW 44	logStream.write(message, 'utf8');	×
45	}
46
47	function yesNo(value: boolean) {
48	return value ? 'Yes' : 'No';	×
49	}
50
51	function affWordToInfix(aff: AffWord): AffWord {
52	return { ...aff, word: aff.prefix + '<' + aff.base + '>' + aff.suffix };	×
53	}
54
55	function mapWord(map: (word: string) => string): (aff: AffWord) => AffWord {
56	return (aff: AffWord) => ({ ...aff, word: map(aff.word) });	×
57	}
58
59	function appendRules(aff: AffWord): AffWord {
60	return { ...aff, word: aff.word + '\t[' + aff.rulesApplied + ' ]\t' + '(' + aff.dic + ')' };	×
61	}
62
63	function writeSeqToFile(seq: Sequence<string>, outFile: string \| undefined): Promise<void> {
64	return new Promise((resolve, reject) => {	×
65	let resolved = false;	×
66	const out = outFile ? createWriteStream(outFile) : process.stdout;	×
67	const bufferedSeq = genSequence(batch(seq, 500)).map((batch) => batch.join(''));	×
68	const dataStream = iterableToStream(bufferedSeq);	×
69	const fileStream = dataStream.pipe(out);	×
70	const endEvents = ['finish', 'close', 'end'];	×
71
72	function resolvePromise() {
73	if (!resolved) {	×
74	resolved = true;	×
75	resolve();	×
76	}
77	}
78	const endHandler = () => {	×
79	cleanupStreams();	×
80	setTimeout(resolvePromise, 10);	×
81	};
82	const errorHandler = (e: Error) => {	×
83	cleanupStreams();	×
84	reject(e);	×
85	};
86
87	listenToStreams();	×
88
89	function listenToStreams() {
90	endEvents.forEach((event) => fileStream.addListener(event, endHandler));	×
91	fileStream.addListener('error', errorHandler);	×
92	dataStream.addListener('end', endHandler);	×
93	}
94
95	function cleanupStreams() {
96	endEvents.forEach((event) => fileStream.removeListener(event, endHandler));	×
97	fileStream.removeListener('error', errorHandler);	×
98	dataStream.removeListener('end', endHandler);	×
99	}
100	});
101	}
102
103	interface ErrorWithCode {
104	code?: string;
105	}
106
107	async function action(hunspellDicFilename: string, options: Options): Promise<void> {
108	try {	×
109	await actionPrime(hunspellDicFilename, options);	×
110	} catch (err) {
111	const reason = asError(err);	×
112	if (reason?.code === 'EPIPE') {	×
113	console.log(reason);	×
114	return;	×
115	}
116	throw err;	×
117	}
118	}
119
120	function asError(err: unknown): ErrorWithCode \| undefined {
121	return err && typeof err === 'object' ? (err as ErrorWithCode) : undefined;	×
122	}
123
124	interface Options {
125	sort?: boolean;
126	unique?: boolean;
127	lower_case?: boolean;
128	output?: string;
129	transform?: boolean;
130	infix?: boolean;
131	rules?: boolean; // append rules
132	progress?: boolean; // show progress
133	number?: string; // limit the number to output
134	max_depth?: string; // limit the recursive depth to apply suffixes.
135	forbidden: boolean; // include forbidden words
136	only_forbidden: boolean; // only include forbidden words
137	partial_compounds: boolean; // include partial word compounds
138	}
139
140	async function actionPrime(hunspellDicFilename: string, options: Options) {
141	const {
142	sort = false,	×
143	unique = false,	×
144	output: outputFile,
145	lower_case: lowerCase = false,	×
146	transform = true,	×
147	infix = false,	×
148	rules = false,	×
149	progress: showProgress = false,	×
150	max_depth,
151	forbidden = false,	×
152	only_forbidden: onlyForbidden = false,	×
153	partial_compounds: partialCompoundsAllowed = false,	×
154	} = options;	×
155	logStream = outputFile ? process.stdout : process.stderr;	×
156	const log = notify;	×
157	log('Write words');	×
158	log(`Sort: ${yesNo(sort)}`);	×
159	log(`Unique: ${yesNo(unique)}`);	×
160	const baseFile = hunspellDicFilename.replace(/\.(dic\|aff)$/, '');	×
161	const dicFile = baseFile + '.dic';	×
162	const affFile = baseFile + '.aff';	×
163	log(`Dic file: ${dicFile}`);	×
164	log(`Aff file: ${affFile}`);	×
165	log(`Generating Words...`);	×
166	const reader = await IterableHunspellReaderLegacy.createFromFiles(affFile, dicFile);	×
167	if (max_depth && Number.parseInt(max_depth) >= 0) {	×
168	reader.maxDepth = Number.parseInt(max_depth);	×
169	}
170	const transformers: ((aff: AffWord) => AffWord)[] = [];	×
171	const filters: ((aff: AffWord) => boolean)[] = [];	×
172	if (!forbidden && !onlyForbidden) filters.push((aff) => !aff.flags.isForbiddenWord);	×
173	if (onlyForbidden) filters.push((aff) => !!aff.flags.isForbiddenWord);	×
174	if (!partialCompoundsAllowed) filters.push((aff) => !aff.flags.isOnlyAllowedInCompound);	×
175	if (infix) {	×
176	transformers.push(affWordToInfix);	×
177	}
178	if (lowerCase) {	×
179	transformers.push(mapWord((a) => a.toLowerCase()));	×
180	}
181	if (rules) {	×
182	transformers.push(appendRules);	×
183	}
184	transformers.push(mapWord((a) => a.trim()));	×
185	const dicSize = reader.dic.length;	×
186	let current = 0;	×
187	const calcProgress = () => '\r' + current + ' / ' + dicSize;	×
188	const reportProgressRate = 253;	×
189	const callback = showProgress	×
190	? () => {
191	current++;	×
NEW 192	!(current % reportProgressRate) && process.stderr.write(calcProgress(), 'utf8');	×
193	}
194	: () => {
195	/* void */
196	};
197	const seqWords = transform ? reader.seqAffWords(callback) : reader.seqRootWords().map(asAffWord);	×
198	const filterUnique = unique ? uniqueFilter<string>(uniqueHistorySize) : (_: string) => true;	×
199
200	const applyTransformers = (aff: AffWord) => transformers.reduce((aff, fn) => fn(aff), aff);	×
201	const applyFilters = (aff: AffWord) => filters.reduce((cur, fn) => cur && fn(aff), true);	×
202
203	const allWords = seqWords	×
204	.filter(applyFilters)
205	.map(applyTransformers)
206	.map((a) => a.word)	×
207	.filter((a) => !!a)	×
208	.filter(filterUnique)
209	.map((a) => a + '\n');	×
210
211	const words = options.number ? allWords.take(Number.parseInt(options.number)) : allWords;	×
212
213	if (sort) {	×
214	log('Sorting...');	×
215	const data = words.toArray().sort().join('');	×
216	const fd = outputFile ? openSync(outputFile, 'w') : 1;	×
217	writeSync(fd, data);	×
218	} else {
219	await writeSeqToFile(words, outputFile);	×
220	}
221	if (showProgress) {	×
222	console.error(calcProgress());	×
223	}
224	log('Done.');	×
225	}

streetsidesoftware / cspell / 8745810937

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous