9084625059

Committed 14 May 2024 06:48PM UTC coverage: 93.443% (+0.1%) from 93.333%

Build # 9084625059

Build Type

push

github

Committed by

BitPatty

Commit Message

Merge branch 'master' of github.com:BitPatty/buffered-file-reader

Run Details

34 of 41 branches covered (82.93%)

Branch coverage included in aggregate %.

80 of 81 relevant lines covered (98.77%)

8400.58 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

90.48

/src/buffered-file-reader.ts

import { FileHandle, open } from 'fs/promises';

import { Configuration, ReaderOptions } from './configuration';

export type IteratorResult = {
  /**
   * The state of the chunk cursor for this data buffer.
   *
   * Note: Indexing starts at 0
   */
  chunkCursor: {
    /**
     * The cursor position at the start of where the initial chunk
     * of the current buffer was read.
     *
     * This number is inclusive, meaning that the byte at this
     * position has been read into the current data buffer.
     */
    start: number;
    /**
     * The cursor position at the end of where the last chunk of
     * the current buffer was read.
     *
     * This number is exclusive, meaning the the byte at this
     * position has not been read into the current data buffer.
     */
    end: number;
  };
  /**
   * The data in the current chunk
   */
  data: Buffer;
};

export type Iterator = AsyncGenerator<IteratorResult, null, IteratorResult>;

export class BufferedFileReader {
  /**
   * The reader configuration
   */
  readonly #configuration: Configuration;

  /**
   * The path to the target file
   */
  readonly #filePath: string;

  /**
   * The file handle
   */
  #handle: FileHandle | undefined;

  /**
   * The current position of the file cursor
   */
  #cursorPosition = 0;

  /**
   * Creates a new file reader instance
   *
   * @param filePath  The path to the file
   * @param config    The reader configuration
   */
  private constructor(filePath: string, config: Configuration) {
    this.#filePath = filePath;
    this.#configuration = config;
    this.#cursorPosition = config.startOffset;
  }

  /**
   * Creates a new bufferd file reader
   *
   * @param filePath  The file path
   * @param options   The reader options
   * @returns         The file reader
   */
  public static create(filePath: string, options?: ReaderOptions): Iterator {
    const config = new Configuration(options ?? {});
    return new BufferedFileReader(filePath, config).start();
  }

  /**
   * Starts reading the file
   *
   * @returns  The iterator
   */
  public async *start(): Iterator {
    try {
      if (!this.#handle) await this.#openFileHandle();
      let next: Buffer | null = null;

      do {
        // The starting position of the cursor
        const startingCursorPosition = this.#cursorPosition;

        // Get the next chunk
        next = this.#configuration.separator
          ? await this.#readChunkToSeparator(
              this.#configuration.chunkSize,
              this.#configuration.separator,
              this.#cursorPosition,
            )
          : await this.#readChunk(
              this.#configuration.chunkSize,
              this.#cursorPosition,
            );

        // Done
        if (next == null) return null;

        // Update the cursor position
        // For the next iteration
        this.#cursorPosition += next.length;

        // Trim the separator if necessary
        if (
          this.#configuration.separator &&
          this.#configuration.trimSeparator
        ) {
          next = this.#trimSeparator(next, this.#configuration.separator);
        }

        // Yield the current entry
        yield {
          chunkCursor: {
            start: startingCursorPosition,
            end: this.#cursorPosition,
          },
          data: next,
        };
      } while (next != null);
    } finally {
      await this.#closeFileHandle();
    }

    return null;
  }

  /**
   * Trims the separator from the end of the chunk
   *
   * @param chunk      The chunk
   * @param separator  The separator
   * @returns          The trimmed buffer
   */
  #trimSeparator(chunk: Buffer, separator: Uint8Array): Buffer {
    // Ensure the separator is present
    for (let i = separator.length; i > 0; i--) {
      if (i > chunk.length) return chunk;
      if (chunk[chunk.length - i] !== separator[separator.length - i])
        return chunk;
    }

    // If present, return the trimmed chunk
    return chunk.subarray(0, chunk.length - separator.length);
  }

  /**
   * Finds the index of the specified pattern in the specified buffer
   *
   * @param buff     The buffer
   * @param pattern  The pattern to find
   * @param offset   The offset from which to start searching
   * @returns        The index of the pattern or NULL if it wasn't found
   */
  #findPatternIndex(
    buff: Buffer,
    pattern: Uint8Array,
    offset = 0,
  ): number | null {
    // No pattern
    if (pattern.length === 0) return null;

    // Buffer too small
    if (buff.length < pattern.length) return null;

    for (let i = offset; i < buff.length; i++) {
      // Check if it's the first matching byte
      if (buff[i] !== pattern[0]) continue;

      // If the first byte matches & there's only
      // one byte in the pattern -> done
      if (pattern.length === 1) return i;

      // Check the rest of the pattern
      for (let j = 1; j < pattern.length && i + j < buff.length; j++) {
        // Pattern mismatch
        if (buff[i + j] !== pattern[j]) break;

        // If end of pattern -> done
        if (j === pattern.length - 1) return i;
      }
    }

    return null;
  }

  /**
   * Continues reading the file until it either:
   * - finds a chunk containing the separator
   * - reaches the end of the file
   *
   * @param chunkSize  The chunk size per read operation
   * @param separator  The separator pattern
   * @param offset     The start offset
   * @returns          The bytes starting from the offset until the end of
   *                   end of the pattern or the end of the file
   */
  async #readChunkToSeparator(
    chunkSize: number,
    separator: Uint8Array,
    offset = 0,
  ): Promise<Buffer | null> {
    // Get the current chunk
    const current = await this.#readChunk(chunkSize + separator.length, offset);
    if (current == null) return null;

    // Find the separator, and return the chunk up to the enx
    // of the separator if there is a match
    const sIdx = this.#findPatternIndex(current, separator, 0);
    if (sIdx != null) return current.subarray(0, sIdx + separator.length);

    // Current entry reached the end of the file
    if (current.length < chunkSize + separator.length) return current;

    // Continue with the next chunk
    const next = await this.#readChunkToSeparator(
      chunkSize,
      separator,
      offset + current.length - separator.length,
    );

    // If there is nothing left, return
    if (next == null) return current;

    // Append the next chunk to the current one
    return Buffer.concat([
      current.subarray(0, current.length - separator.length),
      next,
    ]);
  }

  /**
   * Reads a portion of the file
   *
   * @param length  The buffer length
   * @param offset  The read offset
   * @returns       The read bytes or NULL if the offset exceeds the file size
   */
  async #readChunk(length: number, offset = 0): Promise<Buffer | null> {
    if (!this.#handle) throw new Error('Handle not opened');
    const buffer = Buffer.alloc(length, 0);
    const res = await this.#handle.read(buffer, 0, length, offset);
    if (res.bytesRead == 0) return null;
    return res.buffer.subarray(0, res.bytesRead);
  }

  /**
   * Opens the file handle
   */
  async #openFileHandle(): Promise<void> {
    if (this.#handle) throw new Error('Handle already opened');
    this.#handle = await open(this.#filePath);
  }

  /**
   * Closes the file handle
   */
  async #closeFileHandle(): Promise<void> {
    if (this.#handle == null) return;
    await this.#handle.close();
    this.#handle = undefined;
  }
}

1	import { FileHandle, open } from 'fs/promises';	6✔
2
3	import { Configuration, ReaderOptions } from './configuration';	6✔
4
5	export type IteratorResult = {
6	/**
7	* The state of the chunk cursor for this data buffer.
8	*
9	* Note: Indexing starts at 0
10	*/
11	chunkCursor: {
12	/**
13	* The cursor position at the start of where the initial chunk
14	* of the current buffer was read.
15	*
16	* This number is inclusive, meaning that the byte at this
17	* position has been read into the current data buffer.
18	*/
19	start: number;
20	/**
21	* The cursor position at the end of where the last chunk of
22	* the current buffer was read.
23	*
24	* This number is exclusive, meaning the the byte at this
25	* position has not been read into the current data buffer.
26	*/
27	end: number;
28	};
29	/**
30	* The data in the current chunk
31	*/
32	data: Buffer;
33	};
34
35	export type Iterator = AsyncGenerator<IteratorResult, null, IteratorResult>;
36
37	export class BufferedFileReader {	6✔
38	/**
39	* The reader configuration
40	*/
41	readonly #configuration: Configuration;
42
43	/**
44	* The path to the target file
45	*/
46	readonly #filePath: string;
47
48	/**
49	* The file handle
50	*/
51	#handle: FileHandle \| undefined;
52
53	/**
54	* The current position of the file cursor
55	*/
56	#cursorPosition = 0;	851✔
57
58	/**
59	* Creates a new file reader instance
60	*
61	* @param filePath The path to the file
62	* @param config The reader configuration
63	*/
64	private constructor(filePath: string, config: Configuration) {
65	this.#filePath = filePath;	851✔
66	this.#configuration = config;	851✔
67	this.#cursorPosition = config.startOffset;	851✔
68	}
69
70	/**
71	* Creates a new bufferd file reader
72	*
73	* @param filePath The file path
74	* @param options The reader options
75	* @returns The file reader
76	*/
77	public static create(filePath: string, options?: ReaderOptions): Iterator {
78	const config = new Configuration(options ?? {});	855✔
79	return new BufferedFileReader(filePath, config).start();	851✔
80	}
81
82	/**
83	* Starts reading the file
84	*
85	* @returns The iterator
86	*/
87	public async *start(): Iterator {
88	try {	848✔
89	if (!this.#handle) await this.#openFileHandle();	848✔
90	let next: Buffer \| null = null;	847✔
91
92	do {	847✔
93	// The starting position of the cursor
94	const startingCursorPosition = this.#cursorPosition;	41,301✔
95
96	// Get the next chunk
97	next = this.#configuration.separator	41,301✔
98	? await this.#readChunkToSeparator(
99	this.#configuration.chunkSize,
100	this.#configuration.separator,
101	this.#cursorPosition,
102	)
103	: await this.#readChunk(
104	this.#configuration.chunkSize,
105	this.#cursorPosition,
106	);
107
108	// Done
109	if (next == null) return null;	41,300✔
110
111	// Update the cursor position
112	// For the next iteration
113	this.#cursorPosition += next.length;	40,454✔
114
115	// Trim the separator if necessary
116	if (	40,454✔
117	this.#configuration.separator &&	43,527✔
118	this.#configuration.trimSeparator
119	) {
120	next = this.#trimSeparator(next, this.#configuration.separator);	1,535✔
121	}
122
123	// Yield the current entry
124	yield {	40,454✔
125	chunkCursor: {
126	start: startingCursorPosition,
127	end: this.#cursorPosition,
128	},
129	data: next,
130	};
131	} while (next != null);
132	} finally {
133	await this.#closeFileHandle();	848✔
134	}
135
136	return null;	×
137	}
138
139	/**
140	* Trims the separator from the end of the chunk
141	*
142	* @param chunk The chunk
143	* @param separator The separator
144	* @returns The trimmed buffer
145	*/
146	#trimSeparator(chunk: Buffer, separator: Uint8Array): Buffer {
147	// Ensure the separator is present
148	for (let i = separator.length; i > 0; i--) {	1,535✔
149	if (i > chunk.length) return chunk;	5,063✔
150	if (chunk[chunk.length - i] !== separator[separator.length - i])	5,061✔
151	return chunk;	162✔
152	}
153
154	// If present, return the trimmed chunk
155	return chunk.subarray(0, chunk.length - separator.length);	1,371✔
156	}
157
158	/**
159	* Finds the index of the specified pattern in the specified buffer
160	*
161	* @param buff The buffer
162	* @param pattern The pattern to find
163	* @param offset The offset from which to start searching
164	* @returns The index of the pattern or NULL if it wasn't found
165	*/
166	#findPatternIndex(
167	buff: Buffer,
168	pattern: Uint8Array,
169	offset = 0,	×
170	): number \| null {
171	// No pattern
172	if (pattern.length === 0) return null;	7,142!
173
174	// Buffer too small
175	if (buff.length < pattern.length) return null;	7,142✔
176
177	for (let i = offset; i < buff.length; i++) {	7,138✔
178	// Check if it's the first matching byte
179	if (buff[i] !== pattern[0]) continue;	37,591✔
180
181	// If the first byte matches & there's only
182	// one byte in the pattern -> done
183	if (pattern.length === 1) return i;	10,166✔
184
185	// Check the rest of the pattern
186	for (let j = 1; j < pattern.length && i + j < buff.length; j++) {	8,724✔
187	// Pattern mismatch
188	if (buff[i + j] !== pattern[j]) break;	21,854✔
189
190	// If end of pattern -> done
191	if (j === pattern.length - 1) return i;	18,156✔
192	}
193	}
194
195	return null;	4,394✔
196	}
197
198	/**
199	* Continues reading the file until it either:
200	* - finds a chunk containing the separator
201	* - reaches the end of the file
202	*
203	* @param chunkSize The chunk size per read operation
204	* @param separator The separator pattern
205	* @param offset The start offset
206	* @returns The bytes starting from the offset until the end of
207	* end of the pattern or the end of the file
208	*/
209	async #readChunkToSeparator(
210	chunkSize: number,
211	separator: Uint8Array,
212	offset = 0,	×
213	): Promise<Buffer \| null> {
214	// Get the current chunk
215	const current = await this.#readChunk(chunkSize + separator.length, offset);	7,973✔
216	if (current == null) return null;	7,973✔
217
218	// Find the separator, and return the chunk up to the enx
219	// of the separator if there is a match
220	const sIdx = this.#findPatternIndex(current, separator, 0);	7,142✔
221	if (sIdx != null) return current.subarray(0, sIdx + separator.length);	7,142✔
222
223	// Current entry reached the end of the file
224	if (current.length < chunkSize + separator.length) return current;	4,398✔
225
226	// Continue with the next chunk
227	const next = await this.#readChunkToSeparator(	4,069✔
228	chunkSize,
229	separator,
230	offset + current.length - separator.length,
231	);
232
233	// If there is nothing left, return
234	if (next == null) return current;	4,069!
235
236	// Append the next chunk to the current one
237	return Buffer.concat([	4,069✔
238	current.subarray(0, current.length - separator.length),
239	next,
240	]);
241	}
242
243	/**
244	* Reads a portion of the file
245	*
246	* @param length The buffer length
247	* @param offset The read offset
248	* @returns The read bytes or NULL if the offset exceeds the file size
249	*/
250	async #readChunk(length: number, offset = 0): Promise<Buffer \| null> {	×
251	if (!this.#handle) throw new Error('Handle not opened');	45,370!
252	const buffer = Buffer.alloc(length, 0);	45,370✔
253	const res = await this.#handle.read(buffer, 0, length, offset);	45,370✔
254	if (res.bytesRead == 0) return null;	45,369✔
255	return res.buffer.subarray(0, res.bytesRead);	44,523✔
256	}
257
258	/**
259	* Opens the file handle
260	*/
261	async #openFileHandle(): Promise<void> {
262	if (this.#handle) throw new Error('Handle already opened');	848!
263	this.#handle = await open(this.#filePath);	848✔
264	}
265
266	/**
267	* Closes the file handle
268	*/
269	async #closeFileHandle(): Promise<void> {
270	if (this.#handle == null) return;	848✔
271	await this.#handle.close();	847✔
272	this.#handle = undefined;	847✔
273	}
274	}

BitPatty / buffered-file-reader / 9084625059

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous