3525226766

Build Type

github

Committed by GitHub

Commit Message

<a href="https://github.com/cheeriojs/cheerio/commit/<a class=hub.com/cheeriojs/cheerio/commit/<a class="double-link" href="https://git"><a class=hub.com/cheeriojs/cheerio/commit/<a class="double-link" href="https://git"><a class=hub.com/cheeriojs/cheerio/commit/18348f037a01bced63d18a88a0628e5d803be167">18348f037<a href="https://github.com/cheeriojs/cheerio/commit/18348f037a01bced63d18a88a0628e5d803be167">&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/cheeriojs/cheerio/commit/&lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/cheeriojs/cheerio/commit/&amp;lt;a class=&amp;quot;double-link&amp;quot; href=&amp;quot;https://git&quot;&gt;&amp;lt;a class=&lt;/a&gt;hub.com/cheeriojs/cheerio/commit/18348f037a01bced63d18a88a0628e5d803be167&quot;&gt;18348f037&lt;/a&gt;&lt;a href=&quot;https://github.com/cheeriojs/cheerio/commit/18348f037a01bced63d18a88a0628e5d803be167&quot;&gt;&amp;lt;a href=&amp;quot;https://github.com/cheeriojs/cheerio/commit/18348f037a01bced63d18a88a0628e5d&lt;/a&gt;803be167">&quot;&gt;Merge &lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/cheeriojs/cheerio/commit/&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/cheeriojs/cheerio/commit/<a class="double-link" href="https://github.com/cheeriojs/cheerio/commit/99b9e0b9f877702e620f32e0be2b996a6ea86796">99b9e0b9f</a><a href="https://github.com/cheeriojs/cheerio/commit/18348f037a01bced63d18a88a0628e5d803be167">&amp;quot;&amp;gt;99b9e0b9f&amp;lt;/a&amp;gt;&amp;quot;&amp;gt;99b9e0b9f&amp;lt;/a&amp;gt;&amp;lt;a href=&amp;quot;https://github.com/cheeriojs/cheerio/commit/18348f037a01bced63d18a88a0628e5d803be&lt;/a&gt;167&quot;&gt; into &lt;/a&gt;&lt;a class=&quot;double-link&quot; href=&quot;https://github.com/cheeriojs/cheerio/commit/<a class="double-link" href="https://github.com/cheeriojs/cheerio/commit/0596d22c2">0596d22c2">0596d22c2</a>

Pull Request Pull Request #2857: feat: Add loading methods for NodeJS

Run Details

819 of 849 branches covered (96.47%)

Branch coverage included in aggregate %.

203 of 203 new or added lines in 3 files covered. (100.0%)

5163 of 5169 relevant lines covered (99.88%)

188.66 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

93.04

/src/node.ts

export * from './index.js';

/* eslint-disable n/no-unsupported-features/node-builtins, n/file-extension-in-import */

import type { CheerioAPI, CheerioOptions } from './index.js';
import { load } from './index.js';
import { flatten as flattenOptions, InternalOptions } from './options.js';
import { adapter as htmlparser2Adapter } from 'parse5-htmlparser2-tree-adapter';

import { WritableStream as Htmlparser2Stream } from 'htmlparser2/lib/WritableStream';
import DomHandler from 'domhandler';
import { ParserStream as Parse5Stream } from 'parse5-parser-stream';
import {
  decodeBuffer,
  DecodeStream,
  type SnifferOptions,
} from 'encoding-sniffer';
import * as undici from 'undici';
import MIMEType from 'whatwg-mimetype';
import { type Writable, finished } from 'node:stream';

/**
 * Sniffs the encoding of a buffer, then creates a querying function bound to a
 * document created from the buffer.
 *
 * @param buffer - The buffer to sniff the encoding of.
 * @param options - The options to pass to Cheerio.
 * @returns The loaded document.
 */
export function loadBuffer(
  buffer: Buffer,
  options: DecodeStreamOptions = {}
): CheerioAPI {
  const opts = flattenOptions(options);
  const str = decodeBuffer(buffer, {
    defaultEncoding: opts?.xmlMode ? 'utf8' : 'windows-1252',
    ...options.encoding,
  });

  return load(str, opts);
}

function _stringStream(
  options: InternalOptions | undefined,
  cb: (err: Error | null | undefined, $: CheerioAPI) => void
): Writable {
  if (options && (options.xmlMode || options._useHtmlParser2)) {
    const handler: DomHandler = new DomHandler(
      (err) => cb(err, load(handler.root)),
      options
    );

    return new Htmlparser2Stream(handler, options);
  }

  const stream = new Parse5Stream({
    ...options,
    treeAdapter: htmlparser2Adapter,
  });

  finished(stream, (err) => cb(err, load(stream.document)));

  return stream;
}

/**
 * Creates a stream that parses a sequence of strings into a document.
 *
 * The stream is a `Writable` stream that accepts strings. When the stream is
 * finished, the callback is called with the loaded document.
 *
 * @param options - The options to pass to Cheerio.
 * @param cb - The callback to call when the stream is finished.
 * @returns The writable stream.
 */
export function stringStream(
  options: CheerioOptions,
  cb: (err: Error | null | undefined, $: CheerioAPI) => void
): Writable {
  return _stringStream(flattenOptions(options), cb);
}

export interface DecodeStreamOptions extends CheerioOptions {
  encoding?: SnifferOptions;
}

/**
 * Parses a stream of buffers into a document.
 *
 * The stream is a `Writable` stream that accepts buffers. When the stream is
 * finished, the callback is called with the loaded document.
 *
 * @param options - The options to pass to Cheerio.
 * @param cb - The callback to call when the stream is finished.
 * @returns The writable stream.
 */
export function decodeStream(
  options: DecodeStreamOptions,
  cb: (err: Error | null | undefined, $: CheerioAPI) => void
): Writable {
  const { encoding = {}, ...cheerioOptions } = options;
  const opts = flattenOptions(cheerioOptions);

  // Set the default encoding to UTF-8 for XML mode
  encoding.defaultEncoding ??= opts?.xmlMode ? 'utf8' : 'windows-1252';

  const decodeStream = new DecodeStream(encoding);
  const loadStream = _stringStream(opts, cb);

  decodeStream.pipe(loadStream);

  return decodeStream;
}

type UndiciStreamOptions = Parameters<typeof undici.stream>[1];

export interface CheerioRequestOptions extends DecodeStreamOptions {
  /** The options passed to `undici`'s `stream` method. */
  requestOptions?: UndiciStreamOptions;
}

const defaultRequestOptions: UndiciStreamOptions = {
  method: 'GET',
  // Allow redirects by default
  maxRedirections: 5,
  // NOTE: `throwOnError` currently doesn't work https://github.com/nodejs/undici/issues/1753
  throwOnError: true,
  // Set an Accept header
  headers: {
    accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  },
};

/**
 * `fromURL` loads a document from a URL.
 *
 * By default, redirects are allowed and non-2xx responses are rejected.
 *
 * @param url - The URL to load the document from.
 * @param options - The options to pass to Cheerio.
 * @returns The loaded document.
 */
export async function fromURL(
  url: string | URL,
  options: CheerioRequestOptions = {}
): Promise<CheerioAPI> {
  const {
    requestOptions = defaultRequestOptions,
    encoding = {},
    ...cheerioOptions
  } = options;
  let undiciStream: Promise<undici.Dispatcher.StreamData> | undefined;

  // Add headers if none were supplied.
  requestOptions.headers ??= defaultRequestOptions.headers;

  const promise = new Promise<CheerioAPI>((resolve, reject) => {
    undiciStream = undici.stream(url, requestOptions, (res) => {
      const contentType = res.headers['content-type'];
      const mimeType = new MIMEType(contentType ?? 'text/html');

      if (!mimeType.isHTML() && !mimeType.isXML()) {
        throw new RangeError(
          `The content-type "${contentType}" is neither HTML nor XML.`
        );
      }

      // Forward the charset from the header to the decodeStream.
      encoding.transportLayerEncodingLabel = mimeType.parameters.get('charset');

      /*
       * If we allow redirects, we will have entries in the history.
       * The last entry will be the final URL.
       */
      const history = (
        res.context as
          | {
              history?: URL[];
            }
          | undefined
      )?.history;

      const opts = {
        encoding,
        // Set XML mode based on the MIME type.
        xmlMode: mimeType.isXML(),
        // Set the `baseURL` to the final URL.
        baseURL: history ? history[history.length - 1] : url,
        ...cheerioOptions,
      };

      return decodeStream(opts, (err, $) => (err ? reject(err) : resolve($)));
    });
  });

  // Let's make sure the request is completed before returning the promise.
  await undiciStream;

  return promise;
}

1	export * from './index.js';	1✔
2		1✔
3	/* eslint-disable n/no-unsupported-features/node-builtins, n/file-extension-in-import */	1✔
4		1✔
5	import type { CheerioAPI, CheerioOptions } from './index.js';	1✔
6	import { load } from './index.js';	1✔
7	import { flatten as flattenOptions, InternalOptions } from './options.js';	1✔
8	import { adapter as htmlparser2Adapter } from 'parse5-htmlparser2-tree-adapter';	1✔
9		1✔
10	import { WritableStream as Htmlparser2Stream } from 'htmlparser2/lib/WritableStream';	1✔
11	import DomHandler from 'domhandler';	1✔
12	import { ParserStream as Parse5Stream } from 'parse5-parser-stream';	1✔
13	import {	1✔
14	decodeBuffer,	1✔
15	DecodeStream,	1✔
16	type SnifferOptions,	1✔
17	} from 'encoding-sniffer';	1✔
18	import * as undici from 'undici';	1✔
19	import MIMEType from 'whatwg-mimetype';	1✔
20	import { type Writable, finished } from 'node:stream';	1✔
21		1✔
22	/**	1✔
23	* Sniffs the encoding of a buffer, then creates a querying function bound to a	1✔
24	* document created from the buffer.	1✔
25	*	1✔
26	* @param buffer - The buffer to sniff the encoding of.	1✔
27	* @param options - The options to pass to Cheerio.	1✔
28	* @returns The loaded document.	1✔
29	*/	1✔
30	export function loadBuffer(	2✔
31	buffer: Buffer,	2✔
32	options: DecodeStreamOptions = {}	2✔
33	): CheerioAPI {	2✔
34	const opts = flattenOptions(options);	2✔
35	const str = decodeBuffer(buffer, {	2✔
36	defaultEncoding: opts?.xmlMode ? 'utf8' : 'windows-1252',	2!
37	...options.encoding,	2✔
38	});	2✔
39		2✔
40	return load(str, opts);	2✔
41	}	2✔
42		1✔
43	function _stringStream(	8✔
44	options: InternalOptions \| undefined,	8✔
45	cb: (err: Error \| null \| undefined, $: CheerioAPI) => void	8✔
46	): Writable {	8✔
47	if (options && (options.xmlMode \|\| options._useHtmlParser2)) {	8✔
48	const handler: DomHandler = new DomHandler(	3✔
49	(err) => cb(err, load(handler.root)),	3✔
50	options	3✔
51	);	3✔
52		3✔
53	return new Htmlparser2Stream(handler, options);	3✔
54	}	3✔
55		5✔
56	const stream = new Parse5Stream({	5✔
57	...options,	5✔
58	treeAdapter: htmlparser2Adapter,	5✔
59	});	5✔
60		5✔
61	finished(stream, (err) => cb(err, load(stream.document)));	5✔
62		5✔
63	return stream;	5✔
64	}	5✔
65		1✔
66	/**	1✔
67	* Creates a stream that parses a sequence of strings into a document.	1✔
68	*	1✔
69	* The stream is a `Writable` stream that accepts strings. When the stream is	1✔
70	* finished, the callback is called with the loaded document.	1✔
71	*	1✔
72	* @param options - The options to pass to Cheerio.	1✔
73	* @param cb - The callback to call when the stream is finished.	1✔
74	* @returns The writable stream.	1✔
75	*/	1✔
76	export function stringStream(	3✔
77	options: CheerioOptions,	3✔
78	cb: (err: Error \| null \| undefined, $: CheerioAPI) => void	3✔
79	): Writable {	3✔
80	return _stringStream(flattenOptions(options), cb);	3✔
81	}	3✔
82		1✔
83	export interface DecodeStreamOptions extends CheerioOptions {	1✔
84	encoding?: SnifferOptions;	1✔
85	}	1✔
86		1✔
87	/**	1✔
88	* Parses a stream of buffers into a document.	1✔
89	*	1✔
90	* The stream is a `Writable` stream that accepts buffers. When the stream is	1✔
91	* finished, the callback is called with the loaded document.	1✔
92	*	1✔
93	* @param options - The options to pass to Cheerio.	1✔
94	* @param cb - The callback to call when the stream is finished.	1✔
95	* @returns The writable stream.	1✔
96	*/	1✔
97	export function decodeStream(	5✔
98	options: DecodeStreamOptions,	5✔
99	cb: (err: Error \| null \| undefined, $: CheerioAPI) => void	5✔
100	): Writable {	5✔
101	const { encoding = {}, ...cheerioOptions } = options;	5✔
102	const opts = flattenOptions(cheerioOptions);	5✔
103		5✔
104	// Set the default encoding to UTF-8 for XML mode	5✔
105	encoding.defaultEncoding ??= opts?.xmlMode ? 'utf8' : 'windows-1252';	5!
106		5✔
107	const decodeStream = new DecodeStream(encoding);	5✔
108	const loadStream = _stringStream(opts, cb);	5✔
109		5✔
110	decodeStream.pipe(loadStream);	5✔
111		5✔
112	return decodeStream;	5✔
113	}	5✔
114		1✔
115	type UndiciStreamOptions = Parameters<typeof undici.stream>[1];	1✔
116		1✔
117	export interface CheerioRequestOptions extends DecodeStreamOptions {	1✔
118	/** The options passed to `undici`'s `stream` method. */	1✔
119	requestOptions?: UndiciStreamOptions;	1✔
120	}	1✔
121		1✔
122	const defaultRequestOptions: UndiciStreamOptions = {	1✔
123	method: 'GET',	1✔
124	// Allow redirects by default	1✔
125	maxRedirections: 5,	1✔
126	// NOTE: `throwOnError` currently doesn't work https://github.com/nodejs/undici/issues/1753	1✔
127	throwOnError: true,	1✔
128	// Set an Accept header	1✔
129	headers: {	1✔
130	accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,/;q=0.8',	1✔
131	},	1✔
132	};	1✔
133		1✔
134	/**	1✔
135	* `fromURL` loads a document from a URL.	1✔
136	*	1✔
137	* By default, redirects are allowed and non-2xx responses are rejected.	1✔
138	*	1✔
139	* @param url - The URL to load the document from.	1✔
140	* @param options - The options to pass to Cheerio.	1✔
141	* @returns The loaded document.	1✔
142	*/	1✔
143	export async function fromURL(	3✔
144	url: string \| URL,	3✔
145	options: CheerioRequestOptions = {}	3✔
146	): Promise<CheerioAPI> {	3✔
147	const {	3✔
148	requestOptions = defaultRequestOptions,	3!
149	encoding = {},	3!
150	...cheerioOptions	3✔
151	} = options;	3✔
152	let undiciStream: Promise<undici.Dispatcher.StreamData> \| undefined;	3✔
153		3✔
154	// Add headers if none were supplied.	3✔
155	requestOptions.headers ??= defaultRequestOptions.headers;	3!
156		3✔
157	const promise = new Promise<CheerioAPI>((resolve, reject) => {	3✔
158	undiciStream = undici.stream(url, requestOptions, (res) => {	3✔
159	const contentType = res.headers['content-type'];	3✔
160	const mimeType = new MIMEType(contentType ?? 'text/html');	3!
161		3✔
162	if (!mimeType.isHTML() && !mimeType.isXML()) {	3!
163	throw new RangeError(	×
164	`The content-type "${contentType}" is neither HTML nor XML.`	×
165	);	×
166	}	×
167		3✔
168	// Forward the charset from the header to the decodeStream.	3✔
169	encoding.transportLayerEncodingLabel = mimeType.parameters.get('charset');	3✔
170		3✔
171	/*	3✔
172	* If we allow redirects, we will have entries in the history.	3✔
173	* The last entry will be the final URL.	3✔
174	*/	3✔
175	const history = (	3✔
176	res.context as	3✔
177	\| {	3✔
178	history?: URL[];	3✔
179	}	3✔
180	\| undefined	3✔
181	)?.history;	3!
182		3✔
183	const opts = {	3✔
184	encoding,	3✔
185	// Set XML mode based on the MIME type.	3✔
186	xmlMode: mimeType.isXML(),	3✔
187	// Set the `baseURL` to the final URL.	3✔
188	baseURL: history ? history[history.length - 1] : url,	3!
189	...cheerioOptions,	3✔
190	};	3✔
191		3✔
192	return decodeStream(opts, (err, $) => (err ? reject(err) : resolve($)));	3!
193	});	3✔
194	});	3✔
195		3✔
196	// Let's make sure the request is completed before returning the promise.	3✔
197	await undiciStream;	3✔
198		3✔
199	return promise;	3✔
200	}	3✔

cheeriojs / cheerio / 3525226766

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous