10535348920

Committed 24 Aug 2024 03:23AM UTC coverage: 32.159% (-0.04%) from 32.195%

Build # 10535348920

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #9628 from ForumMagnum/master

[EA] deploy removing survey banner and post sidenotes (not enabled)

Run Details

5017 of 40425 branches covered (12.41%)

Branch coverage included in aggregate %.

68 of 256 new or added lines in 25 files covered. (26.56%)

3 existing lines in 2 files now uncovered.

29025 of 65429 relevant lines covered (44.36%)

190.34 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

83.87

/packages/lesswrong/server/utils/htmlUtil.ts

import cheerio from 'cheerio';
import HtmlLexer from 'html-lexer';


/**
 * Parse an HTML string with cheerio. Server-side only. Provides some cheerio
 * config options, and works around a Typescript annotation problem (which
 * should go away when we version-upgrade Cheerio at some point).
 */
export function cheerioParse(html: string|null) {
  //@ts-ignore (cheerio type annotations sadly don't quite match the actual imported library)
  return cheerio.load(html ?? "", null, false);
}

/**
 * Tokenize HTML. Wraps around the html-lexer library to give it a conventional
 * API taking a string, rather than an awkward streaming thing.
 */
export function tokenizeHtml(html: string): [string,string][] {
  const result: [string,string][] = [];
  const lexer = new HtmlLexer({
    write: (token: [string,string]) => result.push(token),
    end: () => null
  })
  lexer.write(html);
  return result;
}

/**
 * Parse an HTML string with cheerio, and annotate nodes with offsets
 * into the original HTML string.
 *
 * Works by preprocessing the tree with a lexer to insert empty tags
 that look like:
 *   <span class="__offset_marker" offset="123"></span>
 * then finding those spans, transferring their offsets onto text and tag
 * nodes (as a monkeypatched field), then deleting them.
 */
export function cheerioParseAndMarkOffsets(html: string) {
  const tokens = tokenizeHtml(html);
  let preprocessedHtmlSb: string[] = [];
  let offset = 0;
  let isInTag = false;
  
  for (let [tokenType,tokenStr] of tokens) {
    switch(tokenType) {
      case 'startTagStart':
      case 'endTagStart':
        if (offset > 0) {
          preprocessedHtmlSb.push(`<span class="__offset_marker" offset="${offset}"></span>`);
        }
        isInTag = true;
        break;
      case 'tagEnd':
      case 'tagEndAutoclose':
        isInTag = false;
        break;
      case 'space':
        if (isInTag) break;
        // FALLTHROUGH
      case 'data':
        if (tokenStr.length > 0) {
          preprocessedHtmlSb.push(`<span class="__offset_marker" offset="${offset}"></span>`);
        }
        break;
      default:
        break;
    }
    
    preprocessedHtmlSb.push(tokenStr);
    offset += tokenStr.length;
  }
  
  const preprocessedHtml = preprocessedHtmlSb.join('');
  const parsed = cheerioParse(preprocessedHtml);
  let offsetMarkers = parsed('.__offset_marker');
  for (let i=0; i<offsetMarkers.length; i++) {
    let offsetMarker = offsetMarkers[i];
    let offset = parseInt((offsetMarker as cheerio.TagElement).attribs.offset);
    if (offsetMarker.next) {
      (offsetMarker.next as any).offset = offset;
    }
  }
  offsetMarkers.remove();
  
  return parsed;
}

export function htmlContainsFootnotes(html: string) {
  // Post HTML contains footnotes if it contains an element with the class
  // "footnote-reference". Before spending CPU to parse HTML, check whether that
  // string is present at all as a faster check
  if (!html.includes("footnote-reference")) {
    return false;
  }
  const $ = cheerioParse(html);
  const footnotes = $(".footnote-reference");
  return footnotes.length > 0;
}

1	import cheerio from 'cheerio';	289✔
2	import HtmlLexer from 'html-lexer';	24✔
3
4
5	/**
6	* Parse an HTML string with cheerio. Server-side only. Provides some cheerio
7	* config options, and works around a Typescript annotation problem (which
8	* should go away when we version-upgrade Cheerio at some point).
9	*/
10	export function cheerioParse(html: string\|null) {
11	//@ts-ignore (cheerio type annotations sadly don't quite match the actual imported library)
12	return cheerio.load(html ?? "", null, false);	209!
13	}
14
15	/**
16	* Tokenize HTML. Wraps around the html-lexer library to give it a conventional
17	* API taking a string, rather than an awkward streaming thing.
18	*/
19	export function tokenizeHtml(html: string): [string,string][] {
20	const result: [string,string][] = [];	10✔
21	const lexer = new HtmlLexer({	10✔
22	write: (token: [string,string]) => result.push(token),	180✔
23	end: () => null	×
24	})
25	lexer.write(html);	10✔
26	return result;	10✔
27	}
28
29	/**
30	* Parse an HTML string with cheerio, and annotate nodes with offsets
31	* into the original HTML string.
32	*
33	* Works by preprocessing the tree with a lexer to insert empty tags
34	that look like:
35	* <span class="__offset_marker" offset="123"></span>
36	* then finding those spans, transferring their offsets onto text and tag
37	* nodes (as a monkeypatched field), then deleting them.
38	*/
39	export function cheerioParseAndMarkOffsets(html: string) {
40	const tokens = tokenizeHtml(html);	2✔
41	let preprocessedHtmlSb: string[] = [];	2✔
42	let offset = 0;	2✔
43	let isInTag = false;	2✔
44
45	for (let [tokenType,tokenStr] of tokens) {	2✔
46	switch(tokenType) {	38✔
47	case 'startTagStart':
48	case 'endTagStart':
49	if (offset > 0) {	8✔
50	preprocessedHtmlSb.push(`<span class="__offset_marker" offset="${offset}"></span>`);	6✔
51	}
52	isInTag = true;	8✔
53	break;	8✔
54	case 'tagEnd':
55	case 'tagEndAutoclose':
56	isInTag = false;	8✔
57	break;	8✔
58	case 'space':
59	if (isInTag) break;	1!
60	// FALLTHROUGH
61	case 'data':
62	if (tokenStr.length > 0) {	8✔
63	preprocessedHtmlSb.push(`<span class="__offset_marker" offset="${offset}"></span>`);	4✔
64	}
65	break;	8✔
66	default:
67	break;	13✔
68	}
69
70	preprocessedHtmlSb.push(tokenStr);	38✔
71	offset += tokenStr.length;	38✔
72	}
73
74	const preprocessedHtml = preprocessedHtmlSb.join('');	2✔
75	const parsed = cheerioParse(preprocessedHtml);	2✔
76	let offsetMarkers = parsed('.__offset_marker');	2✔
77	for (let i=0; i<offsetMarkers.length; i++) {	2✔
78	let offsetMarker = offsetMarkers[i];	10✔
79	let offset = parseInt((offsetMarker as cheerio.TagElement).attribs.offset);	10✔
80	if (offsetMarker.next) {	10✔
81	(offsetMarker.next as any).offset = offset;	6✔
82	}
83	}
84	offsetMarkers.remove();	2✔
85
86	return parsed;	2✔
87	}
88
89	export function htmlContainsFootnotes(html: string) {
90	// Post HTML contains footnotes if it contains an element with the class
91	// "footnote-reference". Before spending CPU to parse HTML, check whether that
92	// string is present at all as a faster check
NEW 93	if (!html.includes("footnote-reference")) {	×
NEW 94	return false;	×
95	}
NEW 96	const $ = cheerioParse(html);	×
NEW 97	const footnotes = $(".footnote-reference");	×
NEW 98	return footnotes.length > 0;	×
99	}

ForumMagnum / ForumMagnum / 10535348920

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous