10100799001

Committed 25 Jul 2024 08:11PM UTC coverage: 31.861% (+0.003%) from 31.858%

Build # 10100799001

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #9538 from ForumMagnum/some-ModGPT-fixes

[EA Forum only] update ModGPT to GPT-4o, plus some bug fixes

Run Details

4939 of 40606 branches covered (12.16%)

Branch coverage included in aggregate %.

3 of 16 new or added lines in 2 files covered. (18.75%)

6 existing lines in 2 files now uncovered.

28886 of 65559 relevant lines covered (44.06%)

179.46 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

15.79

/packages/lesswrong/server/scripts/generativeModels/testModGPTOnComments.ts

/* eslint-disable no-console */
import sanitizeHtml from 'sanitize-html';
import { Comments } from '../../../lib/collections/comments';
import { sanitizeAllowedTags } from '../../../lib/vulcan-lib/utils';
import { getOpenAI } from '../../languageModels/languageModelIntegration';
import { Vulcan } from '../../vulcan-lib';
import { wrapVulcanAsyncScript } from '../utils';
import * as _ from 'underscore';
import { htmlToText } from 'html-to-text';
import { modGPTPrompt, sanitizeHtmlOptions } from '../../languageModels/modGPT';
import Posts from '../../../lib/collections/posts/collection';
import difference from 'lodash/difference';
import { truncatise } from '../../../lib/truncatise';

/**
 * This was written for the EA Forum to test out having GPT-4o help moderate comments.
 */
Vulcan.testModGPT = wrapVulcanAsyncScript(
  'testModGPT',
  async () => {
    const api = await getOpenAI()
    if (!api) throw new Error("OpenAI API not configured")
    
    const comments = await Comments.find({
      postId: {$exists: true},
      deleted: false
    }, {
      sort: {createdAt: -1},
      limit: 1,
    }).fetch()
  
    for (const comment of comments) {
      const post = await Posts.findOne(comment.postId)
      if (!post) continue
      
      const commentText = sanitizeHtml(comment.contents?.html ?? "", sanitizeHtmlOptions)
      const postText = sanitizeHtml(post.contents?.html ?? "", sanitizeHtmlOptions)
      const postExcerpt = truncatise(postText, {TruncateBy: 'characters', TruncateLength: 300, Strict: true, Suffix: ''})
      
      // Build the message that will be attributed to the user
      let userText = `<post>
        Title: ${post.title}
        Excerpt: ${postExcerpt}
        </post>`
      if (comment.parentCommentId) {
        // If this comment has a parent, include that as well
        const parentComment = await Comments.findOne({_id: comment.parentCommentId})
        if (parentComment) {
          const parentCommentText = sanitizeHtml(parentComment.contents?.html ?? "", sanitizeHtmlOptions)
          userText += `<parent>${parentCommentText}</parent>`
        }
      }
    
      userText += `<comment>${commentText}</comment>`
      
      console.log('============ check comment', comment._id)
      console.log(userText)
      
      const response = await api.chat.completions.create({
        model: 'gpt-4o',
        messages: [
          {role: 'system', content: modGPTPrompt},
          {role: 'user', content: userText},
        ],
      })
      
      const topResult = response.choices[0].message?.content
      if (topResult) {
        console.log('----- ModGPT response:')
        console.log(topResult)
      }
      else throw new Error("API did not return a top result")
    }
  }
)

1	/* eslint-disable no-console */
2	import sanitizeHtml from 'sanitize-html';	18✔
3	import { Comments } from '../../../lib/collections/comments';	18✔
4	import { sanitizeAllowedTags } from '../../../lib/vulcan-lib/utils';
5	import { getOpenAI } from '../../languageModels/languageModelIntegration';	18✔
6	import { Vulcan } from '../../vulcan-lib';	18✔
7	import { wrapVulcanAsyncScript } from '../utils';	18✔
8	import * as _ from 'underscore';
9	import { htmlToText } from 'html-to-text';
10	import { modGPTPrompt, sanitizeHtmlOptions } from '../../languageModels/modGPT';	18✔
11	import Posts from '../../../lib/collections/posts/collection';	18✔
12	import difference from 'lodash/difference';
13	import { truncatise } from '../../../lib/truncatise';	18✔
14
15	/**
16	* This was written for the EA Forum to test out having GPT-4o help moderate comments.
17	*/
18	Vulcan.testModGPT = wrapVulcanAsyncScript(	18✔
19	'testModGPT',
20	async () => {
21	const api = await getOpenAI()	×
22	if (!api) throw new Error("OpenAI API not configured")	×
23
24	const comments = await Comments.find({	×
25	postId: {$exists: true},
26	deleted: false
27	}, {
28	sort: {createdAt: -1},
29	limit: 1,
30	}).fetch()
31
32	for (const comment of comments) {	×
33	const post = await Posts.findOne(comment.postId)	×
34	if (!post) continue	×
35
NEW 36	const commentText = sanitizeHtml(comment.contents?.html ?? "", sanitizeHtmlOptions)	×
NEW 37	const postText = sanitizeHtml(post.contents?.html ?? "", sanitizeHtmlOptions)	×
UNCOV 38	const postExcerpt = truncatise(postText, {TruncateBy: 'characters', TruncateLength: 300, Strict: true, Suffix: ''})	×
39
40	// Build the message that will be attributed to the user
NEW 41	let userText = `<post>	×
42	Title: ${post.title}
43	Excerpt: ${postExcerpt}
44	</post>`
UNCOV 45	if (comment.parentCommentId) {	×
46	// If this comment has a parent, include that as well
47	const parentComment = await Comments.findOne({_id: comment.parentCommentId})	×
48	if (parentComment) {	×
NEW 49	const parentCommentText = sanitizeHtml(parentComment.contents?.html ?? "", sanitizeHtmlOptions)	×
UNCOV 50	userText += `<parent>${parentCommentText}</parent>`	×
51	}
52	}
53
54	userText += `<comment>${commentText}</comment>`	×
55
56	console.log('============ check comment', comment._id)	×
57	console.log(userText)	×
58
59	const response = await api.chat.completions.create({	×
60	model: 'gpt-4o',
61	messages: [
62	{role: 'system', content: modGPTPrompt},
63	{role: 'user', content: userText},
64	],
65	})
66
67	const topResult = response.choices[0].message?.content	×
68	if (topResult) {	×
69	console.log('----- ModGPT response:')	×
70	console.log(topResult)	×
71	}
72	else throw new Error("API did not return a top result")	×
73	}
74	}
75	)

ForumMagnum / ForumMagnum / 10100799001

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous