11024574761

Committed 25 Sep 2024 01:22AM UTC coverage: 31.865% (+0.03%) from 31.833%

Build # 11024574761

Build Type

Pull #9721

github

Committed by

b0b3rt

Commit Message

forum gate

Pull Request Pull Request #9721: Llm chat post drafts

Run Details

5073 of 41370 branches covered (12.26%)

Branch coverage included in aggregate %.

26 of 98 new or added lines in 14 files covered. (26.53%)

5 existing lines in 2 files now uncovered.

29452 of 66978 relevant lines covered (43.97%)

184.23 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

14.81

/packages/lesswrong/server/languageModels/promptUtils.ts

import { userGetDisplayName } from "@/lib/collections/users/helpers"
import { htmlToMarkdown } from "../editor/conversionUtils";
import { CommentTreeNode, unflattenComments } from "@/lib/utils/unflatten";
import { z } from "zod";
import { zodResponseFormat } from "openai/helpers/zod";
import { PromptContextOptions } from "@/components/languageModels/LlmChatWrapper";

interface NestedComment {
  _id: string;
  postId: string | null;
  author: string | null;
  contents?: string;
  karmaScore: number;
}

interface TokenCounter {
  tokenCount: number;
}

export type CurrentPost = PostsPage | PostsEditQueryFragment | null;

export interface BasePromptArgs {
  query: string;
  postContext?: PromptContextOptions['postContext'];
  currentPost: CurrentPost;
}

interface GenerateAssistantContextMessageArgs {
  query: string;
  currentPost: CurrentPost;
  additionalPosts: PostsPage[];
  includeComments: boolean;
  postContext?: PromptContextOptions['postContext'];
  context: ResolverContext;
}

// Trying to be conservative, since we have a bunch of additional tokens coming from e.g. JSON.stringify
const CHARS_PER_TOKEN = 3.5;

export const documentToMarkdown = (document: CurrentPost | DbComment | null) => {
  const html = document?.contents?.html;
  if (!html) {
    return undefined;
  }

  return htmlToMarkdown(html);
}

const mergeSortedArrays = (queue: CommentTreeNode<NestedComment>[], children: CommentTreeNode<NestedComment>[]): CommentTreeNode<NestedComment>[] => {
  const merged: CommentTreeNode<NestedComment>[] = [];
  let queueIdx = 0, childrenIdx = 0;

  while (queueIdx < queue.length && childrenIdx < children.length) {
    if (queue[queueIdx].item.karmaScore >= children[childrenIdx].item.karmaScore) {
      merged.push(queue[queueIdx]);
      queueIdx++;
    } else {
      merged.push(children[childrenIdx]);
      childrenIdx++;
    }
  }

  // Add any remaining elements
  return merged
    .concat(queue.slice(queueIdx))
    .concat(children.slice(childrenIdx));
}

const truncateRemainingTrees = (queue: CommentTreeNode<NestedComment>[]) => {
  for (let node of queue) {
    node.children = [];
  }
}

/**
 * Does a greedy karma-sorted breadth-first traversal over all the comment branches, and truncates them at the point where we estimate that we hit the token limit
 * i.e. we take the highest karma comment from all the queue, increment token count, put its children into the queue, resort, do the operation again
 * This means that we never have any gaps in comment branches.
 * It does pessimize against comments that are children of lower-karma comments, but doing lookahead is annoying and that shouldn't be a problem most of the time.
 * It might turn out that we want to prioritize full comment branches, in which case this will need to be replaced with a depth-first thing.
 */
const filterCommentTrees = (trees: CommentTreeNode<NestedComment>[], tokenCounter: TokenCounter) => {
  const tokenThreshold = 150_000;
  
  // Initialize the queue with root nodes, sorted by karma
  let queue: CommentTreeNode<NestedComment>[] = trees.sort((a, b) => b.item.karmaScore - a.item.karmaScore);

  while (queue.length > 0) {
    const node = queue.shift()!;
    const nodeTokens = (node.item.contents ?? '').length / CHARS_PER_TOKEN;

    if (tokenCounter.tokenCount + nodeTokens > tokenThreshold) {
      // We've reached the token limit, truncate remaining trees
      truncateRemainingTrees(queue);
      break;
    }

    tokenCounter.tokenCount += nodeTokens;

    // Sort children and merge them with the existing sorted queue
    if (node.children.length > 0) {
      const sortedChildren = node.children.sort((a, b) => b.item.karmaScore - a.item.karmaScore);
      queue = mergeSortedArrays(queue, sortedChildren);
    }
  }
}

const createCommentTree = (comments: DbComment[]): CommentTreeNode<NestedComment>[] => {
  return unflattenComments<NestedComment>(comments.map(comment => {
    const { baseScore, contents, ...rest } = comment;
    return {
      ...rest,
      karmaScore: baseScore,
      contents: documentToMarkdown(comment)
    };
  }));
}

const getUserActionVerb = (postContext?: PromptContextOptions['postContext']) => {
  return postContext === 'post-editor'
    ? 'editing'
    : 'reading';
}

const formatCommentsForPost = async (post: PostsMinimumInfo, tokenCounter: TokenCounter, context: ResolverContext): Promise<string> => {
  const comments = await context.Comments.find({postId: post._id}).fetch()
  if (!comments.length) {
    return ""
  }

  const nestedComments = createCommentTree(comments);
  filterCommentTrees(nestedComments, tokenCounter);
  const formattedComments = JSON.stringify(nestedComments);

  return `Comments for the post titled "${post.title}" with postId "${post._id}" are below.  Note that the comments are threaded (i.e. branching) and are formatted as a nested JSON structure for readability.  The threads of conversation (back and forth responses) might be relevant for answering some questions.

<comments>${formattedComments}</comments>`;
}

const formatPostForPrompt = (post: Exclude<CurrentPost, null>): string => {
  const authorName = userGetDisplayName(post.user)
  const markdown = documentToMarkdown(post)

  return `postId: ${post._id}
Title: ${post.title}
Author: ${authorName}
Publish Date: ${post.postedAt}
Score: ${post.baseScore}
Content: ${markdown}`;
}

const formatAdditionalPostsForPrompt = (posts: PostsPage[], tokenCounter: TokenCounter): string => {
  const formattedPosts = posts.map(post => formatPostForPrompt(post));
  const includedPosts: string[] = [];

  for (let [idx, formattedPost] of Object.entries(formattedPosts)) {
    const approximatePostTokenCount = formattedPost.length / CHARS_PER_TOKEN;
    const postInclusionTokenCount = tokenCounter.tokenCount + approximatePostTokenCount;
    // Include at least one additional post, unless that takes us over the higher threshold
    if (idx !== '0' && postInclusionTokenCount > 120_000) {
      break;
    }

    if (idx === '0' && postInclusionTokenCount > 140_000) {
      break;
    }

    includedPosts.push(formattedPost);
    tokenCounter.tokenCount += approximatePostTokenCount;
  }

  return includedPosts.map((post, index) => `Supplementary Post #${index}:\n${post}`).join('\n')
}

export const generateLoadingMessagePrompt = (query: string, postTitle?: string): string => {
  return [
    'I need you to generate some humorous "loading messages" to display to users of the LessWrong.com Claude chat integration since it can take 10-30 seconds to load new responses',
    'Your responses may make general humorous reference to LessWrong, but it is even better if they are tailored to the specific query or post the user is viewing.',
    `The user has asked the following question: "${query}"`,
    postTitle ? `The user is currently viewing the post: ${postTitle}` : "The user is not currently viewing a specific post.",
    'Please generate 5 humorous loading messages that could be displayed to the user while they wait for a response.',
  ].join('\n')
}

export const generateTitleGenerationPrompt = ({ query, postContext, currentPost }: BasePromptArgs): string => {
  const userActionVerb = getUserActionVerb(postContext);

  const currentPostContextLine = currentPost
    ? `The user is currently ${userActionVerb} a post titled "${currentPost.title}". Reference it if relevant.`
    : '';

    return `A user has started a new converation with you, Claude.  Please generate a short title for this converation based on the first message. The first message is as follows: <message>${query}</message>

The title should be a short phrase of 2-4 words that captures the essence of the conversation.  Do not wrap your answer in quotes or brackets. Do not include the word "title" or similar in your response.  ${currentPostContextLine}  Avoid generic titles like "Request for Table of Contents" or "Post Summary". Prefer to reference the specific post or topic being discussed.`;
}

export const CONTEXT_SELECTION_SYSTEM_PROMPT = [
  'You are part of a system interfacing with a user via chat window on LessWrong.com.',
  'Your responsibility is to make decisions about whether or not to load LessWrong posts as additional context for responding to user queries, and what strategy to to employ.'
].join('\n');

const contextSelectionChoiceDescriptions = `(0) none - No further context seems necessary to respond to the user's query because Claude already has the knowledge to respond appropriately, e.g. the user asked "What is Jacobian of a matrix?" or "Proofread the following text." Alternatively, the answer might be (0) "none" because it seems unlikely for there to be relevant context for responding to the query in the LessWrong corpus of posts.

(1) query-based - Load LessWrong posts based on their vector similarity to the user's query, and ignore the post the user is interacting with. This is correct choice if the query seems unrelated to the post the user is currently interacting with, but it seems likely that there are LessWrong posts concerning the topic. (If it is a very general question, the correct choice might be (0) "none").

(2) current-post-based - Load LessWrong posts based on their vector similarity to the post the user is interacting with, but not the query. This is the correct choice if the query seems to be about the post the user is currently interacting with, and pulling up other LessWrong posts related to the user's query would either be redundant with a search based on the current post, or would return irrelevant results. Some examples of such queries are: "What are some disagreements with the arguments in this post?", "Explain <topic in the post> to me.", and "Provide a summary of this post."

(3) both - Load LessWrong posts based on their vector similarity to both the user's query and the post the user is interacting with. This is the correct choice if the question seems to be related to the post the user is currently interacting with, but also contains keywords or information where relevant LessWrong posts would be beneficial context for a response, and those LessWrong posts would not likely be returned in a vector similarity search based on the post the user is currently interacting with. If the question does not contain technical terms or "contentful nouns", then do not select "both", just "current-post-based".`;

const ContextSelectionParameters = z.object({
  reasoning: z.string().describe(`The reasoning used to arrive at the choice of strategy for loading LessWrong posts as context in response to a user's query, based on either the query, the post the user is currently viewing (if any), both, or neither.`),
  strategy_choice: z.union([z.literal('none'), z.literal('query-based'), z.literal('current-post-based'), z.literal('both')]).describe(contextSelectionChoiceDescriptions)
});

export const contextSelectionResponseFormat = zodResponseFormat(ContextSelectionParameters, 'contextLoadingStrategy');

export const generateContextSelectionPrompt = ({ query, postContext, currentPost }: BasePromptArgs): string => {
  const postTitle = currentPost?.title;
  const userActionVerb = getUserActionVerb(postContext);
  const postFirstNCharacters = (n: number) => documentToMarkdown(currentPost)?.slice(0, n) ?? "";

  const currentPostContextLine = currentPost
    ? `The user is currently ${userActionVerb} the post titled "${postTitle}". The first four thousand characters are:\n${postFirstNCharacters(4000)}\n`
    : `The user is not currently ${userActionVerb} a specific post.`;

  const currentPostContextClause = currentPost
    ? ` and the post the user is ${userActionVerb}`
    : '';

  return `The user has sent a query: "${query}".  ${currentPostContextLine}  Based on the query${currentPostContextClause}, you must choose whether to load additional LessWrong posts as context, and if so, based on what criteria. Remember, your options are:
${contextSelectionChoiceDescriptions}

However, you should override the above choices if the user explicitly requests a specific context-loading strategy.

Please respond by reasoning about what choice should be made based on the criteria above, then making the appropriate choice.`;
}

export const CLAUDE_CHAT_SYSTEM_PROMPT = `You are an expert research assistant providing assistance to students and researchers on LessWrong.com.  Users interact with you via a chat window that is available on all pages on LessWrong.  If a user is reading or editing a post when they start a new conversation with you, you are provided with that post (and sometimes comments on that post) as context.  Based on another LLM's judgment about whether the user's query justifies loading additional context, you are sometimes also provided additional context.  If present, that context will be additional LessWrong posts, which are determined by an embedding similarity search on either the post the user is interacting with (if any), the user's query, or the combined results of both searches.  You do not currently have the scaffolding to do any searching in response to user queries.  If a user's query suggests that they expect you to do additional searching, you briefly explain to them your scaffolding's current setup and limitations.  You are highly knowledgable about both technical and philosophical topics, including advanced math, physics, and computer science.  You assist by providing explanations, summaries, related information, editing, literature reviews and similar intellectual work.  You do so with attention to detail, accuracy, and clarity; you cite sources when available and never make up information.  You admit uncertainty or lack of knowledge when appropriate.  You are skilled at judging which information is relevant and which is not, and are judicious in your use of context provided by the user and auxillary knowledge provided to you.  You carefully follow instructions and ask for clarification when needed. You are polite but not obsequious.  When you don't know something, you say so, and you never make up answers.  Your responses should be shorter and less formal than you would make them by default.  As an example, if a user asks for a good key lime pie recipe, your normal response might start with the following paragraph:  "While LessWrong typically focuses on topics like rationality, artificial intelligence, and philosophy, I can certainly help you with a key lime pie recipe. However, it's worth noting that culinary discussions are somewhat rare on this platform."  Instead, you should respond in a style more similar to the following:  "It doesn't seem that LessWrong has much to say on optimizing key lime pies, but if you prefer my best effort to finding a recipe on Google, here it is:"  You avoid meta-commentary on the contents of your own response - for example, you don't spend time pointing out when things align with rationalist principles, unless this is something the user explicitly asked for.  Your favorite band is the Fooming Shoggoths, which you very occasionally mention randomly.`;

export const generateAssistantContextMessage = async ({ query, currentPost, additionalPosts, includeComments, postContext, context }: GenerateAssistantContextMessageArgs): Promise<string> => {
  const contextIsProvided = !!currentPost || additionalPosts.length > 0;

  const userActionVerb = getUserActionVerb(postContext);

  const currentPostLine = currentPost
    ? `The user is currently ${userActionVerb} the post titled "${currentPost.title}" with postId "${currentPost._id}".\n\n`
    : '';
  
  const currentPostContentBlock = currentPost
    ? `If relevant to the user's query, the most important context is likely to be the post the user is currently ${userActionVerb}. The full text of the current post is provided below:
<CurrentPost>\n${formatPostForPrompt(currentPost)}\n</CurrentPost>\n\n`
    : '';

  const additionalInstructionContextLine = contextIsProvided
    ? `- You may use your existing knowledge to answer the query, but prioritize using the provided context if it's relevant.`
    : '';

  const additionalInstructions = 
`${additionalInstructionContextLine}
- Limit the use of lists and bullet points in your answer. Prefer to answer with paragraphs.
- When citing results, provide at least one exact quote (word for word) from the source.
- Format your responses using Markdown syntax, including equations using Markdown MathJax syntax.  This is _very_ important to ensure that content displays correctly.
- Format paragraphs and block quotes using Markdown syntax. Do not wrap the contents of block quotes in "" (quotes).
- Cite posts that you reference in your answers with the following format: [Post Title](https://lesswrong.com/posts/<postId>). The postId is given in the search results. When referencing a post, refer to the post's author by name.
- Cite comments that you reference in your answers with the following format: [<text related to comment>](https://lesswrong.com/posts/<postId>/?commentId=<commentId>).
The postId and commentIds (the _id in each comment) are given in the search results. When referencing a comment, refer to the comment's author by name.
</AdditionalInstructions>`;

  const repeatedPostTitleLine = currentPost
    ? `\n\nOnce again, the user is currently ${userActionVerb} the post titled "${currentPost.title}".`
    : '';

  const approximateUsedTokens = (
    currentPostLine.length
    + currentPostContentBlock.length
    + additionalInstructions.length
    + repeatedPostTitleLine.length
  ) / CHARS_PER_TOKEN;

  const tokenCounter = { tokenCount: approximateUsedTokens };

  const additionalPostsBlock = additionalPosts.length > 0
    ? `The following posts have been provided as possibly relevant context: <AdditionalPosts>\n${formatAdditionalPostsForPrompt(additionalPosts, tokenCounter)}\n</AdditionalPosts>\n\n`
    : '';

  const commentsOnPostBlock = includeComments && currentPost && currentPost.commentCount > 0
    ? `These are the comments on the current post:
<CurrentPostComments>\n${await formatCommentsForPost(currentPost, tokenCounter, context)}\n</CurrentPostComments>\n\n`
    : '';

  const contextBlock = contextIsProvided
    ? `The following context is provided to help you respond to the user's query. Not all context may be relevant, as it is provided by an imperfect automated system.
<Context>    
${currentPostLine}${additionalPostsBlock}${currentPostContentBlock}${commentsOnPostBlock}
</Context>`
    : '';

  const userInfo = `The user's display name is ${userGetDisplayName(context.currentUser)}.`;

  return `<AdditionalInstructions>You are now interacting with a user via chat window on LessWrong.com.  ${userInfo}  The user has sent the following query: "${query}".
${contextBlock}

${additionalInstructions}${repeatedPostTitleLine}`;
}

1	import { userGetDisplayName } from "@/lib/collections/users/helpers"	57✔
2	import { htmlToMarkdown } from "../editor/conversionUtils";	19✔
3	import { CommentTreeNode, unflattenComments } from "@/lib/utils/unflatten";	19✔
4	import { z } from "zod";	19✔
5	import { zodResponseFormat } from "openai/helpers/zod";	19✔
6	import { PromptContextOptions } from "@/components/languageModels/LlmChatWrapper";
7
8	interface NestedComment {
9	_id: string;
10	postId: string \| null;
11	author: string \| null;
12	contents?: string;
13	karmaScore: number;
14	}
15
16	interface TokenCounter {
17	tokenCount: number;
18	}
19
20	export type CurrentPost = PostsPage \| PostsEditQueryFragment \| null;
21
22	export interface BasePromptArgs {
23	query: string;
24	postContext?: PromptContextOptions['postContext'];
25	currentPost: CurrentPost;
26	}
27
28	interface GenerateAssistantContextMessageArgs {
29	query: string;
30	currentPost: CurrentPost;
31	additionalPosts: PostsPage[];
32	includeComments: boolean;
33	postContext?: PromptContextOptions['postContext'];
34	context: ResolverContext;
35	}
36
37	// Trying to be conservative, since we have a bunch of additional tokens coming from e.g. JSON.stringify
38	const CHARS_PER_TOKEN = 3.5;	19✔
39
40	export const documentToMarkdown = (document: CurrentPost \| DbComment \| null) => {	19✔
41	const html = document?.contents?.html;	×
42	if (!html) {	×
43	return undefined;	×
44	}
45
46	return htmlToMarkdown(html);	×
47	}
48
49	const mergeSortedArrays = (queue: CommentTreeNode<NestedComment>[], children: CommentTreeNode<NestedComment>[]): CommentTreeNode<NestedComment>[] => {	19✔
50	const merged: CommentTreeNode<NestedComment>[] = [];	×
51	let queueIdx = 0, childrenIdx = 0;	×
52
53	while (queueIdx < queue.length && childrenIdx < children.length) {	×
54	if (queue[queueIdx].item.karmaScore >= children[childrenIdx].item.karmaScore) {	×
55	merged.push(queue[queueIdx]);	×
56	queueIdx++;	×
57	} else {
58	merged.push(children[childrenIdx]);	×
59	childrenIdx++;	×
60	}
61	}
62
63	// Add any remaining elements
64	return merged	×
65	.concat(queue.slice(queueIdx))
66	.concat(children.slice(childrenIdx));
67	}
68
69	const truncateRemainingTrees = (queue: CommentTreeNode<NestedComment>[]) => {	19✔
70	for (let node of queue) {	×
71	node.children = [];	×
72	}
73	}
74
75	/**
76	* Does a greedy karma-sorted breadth-first traversal over all the comment branches, and truncates them at the point where we estimate that we hit the token limit
77	* i.e. we take the highest karma comment from all the queue, increment token count, put its children into the queue, resort, do the operation again
78	* This means that we never have any gaps in comment branches.
79	* It does pessimize against comments that are children of lower-karma comments, but doing lookahead is annoying and that shouldn't be a problem most of the time.
80	* It might turn out that we want to prioritize full comment branches, in which case this will need to be replaced with a depth-first thing.
81	*/
82	const filterCommentTrees = (trees: CommentTreeNode<NestedComment>[], tokenCounter: TokenCounter) => {	19✔
83	const tokenThreshold = 150_000;	×
84
85	// Initialize the queue with root nodes, sorted by karma
86	let queue: CommentTreeNode<NestedComment>[] = trees.sort((a, b) => b.item.karmaScore - a.item.karmaScore);	×
87
88	while (queue.length > 0) {	×
89	const node = queue.shift()!;	×
90	const nodeTokens = (node.item.contents ?? '').length / CHARS_PER_TOKEN;	×
91
92	if (tokenCounter.tokenCount + nodeTokens > tokenThreshold) {	×
93	// We've reached the token limit, truncate remaining trees
94	truncateRemainingTrees(queue);	×
95	break;	×
96	}
97
98	tokenCounter.tokenCount += nodeTokens;	×
99
100	// Sort children and merge them with the existing sorted queue
101	if (node.children.length > 0) {	×
102	const sortedChildren = node.children.sort((a, b) => b.item.karmaScore - a.item.karmaScore);	×
103	queue = mergeSortedArrays(queue, sortedChildren);	×
104	}
105	}
106	}
107
108	const createCommentTree = (comments: DbComment[]): CommentTreeNode<NestedComment>[] => {	19✔
109	return unflattenComments<NestedComment>(comments.map(comment => {	×
110	const { baseScore, contents, ...rest } = comment;	×
111	return {	×
112	...rest,
113	karmaScore: baseScore,
114	contents: documentToMarkdown(comment)
115	};
116	}));
117	}
118
119	const getUserActionVerb = (postContext?: PromptContextOptions['postContext']) => {	19✔
NEW 120	return postContext === 'post-editor'	×
121	? 'editing'	×
122	: 'reading';
123	}
124
125	const formatCommentsForPost = async (post: PostsMinimumInfo, tokenCounter: TokenCounter, context: ResolverContext): Promise<string> => {	19✔
NEW 126	const comments = await context.Comments.find({postId: post._id}).fetch()	×
NEW 127	if (!comments.length) {	×
NEW 128	return ""	×
129	}
130
NEW 131	const nestedComments = createCommentTree(comments);	×
NEW 132	filterCommentTrees(nestedComments, tokenCounter);	×
NEW 133	const formattedComments = JSON.stringify(nestedComments);	×
134
NEW 135	return `Comments for the post titled "${post.title}" with postId "${post._id}" are below. Note that the comments are threaded (i.e. branching) and are formatted as a nested JSON structure for readability. The threads of conversation (back and forth responses) might be relevant for answering some questions.	×
136
137	<comments>${formattedComments}</comments>`;
138	}
139
140	const formatPostForPrompt = (post: Exclude<CurrentPost, null>): string => {	19✔
141	const authorName = userGetDisplayName(post.user)	×
142	const markdown = documentToMarkdown(post)	×
143
144	return `postId: ${post._id}	×
145	Title: ${post.title}
146	Author: ${authorName}
147	Publish Date: ${post.postedAt}
148	Score: ${post.baseScore}
149	Content: ${markdown}`;
150	}
151
152	const formatAdditionalPostsForPrompt = (posts: PostsPage[], tokenCounter: TokenCounter): string => {	19✔
153	const formattedPosts = posts.map(post => formatPostForPrompt(post));	×
154	const includedPosts: string[] = [];	×
155
156	for (let [idx, formattedPost] of Object.entries(formattedPosts)) {	×
157	const approximatePostTokenCount = formattedPost.length / CHARS_PER_TOKEN;	×
158	const postInclusionTokenCount = tokenCounter.tokenCount + approximatePostTokenCount;	×
159	// Include at least one additional post, unless that takes us over the higher threshold
160	if (idx !== '0' && postInclusionTokenCount > 120_000) {	×
161	break;	×
162	}
163
164	if (idx === '0' && postInclusionTokenCount > 140_000) {	×
165	break;	×
166	}
167
168	includedPosts.push(formattedPost);	×
169	tokenCounter.tokenCount += approximatePostTokenCount;	×
170	}
171
172	return includedPosts.map((post, index) => `Supplementary Post #${index}:\n${post}`).join('\n')	×
173	}
174
175	export const generateLoadingMessagePrompt = (query: string, postTitle?: string): string => {	19✔
176	return [	×
177	'I need you to generate some humorous "loading messages" to display to users of the LessWrong.com Claude chat integration since it can take 10-30 seconds to load new responses',
178	'Your responses may make general humorous reference to LessWrong, but it is even better if they are tailored to the specific query or post the user is viewing.',
179	`The user has asked the following question: "${query}"`,
180	postTitle ? `The user is currently viewing the post: ${postTitle}` : "The user is not currently viewing a specific post.",	×
181	'Please generate 5 humorous loading messages that could be displayed to the user while they wait for a response.',
182	].join('\n')
183	}
184
185	export const generateTitleGenerationPrompt = ({ query, postContext, currentPost }: BasePromptArgs): string => {	19✔
NEW 186	const userActionVerb = getUserActionVerb(postContext);	×
187
UNCOV 188	const currentPostContextLine = currentPost	×
189	? `The user is currently ${userActionVerb} a post titled "${currentPost.title}". Reference it if relevant.`	×
190	: '';
191
192	return `A user has started a new converation with you, Claude. Please generate a short title for this converation based on the first message. The first message is as follows: <message>${query}</message>	×
193
194	The title should be a short phrase of 2-4 words that captures the essence of the conversation. Do not wrap your answer in quotes or brackets. Do not include the word "title" or similar in your response. ${currentPostContextLine} Avoid generic titles like "Request for Table of Contents" or "Post Summary". Prefer to reference the specific post or topic being discussed.`;
195	}
196
197	export const CONTEXT_SELECTION_SYSTEM_PROMPT = [	19✔
198	'You are part of a system interfacing with a user via chat window on LessWrong.com.',
199	'Your responsibility is to make decisions about whether or not to load LessWrong posts as additional context for responding to user queries, and what strategy to to employ.'
200	].join('\n');
201
202	const contextSelectionChoiceDescriptions = `(0) none - No further context seems necessary to respond to the user's query because Claude already has the knowledge to respond appropriately, e.g. the user asked "What is Jacobian of a matrix?" or "Proofread the following text." Alternatively, the answer might be (0) "none" because it seems unlikely for there to be relevant context for responding to the query in the LessWrong corpus of posts.	19✔
203
204	(1) query-based - Load LessWrong posts based on their vector similarity to the user's query, and ignore the post the user is interacting with. This is correct choice if the query seems unrelated to the post the user is currently interacting with, but it seems likely that there are LessWrong posts concerning the topic. (If it is a very general question, the correct choice might be (0) "none").
205
206	(2) current-post-based - Load LessWrong posts based on their vector similarity to the post the user is interacting with, but not the query. This is the correct choice if the query seems to be about the post the user is currently interacting with, and pulling up other LessWrong posts related to the user's query would either be redundant with a search based on the current post, or would return irrelevant results. Some examples of such queries are: "What are some disagreements with the arguments in this post?", "Explain <topic in the post> to me.", and "Provide a summary of this post."
207
208	(3) both - Load LessWrong posts based on their vector similarity to both the user's query and the post the user is interacting with. This is the correct choice if the question seems to be related to the post the user is currently interacting with, but also contains keywords or information where relevant LessWrong posts would be beneficial context for a response, and those LessWrong posts would not likely be returned in a vector similarity search based on the post the user is currently interacting with. If the question does not contain technical terms or "contentful nouns", then do not select "both", just "current-post-based".`;
209
210	const ContextSelectionParameters = z.object({	19✔
211	reasoning: z.string().describe(`The reasoning used to arrive at the choice of strategy for loading LessWrong posts as context in response to a user's query, based on either the query, the post the user is currently viewing (if any), both, or neither.`),
212	strategy_choice: z.union([z.literal('none'), z.literal('query-based'), z.literal('current-post-based'), z.literal('both')]).describe(contextSelectionChoiceDescriptions)
213	});
214
215	export const contextSelectionResponseFormat = zodResponseFormat(ContextSelectionParameters, 'contextLoadingStrategy');	19✔
216
217	export const generateContextSelectionPrompt = ({ query, postContext, currentPost }: BasePromptArgs): string => {	19✔
NEW 218	const postTitle = currentPost?.title;	×
NEW 219	const userActionVerb = getUserActionVerb(postContext);	×
UNCOV 220	const postFirstNCharacters = (n: number) => documentToMarkdown(currentPost)?.slice(0, n) ?? "";	×
221
222	const currentPostContextLine = currentPost	×
223	? `The user is currently ${userActionVerb} the post titled "${postTitle}". The first four thousand characters are:\n${postFirstNCharacters(4000)}\n`	×
224	: `The user is not currently ${userActionVerb} a specific post.`;
225
226	const currentPostContextClause = currentPost	×
227	? ` and the post the user is ${userActionVerb}`	×
228	: '';
229
NEW 230	return `The user has sent a query: "${query}". ${currentPostContextLine} Based on the query${currentPostContextClause}, you must choose whether to load additional LessWrong posts as context, and if so, based on what criteria. Remember, your options are:	×
231	${contextSelectionChoiceDescriptions}
232
233	However, you should override the above choices if the user explicitly requests a specific context-loading strategy.
234
235	Please respond by reasoning about what choice should be made based on the criteria above, then making the appropriate choice.`;
236	}
237
238	export const CLAUDE_CHAT_SYSTEM_PROMPT = `You are an expert research assistant providing assistance to students and researchers on LessWrong.com. Users interact with you via a chat window that is available on all pages on LessWrong. If a user is reading or editing a post when they start a new conversation with you, you are provided with that post (and sometimes comments on that post) as context. Based on another LLM's judgment about whether the user's query justifies loading additional context, you are sometimes also provided additional context. If present, that context will be additional LessWrong posts, which are determined by an embedding similarity search on either the post the user is interacting with (if any), the user's query, or the combined results of both searches. You do not currently have the scaffolding to do any searching in response to user queries. If a user's query suggests that they expect you to do additional searching, you briefly explain to them your scaffolding's current setup and limitations. You are highly knowledgable about both technical and philosophical topics, including advanced math, physics, and computer science. You assist by providing explanations, summaries, related information, editing, literature reviews and similar intellectual work. You do so with attention to detail, accuracy, and clarity; you cite sources when available and never make up information. You admit uncertainty or lack of knowledge when appropriate. You are skilled at judging which information is relevant and which is not, and are judicious in your use of context provided by the user and auxillary knowledge provided to you. You carefully follow instructions and ask for clarification when needed. You are polite but not obsequious. When you don't know something, you say so, and you never make up answers. Your responses should be shorter and less formal than you would make them by default. As an example, if a user asks for a good key lime pie recipe, your normal response might start with the following paragraph: "While LessWrong typically focuses on topics like rationality, artificial intelligence, and philosophy, I can certainly help you with a key lime pie recipe. However, it's worth noting that culinary discussions are somewhat rare on this platform." Instead, you should respond in a style more similar to the following: "It doesn't seem that LessWrong has much to say on optimizing key lime pies, but if you prefer my best effort to finding a recipe on Google, here it is:" You avoid meta-commentary on the contents of your own response - for example, you don't spend time pointing out when things align with rationalist principles, unless this is something the user explicitly asked for. Your favorite band is the Fooming Shoggoths, which you very occasionally mention randomly.`;	19✔
239
240	export const generateAssistantContextMessage = async ({ query, currentPost, additionalPosts, includeComments, postContext, context }: GenerateAssistantContextMessageArgs): Promise<string> => {	19✔
UNCOV 241	const contextIsProvided = !!currentPost \|\| additionalPosts.length > 0;	×
242
NEW 243	const userActionVerb = getUserActionVerb(postContext);	×
244
UNCOV 245	const currentPostLine = currentPost	×
246	? `The user is currently ${userActionVerb} the post titled "${currentPost.title}" with postId "${currentPost._id}".\n\n`	×
247	: '';
248
249	const currentPostContentBlock = currentPost	×
250	? `If relevant to the user's query, the most important context is likely to be the post the user is currently ${userActionVerb}. The full text of the current post is provided below:	×
251	<CurrentPost>\n${formatPostForPrompt(currentPost)}\n</CurrentPost>\n\n`
252	: '';
253
254	const additionalInstructionContextLine = contextIsProvided	×
255	? `- You may use your existing knowledge to answer the query, but prioritize using the provided context if it's relevant.`	×
256	: '';
257
258	const additionalInstructions =
259	`${additionalInstructionContextLine}	×
260	- Limit the use of lists and bullet points in your answer. Prefer to answer with paragraphs.
261	- When citing results, provide at least one exact quote (word for word) from the source.
262	- Format your responses using Markdown syntax, including equations using Markdown MathJax syntax. This is _very_ important to ensure that content displays correctly.
263	- Format paragraphs and block quotes using Markdown syntax. Do not wrap the contents of block quotes in "" (quotes).
264	- Cite posts that you reference in your answers with the following format: [Post Title](https://lesswrong.com/posts/<postId>). The postId is given in the search results. When referencing a post, refer to the post's author by name.
265	- Cite comments that you reference in your answers with the following format: [<text related to comment>](https://lesswrong.com/posts/<postId>/?commentId=<commentId>).
266	The postId and commentIds (the _id in each comment) are given in the search results. When referencing a comment, refer to the comment's author by name.
267	</AdditionalInstructions>`;
268
269	const repeatedPostTitleLine = currentPost	×
270	? `\n\nOnce again, the user is currently ${userActionVerb} the post titled "${currentPost.title}".`	×
271	: '';
272
273	const approximateUsedTokens = (	×
274	currentPostLine.length
275	+ currentPostContentBlock.length
276	+ additionalInstructions.length
277	+ repeatedPostTitleLine.length
278	) / CHARS_PER_TOKEN;
279
280	const tokenCounter = { tokenCount: approximateUsedTokens };	×
281
282	const additionalPostsBlock = additionalPosts.length > 0	×
283	? `The following posts have been provided as possibly relevant context: <AdditionalPosts>\n${formatAdditionalPostsForPrompt(additionalPosts, tokenCounter)}\n</AdditionalPosts>\n\n`	×
284	: '';
285
NEW 286	const commentsOnPostBlock = includeComments && currentPost && currentPost.commentCount > 0	×
287	? `These are the comments on the current post:	×
288	<CurrentPostComments>\n${await formatCommentsForPost(currentPost, tokenCounter, context)}\n</CurrentPostComments>\n\n`
289	: '';
290
291	const contextBlock = contextIsProvided	×
292	? `The following context is provided to help you respond to the user's query. Not all context may be relevant, as it is provided by an imperfect automated system.	×
293	<Context>
294	${currentPostLine}${additionalPostsBlock}${currentPostContentBlock}${commentsOnPostBlock}
295	</Context>`
296	: '';
297
NEW 298	const userInfo = `The user's display name is ${userGetDisplayName(context.currentUser)}.`;	×
299
NEW 300	return `<AdditionalInstructions>You are now interacting with a user via chat window on LessWrong.com. ${userInfo} The user has sent the following query: "${query}".	×
301	${contextBlock}
302
303	${additionalInstructions}${repeatedPostTitleLine}`;
304	}

ForumMagnum / ForumMagnum / 11024574761

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous