11599960914

Committed 30 Oct 2024 07:04PM UTC coverage: 31.324% (-0.005%) from 31.329%

Build # 11599960914

Build Type

Pull #9932

github

Committed by

web-flow

Commit Message

Merge d9a64dc54 into f175ebd6e

Pull Request Pull Request #9932: Deploy LessWrong master

Run Details

5031 of 42396 branches covered (11.87%)

Branch coverage included in aggregate %.

3 of 19 new or added lines in 2 files covered. (15.79%)

1 existing line in 1 file now uncovered.

29649 of 68319 relevant lines covered (43.4%)

173.8 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

13.74

/packages/lesswrong/server/resolvers/jargonResolvers/jargonTermMutations.ts

import { jargonBotClaudeKey } from '@/lib/instanceSettings';
import { defineMutation } from '../../utils/serverGraphqlUtil';
import { getAnthropicPromptCachingClientOrThrow } from '../../languageModels/anthropicClient';
import JargonTerms from '@/lib/collections/jargonTerms/collection';
import { createMutator, sanitize } from '../../vulcan-lib';
import { initialGlossaryPrompt } from './jargonPrompts';
import { fetchFragmentSingle } from '@/server/fetchFragment';
import { htmlToMarkdown } from '@/server/editor/conversionUtils';
import { userCanCreateAndEditJargonTerms } from '@/lib/betas';
import { getAdminTeamAccount } from '@/server/callbacks/commentCallbacks';
import { z } from 'zod';
import { getSqlClientOrThrow } from '@/server/sql/sqlClient';
import { cyrb53Rand } from '@/server/perfMetrics';
import JargonTermsRepo from '@/server/repos/JargonTermsRepo';
import keyBy from 'lodash/keyBy';
import { randomId } from '@/lib/random';
import { filterNonnull } from '@/lib/utils/typeGuardUtils';
import { defaultExampleTerm, defaultExamplePost, defaultGlossaryPrompt, defaultExampleAltTerm, defaultExampleDefinition } from '@/components/jargon/GlossaryEditForm';

import type { Tool } from '@anthropic-ai/sdk/resources';
import type { PromptCachingBetaMessageParam } from '@anthropic-ai/sdk/resources/beta/prompt-caching/messages';
import uniq from 'lodash/uniq';

interface JargonTermGenerationExampleParams {
  glossaryPrompt?: string;
  examplePost?: string;
  exampleTerm?: string;
  exampleAltTerm?: string;
  exampleDefinition?: string;
}

interface CreateJargonTermsQueryParams extends JargonTermGenerationExampleParams {
  postId: string;
  currentUser: DbUser;
}

interface SingleJargonTermExplanationQueryParams extends JargonTermGenerationExampleParams {
  markdown: string;
  term: string;
  toolUseId: string;
}

interface ExplanationsGenerationQueryParams extends JargonTermGenerationExampleParams {
  post: PostsPage;
  excludeTerms: string[];
}

const jargonTermListResponseSchema = z.object({
  jargonTerms: z.array(z.string())
});

const jargonTermSchema = z.object({
  term: z.string(),
  altTerms: z.array(z.string()).optional(),
  htmlContent: z.string(),
});

type LLMGeneratedJargonTerm = z.infer<typeof jargonTermSchema>;

const returnJargonTermsTool: Tool = {
  name: "return_jargon_terms",
  description: "A tool that allows Claude to return a list of jargon terms extracted from a post.",
  input_schema: {
    type: "object",
    properties: {
      jargonTerms: { type: "array", items: { type: "string" } }
    }
  }
};

const generateSingleJargonGlossaryItemTool: Tool = {
  name: "generate_jargon_glossary_item",
  description: "A tool that generates a jargon glossary item for a given post.  It should be provided with a glossary item, which include the term (the identifier), altTerms (alternate spellings or forms of the term), and htmlContent (the explanation of the term).",
  input_schema: {
    type: "object",
    properties: {
      term: { type: "string" },
      altTerms: { type: "array", items: { type: "string" } },
      htmlContent: { type: "string" }
    }
  }
};

class AdvisoryLockError extends Error {}

/**
 * Attempts to acquire an advisory lock and execute a callback.
 * 
 * @param rawLockId - A unique identifier for the lock, such that attempting to run the "same" operation as determined by the ID will fail if another such operation is already running.
 * @param callback - The async function to be executed if the lock is acquired.
 * @returns The result of the callback function.
 * @throws {AdvisoryLockError} If the lock cannot be acquired.
 * @throws {Error} If the callback throws an error (after releasing the lock).
 * 
 * @remarks
 * - If the lock is not acquired, it throws a LockError.
 * - If the lock is acquired but the callback throws an error, the lock is released and the error is thrown.
 * - If the lock is acquired and the callback completes successfully, the lock is released and the callback's return value is returned.
 */
async function executeWithLock<T>(rawLockId: string, callback: () => Promise<T>): Promise<T> {
  const db = getSqlClientOrThrow();
  // We need to convert the lockId to a number because the advisory lock is a 64-bit integer
  // cyrb53Rand returns a double between 0 and 1, so multiplying by 1e15 gives us a number between 0 and 1e15
  const lockId = Math.floor(cyrb53Rand(rawLockId) * 1e15);

  return await db.task(async (task) => {  
    let lockAcquired = false;
    let lockResult;
    try {
      // Attempt to acquire the advisory lock.  Four possible outcomes:
      // 1. The lock is acquired
      // 2. The lock is not acquired
      // 3. The lock is acquired, but the query throws an error
      // 4. The lock is not acquired, and the query throws an error
      lockResult = await task.any<{ pg_try_advisory_lock: boolean }>('SELECT pg_try_advisory_lock($1)', [lockId]);
    } catch (err) {
      // This deals with cases 3 and 4 (query threw an error, lock may or may not be acquired)
      // Case 4 is unfortunate, since we might release someone else's lock, but it's better than leaving the lock in place
      await task.any('SELECT pg_advisory_unlock($1)', [lockId]);
      throw err;
    }
    
    try {
      // If case 1, we note that the lock was acquired and run the callback
      // Case 2 falls through to the error throw below
      if (lockResult[0].pg_try_advisory_lock) {
        lockAcquired = true;
        return await callback();
      }
    } finally {
      // Only release the lock if we acquired it, since we don't want to release someone else's lock
      if (lockAcquired) {
        await task.any('SELECT pg_advisory_unlock($1)', [lockId]);
      }
    }

    // Case 2: Lock not acquired.  This is the only case where we don't release the lock, since we don't want to release someone else's lock.
    // We throw an error so the caller knows they tried to run an operation that required the lock, but it was already locked by someone else.
    throw new AdvisoryLockError(`Lock could not be acquired for lockId: ${rawLockId}`);
  });
}

function sanitizeJargonTerms(jargonTerms: LLMGeneratedJargonTerm[]) {
  return jargonTerms.map(jargonTerm => ({
    ...jargonTerm,
    // In the case where the term is LaTeX, it's probably represented as HTML, so also needs to be sanitized
    term: sanitize(jargonTerm.term),
    htmlContent: sanitize(jargonTerm.htmlContent)
  }));
}

export const queryClaudeForTerms = async (markdown: string) => {
  const client = getAnthropicPromptCachingClientOrThrow(jargonBotClaudeKey.get());
  const messages = [{
    role: "user" as const, 
    content: [{
      type: "text" as const,
      text: `${initialGlossaryPrompt} The post is: <Post>${markdown}</Post>.
      
The jargon terms are:`
    }]
  }];

  const termsResponse = await client.messages.create({
    model: "claude-3-5-sonnet-20240620",
    max_tokens: 5000,
    messages,
    tools: [returnJargonTermsTool],
    tool_choice: { type: "tool", name: "return_jargon_terms" }
  });

  if (termsResponse.content[0].type === "text") {
    // eslint-disable-next-line no-console
    console.error(`Claude responded with text, but we expected a tool use.`)
    return [];
  }
  
  const responseContent = termsResponse.content[0]?.input;
  const parsedResponse = jargonTermListResponseSchema.safeParse(responseContent);
  if (!parsedResponse.success) {
    // eslint-disable-next-line no-console
    console.error(`Claude's response when getting jargon terms doesn't match the expected format.`)
    return [];
  }

  return parsedResponse.data.jargonTerms;
}

async function createSingleExplanationMessageWithExample({ markdown, term, toolUseId, ...exampleParams }: SingleJargonTermExplanationQueryParams): Promise<PromptCachingBetaMessageParam[]> {
  const finalSystemPrompt = exampleParams.glossaryPrompt ?? defaultGlossaryPrompt
  const finalExamplePost = exampleParams.examplePost ?? defaultExamplePost
  const finalExampleTerm = exampleParams.exampleTerm ?? defaultExampleTerm
  const finalExampleAltTerm = exampleParams.exampleAltTerm ?? defaultExampleAltTerm
  const finalExampleDefinition = exampleParams.exampleDefinition ?? defaultExampleDefinition
  
  const finalExampleGlossary = {
    term: finalExampleTerm,
    altTerms: [finalExampleAltTerm],
    htmlContent: finalExampleDefinition
  }

  return [{
    role: "user",
    content: [{
      type: "text",
      text: `${finalSystemPrompt}\n\nHere's an example post: <Post>${finalExamplePost}</Post>. And here's an example term: <Term>${finalExampleTerm}</Term>`
    }]
  },
  {
    role: "assistant",
    content: [{
      type: "tool_use",
      id: toolUseId,
      name: "generate_jargon_glossary",
      input: finalExampleGlossary
    }]
  },
  {
    role: "user",
    content: [{
      type: "tool_result",
      tool_use_id: toolUseId,
    }, {
      type: "text",
      text: `Thanks!  Now can you do the following post?  This time, you'll only need to do one term. <Post>${markdown}</Post>.`,
      cache_control: { type: 'ephemeral' }
    }]
  }, {
    role: "user",
    content: [{
      type: "text",
      text: `  The jargon term is: <Term>${term}</Term>`
    }]
  }];
}

const queryClaudeForSingleJargonExplanation = async ({ markdown, term, toolUseId, ...exampleParams }: SingleJargonTermExplanationQueryParams): Promise<LLMGeneratedJargonTerm | null> => {
  const client = getAnthropicPromptCachingClientOrThrow(jargonBotClaudeKey.get());
  const messages: PromptCachingBetaMessageParam[] = await createSingleExplanationMessageWithExample({ markdown, term, toolUseId, ...exampleParams });

  const response = await client.messages.create({
    model: "claude-3-5-sonnet-20240620",
    max_tokens: 512,
    messages,
    tools: [generateSingleJargonGlossaryItemTool],
    tool_choice: { type: "tool", name: "generate_jargon_glossary_item" }
  });

  if (response.content[0].type === "text") {
    // eslint-disable-next-line no-console
    console.error(`Claude responded with text, but we expected a tool use.`)
    return null;
  }

  const responseContent = response.content[0]?.input;
  const validatedTerm = jargonTermSchema.safeParse(responseContent);
  if (!validatedTerm.success) {
    // eslint-disable-next-line no-console
    console.error('Invalid jargon term:', validatedTerm.error);
    return null;
  }
  const parsedJargonTerm = validatedTerm.data;
  return sanitizeJargonTerms([parsedJargonTerm])[0];
}

export async function createEnglishExplanations({ post, excludeTerms, ...exampleParams }: ExplanationsGenerationQueryParams): Promise<LLMGeneratedJargonTerm[]> {
  const originalHtml = post.contents?.html ?? "";
  const originalMarkdown = htmlToMarkdown(originalHtml);
  const markdown = (originalMarkdown.length < 200_000) ? originalMarkdown : originalMarkdown.slice(0, 200_000);

  const terms = await queryClaudeForTerms(markdown);
  if (!terms.length) {
    return [];
  }

  const newTerms = terms.filter(term => {
    const lowerCaseTerm = term.toLowerCase();
    return !excludeTerms.includes(lowerCaseTerm) && post.contents?.html?.toLowerCase().includes(lowerCaseTerm);
  });

  const toolUseId = randomId();
  const [firstTerm, ...remainingTerms] = newTerms;
  // Do one term first to ensure the shared prompt is cached, before doing the others in parallel
  const firstExplanation = await queryClaudeForSingleJargonExplanation({ markdown, term: firstTerm, toolUseId, ...exampleParams });
  const remainingExplanations = await Promise.all(remainingTerms.map((term) => queryClaudeForSingleJargonExplanation({ markdown, term, toolUseId, ...exampleParams })));
  return filterNonnull([firstExplanation, ...remainingExplanations]);
}

const processedTerms = (jargonTerms: DbJargonTerm[]) => {
  return jargonTerms.flatMap(jargonTerm => [jargonTerm.term.toLowerCase(), ...jargonTerm.altTerms.map(altTerm => altTerm.toLowerCase())]);
}

export const createNewJargonTerms = async ({ postId, currentUser, ...exampleParams }: CreateJargonTermsQueryParams) => {
  const post = await fetchFragmentSingle({
    collectionName: 'Posts',
    fragmentName: 'PostsPage',
    currentUser,
    selector: { _id: postId },
  });

  if (!post) {
    throw new Error('Post not found');
  }

  const [authorsOtherPostJargonTerms, jargonTermsFromThisPost] = await Promise.all([ 
    (new JargonTermsRepo()).getAuthorsOtherJargonTerms(currentUser._id, postId),
    JargonTerms.find({ postId }).fetch()
  ]);
  const existingJargonTerms = [...authorsOtherPostJargonTerms, ...jargonTermsFromThisPost];

  const termsToExclude = uniq(processedTerms(existingJargonTerms));

  const presentTerms = existingJargonTerms.filter(jargonTerm => post.contents?.html?.includes(jargonTerm.term));
  const jargonTermsToCopy = presentTerms.filter(jargonTerm => {
    const terms = processedTerms([jargonTerm]);
    return !termsToExclude.some(excludedTerm => terms.includes(excludedTerm));
  });

  let newJargonTerms;
  try {
    const rawLockId = `jargonTermsLock:${postId}`;
    // Test lock by sleeping for 10 seconds

    newJargonTerms = await executeWithLock(rawLockId, async () => {
      const newEnglishJargon = await createEnglishExplanations({ post, excludeTerms: termsToExclude, ...exampleParams });

      const botAccount = await getAdminTeamAccount();
      
      const createdTerms = await Promise.all([
        ...newEnglishJargon.map((term) =>
          createMutator({
            collection: JargonTerms,
            document: {
              postId: postId,
              term: term.term,
              approved: false,
              deleted: false,
              contents: {
                originalContents: {
                  data: term.htmlContent,
                  type: "ckEditorMarkup",
                },
              },
              altTerms: term.altTerms,
            },
            currentUser: botAccount,
            validate: false,
          })
        ),
        ...jargonTermsToCopy.map((jargonTerm) =>
          createMutator({
            collection: JargonTerms,
            document: {
              postId: postId,
              term: jargonTerm.term,
              approved: jargonTerm.approved,
              deleted: jargonTerm.deleted,
              contents: { originalContents: jargonTerm.contents!.originalContents },
              altTerms: jargonTerm.altTerms,
            },
            currentUser: botAccount,
            validate: false,
          })
        ),
      ]);
      
      return createdTerms;
    });
  } catch (err) {
    if (err instanceof AdvisoryLockError) {
      throw new Error('Already in the process of creating jargon terms for this post, please refresh in a minute', { cause: err });
    }

    throw err;
  }

  return newJargonTerms.map(result => result.data);
}


defineMutation({
  name: 'getNewJargonTerms',
  argTypes: '(postId: String!, glossaryPrompt: String, examplePost: String, exampleTerm: String, exampleAltTerm: String, exampleDefinition: String)',
  resultType: '[JargonTerm]',
  fn: async (_, { postId, ...exampleParams }: Omit<CreateJargonTermsQueryParams, 'currentUser'>, { currentUser }: ResolverContext) => {
    if (!currentUser) {
      throw new Error('You need to be logged in to generate jargon terms');
    }
    if (!userCanCreateAndEditJargonTerms(currentUser)) {
      throw new Error('This is a prototype feature that is not yet available to all users');
    }
    return await createNewJargonTerms({ postId, currentUser, ...exampleParams });
  },
});

1	import { jargonBotClaudeKey } from '@/lib/instanceSettings';	18✔
2	import { defineMutation } from '../../utils/serverGraphqlUtil';	18✔
3	import { getAnthropicPromptCachingClientOrThrow } from '../../languageModels/anthropicClient';	18✔
4	import JargonTerms from '@/lib/collections/jargonTerms/collection';	18✔
5	import { createMutator, sanitize } from '../../vulcan-lib';	18✔
6	import { initialGlossaryPrompt } from './jargonPrompts';	18✔
7	import { fetchFragmentSingle } from '@/server/fetchFragment';	18✔
8	import { htmlToMarkdown } from '@/server/editor/conversionUtils';	18✔
9	import { userCanCreateAndEditJargonTerms } from '@/lib/betas';	18✔
10	import { getAdminTeamAccount } from '@/server/callbacks/commentCallbacks';	18✔
11	import { z } from 'zod';	18✔
12	import { getSqlClientOrThrow } from '@/server/sql/sqlClient';	18✔
13	import { cyrb53Rand } from '@/server/perfMetrics';	18✔
14	import JargonTermsRepo from '@/server/repos/JargonTermsRepo';	18✔
15	import keyBy from 'lodash/keyBy';
16	import { randomId } from '@/lib/random';	18✔
17	import { filterNonnull } from '@/lib/utils/typeGuardUtils';	18✔
18	import { defaultExampleTerm, defaultExamplePost, defaultGlossaryPrompt, defaultExampleAltTerm, defaultExampleDefinition } from '@/components/jargon/GlossaryEditForm';	18✔
19
20	import type { Tool } from '@anthropic-ai/sdk/resources';
21	import type { PromptCachingBetaMessageParam } from '@anthropic-ai/sdk/resources/beta/prompt-caching/messages';
22	import uniq from 'lodash/uniq';	18✔
23
24	interface JargonTermGenerationExampleParams {
25	glossaryPrompt?: string;
26	examplePost?: string;
27	exampleTerm?: string;
28	exampleAltTerm?: string;
29	exampleDefinition?: string;
30	}
31
32	interface CreateJargonTermsQueryParams extends JargonTermGenerationExampleParams {
33	postId: string;
34	currentUser: DbUser;
35	}
36
37	interface SingleJargonTermExplanationQueryParams extends JargonTermGenerationExampleParams {
38	markdown: string;
39	term: string;
40	toolUseId: string;
41	}
42
43	interface ExplanationsGenerationQueryParams extends JargonTermGenerationExampleParams {
44	post: PostsPage;
45	excludeTerms: string[];
46	}
47
48	const jargonTermListResponseSchema = z.object({	18✔
49	jargonTerms: z.array(z.string())
50	});
51
52	const jargonTermSchema = z.object({	18✔
53	term: z.string(),
54	altTerms: z.array(z.string()).optional(),
55	htmlContent: z.string(),
56	});
57
58	type LLMGeneratedJargonTerm = z.infer<typeof jargonTermSchema>;
59
60	const returnJargonTermsTool: Tool = {	18✔
61	name: "return_jargon_terms",
62	description: "A tool that allows Claude to return a list of jargon terms extracted from a post.",
63	input_schema: {
64	type: "object",
65	properties: {
66	jargonTerms: { type: "array", items: { type: "string" } }
67	}
68	}
69	};
70
71	const generateSingleJargonGlossaryItemTool: Tool = {	18✔
72	name: "generate_jargon_glossary_item",
73	description: "A tool that generates a jargon glossary item for a given post. It should be provided with a glossary item, which include the term (the identifier), altTerms (alternate spellings or forms of the term), and htmlContent (the explanation of the term).",
74	input_schema: {
75	type: "object",
76	properties: {
77	term: { type: "string" },
78	altTerms: { type: "array", items: { type: "string" } },
79	htmlContent: { type: "string" }
80	}
81	}
82	};
83
84	class AdvisoryLockError extends Error {}	18✔
85
86	/**
87	* Attempts to acquire an advisory lock and execute a callback.
88	*
89	* @param rawLockId - A unique identifier for the lock, such that attempting to run the "same" operation as determined by the ID will fail if another such operation is already running.
90	* @param callback - The async function to be executed if the lock is acquired.
91	* @returns The result of the callback function.
92	* @throws {AdvisoryLockError} If the lock cannot be acquired.
93	* @throws {Error} If the callback throws an error (after releasing the lock).
94	*
95	* @remarks
96	* - If the lock is not acquired, it throws a LockError.
97	* - If the lock is acquired but the callback throws an error, the lock is released and the error is thrown.
98	* - If the lock is acquired and the callback completes successfully, the lock is released and the callback's return value is returned.
99	*/
100	async function executeWithLock<T>(rawLockId: string, callback: () => Promise<T>): Promise<T> {	×
101	const db = getSqlClientOrThrow();	×
102	// We need to convert the lockId to a number because the advisory lock is a 64-bit integer
103	// cyrb53Rand returns a double between 0 and 1, so multiplying by 1e15 gives us a number between 0 and 1e15
104	const lockId = Math.floor(cyrb53Rand(rawLockId) * 1e15);	×
105
106	return await db.task(async (task) => {	×
107	let lockAcquired = false;	×
108	let lockResult;
109	try {
110	// Attempt to acquire the advisory lock. Four possible outcomes:
111	// 1. The lock is acquired
112	// 2. The lock is not acquired
113	// 3. The lock is acquired, but the query throws an error
114	// 4. The lock is not acquired, and the query throws an error
115	lockResult = await task.any<{ pg_try_advisory_lock: boolean }>('SELECT pg_try_advisory_lock($1)', [lockId]);	×
116	} catch (err) {
117	// This deals with cases 3 and 4 (query threw an error, lock may or may not be acquired)
118	// Case 4 is unfortunate, since we might release someone else's lock, but it's better than leaving the lock in place
119	await task.any('SELECT pg_advisory_unlock($1)', [lockId]);
120	throw err;
121	}
122
123	try {
124	// If case 1, we note that the lock was acquired and run the callback
125	// Case 2 falls through to the error throw below
126	if (lockResult[0].pg_try_advisory_lock) {
127	lockAcquired = true;	×
128	return await callback();
129	}
130	} finally {
131	// Only release the lock if we acquired it, since we don't want to release someone else's lock
132	if (lockAcquired) {
133	await task.any('SELECT pg_advisory_unlock($1)', [lockId]);
134	}
135	}
136
137	// Case 2: Lock not acquired. This is the only case where we don't release the lock, since we don't want to release someone else's lock.
138	// We throw an error so the caller knows they tried to run an operation that required the lock, but it was already locked by someone else.
139	throw new AdvisoryLockError(`Lock could not be acquired for lockId: ${rawLockId}`);
140	});
141	}
142
143	function sanitizeJargonTerms(jargonTerms: LLMGeneratedJargonTerm[]) {
144	return jargonTerms.map(jargonTerm => ({	×
145	...jargonTerm,
146	// In the case where the term is LaTeX, it's probably represented as HTML, so also needs to be sanitized
147	term: sanitize(jargonTerm.term),
148	htmlContent: sanitize(jargonTerm.htmlContent)
149	}));
150	}
151
152	export const queryClaudeForTerms = async (markdown: string) => {	54!
153	const client = getAnthropicPromptCachingClientOrThrow(jargonBotClaudeKey.get());	×
154	const messages = [{	×
155	role: "user" as const,
156	content: [{
157	type: "text" as const,
158	text: `${initialGlossaryPrompt} The post is: <Post>${markdown}</Post>.
159
160	The jargon terms are:`
161	}]
162	}];
163
164	const termsResponse = await client.messages.create({	×
165	model: "claude-3-5-sonnet-20240620",
166	max_tokens: 5000,
167	messages,
168	tools: [returnJargonTermsTool],
169	tool_choice: { type: "tool", name: "return_jargon_terms" }
170	});
171
172	if (termsResponse.content[0].type === "text") {
173	// eslint-disable-next-line no-console
174	console.error(`Claude responded with text, but we expected a tool use.`)	×
175	return [];
176	}
177
178	const responseContent = termsResponse.content[0]?.input;	×
179	const parsedResponse = jargonTermListResponseSchema.safeParse(responseContent);	×
180	if (!parsedResponse.success) {
181	// eslint-disable-next-line no-console
182	console.error(`Claude's response when getting jargon terms doesn't match the expected format.`)	×
183	return [];
184	}
185
186	return parsedResponse.data.jargonTerms;
187	}
188
189	async function createSingleExplanationMessageWithExample({ markdown, term, toolUseId, ...exampleParams }: SingleJargonTermExplanationQueryParams): Promise<PromptCachingBetaMessageParam[]> {	×
190	const finalSystemPrompt = exampleParams.glossaryPrompt ?? defaultGlossaryPrompt	×
191	const finalExamplePost = exampleParams.examplePost ?? defaultExamplePost	×
192	const finalExampleTerm = exampleParams.exampleTerm ?? defaultExampleTerm	×
193	const finalExampleAltTerm = exampleParams.exampleAltTerm ?? defaultExampleAltTerm	×
194	const finalExampleDefinition = exampleParams.exampleDefinition ?? defaultExampleDefinition	×
195
196	const finalExampleGlossary = {	×
197	term: finalExampleTerm,
198	altTerms: [finalExampleAltTerm],
199	htmlContent: finalExampleDefinition
200	}
201
202	return [{
203	role: "user",
204	content: [{
205	type: "text",
206	text: `${finalSystemPrompt}\n\nHere's an example post: <Post>${finalExamplePost}</Post>. And here's an example term: <Term>${finalExampleTerm}</Term>`
207	}]
208	},
209	{
210	role: "assistant",
211	content: [{
212	type: "tool_use",
213	id: toolUseId,
214	name: "generate_jargon_glossary",
215	input: finalExampleGlossary
216	}]
217	},
218	{
219	role: "user",
220	content: [{
221	type: "tool_result",
222	tool_use_id: toolUseId,
223	}, {
224	type: "text",
225	text: `Thanks! Now can you do the following post? This time, you'll only need to do one term. <Post>${markdown}</Post>.`,
226	cache_control: { type: 'ephemeral' }
227	}]
228	}, {
229	role: "user",
230	content: [{
231	type: "text",
232	text: ` The jargon term is: <Term>${term}</Term>`
233	}]
234	}];
235	}
236
237	const queryClaudeForSingleJargonExplanation = async ({ markdown, term, toolUseId, ...exampleParams }: SingleJargonTermExplanationQueryParams): Promise<LLMGeneratedJargonTerm \| null> => {	54!
238	const client = getAnthropicPromptCachingClientOrThrow(jargonBotClaudeKey.get());	×
239	const messages: PromptCachingBetaMessageParam[] = await createSingleExplanationMessageWithExample({ markdown, term, toolUseId, ...exampleParams });	×
240
241	const response = await client.messages.create({	×
242	model: "claude-3-5-sonnet-20240620",
243	max_tokens: 512,
244	messages,
245	tools: [generateSingleJargonGlossaryItemTool],
246	tool_choice: { type: "tool", name: "generate_jargon_glossary_item" }
247	});
248
249	if (response.content[0].type === "text") {
250	// eslint-disable-next-line no-console
251	console.error(`Claude responded with text, but we expected a tool use.`)	×
252	return null;
253	}
254
255	const responseContent = response.content[0]?.input;	×
256	const validatedTerm = jargonTermSchema.safeParse(responseContent);	×
257	if (!validatedTerm.success) {
258	// eslint-disable-next-line no-console
259	console.error('Invalid jargon term:', validatedTerm.error);	×
260	return null;
261	}
262	const parsedJargonTerm = validatedTerm.data;	×
263	return sanitizeJargonTerms([parsedJargonTerm])[0];
264	}
265
266	export async function createEnglishExplanations({ post, excludeTerms, ...exampleParams }: ExplanationsGenerationQueryParams): Promise<LLMGeneratedJargonTerm[]> {	×
267	const originalHtml = post.contents?.html ?? "";	×
268	const originalMarkdown = htmlToMarkdown(originalHtml);	×
269	const markdown = (originalMarkdown.length < 200_000) ? originalMarkdown : originalMarkdown.slice(0, 200_000);	×
270
271	const terms = await queryClaudeForTerms(markdown);	×
272	if (!terms.length) {
273	return [];
274	}
275
NEW 276	const newTerms = terms.filter(term => {	×
NEW 277	const lowerCaseTerm = term.toLowerCase();	×
NEW 278	return !excludeTerms.includes(lowerCaseTerm) && post.contents?.html?.toLowerCase().includes(lowerCaseTerm);	×
279	});
280
281	const toolUseId = randomId();	×
282	const [firstTerm, ...remainingTerms] = newTerms;
283	// Do one term first to ensure the shared prompt is cached, before doing the others in parallel
284	const firstExplanation = await queryClaudeForSingleJargonExplanation({ markdown, term: firstTerm, toolUseId, ...exampleParams });	×
285	const remainingExplanations = await Promise.all(remainingTerms.map((term) => queryClaudeForSingleJargonExplanation({ markdown, term, toolUseId, ...exampleParams })));	×
286	return filterNonnull([firstExplanation, ...remainingExplanations]);
287	}
288
289	const processedTerms = (jargonTerms: DbJargonTerm[]) => {	18✔
NEW 290	return jargonTerms.flatMap(jargonTerm => [jargonTerm.term.toLowerCase(), ...jargonTerm.altTerms.map(altTerm => altTerm.toLowerCase())]);	×
291	}
292
293	export const createNewJargonTerms = async ({ postId, currentUser, ...exampleParams }: CreateJargonTermsQueryParams) => {	54!
294	const post = await fetchFragmentSingle({	×
295	collectionName: 'Posts',
296	fragmentName: 'PostsPage',
297	currentUser,
298	selector: { _id: postId },
299	});
300
301	if (!post) {
302	throw new Error('Post not found');
303	}
304
NEW 305	const [authorsOtherPostJargonTerms, jargonTermsFromThisPost] = await Promise.all([	×
306	(new JargonTermsRepo()).getAuthorsOtherJargonTerms(currentUser._id, postId),
307	JargonTerms.find({ postId }).fetch()
308	]);
NEW 309	const existingJargonTerms = [...authorsOtherPostJargonTerms, ...jargonTermsFromThisPost];	×
310
NEW 311	const termsToExclude = uniq(processedTerms(existingJargonTerms));	×
312
NEW 313	const presentTerms = existingJargonTerms.filter(jargonTerm => post.contents?.html?.includes(jargonTerm.term));	×
NEW 314	const jargonTermsToCopy = presentTerms.filter(jargonTerm => {	×
NEW 315	const terms = processedTerms([jargonTerm]);	×
NEW 316	return !termsToExclude.some(excludedTerm => terms.includes(excludedTerm));	×
317	});
318
319	let newJargonTerms;
320	try {
321	const rawLockId = `jargonTermsLock:${postId}`;	×
322	// Test lock by sleeping for 10 seconds
323
324	newJargonTerms = await executeWithLock(rawLockId, async () => {	×
325	const newEnglishJargon = await createEnglishExplanations({ post, excludeTerms: termsToExclude, ...exampleParams });	×
326
327	const botAccount = await getAdminTeamAccount();	×
328
329	const createdTerms = await Promise.all([	×
330	...newEnglishJargon.map((term) =>	×
331	createMutator({
332	collection: JargonTerms,
333	document: {
334	postId: postId,
335	term: term.term,
336	approved: false,
337	deleted: false,
338	contents: {
339	originalContents: {
340	data: term.htmlContent,
341	type: "ckEditorMarkup",
342	},
343	},
344	altTerms: term.altTerms,
345	},
346	currentUser: botAccount,
347	validate: false,
348	})
349	),
350	...jargonTermsToCopy.map((jargonTerm) =>	×
351	createMutator({
352	collection: JargonTerms,
353	document: {
354	postId: postId,
355	term: jargonTerm.term,
356	approved: jargonTerm.approved,
357	deleted: jargonTerm.deleted,
358	contents: { originalContents: jargonTerm.contents!.originalContents },
359	altTerms: jargonTerm.altTerms,
360	},
361	currentUser: botAccount,
362	validate: false,
363	})
364	),
365	]);
366
367	return createdTerms;
368	});
369	} catch (err) {
370	if (err instanceof AdvisoryLockError) {
371	throw new Error('Already in the process of creating jargon terms for this post, please refresh in a minute', { cause: err });
372	}
373
374	throw err;
375	}
376
377	return newJargonTerms.map(result => result.data);	×
378	}
379
380
381	defineMutation({	18✔
382	name: 'getNewJargonTerms',
383	argTypes: '(postId: String!, glossaryPrompt: String, examplePost: String, exampleTerm: String, exampleAltTerm: String, exampleDefinition: String)',
384	resultType: '[JargonTerm]',
385	fn: async (_, { postId, ...exampleParams }: Omit<CreateJargonTermsQueryParams, 'currentUser'>, { currentUser }: ResolverContext) => {	36!
386	if (!currentUser) {
387	throw new Error('You need to be logged in to generate jargon terms');
388	}
389	if (!userCanCreateAndEditJargonTerms(currentUser)) {
390	throw new Error('This is a prototype feature that is not yet available to all users');
391	}
392	return await createNewJargonTerms({ postId, currentUser, ...exampleParams });
393	},
394	});

ForumMagnum / ForumMagnum / 11599960914

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous