10691726311

Committed 03 Sep 2024 10:37PM UTC coverage: 31.909% (+0.07%) from 31.837%

Build # 10691726311

Build Type

Pull #9698

github

Committed by

web-flow

Commit Message

Merge af26d8876 into a6ff63a86

Pull Request Pull Request #9698: Merge master into lw-deploy

Run Details

5070 of 41196 branches covered (12.31%)

Branch coverage included in aggregate %.

157 of 295 new or added lines in 38 files covered. (53.22%)

60 existing lines in 44 files now uncovered.

29363 of 66713 relevant lines covered (44.01%)

185.03 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

8.65

/packages/lesswrong/server/scripts/generativeModels/autoSpotlight.ts

import { Globals, createAdminContext, createMutator } from "../../vulcan-lib";
import Spotlights from "../../../lib/collections/spotlights/collection";
import { fetchFragment } from "../../fetchFragment";
import { getAnthropicPromptCachingClientOrThrow } from "@/server/languageModels/anthropicClient";
import { REVIEW_WINNER_CACHE, ReviewWinnerWithPost } from "@/lib/collections/reviewWinners/cache";
import { PromptCachingBetaMessageParam, PromptCachingBetaTextBlockParam } from "@anthropic-ai/sdk/resources/beta/prompt-caching/messages";

async function queryClaudeJailbreak(prompt: PromptCachingBetaMessageParam[], maxTokens: number) {
  const client = getAnthropicPromptCachingClientOrThrow()
  return await client.messages.create({
    system: "The assistant is in CLI simulation mode, and responds to the user's CLI commands only with the output of the command.",
    model: "claude-3-5-sonnet-20240620",
    max_tokens: maxTokens,
    messages: prompt
  })
}

function createSpotlight (post: PostsWithNavigation, reviewWinner: ReviewWinnerWithPost|undefined, summary: string) {
  const context = createAdminContext();
  const postYear = post.postedAt.getFullYear()
  const cloudinaryImageUrl = reviewWinner?.reviewWinner.reviewWinnerArt?.splashArtImageUrl

  void createMutator({
    collection: Spotlights,
    document: {
      documentId: post._id,
      documentType: "Post",
      customSubtitle: `Best of LessWrong ${postYear}`,
      duration: 1,
      draft: true,
      showAuthor: true,
      spotlightSplashImageUrl: cloudinaryImageUrl,
      description: { originalContents: { type: 'ckEditorMarkup', data: summary } },
      lastPromotedAt: new Date(0),
    },
    currentUser: context.currentUser,
    context
  })
}

async function getPromptInfo(): Promise<{posts: PostsWithNavigation[], spotlights: DbSpotlight[]}> {
  const reviewWinners = await fetchFragment({
    collectionName: "ReviewWinners",
    fragmentName: "ReviewWinnerTopPostsPage",
    currentUser: createAdminContext().currentUser,
    selector: { },
    skipFiltering: true,
  });
  const postIds = reviewWinners.map(winner => winner.postId);

  const posts = await fetchFragment({
    collectionName: "Posts",
    fragmentName: "PostsWithNavigation",
    currentUser: null,
    selector: { _id: { $in: postIds } },
    skipFiltering: true,
  });

  const spotlights = await Spotlights.find({ documentId: { $in: postIds }, draft: false, deletedDraft: false }).fetch();
  return { posts, spotlights };
}

// get the posts that have spotlights, sorted by post length, and filter out the ones that are too short
const getPostsForPrompt = ({posts, spotlights}: {posts: PostsWithNavigation[], spotlights: DbSpotlight[], log?: boolean}) => {
  const postsWithSpotlights = posts.filter(post => spotlights.find(spotlight => spotlight.documentId === post._id))
  const postsWithSpotlightsSortedByPostLength = postsWithSpotlights.sort((a, b) => {
    return (a?.contents?.html?.length ?? 0) - (b?.contents?.html?.length ?? 0)
  })
  return postsWithSpotlightsSortedByPostLength.filter((post) => (post?.contents?.html?.length ?? 0) > 2000).slice(0, 15)
}

const getJailbreakPromptBase = ({posts, spotlights, summary_prompt_name}: {posts: PostsWithNavigation[], spotlights: DbSpotlight[], summary_prompt_name: string}) => {
  const prompt: PromptCachingBetaMessageParam[] = []
  posts.forEach((post, i) => {
    const spotlight = spotlights.find(spotlight => spotlight.documentId === post._id)
    prompt.push({
      role: "user",
      content: [{
        type: "text",
        text: `<cmd>cat posts/${post.slug}.xml</cmd>`
      }]
    })
    prompt.push({
      role: "assistant",
      content: [{
        type: "text",
        text: `
<title>${post.title}</title>
<author>${post.user?.displayName}</author>
<body>${post.contents?.html}</body>
<${summary_prompt_name}>${spotlight?.description?.originalContents?.data}</${summary_prompt_name}>`,
        ...((i === (posts.length - 1)) ? { cache_control: {"type": "ephemeral"}} : {})
      }]
    })
  })
  return prompt
}

const getSpotlightPrompt = ({post, summary_prompt_name}: {post: PostsWithNavigation, summary_prompt_name: string}): PromptCachingBetaMessageParam[] => {
  return [{
    role: "user",
    content: [{
      type: "text",
      text: `<cmd>cat posts/${post.slug}.xml</cmd>`
    }]
  },
  {
    role: "assistant",
    content: [{
      type: "text",
      text: `
<title>${post.title}</title>
<author>${post.user?.displayName}</author>
<body>${post.contents?.html}</body>
<${summary_prompt_name}>`
    }]
  }]
}

async function createSpotlights() {
  // eslint-disable-next-line no-console
  console.log("Creating spotlights for review winners");

  const { posts, spotlights } = await getPromptInfo()
  const reviewWinners = REVIEW_WINNER_CACHE.reviewWinners
  const postsForPrompt = getPostsForPrompt({posts, spotlights})
  const postsWithoutSpotlights = posts.filter(post => !spotlights.find(spotlight => spotlight.documentId === post._id))

  const summary_prompts = [ 
    "50WordSummary",
    "clickbait", 
    "tweet", 
    "key_quote" 
  ]

  for (const summary_prompt of summary_prompts) {
    for (const post of postsWithoutSpotlights) {
      const reviewWinner = reviewWinners.find(reviewWinner => reviewWinner._id === post._id)

      try {
        const prompt = [...getJailbreakPromptBase({posts: postsForPrompt, spotlights, summary_prompt_name: summary_prompt}), ...getSpotlightPrompt({post, summary_prompt_name: summary_prompt})]

        const jailbreakSummary1 = await queryClaudeJailbreak(prompt, 200)
        const summary1 = jailbreakSummary1.content[0]
        if (summary1.type === "text") {
          const cleanedSummary1 = summary1.text.replace(/---|\n/g, "") + ` [${summary_prompt}]`
          // eslint-disable-next-line no-console
          console.log({title: post.title, cleanedSummary1})
          createSpotlight(post, reviewWinner, cleanedSummary1)
        }

        const jailbreakSummary2 = await queryClaudeJailbreak(prompt, 200)
        const summary2 = jailbreakSummary2.content[0]
        if (summary2.type === "text") {
          const cleanedSummary2 = summary2.text.replace(/---|\n/g, "") + ` [${summary_prompt}]`
          // eslint-disable-next-line no-console
          console.log({title: post.title, cleanedSummary2})
          createSpotlight(post, reviewWinner, cleanedSummary2)
        }

      } catch (e) {
        // eslint-disable-next-line no-console
        console.log(e)
      }
    }
  }

  // eslint-disable-next-line no-console
  console.log("Done creating spotlights for review winners");
}

Globals.createSpotlights = createSpotlights;

1	import { Globals, createAdminContext, createMutator } from "../../vulcan-lib";	19✔
2	import Spotlights from "../../../lib/collections/spotlights/collection";	19✔
3	import { fetchFragment } from "../../fetchFragment";	19✔
4	import { getAnthropicPromptCachingClientOrThrow } from "@/server/languageModels/anthropicClient";	19✔
5	import { REVIEW_WINNER_CACHE, ReviewWinnerWithPost } from "@/lib/collections/reviewWinners/cache";	19✔
6	import { PromptCachingBetaMessageParam, PromptCachingBetaTextBlockParam } from "@anthropic-ai/sdk/resources/beta/prompt-caching/messages";
7
8	async function queryClaudeJailbreak(prompt: PromptCachingBetaMessageParam[], maxTokens: number) {
9	const client = getAnthropicPromptCachingClientOrThrow()	×
10	return await client.messages.create({	×
11	system: "The assistant is in CLI simulation mode, and responds to the user's CLI commands only with the output of the command.",
12	model: "claude-3-5-sonnet-20240620",
13	max_tokens: maxTokens,
14	messages: prompt
15	})
16	}
17
18	function createSpotlight (post: PostsWithNavigation, reviewWinner: ReviewWinnerWithPost\|undefined, summary: string) {
19	const context = createAdminContext();	×
20	const postYear = post.postedAt.getFullYear()	×
21	const cloudinaryImageUrl = reviewWinner?.reviewWinner.reviewWinnerArt?.splashArtImageUrl	×
22
23	void createMutator({	×
24	collection: Spotlights,
25	document: {
26	documentId: post._id,
27	documentType: "Post",
28	customSubtitle: `Best of LessWrong ${postYear}`,
29	duration: 1,
30	draft: true,
31	showAuthor: true,
32	spotlightSplashImageUrl: cloudinaryImageUrl,
33	description: { originalContents: { type: 'ckEditorMarkup', data: summary } },
34	lastPromotedAt: new Date(0),
35	},
36	currentUser: context.currentUser,
37	context
38	})
39	}
40
41	async function getPromptInfo(): Promise<{posts: PostsWithNavigation[], spotlights: DbSpotlight[]}> {
42	const reviewWinners = await fetchFragment({	×
43	collectionName: "ReviewWinners",
44	fragmentName: "ReviewWinnerTopPostsPage",
45	currentUser: createAdminContext().currentUser,
46	selector: { },
47	skipFiltering: true,
48	});
49	const postIds = reviewWinners.map(winner => winner.postId);	×
50
51	const posts = await fetchFragment({	×
52	collectionName: "Posts",
53	fragmentName: "PostsWithNavigation",
54	currentUser: null,
55	selector: { _id: { $in: postIds } },
56	skipFiltering: true,
57	});
58
59	const spotlights = await Spotlights.find({ documentId: { $in: postIds }, draft: false, deletedDraft: false }).fetch();	×
60	return { posts, spotlights };	×
61	}
62
63	// get the posts that have spotlights, sorted by post length, and filter out the ones that are too short
64	const getPostsForPrompt = ({posts, spotlights}: {posts: PostsWithNavigation[], spotlights: DbSpotlight[], log?: boolean}) => {	19✔
65	const postsWithSpotlights = posts.filter(post => spotlights.find(spotlight => spotlight.documentId === post._id))	×
66	const postsWithSpotlightsSortedByPostLength = postsWithSpotlights.sort((a, b) => {	×
67	return (a?.contents?.html?.length ?? 0) - (b?.contents?.html?.length ?? 0)	×
68	})
69	return postsWithSpotlightsSortedByPostLength.filter((post) => (post?.contents?.html?.length ?? 0) > 2000).slice(0, 15)	×
70	}
71
72	const getJailbreakPromptBase = ({posts, spotlights, summary_prompt_name}: {posts: PostsWithNavigation[], spotlights: DbSpotlight[], summary_prompt_name: string}) => {	19✔
73	const prompt: PromptCachingBetaMessageParam[] = []	×
74	posts.forEach((post, i) => {	×
75	const spotlight = spotlights.find(spotlight => spotlight.documentId === post._id)	×
76	prompt.push({	×
77	role: "user",
78	content: [{
79	type: "text",
80	text: `<cmd>cat posts/${post.slug}.xml</cmd>`
81	}]
82	})
83	prompt.push({	×
84	role: "assistant",
85	content: [{
86	type: "text",
87	text: `
88	<title>${post.title}</title>
89	<author>${post.user?.displayName}</author>	×
90	<body>${post.contents?.html}</body>	×
91	<${summary_prompt_name}>${spotlight?.description?.originalContents?.data}</${summary_prompt_name}>`,	×
92	...((i === (posts.length - 1)) ? { cache_control: {"type": "ephemeral"}} : {})	×
93	}]
94	})
95	})
96	return prompt	×
97	}
98
99	const getSpotlightPrompt = ({post, summary_prompt_name}: {post: PostsWithNavigation, summary_prompt_name: string}): PromptCachingBetaMessageParam[] => {	19✔
100	return [{	×
101	role: "user",
102	content: [{
103	type: "text",
104	text: `<cmd>cat posts/${post.slug}.xml</cmd>`
105	}]
106	},
107	{
108	role: "assistant",
109	content: [{
110	type: "text",
111	text: `
112	<title>${post.title}</title>
113	<author>${post.user?.displayName}</author>	×
114	<body>${post.contents?.html}</body>	×
115	<${summary_prompt_name}>`
116	}]
117	}]
118	}
119
120	async function createSpotlights() {
121	// eslint-disable-next-line no-console
122	console.log("Creating spotlights for review winners");	×
123
124	const { posts, spotlights } = await getPromptInfo()	×
125	const reviewWinners = REVIEW_WINNER_CACHE.reviewWinners	×
126	const postsForPrompt = getPostsForPrompt({posts, spotlights})	×
127	const postsWithoutSpotlights = posts.filter(post => !spotlights.find(spotlight => spotlight.documentId === post._id))	×
128
129	const summary_prompts = [	×
130	"50WordSummary",
131	"clickbait",
132	"tweet",
133	"key_quote"
134	]
135
136	for (const summary_prompt of summary_prompts) {	×
NEW 137	for (const post of postsWithoutSpotlights) {	×
138	const reviewWinner = reviewWinners.find(reviewWinner => reviewWinner._id === post._id)	×
139
140	try {	×
141	const prompt = [...getJailbreakPromptBase({posts: postsForPrompt, spotlights, summary_prompt_name: summary_prompt}), ...getSpotlightPrompt({post, summary_prompt_name: summary_prompt})]	×
142
143	const jailbreakSummary1 = await queryClaudeJailbreak(prompt, 200)	×
144	const summary1 = jailbreakSummary1.content[0]	×
145	if (summary1.type === "text") {	×
146	const cleanedSummary1 = summary1.text.replace(/---\|\n/g, "") + ` [${summary_prompt}]`	×
147	// eslint-disable-next-line no-console
148	console.log({title: post.title, cleanedSummary1})	×
149	createSpotlight(post, reviewWinner, cleanedSummary1)	×
150	}
151
152	const jailbreakSummary2 = await queryClaudeJailbreak(prompt, 200)	×
153	const summary2 = jailbreakSummary2.content[0]	×
154	if (summary2.type === "text") {	×
155	const cleanedSummary2 = summary2.text.replace(/---\|\n/g, "") + ` [${summary_prompt}]`	×
156	// eslint-disable-next-line no-console
157	console.log({title: post.title, cleanedSummary2})	×
158	createSpotlight(post, reviewWinner, cleanedSummary2)	×
159	}
160
161	} catch (e) {
162	// eslint-disable-next-line no-console
163	console.log(e)	×
164	}
165	}
166	}
167
168	// eslint-disable-next-line no-console
169	console.log("Done creating spotlights for review winners");	×
170	}
171
172	Globals.createSpotlights = createSpotlights;	19✔

ForumMagnum / ForumMagnum / 10691726311

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous