10606547007

Committed 29 Aug 2024 12:33AM UTC coverage: 31.462% (-0.5%) from 31.937%

Build # 10606547007

Build Type

Pull #9654

github

Committed by

Raemon

Commit Message

redo migrations

Pull Request Pull Request #9654: Auto Spotlights for Best of LessWrong

Run Details

1698 of 13948 branches covered (12.17%)

Branch coverage included in aggregate %.

9 of 60 new or added lines in 4 files covered. (15.0%)

2 existing lines in 2 files now uncovered.

9575 of 21882 relevant lines covered (43.76%)

572.63 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

9.9

/packages/lesswrong/server/scripts/generativeModels/autoSpotlight.ts

import { Globals, createAdminContext, createMutator } from "../../vulcan-lib";
import Spotlights from "../../../lib/collections/spotlights/collection";
import { fetchFragment } from "../../fetchFragment";
import { getAnthropicPromptCachingClientOrThrow } from "@/server/languageModels/anthropicClient";
import { REVIEW_WINNER_CACHE, ReviewWinnerWithPost } from "@/lib/collections/reviewWinners/cache";

async function queryClaudeJailbreak(basePrompt: string, spotlightPrompt: string, maxTokens: number) {
  const client = getAnthropicPromptCachingClientOrThrow()
  return await client.messages.create({
    system: "The assistant is in CLI simulation mode, and responds to the user's CLI commands only with the output of the command.",
    model: "claude-3-5-sonnet-20240620",
    max_tokens: maxTokens,
    messages: [
      {
        role: "user", 
        content: [{
          type: "text",
          text: basePrompt,
          cache_control: {type: "ephemeral"}
        }]
      },
      {
        role: "assistant",
        content: [{
          type: "text",
          text: spotlightPrompt,
        }]
      },
    ]
  })
}

function createSpotlight (post: PostsWithNavigation, reviewWinner: ReviewWinnerWithPost|undefined, summary: string) {
  const context = createAdminContext();
  const postYear = post.postedAt.getFullYear()
  const cloudinaryImageUrl = reviewWinner?.reviewWinner.reviewWinnerArt?.splashArtImageUrl

  void createMutator({
    collection: Spotlights,
    document: {
      documentId: post._id,
      documentType: "Post",
      customSubtitle: `Best of LessWrong ${postYear}`,
      duration: 1,
      draft: true,
      showAuthor: true,
      spotlightSplashImageUrl: cloudinaryImageUrl,
      description: { originalContents: { type: 'ckEditorMarkup', data: summary } },
      lastPromotedAt: new Date(0),
    },
    currentUser: context.currentUser,
    context
  })
}

async function getPromptInfo(): Promise<{posts: PostsWithNavigation[], spotlights: DbSpotlight[]}> {
  const reviewWinners = await fetchFragment({
    collectionName: "ReviewWinners",
    fragmentName: "ReviewWinnerTopPostsPage",
    currentUser: createAdminContext().currentUser,
    selector: { },
    skipFiltering: true,
  });
  const postIds = reviewWinners.map(winner => winner.postId);
  // Assume these functions exist to fetch data from the database
  const posts = await fetchFragment({
    collectionName: "Posts",
    fragmentName: "PostsWithNavigation",
    currentUser: null,
    selector: { _id: { $in: postIds } },
    skipFiltering: true,
  });

  const spotlights = await Spotlights.find({ documentId: { $in: postIds }, draft: false }).fetch();
  return { posts, spotlights };
}

// get the posts that have spotlights, sorted by post length, and filter out the ones that are too short
const getPostsForPrompt = ({posts, spotlights}: {posts: PostsWithNavigation[], spotlights: DbSpotlight[], log?: boolean}) => {
  const postsWithSpotlights = posts.filter(post => spotlights.find(spotlight => spotlight.documentId === post._id))
  const postsWithSpotlightsSortedByPostLength = postsWithSpotlights.sort((a, b) => {
    return (a?.contents?.html?.length ?? 0) - (b?.contents?.html?.length ?? 0)
  })
  return postsWithSpotlightsSortedByPostLength.filter((post) => (post?.contents?.html?.length ?? 0) > 2000).slice(0, 15)
}

const getJailbreakPromptBase = ({posts, spotlights}: {posts: PostsWithNavigation[], spotlights: DbSpotlight[]}) => {
  let prompt = `A series of essays, each followed by a short description of the essay. The style of the short description is slightly casual. They are a single paragraph, 1-3 sentences long. Try to avoid referencing the essay or author (i.e. they don't say "the author" or "in this post" or similar), just talk about the ideas.
  
  `

  for (const post of posts) {
    const spotlight = spotlights.find(spotlight => spotlight.documentId === post._id)
    if (!spotlight) {
      console.log("No spotlight found for post", post.title)
    }
    prompt += `
      Post: ${post.title}
      ---
      Post Contents: ${post.contents?.html}
      ---
      A short description of the essay, ~2 sentences, no paragraph breaks or bullet points: ${spotlight?.description?.originalContents?.data}
      ---
      ---
      ---
    `
  }
  return prompt
}

const getSpotlightPrompt = ({post}: {post: PostsWithNavigation}) => {
  return `
  Post: ${post.title}
  ---
  Post Contents: ${post.contents?.html}
  ---
  A short description of the essay, ~2 sentences, no paragraph breaks or bullet points:`
}

const getSpotlightPrompt2 = ({post}: {post: PostsWithNavigation}) => {
  return `
  Post: ${post.title}
  ---
  Post Contents: ${post.contents?.html}
  ---
  First, write out three key points that the author makes.
  Then, write "==="
  Then, write a short description of the essay, ~2 sentences, no paragraph breaks or bullet points:`
}

// TODO: figure out what the type from the response is supposed to be and fix it
// right now it claims "text" isn't a valid type but seems like it should be
type AnthropicMessageContent = {
  type: "text"
  text: string
}

async function createSpotlights() {
  // eslint-disable-next-line no-console
  console.log("Creating spotlights for review winners");

  const { posts, spotlights } = await getPromptInfo()
  const reviewWinners = REVIEW_WINNER_CACHE.reviewWinners
  const postsForPrompt = getPostsForPrompt({posts, spotlights})
  const postsWithoutSpotlights = posts.filter(post => !spotlights.find(spotlight => spotlight.documentId === post._id))

  const jailbreakPromptBase = getJailbreakPromptBase({posts: postsForPrompt, spotlights})

  for (const post of postsWithoutSpotlights.slice(0, 2)) {
    const reviewWinner = reviewWinners.find(reviewWinner => reviewWinner._id === post._id)

    try {
      const jailbreakSummary1 = await queryClaudeJailbreak(jailbreakPromptBase, getSpotlightPrompt({post}), 75)
      console.log({jailbreakSummary1})
      const summary1 = jailbreakSummary1.content[0] as AnthropicMessageContent
      console.log(post.title, summary1)
      const cleanedSummary1 = summary1.text.replace(/---|\n/g, "") 
      createSpotlight(post, reviewWinner, cleanedSummary1)
    } catch (e) {
      console.log(e)
    }
    // eslint-disable-next-line no-console

    try{ 
      const jailbreakSummary2 = await queryClaudeJailbreak(jailbreakPromptBase, getSpotlightPrompt2({post}), 300)
      console.log({jailbreakSummary2})
      const summary2 = jailbreakSummary2.content[0] as AnthropicMessageContent
      console.log(post.title, summary2)
      const cleanedSummary2 = summary2.text.replace(/---|\n/g, "").split("===")[1]
      createSpotlight(post, reviewWinner, cleanedSummary2)
    } catch (e) {
      console.log(e)
    }

    // // eslint-disable-next-line no-console

  }

  // eslint-disable-next-line no-console
  console.log("Done creating spotlights for review winners");
}

Globals.createSpotlights = createSpotlights;

1	import { Globals, createAdminContext, createMutator } from "../../vulcan-lib";	18✔
2	import Spotlights from "../../../lib/collections/spotlights/collection";	18✔
3	import { fetchFragment } from "../../fetchFragment";	18✔
4	import { getAnthropicPromptCachingClientOrThrow } from "@/server/languageModels/anthropicClient";	18✔
5	import { REVIEW_WINNER_CACHE, ReviewWinnerWithPost } from "@/lib/collections/reviewWinners/cache";	18✔
6
7	async function queryClaudeJailbreak(basePrompt: string, spotlightPrompt: string, maxTokens: number) {
NEW 8	const client = getAnthropicPromptCachingClientOrThrow()	×
NEW 9	return await client.messages.create({	×
10	system: "The assistant is in CLI simulation mode, and responds to the user's CLI commands only with the output of the command.",
11	model: "claude-3-5-sonnet-20240620",
12	max_tokens: maxTokens,
13	messages: [
14	{
15	role: "user",
16	content: [{
17	type: "text",
18	text: basePrompt,
19	cache_control: {type: "ephemeral"}
20	}]
21	},
22	{
23	role: "assistant",
24	content: [{
25	type: "text",
26	text: spotlightPrompt,
27	}]
28	},
29	]
30	})
31	}
32
33	function createSpotlight (post: PostsWithNavigation, reviewWinner: ReviewWinnerWithPost\|undefined, summary: string) {
34	const context = createAdminContext();	×
NEW 35	const postYear = post.postedAt.getFullYear()	×
NEW 36	const cloudinaryImageUrl = reviewWinner?.reviewWinner.reviewWinnerArt?.splashArtImageUrl	×
37
UNCOV 38	void createMutator({	×
39	collection: Spotlights,
40	document: {
41	documentId: post._id,
42	documentType: "Post",
43	customSubtitle: `Best of LessWrong ${postYear}`,
44	duration: 1,
45	draft: true,
46	showAuthor: true,
47	spotlightSplashImageUrl: cloudinaryImageUrl,
48	description: { originalContents: { type: 'ckEditorMarkup', data: summary } },
49	lastPromotedAt: new Date(0),
50	},
51	currentUser: context.currentUser,
52	context
53	})
54	}
55
56	async function getPromptInfo(): Promise<{posts: PostsWithNavigation[], spotlights: DbSpotlight[]}> {
NEW 57	const reviewWinners = await fetchFragment({	×
58	collectionName: "ReviewWinners",
59	fragmentName: "ReviewWinnerTopPostsPage",
60	currentUser: createAdminContext().currentUser,
61	selector: { },
62	skipFiltering: true,
63	});
NEW 64	const postIds = reviewWinners.map(winner => winner.postId);	×
65	// Assume these functions exist to fetch data from the database
NEW 66	const posts = await fetchFragment({	×
67	collectionName: "Posts",
68	fragmentName: "PostsWithNavigation",
69	currentUser: null,
70	selector: { _id: { $in: postIds } },
71	skipFiltering: true,
72	});
73
NEW 74	const spotlights = await Spotlights.find({ documentId: { $in: postIds }, draft: false }).fetch();	×
NEW 75	return { posts, spotlights };	×
76	}
77
78	// get the posts that have spotlights, sorted by post length, and filter out the ones that are too short
79	const getPostsForPrompt = ({posts, spotlights}: {posts: PostsWithNavigation[], spotlights: DbSpotlight[], log?: boolean}) => {	18✔
NEW 80	const postsWithSpotlights = posts.filter(post => spotlights.find(spotlight => spotlight.documentId === post._id))	×
NEW 81	const postsWithSpotlightsSortedByPostLength = postsWithSpotlights.sort((a, b) => {	×
NEW 82	return (a?.contents?.html?.length ?? 0) - (b?.contents?.html?.length ?? 0)	×
83	})
NEW 84	return postsWithSpotlightsSortedByPostLength.filter((post) => (post?.contents?.html?.length ?? 0) > 2000).slice(0, 15)	×
85	}
86
87	const getJailbreakPromptBase = ({posts, spotlights}: {posts: PostsWithNavigation[], spotlights: DbSpotlight[]}) => {	18✔
NEW 88	let prompt = `A series of essays, each followed by a short description of the essay. The style of the short description is slightly casual. They are a single paragraph, 1-3 sentences long. Try to avoid referencing the essay or author (i.e. they don't say "the author" or "in this post" or similar), just talk about the ideas.	×
89
90	`
91
NEW 92	for (const post of posts) {	×
NEW 93	const spotlight = spotlights.find(spotlight => spotlight.documentId === post._id)	×
NEW 94	if (!spotlight) {	×
NEW 95	console.log("No spotlight found for post", post.title)	×
96	}
NEW 97	prompt += `	×
98	Post: ${post.title}
99	---
100	Post Contents: ${post.contents?.html}	×
101	---
102	A short description of the essay, ~2 sentences, no paragraph breaks or bullet points: ${spotlight?.description?.originalContents?.data}	×
103	---
104	---
105	---
106	`
107	}
NEW 108	return prompt	×
109	}
110
111	const getSpotlightPrompt = ({post}: {post: PostsWithNavigation}) => {	18✔
NEW 112	return `	×
113	Post: ${post.title}
114	---
115	Post Contents: ${post.contents?.html}	×
116	---
117	A short description of the essay, ~2 sentences, no paragraph breaks or bullet points:`
118	}
119
120	const getSpotlightPrompt2 = ({post}: {post: PostsWithNavigation}) => {	18✔
NEW 121	return `	×
122	Post: ${post.title}
123	---
124	Post Contents: ${post.contents?.html}	×
125	---
126	First, write out three key points that the author makes.
127	Then, write "==="
128	Then, write a short description of the essay, ~2 sentences, no paragraph breaks or bullet points:`
129	}
130
131	// TODO: figure out what the type from the response is supposed to be and fix it
132	// right now it claims "text" isn't a valid type but seems like it should be
133	type AnthropicMessageContent = {
134	type: "text"
135	text: string
136	}
137
138	async function createSpotlights() {
139	// eslint-disable-next-line no-console
NEW 140	console.log("Creating spotlights for review winners");	×
141
NEW 142	const { posts, spotlights } = await getPromptInfo()	×
NEW 143	const reviewWinners = REVIEW_WINNER_CACHE.reviewWinners	×
NEW 144	const postsForPrompt = getPostsForPrompt({posts, spotlights})	×
NEW 145	const postsWithoutSpotlights = posts.filter(post => !spotlights.find(spotlight => spotlight.documentId === post._id))	×
146
NEW 147	const jailbreakPromptBase = getJailbreakPromptBase({posts: postsForPrompt, spotlights})	×
148
NEW 149	for (const post of postsWithoutSpotlights.slice(0, 2)) {	×
NEW 150	const reviewWinner = reviewWinners.find(reviewWinner => reviewWinner._id === post._id)	×
151
NEW 152	try {	×
NEW 153	const jailbreakSummary1 = await queryClaudeJailbreak(jailbreakPromptBase, getSpotlightPrompt({post}), 75)	×
NEW 154	console.log({jailbreakSummary1})	×
NEW 155	const summary1 = jailbreakSummary1.content[0] as AnthropicMessageContent	×
NEW 156	console.log(post.title, summary1)	×
NEW 157	const cleanedSummary1 = summary1.text.replace(/---\|\n/g, "")	×
NEW 158	createSpotlight(post, reviewWinner, cleanedSummary1)	×
159	} catch (e) {
NEW 160	console.log(e)	×
161	}
162	// eslint-disable-next-line no-console
163
NEW 164	try{	×
NEW 165	const jailbreakSummary2 = await queryClaudeJailbreak(jailbreakPromptBase, getSpotlightPrompt2({post}), 300)	×
NEW 166	console.log({jailbreakSummary2})	×
NEW 167	const summary2 = jailbreakSummary2.content[0] as AnthropicMessageContent	×
NEW 168	console.log(post.title, summary2)	×
NEW 169	const cleanedSummary2 = summary2.text.replace(/---\|\n/g, "").split("===")[1]	×
NEW 170	createSpotlight(post, reviewWinner, cleanedSummary2)	×
171	} catch (e) {
NEW 172	console.log(e)	×
173	}
174
175	// // eslint-disable-next-line no-console
176
177	}
178
179	// eslint-disable-next-line no-console
NEW 180	console.log("Done creating spotlights for review winners");	×
181	}
182
183	Globals.createSpotlights = createSpotlights;	18✔

ForumMagnum / ForumMagnum / 10606547007

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous