Build has been set to done!

10050044142

Committed 22 Jul 2024 11:43PM UTC coverage: 31.859% (-0.01%) from 31.869%

Build # 10050044142

Build Type

push

github

Committed by

jimrandomh

Commit Message

Add post-response-pairs dataset generation

Run Details

4930 of 40541 branches covered (12.16%)

Branch coverage included in aggregate %.

5 of 67 new or added lines in 1 file covered. (7.46%)

1 existing line in 1 file now uncovered.

28854 of 65501 relevant lines covered (44.05%)

179.6 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

7.88

/packages/lesswrong/server/scripts/generativeModels/generateFinetuneDataset.ts

import { Globals } from '@/lib/vulcan-lib/config';
import { getDefaultViewSelector, mergeSelectors } from "@/lib/utils/viewUtils";
import { Comments } from '@/lib/collections/comments/collection';
import { Posts } from '@/lib/collections/posts/collection';
import { Users } from '@/lib/collections/users/collection';
import { dataToMarkdown } from '@/server/editor/conversionUtils';
import path from 'path';
import fs from 'fs';
import fsPromises from 'fs/promises';
import chunk from 'lodash/chunk';
import groupBy from 'lodash/groupBy';
import sortBy from 'lodash/sortBy';
import keyBy from 'lodash/keyBy';
import { postprocessMarkdown } from '@/server/languageModels/llmUtil';
import shuffle from 'lodash/shuffle';

type HighKarmaContentDatasetOptions = {
  postKarmaThreshold: number
  commentKarmaThreshold: number
  outputFilename: string

  // Filter for posts to include. This will be merged with the default post
  // view, which takes care of excluding drafts, spam, etc, and with the
  // postKarmaThreshold option above.
  postFilter: ViewQueryAndOptions<"Posts",DbPost>["selector"]
  
  // Filter for comments to include. This will be merged with the default
  // comment view, which takes care of excluding deleted comments.
  commentFilter: ViewQueryAndOptions<"Comments",DbComment>["selector"]
  
  // Whether to include shortform posts (ie top-level comments on shortform
  // containers.)
  includeShortformTopLevelComments: boolean
  
  // Whether to include comments (other than top-level shortform). If this is
  // "onlyIfPostIncluded", a comment will be included only if the post it's on
  // is included (or if it's under a shortform, the top-level shortform
  // comment).
  //
  // (Regardless of this setting, commentFilter and commentKarmaThreshold are
  // still applied.)
  includeReplyComments: "all"|"onlyIfPostIncluded"|"none"
  
  documentSeparator: string,
}
const defaultHighKarmaContentDatasetOptions: HighKarmaContentDatasetOptions = {
  postKarmaThreshold: 60,
  commentKarmaThreshold: 30,
  outputFilename: "ml/finetuneData.txt",
  postFilter: {
    frontpageDate: {$gt: new Date(0)}
  },
  commentFilter: {},
  includeShortformTopLevelComments: true,
  includeReplyComments: "all",
  documentSeparator: "========",
}

async function generateHighKarmaContentDataset(_options?: Partial<HighKarmaContentDatasetOptions>) {
  const options = {...defaultHighKarmaContentDatasetOptions, ..._options};
  const startTime = new Date();

  await withStreamingWrite(options.outputFilename, async (write) => {
    await writeHighKarmaContentDatasetTo(options, write);
    // eslint-disable-next-line no-console
    console.log(`Finetune dataset generated in ${new Date().getTime() - startTime.getTime()}ms`);
  });
}

async function withStreamingWrite(
  filename: string,
  fn: (write: (s: string) => Promise<void>) => Promise<void>
): Promise<void> {
  ensurePathExists(filename);
  const fileHandle = await fsPromises.open(filename, 'w');
  if (!fileHandle) {
    throw new Error(`Unable to open ${filename} for writing`);
  }
  try {
    const writeStream = fileHandle.createWriteStream();
    // eslint-disable-next-line no-inner-declarations
    async function write(s: string) {
      return new Promise<void>((resolve, reject) => {
        writeStream.write(s, (err) => {
          if (err) reject(err);
          else resolve();
        });
      });
    }
    await fn(write);
  } finally {
    await fileHandle.close();
  }
}

async function writeHighKarmaContentDatasetTo(options: HighKarmaContentDatasetOptions, write: (s: string) => Promise<void>) {
  // eslint-disable-next-line no-console
  const log = console.log;
  
  ////////////////////////////////////////////////////////////////////
  // Data selection stage
  // First we collect IDs of posts and comments that will be included (without
  // downloading their bodies from the DB). This is split out so that you can
  // iterate on filters to see how much will be included, without waiting for
  // the slow parts (downloading contents from the DB and doing format
  // conversions), and to work around the possibility of out-of-memory issues.
  
  log("Generating fine-tune dataset", options);

  log("Collecting list of posts to include");
  const postSelector = mergeSelectors(
    getDefaultViewSelector("Posts"),
    {baseScore: {$gte: options.postKarmaThreshold}},
    options.postFilter
  );
  
  const includedPostIds = new Set<string>();
  const commentIdsByPost = new Map<string,string[]>();
  const commentIdsByShortform = new Map<string,string[]>();
  const userIds = new Set<string>();

  for (let post of await Posts.find(postSelector, {projection: {_id: true, userId: true}}).fetch()) {
    includedPostIds.add(post._id);
    commentIdsByPost.set(post._id, []);
    userIds.add(post.userId);
  }
  log(`Found ${includedPostIds.size} posts`);
  
  log("Collecting list of comments to include");
  const commentSelector = mergeSelectors(
    getDefaultViewSelector("Comments"),
    {baseScore: {$gte: options.commentKarmaThreshold}},
    options.commentFilter
  );
  
  if (options.includeShortformTopLevelComments) {
    log("Collecting shortform top-level comments");
    for (let comment of await Comments.find(
      {...commentSelector, shortform: true, parentCommentId: null},
      {projection: {_id: true, userId: true}}).fetch()
    ) {
      commentIdsByShortform.set(comment._id, [comment._id]);
      userIds.add(comment.userId);
    }
  }

  if (options.includeReplyComments === "all") {
    for (let comment of await Comments.find(commentSelector, {projection: {_id: true, postId: true}}).fetch()) {
      const postId = comment.postId ?? "";
      if (!commentIdsByPost.has(postId))
        commentIdsByPost.set(postId, []);
      commentIdsByPost.get(postId)!.push(comment._id);
      userIds.add(comment.userId);
    }
  } else if (options.includeReplyComments === "onlyIfPostIncluded") {
    const potentiallyIncludedComments = new Map<string, {parentCommentId: string|null, postId: string|null, topLevelCommentId: string|null, userId: string}>();
    for (let comment of await Comments.find(
      commentSelector,
      {projection: {_id: true, parentCommentId: true, postId: true, topLevelCommentId: true, userId: true}}
    ).fetch()) {
      potentiallyIncludedComments.set(comment._id, {parentCommentId: comment.parentCommentId, postId: comment.postId, topLevelCommentId: comment.topLevelCommentId, userId: comment.userId});
      userIds.add(comment.userId);
    }
    
    for (let comment of potentiallyIncludedComments) {
      const [commentId, {parentCommentId, postId, topLevelCommentId}] = comment;
      
      // Comment must be on an included post or an included shortform
      if (!(postId && includedPostIds.has(postId)) && !(topLevelCommentId && commentIdsByShortform.has(topLevelCommentId))) {
        continue;
      }
      
      if (topLevelCommentId && commentIdsByShortform.has(topLevelCommentId)) {
        commentIdsByShortform.get(topLevelCommentId)!.push(commentId);
      } else if (postId) {
        if (!commentIdsByPost.has(postId))
          commentIdsByPost.set(postId, []);
        commentIdsByPost.get(postId)!.push(commentId);
      }
    }
  }
  
  log(`Finished selecting posts and comments to include: ${includedPostIds.size} posts, ${[...commentIdsByPost.values()].flat().length} comments on posts, and ${[...commentIdsByShortform.values()].flat().length} comments on shortform`);
  
  const authorNameCache = new Map<string,string>();
  const usersById = keyBy(
    await Users.find(
      {_id: {$in: [...userIds.values()]}},
      {projection: {_id: true, deleted: true, displayName: true}}
    ).fetch(),
    u => u._id
  );
  
  async function getAuthorName(userId: string): Promise<string> {
    if (authorNameCache.has(userId)) {
      return authorNameCache.get(userId)!;
    }
    
    const author = (userId in usersById) ? usersById[userId] : await Users.findOne({_id: userId});
    const name = (!author || author.deleted || !author.displayName)
      ? "[anonymous]"
      : author.displayName;
    authorNameCache.set(userId, name);
    return name;
  }

  for (let postBatch of chunk([...commentIdsByPost.keys()], 50)) {
    // Fetch posts and comments related to a batch. Note that depending on
    // filter options, a comment may be included without the corresponding post.
    // If there are comments on an excluded post, postBatch contains the post ID
    // but postIds does not.
    const postIds = postBatch.filter(postId => includedPostIds.has(postId));
    const commentIds = postBatch.map(postId => commentIdsByPost.get(postId)!).flat();
    const [posts, comments] = await Promise.all([
      Posts.find({_id: {$in: postIds}}).fetch(),
      Comments.find({_id: {$in: commentIds}}).fetch()
    ]);
    
    const commentsByPost = groupBy(comments, c=>c.postId);
    let first = true;
    for (const postId of postBatch) {
      const rendered = await renderForFinetune({
        options,
        post: posts.find(p=>p._id===postId) ?? null,
        comments: commentsByPost[postId] ?? [],
        getAuthorName,
      });
      await write(rendered);
    }
  }
  
  const commentsOnShortform = await Comments.find({
    _id: {$in: [...commentIdsByShortform.values()].flat()}
  }).fetch();
  const commentsByShortform = groupBy(commentsOnShortform, c=>c.topLevelCommentId ?? c._id);
  let first = true;
  for (const shortformCommentId of [...commentIdsByShortform.keys()]) {
    const rendered = await renderForFinetune({
      options,
      post: null,
      comments: commentsByShortform[shortformCommentId],
      getAuthorName,
    });
    await write(rendered);
  }

  // eslint-disable-next-line no-console
  log(`Finished generating fine-tune data: ${options.outputFilename}`);
}

async function renderForFinetune({options, post, comments, getAuthorName}: {
  options: HighKarmaContentDatasetOptions,
  post: DbPost|null
  comments: DbComment[],
  getAuthorName: (authorId: string) => Promise<string>
}) {
  const sb: string[] = [];
  
  if (post) {
    sb.push(await renderPostForFinetune({post, getAuthorName}));
  } else {
    sb.push("Shortform\n\n");
  }
  
  const sortedComments: DbComment[] = sortBy(comments, c=>c.postedAt);
  for (let comment of sortedComments) {
    sb.push(options.documentSeparator+'\n\n');
    sb.push(await renderCommentForFinetune({comment, getAuthorName}));
  }
  
  sb.push(options.documentSeparator+"\n\n");
  return sb.join("");
}

async function renderPostForFinetune({ post, getAuthorName }: {
  post: DbPost
  getAuthorName: (authorId: string) => Promise<string>
}) {
  const sb: string[] = [];
  sb.push(`${post.title}\n`)
  sb.push(`by ${await getAuthorName(post.userId)}\n`)
  sb.push(`${post.baseScore} points\n`);
  sb.push(`\n`);
  if ('url' in post && post.url) {
    sb.push(`This is a linkpost for ${post.url}\n\n`);
  }
  const body = post.contents?.originalContents;
  if (body) {
    const bodyMarkdown = dataToMarkdown(body.data, body.type);
    sb.push(postprocessMarkdown(bodyMarkdown));
    sb.push('\n\n');
  }
  return sb.join('');
}

async function renderCommentForFinetune({ comment, getAuthorName }: {
  comment: DbComment
  getAuthorName: (authorId: string) => Promise<string>
}) {
  const sb: string[] = [];
  sb.push(`Comment by ${await getAuthorName(comment.userId)}\n\n`);
  const body = comment.contents?.originalContents;
  if (body) {
    const bodyMarkdown = dataToMarkdown(body.data, body.type);
    sb.push(postprocessMarkdown(bodyMarkdown));
    sb.push('\n\n');
  }
  return sb.join('');
}

/**
 * Ensure the given directory exists and is a directory. Checks path components
 * in order; if any don't exist, creates them with fs.mkdirSync. If any path
 * component exists but isn't a directory, throws an exception.
 */
function ensurePathExists(filename: string): void {
  const dirname = path.dirname(filename);
  const normalizedPath = path.normalize(dirname);
  const pathComponents = normalizedPath.split(path.sep);
  
  for (let i=0; i<pathComponents.length; i++) {
    const currentPath = pathComponents.slice(0,i+1).join(path.sep);
    
    if (!fs.existsSync(currentPath)) {
      fs.mkdirSync(currentPath);
    } else {
      const stats = fs.statSync(currentPath);
      if (!stats.isDirectory()) {
        throw new Error(`Path exists but is not a directory: ${currentPath}`);
      }
    }
  }
}

Globals.generateHighKarmaContentDataset = generateHighKarmaContentDataset



type PostResponsePairsOptions = {
  outputFilename: string

  postKarmaThreshold: number
  commentKarmaThreshold: number
  examplesCount: number

  // Filter for posts to include. This will be merged with the default post
  // view, which takes care of excluding drafts, spam, etc, and with the
  // postKarmaThreshold option above.
  postFilter: ViewQueryAndOptions<"Posts",DbPost>["selector"]
  
  // Filter for comments to include. This will be merged with the default
  // comment view, which takes care of excluding deleted comments.
  commentFilter: ViewQueryAndOptions<"Comments",DbComment>["selector"]
  
  systemPrompt: "You are highly capable writing assistant that helps authors by replying to their essays as other people will. Your replies may riff on, refute, or comment on the essay or the essay's topic."
}

const defaultPostResponsePairsOptions = {
  outputFilename: "ml/postResponsePairs.json",
  commentKarmaThreshold: 50,
  postKarmaThreshold: -20,
  examplesCount: 500,

  postFilter: {
    frontpageDate: {$gt: new Date(0)}
  },
}

async function generatePostResponsePairsDataset(_options: PostResponsePairsOptions) {
  const options = {...defaultPostResponsePairsOptions, ..._options};
  const startTime = new Date();
  await withStreamingWrite(options.outputFilename, async (write) => {
    console.log("Selecting comments for post-comment pairs data");
    // Pick random high-karma comments that meet the requirements
    const commentSelector = mergeSelectors(
      getDefaultViewSelector("Comments"),
      options.commentFilter,
      {
        baseScore: {$gte: options.commentKarmaThreshold},
        postId: {$ne: null},
        parentCommentId: null,
      },
    );
    const eligibleComments = await Comments.find(commentSelector, {
      projection: {_id: true, postId: true}
    }).fetch();

    console.log("Getting posts for post-comment pairs data");
    const postSelector = mergeSelectors(
      getDefaultViewSelector("Posts"),
      options.postFilter
    );
    const eligiblePostIds = (await Posts.find(postSelector, {
      projection: {_id: true}
    }).fetch()).map(p => p._id);
    const eligiblePostIdsSet = new Set<string>(eligiblePostIds);
    
    const eligibleCommentsOnEligiblePosts = eligibleComments
      .filter(c => c.postId && eligiblePostIdsSet.has(c.postId));
    const commentIds = shuffle(eligibleCommentsOnEligiblePosts).slice(0, options.examplesCount);
    
    console.log("Downloading full posts/comments");
    const comments = await Comments.find({
      _id: {$in: commentIds.map(c=>c._id)}
    }).fetch();
    const postIds = new Set(comments.map(c => c.postId));
    const posts = await Posts.find({
      _id: {$in: [...postIds.values()]}
    }).fetch();
    const postsById = keyBy(posts, p=>p._id);
    
    const getAuthorName = async (userId: string) => "AnonymizedUser"

    console.log("Writing out examples");
    const data: any[] = [];
    for (let comment of comments) {
      const post = postsById[comment.postId!];
      const renderedPost = await renderPostForFinetune({post, getAuthorName})
      const renderedComment = await renderCommentForFinetune({comment, getAuthorName})
      const example = {
        messages: [
          {role: "system", content: options.systemPrompt},
          {role: "user", content: renderedPost},
          {role: "assistant", content: renderedComment},
        ],
      };
      await write(JSON.stringify(example)+"\n");
    }
    // eslint-disable-next-line no-console
    console.log(`Finetune dataset generated in ${new Date().getTime() - startTime.getTime()}ms`);
  });
}



Globals.generatePostResponsePairs = generatePostResponsePairsDataset

1	import { Globals } from '@/lib/vulcan-lib/config';	18✔
2	import { getDefaultViewSelector, mergeSelectors } from "@/lib/utils/viewUtils";	18✔
3	import { Comments } from '@/lib/collections/comments/collection';	18✔
4	import { Posts } from '@/lib/collections/posts/collection';	18✔
5	import { Users } from '@/lib/collections/users/collection';	18✔
6	import { dataToMarkdown } from '@/server/editor/conversionUtils';	18✔
7	import path from 'path';	18✔
8	import fs from 'fs';	18✔
9	import fsPromises from 'fs/promises';	18✔
10	import chunk from 'lodash/chunk';	18✔
11	import groupBy from 'lodash/groupBy';	18✔
12	import sortBy from 'lodash/sortBy';	18✔
13	import keyBy from 'lodash/keyBy';	18✔
14	import { postprocessMarkdown } from '@/server/languageModels/llmUtil';	18✔
15	import shuffle from 'lodash/shuffle';	18✔
16
17	type HighKarmaContentDatasetOptions = {
18	postKarmaThreshold: number
19	commentKarmaThreshold: number
20	outputFilename: string
21
22	// Filter for posts to include. This will be merged with the default post
23	// view, which takes care of excluding drafts, spam, etc, and with the
24	// postKarmaThreshold option above.
25	postFilter: ViewQueryAndOptions<"Posts",DbPost>["selector"]
26
27	// Filter for comments to include. This will be merged with the default
28	// comment view, which takes care of excluding deleted comments.
29	commentFilter: ViewQueryAndOptions<"Comments",DbComment>["selector"]
30
31	// Whether to include shortform posts (ie top-level comments on shortform
32	// containers.)
33	includeShortformTopLevelComments: boolean
34
35	// Whether to include comments (other than top-level shortform). If this is
36	// "onlyIfPostIncluded", a comment will be included only if the post it's on
37	// is included (or if it's under a shortform, the top-level shortform
38	// comment).
39	//
40	// (Regardless of this setting, commentFilter and commentKarmaThreshold are
41	// still applied.)
42	includeReplyComments: "all"\|"onlyIfPostIncluded"\|"none"
43
44	documentSeparator: string,
45	}
46	const defaultHighKarmaContentDatasetOptions: HighKarmaContentDatasetOptions = {	18✔
47	postKarmaThreshold: 60,
48	commentKarmaThreshold: 30,
49	outputFilename: "ml/finetuneData.txt",
50	postFilter: {
51	frontpageDate: {$gt: new Date(0)}
52	},
53	commentFilter: {},
54	includeShortformTopLevelComments: true,
55	includeReplyComments: "all",
56	documentSeparator: "========",
57	}
58
59	async function generateHighKarmaContentDataset(_options?: Partial<HighKarmaContentDatasetOptions>) {
NEW 60	const options = {...defaultHighKarmaContentDatasetOptions, ..._options};	×
UNCOV 61	const startTime = new Date();	×
62
NEW 63	await withStreamingWrite(options.outputFilename, async (write) => {	×
NEW 64	await writeHighKarmaContentDatasetTo(options, write);	×
65	// eslint-disable-next-line no-console
NEW 66	console.log(`Finetune dataset generated in ${new Date().getTime() - startTime.getTime()}ms`);	×
67	});
68	}
69
70	async function withStreamingWrite(
71	filename: string,
72	fn: (write: (s: string) => Promise<void>) => Promise<void>
73	): Promise<void> {
NEW 74	ensurePathExists(filename);	×
NEW 75	const fileHandle = await fsPromises.open(filename, 'w');	×
76	if (!fileHandle) {	×
NEW 77	throw new Error(`Unable to open ${filename} for writing`);	×
78	}
79	try {	×
80	const writeStream = fileHandle.createWriteStream();	×
81	// eslint-disable-next-line no-inner-declarations
82	async function write(s: string) {
83	return new Promise<void>((resolve, reject) => {	×
84	writeStream.write(s, (err) => {	×
85	if (err) reject(err);	×
86	else resolve();	×
87	});
88	});
89	}
NEW 90	await fn(write);	×
91	} finally {
92	await fileHandle.close();	×
93	}
94	}
95
96	async function writeHighKarmaContentDatasetTo(options: HighKarmaContentDatasetOptions, write: (s: string) => Promise<void>) {
97	// eslint-disable-next-line no-console
98	const log = console.log;	×
99
100	////////////////////////////////////////////////////////////////////
101	// Data selection stage
102	// First we collect IDs of posts and comments that will be included (without
103	// downloading their bodies from the DB). This is split out so that you can
104	// iterate on filters to see how much will be included, without waiting for
105	// the slow parts (downloading contents from the DB and doing format
106	// conversions), and to work around the possibility of out-of-memory issues.
107
108	log("Generating fine-tune dataset", options);	×
109
110	log("Collecting list of posts to include");	×
111	const postSelector = mergeSelectors(	×
112	getDefaultViewSelector("Posts"),
113	{baseScore: {$gte: options.postKarmaThreshold}},
114	options.postFilter
115	);
116
117	const includedPostIds = new Set<string>();	×
118	const commentIdsByPost = new Map<string,string[]>();	×
119	const commentIdsByShortform = new Map<string,string[]>();	×
120	const userIds = new Set<string>();	×
121
122	for (let post of await Posts.find(postSelector, {projection: {_id: true, userId: true}}).fetch()) {	×
123	includedPostIds.add(post._id);	×
124	commentIdsByPost.set(post._id, []);	×
125	userIds.add(post.userId);	×
126	}
127	log(`Found ${includedPostIds.size} posts`);	×
128
129	log("Collecting list of comments to include");	×
130	const commentSelector = mergeSelectors(	×
131	getDefaultViewSelector("Comments"),
132	{baseScore: {$gte: options.commentKarmaThreshold}},
133	options.commentFilter
134	);
135
136	if (options.includeShortformTopLevelComments) {	×
137	log("Collecting shortform top-level comments");	×
138	for (let comment of await Comments.find(	×
139	{...commentSelector, shortform: true, parentCommentId: null},
140	{projection: {_id: true, userId: true}}).fetch()
141	) {
142	commentIdsByShortform.set(comment._id, [comment._id]);	×
143	userIds.add(comment.userId);	×
144	}
145	}
146
147	if (options.includeReplyComments === "all") {	×
148	for (let comment of await Comments.find(commentSelector, {projection: {_id: true, postId: true}}).fetch()) {	×
149	const postId = comment.postId ?? "";	×
150	if (!commentIdsByPost.has(postId))	×
151	commentIdsByPost.set(postId, []);	×
152	commentIdsByPost.get(postId)!.push(comment._id);	×
153	userIds.add(comment.userId);	×
154	}
155	} else if (options.includeReplyComments === "onlyIfPostIncluded") {	×
156	const potentiallyIncludedComments = new Map<string, {parentCommentId: string\|null, postId: string\|null, topLevelCommentId: string\|null, userId: string}>();	×
157	for (let comment of await Comments.find(	×
158	commentSelector,
159	{projection: {_id: true, parentCommentId: true, postId: true, topLevelCommentId: true, userId: true}}
160	).fetch()) {
161	potentiallyIncludedComments.set(comment._id, {parentCommentId: comment.parentCommentId, postId: comment.postId, topLevelCommentId: comment.topLevelCommentId, userId: comment.userId});	×
162	userIds.add(comment.userId);	×
163	}
164
165	for (let comment of potentiallyIncludedComments) {	×
166	const [commentId, {parentCommentId, postId, topLevelCommentId}] = comment;	×
167
168	// Comment must be on an included post or an included shortform
169	if (!(postId && includedPostIds.has(postId)) && !(topLevelCommentId && commentIdsByShortform.has(topLevelCommentId))) {	×
170	continue;	×
171	}
172
173	if (topLevelCommentId && commentIdsByShortform.has(topLevelCommentId)) {	×
174	commentIdsByShortform.get(topLevelCommentId)!.push(commentId);	×
175	} else if (postId) {	×
176	if (!commentIdsByPost.has(postId))	×
177	commentIdsByPost.set(postId, []);	×
178	commentIdsByPost.get(postId)!.push(commentId);	×
179	}
180	}
181	}
182
183	log(`Finished selecting posts and comments to include: ${includedPostIds.size} posts, ${[...commentIdsByPost.values()].flat().length} comments on posts, and ${[...commentIdsByShortform.values()].flat().length} comments on shortform`);	×
184
185	const authorNameCache = new Map<string,string>();	×
186	const usersById = keyBy(	×
187	await Users.find(
188	{_id: {$in: [...userIds.values()]}},
189	{projection: {_id: true, deleted: true, displayName: true}}
190	).fetch(),
191	u => u._id	×
192	);
193
194	async function getAuthorName(userId: string): Promise<string> {
195	if (authorNameCache.has(userId)) {	×
196	return authorNameCache.get(userId)!;	×
197	}
198
199	const author = (userId in usersById) ? usersById[userId] : await Users.findOne({_id: userId});	×
200	const name = (!author \|\| author.deleted \|\| !author.displayName)	×
201	? "[anonymous]"	×
202	: author.displayName;
203	authorNameCache.set(userId, name);	×
204	return name;	×
205	}
206
207	for (let postBatch of chunk([...commentIdsByPost.keys()], 50)) {	×
208	// Fetch posts and comments related to a batch. Note that depending on
209	// filter options, a comment may be included without the corresponding post.
210	// If there are comments on an excluded post, postBatch contains the post ID
211	// but postIds does not.
212	const postIds = postBatch.filter(postId => includedPostIds.has(postId));	×
213	const commentIds = postBatch.map(postId => commentIdsByPost.get(postId)!).flat();	×
214	const [posts, comments] = await Promise.all([	×
215	Posts.find({_id: {$in: postIds}}).fetch(),
216	Comments.find({_id: {$in: commentIds}}).fetch()
217	]);
218
219	const commentsByPost = groupBy(comments, c=>c.postId);	×
220	let first = true;	×
221	for (const postId of postBatch) {	×
222	const rendered = await renderForFinetune({	×
223	options,
224	post: posts.find(p=>p._id===postId) ?? null,	×
225	comments: commentsByPost[postId] ?? [],	×
226	getAuthorName,
227	});
228	await write(rendered);	×
229	}
230	}
231
232	const commentsOnShortform = await Comments.find({	×
233	_id: {$in: [...commentIdsByShortform.values()].flat()}
234	}).fetch();
235	const commentsByShortform = groupBy(commentsOnShortform, c=>c.topLevelCommentId ?? c._id);	×
236	let first = true;	×
237	for (const shortformCommentId of [...commentIdsByShortform.keys()]) {	×
238	const rendered = await renderForFinetune({	×
239	options,
240	post: null,
241	comments: commentsByShortform[shortformCommentId],
242	getAuthorName,
243	});
244	await write(rendered);	×
245	}
246
247	// eslint-disable-next-line no-console
248	log(`Finished generating fine-tune data: ${options.outputFilename}`);	×
249	}
250
251	async function renderForFinetune({options, post, comments, getAuthorName}: {
252	options: HighKarmaContentDatasetOptions,
253	post: DbPost\|null
254	comments: DbComment[],
255	getAuthorName: (authorId: string) => Promise<string>
256	}) {
257	const sb: string[] = [];	×
258
259	if (post) {	×
NEW 260	sb.push(await renderPostForFinetune({post, getAuthorName}));	×
261	} else {
262	sb.push("Shortform\n\n");	×
263	}
264
265	const sortedComments: DbComment[] = sortBy(comments, c=>c.postedAt);	×
266	for (let comment of sortedComments) {	×
267	sb.push(options.documentSeparator+'\n\n');	×
NEW 268	sb.push(await renderCommentForFinetune({comment, getAuthorName}));	×
269	}
270
271	sb.push(options.documentSeparator+"\n\n");	×
272	return sb.join("");	×
273	}
274
275	async function renderPostForFinetune({ post, getAuthorName }: {
276	post: DbPost
277	getAuthorName: (authorId: string) => Promise<string>
278	}) {
NEW 279	const sb: string[] = [];	×
NEW 280	sb.push(`${post.title}\n`)	×
NEW 281	sb.push(`by ${await getAuthorName(post.userId)}\n`)	×
NEW 282	sb.push(`${post.baseScore} points\n`);	×
NEW 283	sb.push(`\n`);	×
NEW 284	if ('url' in post && post.url) {	×
NEW 285	sb.push(`This is a linkpost for ${post.url}\n\n`);	×
286	}
NEW 287	const body = post.contents?.originalContents;	×
NEW 288	if (body) {	×
NEW 289	const bodyMarkdown = dataToMarkdown(body.data, body.type);	×
NEW 290	sb.push(postprocessMarkdown(bodyMarkdown));	×
NEW 291	sb.push('\n\n');	×
292	}
NEW 293	return sb.join('');	×
294	}
295
296	async function renderCommentForFinetune({ comment, getAuthorName }: {
297	comment: DbComment
298	getAuthorName: (authorId: string) => Promise<string>
299	}) {
NEW 300	const sb: string[] = [];	×
NEW 301	sb.push(`Comment by ${await getAuthorName(comment.userId)}\n\n`);	×
NEW 302	const body = comment.contents?.originalContents;	×
NEW 303	if (body) {	×
NEW 304	const bodyMarkdown = dataToMarkdown(body.data, body.type);	×
NEW 305	sb.push(postprocessMarkdown(bodyMarkdown));	×
NEW 306	sb.push('\n\n');	×
307	}
NEW 308	return sb.join('');	×
309	}
310
311	/**
312	* Ensure the given directory exists and is a directory. Checks path components
313	* in order; if any don't exist, creates them with fs.mkdirSync. If any path
314	* component exists but isn't a directory, throws an exception.
315	*/
316	function ensurePathExists(filename: string): void {
NEW 317	const dirname = path.dirname(filename);	×
318	const normalizedPath = path.normalize(dirname);	×
319	const pathComponents = normalizedPath.split(path.sep);	×
320
321	for (let i=0; i<pathComponents.length; i++) {	×
322	const currentPath = pathComponents.slice(0,i+1).join(path.sep);	×
323
324	if (!fs.existsSync(currentPath)) {	×
325	fs.mkdirSync(currentPath);	×
326	} else {
327	const stats = fs.statSync(currentPath);	×
328	if (!stats.isDirectory()) {	×
329	throw new Error(`Path exists but is not a directory: ${currentPath}`);	×
330	}
331	}
332	}
333	}
334
335	Globals.generateHighKarmaContentDataset = generateHighKarmaContentDataset	18✔
336
337
338
339	type PostResponsePairsOptions = {
340	outputFilename: string
341
342	postKarmaThreshold: number
343	commentKarmaThreshold: number
344	examplesCount: number
345
346	// Filter for posts to include. This will be merged with the default post
347	// view, which takes care of excluding drafts, spam, etc, and with the
348	// postKarmaThreshold option above.
349	postFilter: ViewQueryAndOptions<"Posts",DbPost>["selector"]
350
351	// Filter for comments to include. This will be merged with the default
352	// comment view, which takes care of excluding deleted comments.
353	commentFilter: ViewQueryAndOptions<"Comments",DbComment>["selector"]
354
355	systemPrompt: "You are highly capable writing assistant that helps authors by replying to their essays as other people will. Your replies may riff on, refute, or comment on the essay or the essay's topic."
356	}
357
358	const defaultPostResponsePairsOptions = {	18✔
359	outputFilename: "ml/postResponsePairs.json",
360	commentKarmaThreshold: 50,
361	postKarmaThreshold: -20,
362	examplesCount: 500,
363
364	postFilter: {
365	frontpageDate: {$gt: new Date(0)}
366	},
367	}
368
369	async function generatePostResponsePairsDataset(_options: PostResponsePairsOptions) {
NEW 370	const options = {...defaultPostResponsePairsOptions, ..._options};	×
NEW 371	const startTime = new Date();	×
NEW 372	await withStreamingWrite(options.outputFilename, async (write) => {	×
NEW 373	console.log("Selecting comments for post-comment pairs data");	×
374	// Pick random high-karma comments that meet the requirements
NEW 375	const commentSelector = mergeSelectors(	×
376	getDefaultViewSelector("Comments"),
377	options.commentFilter,
378	{
379	baseScore: {$gte: options.commentKarmaThreshold},
380	postId: {$ne: null},
381	parentCommentId: null,
382	},
383	);
NEW 384	const eligibleComments = await Comments.find(commentSelector, {	×
385	projection: {_id: true, postId: true}
386	}).fetch();
387
NEW 388	console.log("Getting posts for post-comment pairs data");	×
NEW 389	const postSelector = mergeSelectors(	×
390	getDefaultViewSelector("Posts"),
391	options.postFilter
392	);
NEW 393	const eligiblePostIds = (await Posts.find(postSelector, {	×
394	projection: {_id: true}
NEW 395	}).fetch()).map(p => p._id);	×
NEW 396	const eligiblePostIdsSet = new Set<string>(eligiblePostIds);	×
397
NEW 398	const eligibleCommentsOnEligiblePosts = eligibleComments	×
NEW 399	.filter(c => c.postId && eligiblePostIdsSet.has(c.postId));	×
NEW 400	const commentIds = shuffle(eligibleCommentsOnEligiblePosts).slice(0, options.examplesCount);	×
401
NEW 402	console.log("Downloading full posts/comments");	×
NEW 403	const comments = await Comments.find({	×
NEW 404	_id: {$in: commentIds.map(c=>c._id)}	×
405	}).fetch();
NEW 406	const postIds = new Set(comments.map(c => c.postId));	×
NEW 407	const posts = await Posts.find({	×
408	_id: {$in: [...postIds.values()]}
409	}).fetch();
NEW 410	const postsById = keyBy(posts, p=>p._id);	×
411
NEW 412	const getAuthorName = async (userId: string) => "AnonymizedUser"	×
413
NEW 414	console.log("Writing out examples");	×
NEW 415	const data: any[] = [];	×
NEW 416	for (let comment of comments) {	×
NEW 417	const post = postsById[comment.postId!];	×
NEW 418	const renderedPost = await renderPostForFinetune({post, getAuthorName})	×
NEW 419	const renderedComment = await renderCommentForFinetune({comment, getAuthorName})	×
NEW 420	const example = {	×
421	messages: [
422	{role: "system", content: options.systemPrompt},
423	{role: "user", content: renderedPost},
424	{role: "assistant", content: renderedComment},
425	],
426	};
NEW 427	await write(JSON.stringify(example)+"\n");	×
428	}
429	// eslint-disable-next-line no-console
NEW 430	console.log(`Finetune dataset generated in ${new Date().getTime() - startTime.getTime()}ms`);	×
431	});
432	}
433
434
435
436	Globals.generatePostResponsePairs = generatePostResponsePairsDataset	18✔

ForumMagnum / ForumMagnum / 10050044142

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous