10589248057

Committed 28 Aug 2024 02:42AM UTC coverage: 31.951% (-0.09%) from 32.037%

Build # 10589248057

Build Type

Pull #9633

github

Committed by

Discordius

Commit Message

More lint

Pull Request Pull Request #9633: LLM Autocomplete for admins

Run Details

5020 of 40912 branches covered (12.27%)

Branch coverage included in aggregate %.

49 of 280 new or added lines in 7 files covered. (17.5%)

14 existing lines in 2 files now uncovered.

29254 of 66360 relevant lines covered (44.08%)

188.61 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

8.15

/packages/lesswrong/server/autocompleteEndpoint.ts

import type { Express } from "express";
import { getUserFromReq } from "./vulcan-lib/apollo-server/context";
import { userIsAdmin } from "@/lib/vulcan-users/permissions";
import { getAnthropicPromptCachingClientOrThrow } from "./languageModels/anthropicClient";
import express from "express";
import { PromptCachingBetaMessageParam } from "@anthropic-ai/sdk/resources/beta/prompt-caching/messages";
import { Posts } from "@/lib/collections/posts";
import { Comments } from "@/lib/collections/comments";
import Revisions from "@/lib/collections/revisions/collection";
import { turndownService } from "./editor/conversionUtils";
import { formatRelative } from "@/lib/utils/timeFormat";
import Users from "@/lib/vulcan-users";
const { Readable } = require('stream');
import { pipeline } from 'stream/promises'
import { hyperbolicApiKey } from "@/lib/instanceSettings";



const getParentComments = async (comment: DbComment) => {
  const parentComments: DbComment[] = [];
  let currentComment: DbComment | null = comment;
  while (currentComment.parentCommentId && currentComment.parentCommentId !== currentComment._id) {
    currentComment = await Comments.findOne({ _id: currentComment.parentCommentId });
    if (currentComment) {
      parentComments.push(currentComment);
    } else {
      break;
    }
  }

  return parentComments.reverse();
}

const getPostBodyFormatted = (post: DbPost, revisionsMap: Map<string, DbRevision>, authorsMap: Map<string, DbUser>) => {
  const markdownBody = turndownService.turndown(revisionsMap.get(post._id)?.html ?? "<not available/>");
  const author = authorsMap.get(post.userId)?.displayName;
  return `${post.title}
by ${author}
${post.baseScore}
${markdownBody}`.trim();
}

const getCommentBodyFormatted = (comment: DbComment, revisionsMap: Map<string, DbRevision>, authorsMap: Map<string, DbUser>) => {
  const markdownBody = turndownService.turndown(revisionsMap.get(comment._id)?.html ?? comment.contents?.html ?? "<not available/>");
  const dateString = formatRelative(new Date(comment.createdAt), new Date(), false)
  const author = authorsMap.get(comment.userId)?.displayName;
  return `${author} ${dateString} ${comment.baseScore} ${comment.extendedScore?.agreement}
${markdownBody}`.trim();
}

const getPostReplyMessageFormatted = (post: DbPost, revisionsMap: Map<string, DbRevision>, authorsMap: Map<string, DbUser>, currentUser: DbUser, prefix: string) => {
  return `${getPostBodyFormatted(post, revisionsMap, authorsMap)}
---
${currentUser.displayName} 1h ${Math.floor(20 + (Math.random() * 75))} ${Math.floor((Math.random() - 0.5) * 100)}
${prefix}`.trim();
}

const getCommentReplyMessageFormatted = (comment: DbComment, parentComments: DbComment[], parentPost: DbPost, revisionsMap: Map<string, DbRevision>, authorsMap: Map<string, DbUser>, prefix: string, currentUser: DbUser) => {
  const parentComment = parentComments[parentComments.length - 1];
  return `${getPostBodyFormatted(parentPost, revisionsMap, authorsMap)}
---
${parentComments.map((comment) => getCommentBodyFormatted(comment, revisionsMap, authorsMap)).join("\n---\n")}
---
${getCommentBodyFormatted(comment, revisionsMap, authorsMap)}
---
${currentUser.displayName} 1h ${Math.floor(20 + (Math.random() * 75))} ${Math.floor((Math.random() - 0.5) * 100)}
${prefix}`.trim();
}

async function constructMessageHistory(
  prefix: string,
  commentIds: string[],
  postIds: string[],
  currentUser: any,
  replyingCommentId?: string,
  postId?: string
): Promise<PromptCachingBetaMessageParam[]> {
  const messages: PromptCachingBetaMessageParam[] = [];

  // Make the fetches parallel to save time
  const [revisions, posts, comments] = await Promise.all([
    Revisions.find({ documentId: { $in: [...postIds, ...commentIds] }, fieldName: "contents", version: "1.0.0" }).fetch(),
    Posts.find({ _id: { $in: postIds } }).fetch(),
    Comments.find({ _id: { $in: commentIds } }).fetch(),
  ]);

  const authors = await Users.find({ _id: { $in: [...posts.map((post) => post.userId), ...comments.map((comment) => comment.userId)] } }).fetch();

  const authorsMap = new Map(authors.map((author) => [author._id, author]));
  const revisionsMap = new Map(revisions.map((revision) => [revision.documentId!, revision]));

  // eslint-disable-next-line no-console
  console.log(`Converting ${posts.length} posts and ${comments.length} comments to messages`);
  // eslint-disable-next-line no-console
  console.time("constructPostMessageHistory");

  // Add fetched posts and comments to message history
  for (const post of posts) {
    messages.push({
      role: "user",
      content: [{ type: "text", text: `<cmd>cat lw/${post._id}.txt</cmd>` }],
    });

    messages.push({
      role: "assistant",
      content: [
        {
          type: "text",
          text: getPostBodyFormatted(post, revisionsMap, authorsMap),
        },
      ],
    });
  }

  // eslint-disable-next-line no-console
  console.timeEnd("constructPostMessageHistory");

  for (const comment of comments) {
    messages.push({
      role: "user",
      content: [{ type: "text", text: `<cmd>cat lw/${comment._id}.txt</cmd>` }],
    });

    messages.push({
      role: "assistant",
      content: [{ type: "text", text: getCommentBodyFormatted(comment, revisionsMap, authorsMap) }],
    });
  }

  // Add final user message with prefix
  messages.push({
    role: "user",
    content: [{ type: "text", text: `<cmd>cat lw/hsqKp56whpPEQns3Z.txt</cmd>`, cache_control: {type: "ephemeral"}}]
  });

  if (replyingCommentId) {
    // Fetch the comment we're replying to
    const replyingToComment = await Comments.findOne({ _id: replyingCommentId });
    if (!replyingToComment) {
      throw new Error("Comment not found");
    }
    const parentComments = await getParentComments(replyingToComment);
    const parentPost = await Posts.findOne({ _id: replyingToComment.postId });
    const parentPostRevision = await Revisions.findOne({ documentId: parentPost?._id, fieldName: "contents", version: "1.0.0" });

    if (!parentPost) {
      throw new Error(`Parent post or revision not found ${replyingToComment.postId}`);
    }

    const authors = await Users.find({ _id: { $in: [parentPost.userId, ...parentComments.map((comment) => comment.userId), replyingToComment.userId] } }).fetch();

    authors.forEach((author) => authorsMap.set(author._id, author));
    revisionsMap.set(parentPost._id!, parentPostRevision!);
    if (!parentPost) {
      throw new Error("Post not found");
    }

    const message = getCommentReplyMessageFormatted(replyingToComment, parentComments, parentPost, revisionsMap, authorsMap, prefix, currentUser)

    messages.push({
      role: "assistant",
      content: [
        {
          type: "text",
          text: message
        },
      ],
    });
  }
  else if (postId) {
    const post = await Posts.findOne({ _id: postId });
    if (!post) {
      throw new Error("Post not found");
    }
    const postRevision = await Revisions.findOne({ documentId: post._id, fieldName: "contents", version: "1.0.0" });
    if (!postRevision) {
      throw new Error("Post revision not found");
    }

    const authors = await Users.find({ _id: post.userId }).fetch();
    authors.forEach((author) => authorsMap.set(author._id, author));
    revisionsMap.set(post._id!, postRevision);

    const message = getPostReplyMessageFormatted(post, revisionsMap, authorsMap, currentUser, prefix);

    messages.push({
      role: "assistant",
      content: [
        {
          type: "text",
          text: getPostReplyMessageFormatted(post, revisionsMap, authorsMap, currentUser, prefix)
        },
      ],
    });
  }
  else {
    messages.push({
      role: "assistant",
      content: [
        {
          type: "text",
          text: `${prefix}`
        }
      ]
    });
  }

  return messages;
}

async function construct405bPrompt(
  prefix: string,
  commentIds: string[],
  postIds: string[],
  currentUser: any,
  replyingCommentId?: string,
  postId?: string
): Promise<string> {

  // Make the fetches parallel to save time
  const [revisions, posts, comments] = await Promise.all([
    Revisions.find({ documentId: { $in: [...postIds, ...commentIds] }, fieldName: "contents", version: "1.0.0" }).fetch(),
    Posts.find({ _id: { $in: postIds } }).fetch(),
    Comments.find({ _id: { $in: commentIds } }).fetch(),
  ]);

  const authors = await Users.find({ _id: { $in: [...posts.map((post) => post.userId), ...comments.map((comment) => comment.userId)] } }).fetch();

  const authorsMap = new Map(authors.map((author) => [author._id, author]));
  const revisionsMap = new Map(revisions.map((revision) => [revision.documentId!, revision]));

  // eslint-disable-next-line no-console
  console.log(`Converting ${posts.length} posts and ${comments.length} comments to messages`);

  let finalSection = ''

  if (replyingCommentId) {
    // Fetch the comment we're replying to
    const replyingToComment = await
    Comments.findOne({ _id: replyingCommentId });
    if (!replyingToComment) {
      throw new Error("Comment not found");
    }
    const parentComments = await getParentComments(replyingToComment);
    const parentPost = await Posts.findOne({ _id: replyingToComment.postId });
    const parentPostRevision = await Revisions.findOne({ documentId: parentPost?._id, fieldName: "contents", version: "1.0.0" });

    if (!parentPost) {
      throw new Error(`Parent post or revision not found ${replyingToComment.postId}`);
    }

    const authors = await Users.find({ _id: { $in: [parentPost.userId, ...parentComments.map((comment) => comment.userId), replyingToComment.userId] } }).fetch();

    authors.forEach((author) => authorsMap.set(author._id, author));
    revisionsMap.set(parentPost._id!, parentPostRevision!);

    finalSection = getCommentReplyMessageFormatted(replyingToComment, parentComments, parentPost, revisionsMap, authorsMap, prefix, currentUser)    
  } else if (postId) {
    const post = await Posts.findOne({ _id: postId });
    if (!post) {
      throw new Error("Post not found");
    }
    const postRevision = await Revisions.findOne({ documentId: post._id, fieldName: "contents", version: "1.0.0" });
    if (!postRevision) {
      throw new Error("Post revision not found");
    }

    const authors = await Users.find({ _id: post.userId }).fetch();
    authors.forEach((author) => authorsMap.set(author._id, author));
    revisionsMap.set(post._id!, postRevision);

    finalSection = getPostReplyMessageFormatted(post, revisionsMap, authorsMap, currentUser, prefix)
  } else {
    finalSection = `${prefix}`
  }


  return `
${posts.map((post) => getPostBodyFormatted(post, revisionsMap, authorsMap)).join("\n---\n")}
---
${comments.map((comment) => getCommentBodyFormatted(comment, revisionsMap, authorsMap)).join("\n---\n")}
---
${finalSection}`.trim();
}


export function addAutocompleteEndpoint(app: Express) {
  app.use("/api/autocomplete", express.json());
  app.post("/api/autocomplete", async (req, res) => {
    const currentUser = getUserFromReq(req);
    if (!currentUser) {
      return res.status(401).json({ error: "Unauthorized" });
    }

    try {
      if (!userIsAdmin(currentUser)) {
        throw new Error("Claude Completion is for admins only");
      }

      const client = getAnthropicPromptCachingClientOrThrow();

      const { prefix = '', commentIds, postIds, replyingCommentId, postId } = req.body;

      // Set headers for streaming response
      res.writeHead(200, {
        "Content-Type": "text/event-stream",
        "Cache-Control": "no-cache",
        Connection: "keep-alive",
      });

      const loadingMessagesStream = client.messages.stream({
        model: "claude-3-5-sonnet-20240620",
        max_tokens: 1000,
        system: "The assistant is in CLI simulation mode, and responds to the user's CLI commands only with the output of the command.",
        messages: await constructMessageHistory(
          prefix,
          commentIds,
          postIds,
          currentUser,
          replyingCommentId,
          postId
        ),
      });

      loadingMessagesStream.on("text", (delta, snapshot) => {
        res.write(
          `data: ${JSON.stringify({ type: "text", content: delta })}\n\n`,
        );
      });

      loadingMessagesStream.on("end", () => {
        res.write(`data: ${JSON.stringify({ type: "end" })}\n\n`);
        res.end();
      });

      loadingMessagesStream.on("error", (error) => {
        // eslint-disable-next-line no-console
        console.error("Stream error:", JSON.stringify(error));
        res.write(
          `data: ${JSON.stringify({ type: "error", message: "An error occurred" })}\n\n`,
        );
        res.end();
      });
    } catch (error) {
      // eslint-disable-next-line no-console
      console.error("Autocomplete error:", error);
      res.status(500).json({ error: "An error occurred during autocomplete" });
    }
  });
  app.use("/api/autocomplete405b", express.json());
  app.post("/api/autocomplete405b", async (req, res) => {
    const currentUser = getUserFromReq(req);
    if (!currentUser) {
      return res.status(401).json({ error: "Unauthorized" });
    }

    const { prefix = '', commentIds, postIds, replyingCommentId, postId } = req.body;
    // Set headers for streaming response
    res.writeHead(200, {
      "Content-Type": "text/event-stream",
      "Cache-Control": "no-cache",
      Connection: "keep-alive",
    });

    const url = 'https://api.hyperbolic.xyz/v1/completions';

    const response = await fetch(url, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        Authorization: `Bearer ${hyperbolicApiKey.get()}`,
      },
      body: JSON.stringify({
        model: 'meta-llama/Meta-Llama-3.1-405B',
        prompt: await construct405bPrompt(
          prefix,
          commentIds,
          postIds,
          currentUser,
          replyingCommentId,
          postId
        ),
        max_tokens: 256,
        temperature: 0.7,
        top_p: 0.9,
        stream: true
      }),
    });

    if (!response.ok || !response.body) {
      res.writeHead(response.status, response.statusText);
      res.end(`Error from API: ${response.statusText}`);
      return;
    }

    try {
      if (!userIsAdmin(currentUser)) {
        throw new Error("Claude Completion is for admins only");
      }
      const reader = response.body.getReader();
      const readableNodeStream = new Readable({
        async read() {
          const { done, value } = await reader.read();
          if (done) {
            this.push(null);
          } else {
            this.push(Buffer.from(value));
          }
        }
      });

      await pipeline(readableNodeStream, res);
    } catch (error) {
      // eslint-disable-next-line no-console
      console.error("Autocomplete error:", error);
      res.status(500).json({ error: "An error occurred during autocomplete" });
    }
  });
}

1	import type { Express } from "express";	54✔
2	import { getUserFromReq } from "./vulcan-lib/apollo-server/context";	18✔
3	import { userIsAdmin } from "@/lib/vulcan-users/permissions";	18✔
4	import { getAnthropicPromptCachingClientOrThrow } from "./languageModels/anthropicClient";	18✔
5	import express from "express";	18✔
6	import { PromptCachingBetaMessageParam } from "@anthropic-ai/sdk/resources/beta/prompt-caching/messages";
7	import { Posts } from "@/lib/collections/posts";	18✔
8	import { Comments } from "@/lib/collections/comments";	18✔
9	import Revisions from "@/lib/collections/revisions/collection";	18✔
10	import { turndownService } from "./editor/conversionUtils";	18✔
11	import { formatRelative } from "@/lib/utils/timeFormat";	18✔
12	import Users from "@/lib/vulcan-users";	18✔
13	const { Readable } = require('stream');	18✔
14	import { pipeline } from 'stream/promises'	18✔
15	import { hyperbolicApiKey } from "@/lib/instanceSettings";	18✔
16
17
18
19	const getParentComments = async (comment: DbComment) => {	18✔
NEW 20	const parentComments: DbComment[] = [];	×
NEW 21	let currentComment: DbComment \| null = comment;	×
NEW 22	while (currentComment.parentCommentId && currentComment.parentCommentId !== currentComment._id) {	×
NEW 23	currentComment = await Comments.findOne({ _id: currentComment.parentCommentId });	×
NEW 24	if (currentComment) {	×
NEW 25	parentComments.push(currentComment);	×
26	} else {
NEW 27	break;	×
28	}
29	}
30
NEW 31	return parentComments.reverse();	×
32	}
33
34	const getPostBodyFormatted = (post: DbPost, revisionsMap: Map<string, DbRevision>, authorsMap: Map<string, DbUser>) => {	18✔
NEW 35	const markdownBody = turndownService.turndown(revisionsMap.get(post._id)?.html ?? "<not available/>");	×
NEW 36	const author = authorsMap.get(post.userId)?.displayName;	×
NEW 37	return `${post.title}	×
38	by ${author}
39	${post.baseScore}
40	${markdownBody}`.trim();
41	}
42
43	const getCommentBodyFormatted = (comment: DbComment, revisionsMap: Map<string, DbRevision>, authorsMap: Map<string, DbUser>) => {	18✔
NEW 44	const markdownBody = turndownService.turndown(revisionsMap.get(comment._id)?.html ?? comment.contents?.html ?? "<not available/>");	×
NEW 45	const dateString = formatRelative(new Date(comment.createdAt), new Date(), false)	×
NEW 46	const author = authorsMap.get(comment.userId)?.displayName;	×
NEW 47	return `${author} ${dateString} ${comment.baseScore} ${comment.extendedScore?.agreement}	×
48	${markdownBody}`.trim();
49	}
50
51	const getPostReplyMessageFormatted = (post: DbPost, revisionsMap: Map<string, DbRevision>, authorsMap: Map<string, DbUser>, currentUser: DbUser, prefix: string) => {	18✔
NEW 52	return `${getPostBodyFormatted(post, revisionsMap, authorsMap)}	×
53	---
54	${currentUser.displayName} 1h ${Math.floor(20 + (Math.random() * 75))} ${Math.floor((Math.random() - 0.5) * 100)}
55	${prefix}`.trim();
56	}
57
58	const getCommentReplyMessageFormatted = (comment: DbComment, parentComments: DbComment[], parentPost: DbPost, revisionsMap: Map<string, DbRevision>, authorsMap: Map<string, DbUser>, prefix: string, currentUser: DbUser) => {	18✔
NEW 59	const parentComment = parentComments[parentComments.length - 1];	×
NEW 60	return `${getPostBodyFormatted(parentPost, revisionsMap, authorsMap)}	×
61	---
NEW 62	${parentComments.map((comment) => getCommentBodyFormatted(comment, revisionsMap, authorsMap)).join("\n---\n")}	×
63	---
64	${getCommentBodyFormatted(comment, revisionsMap, authorsMap)}
65	---
66	${currentUser.displayName} 1h ${Math.floor(20 + (Math.random() * 75))} ${Math.floor((Math.random() - 0.5) * 100)}
67	${prefix}`.trim();
68	}
69
70	async function constructMessageHistory(
71	prefix: string,
72	commentIds: string[],
73	postIds: string[],
74	currentUser: any,
75	replyingCommentId?: string,
76	postId?: string
77	): Promise<PromptCachingBetaMessageParam[]> {
NEW 78	const messages: PromptCachingBetaMessageParam[] = [];	×
79
80	// Make the fetches parallel to save time
NEW 81	const [revisions, posts, comments] = await Promise.all([	×
82	Revisions.find({ documentId: { $in: [...postIds, ...commentIds] }, fieldName: "contents", version: "1.0.0" }).fetch(),
83	Posts.find({ _id: { $in: postIds } }).fetch(),
84	Comments.find({ _id: { $in: commentIds } }).fetch(),
85	]);
86
NEW 87	const authors = await Users.find({ _id: { $in: [...posts.map((post) => post.userId), ...comments.map((comment) => comment.userId)] } }).fetch();	×
88
NEW 89	const authorsMap = new Map(authors.map((author) => [author._id, author]));	×
NEW 90	const revisionsMap = new Map(revisions.map((revision) => [revision.documentId!, revision]));	×
91
92	// eslint-disable-next-line no-console
NEW 93	console.log(`Converting ${posts.length} posts and ${comments.length} comments to messages`);	×
94	// eslint-disable-next-line no-console
NEW 95	console.time("constructPostMessageHistory");	×
96
97	// Add fetched posts and comments to message history
NEW 98	for (const post of posts) {	×
NEW 99	messages.push({	×
100	role: "user",
101	content: [{ type: "text", text: `<cmd>cat lw/${post._id}.txt</cmd>` }],
102	});
103
NEW 104	messages.push({	×
105	role: "assistant",
106	content: [
107	{
108	type: "text",
109	text: getPostBodyFormatted(post, revisionsMap, authorsMap),
110	},
111	],
112	});
113	}
114
115	// eslint-disable-next-line no-console
NEW 116	console.timeEnd("constructPostMessageHistory");	×
117
NEW 118	for (const comment of comments) {	×
NEW 119	messages.push({	×
120	role: "user",
121	content: [{ type: "text", text: `<cmd>cat lw/${comment._id}.txt</cmd>` }],
122	});
123
NEW 124	messages.push({	×
125	role: "assistant",
126	content: [{ type: "text", text: getCommentBodyFormatted(comment, revisionsMap, authorsMap) }],
127	});
128	}
129
130	// Add final user message with prefix
NEW 131	messages.push({	×
132	role: "user",
133	content: [{ type: "text", text: `<cmd>cat lw/hsqKp56whpPEQns3Z.txt</cmd>`, cache_control: {type: "ephemeral"}}]
134	});
135
NEW 136	if (replyingCommentId) {	×
137	// Fetch the comment we're replying to
NEW 138	const replyingToComment = await Comments.findOne({ _id: replyingCommentId });	×
NEW 139	if (!replyingToComment) {	×
NEW 140	throw new Error("Comment not found");	×
141	}
NEW 142	const parentComments = await getParentComments(replyingToComment);	×
NEW 143	const parentPost = await Posts.findOne({ _id: replyingToComment.postId });	×
NEW 144	const parentPostRevision = await Revisions.findOne({ documentId: parentPost?._id, fieldName: "contents", version: "1.0.0" });	×
145
NEW 146	if (!parentPost) {	×
NEW 147	throw new Error(`Parent post or revision not found ${replyingToComment.postId}`);	×
148	}
149
NEW 150	const authors = await Users.find({ _id: { $in: [parentPost.userId, ...parentComments.map((comment) => comment.userId), replyingToComment.userId] } }).fetch();	×
151
NEW 152	authors.forEach((author) => authorsMap.set(author._id, author));	×
NEW 153	revisionsMap.set(parentPost._id!, parentPostRevision!);	×
NEW 154	if (!parentPost) {	×
NEW 155	throw new Error("Post not found");	×
156	}
157
NEW 158	const message = getCommentReplyMessageFormatted(replyingToComment, parentComments, parentPost, revisionsMap, authorsMap, prefix, currentUser)	×
159
NEW 160	messages.push({	×
161	role: "assistant",
162	content: [
163	{
164	type: "text",
165	text: message
166	},
167	],
168	});
NEW 169	}	×
170	else if (postId) {
NEW 171	const post = await Posts.findOne({ _id: postId });	×
NEW 172	if (!post) {	×
NEW 173	throw new Error("Post not found");	×
174	}
NEW 175	const postRevision = await Revisions.findOne({ documentId: post._id, fieldName: "contents", version: "1.0.0" });	×
NEW 176	if (!postRevision) {	×
NEW 177	throw new Error("Post revision not found");	×
178	}
179
NEW 180	const authors = await Users.find({ _id: post.userId }).fetch();	×
NEW 181	authors.forEach((author) => authorsMap.set(author._id, author));	×
NEW 182	revisionsMap.set(post._id!, postRevision);	×
183
NEW 184	const message = getPostReplyMessageFormatted(post, revisionsMap, authorsMap, currentUser, prefix);	×
185
NEW 186	messages.push({	×
187	role: "assistant",
188	content: [
189	{
190	type: "text",
191	text: getPostReplyMessageFormatted(post, revisionsMap, authorsMap, currentUser, prefix)
192	},
193	],
194	});
195	}
196	else {
NEW 197	messages.push({	×
198	role: "assistant",
199	content: [
200	{
201	type: "text",
202	text: `${prefix}`
203	}
204	]
205	});
206	}
207
NEW 208	return messages;	×
209	}
210
211	async function construct405bPrompt(
212	prefix: string,
213	commentIds: string[],
214	postIds: string[],
215	currentUser: any,
216	replyingCommentId?: string,
217	postId?: string
218	): Promise<string> {
219
220	// Make the fetches parallel to save time
NEW 221	const [revisions, posts, comments] = await Promise.all([	×
222	Revisions.find({ documentId: { $in: [...postIds, ...commentIds] }, fieldName: "contents", version: "1.0.0" }).fetch(),
223	Posts.find({ _id: { $in: postIds } }).fetch(),
224	Comments.find({ _id: { $in: commentIds } }).fetch(),
225	]);
226
NEW 227	const authors = await Users.find({ _id: { $in: [...posts.map((post) => post.userId), ...comments.map((comment) => comment.userId)] } }).fetch();	×
228
NEW 229	const authorsMap = new Map(authors.map((author) => [author._id, author]));	×
NEW 230	const revisionsMap = new Map(revisions.map((revision) => [revision.documentId!, revision]));	×
231
232	// eslint-disable-next-line no-console
NEW 233	console.log(`Converting ${posts.length} posts and ${comments.length} comments to messages`);	×
234
NEW 235	let finalSection = ''	×
236
NEW 237	if (replyingCommentId) {	×
238	// Fetch the comment we're replying to
NEW 239	const replyingToComment = await	×
240	Comments.findOne({ _id: replyingCommentId });
NEW 241	if (!replyingToComment) {	×
NEW 242	throw new Error("Comment not found");	×
243	}
NEW 244	const parentComments = await getParentComments(replyingToComment);	×
NEW 245	const parentPost = await Posts.findOne({ _id: replyingToComment.postId });	×
NEW 246	const parentPostRevision = await Revisions.findOne({ documentId: parentPost?._id, fieldName: "contents", version: "1.0.0" });	×
247
NEW 248	if (!parentPost) {	×
NEW 249	throw new Error(`Parent post or revision not found ${replyingToComment.postId}`);	×
250	}
251
NEW 252	const authors = await Users.find({ _id: { $in: [parentPost.userId, ...parentComments.map((comment) => comment.userId), replyingToComment.userId] } }).fetch();	×
253
NEW 254	authors.forEach((author) => authorsMap.set(author._id, author));	×
NEW 255	revisionsMap.set(parentPost._id!, parentPostRevision!);	×
256
NEW 257	finalSection = getCommentReplyMessageFormatted(replyingToComment, parentComments, parentPost, revisionsMap, authorsMap, prefix, currentUser)	×
NEW 258	} else if (postId) {	×
NEW 259	const post = await Posts.findOne({ _id: postId });	×
NEW 260	if (!post) {	×
NEW 261	throw new Error("Post not found");	×
262	}
NEW 263	const postRevision = await Revisions.findOne({ documentId: post._id, fieldName: "contents", version: "1.0.0" });	×
NEW 264	if (!postRevision) {	×
NEW 265	throw new Error("Post revision not found");	×
266	}
267
NEW 268	const authors = await Users.find({ _id: post.userId }).fetch();	×
NEW 269	authors.forEach((author) => authorsMap.set(author._id, author));	×
NEW 270	revisionsMap.set(post._id!, postRevision);	×
271
NEW 272	finalSection = getPostReplyMessageFormatted(post, revisionsMap, authorsMap, currentUser, prefix)	×
273	} else {
NEW 274	finalSection = `${prefix}`	×
275	}
276
277
NEW 278	return `	×
NEW 279	${posts.map((post) => getPostBodyFormatted(post, revisionsMap, authorsMap)).join("\n---\n")}	×
280	---
NEW 281	${comments.map((comment) => getCommentBodyFormatted(comment, revisionsMap, authorsMap)).join("\n---\n")}	×
282	---
283	${finalSection}`.trim();
284	}
285
286
287	export function addAutocompleteEndpoint(app: Express) {
NEW 288	app.use("/api/autocomplete", express.json());	×
NEW 289	app.post("/api/autocomplete", async (req, res) => {	×
NEW 290	const currentUser = getUserFromReq(req);	×
NEW 291	if (!currentUser) {	×
NEW 292	return res.status(401).json({ error: "Unauthorized" });	×
293	}
294
NEW 295	try {	×
NEW 296	if (!userIsAdmin(currentUser)) {	×
NEW 297	throw new Error("Claude Completion is for admins only");	×
298	}
299
NEW 300	const client = getAnthropicPromptCachingClientOrThrow();	×
301
NEW 302	const { prefix = '', commentIds, postIds, replyingCommentId, postId } = req.body;	×
303
304	// Set headers for streaming response
NEW 305	res.writeHead(200, {	×
306	"Content-Type": "text/event-stream",
307	"Cache-Control": "no-cache",
308	Connection: "keep-alive",
309	});
310
NEW 311	const loadingMessagesStream = client.messages.stream({	×
312	model: "claude-3-5-sonnet-20240620",
313	max_tokens: 1000,
314	system: "The assistant is in CLI simulation mode, and responds to the user's CLI commands only with the output of the command.",
315	messages: await constructMessageHistory(
316	prefix,
317	commentIds,
318	postIds,
319	currentUser,
320	replyingCommentId,
321	postId
322	),
323	});
324
NEW 325	loadingMessagesStream.on("text", (delta, snapshot) => {	×
NEW 326	res.write(	×
327	`data: ${JSON.stringify({ type: "text", content: delta })}\n\n`,
328	);
329	});
330
NEW 331	loadingMessagesStream.on("end", () => {	×
NEW 332	res.write(`data: ${JSON.stringify({ type: "end" })}\n\n`);	×
NEW 333	res.end();	×
334	});
335
NEW 336	loadingMessagesStream.on("error", (error) => {	×
337	// eslint-disable-next-line no-console
NEW 338	console.error("Stream error:", JSON.stringify(error));	×
NEW 339	res.write(	×
340	`data: ${JSON.stringify({ type: "error", message: "An error occurred" })}\n\n`,
341	);
NEW 342	res.end();	×
343	});
344	} catch (error) {
345	// eslint-disable-next-line no-console
NEW 346	console.error("Autocomplete error:", error);	×
NEW 347	res.status(500).json({ error: "An error occurred during autocomplete" });	×
348	}
349	});
NEW 350	app.use("/api/autocomplete405b", express.json());	×
NEW 351	app.post("/api/autocomplete405b", async (req, res) => {	×
NEW 352	const currentUser = getUserFromReq(req);	×
NEW 353	if (!currentUser) {	×
NEW 354	return res.status(401).json({ error: "Unauthorized" });	×
355	}
356
NEW 357	const { prefix = '', commentIds, postIds, replyingCommentId, postId } = req.body;	×
358	// Set headers for streaming response
NEW 359	res.writeHead(200, {	×
360	"Content-Type": "text/event-stream",
361	"Cache-Control": "no-cache",
362	Connection: "keep-alive",
363	});
364
NEW 365	const url = 'https://api.hyperbolic.xyz/v1/completions';	×
366
NEW 367	const response = await fetch(url, {	×
368	method: 'POST',
369	headers: {
370	'Content-Type': 'application/json',
371	Authorization: `Bearer ${hyperbolicApiKey.get()}`,
372	},
373	body: JSON.stringify({
374	model: 'meta-llama/Meta-Llama-3.1-405B',
375	prompt: await construct405bPrompt(
376	prefix,
377	commentIds,
378	postIds,
379	currentUser,
380	replyingCommentId,
381	postId
382	),
383	max_tokens: 256,
384	temperature: 0.7,
385	top_p: 0.9,
386	stream: true
387	}),
388	});
389
NEW 390	if (!response.ok \|\| !response.body) {	×
NEW 391	res.writeHead(response.status, response.statusText);	×
NEW 392	res.end(`Error from API: ${response.statusText}`);	×
NEW 393	return;	×
394	}
395
NEW 396	try {	×
NEW 397	if (!userIsAdmin(currentUser)) {	×
NEW 398	throw new Error("Claude Completion is for admins only");	×
399	}
NEW 400	const reader = response.body.getReader();	×
NEW 401	const readableNodeStream = new Readable({	×
402	async read() {
NEW 403	const { done, value } = await reader.read();	×
NEW 404	if (done) {	×
NEW 405	this.push(null);	×
406	} else {
NEW 407	this.push(Buffer.from(value));	×
408	}
409	}
410	});
411
NEW 412	await pipeline(readableNodeStream, res);	×
413	} catch (error) {
414	// eslint-disable-next-line no-console
NEW 415	console.error("Autocomplete error:", error);	×
NEW 416	res.status(500).json({ error: "An error occurred during autocomplete" });	×
417	}
418	});
419	}

ForumMagnum / ForumMagnum / 10589248057

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous