10605510383

Committed 28 Aug 2024 10:43PM UTC coverage: 31.945% (-0.07%) from 32.018%

Build # 10605510383

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #9633 from ForumMagnum/autocomplete-new

LLM Autocomplete for admins

Run Details

5023 of 40983 branches covered (12.26%)

Branch coverage included in aggregate %.

47 of 241 new or added lines in 8 files covered. (19.5%)

1 existing line in 1 file now uncovered.

29286 of 66418 relevant lines covered (44.09%)

188.59 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

8.24

/packages/lesswrong/server/autocompleteEndpoint.ts

import type { Express } from "express";
import { getContextFromReqAndRes, getUserFromReq } from "./vulcan-lib/apollo-server/context";
import { userIsAdmin } from "@/lib/vulcan-users/permissions";
import { getAnthropicPromptCachingClientOrThrow } from "./languageModels/anthropicClient";
import express from "express";
import { PromptCachingBetaMessageParam } from "@anthropic-ai/sdk/resources/beta/prompt-caching/messages";
import { formatRelative } from "@/lib/utils/timeFormat";
import { Readable } from 'stream';
import { pipeline } from 'stream/promises'
import { hyperbolicApiKey } from "@/lib/instanceSettings";
import { runFragmentQuery } from "./vulcan-lib/query";



const getParentComments = (comment: CommentsForAutocompleteWithParents) => {
  const parentComments: CommentsForAutocompleteWithParents[] = [];
  let currentComment: CommentsForAutocompleteWithParents | null = comment;
  while (currentComment.parentComment && currentComment.parentComment._id !== currentComment._id) {
    currentComment = currentComment.parentComment as CommentsForAutocompleteWithParents;
    if (currentComment) {
      parentComments.push(currentComment);
    } else {
      break;
    }
  }
  return parentComments.reverse();
}

const getPostBodyFormatted = (post: PostsForAutocomplete) => {
  return `${post.title}
by ${post.user?.displayName}
${post.baseScore}
${post.contents?.markdown}`.trim();
}

const getCommentBodyFormatted = (comment: CommentsForAutocomplete) => {
  const dateString = formatRelative(new Date(comment.createdAt), new Date(), false)
  return `Comment on ${comment.post?.title}
${comment.user?.displayName} ${dateString} ${comment.baseScore} ${comment.extendedScore?.agreement}
${comment.contents?.markdown}`.trim();
}

const getPostReplyMessageFormatted = (post: PostsForAutocomplete, currentUser: DbUser, prefix: string) => {
  return `${getPostBodyFormatted(post)}
---
Comment on ${post.title}
${currentUser.displayName} 1h ${Math.floor(20 + (Math.random() * 75))} ${Math.floor((Math.random() - 0.5) * 100)}
${prefix}`.trim();
}

const getCommentReplyMessageFormatted = (comment: CommentsForAutocompleteWithParents, prefix: string, currentUser: DbUser) => {
  const parentComments = getParentComments(comment);
  return `${getPostBodyFormatted(comment.post!)}
---
${parentComments.map((comment) => getCommentBodyFormatted(comment)).join("\n---\n")}
---
${getCommentBodyFormatted(comment)}
---
Comment on ${comment.post?.title}
${currentUser.displayName} 1h ${Math.floor(20 + (Math.random() * 75))} ${Math.floor((Math.random() - 0.5) * 100)}
${prefix}`.trim();
}

async function constructMessageHistory(
  prefix: string,
  commentIds: string[],
  postIds: string[],
  currentUser: any,
  context: ResolverContext,
  replyingCommentId?: string,
  postId?: string
): Promise<PromptCachingBetaMessageParam[]> {
  const messages: PromptCachingBetaMessageParam[] = [];

  // Make the fetches parallel to save time
  const [posts, comments] = await Promise.all([
    runFragmentQuery({
      collectionName: "Posts",
      fragmentName: "PostsForAutocomplete",
      terms: { postIds },
      context,
    }),
    runFragmentQuery({
      collectionName: "Comments",
      fragmentName: "CommentsForAutocomplete",
      terms: { commentIds },
      context,
    }),
  ]);

  // eslint-disable-next-line no-console
  console.log(`Converting ${posts.length} posts and ${comments.length} comments to messages`);

  // Add fetched posts and comments to message history
  for (const post of posts) {
    messages.push({
      role: "user",
      content: [{ type: "text", text: `<cmd>cat lw/${post._id}.txt</cmd>` }],
    });

    messages.push({
      role: "assistant",
      content: [
        {
          type: "text",
          text: getPostBodyFormatted(post),
        },
      ],
    });
  }

  for (const comment of comments) {
    messages.push({
      role: "user",
      content: [{ type: "text", text: `<cmd>cat lw/${comment._id}.txt</cmd>` }],
    });

    messages.push({
      role: "assistant",
      content: [{ type: "text", text: getCommentBodyFormatted(comment) }],
    });
  }

  // Add final user message with prefix
  messages.push({
    role: "user",
    content: [{ type: "text", text: `<cmd>cat lw/hsqKp56whpPEQns3Z.txt</cmd>`, cache_control: {type: "ephemeral"}}]
  });

  if (replyingCommentId) {
    // Fetch the comment we're replying to
    const replyingToCommentResponse = await runFragmentQuery({
      collectionName: "Comments",
      fragmentName: "CommentsForAutocompleteWithParents",
      terms: { commentIds: [replyingCommentId] },
      context,
    })
    if (!replyingToCommentResponse || replyingToCommentResponse.length === 0) {
      throw new Error("Comment not found");
    }
    const replyingToComment = replyingToCommentResponse[0];
    const message = getCommentReplyMessageFormatted(replyingToComment, prefix, currentUser)

    messages.push({
      role: "assistant",
      content: [
        {
          type: "text",
          text: message
        },
      ],
    });
  }
  else if (postId) {
    const postResponse = await runFragmentQuery({
      collectionName: "Posts",
      fragmentName: "PostsForAutocomplete",
      terms: { postIds: [postId] },
      context,
    })
    if (!postResponse || postResponse.length === 0) {
      throw new Error("Post not found");
    }
    const post = postResponse[0];

    messages.push({
      role: "assistant",
      content: [
        {
          type: "text",
          text: getPostReplyMessageFormatted(post, currentUser, prefix)
        },
      ],
    });
  }
  else {
    messages.push({
      role: "assistant",
      content: [
        {
          type: "text",
          text: `${prefix}`
        }
      ]
    });
  }

  return messages;
}

async function construct405bPrompt(
  prefix: string,
  commentIds: string[],
  postIds: string[],
  currentUser: any,
  context: ResolverContext,
  replyingCommentId?: string,
  postId?: string
): Promise<string> {

  // Make the fetches parallel to save time
  const [posts, comments] = await Promise.all([
    runFragmentQuery({
      collectionName: "Posts",
      fragmentName: "PostsForAutocomplete",
      terms: { postIds },
      context,
    }),
    runFragmentQuery({
      collectionName: "Comments",
      fragmentName: "CommentsForAutocomplete",
      terms: { commentIds },
      context,
    }),
  ]);


  // eslint-disable-next-line no-console
  console.log(`Converting ${posts.length} posts and ${comments.length} comments to messages`, postId, replyingCommentId);

  let finalSection = ''

  if (replyingCommentId) {
    // Fetch the comment we're replying to
    const replyingToCommentResponse = await runFragmentQuery({
      collectionName: "Comments",
      fragmentName: "CommentsForAutocompleteWithParents",
      terms: { commentIds: [replyingCommentId] },
      context,
    })
    if (!replyingToCommentResponse || replyingToCommentResponse.length === 0) {
      throw new Error("Comment not found");
    }
    const replyingToComment = replyingToCommentResponse[0];

    finalSection = getCommentReplyMessageFormatted(replyingToComment, prefix, currentUser)    
  } else if (postId) {
    const postResponse = await runFragmentQuery({
      collectionName: "Posts",
      fragmentName: "PostsForAutocomplete",
      terms: { postIds: [postId] },
      context,
    })
    if (!postResponse || postResponse.length === 0) {
      throw new Error("Post not found");
    }
    const post = postResponse[0];

    finalSection = getPostReplyMessageFormatted(post, currentUser, prefix)
  } else {
    finalSection = `${prefix}`
  }


  return `
${posts.map((post) => getPostBodyFormatted(post)).join("\n---\n")}
====
${comments.map((comment) => getCommentBodyFormatted(comment)).join("\n---\n")}
====
${finalSection}`.trim();
}


export function addAutocompleteEndpoint(app: Express) {
  app.use("/api/autocomplete", express.json());
  app.post("/api/autocomplete", async (req, res) => {
    const context = await getContextFromReqAndRes(req, res);
    const currentUser = context.currentUser
    if (!currentUser) {
      return res.status(401).json({ error: "Unauthorized" });
    }

    try {
      if (!userIsAdmin(currentUser)) {
        throw new Error("Claude Completion is for admins only");
      }

      const client = getAnthropicPromptCachingClientOrThrow();

      const { prefix = '', commentIds, postIds, replyingCommentId, postId } = req.body;

      // Set headers for streaming response
      res.writeHead(200, {
        "Content-Type": "text/event-stream",
        "Cache-Control": "no-cache",
        Connection: "keep-alive",
      });

      const loadingMessagesStream = client.messages.stream({
        model: "claude-3-5-sonnet-20240620",
        max_tokens: 1000,
        system: "The assistant is in CLI simulation mode, and responds to the user's CLI commands only with the output of the command.",
        messages: await constructMessageHistory(
          prefix,
          commentIds,
          postIds,
          currentUser,
          context,
          replyingCommentId,
          postId
        ),
      });

      loadingMessagesStream.on("text", (delta, snapshot) => {
        res.write(
          `data: ${JSON.stringify({ type: "text", content: delta })}\n\n`,
        );
      });

      loadingMessagesStream.on("end", () => {
        res.write(`data: ${JSON.stringify({ type: "end" })}\n\n`);
        res.end();
      });

      loadingMessagesStream.on("error", (error) => {
        // eslint-disable-next-line no-console
        console.error("Stream error:", JSON.stringify(error));
        res.write(
          `data: ${JSON.stringify({ type: "error", message: "An error occurred" })}\n\n`,
        );
        res.end();
      });
    } catch (error) {
      // eslint-disable-next-line no-console
      console.error("Autocomplete error:", error);
      res.status(500).json({ error: "An error occurred during autocomplete" });
    }
  });
  app.use("/api/autocomplete405b", express.json());
  app.post("/api/autocomplete405b", async (req, res) => {
    const context = await getContextFromReqAndRes(req, res);
    const currentUser = context.currentUser
    if (!currentUser) {
      return res.status(401).json({ error: "Unauthorized" });
    }
    
    try {
      if (!userIsAdmin(currentUser)) {
        throw new Error("Claude Completion is for admins only");
      }
  
      const { prefix = '', commentIds, postIds, replyingCommentId, postId } = req.body;
      // Set headers for streaming response
      res.writeHead(200, {
        "Content-Type": "text/event-stream",
        "Cache-Control": "no-cache",
        Connection: "keep-alive",
      });
  
      const url = 'https://api.hyperbolic.xyz/v1/completions';
  
      const response = await fetch(url, {
        method: 'POST',
        headers: {
          'Content-Type': 'application/json',
          Authorization: `Bearer ${hyperbolicApiKey.get()}`,
        },
        body: JSON.stringify({
          model: 'meta-llama/Meta-Llama-3.1-405B',
          prompt: await construct405bPrompt(
            prefix,
            commentIds,
            postIds,
            currentUser,
            context,
            replyingCommentId,
            postId
          ),
          max_tokens: 256,
          temperature: 0.7,
          top_p: 0.9,
          stream: true
        }),
      });
  
      if (!response.ok || !response.body) {
        res.writeHead(response.status, response.statusText);
        res.end(`Error from API: ${response.statusText}`);
        return;
      }
      const reader = response.body.getReader();
      const readableNodeStream = new Readable({
        async read() {
          const { done, value } = await reader.read();
          if (done) {
            this.push(null);
          } else {
            this.push(Buffer.from(value));
          }
        }
      });

      await pipeline(readableNodeStream, res);
    } catch (error) {
      // eslint-disable-next-line no-console
      console.error("Autocomplete error:", error);
      res.status(500).json({ error: "An error occurred during autocomplete" });
    }
  });
}

1	import type { Express } from "express";	54✔
2	import { getContextFromReqAndRes, getUserFromReq } from "./vulcan-lib/apollo-server/context";	18✔
3	import { userIsAdmin } from "@/lib/vulcan-users/permissions";	18✔
4	import { getAnthropicPromptCachingClientOrThrow } from "./languageModels/anthropicClient";	18✔
5	import express from "express";	18✔
6	import { PromptCachingBetaMessageParam } from "@anthropic-ai/sdk/resources/beta/prompt-caching/messages";
7	import { formatRelative } from "@/lib/utils/timeFormat";	18✔
8	import { Readable } from 'stream';	18✔
9	import { pipeline } from 'stream/promises'	18✔
10	import { hyperbolicApiKey } from "@/lib/instanceSettings";	18✔
11	import { runFragmentQuery } from "./vulcan-lib/query";	18✔
12
13
14
15	const getParentComments = (comment: CommentsForAutocompleteWithParents) => {	18✔
NEW 16	const parentComments: CommentsForAutocompleteWithParents[] = [];	×
NEW 17	let currentComment: CommentsForAutocompleteWithParents \| null = comment;	×
NEW 18	while (currentComment.parentComment && currentComment.parentComment._id !== currentComment._id) {	×
NEW 19	currentComment = currentComment.parentComment as CommentsForAutocompleteWithParents;	×
NEW 20	if (currentComment) {	×
NEW 21	parentComments.push(currentComment);	×
22	} else {
NEW 23	break;	×
24	}
25	}
NEW 26	return parentComments.reverse();	×
27	}
28
29	const getPostBodyFormatted = (post: PostsForAutocomplete) => {	18✔
NEW 30	return `${post.title}	×
31	by ${post.user?.displayName}	×
32	${post.baseScore}
33	${post.contents?.markdown}`.trim();	×
34	}
35
36	const getCommentBodyFormatted = (comment: CommentsForAutocomplete) => {	18✔
NEW 37	const dateString = formatRelative(new Date(comment.createdAt), new Date(), false)	×
NEW 38	return `Comment on ${comment.post?.title}	×
39	${comment.user?.displayName} ${dateString} ${comment.baseScore} ${comment.extendedScore?.agreement}	×
40	${comment.contents?.markdown}`.trim();	×
41	}
42
43	const getPostReplyMessageFormatted = (post: PostsForAutocomplete, currentUser: DbUser, prefix: string) => {	18✔
NEW 44	return `${getPostBodyFormatted(post)}	×
45	---
46	Comment on ${post.title}
47	${currentUser.displayName} 1h ${Math.floor(20 + (Math.random() * 75))} ${Math.floor((Math.random() - 0.5) * 100)}
48	${prefix}`.trim();
49	}
50
51	const getCommentReplyMessageFormatted = (comment: CommentsForAutocompleteWithParents, prefix: string, currentUser: DbUser) => {	18✔
NEW 52	const parentComments = getParentComments(comment);	×
NEW 53	return `${getPostBodyFormatted(comment.post!)}	×
54	---
NEW 55	${parentComments.map((comment) => getCommentBodyFormatted(comment)).join("\n---\n")}	×
56	---
57	${getCommentBodyFormatted(comment)}
58	---
59	Comment on ${comment.post?.title}	×
60	${currentUser.displayName} 1h ${Math.floor(20 + (Math.random() * 75))} ${Math.floor((Math.random() - 0.5) * 100)}
61	${prefix}`.trim();
62	}
63
64	async function constructMessageHistory(
65	prefix: string,
66	commentIds: string[],
67	postIds: string[],
68	currentUser: any,
69	context: ResolverContext,
70	replyingCommentId?: string,
71	postId?: string
72	): Promise<PromptCachingBetaMessageParam[]> {
NEW 73	const messages: PromptCachingBetaMessageParam[] = [];	×
74
75	// Make the fetches parallel to save time
NEW 76	const [posts, comments] = await Promise.all([	×
77	runFragmentQuery({
78	collectionName: "Posts",
79	fragmentName: "PostsForAutocomplete",
80	terms: { postIds },
81	context,
82	}),
83	runFragmentQuery({
84	collectionName: "Comments",
85	fragmentName: "CommentsForAutocomplete",
86	terms: { commentIds },
87	context,
88	}),
89	]);
90
91	// eslint-disable-next-line no-console
NEW 92	console.log(`Converting ${posts.length} posts and ${comments.length} comments to messages`);	×
93
94	// Add fetched posts and comments to message history
NEW 95	for (const post of posts) {	×
NEW 96	messages.push({	×
97	role: "user",
98	content: [{ type: "text", text: `<cmd>cat lw/${post._id}.txt</cmd>` }],
99	});
100
NEW 101	messages.push({	×
102	role: "assistant",
103	content: [
104	{
105	type: "text",
106	text: getPostBodyFormatted(post),
107	},
108	],
109	});
110	}
111
NEW 112	for (const comment of comments) {	×
NEW 113	messages.push({	×
114	role: "user",
115	content: [{ type: "text", text: `<cmd>cat lw/${comment._id}.txt</cmd>` }],
116	});
117
NEW 118	messages.push({	×
119	role: "assistant",
120	content: [{ type: "text", text: getCommentBodyFormatted(comment) }],
121	});
122	}
123
124	// Add final user message with prefix
NEW 125	messages.push({	×
126	role: "user",
127	content: [{ type: "text", text: `<cmd>cat lw/hsqKp56whpPEQns3Z.txt</cmd>`, cache_control: {type: "ephemeral"}}]
128	});
129
NEW 130	if (replyingCommentId) {	×
131	// Fetch the comment we're replying to
NEW 132	const replyingToCommentResponse = await runFragmentQuery({	×
133	collectionName: "Comments",
134	fragmentName: "CommentsForAutocompleteWithParents",
135	terms: { commentIds: [replyingCommentId] },
136	context,
137	})
NEW 138	if (!replyingToCommentResponse \|\| replyingToCommentResponse.length === 0) {	×
NEW 139	throw new Error("Comment not found");	×
140	}
NEW 141	const replyingToComment = replyingToCommentResponse[0];	×
NEW 142	const message = getCommentReplyMessageFormatted(replyingToComment, prefix, currentUser)	×
143
NEW 144	messages.push({	×
145	role: "assistant",
146	content: [
147	{
148	type: "text",
149	text: message
150	},
151	],
152	});
NEW 153	}	×
154	else if (postId) {
NEW 155	const postResponse = await runFragmentQuery({	×
156	collectionName: "Posts",
157	fragmentName: "PostsForAutocomplete",
158	terms: { postIds: [postId] },
159	context,
160	})
NEW 161	if (!postResponse \|\| postResponse.length === 0) {	×
NEW 162	throw new Error("Post not found");	×
163	}
NEW 164	const post = postResponse[0];	×
165
NEW 166	messages.push({	×
167	role: "assistant",
168	content: [
169	{
170	type: "text",
171	text: getPostReplyMessageFormatted(post, currentUser, prefix)
172	},
173	],
174	});
175	}
176	else {
NEW 177	messages.push({	×
178	role: "assistant",
179	content: [
180	{
181	type: "text",
182	text: `${prefix}`
183	}
184	]
185	});
186	}
187
NEW 188	return messages;	×
189	}
190
191	async function construct405bPrompt(
192	prefix: string,
193	commentIds: string[],
194	postIds: string[],
195	currentUser: any,
196	context: ResolverContext,
197	replyingCommentId?: string,
198	postId?: string
199	): Promise<string> {
200
201	// Make the fetches parallel to save time
NEW 202	const [posts, comments] = await Promise.all([	×
203	runFragmentQuery({
204	collectionName: "Posts",
205	fragmentName: "PostsForAutocomplete",
206	terms: { postIds },
207	context,
208	}),
209	runFragmentQuery({
210	collectionName: "Comments",
211	fragmentName: "CommentsForAutocomplete",
212	terms: { commentIds },
213	context,
214	}),
215	]);
216
217
218	// eslint-disable-next-line no-console
NEW 219	console.log(`Converting ${posts.length} posts and ${comments.length} comments to messages`, postId, replyingCommentId);	×
220
NEW 221	let finalSection = ''	×
222
NEW 223	if (replyingCommentId) {	×
224	// Fetch the comment we're replying to
NEW 225	const replyingToCommentResponse = await runFragmentQuery({	×
226	collectionName: "Comments",
227	fragmentName: "CommentsForAutocompleteWithParents",
228	terms: { commentIds: [replyingCommentId] },
229	context,
230	})
NEW 231	if (!replyingToCommentResponse \|\| replyingToCommentResponse.length === 0) {	×
NEW 232	throw new Error("Comment not found");	×
233	}
NEW 234	const replyingToComment = replyingToCommentResponse[0];	×
235
NEW 236	finalSection = getCommentReplyMessageFormatted(replyingToComment, prefix, currentUser)	×
NEW 237	} else if (postId) {	×
NEW 238	const postResponse = await runFragmentQuery({	×
239	collectionName: "Posts",
240	fragmentName: "PostsForAutocomplete",
241	terms: { postIds: [postId] },
242	context,
243	})
NEW 244	if (!postResponse \|\| postResponse.length === 0) {	×
NEW 245	throw new Error("Post not found");	×
246	}
NEW 247	const post = postResponse[0];	×
248
NEW 249	finalSection = getPostReplyMessageFormatted(post, currentUser, prefix)	×
250	} else {
NEW 251	finalSection = `${prefix}`	×
252	}
253
254
NEW 255	return `	×
NEW 256	${posts.map((post) => getPostBodyFormatted(post)).join("\n---\n")}	×
257	====
NEW 258	${comments.map((comment) => getCommentBodyFormatted(comment)).join("\n---\n")}	×
259	====
260	${finalSection}`.trim();
261	}
262
263
264	export function addAutocompleteEndpoint(app: Express) {
NEW 265	app.use("/api/autocomplete", express.json());	×
NEW 266	app.post("/api/autocomplete", async (req, res) => {	×
NEW 267	const context = await getContextFromReqAndRes(req, res);	×
NEW 268	const currentUser = context.currentUser	×
NEW 269	if (!currentUser) {	×
NEW 270	return res.status(401).json({ error: "Unauthorized" });	×
271	}
272
NEW 273	try {	×
NEW 274	if (!userIsAdmin(currentUser)) {	×
NEW 275	throw new Error("Claude Completion is for admins only");	×
276	}
277
NEW 278	const client = getAnthropicPromptCachingClientOrThrow();	×
279
NEW 280	const { prefix = '', commentIds, postIds, replyingCommentId, postId } = req.body;	×
281
282	// Set headers for streaming response
NEW 283	res.writeHead(200, {	×
284	"Content-Type": "text/event-stream",
285	"Cache-Control": "no-cache",
286	Connection: "keep-alive",
287	});
288
NEW 289	const loadingMessagesStream = client.messages.stream({	×
290	model: "claude-3-5-sonnet-20240620",
291	max_tokens: 1000,
292	system: "The assistant is in CLI simulation mode, and responds to the user's CLI commands only with the output of the command.",
293	messages: await constructMessageHistory(
294	prefix,
295	commentIds,
296	postIds,
297	currentUser,
298	context,
299	replyingCommentId,
300	postId
301	),
302	});
303
NEW 304	loadingMessagesStream.on("text", (delta, snapshot) => {	×
NEW 305	res.write(	×
306	`data: ${JSON.stringify({ type: "text", content: delta })}\n\n`,
307	);
308	});
309
NEW 310	loadingMessagesStream.on("end", () => {	×
NEW 311	res.write(`data: ${JSON.stringify({ type: "end" })}\n\n`);	×
NEW 312	res.end();	×
313	});
314
NEW 315	loadingMessagesStream.on("error", (error) => {	×
316	// eslint-disable-next-line no-console
NEW 317	console.error("Stream error:", JSON.stringify(error));	×
NEW 318	res.write(	×
319	`data: ${JSON.stringify({ type: "error", message: "An error occurred" })}\n\n`,
320	);
NEW 321	res.end();	×
322	});
323	} catch (error) {
324	// eslint-disable-next-line no-console
NEW 325	console.error("Autocomplete error:", error);	×
NEW 326	res.status(500).json({ error: "An error occurred during autocomplete" });	×
327	}
328	});
NEW 329	app.use("/api/autocomplete405b", express.json());	×
NEW 330	app.post("/api/autocomplete405b", async (req, res) => {	×
NEW 331	const context = await getContextFromReqAndRes(req, res);	×
NEW 332	const currentUser = context.currentUser	×
NEW 333	if (!currentUser) {	×
NEW 334	return res.status(401).json({ error: "Unauthorized" });	×
335	}
336
NEW 337	try {	×
NEW 338	if (!userIsAdmin(currentUser)) {	×
NEW 339	throw new Error("Claude Completion is for admins only");	×
340	}
341
NEW 342	const { prefix = '', commentIds, postIds, replyingCommentId, postId } = req.body;	×
343	// Set headers for streaming response
NEW 344	res.writeHead(200, {	×
345	"Content-Type": "text/event-stream",
346	"Cache-Control": "no-cache",
347	Connection: "keep-alive",
348	});
349
NEW 350	const url = 'https://api.hyperbolic.xyz/v1/completions';	×
351
NEW 352	const response = await fetch(url, {	×
353	method: 'POST',
354	headers: {
355	'Content-Type': 'application/json',
356	Authorization: `Bearer ${hyperbolicApiKey.get()}`,
357	},
358	body: JSON.stringify({
359	model: 'meta-llama/Meta-Llama-3.1-405B',
360	prompt: await construct405bPrompt(
361	prefix,
362	commentIds,
363	postIds,
364	currentUser,
365	context,
366	replyingCommentId,
367	postId
368	),
369	max_tokens: 256,
370	temperature: 0.7,
371	top_p: 0.9,
372	stream: true
373	}),
374	});
375
NEW 376	if (!response.ok \|\| !response.body) {	×
NEW 377	res.writeHead(response.status, response.statusText);	×
NEW 378	res.end(`Error from API: ${response.statusText}`);	×
NEW 379	return;	×
380	}
NEW 381	const reader = response.body.getReader();	×
NEW 382	const readableNodeStream = new Readable({	×
383	async read() {
NEW 384	const { done, value } = await reader.read();	×
NEW 385	if (done) {	×
NEW 386	this.push(null);	×
387	} else {
NEW 388	this.push(Buffer.from(value));	×
389	}
390	}
391	});
392
NEW 393	await pipeline(readableNodeStream, res);	×
394	} catch (error) {
395	// eslint-disable-next-line no-console
NEW 396	console.error("Autocomplete error:", error);	×
NEW 397	res.status(500).json({ error: "An error occurred during autocomplete" });	×
398	}
399	});
400	}

ForumMagnum / ForumMagnum / 10605510383

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous