10101329865

Committed 25 Jul 2024 08:53PM UTC coverage: 31.859% (-0.002%) from 31.861%

Build # 10101329865

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #9550 from ForumMagnum/master

[EA] deploy ModGPT update, neuronpedia embeddings, no custom highlights in recent discussion, etc

Run Details

4939 of 40611 branches covered (12.16%)

Branch coverage included in aggregate %.

14 of 51 new or added lines in 11 files covered. (27.45%)

8 existing lines in 3 files now uncovered.

28886 of 65561 relevant lines covered (44.06%)

179.45 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

20.0

/packages/lesswrong/server/users/permanentDeletion.ts

import Users from "@/lib/collections/users/collection";
import { addCronJob } from "../cronUtil";
import { ACCOUNT_DELETION_COOLING_OFF_DAYS, getUserEmail } from "@/lib/collections/users/helpers";
import { Globals, createAdminContext, deleteMutator, updateMutator } from "../vulcan-lib";
import { getAdminTeamAccount } from "../callbacks/commentCallbacks";
import { loggerConstructor } from "@/lib/utils/logging";
import { mailchimpAPIKeySetting } from "../serverSettings";
import { mailchimpEAForumListIdSetting, mailchimpForumDigestListIdSetting } from "@/lib/publicSettings";
import md5 from "md5";
import { captureException } from "@sentry/core";
import { auth0RemoveAssociationAndTryDeleteUser } from "../authentication/auth0";
import { dogstatsd } from "../datadog/tracer";
import { isEAForum } from "@/lib/instanceSettings";

type DeleteOptions = { includingNonForumData: boolean };
const defaultDeleteOptions = { includingNonForumData: false };

/**
 * Additional mailchimp lists to delete a user from if they are requesting removal of all the
 * data CEA Online has on them, rather than just deleting their forum account
 */
const EAF_EXTRA_MAILCHIMP_LISTS = [
  '51c1df13ac', // The EA Newsletter
  // The lists below are very rarely/never used
  '744dc0ccaa', // EA.org
  '32149ddbad', // Top 20% readers of EA Forum AI safety posts
  '4ef6c1c639', // Forum Readers (Non-Active)
  '8671bf958a', // Evergreen posts
  'a8a23445f6', // Forum Beta Users
]

/**
 * Permanently delete a user from a Mailchimp list. Note
 * that this means they can't sign back up using the API, only with the mailchimp
 * signup form, so this should only be used if a user requests to permanently
 * delete their data.
 *
 * See here for discussion of the api/sign-up restriction: https://stackoverflow.com/questions/52198510/mailchimp-resubscribe-a-deleted-member-causes-the-api-to-return-a-400-bad-reques
 */
const permanentlyDeleteFromMailchimpList = async ({
  listId,
  emailHash,
  user,
  logger
}: {
  listId: string;
  emailHash: string;
  user: DbUser;
  logger: ReturnType<typeof loggerConstructor>;
}) => {
  const mailchimpAPIKey = mailchimpAPIKeySetting.get();

  try {
    const response = await fetch(
      `https://us8.api.mailchimp.com/3.0/lists/${listId}/members/${emailHash}/actions/delete-permanent`,
      {
        method: "POST",
        headers: {
          "Content-Type": "application/json",
          Authorization: `API_KEY ${mailchimpAPIKey}`,
        },
      }
    );

    if (!response.ok) {
      const json = await response.json();
      if (json.status !== 404) { // 404 just means they weren't subscribed to begin with
        // eslint-disable-next-line no-console
        console.error(
          `Failed to permanently delete user ${user.displayName} from Mailchimp list ${listId}. Response: `,
          json
        );
      }
    } else {
      logger(`Permanently deleted user with display name "${user.displayName}" from Mailchimp list ${listId}`);
    }
  } catch (e) {
    // eslint-disable-next-line no-console
    console.error(e);
    captureException(e);
  }
};

async function permanentlyDeleteUser(user: DbUser, options: DeleteOptions) {
  const logger = loggerConstructor(`permanentlyDeleteUsers`);
  const adminContext = createAdminContext();
  const adminTeamAccount = await getAdminTeamAccount();
  if (!adminTeamAccount) throw new Error("Couldn't find admin team account");

  // Precaution: Ensure the soft-deleting callbacks have run (unsubscribing from mailchimp, reindexing elasticsearch content)
  await updateMutator({
    collection: Users,
    documentId: user._id,
    set: { deleted: true },
    validate: false,
    context: adminContext,
    currentUser: adminContext.currentUser,
  })
  // Wait until async callbacks finish. This is overcautious, as there should be no need for the callbacks to refetch the user object
  await new Promise(resolve => setTimeout(resolve, 5000));

  // Remove from mailchimp lists
  const mailchimpForumDigestListId = mailchimpForumDigestListIdSetting.get();
  const mailchimpEAForumListId = mailchimpEAForumListIdSetting.get();

  const listIdsToDeleteFrom = [mailchimpEAForumListId, mailchimpForumDigestListId].filter(v => v) as string[];
  if (isEAForum && options.includingNonForumData) {
    listIdsToDeleteFrom.push(...EAF_EXTRA_MAILCHIMP_LISTS);
  }

  const email = getUserEmail(user);
  if (email) {
    const emailHash = md5(email.toLowerCase());

    for (const listId of listIdsToDeleteFrom) {
      await permanentlyDeleteFromMailchimpList({ listId, emailHash, user, logger });
    }
  }

  // Delete in auth0 to the extent possible
  const deletedFromAuth0 = await auth0RemoveAssociationAndTryDeleteUser(user);
  logger(`Removed association with Auth0 for user with display name "${user.displayName}". The user was${deletedFromAuth0 ? "" : " not"} deleted from Auth0`)

  // Permanently delete from the forum itself
  await deleteMutator({
    collection: Users,
    documentId: user._id,
    validate: false,
    currentUser: adminTeamAccount,
    context: adminContext
  })
  logger(`Permanently deleted user with display name "${user.displayName}" from the forum database`)
}

/**
 * Permanently delete a user from the forum, this is sufficient to comply with GDPR deletion
 * requests if their forum account and associated services is the only data we have for them
 */
async function permanentlyDeleteUserById(userId: string, options?: DeleteOptions) {
  const user = await Users.findOne({_id: userId})

  if (!user) {
    // eslint-disable-next-line no-console
    console.log(`User with _id "${userId}" not found`)
    return
  }

  await permanentlyDeleteUser(user, options ?? defaultDeleteOptions)
}

const cutoffOffsetMs = ACCOUNT_DELETION_COOLING_OFF_DAYS * 24 * 60 * 60 * 1000;

addCronJob({
  name: "permanentlyDeleteUsers",
  interval: "every 1 hour",
  job: async () => {
    const deletionRequestCutoff = new Date(Date.now() - cutoffOffsetMs)

    const usersToDelete = await Users.find({ permanentDeletionRequestedAt: { $lt: deletionRequestCutoff } }).fetch();

    if (usersToDelete.length > 10) {
      dogstatsd?.increment("user_deleted", usersToDelete.length, 1.0, {outcome: 'error'})
      throw new Error("Unexpectedly high number of users queued for deletion")
    }

    for (const user of usersToDelete) {
      try {
        await permanentlyDeleteUser(user, defaultDeleteOptions)
        dogstatsd?.increment("user_deleted", 1, 1.0, {outcome: 'success'})
      } catch (e) {
        // eslint-disable-next-line no-console
        console.error(e)
        dogstatsd?.increment("user_deleted", 1, 1.0, {outcome: 'error'})
      }
    }
  },
});

Globals.permanentlyDeleteUserById = permanentlyDeleteUserById

1	import Users from "@/lib/collections/users/collection";	18✔
2	import { addCronJob } from "../cronUtil";	18✔
3	import { ACCOUNT_DELETION_COOLING_OFF_DAYS, getUserEmail } from "@/lib/collections/users/helpers";	18✔
4	import { Globals, createAdminContext, deleteMutator, updateMutator } from "../vulcan-lib";	18✔
5	import { getAdminTeamAccount } from "../callbacks/commentCallbacks";	18✔
6	import { loggerConstructor } from "@/lib/utils/logging";	18✔
7	import { mailchimpAPIKeySetting } from "../serverSettings";	18✔
8	import { mailchimpEAForumListIdSetting, mailchimpForumDigestListIdSetting } from "@/lib/publicSettings";	18✔
9	import md5 from "md5";	18✔
10	import { captureException } from "@sentry/core";	18✔
11	import { auth0RemoveAssociationAndTryDeleteUser } from "../authentication/auth0";	18✔
12	import { dogstatsd } from "../datadog/tracer";	18✔
13	import { isEAForum } from "@/lib/instanceSettings";	18✔
14
15	type DeleteOptions = { includingNonForumData: boolean };
16	const defaultDeleteOptions = { includingNonForumData: false };	18✔
17
18	/**
19	* Additional mailchimp lists to delete a user from if they are requesting removal of all the
20	* data CEA Online has on them, rather than just deleting their forum account
21	*/
22	const EAF_EXTRA_MAILCHIMP_LISTS = [	18✔
23	'51c1df13ac', // The EA Newsletter
24	// The lists below are very rarely/never used
25	'744dc0ccaa', // EA.org
26	'32149ddbad', // Top 20% readers of EA Forum AI safety posts
27	'4ef6c1c639', // Forum Readers (Non-Active)
28	'8671bf958a', // Evergreen posts
29	'a8a23445f6', // Forum Beta Users
30	]
31
32	/**
33	* Permanently delete a user from a Mailchimp list. Note
34	* that this means they can't sign back up using the API, only with the mailchimp
35	* signup form, so this should only be used if a user requests to permanently
36	* delete their data.
37	*
38	* See here for discussion of the api/sign-up restriction: https://stackoverflow.com/questions/52198510/mailchimp-resubscribe-a-deleted-member-causes-the-api-to-return-a-400-bad-reques
39	*/
40	const permanentlyDeleteFromMailchimpList = async ({	18✔
41	listId,
42	emailHash,
43	user,
44	logger
45	}: {
46	listId: string;
47	emailHash: string;
48	user: DbUser;
49	logger: ReturnType<typeof loggerConstructor>;
50	}) => {
51	const mailchimpAPIKey = mailchimpAPIKeySetting.get();	×
52
53	try {	×
54	const response = await fetch(	×
55	`https://us8.api.mailchimp.com/3.0/lists/${listId}/members/${emailHash}/actions/delete-permanent`,
56	{
57	method: "POST",
58	headers: {
59	"Content-Type": "application/json",
60	Authorization: `API_KEY ${mailchimpAPIKey}`,
61	},
62	}
63	);
64
65	if (!response.ok) {	×
66	const json = await response.json();	×
67	if (json.status !== 404) { // 404 just means they weren't subscribed to begin with	×
68	// eslint-disable-next-line no-console
69	console.error(	×
70	`Failed to permanently delete user ${user.displayName} from Mailchimp list ${listId}. Response: `,
71	json
72	);
73	}
74	} else {
75	logger(`Permanently deleted user with display name "${user.displayName}" from Mailchimp list ${listId}`);	×
76	}
77	} catch (e) {
78	// eslint-disable-next-line no-console
79	console.error(e);	×
80	captureException(e);	×
81	}
82	};
83
84	async function permanentlyDeleteUser(user: DbUser, options: DeleteOptions) {
85	const logger = loggerConstructor(`permanentlyDeleteUsers`);	×
86	const adminContext = createAdminContext();	×
87	const adminTeamAccount = await getAdminTeamAccount();	×
88	if (!adminTeamAccount) throw new Error("Couldn't find admin team account");	×
89
90	// Precaution: Ensure the soft-deleting callbacks have run (unsubscribing from mailchimp, reindexing elasticsearch content)
91	await updateMutator({	×
92	collection: Users,
93	documentId: user._id,
94	set: { deleted: true },
95	validate: false,
96	context: adminContext,
97	currentUser: adminContext.currentUser,
98	})
99	// Wait until async callbacks finish. This is overcautious, as there should be no need for the callbacks to refetch the user object
100	await new Promise(resolve => setTimeout(resolve, 5000));	×
101
102	// Remove from mailchimp lists
103	const mailchimpForumDigestListId = mailchimpForumDigestListIdSetting.get();	×
104	const mailchimpEAForumListId = mailchimpEAForumListIdSetting.get();	×
105
NEW 106	const listIdsToDeleteFrom = [mailchimpEAForumListId, mailchimpForumDigestListId].filter(v => v) as string[];	×
NEW 107	if (isEAForum && options.includingNonForumData) {	×
NEW 108	listIdsToDeleteFrom.push(...EAF_EXTRA_MAILCHIMP_LISTS);	×
109	}
110
111	const email = getUserEmail(user);	×
112	if (email) {	×
113	const emailHash = md5(email.toLowerCase());	×
114
115	for (const listId of listIdsToDeleteFrom) {	×
116	await permanentlyDeleteFromMailchimpList({ listId, emailHash, user, logger });	×
117	}
118	}
119
120	// Delete in auth0 to the extent possible
121	const deletedFromAuth0 = await auth0RemoveAssociationAndTryDeleteUser(user);	×
122	logger(`Removed association with Auth0 for user with display name "${user.displayName}". The user was${deletedFromAuth0 ? "" : " not"} deleted from Auth0`)	×
123
124	// Permanently delete from the forum itself
125	await deleteMutator({	×
126	collection: Users,
127	documentId: user._id,
128	validate: false,
129	currentUser: adminTeamAccount,
130	context: adminContext
131	})
132	logger(`Permanently deleted user with display name "${user.displayName}" from the forum database`)	×
133	}
134
135	/**
136	* Permanently delete a user from the forum, this is sufficient to comply with GDPR deletion
137	* requests if their forum account and associated services is the only data we have for them
138	*/
139	async function permanentlyDeleteUserById(userId: string, options?: DeleteOptions) {
140	const user = await Users.findOne({_id: userId})	×
141
142	if (!user) {	×
143	// eslint-disable-next-line no-console
144	console.log(`User with _id "${userId}" not found`)	×
145	return	×
146	}
147
NEW 148	await permanentlyDeleteUser(user, options ?? defaultDeleteOptions)	×
149	}
150
151	const cutoffOffsetMs = ACCOUNT_DELETION_COOLING_OFF_DAYS * 24 * 60 * 60 * 1000;	18✔
152
153	addCronJob({
154	name: "permanentlyDeleteUsers",
155	interval: "every 1 hour",
156	job: async () => {
157	const deletionRequestCutoff = new Date(Date.now() - cutoffOffsetMs)	×
158
159	const usersToDelete = await Users.find({ permanentDeletionRequestedAt: { $lt: deletionRequestCutoff } }).fetch();	×
160
161	if (usersToDelete.length > 10) {	×
162	dogstatsd?.increment("user_deleted", usersToDelete.length, 1.0, {outcome: 'error'})	×
163	throw new Error("Unexpectedly high number of users queued for deletion")	×
164	}
165
166	for (const user of usersToDelete) {	×
167	try {	×
NEW 168	await permanentlyDeleteUser(user, defaultDeleteOptions)	×
169	dogstatsd?.increment("user_deleted", 1, 1.0, {outcome: 'success'})	×
170	} catch (e) {
171	// eslint-disable-next-line no-console
172	console.error(e)	×
173	dogstatsd?.increment("user_deleted", 1, 1.0, {outcome: 'error'})	×
174	}
175	}
176	},
177	});
178
179	Globals.permanentlyDeleteUserById = permanentlyDeleteUserById	18✔

ForumMagnum / ForumMagnum / 10101329865

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous