9988459672

Committed 18 Jul 2024 08:49AM UTC coverage: 31.841% (-0.07%) from 31.911%

Build # 9988459672

Build Type

push

github

Committed by

Will-Howard

Commit Message

ensure userIds are associated with clientIds every time a new pair is encountered

Run Details

4905 of 40412 branches covered (12.14%)

Branch coverage included in aggregate %.

3 of 11 new or added lines in 2 files covered. (27.27%)

1 existing line in 1 file now uncovered.

28713 of 65168 relevant lines covered (44.06%)

179.82 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

30.54

/packages/lesswrong/server/scripts/convertImagesToCloudinary.ts

import { Globals } from '../../lib/vulcan-lib/config';
import { Revisions } from '../../lib/collections/revisions/collection';
import { Images } from '../../lib/collections/images/collection';
import { DatabaseServerSetting } from '../databaseSettings';
import { ckEditorUploadUrlSetting, cloudinaryCloudNameSetting } from '../../lib/publicSettings';
import { randomId } from '../../lib/random';
import cloudinary, { UploadApiResponse } from 'cloudinary';
import cheerio from 'cheerio';
import { cheerioParse } from '../utils/htmlUtil';
import { URL } from 'url';
import { ckEditorUploadUrlOverrideSetting } from '../../lib/instanceSettings';
import { getCollection } from '../../lib/vulcan-lib/getCollection';
import uniq from 'lodash/uniq';
import { loggerConstructor } from '../../lib/utils/logging';
import { Posts } from '../../lib/collections/posts';
import { getAtPath, setAtPath } from '../../lib/helpers';
import { getLatestRev, getNextVersion, htmlToChangeMetrics } from '../editor/utils';
import crypto from 'crypto';

const cloudinaryApiKey = new DatabaseServerSetting<string>("cloudinaryApiKey", "");
const cloudinaryApiSecret = new DatabaseServerSetting<string>("cloudinaryApiSecret", "");

type CloudinaryCredentials = {
  cloud_name: string,
  api_key: string,
  api_secret: string,
}

/**
 * Credentials that can be spread into `cloudinary.v2` functions, like so: `cloudinary.v2.url(publicId, { ...credentials })`
 */
const getCloudinaryCredentials = () => {
  const cloudName = cloudinaryCloudNameSetting.get();
  const apiKey = cloudinaryApiKey.get();
  const apiSecret = cloudinaryApiSecret.get();

  if (!cloudName || !apiKey || !apiSecret) {
    // eslint-disable-next-line no-console
    console.error("Cloudinary credentials not configured");
    return null;
  }

  const credentials: CloudinaryCredentials = {
    cloud_name: cloudName,
    api_key: apiKey,
    api_secret: apiSecret,
  };
  return credentials;
};

/** If an image has already been re-hosted, return its CDN URL. Otherwise null. */
async function findAlreadyMovedImage(identifier: string): Promise<string|null> {
  const image = await Images.findOne({identifier});
  return image?.cdnHostedUrl ?? null;
}

/**
 * Re-upload the given image URL to cloudinary, and return the cloudinary URL. If the image has already been uploaded
 * it will return the existing cloudinary URL.
 */
export async function moveImageToCloudinary({oldUrl, originDocumentId}: {oldUrl: string, originDocumentId: string}): Promise<string|null> {
  const upload = async (credentials: CloudinaryCredentials) => await cloudinary.v2.uploader.upload(
    oldUrl,
    {
      folder: `mirroredImages/${originDocumentId}`,
      ...credentials
    }
  );

  return getOrCreateCloudinaryImage({identifier: oldUrl, identifierType: 'originalUrl', upload})
}

/**
 * Upload the given image buffer to cloudinary, and return the cloudinary URL. If the image has already been uploaded
 * (identified by SHA256 hash) it will return the existing cloudinary URL.
 */
export async function uploadBufferToCloudinary(buffer: Buffer) {
  const hash = crypto.createHash('sha256').update(buffer).digest('hex');
  const upload = async (credentials: CloudinaryCredentials) => new Promise<UploadApiResponse>((resolve) => {
    cloudinary.v2.uploader
      .upload_stream(
        {
          ...credentials,
          folder: `mirroredImages/${hash}`
        },
        (error, result) => {
          if (error || !result) {
            // eslint-disable-next-line no-console
            console.error("Failed to upload buffer to Cloudinary:", error);
            throw error;
          }
          return resolve(result);
        }
      )
      .end(buffer);
  })

  return getOrCreateCloudinaryImage({identifier: hash, identifierType: 'sha256Hash', upload})
}

/**
 * Returns a cloudinary url of the image corresponding to `identifier`, uploads the image if it doesn't
 * already exist in cloudinary
 */
async function getOrCreateCloudinaryImage({
  identifier,
  identifierType,
  upload,
}: {
  identifier: string;
  identifierType: string;
  upload: (credentials: CloudinaryCredentials) => Promise<UploadApiResponse>;
}) {
  const logger = loggerConstructor("image-conversion");
  const alreadyRehosted = await findAlreadyMovedImage(identifier);
  if (alreadyRehosted) return alreadyRehosted;

  const credentials = getCloudinaryCredentials();

  if (!credentials) return null;

  const uploadResponse = await upload(credentials);
  logger(`Result of uploading image: ${uploadResponse.secure_url}`);

  // Serve all images with automatic quality and format transformations to save on bandwidth
  const autoQualityFormatUrl = cloudinary.v2.url(uploadResponse.public_id, {
    ...credentials,
    quality: "auto",
    fetch_format: "auto",
    secure: true,
  });

  await Images.rawInsert({
    identifier,
    identifierType,
    cdnHostedUrl: autoQualityFormatUrl,
  });

  return autoQualityFormatUrl;
}

/**
 * Images on domains not in this list will be mirrored on Cloudinary and have
 * their src updated.
 */
function getImageUrlWhitelist() {
  const localUploadUrl = ckEditorUploadUrlOverrideSetting.get() || ckEditorUploadUrlSetting.get()
  return [
    "cloudinary.com",
    "res.cloudinary.com",
    "www.lesswrong.com",
    "www.alignmentforum.org",
  ].concat(localUploadUrl ? [new URL(localUploadUrl).host] : []);
}

function urlNeedsMirroring(url: string, filterFn: (url: string) => boolean) {
  try {
    const parsedUrl = new URL(url);
    if (getImageUrlWhitelist().indexOf(parsedUrl.hostname) !== -1) {
      return false;
    }
    return filterFn(url);
  } catch (e) {
    return false;
  }
}

export async function convertImagesInHTML(html: string, originDocumentId: string, urlFilterFn: (url: string) => boolean = () => true): Promise<{count: number, html: string}> {
  const parsedHtml = cheerioParse(html);
  const imgTags = parsedHtml("img").toArray();
  const imgUrls: string[] = [];
  
  for (let imgTag of imgTags) {
    const urls = getImageUrlsFromImgTag(cheerio(imgTag));
    for (let url of urls) {
      if (urlNeedsMirroring(url, urlFilterFn)) {
        imgUrls.push(url);
      }
    }
  }

  // Upload all the images to Cloudinary (slow)
  const mirrorUrls: Record<string,string> = {};
  await Promise.all(imgUrls.map(async (url) => {
    // resolve to the url of the image on cloudinary
    const movedImage = await moveImageToCloudinary({oldUrl: url, originDocumentId})
    if (movedImage) {
      mirrorUrls[url] = movedImage;
    }
  }));

  // cheerio is not guarantueed to return the same html so explicitly count
  // the number of images that were converted
  let count = 0
  
  for (let i=0; i<imgTags.length; i++) {
    const imgTag = cheerio(imgTags[i]);
    const src: string|undefined = imgTag.attr("src");
    if (src) {
      const replacement = mirrorUrls[src];
      if (replacement) {
        imgTag.attr("src", replacement);
        count++;
      }
    }
    
    const srcset: string|undefined = imgTag.attr("srcset");
    if (srcset) {
      const replacement = rewriteSrcset(srcset, mirrorUrls);
      if (replacement) {
        imgTag.attr("srcset", replacement);
        count++;
      }
    }
  }
  
  return {count, html: parsedHtml.html()};
}

function getImageUrlsFromImgTag(tag: any): string[] {
  let imageUrls: string[] = [];
  const src: string = tag.attr("src");
  if (src) {
    imageUrls.push(src);
  }
  const srcset: string = tag.attr("srcset");
  if (srcset) {
    const imageVariants = srcset.split(/,\s/g).map(tok=>tok.trim());
    for (let imageVariant of imageVariants) {
      const [url, _size] = imageVariant.split(" ").map(tok=>tok.trim());
      if (url) imageUrls.push(url);
    }
  }
  
  return uniq(imageUrls);
}

function rewriteSrcset(srcset: string, urlMap: Record<string,string>): string {
  const imageVariants = srcset.split(/,\s/g).map(tok=>tok.trim());
  const rewrittenImageVariants = imageVariants.map(variant => {
    let tokens = variant.split(" ");
    if (tokens[0] in urlMap)
      tokens[0] = urlMap[tokens[0]];
    return tokens.join(" ");
  });
  return rewrittenImageVariants.join(", ");
}

/**
 * Reupload all images in an object (post, tag, user, etc) to Cloudinary, for a
 * specific editable field. Creates a new revision with the updated HTML.
 *
 * @param collectionName - The collection that this object is in
 * @param _id - The object to reupload images for
 * @param fieldName - The content-editable field tos can for images
 * @param urlFilterFn - A function that takes a URL and returns true if it should be mirrored, by default all URLs are mirrored except those in getImageUrlWhitelist()
 * @returns The number of images that were mirrored
 */
export async function convertImagesInObject<N extends CollectionNameString>(
  collectionName: N,
  _id: string,
  fieldName = "contents",
  urlFilterFn: (url: string) => boolean = ()=>true
): Promise<number> {
  const logger = loggerConstructor("image-conversion")
  let totalUploaded = 0;
  try {
    const collection = getCollection(collectionName);
    const obj = await collection.findOne({_id});

    if (!obj) {
      // eslint-disable-next-line no-console
      console.error(`Cannot convert images in ${collectionName}.${_id}: ID not found`);
      return 0;
    }
    
    const latestRev = await getLatestRev(_id, fieldName);
    if (!latestRev) {
      // If this field doesn't have a latest rev, it's empty (common eg for
      // moderation guidelines).
      return 0;
    }
    
    const newVersion = getNextVersion(latestRev, "patch", false);
    const now = new Date();
    // NOTE: we use the post contents rather than the revision contents because we don't
    // create a revision for no-op edits (this is arguably a bug)
    //
    // We also manually downcast the document because it's otherwise a union type of all possible DbObjects, and we can't use a random string as an index accessor
    // This is because `collection` is itself a union of all possible collections
    // I tried to make a mutual constraint between `fieldName` and `collectionName` but it was a bit too finnicky to be worth it; this is mostly being (unsafely) called from Globals anyways
    const oldHtml = (obj as AnyBecauseHard)?.[fieldName]?.html;
    if (!oldHtml) {
      return 0;
    }
    const {count: uploadCount, html: newHtml} = await convertImagesInHTML(oldHtml, _id, urlFilterFn);
    if (!uploadCount) {
      logger("No images to convert.");
      return 0;
    } else {
      logger(`Converted ${uploadCount} images`)
    }
    
    const newRevision = {
      ...latestRev,
      _id: randomId(),
      html: newHtml,
      editedAt: now,
      updateType: "patch",
      version: newVersion,
      commitMessage: "Move images to CDN",
      changeMetrics: htmlToChangeMetrics(oldHtml, newHtml),
    };
    const insertedRevisionId: string = await Revisions.rawInsert(newRevision);
    await collection.rawUpdateOne({_id}, {
      $set: {
        [`${fieldName}_latest`]: insertedRevisionId,
        [fieldName]: {
          ...(obj as AnyBecauseHard)[fieldName],
          html: newHtml,
          version: newVersion,
          editedAt: now,
          updateType: "patch",
        },
      }
    });
    totalUploaded += uploadCount;
    return totalUploaded;
  } catch (e) {
    // Always catch the error because the obj should mostly load fine without rehosting the images
    // eslint-disable-next-line no-console
    console.error("Error in convertImagesInObject", e)
    return 0
  }
}

const postMetaImageFields: string[][] = [
  ["socialPreview", "imageId"],
  ["eventImageId"],
  ["socialPreviewImageAutoUrl"],
  ["socialPreviewImageId"],
];

export const rehostPostMetaImages = async (post: DbPost) => {
  const operations: MongoBulkWriteOperations<DbPost> = [];

  for (const field of postMetaImageFields) {
    const currentUrl = getAtPath<DbPost, string>(post, field);
    if (!currentUrl || !urlNeedsMirroring(currentUrl, () => true)) {
      continue;
    }

    let newUrl: string | null = null;
    try {
      newUrl = await moveImageToCloudinary({oldUrl: currentUrl, originDocumentId: post._id});
    } catch (e) {
      // eslint-disable-next-line
      console.error(`Failed to move image for ${post._id}:`, field, `(error ${e})`);
      continue;
    }
    if (!newUrl) {
      // eslint-disable-next-line
      console.error(`Failed to move image for ${post._id}:`, field);
      continue;
    }

    setAtPath(post, field, newUrl);

    operations.push({
      updateOne: {
        filter: {
          _id: post._id,
        },
        update: {
          $set: {
            [field.join(".")]: newUrl,
          },
        },
      },
    });
  }

  await Posts.rawCollection().bulkWrite(operations);
}

export const rehostPostMetaImagesById = async (postId: string) => {
  const post = await Posts.findOne({_id: postId});
  if (!post) {
    throw new Error("Post not found");
  }
  await rehostPostMetaImages(post);
}

export const rehostAllPostMetaImages = async () => {
  const projection: Partial<Record<keyof DbPost, 1>> = {_id: 1};
  for (const field of postMetaImageFields) {
    projection[field[0] as keyof DbPost] = 1;
  }
  const posts = await Posts.find({
    $or: postMetaImageFields.map((field) => ({[field[0]]: {$exists: true}})),
  }, {projection}).fetch();
  for (const post of posts) {
    await rehostPostMetaImages(post);
  }
}

Globals.moveImageToCloudinary = moveImageToCloudinary;
Globals.convertImagesInHTML = convertImagesInHTML;
Globals.convertImagesInObject = convertImagesInObject;
Globals.rehostPostMetaImages = rehostPostMetaImages;
Globals.rehostPostMetaImagesById = rehostPostMetaImagesById;
Globals.rehostAllPostMetaImages = rehostAllPostMetaImages;

1	import { Globals } from '../../lib/vulcan-lib/config';	61✔
2	import { Revisions } from '../../lib/collections/revisions/collection';	20✔
3	import { Images } from '../../lib/collections/images/collection';	20✔
4	import { DatabaseServerSetting } from '../databaseSettings';	20✔
5	import { ckEditorUploadUrlSetting, cloudinaryCloudNameSetting } from '../../lib/publicSettings';	20✔
6	import { randomId } from '../../lib/random';	20✔
7	import cloudinary, { UploadApiResponse } from 'cloudinary';	20✔
8	import cheerio from 'cheerio';	20✔
9	import { cheerioParse } from '../utils/htmlUtil';	20✔
10	import { URL } from 'url';	20✔
11	import { ckEditorUploadUrlOverrideSetting } from '../../lib/instanceSettings';	20✔
12	import { getCollection } from '../../lib/vulcan-lib/getCollection';	20✔
13	import uniq from 'lodash/uniq';	20✔
14	import { loggerConstructor } from '../../lib/utils/logging';	20✔
15	import { Posts } from '../../lib/collections/posts';	20✔
16	import { getAtPath, setAtPath } from '../../lib/helpers';	20✔
17	import { getLatestRev, getNextVersion, htmlToChangeMetrics } from '../editor/utils';	20✔
18	import crypto from 'crypto';	20✔
19
20	const cloudinaryApiKey = new DatabaseServerSetting<string>("cloudinaryApiKey", "");	20✔
21	const cloudinaryApiSecret = new DatabaseServerSetting<string>("cloudinaryApiSecret", "");	20✔
22
23	type CloudinaryCredentials = {
24	cloud_name: string,
25	api_key: string,
26	api_secret: string,
27	}
28
29	/**
30	* Credentials that can be spread into `cloudinary.v2` functions, like so: `cloudinary.v2.url(publicId, { ...credentials })`
31	*/
32	const getCloudinaryCredentials = () => {	20✔
33	const cloudName = cloudinaryCloudNameSetting.get();	×
34	const apiKey = cloudinaryApiKey.get();	×
35	const apiSecret = cloudinaryApiSecret.get();	×
36
37	if (!cloudName \|\| !apiKey \|\| !apiSecret) {	×
38	// eslint-disable-next-line no-console
39	console.error("Cloudinary credentials not configured");	×
40	return null;	×
41	}
42
43	const credentials: CloudinaryCredentials = {	×
44	cloud_name: cloudName,
45	api_key: apiKey,
46	api_secret: apiSecret,
47	};
48	return credentials;	×
49	};
50
51	/** If an image has already been re-hosted, return its CDN URL. Otherwise null. */
52	async function findAlreadyMovedImage(identifier: string): Promise<string\|null> {
53	const image = await Images.findOne({identifier});	×
54	return image?.cdnHostedUrl ?? null;	×
55	}
56
57	/**
58	* Re-upload the given image URL to cloudinary, and return the cloudinary URL. If the image has already been uploaded
59	* it will return the existing cloudinary URL.
60	*/
61	export async function moveImageToCloudinary({oldUrl, originDocumentId}: {oldUrl: string, originDocumentId: string}): Promise<string\|null> {
62	const upload = async (credentials: CloudinaryCredentials) => await cloudinary.v2.uploader.upload(	×
63	oldUrl,
64	{
65	folder: `mirroredImages/${originDocumentId}`,
66	...credentials
67	}
68	);
69
70	return getOrCreateCloudinaryImage({identifier: oldUrl, identifierType: 'originalUrl', upload})	×
71	}
72
73	/**
74	* Upload the given image buffer to cloudinary, and return the cloudinary URL. If the image has already been uploaded
75	* (identified by SHA256 hash) it will return the existing cloudinary URL.
76	*/
77	export async function uploadBufferToCloudinary(buffer: Buffer) {
78	const hash = crypto.createHash('sha256').update(buffer).digest('hex');	×
79	const upload = async (credentials: CloudinaryCredentials) => new Promise<UploadApiResponse>((resolve) => {	×
80	cloudinary.v2.uploader	×
81	.upload_stream(
82	{
83	...credentials,
84	folder: `mirroredImages/${hash}`
85	},
86	(error, result) => {
87	if (error \|\| !result) {	×
88	// eslint-disable-next-line no-console
89	console.error("Failed to upload buffer to Cloudinary:", error);	×
90	throw error;	×
91	}
92	return resolve(result);	×
93	}
94	)
95	.end(buffer);
96	})
97
98	return getOrCreateCloudinaryImage({identifier: hash, identifierType: 'sha256Hash', upload})	×
99	}
100
101	/**
102	* Returns a cloudinary url of the image corresponding to `identifier`, uploads the image if it doesn't
103	* already exist in cloudinary
104	*/
105	async function getOrCreateCloudinaryImage({
106	identifier,
107	identifierType,
108	upload,
109	}: {
110	identifier: string;
111	identifierType: string;
112	upload: (credentials: CloudinaryCredentials) => Promise<UploadApiResponse>;
113	}) {
114	const logger = loggerConstructor("image-conversion");	×
115	const alreadyRehosted = await findAlreadyMovedImage(identifier);	×
116	if (alreadyRehosted) return alreadyRehosted;	×
117
118	const credentials = getCloudinaryCredentials();	×
119
120	if (!credentials) return null;	×
121
122	const uploadResponse = await upload(credentials);	×
123	logger(`Result of uploading image: ${uploadResponse.secure_url}`);	×
124
125	// Serve all images with automatic quality and format transformations to save on bandwidth
126	const autoQualityFormatUrl = cloudinary.v2.url(uploadResponse.public_id, {	×
127	...credentials,
128	quality: "auto",
129	fetch_format: "auto",
130	secure: true,
131	});
132
133	await Images.rawInsert({	×
134	identifier,
135	identifierType,
136	cdnHostedUrl: autoQualityFormatUrl,
137	});
138
139	return autoQualityFormatUrl;	×
140	}
141
142	/**
143	* Images on domains not in this list will be mirrored on Cloudinary and have
144	* their src updated.
145	*/
146	function getImageUrlWhitelist() {
147	const localUploadUrl = ckEditorUploadUrlOverrideSetting.get() \|\| ckEditorUploadUrlSetting.get()	×
148	return [	×
149	"cloudinary.com",
150	"res.cloudinary.com",
151	"www.lesswrong.com",
152	"www.alignmentforum.org",
153	].concat(localUploadUrl ? [new URL(localUploadUrl).host] : []);	×
154	}
155
156	function urlNeedsMirroring(url: string, filterFn: (url: string) => boolean) {
157	try {	×
158	const parsedUrl = new URL(url);	×
159	if (getImageUrlWhitelist().indexOf(parsedUrl.hostname) !== -1) {	×
160	return false;	×
161	}
162	return filterFn(url);	×
163	} catch (e) {
164	return false;	×
165	}
166	}
167
168	export async function convertImagesInHTML(html: string, originDocumentId: string, urlFilterFn: (url: string) => boolean = () => true): Promise<{count: number, html: string}> {	×
169	const parsedHtml = cheerioParse(html);	32✔
170	const imgTags = parsedHtml("img").toArray();	32✔
171	const imgUrls: string[] = [];	32✔
172
173	for (let imgTag of imgTags) {	32✔
174	const urls = getImageUrlsFromImgTag(cheerio(imgTag));	×
175	for (let url of urls) {	×
176	if (urlNeedsMirroring(url, urlFilterFn)) {	×
177	imgUrls.push(url);	×
178	}
179	}
180	}
181
182	// Upload all the images to Cloudinary (slow)
183	const mirrorUrls: Record<string,string> = {};	32✔
184	await Promise.all(imgUrls.map(async (url) => {	32✔
185	// resolve to the url of the image on cloudinary
186	const movedImage = await moveImageToCloudinary({oldUrl: url, originDocumentId})	×
187	if (movedImage) {	×
188	mirrorUrls[url] = movedImage;	×
189	}
190	}));
191
192	// cheerio is not guarantueed to return the same html so explicitly count
193	// the number of images that were converted
194	let count = 0	32✔
195
196	for (let i=0; i<imgTags.length; i++) {	32✔
197	const imgTag = cheerio(imgTags[i]);	×
198	const src: string\|undefined = imgTag.attr("src");	×
199	if (src) {	×
200	const replacement = mirrorUrls[src];	×
201	if (replacement) {	×
202	imgTag.attr("src", replacement);	×
203	count++;	×
204	}
205	}
206
207	const srcset: string\|undefined = imgTag.attr("srcset");	×
208	if (srcset) {	×
209	const replacement = rewriteSrcset(srcset, mirrorUrls);	×
210	if (replacement) {	×
211	imgTag.attr("srcset", replacement);	×
212	count++;	×
213	}
214	}
215	}
216
217	return {count, html: parsedHtml.html()};	32✔
218	}
219
220	function getImageUrlsFromImgTag(tag: any): string[] {
221	let imageUrls: string[] = [];	×
222	const src: string = tag.attr("src");	×
223	if (src) {	×
224	imageUrls.push(src);	×
225	}
226	const srcset: string = tag.attr("srcset");	×
227	if (srcset) {	×
228	const imageVariants = srcset.split(/,\s/g).map(tok=>tok.trim());	×
229	for (let imageVariant of imageVariants) {	×
230	const [url, _size] = imageVariant.split(" ").map(tok=>tok.trim());	×
231	if (url) imageUrls.push(url);	×
232	}
233	}
234
235	return uniq(imageUrls);	×
236	}
237
238	function rewriteSrcset(srcset: string, urlMap: Record<string,string>): string {
239	const imageVariants = srcset.split(/,\s/g).map(tok=>tok.trim());	×
240	const rewrittenImageVariants = imageVariants.map(variant => {	×
241	let tokens = variant.split(" ");	×
242	if (tokens[0] in urlMap)	×
243	tokens[0] = urlMap[tokens[0]];	×
244	return tokens.join(" ");	×
245	});
246	return rewrittenImageVariants.join(", ");	×
247	}
248
249	/**
250	* Reupload all images in an object (post, tag, user, etc) to Cloudinary, for a
251	* specific editable field. Creates a new revision with the updated HTML.
252	*
253	* @param collectionName - The collection that this object is in
254	* @param _id - The object to reupload images for
255	* @param fieldName - The content-editable field tos can for images
256	* @param urlFilterFn - A function that takes a URL and returns true if it should be mirrored, by default all URLs are mirrored except those in getImageUrlWhitelist()
257	* @returns The number of images that were mirrored
258	*/
259	export async function convertImagesInObject<N extends CollectionNameString>(
260	collectionName: N,
261	_id: string,
262	fieldName = "contents",	×
263	urlFilterFn: (url: string) => boolean = ()=>true	✔
264	): Promise<number> {
265	const logger = loggerConstructor("image-conversion")	997✔
266	let totalUploaded = 0;	997✔
267	try {	997✔
268	const collection = getCollection(collectionName);	997✔
269	const obj = await collection.findOne({_id});	997✔
270
271	if (!obj) {	997!
272	// eslint-disable-next-line no-console
273	console.error(`Cannot convert images in ${collectionName}.${_id}: ID not found`);	×
274	return 0;	×
275	}
276
277	const latestRev = await getLatestRev(_id, fieldName);	997✔
278	if (!latestRev) {	997✔
279	// If this field doesn't have a latest rev, it's empty (common eg for
280	// moderation guidelines).
281	return 0;	966✔
282	}
283
284	const newVersion = getNextVersion(latestRev, "patch", false);	31✔
285	const now = new Date();	31✔
286	// NOTE: we use the post contents rather than the revision contents because we don't
287	// create a revision for no-op edits (this is arguably a bug)
288	//
289	// We also manually downcast the document because it's otherwise a union type of all possible DbObjects, and we can't use a random string as an index accessor
290	// This is because `collection` is itself a union of all possible collections
291	// I tried to make a mutual constraint between `fieldName` and `collectionName` but it was a bit too finnicky to be worth it; this is mostly being (unsafely) called from Globals anyways
292	const oldHtml = (obj as AnyBecauseHard)?.[fieldName]?.html;	31!
293	if (!oldHtml) {	31!
UNCOV 294	return 0;	×
295	}
296	const {count: uploadCount, html: newHtml} = await convertImagesInHTML(oldHtml, _id, urlFilterFn);	31✔
297	if (!uploadCount) {	31!
298	logger("No images to convert.");	31✔
299	return 0;	31✔
300	} else {
301	logger(`Converted ${uploadCount} images`)	×
302	}
303
304	const newRevision = {	×
305	...latestRev,
306	_id: randomId(),
307	html: newHtml,
308	editedAt: now,
309	updateType: "patch",
310	version: newVersion,
311	commitMessage: "Move images to CDN",
312	changeMetrics: htmlToChangeMetrics(oldHtml, newHtml),
313	};
314	const insertedRevisionId: string = await Revisions.rawInsert(newRevision);	×
315	await collection.rawUpdateOne({_id}, {	×
316	$set: {
317	[`${fieldName}_latest`]: insertedRevisionId,
318	[fieldName]: {
319	...(obj as AnyBecauseHard)[fieldName],
320	html: newHtml,
321	version: newVersion,
322	editedAt: now,
323	updateType: "patch",
324	},
325	}
326	});
327	totalUploaded += uploadCount;	×
328	return totalUploaded;	×
329	} catch (e) {
330	// Always catch the error because the obj should mostly load fine without rehosting the images
331	// eslint-disable-next-line no-console
332	console.error("Error in convertImagesInObject", e)	×
333	return 0	×
334	}
335	}
336
337	const postMetaImageFields: string[][] = [	20✔
338	["socialPreview", "imageId"],
339	["eventImageId"],
340	["socialPreviewImageAutoUrl"],
341	["socialPreviewImageId"],
342	];
343
344	export const rehostPostMetaImages = async (post: DbPost) => {	20✔
345	const operations: MongoBulkWriteOperations<DbPost> = [];	294✔
346
347	for (const field of postMetaImageFields) {	294✔
348	const currentUrl = getAtPath<DbPost, string>(post, field);	1,176✔
349	if (!currentUrl \|\| !urlNeedsMirroring(currentUrl, () => true)) {	1,176!
350	continue;	1,176✔
351	}
352
353	let newUrl: string \| null = null;	×
354	try {	×
355	newUrl = await moveImageToCloudinary({oldUrl: currentUrl, originDocumentId: post._id});	×
356	} catch (e) {
357	// eslint-disable-next-line
358	console.error(`Failed to move image for ${post._id}:`, field, `(error ${e})`);	×
359	continue;	×
360	}
361	if (!newUrl) {	×
362	// eslint-disable-next-line
363	console.error(`Failed to move image for ${post._id}:`, field);	×
364	continue;	×
365	}
366
367	setAtPath(post, field, newUrl);	×
368
369	operations.push({	×
370	updateOne: {
371	filter: {
372	_id: post._id,
373	},
374	update: {
375	$set: {
376	[field.join(".")]: newUrl,
377	},
378	},
379	},
380	});
381	}
382
383	await Posts.rawCollection().bulkWrite(operations);	294✔
384	}
385
386	export const rehostPostMetaImagesById = async (postId: string) => {	20✔
387	const post = await Posts.findOne({_id: postId});	×
388	if (!post) {	×
389	throw new Error("Post not found");	×
390	}
391	await rehostPostMetaImages(post);	×
392	}
393
394	export const rehostAllPostMetaImages = async () => {	20✔
395	const projection: Partial<Record<keyof DbPost, 1>> = {_id: 1};	×
396	for (const field of postMetaImageFields) {	×
397	projection[field[0] as keyof DbPost] = 1;	×
398	}
399	const posts = await Posts.find({	×
400	$or: postMetaImageFields.map((field) => ({[field[0]]: {$exists: true}})),	×
401	}, {projection}).fetch();
402	for (const post of posts) {	×
403	await rehostPostMetaImages(post);	×
404	}
405	}
406
407	Globals.moveImageToCloudinary = moveImageToCloudinary;	20✔
408	Globals.convertImagesInHTML = convertImagesInHTML;	20✔
409	Globals.convertImagesInObject = convertImagesInObject;	20✔
410	Globals.rehostPostMetaImages = rehostPostMetaImages;	20✔
411	Globals.rehostPostMetaImagesById = rehostPostMetaImagesById;	20✔
412	Globals.rehostAllPostMetaImages = rehostAllPostMetaImages;	20✔

ForumMagnum / ForumMagnum / 9988459672

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous