34ce2a6a-2d06-4f6f-8054-151fb066ee9e

Build # 34ce2a6a-2d06-4f6f-8054-151fb066ee9e

Build Type

push

circleci

Committed by GitHub

Commit Message

Merge pull request #2766 from mozilla/MNTOR-978

Run Details

282 of 1182 branches covered (23.86%)

Branch coverage included in aggregate %.

40 of 40 new or added lines in 1 file covered. (100.0%)

959 of 3157 relevant lines covered (30.38%)

4.93 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

0.0

/src/scripts/recencyToBreachId.js

/**
 * Executes once
 * Execute after `deprecateBreachResolve.js`
 * The purpose of the script is to convert all recency indices to breach ids for `breach_resolution`
 * For backwards compatibility, all converted `breach_resolution` fields will have a boolean
 * `useBreachId: true/false`
 */

import Knex from 'knex'
import knexConfig from '../db/knexfile.js'
import { getAllBreachesFromDb } from '../utils/hibp.js'
import { getAllEmailsAndBreaches } from '../utils/breaches.js'
import { setBreachResolution } from '../db/tables/subscribers.js'
const knex = Knex(knexConfig)

const LIMIT = 50 // with millions of records, we have to load a few at a time
let offset = 0 // looping through all records with offset
let subscribersArr = []

// load all breaches for ref
const allBreaches = await getAllBreachesFromDb()
if (allBreaches && allBreaches.length > 0) console.log('breaches loaded successfully! ', allBreaches.length)
console.log(JSON.stringify(allBreaches[0]))

// find all subscribers who resolved any breaches in the past,
// replace recency index with breach id
do {
  console.log(`Converting breach_resolution to use breach Id - start: ${offset} limit: ${LIMIT}`)
  subscribersArr = await knex
    .select('id', 'primary_email', 'breach_resolution')
    .from('subscribers')
    .whereNotNull('breach_resolution')
    .limit(LIMIT)
    .offset(offset)

  console.log(`Loaded # of subscribers: ${subscribersArr.length}`)

  for (const subscriber of subscribersArr) {
    const { breach_resolution: v2 } = subscriber
    console.debug({ v2 })

    // if useBreachId is set, skip because this breach_resolution has already been worked on
    if (v2.useBreachId) {
      console.log('Skipping since `useBreachId` is set already, this breach resolution is already converted')
      continue
    }

    const newResolutions = {}

    // fetch subscriber all breaches / email
    const subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)
    console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))

    for (const email in v2) {
      console.debug({ email })
      const resolutions = v2[email]
      console.debug({ resolutions })
      newResolutions[email] = {}

      for (const recencyIndex in resolutions) {
        console.debug({ recencyIndex })

        // find subscriber's relevant recency index breach information
        const ve = subscriberBreachesEmail.verifiedEmails?.filter(ve => ve.email === email)[0] || {}
        const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] || null
        const breachName = subBreach?.Name
        console.debug({ breachName })

        // find breach id for the breach
        const breachId = allBreaches.find(b => b.Name === breachName)?.Id
        console.log({ breachId })
        newResolutions[email][breachId] = v2[email][recencyIndex]
      }
    }

    // check if v2 is changed, if so, upsert the new v2
    newResolutions.useBreachId = true
    console.log(JSON.stringify(newResolutions))
    await setBreachResolution(subscriber, newResolutions)
  }
  offset += LIMIT
} while (subscribersArr.length === LIMIT)

// breaking out of do..while loop
console.log('Reaching the end of the table, offset ended at', offset)
process.exit()

1	/**
2	* Executes once
3	* Execute after `deprecateBreachResolve.js`
4	* The purpose of the script is to convert all recency indices to breach ids for `breach_resolution`
5	* For backwards compatibility, all converted `breach_resolution` fields will have a boolean
6	* `useBreachId: true/false`
7	*/
8
9	import Knex from 'knex'
10	import knexConfig from '../db/knexfile.js'
11	import { getAllBreachesFromDb } from '../utils/hibp.js'
12	import { getAllEmailsAndBreaches } from '../utils/breaches.js'
13	import { setBreachResolution } from '../db/tables/subscribers.js'
14	const knex = Knex(knexConfig)	×
15
16	const LIMIT = 50 // with millions of records, we have to load a few at a time	×
17	let offset = 0 // looping through all records with offset	×
18	let subscribersArr = []	×
19
20	// load all breaches for ref
21	const allBreaches = await getAllBreachesFromDb()	×
22	if (allBreaches && allBreaches.length > 0) console.log('breaches loaded successfully! ', allBreaches.length)	×
23	console.log(JSON.stringify(allBreaches[0]))	×
24
25	// find all subscribers who resolved any breaches in the past,
26	// replace recency index with breach id
27	do {	×
28	console.log(`Converting breach_resolution to use breach Id - start: ${offset} limit: ${LIMIT}`)	×
29	subscribersArr = await knex	×
30	.select('id', 'primary_email', 'breach_resolution')
31	.from('subscribers')
32	.whereNotNull('breach_resolution')
33	.limit(LIMIT)
34	.offset(offset)
35
36	console.log(`Loaded # of subscribers: ${subscribersArr.length}`)	×
37
38	for (const subscriber of subscribersArr) {	×
39	const { breach_resolution: v2 } = subscriber	×
40	console.debug({ v2 })	×
41
42	// if useBreachId is set, skip because this breach_resolution has already been worked on
43	if (v2.useBreachId) {	×
44	console.log('Skipping since `useBreachId` is set already, this breach resolution is already converted')	×
45	continue	×
46	}
47
48	const newResolutions = {}	×
49
50	// fetch subscriber all breaches / email
51	const subscriberBreachesEmail = await getAllEmailsAndBreaches(subscriber, allBreaches)	×
52	console.debug(JSON.stringify(subscriberBreachesEmail.verifiedEmails))	×
53
54	for (const email in v2) {	×
55	console.debug({ email })	×
56	const resolutions = v2[email]	×
57	console.debug({ resolutions })	×
58	newResolutions[email] = {}	×
59
60	for (const recencyIndex in resolutions) {	×
61	console.debug({ recencyIndex })	×
62
63	// find subscriber's relevant recency index breach information
64	const ve = subscriberBreachesEmail.verifiedEmails?.filter(ve => ve.email === email)[0] \|\| {}	×
65	const subBreach = ve.breaches?.filter(b => Number(b.recencyIndex) === Number(recencyIndex))[0] \|\| null	×
66	const breachName = subBreach?.Name	×
67	console.debug({ breachName })	×
68
69	// find breach id for the breach
70	const breachId = allBreaches.find(b => b.Name === breachName)?.Id	×
71	console.log({ breachId })	×
72	newResolutions[email][breachId] = v2[email][recencyIndex]	×
73	}
74	}
75
76	// check if v2 is changed, if so, upsert the new v2
77	newResolutions.useBreachId = true	×
78	console.log(JSON.stringify(newResolutions))	×
79	await setBreachResolution(subscriber, newResolutions)	×
80	}
81	offset += LIMIT	×
82	} while (subscribersArr.length === LIMIT)
83
84	// breaking out of do..while loop
85	console.log('Reaching the end of the table, offset ended at', offset)	×
86	process.exit()	×

mozilla / blurts-server / 34ce2a6a-2d06-4f6f-8054-151fb066ee9e

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous