27606648281

Committed 13 Jun 2026 07:03PM UTC coverage: 97.953% (-0.03%) from 97.979%

Build # 27606648281

Build Type

push

github

Committed by

web-flow

Commit Message

Add telemetry when GitHub token is removed (#11922)

Coverage Stats

5848 of 6118 branches covered (95.59%)

30 of 35 new or added lines in 3 files covered. (85.71%)

58 existing lines in 7 files now uncovered.

49427 of 50460 relevant lines covered (97.95%)

138.27 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

82.91

/services/github/github-api-provider.js

import Joi from 'joi'
import log from '../../core/server/log.js'
import {
  TokenPool,
  sanitizeToken,
} from '../../core/token-pooling/token-pool.js'
import { getUserAgent } from '../../core/base-service/got-config.js'
import { nonNegativeInteger } from '../validators.js'
import { ImproperlyConfigured } from '../index.js'

const userAgent = getUserAgent()

const headerSchema = Joi.object({
  'x-ratelimit-limit': nonNegativeInteger,
  'x-ratelimit-remaining': nonNegativeInteger,
  'x-ratelimit-reset': nonNegativeInteger,
})
  .required()
  .unknown(true)

const bodySchema = Joi.object({
  data: Joi.object({
    rateLimit: Joi.object({
      limit: nonNegativeInteger,
      remaining: nonNegativeInteger,
      resetAt: Joi.date().iso(),
    })
      .required()
      .unknown(true),
  })
    .required()
    .unknown(true),
})
  .required()
  .unknown(true)

// Provides an interface to the Github API. Manages the base URL.
class GithubApiProvider {
  static AUTH_TYPES = {
    NO_AUTH: 'No Auth',
    GLOBAL_TOKEN: 'Global Token',
    TOKEN_POOL: 'Token Pool',
  }

  // reserveFraction: The amount of much of a token's quota we avoid using, to
  //   reserve it for the user.
  constructor({
    baseUrl,
    authType = this.constructor.AUTH_TYPES.NO_AUTH,
    onTokenInvalidated = tokenString => {},
    globalToken,
    reserveFraction = 0.25,
    restApiVersion,
  }) {
    Object.assign(this, {
      baseUrl,
      authType,
      onTokenInvalidated,
      globalToken,
      reserveFraction,
    })

    this.metricInstance = undefined

    if (this.authType === this.constructor.AUTH_TYPES.TOKEN_POOL) {
      this.standardTokens = new TokenPool({ batchSize: 25 })
      this.searchTokens = new TokenPool({ batchSize: 5 })
      this.graphqlTokens = new TokenPool({ batchSize: 25 })
    }
    this.restApiVersion = restApiVersion
  }

  addToken(tokenString) {
    if (this.authType === this.constructor.AUTH_TYPES.TOKEN_POOL) {
      this.standardTokens.add(tokenString)
      this.searchTokens.add(tokenString)
      this.graphqlTokens.add(tokenString)
    } else {
      throw Error('When not using a token pool, do not provide tokens')
    }
  }

  getTokenDebugInfo({ sanitize = true } = {}) {
    if (this.authType === this.constructor.AUTH_TYPES.TOKEN_POOL) {
      return {
        standardTokens: this.standardTokens.serializeDebugInfo({ sanitize }),
        searchTokens: this.searchTokens.serializeDebugInfo({ sanitize }),
        graphqlTokens: this.graphqlTokens.serializeDebugInfo({ sanitize }),
      }
    } else {
      return {}
    }
  }

  getV3RateLimitFromHeaders(headers) {
    const h = Joi.attempt(headers, headerSchema)
    return {
      rateLimit: h['x-ratelimit-limit'],
      totalUsesRemaining: h['x-ratelimit-remaining'],
      nextReset: h['x-ratelimit-reset'],
    }
  }

  getV4RateLimitFromBody(body) {
    const b = Joi.attempt(body, bodySchema)
    return {
      rateLimit: b.data.rateLimit.limit,
      totalUsesRemaining: b.data.rateLimit.remaining,
      nextReset: Date.parse(b.data.rateLimit.resetAt) / 1000,
    }
  }

  updateToken({ token, url, res }) {
    let rateLimit, totalUsesRemaining, nextReset
    if (url.startsWith('/graphql')) {
      try {
        const parsedBody = JSON.parse(res.body)

        if ('message' in parsedBody && !('data' in parsedBody)) {
          if (parsedBody.message === 'Sorry. Your account was suspended.') {
            this.invalidateToken(token, 'account_suspended')
            return
          }
        }

        ;({ rateLimit, totalUsesRemaining, nextReset } =
          this.getV4RateLimitFromBody(parsedBody))
      } catch (e) {
        console.error(
          `Could not extract rate limit info from response body ${res.body}`,
        )
        log.error(e)
        return
      }
    } else {
      try {
        ;({ rateLimit, totalUsesRemaining, nextReset } =
          this.getV3RateLimitFromHeaders(res.headers))
      } catch (e) {
        const logHeaders = {
          'x-ratelimit-limit': res.headers['x-ratelimit-limit'],
          'x-ratelimit-remaining': res.headers['x-ratelimit-remaining'],
          'x-ratelimit-reset': res.headers['x-ratelimit-reset'],
        }
        console.error(
          `Invalid GitHub rate limit headers ${JSON.stringify(
            logHeaders,
            undefined,
            2,
          )}`,
        )
        log.error(e)
        return
      }
    }

    const reserve = Math.ceil(this.reserveFraction * rateLimit)
    const usesRemaining = totalUsesRemaining - reserve

    token.update(usesRemaining, nextReset)
  }

  invalidateToken(token, reason) {
    log.log(
      `GitHub token invalidated and removed from pool (reason: ${reason}, token: ${sanitizeToken(
        token.id,
      )})`,
    )
    if (this.metricInstance) {
      this.metricInstance.noteGithubTokenInvalidation({ reason })
    }
    token.invalidate()
    this.onTokenInvalidated(token.id)
  }

  tokenForUrl(url) {
    if (url.startsWith('/search')) {
      return this.searchTokens.next()
    } else if (url.startsWith('/graphql')) {
      return this.graphqlTokens.next()
    } else {
      return this.standardTokens.next()
    }
  }

  async fetch(requestFetcher, url, options = {}) {
    const { baseUrl } = this

    let token
    let tokenString
    if (this.authType === this.constructor.AUTH_TYPES.TOKEN_POOL) {
      try {
        token = this.tokenForUrl(url)
      } catch (e) {
        log.error(e)
        throw new ImproperlyConfigured({
          prettyMessage: 'Unable to select next GitHub token from pool',
        })
      }
      tokenString = token.id
    } else if (this.authType === this.constructor.AUTH_TYPES.GLOBAL_TOKEN) {
      tokenString = this.globalToken
    }

    const mergedOptions = {
      ...options,
      ...{
        headers: {
          'User-Agent': userAgent,
          'X-GitHub-Api-Version': this.restApiVersion,
          ...options.headers,
        },
      },
    }
    if (
      this.authType === this.constructor.AUTH_TYPES.TOKEN_POOL ||
      this.authType === this.constructor.AUTH_TYPES.GLOBAL_TOKEN
    ) {
      mergedOptions.headers.Authorization = `token ${tokenString}`
    }

    const response = await requestFetcher(`${baseUrl}${url}`, mergedOptions)
    if (this.authType === this.constructor.AUTH_TYPES.TOKEN_POOL) {
      if (response.res.statusCode === 401) {
        this.invalidateToken(token, 'http_401')
      } else if (response.res.statusCode < 500) {
        this.updateToken({ token, url, res: response.res })
      }
    }
    return response
  }
}

export default GithubApiProvider

1	import Joi from 'joi'	4✔
2	import log from '../../core/server/log.js'	4✔
3	import {	4✔
4	TokenPool,	4✔
5	sanitizeToken,	4✔
6	} from '../../core/token-pooling/token-pool.js'	4✔
7	import { getUserAgent } from '../../core/base-service/got-config.js'	4✔
8	import { nonNegativeInteger } from '../validators.js'	4✔
9	import { ImproperlyConfigured } from '../index.js'	4✔
10		4✔
11	const userAgent = getUserAgent()	4✔
12		4✔
13	const headerSchema = Joi.object({	4✔
14	'x-ratelimit-limit': nonNegativeInteger,	4✔
15	'x-ratelimit-remaining': nonNegativeInteger,	4✔
16	'x-ratelimit-reset': nonNegativeInteger,	4✔
17	})	4✔
18	.required()	4✔
19	.unknown(true)	4✔
20		4✔
21	const bodySchema = Joi.object({	4✔
22	data: Joi.object({	4✔
23	rateLimit: Joi.object({	4✔
24	limit: nonNegativeInteger,	4✔
25	remaining: nonNegativeInteger,	4✔
26	resetAt: Joi.date().iso(),	4✔
27	})	4✔
28	.required()	4✔
29	.unknown(true),	4✔
30	})	4✔
31	.required()	4✔
32	.unknown(true),	4✔
33	})	4✔
34	.required()	4✔
35	.unknown(true)	4✔
36		4✔
37	// Provides an interface to the Github API. Manages the base URL.	4✔
38	class GithubApiProvider {	4✔
39	static AUTH_TYPES = {	4✔
40	NO_AUTH: 'No Auth',	4✔
41	GLOBAL_TOKEN: 'Global Token',	4✔
42	TOKEN_POOL: 'Token Pool',	4✔
43	}	4✔
44		4✔
45	// reserveFraction: The amount of much of a token's quota we avoid using, to	4✔
46	// reserve it for the user.	4✔
47	constructor({	4✔
48	baseUrl,	33✔
49	authType = this.constructor.AUTH_TYPES.NO_AUTH,	33✔
50	onTokenInvalidated = tokenString => {},	33✔
51	globalToken,	33✔
52	reserveFraction = 0.25,	33✔
53	restApiVersion,	33✔
54	}) {	33✔
55	Object.assign(this, {	33✔
56	baseUrl,	33✔
57	authType,	33✔
58	onTokenInvalidated,	33✔
59	globalToken,	33✔
60	reserveFraction,	33✔
61	})	33✔
62		33✔
63	this.metricInstance = undefined	33✔
64		33✔
65	if (this.authType === this.constructor.AUTH_TYPES.TOKEN_POOL) {	33✔
66	this.standardTokens = new TokenPool({ batchSize: 25 })	13✔
67	this.searchTokens = new TokenPool({ batchSize: 5 })	13✔
68	this.graphqlTokens = new TokenPool({ batchSize: 25 })	13✔
69	}	13✔
70	this.restApiVersion = restApiVersion	33✔
71	}	33✔
72		4✔
73	addToken(tokenString) {	4✔
74	if (this.authType === this.constructor.AUTH_TYPES.TOKEN_POOL) {	1✔
75	this.standardTokens.add(tokenString)	1✔
76	this.searchTokens.add(tokenString)	1✔
77	this.graphqlTokens.add(tokenString)	1✔
78	} else {	1!
UNCOV 79	throw Error('When not using a token pool, do not provide tokens')	×
UNCOV 80	}	×
81	}	1✔
82		4✔
83	getTokenDebugInfo({ sanitize = true } = {}) {	4✔
84	if (this.authType === this.constructor.AUTH_TYPES.TOKEN_POOL) {	×
85	return {	×
UNCOV 86	standardTokens: this.standardTokens.serializeDebugInfo({ sanitize }),	×
UNCOV 87	searchTokens: this.searchTokens.serializeDebugInfo({ sanitize }),	×
UNCOV 88	graphqlTokens: this.graphqlTokens.serializeDebugInfo({ sanitize }),	×
89	}	×
90	} else {	×
91	return {}	×
92	}	×
93	}	×
94		4✔
95	getV3RateLimitFromHeaders(headers) {	4✔
96	const h = Joi.attempt(headers, headerSchema)	13✔
97	return {	13✔
98	rateLimit: h['x-ratelimit-limit'],	13✔
99	totalUsesRemaining: h['x-ratelimit-remaining'],	13✔
100	nextReset: h['x-ratelimit-reset'],	13✔
101	}	13✔
102	}	13✔
103		4✔
104	getV4RateLimitFromBody(body) {	4✔
105	const b = Joi.attempt(body, bodySchema)	2✔
106	return {	2✔
107	rateLimit: b.data.rateLimit.limit,	2✔
108	totalUsesRemaining: b.data.rateLimit.remaining,	2✔
109	nextReset: Date.parse(b.data.rateLimit.resetAt) / 1000,	2✔
110	}	2✔
111	}	2✔
112		4✔
113	updateToken({ token, url, res }) {	4✔
114	let rateLimit, totalUsesRemaining, nextReset	16✔
115	if (url.startsWith('/graphql')) {	16✔
116	try {	3✔
117	const parsedBody = JSON.parse(res.body)	3✔
118		3✔
119	if ('message' in parsedBody && !('data' in parsedBody)) {	3✔
120	if (parsedBody.message === 'Sorry. Your account was suspended.') {	1✔
121	this.invalidateToken(token, 'account_suspended')	1✔
122	return	1✔
123	}	1✔
124	}	1✔
125		2✔
126	;({ rateLimit, totalUsesRemaining, nextReset } =	2✔
127	this.getV4RateLimitFromBody(parsedBody))	2✔
128	} catch (e) {	3!
UNCOV 129	console.error(	×
UNCOV 130	`Could not extract rate limit info from response body ${res.body}`,	×
UNCOV 131	)	×
UNCOV 132	log.error(e)	×
UNCOV 133	return	×
134	}	×
135	} else {	16✔
136	try {	13✔
137	;({ rateLimit, totalUsesRemaining, nextReset } =	13✔
138	this.getV3RateLimitFromHeaders(res.headers))	13✔
139	} catch (e) {	13!
UNCOV 140	const logHeaders = {	×
UNCOV 141	'x-ratelimit-limit': res.headers['x-ratelimit-limit'],	×
UNCOV 142	'x-ratelimit-remaining': res.headers['x-ratelimit-remaining'],	×
UNCOV 143	'x-ratelimit-reset': res.headers['x-ratelimit-reset'],	×
UNCOV 144	}	×
145	console.error(	×
146	`Invalid GitHub rate limit headers ${JSON.stringify(	×
147	logHeaders,	×
148	undefined,	×
149	2,	×
150	)}`,	×
151	)	×
152	log.error(e)	×
153	return	×
154	}	×
155	}	13✔
156		15✔
157	const reserve = Math.ceil(this.reserveFraction * rateLimit)	15✔
158	const usesRemaining = totalUsesRemaining - reserve	15✔
159		15✔
160	token.update(usesRemaining, nextReset)	15✔
161	}	16✔
162		4✔
163	invalidateToken(token, reason) {	4✔
164	log.log(	3✔
165	`GitHub token invalidated and removed from pool (reason: ${reason}, token: ${sanitizeToken(	3✔
166	token.id,	3✔
167	)})`,	3✔
168	)	3✔
169	if (this.metricInstance) {	3!
NEW 170	this.metricInstance.noteGithubTokenInvalidation({ reason })	×
NEW 171	}	×
172	token.invalidate()	3✔
173	this.onTokenInvalidated(token.id)	3✔
174	}	3✔
175		4✔
176	tokenForUrl(url) {	4✔
177	if (url.startsWith('/search')) {	22✔
178	return this.searchTokens.next()	1✔
179	} else if (url.startsWith('/graphql')) {	22✔
180	return this.graphqlTokens.next()	5✔
181	} else {	21✔
182	return this.standardTokens.next()	16✔
183	}	16✔
184	}	22✔
185		4✔
186	async fetch(requestFetcher, url, options = {}) {	4✔
187	const { baseUrl } = this	368✔
188		368✔
189	let token	368✔
190	let tokenString	368✔
191	if (this.authType === this.constructor.AUTH_TYPES.TOKEN_POOL) {	368✔
192	try {	22✔
193	token = this.tokenForUrl(url)	22✔
194	} catch (e) {	22!
UNCOV 195	log.error(e)	×
UNCOV 196	throw new ImproperlyConfigured({	×
UNCOV 197	prettyMessage: 'Unable to select next GitHub token from pool',	×
UNCOV 198	})	×
UNCOV 199	}	×
200	tokenString = token.id	22✔
201	} else if (this.authType === this.constructor.AUTH_TYPES.GLOBAL_TOKEN) {	368✔
202	tokenString = this.globalToken	336✔
203	}	336✔
204		368✔
205	const mergedOptions = {	368✔
206	...options,	368✔
207	...{	368✔
208	headers: {	368✔
209	'User-Agent': userAgent,	368✔
210	'X-GitHub-Api-Version': this.restApiVersion,	368✔
211	...options.headers,	368✔
212	},	368✔
213	},	368✔
214	}	368✔
215	if (	368✔
216	this.authType === this.constructor.AUTH_TYPES.TOKEN_POOL \|\|	368✔
217	this.authType === this.constructor.AUTH_TYPES.GLOBAL_TOKEN	346✔
218	) {	368✔
219	mergedOptions.headers.Authorization = `token ${tokenString}`	358✔
220	}	358✔
221		368✔
222	const response = await requestFetcher(`${baseUrl}${url}`, mergedOptions)	368✔
223	if (this.authType === this.constructor.AUTH_TYPES.TOKEN_POOL) {	367✔
224	if (response.res.statusCode === 401) {	21✔
225	this.invalidateToken(token, 'http_401')	2✔
226	} else if (response.res.statusCode < 500) {	21✔
227	this.updateToken({ token, url, res: response.res })	16✔
228	}	16✔
229	}	21✔
230	return response	367✔
231	}	368✔
232	}	4✔
233		4✔
234	export default GithubApiProvider	4✔

badges / shields / 27606648281

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous