14112918855

Committed 27 Mar 2025 05:25PM UTC coverage: 79.429% (-1.2%) from 80.637%

Build # 14112918855

Build Type

push

github

Committed by

web-flow

Commit Message

Merge pull request #27 from bmaupin/feat/validation-minimum-score

Run Details

62 of 95 branches covered (65.26%)

Branch coverage included in aggregate %.

2 of 5 new or added lines in 2 files covered. (40.0%)

1 existing line in 1 file now uncovered.

216 of 255 relevant lines covered (84.71%)

48.23 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

66.2

/src/StackOverflow.ts

'use strict';

import https from 'https';
import util from 'util';
import zlib from 'zlib';

// Uses a custom filter that only returns backoff, quota_remaining, and total
// (https://api.stackexchange.com/docs/create-filter#unsafe=false&filter=!.UE8F0bVg4M-_Ii4&run=true)
const API_URL =
  'https://api.stackexchange.com/2.2/search?fromdate=%s&todate=%s&site=stackoverflow&tagged=%s&filter=!.UE8F0bVg4M-_Ii4';

interface StackOverflowData {
  backoff?: number;
  quota_remaining: number;
  total: number;
}

export default class StackOverflow {
  private apiKey?: string;

  constructor(apiKey?: string) {
    this.apiKey = apiKey;
  }

  // Get the number of tags between fromDate (inclusive) and toDate (exclusive)
  public async getScore(
    languageName: string,
    fromDate: Date,
    toDate: Date
  ): Promise<number> {
    let url = this.buildUrl(languageName, fromDate, toDate);
    let body = await this.callApi(url);

    StackOverflow.handleApiLimits(body);

    return body.total;
  }

  private buildUrl(languageName: string, fromDate: Date, toDate: Date): string {
    let url = util.format(
      API_URL,
      StackOverflow.encodeDate(fromDate),
      StackOverflow.encodeDate(toDate),
      StackOverflow.encodeLanguageName(languageName)
    );
    url = this.addApiKey(url);

    return url;
  }

  private static encodeDate(date: Date): number {
    // All dates in the API are in unix epoch time, which is the number of seconds since midnight UTC January 1st, 1970.
    // (https://api.stackexchange.com/docs/dates)
    return Math.floor(Number(date) / 1000);
  }

  private static encodeLanguageName(languageName: string): string {
    return encodeURIComponent(languageName.toLowerCase().replace(/ /g, '-'));
  }

  private addApiKey(url: string): string {
    const KEY_PARAMETER = '&key=';
    if (typeof this.apiKey !== 'undefined') {
      url = `${url}${KEY_PARAMETER}${this.apiKey}`;
    }

    return url;
  }

  private async callApi(url: string): Promise<StackOverflowData> {
    const options = {
      headers: {
        // Stackoverflow API now requires a user agent
        'User-Agent': 'node.js',
      },
    };
    const bodyJson = await this.httpsRequest(options, url);
    return JSON.parse(bodyJson);
  }

  // Based on https://stackoverflow.com/a/38543075/399105
  private httpsRequest(
    options: https.RequestOptions,
    url: string
  ): Promise<string> {
    return new Promise((resolve, reject) => {
      const request = https.request(url, options, async (response) => {
        if (
          response.statusCode &&
          (response.statusCode < 200 || response.statusCode >= 300)
        ) {
          if (response.statusCode === 400) {
            console.warn(
              'Warning: Stackoverflow API daily limit exceeded or API key incorrect'
            );
          } else if (response.statusCode === 403) {
            console.warn(
              'Warning: Stackoverflow API returned 403; please set the user agent'
            );
          } else if (response.statusCode === 503) {
            // Stackoverflow might throw a 503 if it feels there are too many requests
            console.warn(
              'Warning: Stackoverflow API returned 503; wait a bit and try again'
            );
          }
          reject(
            new Error(
              `statusCode=${response.statusCode}, URL=${url.replace(
                this.apiKey || '',
                'REDACTED'
              )}`
            )
          );
        }

        const body = [] as Buffer[];
        response.on('data', function (chunk) {
          body.push(chunk);
        });

        response.on('end', () => {
          switch (response.headers['content-encoding']) {
            case 'gzip':
              zlib.gunzip(Buffer.concat(body), (error, uncompressedData) => {
                resolve(uncompressedData.toString());
              });
              break;
            case undefined:
              resolve(Buffer.concat(body).toString());
              break;
            default:
              // If we get here it's likely due to another issue, normally a 503 error due to too many requests
              reject(
                new Error(
                  `Incorrect content encoding: ${response.headers['content-encoding']}`
                )
              );
              break;
          }
        });
      });

      request.on('error', (err: NodeJS.ErrnoException) => {
        // Stack Overflow might close the connection for any outstanding requests and return a 503 for new ones if it
        // feels there are too many requests
        if (err.code === 'ECONNRESET') {
          console.warn(
            'Warning: Stack Overflow API closed connection; wait a bit and try again'
          );
        }
        // Use the original message and code but our stack trace since the original stack trace won't point back to
        // here
        reject(new Error(`${err.message} (${err.code})`));
      });

      request.end();
    });
  }

  private static handleApiLimits(body: StackOverflowData) {
    if (body.quota_remaining <= 0) {
      throw new Error('Stack Overflow API daily limit exceeded');
    }

    // The backoff field never seems to be sent, but throw if it happens so we can add logic for it (https://stackapps.com/a/3057/41977)
    if (body.backoff) {
      throw new Error(
        `Stack Overflow API backoff field not handled: ${body.backoff}`
      );
    }
  }
}

1	'use strict';
2
3	import https from 'https';	2✔
4	import util from 'util';	2✔
5	import zlib from 'zlib';	2✔
6
7	// Uses a custom filter that only returns backoff, quota_remaining, and total
8	// (https://api.stackexchange.com/docs/create-filter#unsafe=false&filter=!.UE8F0bVg4M-_Ii4&run=true)
9	const API_URL =
10	'https://api.stackexchange.com/2.2/search?fromdate=%s&todate=%s&site=stackoverflow&tagged=%s&filter=!.UE8F0bVg4M-_Ii4';	2✔
11
12	interface StackOverflowData {
13	backoff?: number;
14	quota_remaining: number;
15	total: number;
16	}
17
18	export default class StackOverflow {	2✔
19	private apiKey?: string;
20
21	constructor(apiKey?: string) {
22	this.apiKey = apiKey;	6✔
23	}
24
25	// Get the number of tags between fromDate (inclusive) and toDate (exclusive)
26	public async getScore(
27	languageName: string,
28	fromDate: Date,
29	toDate: Date
30	): Promise<number> {
31	let url = this.buildUrl(languageName, fromDate, toDate);	59✔
32	let body = await this.callApi(url);	59✔
33
34	StackOverflow.handleApiLimits(body);	58✔
35
36	return body.total;	58✔
37	}
38
39	private buildUrl(languageName: string, fromDate: Date, toDate: Date): string {
40	let url = util.format(	59✔
41	API_URL,
42	StackOverflow.encodeDate(fromDate),
43	StackOverflow.encodeDate(toDate),
44	StackOverflow.encodeLanguageName(languageName)
45	);
46	url = this.addApiKey(url);	59✔
47
48	return url;	59✔
49	}
50
51	private static encodeDate(date: Date): number {
52	// All dates in the API are in unix epoch time, which is the number of seconds since midnight UTC January 1st, 1970.
53	// (https://api.stackexchange.com/docs/dates)
54	return Math.floor(Number(date) / 1000);	118✔
55	}
56
57	private static encodeLanguageName(languageName: string): string {
58	return encodeURIComponent(languageName.toLowerCase().replace(/ /g, '-'));	59✔
59	}
60
61	private addApiKey(url: string): string {
62	const KEY_PARAMETER = '&key=';	59✔
63	if (typeof this.apiKey !== 'undefined') {	59✔
64	url = `${url}${KEY_PARAMETER}${this.apiKey}`;	58✔
65	}
66
67	return url;	59✔
68	}
69
70	private async callApi(url: string): Promise<StackOverflowData> {
71	const options = {	59✔
72	headers: {
73	// Stackoverflow API now requires a user agent
74	'User-Agent': 'node.js',
75	},
76	};
77	const bodyJson = await this.httpsRequest(options, url);	59✔
78	return JSON.parse(bodyJson);	58✔
79	}
80
81	// Based on https://stackoverflow.com/a/38543075/399105
82	private httpsRequest(
83	options: https.RequestOptions,
84	url: string
85	): Promise<string> {
86	return new Promise((resolve, reject) => {	59✔
87	const request = https.request(url, options, async (response) => {	59✔
88	if (	59✔
89	response.statusCode &&	177✔
90	(response.statusCode < 200 \|\| response.statusCode >= 300)
91	) {
92	if (response.statusCode === 400) {	1!
93	console.warn(	1✔
94	'Warning: Stackoverflow API daily limit exceeded or API key incorrect'
95	);
NEW 96	} else if (response.statusCode === 403) {	×
NEW 97	console.warn(	×
98	'Warning: Stackoverflow API returned 403; please set the user agent'
99	);
UNCOV 100	} else if (response.statusCode === 503) {	×
101	// Stackoverflow might throw a 503 if it feels there are too many requests
102	console.warn(	×
103	'Warning: Stackoverflow API returned 503; wait a bit and try again'
104	);
105	}
106	reject(	1✔
107	new Error(
108	`statusCode=${response.statusCode}, URL=${url.replace(
109	this.apiKey \|\| '',	1!
110	'REDACTED'
111	)}`
112	)
113	);
114	}
115
116	const body = [] as Buffer[];	59✔
117	response.on('data', function (chunk) {	59✔
118	body.push(chunk);	59✔
119	});
120
121	response.on('end', () => {	59✔
122	switch (response.headers['content-encoding']) {	59!
123	case 'gzip':
124	zlib.gunzip(Buffer.concat(body), (error, uncompressedData) => {	×
125	resolve(uncompressedData.toString());	×
126	});
127	break;	×
128	case undefined:
129	resolve(Buffer.concat(body).toString());	59✔
130	break;	59✔
131	default:
132	// If we get here it's likely due to another issue, normally a 503 error due to too many requests
133	reject(	×
134	new Error(
135	`Incorrect content encoding: ${response.headers['content-encoding']}`
136	)
137	);
138	break;	×
139	}
140	});
141	});
142
143	request.on('error', (err: NodeJS.ErrnoException) => {	59✔
144	// Stack Overflow might close the connection for any outstanding requests and return a 503 for new ones if it
145	// feels there are too many requests
146	if (err.code === 'ECONNRESET') {	×
147	console.warn(	×
148	'Warning: Stack Overflow API closed connection; wait a bit and try again'
149	);
150	}
151	// Use the original message and code but our stack trace since the original stack trace won't point back to
152	// here
153	reject(new Error(`${err.message} (${err.code})`));	×
154	});
155
156	request.end();	59✔
157	});
158	}
159
160	private static handleApiLimits(body: StackOverflowData) {
161	if (body.quota_remaining <= 0) {	58!
162	throw new Error('Stack Overflow API daily limit exceeded');	×
163	}
164
165	// The backoff field never seems to be sent, but throw if it happens so we can add logic for it (https://stackapps.com/a/3057/41977)
166	if (body.backoff) {	58!
167	throw new Error(	×
168	`Stack Overflow API backoff field not handled: ${body.backoff}`
169	);
170	}
171	}
172	}

bmaupin / langtrends-data / 14112918855

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous