10901687640

Committed 17 Sep 2024 11:00AM UTC coverage: 91.575% (-8.4%) from 100.0%

Build # 10901687640

Build Type

Pull #36

github

Committed by

web-flow

Commit Message

Merge d5f8b28d9 into 21c66bfd7

Pull Request Pull Request #36: Add explicit disallow feature

Run Details

103 of 116 branches covered (88.79%)

Branch coverage included in aggregate %.

13 of 23 new or added lines in 1 file covered. (56.52%)

7 existing lines in 1 file now uncovered.

147 of 157 relevant lines covered (93.63%)

242560.76 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

91.48

/Robots.js

/**
 * Trims the white space from the start and end of the line.
 *
 * If the line is an array it will strip the white space from
 * the start and end of each element of the array.
 *
 * @param  {string|Array} line
 * @return {string|Array}
 * @private
 */
function trimLine(line) {
        if (!line) {
                return null;
        }

        if (Array.isArray(line)) {
                return line.map(trimLine);
        }

        return String(line).trim();
}

/**
 * Remove comments from lines
 *
 * @param {string} line
 * @return {string}
 * @private
 */
function removeComments(line) {
        var commentStartIndex = line.indexOf('#');
        if (commentStartIndex > -1) {
                return line.substr(0, commentStartIndex);
        }

        return line;
}

/**
 * Splits a line at the first occurrence of :
 *
 * @param  {string} line
 * @return {Array.<string>}
 * @private
 */
function splitLine(line) {
        var idx = String(line).indexOf(':');

        if (!line || idx < 0) {
                return null;
        }

        return [line.slice(0, idx), line.slice(idx + 1)];
}

/**
 * Normalises the user-agent string by converting it to
 * lower case and removing any version numbers.
 *
 * @param  {string} userAgent
 * @return {string}
 * @private
 */
function formatUserAgent(userAgent) {
        var formattedUserAgent = userAgent.toLowerCase();

        // Strip the version number from robot/1.0 user agents
        var idx = formattedUserAgent.indexOf('/');
        if (idx > -1) {
                formattedUserAgent = formattedUserAgent.substr(0, idx);
        }

        return formattedUserAgent.trim();
}

/**
 * Normalises the URL encoding of a path by encoding
 * unicode characters.
 *
 * @param {string} path
 * @return {string}
 * @private
 */
function normaliseEncoding(path) {
        try {
                return urlEncodeToUpper(encodeURI(path).replace(/%25/g, '%'));
        } catch (e) {
                return path;
        }
}

/**
 * Convert URL encodings to support case.
 *
 * e.g.: %2a%ef becomes %2A%EF
 *
 * @param {string} path
 * @return {string}
 * @private
 */
function urlEncodeToUpper(path) {
        return path.replace(/%[0-9a-fA-F]{2}/g, function (match) {
                return match.toUpperCase();
        });
}

/**
 * Matches a pattern with the specified path
 *
 * Uses same algorithm to match patterns as the Google implementation in
 * google/robotstxt so it should be consistent with the spec.
 *
 * @see https://github.com/google/robotstxt/blob/f465f0ede81099dd8bc4aeb2966b3a892bd488b3/robots.cc#L74
 * @param {string} pattern
 * @param {string} path
 * @return {boolean}
 * @private
 */
function matches(pattern, path) {
        // I've added extra comments to try make this easier to understand

        // Stores the lengths of all the current matching substrings.
        // Maximum number of possible matching lengths is every length in path plus
        // 1 to handle 0 length too (if pattern starts with * which is zero or more)
        var matchingLengths = new Array(path.length + 1);
        var numMatchingLengths = 1;

        // Initially longest match is 0
        matchingLengths[0] = 0;

        for (var p = 0; p < pattern.length; p++) {
                // If $ is at the end of pattern then we must match the whole path.
                // Which is true if the longest matching length matches path length
                if (pattern[p] === '$' && p + 1 === pattern.length) {
                        return matchingLengths[numMatchingLengths - 1] === path.length;
                }

                // Handle wildcards
                if (pattern[p] == '*') {
                        // Wildcard so all substrings minus the current smallest matching
                        // length are matches
                        numMatchingLengths = path.length - matchingLengths[0] + 1;

                        // Update matching lengths to include the smallest all the way up
                        // to numMatchingLengths
                        // Don't update smallest possible match as * matches zero or more
                        // so the smallest current match is also valid
                        for (var i = 1; i < numMatchingLengths; i++) {
                                matchingLengths[i] = matchingLengths[i - 1] + 1;
                        }
                } else {
                        // Check the char at the matching length matches the pattern, if it
                        // does increment it and add it as a valid length, ignore if not.
                        var numMatches = 0;
                        for (var i = 0; i < numMatchingLengths; i++) {
                                if (
                                        matchingLengths[i] < path.length &&
                                        path[matchingLengths[i]] === pattern[p]
                                ) {
                                        matchingLengths[numMatches++] = matchingLengths[i] + 1;
                                }
                        }

                        // No paths matched the current pattern char so not a match
                        if (numMatches == 0) {
                                return false;
                        }

                        numMatchingLengths = numMatches;
                }
        }

        return true;
}

function parseRobots(contents, robots) {
        var newlineRegex = /\r\n|\r|\n/;
        var lines = contents
                .split(newlineRegex)
                .map(removeComments)
                .map(splitLine)
                .map(trimLine);

        var currentUserAgents = [];
        var isNoneUserAgentState = true;
        for (var i = 0; i < lines.length; i++) {
                var line = lines[i];

                if (!line || !line[0]) {
                        continue;
                }

                switch (line[0].toLowerCase()) {
                        case 'user-agent':
                                if (isNoneUserAgentState) {
                                        currentUserAgents.length = 0;
                                }

                                if (line[1]) {
                                        currentUserAgents.push(formatUserAgent(line[1]));
                                }
                                break;
                        case 'disallow':
                                robots.addRule(currentUserAgents, line[1], false, i + 1);
                                break;
                        case 'allow':
                                robots.addRule(currentUserAgents, line[1], true, i + 1);
                                break;
                        case 'crawl-delay':
                                robots.setCrawlDelay(currentUserAgents, line[1]);
                                break;
                        case 'sitemap':
                                if (line[1]) {
                                        robots.addSitemap(line[1]);
                                }
                                break;
                        case 'host':
                                if (line[1]) {
                                        robots.setPreferredHost(line[1].toLowerCase());
                                }
                                break;
                }

                isNoneUserAgentState = line[0].toLowerCase() !== 'user-agent';
        }
}

/**
 * Returns if a pattern is allowed by the specified rules.
 *
 * @param  {string}  path
 * @param  {Array.<Object>}  rules
 * @return {Object?}
 * @private
 */
function findRule(path, rules) {
        var matchedRule = null;

        for (var i = 0; i < rules.length; i++) {
                var rule = rules[i];

                if (!matches(rule.pattern, path)) {
                        continue;
                }

                // The longest matching rule takes precedence
                // If rules are the same length then allow takes precedence
                if (!matchedRule || rule.pattern.length > matchedRule.pattern.length) {
                        matchedRule = rule;
                } else if (
                        rule.pattern.length == matchedRule.pattern.length &&
                        rule.allow &&
                        !matchedRule.allow
                ) {
                        matchedRule = rule;
                }
        }

        return matchedRule;
}

/**
 * Converts provided string into an URL object.
 *
 * Will return null if provided string is not a valid URL.
 *
 * @param {string} url
 * @return {?URL}
 * @private
 */
function parseUrl(url) {
        try {
                // Specify a URL to be used with relative paths
                // Using non-existent subdomain so can never cause conflict unless
                // trying to crawl it but doesn't exist and even if tried worst that can
                // happen is it allows relative URLs on it.
                var url = new URL(url, 'http://robots-relative.samclarke.com/');

                if (!url.port) {
                        url.port = url.protocol === 'https:' ? 443 : 80;
                }

                return url;
        } catch (e) {
                return null;
        }
}

function Robots(url, contents) {
        this._url = parseUrl(url) || {};
        this._rules = Object.create(null);
        this._sitemaps = [];
        this._preferredHost = null;

        parseRobots(contents || '', this);
}

/**
 * Adds the specified allow/deny rule to the rules
 * for the specified user-agents.
 *
 * @param {Array.<string>} userAgents
 * @param {string} pattern
 * @param {boolean} allow
 * @param {number} [lineNumber] Should use 1-based indexing
 */
Robots.prototype.addRule = function (userAgents, pattern, allow, lineNumber) {
        var rules = this._rules;

        userAgents.forEach(function (userAgent) {
                rules[userAgent] = rules[userAgent] || [];

                if (!pattern) {
                        return;
                }

                rules[userAgent].push({
                        pattern: normaliseEncoding(pattern),
                        allow: allow,
                        lineNumber: lineNumber
                });
        });
};

/**
 * Adds the specified delay to the specified user agents.
 *
 * @param {Array.<string>} userAgents
 * @param {string} delayStr
 */
Robots.prototype.setCrawlDelay = function (userAgents, delayStr) {
        var rules = this._rules;
        var delay = Number(delayStr);

        userAgents.forEach(function (userAgent) {
                rules[userAgent] = rules[userAgent] || [];

                if (isNaN(delay)) {
                        return;
                }

                rules[userAgent].crawlDelay = delay;
        });
};

/**
 * Add a sitemap
 *
 * @param {string} url
 */
Robots.prototype.addSitemap = function (url) {
        this._sitemaps.push(url);
};

/**
 * Sets the preferred host name
 *
 * @param {string} url
 */
Robots.prototype.setPreferredHost = function (url) {
        this._preferredHost = url;
};

Robots.prototype._getRule = function (url, ua, explicit) {
        var parsedUrl = parseUrl(url) || {};
        var userAgent = formatUserAgent(ua || '*');

        // The base URL must match otherwise this robots.txt is not valid for it.
        if (
                parsedUrl.protocol !== this._url.protocol ||
                parsedUrl.hostname !== this._url.hostname ||
                parsedUrl.port !== this._url.port
        ) {
                return;
        }

        var rules = this._rules[userAgent];
        if (!explicit) {
                rules = rules || this._rules['*']
        }
        rules = rules || []

        var path = urlEncodeToUpper(parsedUrl.pathname + parsedUrl.search);
        var rule = findRule(path, rules);

        return rule;
};

/**
 * Returns true if allowed, false if not allowed.
 *
 * Will return undefined if the URL is not valid for
 * this robots.txt file.
 *
 * @param  {string}  url
 * @param  {string?}  ua
 * @return {boolean?}
 */
Robots.prototype.isAllowed = function (url, ua) {
        var rule = this._getRule(url, ua);

        if (typeof rule === 'undefined') {
                return;
        }

        return !rule || rule.allow;
};

/**
 * Returns the line number of the matching directive for the specified
 * URL and user-agent if any.
 *
 * The line numbers start at 1 and go up (1-based indexing).
 *
 * Return -1 if there is no matching directive. If a rule is manually
 * added without a lineNumber then this will return undefined for that
 * rule.
 *
 * @param  {string}  url
 * @param  {string?}  ua
 * @return {number?}
 */
Robots.prototype.getMatchingLineNumber = function (url, ua) {
        var rule = this._getRule(url, ua);

        return rule ? rule.lineNumber : -1;
};

/**
 * In standard mode, it returns the opposite of is allowed().
 * In explicit mode, it will return:
 *  - true if the the agent is explicitly disallowed (wildcard non included),
 *         - throws an error if the user agent is not specified,
 *         - and false otherwise.
 * @param  {string}  url
 * @param  {string}  ua
 * @return {boolean}
 */
Robots.prototype.isDisallowed = function (url, ua, explicit) {
        if ((explicit === true) && (ua === undefined)) {
                throw new Error("User Agent must be specified in explicit mode")
        }

        var rule = this._getRule(url, ua, explicit);
        if (typeof rule === 'undefined') {
                return true;
        }
        return !(!rule || rule.allow);
};

Robots.prototype.isExplicitlyDisallowed = function(url, ua) {        
        var parsedUrl = parseUrl(url) || {};
        var userAgent = formatUserAgent(ua);

        // The base URL must match otherwise this robots.txt is not valid for it.
        if (
                parsedUrl.protocol !== this._url.protocol ||
                parsedUrl.hostname !== this._url.hostname ||
                parsedUrl.port !== this._url.port
        ) {
                return;
        }

        var rules = this._rules[userAgent] || [];
        var path = urlEncodeToUpper(parsedUrl.pathname + parsedUrl.search);
        var rule = findRule(path, rules);

        if (typeof rule === 'undefined') {
                return;
        }

        return !(!rule || rule.allow);
}

/**
 * Gets the crawl delay if there is one.
 *
 * Will return undefined if there is no crawl delay set.
 *
 * @param  {string} ua
 * @return {number?}
 */
Robots.prototype.getCrawlDelay = function (ua) {
        var userAgent = formatUserAgent(ua || '*');

        return (this._rules[userAgent] || this._rules['*'] || {}).crawlDelay;
};

/**
 * Returns the preferred host if there is one.
 *
 * @return {string?}
 */
Robots.prototype.getPreferredHost = function () {
        return this._preferredHost;
};

/**
 * Returns an array of sitemap URLs if there are any.
 *
 * @return {Array.<string>}
 */
Robots.prototype.getSitemaps = function () {
        return this._sitemaps.slice(0);
};

module.exports = Robots;

1	/**
2	* Trims the white space from the start and end of the line.
3	*
4	* If the line is an array it will strip the white space from
5	* the start and end of each element of the array.
6	*
7	* @param {string\|Array} line
8	* @return {string\|Array}
9	* @private
10	*/
11	function trimLine(line) {
12	if (!line) {	1,881✔
13	return null;	72✔
14	}
15
16	if (Array.isArray(line)) {	1,809✔
17	return line.map(trimLine);	612✔
18	}
19
20	return String(line).trim();	1,197✔
21	}
22
23	/**
24	* Remove comments from lines
25	*
26	* @param {string} line
27	* @return {string}
28	* @private
29	*/
30	function removeComments(line) {
31	var commentStartIndex = line.indexOf('#');	657✔
32	if (commentStartIndex > -1) {	657✔
33	return line.substr(0, commentStartIndex);	18✔
34	}
35
36	return line;	639✔
37	}
38
39	/**
40	* Splits a line at the first occurrence of :
41	*
42	* @param {string} line
43	* @return {Array.<string>}
44	* @private
45	*/
46	function splitLine(line) {
47	var idx = String(line).indexOf(':');	657✔
48
49	if (!line \|\| idx < 0) {	657✔
50	return null;	45✔
51	}
52
53	return [line.slice(0, idx), line.slice(idx + 1)];	612✔
54	}
55
56	/**
57	* Normalises the user-agent string by converting it to
58	* lower case and removing any version numbers.
59	*
60	* @param {string} userAgent
61	* @return {string}
62	* @private
63	*/
64	function formatUserAgent(userAgent) {
65	var formattedUserAgent = userAgent.toLowerCase();	798✔
66
67	// Strip the version number from robot/1.0 user agents
68	var idx = formattedUserAgent.indexOf('/');	798✔
69	if (idx > -1) {	798✔
70	formattedUserAgent = formattedUserAgent.substr(0, idx);	12✔
71	}
72
73	return formattedUserAgent.trim();	798✔
74	}
75
76	/**
77	* Normalises the URL encoding of a path by encoding
78	* unicode characters.
79	*
80	* @param {string} path
81	* @return {string}
82	* @private
83	*/
84	function normaliseEncoding(path) {
85	try {	270✔
86	return urlEncodeToUpper(encodeURI(path).replace(/%25/g, '%'));	270✔
87	} catch (e) {
88	return path;	6✔
89	}
90	}
91
92	/**
93	* Convert URL encodings to support case.
94	*
95	* e.g.: %2a%ef becomes %2A%EF
96	*
97	* @param {string} path
98	* @return {string}
99	* @private
100	*/
101	function urlEncodeToUpper(path) {
102	return path.replace(/%[0-9a-fA-F]{2}/g, function (match) {	732✔
103	return match.toUpperCase();	300✔
104	});
105	}
106
107	/**
108	* Matches a pattern with the specified path
109	*
110	* Uses same algorithm to match patterns as the Google implementation in
111	* google/robotstxt so it should be consistent with the spec.
112	*
113	* @see https://github.com/google/robotstxt/blob/f465f0ede81099dd8bc4aeb2966b3a892bd488b3/robots.cc#L74
114	* @param {string} pattern
115	* @param {string} path
116	* @return {boolean}
117	* @private
118	*/
119	function matches(pattern, path) {
120	// I've added extra comments to try make this easier to understand
121
122	// Stores the lengths of all the current matching substrings.
123	// Maximum number of possible matching lengths is every length in path plus
124	// 1 to handle 0 length too (if pattern starts with * which is zero or more)
125	var matchingLengths = new Array(path.length + 1);	1,581✔
126	var numMatchingLengths = 1;	1,581✔
127
128	// Initially longest match is 0
129	matchingLengths[0] = 0;	1,581✔
130
131	for (var p = 0; p < pattern.length; p++) {	1,581✔
132	// If $ is at the end of pattern then we must match the whole path.
133	// Which is true if the longest matching length matches path length
134	if (pattern[p] === '$' && p + 1 === pattern.length) {	19,020✔
135	return matchingLengths[numMatchingLengths - 1] === path.length;	33✔
136	}
137
138	// Handle wildcards
139	if (pattern[p] == '*') {	18,987✔
140	// Wildcard so all substrings minus the current smallest matching
141	// length are matches
142	numMatchingLengths = path.length - matchingLengths[0] + 1;	12,387✔
143
144	// Update matching lengths to include the smallest all the way up
145	// to numMatchingLengths
146	// Don't update smallest possible match as * matches zero or more
147	// so the smallest current match is also valid
148	for (var i = 1; i < numMatchingLengths; i++) {	12,387✔
149	matchingLengths[i] = matchingLengths[i - 1] + 1;	37,915,629✔
150	}
151	} else {
152	// Check the char at the matching length matches the pattern, if it
153	// does increment it and add it as a valid length, ignore if not.
154	var numMatches = 0;	6,600✔
155	for (var i = 0; i < numMatchingLengths; i++) {	6,600✔
156	if (	26,067✔
157	matchingLengths[i] < path.length &&	51,981✔
158	path[matchingLengths[i]] === pattern[p]
159	) {
160	matchingLengths[numMatches++] = matchingLengths[i] + 1;	5,382✔
161	}
162	}
163
164	// No paths matched the current pattern char so not a match
165	if (numMatches == 0) {	6,600✔
166	return false;	1,260✔
167	}
168
169	numMatchingLengths = numMatches;	5,340✔
170	}
171	}
172
173	return true;	288✔
174	}
175
176	function parseRobots(contents, robots) {
177	var newlineRegex = /\r\n\|\r\|\n/;	147✔
178	var lines = contents	147✔
179	.split(newlineRegex)
180	.map(removeComments)
181	.map(splitLine)
182	.map(trimLine);
183
184	var currentUserAgents = [];	147✔
185	var isNoneUserAgentState = true;	147✔
186	for (var i = 0; i < lines.length; i++) {	147✔
187	var line = lines[i];	657✔
188
189	if (!line \|\| !line[0]) {	657✔
190	continue;	48✔
191	}
192
193	switch (line[0].toLowerCase()) {	609✔
194	case 'user-agent':
195	if (isNoneUserAgentState) {	204✔
196	currentUserAgents.length = 0;	186✔
197	}
198
199	if (line[1]) {	204✔
200	currentUserAgents.push(formatUserAgent(line[1]));	198✔
201	}
202	break;	204✔
203	case 'disallow':
204	robots.addRule(currentUserAgents, line[1], false, i + 1);	237✔
205	break;	237✔
206	case 'allow':
207	robots.addRule(currentUserAgents, line[1], true, i + 1);	51✔
208	break;	51✔
209	case 'crawl-delay':
210	robots.setCrawlDelay(currentUserAgents, line[1]);	54✔
211	break;	54✔
212	case 'sitemap':
213	if (line[1]) {	27✔
214	robots.addSitemap(line[1]);	24✔
215	}
216	break;	27✔
217	case 'host':
218	if (line[1]) {	33✔
219	robots.setPreferredHost(line[1].toLowerCase());	30✔
220	}
221	break;	33✔
222	}
223
224	isNoneUserAgentState = line[0].toLowerCase() !== 'user-agent';	609✔
225	}
226	}
227
228	/**
229	* Returns if a pattern is allowed by the specified rules.
230	*
231	* @param {string} path
232	* @param {Array.<Object>} rules
233	* @return {Object?}
234	* @private
235	*/
236	function findRule(path, rules) {
237	var matchedRule = null;	468✔
238
239	for (var i = 0; i < rules.length; i++) {	468✔
240	var rule = rules[i];	1,581✔
241
242	if (!matches(rule.pattern, path)) {	1,581✔
243	continue;	1,275✔
244	}
245
246	// The longest matching rule takes precedence
247	// If rules are the same length then allow takes precedence
248	if (!matchedRule \|\| rule.pattern.length > matchedRule.pattern.length) {	306✔
249	matchedRule = rule;	291✔
250	} else if (	15✔
251	rule.pattern.length == matchedRule.pattern.length &&	42✔
252	rule.allow &&
253	!matchedRule.allow
254	) {
255	matchedRule = rule;	12✔
256	}
257	}
258
259	return matchedRule;	468✔
260	}
261
262	/**
263	* Converts provided string into an URL object.
264	*
265	* Will return null if provided string is not a valid URL.
266	*
267	* @param {string} url
268	* @return {?URL}
269	* @private
270	*/
271	function parseUrl(url) {
272	try {	696✔
273	// Specify a URL to be used with relative paths
274	// Using non-existent subdomain so can never cause conflict unless
275	// trying to crawl it but doesn't exist and even if tried worst that can
276	// happen is it allows relative URLs on it.
277	var url = new URL(url, 'http://robots-relative.samclarke.com/');	696✔
278
279	if (!url.port) {	690✔
280	url.port = url.protocol === 'https:' ? 443 : 80;	666✔
281	}
282
283	return url;	690✔
284	} catch (e) {
285	return null;	6✔
286	}
287	}
288
289	function Robots(url, contents) {
290	this._url = parseUrl(url) \|\| {};	147✔
291	this._rules = Object.create(null);	147✔
292	this._sitemaps = [];	147✔
293	this._preferredHost = null;	147✔
294
295	parseRobots(contents \|\| '', this);	147✔
296	}
297
298	/**
299	* Adds the specified allow/deny rule to the rules
300	* for the specified user-agents.
301	*
302	* @param {Array.<string>} userAgents
303	* @param {string} pattern
304	* @param {boolean} allow
305	* @param {number} [lineNumber] Should use 1-based indexing
306	*/
307	Robots.prototype.addRule = function (userAgents, pattern, allow, lineNumber) {	3✔
308	var rules = this._rules;	288✔
309
310	userAgents.forEach(function (userAgent) {	288✔
311	rules[userAgent] = rules[userAgent] \|\| [];	279✔
312
313	if (!pattern) {	279✔
314	return;	9✔
315	}
316
317	rules[userAgent].push({	270✔
318	pattern: normaliseEncoding(pattern),
319	allow: allow,
320	lineNumber: lineNumber
321	});
322	});
323	};
324
325	/**
326	* Adds the specified delay to the specified user agents.
327	*
328	* @param {Array.<string>} userAgents
329	* @param {string} delayStr
330	*/
331	Robots.prototype.setCrawlDelay = function (userAgents, delayStr) {	3✔
332	var rules = this._rules;	54✔
333	var delay = Number(delayStr);	54✔
334
335	userAgents.forEach(function (userAgent) {	54✔
336	rules[userAgent] = rules[userAgent] \|\| [];	66✔
337
338	if (isNaN(delay)) {	66✔
339	return;	15✔
340	}
341
342	rules[userAgent].crawlDelay = delay;	51✔
343	});
344	};
345
346	/**
347	* Add a sitemap
348	*
349	* @param {string} url
350	*/
351	Robots.prototype.addSitemap = function (url) {	3✔
352	this._sitemaps.push(url);	24✔
353	};
354
355	/**
356	* Sets the preferred host name
357	*
358	* @param {string} url
359	*/
360	Robots.prototype.setPreferredHost = function (url) {	3✔
361	this._preferredHost = url;	30✔
362	};
363
364	Robots.prototype._getRule = function (url, ua, explicit) {	3✔
365	var parsedUrl = parseUrl(url) \|\| {};	549✔
366	var userAgent = formatUserAgent(ua \|\| '*');	549✔
367
368	// The base URL must match otherwise this robots.txt is not valid for it.
369	if (	549✔
370	parsedUrl.protocol !== this._url.protocol \|\|	1,560✔
371	parsedUrl.hostname !== this._url.hostname \|\|
372	parsedUrl.port !== this._url.port
373	) {
374	return;	81✔
375	}
376
377	var rules = this._rules[userAgent];	468✔
378	if (!explicit) {	468✔
379	rules = rules \|\| this._rules['*']	462✔
380	}
381	rules = rules \|\| []	468✔
382
383	var path = urlEncodeToUpper(parsedUrl.pathname + parsedUrl.search);	468✔
384	var rule = findRule(path, rules);	468✔
385
386	return rule;	468✔
387	};
388
389	/**
390	* Returns true if allowed, false if not allowed.
391	*
392	* Will return undefined if the URL is not valid for
393	* this robots.txt file.
394	*
395	* @param {string} url
396	* @param {string?} ua
397	* @return {boolean?}
398	*/
399	Robots.prototype.isAllowed = function (url, ua) {	3✔
400	var rule = this._getRule(url, ua);	276✔
401
402	if (typeof rule === 'undefined') {	276✔
403	return;	48✔
404	}
405
406	return !rule \|\| rule.allow;	228✔
407	};
408
409	/**
410	* Returns the line number of the matching directive for the specified
411	* URL and user-agent if any.
412	*
413	* The line numbers start at 1 and go up (1-based indexing).
414	*
415	* Return -1 if there is no matching directive. If a rule is manually
416	* added without a lineNumber then this will return undefined for that
417	* rule.
418	*
419	* @param {string} url
420	* @param {string?} ua
421	* @return {number?}
422	*/
423	Robots.prototype.getMatchingLineNumber = function (url, ua) {	3✔
424	var rule = this._getRule(url, ua);	27✔
425
426	return rule ? rule.lineNumber : -1;	27✔
427	};
428
429	/**
430	* In standard mode, it returns the opposite of is allowed().
431	* In explicit mode, it will return:
432	* - true if the the agent is explicitly disallowed (wildcard non included),
433	* - throws an error if the user agent is not specified,
434	* - and false otherwise.
435	* @param {string} url
436	* @param {string} ua
437	* @return {boolean}
438	*/
439	Robots.prototype.isDisallowed = function (url, ua, explicit) {	3✔
440	if ((explicit === true) && (ua === undefined)) {	249✔
441	throw new Error("User Agent must be specified in explicit mode")	3✔
442	}
443
444	var rule = this._getRule(url, ua, explicit);	246✔
445	if (typeof rule === 'undefined') {	246✔
446	return true;	33✔
447	}
448	return !(!rule \|\| rule.allow);	213✔
449	};
450
451	Robots.prototype.isExplicitlyDisallowed = function(url, ua) {	3✔
NEW 452	var parsedUrl = parseUrl(url) \|\| {};	×
NEW 453	var userAgent = formatUserAgent(ua);	×
454
455	// The base URL must match otherwise this robots.txt is not valid for it.
NEW 456	if (	×
457	parsedUrl.protocol !== this._url.protocol \|\|	×
458	parsedUrl.hostname !== this._url.hostname \|\|
459	parsedUrl.port !== this._url.port
460	) {
NEW UNCOV 461	return;	×
462	}
463
NEW UNCOV 464	var rules = this._rules[userAgent] \|\| [];	×
NEW UNCOV 465	var path = urlEncodeToUpper(parsedUrl.pathname + parsedUrl.search);	×
NEW UNCOV 466	var rule = findRule(path, rules);	×
467
NEW UNCOV 468	if (typeof rule === 'undefined') {	×
NEW UNCOV 469	return;	×
470	}
471
NEW UNCOV 472	return !(!rule \|\| rule.allow);	×
473	}
474
475	/**
476	* Gets the crawl delay if there is one.
477	*
478	* Will return undefined if there is no crawl delay set.
479	*
480	* @param {string} ua
481	* @return {number?}
482	*/
483	Robots.prototype.getCrawlDelay = function (ua) {	3✔
484	var userAgent = formatUserAgent(ua \|\| '*');	51✔
485
486	return (this._rules[userAgent] \|\| this._rules['*'] \|\| {}).crawlDelay;	51✔
487	};
488
489	/**
490	* Returns the preferred host if there is one.
491	*
492	* @return {string?}
493	*/
494	Robots.prototype.getPreferredHost = function () {	3✔
495	return this._preferredHost;	21✔
496	};
497
498	/**
499	* Returns an array of sitemap URLs if there are any.
500	*
501	* @return {Array.<string>}
502	*/
503	Robots.prototype.getSitemaps = function () {	3✔
504	return this._sitemaps.slice(0);	15✔
505	};
506
507	module.exports = Robots;	3✔

samclarke / robots-parser / 10901687640

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous