2428604161

Build Type

push

github

Committed by GitHub

Commit Message

Merge pull request #48 from get-set-fetch/next

Run Details

657 of 852 branches covered (77.11%)

Branch coverage included in aggregate %.

144 of 176 new or added lines in 5 files covered. (81.82%)

1 existing line in 1 file now uncovered.

1574 of 1796 relevant lines covered (87.64%)

1267.02 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

85.37

/src/scraper/QueueBuffer.ts

import Project from '../storage/base/Project';
import Resource from '../storage/base/Resource';

export default class QueueBuffer {
  project: Project;
  resources: Resource[];
  size: number;

  /**
   * Prevents parallel project.queue.getResourcesToScrape calls
   */
  refillInProgress:boolean;

  error;

  constructor(size: number) {
    this.size = size;
    this.resources = [];
  }

  init(project, resources: Resource[]) {
    this.project = project;
    this.resources = resources;
  }

  async refill():Promise<void> {
    // buffer is only filled sequentially
    if (this.refillInProgress) return;

    try {
      this.refillInProgress = true;
      const toBeScrapedResources = await this.project.queue.getResourcesToScrape(this.size - this.resources.length);
      this.addResources(toBeScrapedResources);
      this.refillInProgress = false;
    }
    catch (err) {
      // parent call doesn't wait for this async to finish thus can't catch it, store err separately
      this.error = err;
    }
  }

  async getResource(stop:boolean):Promise<Resource> {
    /*
    stop signal has been received
    gracefully stop scraping, allow all scrape-in-progress resources to be scraped
    */
    if (stop) {
      if (this.resources.length > 0) {
        // re-make to-be-scraped buffered resources eligible for scraping by reseting their status flag
        await Promise.all(this.resources.map(resource => this.project.queue.updateStatus(resource.queueEntryId, null)));
        this.resources = [];
      }

      return null;
    }

    /*
    stop signal was received due to buffer error in an independent async call, throw the error up
    parent scraper will catch any errors and stop the process via the `stop` flag
    */
    if (this.error) throw (this.error);

    // attemp to re-fill buffer before it's completely empty
    if (this.resources.length < this.size / 2) {
      // buffer needs to be refilled now, can't refill it independently, we risk isScrapingComplete condition to pass
      if (this.resources.length === 0) {
        await this.refill();
        // take advantage of waiting for refillBuffer, directly thrown the error if one was caught
        // avoid isScrapingComplete returning true on empty buffer due to refillBuffer error
        if (this.error) throw (this.error);
      }
      // refill buffer independently
      else {
        this.refill();
      }
    }

    // in the future, don't just retrieve the 1st resource, attempt to search for one meeting the concurrency conditions
    return this.resources.length > 0 ? this.resources.shift() : null;
  }

  addResources(resources: Resource[]) {
    this.resources.push(...resources);
  }
}

1	import Project from '../storage/base/Project';
2	import Resource from '../storage/base/Resource';
3
4	export default class QueueBuffer {	15✔
5	project: Project;
6	resources: Resource[];
7	size: number;
8
9	/**
10	* Prevents parallel project.queue.getResourcesToScrape calls
11	*/
12	refillInProgress:boolean;
13
14	error;
15
16	constructor(size: number) {
17	this.size = size;	423✔
18	this.resources = [];	423✔
19	}
20
21	init(project, resources: Resource[]) {
22	this.project = project;	423✔
23	this.resources = resources;	423✔
24	}
25
26	async refill():Promise<void> {
27	// buffer is only filled sequentially
28	if (this.refillInProgress) return;	3,460!
29
30	try {	3,460✔
31	this.refillInProgress = true;	3,460✔
32	const toBeScrapedResources = await this.project.queue.getResourcesToScrape(this.size - this.resources.length);	3,460✔
33	this.addResources(toBeScrapedResources);	3,459✔
34	this.refillInProgress = false;	3,459✔
35	}
36	catch (err) {
37	// parent call doesn't wait for this async to finish thus can't catch it, store err separately
38	this.error = err;	1✔
39	}
40	}
41
42	async getResource(stop:boolean):Promise<Resource> {
43	/*
44	stop signal has been received
45	gracefully stop scraping, allow all scrape-in-progress resources to be scraped
46	*/
47	if (stop) {	3,470✔
48	if (this.resources.length > 0) {	10!
49	// re-make to-be-scraped buffered resources eligible for scraping by reseting their status flag
NEW 50	await Promise.all(this.resources.map(resource => this.project.queue.updateStatus(resource.queueEntryId, null)));	×
NEW 51	this.resources = [];	×
52	}
53
54	return null;	10✔
55	}
56
57	/*
58	stop signal was received due to buffer error in an independent async call, throw the error up
59	parent scraper will catch any errors and stop the process via the `stop` flag
60	*/
61	if (this.error) throw (this.error);	3,460!
62
63	// attemp to re-fill buffer before it's completely empty
64	if (this.resources.length < this.size / 2) {	3,460!
65	// buffer needs to be refilled now, can't refill it independently, we risk isScrapingComplete condition to pass
66	if (this.resources.length === 0) {	3,460✔
67	await this.refill();	2,690✔
68	// take advantage of waiting for refillBuffer, directly thrown the error if one was caught
69	// avoid isScrapingComplete returning true on empty buffer due to refillBuffer error
70	if (this.error) throw (this.error);	2,690✔
71	}
72	// refill buffer independently
73	else {
74	this.refill();	770✔
75	}
76	}
77
78	// in the future, don't just retrieve the 1st resource, attempt to search for one meeting the concurrency conditions
79	return this.resources.length > 0 ? this.resources.shift() : null;	3,459✔
80	}
81
82	addResources(resources: Resource[]) {
83	this.resources.push(...resources);	4,075✔
84	}
85	}

get-set-fetch / scraper / 2428604161

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous