Puppeteer Request
This library provides a simple way to utilize Puppeteer to automate/scrape web pages. It helps create a browser for puppeteer and then provides a way to make a request. Since this is part of the request-group family, it's trivial to create a group of these request and fire them off to effectively pull data across a number of web pages. Be sure to check out the example below to understand how to utilize this library
Installing
npm install request-group-puppeteerBuilding
git clone https://github.com/GabrieleNunez/request-group-puppeteer.git
cd request-group-puppeteer
npm install
npm run buildExample ( Making a singular request )
import { PuppeteerManager, PuppeteerRequest } from 'request-group-puppeteer';
function puppeteerExample(): Promise<void> {
return new Promise(
async (resolve): Promise<void> => {
// create our puppeteer manager, this can be shared across multiple request no problem
// you really only need 1. Since 1 manager = 1 puppeteer browser and 1 request = 1 tab in that browser
console.log('Starting puppeteer up');
let puppeteerManager: PuppeteerManager = new PuppeteerManager();
await puppeteerManager.initialize();
let genericRequest: PuppeteerRequest = new PuppeteerRequest(
'https://github.com/GabrieleNunez/request-group',
puppeteerManager,
);
console.log('Running request');
await genericRequest.run();
// once the request is run at this point its just like any other Puppeteer Page object.
// do with it whatever you want
let page = genericRequest.getPage();
let lines: string[] = await page.evaluate(function(): Promise<string[]> {
return new Promise((evalResolve): void => {
let resultLines: string[] = [];
let tableMessages: NodeListOf<Element> = document.querySelectorAll('table.files td.message');
for (var i = 0; i < tableMessages.length; i++) {
let tableCell = tableMessages[i];
let message: string = tableCell.textContent ? (tableCell.textContent as string) : '';
resultLines.push(message.trim());
}
evalResolve(resultLines);
});
});
console.log('Results');
console.log(lines);
// make sure to free up up the request. Memory leaks and runaway processes are a thing to watch for
console.log('Freeing up resources');
await genericRequest.dispose();
await puppeteerManager.dispose();
resolve();
},
);
}
// puppeteer
puppeteerExample().then((): void => {
console.log('Completed');
});
