Added callout for crawler

crawler
Edward Peterson 3 years ago
parent 2f5cc56b5f
commit 32068b1338

@ -1,7 +1,8 @@
const puppeteer = require('puppeteer'); const puppeteer = require('puppeteer');
const genericVinParserFactory = require('../processors/generics/generic-vin-parser'); const genericVinParserFactory = require('../processors/generics/generic-vin-parser');
const superagent = require('superagent');
module.exports = { module.exports = {
cronString: '46 * * * *', cronString: '0 22 * * *',
run: async function () { run: async function () {
const startingPoint = 'https://topclassiccarsforsale.com/amc'; const startingPoint = 'https://topclassiccarsforsale.com/amc';
const browser = await puppeteer.launch({ const browser = await puppeteer.launch({
@ -25,6 +26,16 @@ module.exports = {
await page.goto(`${startingPoint}/page/${pageNumber}/`, {timeout: 60000}); await page.goto(`${startingPoint}/page/${pageNumber}/`, {timeout: 60000});
const cars = await module.exports.processPage(page); const cars = await module.exports.processPage(page);
console.log(cars); console.log(cars);
cars.forEach(car => {
superagent.post('http://localhost:3000/lead/createFromCrawler')
.send({
url: car.url
}).end((err, res) => {
if(err){
console.error('Failed to send lead', err);
}
});
})
} }
}, },

Loading…
Cancel
Save