diff --git a/crawlers/topclassiccarsforsale.crawler.js b/crawlers/topclassiccarsforsale.crawler.js index 40e145b..bb55499 100644 --- a/crawlers/topclassiccarsforsale.crawler.js +++ b/crawlers/topclassiccarsforsale.crawler.js @@ -1,7 +1,8 @@ const puppeteer = require('puppeteer'); const genericVinParserFactory = require('../processors/generics/generic-vin-parser'); +const superagent = require('superagent'); module.exports = { - cronString: '46 * * * *', + cronString: '0 22 * * *', run: async function () { const startingPoint = 'https://topclassiccarsforsale.com/amc'; const browser = await puppeteer.launch({ @@ -25,6 +26,16 @@ module.exports = { await page.goto(`${startingPoint}/page/${pageNumber}/`, {timeout: 60000}); const cars = await module.exports.processPage(page); console.log(cars); + cars.forEach(car => { + superagent.post('http://localhost:3000/lead/createFromCrawler') + .send({ + url: car.url + }).end((err, res) => { + if(err){ + console.error('Failed to send lead', err); + } + }); + }) } },