|
|
|
@ -1,7 +1,8 @@
|
|
|
|
const puppeteer = require('puppeteer');
|
|
|
|
const puppeteer = require('puppeteer');
|
|
|
|
const genericVinParserFactory = require('../processors/generics/generic-vin-parser');
|
|
|
|
const genericVinParserFactory = require('../processors/generics/generic-vin-parser');
|
|
|
|
|
|
|
|
const superagent = require('superagent');
|
|
|
|
module.exports = {
|
|
|
|
module.exports = {
|
|
|
|
cronString: '46 * * * *',
|
|
|
|
cronString: '0 22 * * *',
|
|
|
|
run: async function () {
|
|
|
|
run: async function () {
|
|
|
|
const startingPoint = 'https://topclassiccarsforsale.com/amc';
|
|
|
|
const startingPoint = 'https://topclassiccarsforsale.com/amc';
|
|
|
|
const browser = await puppeteer.launch({
|
|
|
|
const browser = await puppeteer.launch({
|
|
|
|
@ -25,6 +26,16 @@ module.exports = {
|
|
|
|
await page.goto(`${startingPoint}/page/${pageNumber}/`, {timeout: 60000});
|
|
|
|
await page.goto(`${startingPoint}/page/${pageNumber}/`, {timeout: 60000});
|
|
|
|
const cars = await module.exports.processPage(page);
|
|
|
|
const cars = await module.exports.processPage(page);
|
|
|
|
console.log(cars);
|
|
|
|
console.log(cars);
|
|
|
|
|
|
|
|
cars.forEach(car => {
|
|
|
|
|
|
|
|
superagent.post('http://localhost:3000/lead/createFromCrawler')
|
|
|
|
|
|
|
|
.send({
|
|
|
|
|
|
|
|
url: car.url
|
|
|
|
|
|
|
|
}).end((err, res) => {
|
|
|
|
|
|
|
|
if(err){
|
|
|
|
|
|
|
|
console.error('Failed to send lead', err);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
});
|
|
|
|
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
},
|
|
|
|
},
|
|
|
|
|