Added API endpoints to trigger process
Edward Peterson
3 years ago
parent
5e981cedca
commit
be1ab59bac
@ -1,158 +1,85 @@
|
||||
const puppeteer = require('puppeteer')
|
||||
const fs = require('fs');
|
||||
const superagent = require('superagent');
|
||||
const path = require('path');
|
||||
const uuid = require('uuid');
|
||||
const express = require('express');
|
||||
const glob = require('glob');
|
||||
const _ = require('lodash');
|
||||
const app = express();
|
||||
var bodyParser = require('body-parser')
|
||||
|
||||
var url = '';
|
||||
process.argv.forEach((val, index, array) => {
|
||||
console.log(val)
|
||||
if(val.startsWith('url=')){
|
||||
url = val.substring(4);
|
||||
}
|
||||
|
||||
const processors = [];
|
||||
//load the processor files
|
||||
glob('processors/*.processor.js', (error, matches) => {
|
||||
console.log(matches);
|
||||
_.forEach(matches, file => {
|
||||
const processor = require(path.resolve('./', file));
|
||||
processors.push(processor);
|
||||
})
|
||||
console.log(`${matches.length} processors loaded`);
|
||||
})
|
||||
if(!url || url.length === 0) {
|
||||
console.log('Need to supply url');
|
||||
process.exit();
|
||||
}
|
||||
|
||||
|
||||
const processors = [{
|
||||
baseUrl: 'bringatrailer.com',
|
||||
pageLoadIndicator: '.gallery',
|
||||
vinSelector: 'body > main > div.listing > div:nth-child(3) > div.column.column-right.column-right-force > div.essentials > div:nth-child(5) > ul > li:nth-child(1) > a',
|
||||
carouselTrigger: '.gallery > a:nth-child(1)'
|
||||
app.use(bodyParser.json())
|
||||
app.post('/convertGalleryToHar', async (req, res) => {
|
||||
const url = req.body.url;
|
||||
console.log(url);
|
||||
// get processor
|
||||
const processor = _.find(processors, (processor) => url.includes(processor.baseUrl));
|
||||
if (!processor) {
|
||||
return res.status(400).json({
|
||||
message: 'Could not find processor for url'
|
||||
})
|
||||
}
|
||||
console.log('Processor found');
|
||||
try {
|
||||
const payloads = await run(url, processor);
|
||||
console.log(payloads);
|
||||
|
||||
}, {
|
||||
baseUrl: 'classiccars.com',
|
||||
pageLoadIndicator: '#ListingCarousel',
|
||||
vinSelector: 'li.p-vin > span:nth-child(2)',
|
||||
carouselTrigger: 'div.swiper-slide-active > div > img.u-photo'
|
||||
}]
|
||||
var processorConfig = {}
|
||||
processors.forEach(config => {
|
||||
if(url.includes(config.baseUrl)){
|
||||
processorConfig = config;
|
||||
res.status(200).json({
|
||||
log: {
|
||||
entries: payloads
|
||||
}
|
||||
// payloads
|
||||
})
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
res.status(500).json(error);
|
||||
}
|
||||
|
||||
})
|
||||
if(!processorConfig) {
|
||||
console.log('Unsupported URL');
|
||||
process.exit();
|
||||
}
|
||||
app.listen(2667);
|
||||
|
||||
const run = async () => {
|
||||
const browser = await puppeteer.launch({
|
||||
headless:true
|
||||
});
|
||||
|
||||
const page = await browser.newPage();
|
||||
|
||||
await page.goto(url);
|
||||
|
||||
// Type into search box.
|
||||
// await page.type('.devsite-search-field', 'Headless Chrome');
|
||||
|
||||
// Wait for suggest overlay to appear and click "show all results".
|
||||
const allResultsSelector = processorConfig.pageLoadIndicator;
|
||||
await page.waitForSelector(allResultsSelector);
|
||||
var vin;
|
||||
async function run(url, processor) {
|
||||
const browser = await puppeteer.launch({
|
||||
headless: true
|
||||
});
|
||||
|
||||
const vinSelector = processorConfig.vinSelector;
|
||||
await page.waitForSelector(vinSelector).then(async () => {
|
||||
let element = await page.$(vinSelector)
|
||||
vin = await page.evaluate(el => el.textContent, element);
|
||||
}).catch(error => {
|
||||
console.error('Unable to grab VIN, falling back to UUID');
|
||||
vin = uuid.v4();
|
||||
});
|
||||
console.log(vin);
|
||||
// await page.click(allResultsSelector);
|
||||
const page = await browser.newPage();
|
||||
console.log('Loading page...');
|
||||
await page.goto(url, { timeout: 60000 });
|
||||
console.log('page loaded')
|
||||
const galleryUrls = await processor.execute(page);
|
||||
|
||||
const client = await page.target().createCDPSession()
|
||||
await client.send('Page.setDownloadBehavior', {
|
||||
behavior: 'allow',
|
||||
downloadPath: './images',
|
||||
})
|
||||
const firstImageLinkSelector = processorConfig.carouselTrigger;
|
||||
await page.click(firstImageLinkSelector);
|
||||
await page.waitForSelector('.pswp__img')
|
||||
console.log('Gallery is loaded, fetching URLS')
|
||||
const galleryUrls = await page.evaluate(downloadGallery, vin);
|
||||
console.log('Done collecting URLS', galleryUrls.length);
|
||||
await Promise.all(galleryUrls.map(image => new Promise(async (resolve, reject) => {
|
||||
const fileType = path.extname(image.url);
|
||||
var stream = fs.createWriteStream(path.resolve('images', `${image.fileName}${fileType}`));
|
||||
stream.on('finish', resolve);
|
||||
superagent.get(image.url).pipe(stream);
|
||||
})))
|
||||
console.log('URLS done downloading')
|
||||
|
||||
await browser.close();
|
||||
};
|
||||
|
||||
downloadGallery = async (vin) => {
|
||||
var galleryUrls = [];
|
||||
async function ensureCarouselVisible() {
|
||||
var imgWrap;
|
||||
do {
|
||||
imgWrap = document.elementFromPoint(100, 100);
|
||||
console.log(imgWrap.classList);
|
||||
await delay(50);
|
||||
} while(imgWrap.classList.contains('pswp__img--placeholder'));
|
||||
|
||||
}
|
||||
|
||||
function delay(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
// Find the active image, surround it in an anchor tag, then click it.
|
||||
async function downloadImage(id) {
|
||||
const imgWrap = document.elementFromPoint(300, 300);
|
||||
const children = imgWrap.querySelectorAll('.pswp__img');
|
||||
const img = children[children.length - 1];
|
||||
console.log(img, imgWrap);
|
||||
// debugger;
|
||||
// Full image hasn't loaded yet
|
||||
const src = imgWrap.src.split('?')[0]// get rid of querystring
|
||||
return downloadSrc(src, id);
|
||||
}
|
||||
|
||||
function downloadSrc(src, id) {
|
||||
galleryUrls.push({
|
||||
fileName: id,
|
||||
url:src
|
||||
})
|
||||
}
|
||||
|
||||
function nextImage() {
|
||||
document.querySelector('.pswp__button.pswp__button--arrow--right').click();
|
||||
console.log('Done collecting URLS', galleryUrls.length);
|
||||
const payloads = await Promise.all(galleryUrls.map(image => new Promise(async (resolve, reject) => {
|
||||
superagent.get(image.url).responseType('blob').then(function (response) {
|
||||
if (response.statusCode == 200) {
|
||||
return resolve({
|
||||
response: {
|
||||
content: {
|
||||
mimeType: response.headers["content-type"],
|
||||
encoding: 'base64',
|
||||
text: response.body.toString('base64')
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function getCounterValue() {
|
||||
const [position, total] = document.querySelector('.pswp__counter').textContent.split('/');
|
||||
return parseInt(position.trim(), 10);
|
||||
}
|
||||
|
||||
function run() {
|
||||
return new Promise(async (resolve, reject) => {
|
||||
await ensureCarouselVisible();
|
||||
await delay(500);
|
||||
const firstValue = getCounterValue();
|
||||
do {
|
||||
await downloadImage(`${vin}-${getCounterValue()}`);
|
||||
nextImage();
|
||||
await delay(100);
|
||||
} while(getCounterValue() !== firstValue);
|
||||
return resolve(galleryUrls)
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
run().then(resolve);
|
||||
});
|
||||
}
|
||||
reject();
|
||||
});
|
||||
})))
|
||||
console.log('URLS done downloading')
|
||||
await browser.close();
|
||||
return payloads;
|
||||
}
|
||||
|
||||
run();
|
||||
// run();
|
||||
@ -0,0 +1,11 @@
|
||||
const genericBootstrapFactory = require("./generic-bootstrap")
|
||||
|
||||
module.exports = {
|
||||
execute: genericBootstrapFactory({
|
||||
pageLoadIndicator: '.gallery',
|
||||
vinSelector: 'body > main > div.listing > div:nth-child(3) > div.column.column-right.column-right-force > div.essentials > div:nth-child(5) > ul > li:nth-child(1) > a',
|
||||
carouselTrigger: '.gallery > a:nth-child(1)'
|
||||
|
||||
}),
|
||||
baseUrl: 'bringatrailer.com'
|
||||
}
|
||||
@ -0,0 +1,11 @@
|
||||
const genericBootstrapFactory = require("./generic-bootstrap")
|
||||
|
||||
module.exports = {
|
||||
execute: genericBootstrapFactory({
|
||||
baseUrl: 'classiccars.com',
|
||||
pageLoadIndicator: '#ListingCarousel',
|
||||
vinSelector: 'li.p-vin > span:nth-child(2)',
|
||||
carouselTrigger: 'div.swiper-slide-active > div > img.u-photo'
|
||||
}),
|
||||
baseUrl :'classiccars.com'
|
||||
}
|
||||
@ -0,0 +1,105 @@
|
||||
const uuid = require('uuid');
|
||||
module.exports = function (processorConfig) {
|
||||
|
||||
downloadGallery = async (vin) => {
|
||||
var galleryUrls = [];
|
||||
async function ensureCarouselVisible() {
|
||||
var imgWrap;
|
||||
do {
|
||||
imgWrap = document.elementFromPoint(100, 100);
|
||||
console.log(imgWrap.classList);
|
||||
await delay(50);
|
||||
} while (imgWrap.classList.contains('pswp__img--placeholder'));
|
||||
|
||||
}
|
||||
|
||||
function delay(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
// Find the active image, surround it in an anchor tag, then click it.
|
||||
async function downloadImage(id) {
|
||||
const imgWrap = document.elementFromPoint(300, 300);
|
||||
const children = imgWrap.querySelectorAll('.pswp__img');
|
||||
const img = children[children.length - 1];
|
||||
console.log(img, imgWrap);
|
||||
// debugger;
|
||||
// Full image hasn't loaded yet
|
||||
const src = imgWrap.src.split('?')[0]// get rid of querystring
|
||||
return downloadSrc(src, id);
|
||||
}
|
||||
|
||||
function downloadSrc(src, id) {
|
||||
galleryUrls.push({
|
||||
fileName: id,
|
||||
url: src
|
||||
})
|
||||
}
|
||||
|
||||
function nextImage() {
|
||||
document.querySelector('.pswp__button.pswp__button--arrow--right').click();
|
||||
}
|
||||
|
||||
function getCounterValue() {
|
||||
const [position, total] = document.querySelector('.pswp__counter').textContent.split('/');
|
||||
return parseInt(position.trim(), 10);
|
||||
}
|
||||
|
||||
function run() {
|
||||
return new Promise(async (resolve, reject) => {
|
||||
await ensureCarouselVisible();
|
||||
await delay(500);
|
||||
const firstValue = getCounterValue();
|
||||
do {
|
||||
await downloadImage(`${vin}-${getCounterValue()}`);
|
||||
nextImage();
|
||||
await delay(100);
|
||||
} while (getCounterValue() !== firstValue);
|
||||
return resolve(galleryUrls)
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
run().then(resolve);
|
||||
});
|
||||
}
|
||||
|
||||
return async function (page) {
|
||||
console.log('Running generic boostrap extractor')
|
||||
|
||||
// Type into search box.
|
||||
// await page.type('.devsite-search-field', 'Headless Chrome');
|
||||
|
||||
// Wait for suggest overlay to appear and click "show all results".
|
||||
const allResultsSelector = processorConfig.pageLoadIndicator;
|
||||
await page.waitForSelector(allResultsSelector);
|
||||
var vin;
|
||||
|
||||
const vinSelector = processorConfig.vinSelector;
|
||||
await page.waitForSelector(vinSelector).then(async () => {
|
||||
let element = await page.$(vinSelector)
|
||||
vin = await page.evaluate(el => el.textContent, element);
|
||||
}).catch(error => {
|
||||
console.error('Unable to grab VIN, falling back to UUID');
|
||||
vin = uuid.v4();
|
||||
});
|
||||
console.log(vin);
|
||||
// await page.click(allResultsSelector);
|
||||
|
||||
const client = await page.target().createCDPSession()
|
||||
await client.send('Page.setDownloadBehavior', {
|
||||
behavior: 'allow',
|
||||
downloadPath: './images',
|
||||
})
|
||||
const firstImageLinkSelector = processorConfig.carouselTrigger;
|
||||
await page.click(firstImageLinkSelector);
|
||||
await page.waitForSelector('.pswp__img')
|
||||
console.log('Gallery is loaded, fetching URLS')
|
||||
const galleryUrls = await page.evaluate(downloadGallery, vin);
|
||||
|
||||
|
||||
|
||||
return galleryUrls
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue