Added filmstrip option for ebay processor.

Reduced redundant calls to crawled websites.

.gitignore

@@ -1,2 +1,3 @@
 node_modules/
 images/
+stack.env

Dockerfile

@@ -0,0 +1,7 @@
+FROM ghcr.io/puppeteer/puppeteer:21.5.2
+WORKDIR /app
+
+COPY /server ./
+COPY /package.json ./package.json
+RUN npm install
+CMD ["node", "index.js"]

README.md

@@ -1,2 +1,9 @@
 # bringatrailerbot
 
+
+Deploy instructions:
+- Run `./build.sh` script and pass new version number as parameter.
+- Commit and push changes
+- Upgrade BUILD_VERSION env var in Portainer
+- Redeploy
+<!-- - From portainer, open the amxregistry stack and select Pull image and Deploy -->

build.sh

@@ -0,0 +1,6 @@
+
+export BUILD_VERSION=$1
+docker login git.edwardpeterson.dev
+docker-compose -f docker-compose-build.yml build --pull
+docker-compose -f docker-compose-build.yml push
+echo "Upload complete. Make sure to update the BUILD_VERSION env var in Portainer to $1"

docker-compose-build.yml

@@ -0,0 +1,16 @@
+version: '3.7'
+
+services:
+  
+  amxregistry-bundler:
+    image: git.edwardpeterson.dev/cubemaster21/amxregistry-bundler:${BUILD_VERSION}
+    pull_policy: build
+    restart: always
+    build:
+      no_cache: true
+      context: .
+      dockerfile: ./Dockerfile
+    ports:
+      - 2667:2667
+    env_file:
+      - stack.env

docker-compose.yml

@@ -0,0 +1,16 @@
+version: '3.7'
+
+services:
+  
+  amxregistry-bundler:
+    image: git.edwardpeterson.dev/cubemaster21/amxregistry-bundler:${BUILD_VERSION}
+    # pull_policy: build
+    restart: always
+    build:
+      no_cache: true
+      context: .
+      dockerfile: ./Dockerfile
+    ports:
+      - 2667:2667
+    env_file:
+      - stack.env

index.js

@@ -1,123 +0,0 @@
-const puppeteer = require('puppeteer')
-const superagent = require('superagent');
-const path = require('path');
-const express = require('express');
-const glob = require('glob');
-const _ = require('lodash');
-const app = express();
-var bodyParser = require('body-parser')
-const CronJob = require('cron').CronJob;
-
-const processors = [];
-//load the processor files
-glob(path.resolve(__dirname, 'processors/*.processor.js'), (error, matches) => {
-  console.log(matches);
-  _.forEach(matches, file => {
-    const processor = require(path.resolve(__dirname, file));
-    processors.push(processor);
-  })
-  console.log(`${matches.length} processors loaded`);
-})
-//load the crawler files
-glob(path.resolve(__dirname, 'crawlers/*.crawler.js'), (error, matches) => {
-  console.log(matches);
-  _.forEach(matches, file => {
-    const crawler = require(path.resolve(__dirname, file));
-    const cronJob = new CronJob(crawler.cronString, crawler.run);
-    cronJob.start();
-    
-  })
-  console.log(`${matches.length} crawlers loaded`);
-})
-app.use(bodyParser.json())
-app.post('/convertGalleryToHar', async (req, res) => {
-  const url = req.body.url;
-  console.log(url);
-
-  // get processor
-  const searchableHostname = (new URL(url)).hostname.replace(/^www\./i, '');
-  console.log('Searching for hostname:', searchableHostname)
-  const processor = _.find(processors, (processor) => (searchableHostname) === (processor.baseUrl.replace(/^www\./i, '')));
-  if (!processor) {
-    return res.status(400).json({
-      message: 'Could not find processor for url'
-    })
-  }
-  console.log('Processor found', processor.baseUrl);
-  try {
-    const payloads = await run(url, processor);
-    res.status(200).json({
-      vin: payloads.vin,
-      mileage: payloads.mileage,
-      log: {
-        entries: payloads.payloads
-      }
-      // payloads
-    })
-  } catch (error) {
-    console.log(error);
-    res.status(500).json(error);
-  }
-
-})
-app.get('/supportedSites', async (req, res) => {
-  res.status(200).json(processors.map(p => p.baseUrl));
-})
-app.listen(2667);
-
-async function run(url, processor) {
-  const browser = await puppeteer.launch({
-    headless: true
-  });
-
-  const page = await browser.newPage();
-  await page.setViewport({
-    width: 1200,
-    height: 800
-  });
-  await page.setUserAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36");
-  console.log('Loading page...');
-  await page.goto(url, { timeout: 60000 });
-  console.log('page loaded');
-  console.log('attempting to parse fields');
-  const vin = await processor.parseVIN(page);
-  console.log('parsed VIN:', vin);
-  const mileage = await processor.parseMileage(page);
-  console.log('parsed Mileage:', mileage);
-  const galleryUrls = await processor.execute(page);
-
-
-  console.log('Done collecting URLS', galleryUrls.length);
-  const payloads = await Promise.all(galleryUrls.map(image => new Promise(async (resolve, reject) => {
-    superagent.get(image.url).responseType('blob').then(function (response) {
-      if (response.statusCode == 200) {
-        console.log('Resolving', image.url)
-        return resolve({
-          response: {
-            content: {
-              mimeType: response.headers["content-type"],
-              encoding: 'base64',
-              text: response.body.toString('base64')
-            }
-          }
-        });
-      } else {
-        console.log("Invalid status code", response.statusCode, 'for', image.url);
-        resolve({})
-
-      }
-    }).catch(error => {
-      console.error(error)
-      resolve({})
-    });
-  })))
-  console.log('URLS done downloading')
-  await browser.close();
-  return {
-    vin,
-    payloads,
-    mileage
-  };
-}
-
-// run();

package.json

@@ -10,6 +10,7 @@
   "license": "ISC",
   "dependencies": {
     "body-parser": "^1.20.1",
+    "clew-logger": "^1.0.8",
     "cron": "^2.2.0",
     "express": "^4.18.2",
     "glob": "^8.0.3",

server/crawlers/bringatrailer.crawler.js

@@ -0,0 +1,155 @@
+const puppeteer = require('puppeteer');
+const genericVinParserFactory = require('../processors/generics/generic-vin-parser');
+const superagent = require('superagent');
+const {log} = require('clew-logger');
+
+const selectors = {
+    listingInternalLink: "h3>a.typography-body1",
+    vinSpan: "h2>span.flex-1",
+    listingExternalLink: "#vehicle_gotoAuction>a",
+    totalPageIndicator: ".mx-2>span.typography-subtitle1:nth-child(3)"
+}
+
+module.exports = {
+    cronString: '00 8 * * 0',
+    run: async function () {
+
+        function getQueryBody(pageNum) {
+            return  `page=${pageNum}&per_page=24&get_items=1&get_stats=0&base_filter%5Bkeyword_pages%5D%5B%5D=13387884&base_filter%5Bitems_type%5D=model&sort=td`
+        }
+
+
+        //get first page
+        const backendUrl = 'https://bringatrailer.com/wp-json/bringatrailer/1.0/data/listings-filter';
+        const firstPageResponse = await superagent.post(backendUrl)
+        .send(getQueryBody(1))
+        .set('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8').end();
+        const totalPages = firstPageResponse.body?.pages_total;
+
+
+
+        // const startingPoint = baseUrl;
+        // const browser = await puppeteer.launch({
+        //     headless: true,
+        //     args: ['--no-sandbox']
+        // });
+        // const page = await browser.newPage();
+        // await page.setViewport({
+        //     width: 1200,
+        //     height: 800
+        // });
+        // await page.setUserAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36");
+        // log.info('Loading page...');
+        // await page.goto(startingPoint, { timeout: 60000 });
+        // log.info('page loaded');
+        
+        // //get total page count
+        // const visiblePageNumbers = await Promise.all((await page.$$(selectors.totalPageIndicator)).map(async element => await page.evaluate(el => el.textContent, element)))
+        // const highestPage = Math.max(...visiblePageNumbers.map(num => parseInt(num)));
+        for(let pageNumber = 1;pageNumber < totalPages;pageNumber++) {
+            const pageContent = await superagent.post(backendUrl)
+            .send(getQueryBody(pageNumber))
+            .set('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8').end();
+            
+            let cars = await this.filterCompletedLinks((pageContent.body?.items || []).map(c => c.url)).map(item => {
+                return {
+                    vin: null, //I really don't feel like loading each link to pull the vin right now. Maybe later.
+                    url: item
+                }
+            });
+            log.info(cars);
+            cars.forEach(car => {
+                superagent.post(`${process.env.parentUrl}/lead/createFromCrawler`)
+                    .send({
+                        url: car.url,
+                        vinNumber: car.vin
+                    })
+                    .set('authorization', `Basic ${process.env.crawlerToken}`)
+                    .end((err, res) => {
+                        if(err){
+                            log.error({
+                                message: 'Failed to send lead', 
+                                error: err
+                            });
+                        }
+                    });
+            })
+        }
+
+    },
+    processPage: async function(page) {
+        const candidateLeads = [];
+        const vinParser = genericVinParserFactory({
+            vinElementSelector: selectors.vinSpan,
+            vinRegex: /(?<vin>A\d\w397\w\d{6})/i
+        })
+
+        //start with this page, pull all the carname elements for their links
+        let links = await page.$$(selectors.listingInternalLink);
+        links = await Promise.all(links.map(async element => baseUrl + (await page.evaluate(el => el.href, element))));
+
+        links = await this.filterCompletedLinks(links);
+
+
+        log.info(`Found ${links.length} unexplored links...`);
+
+
+
+        for(let i = 0;i < links.length;i++){
+            let link = links[i];
+            const newTab = await page.browser().newPage();
+            await newTab.setViewport({
+                width: 1200,
+                height: 800
+            });
+            await newTab.setUserAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36");
+            log.info('Loading page...');
+            await newTab.goto(link, { timeout: 60000 });
+            log.info({
+                message: 'loaded new tab',
+                link
+            });
+            const possibleVin = await vinParser(newTab);
+            if(possibleVin){
+
+                //get the external link
+                const externalAnchor = await newTab.$(selectors.listingExternalLink);
+                if(externalAnchor) {
+                    //candidate found
+
+                    const externalLink = await page.evaluate(el => el.href, externalAnchor);
+                    candidateLeads.push({
+                        vin: possibleVin, 
+                        url: externalLink
+                    });
+                }
+
+                
+            }
+            await newTab.close();
+
+        }
+        return candidateLeads;
+    },
+    filterCompletedLinks: function(links){
+        return new Promise((resolve, reject) => {
+            superagent.post(`${process.env.parentUrl}/lead/crawler/filterUrls`)
+            .send({
+                urls: links
+            })
+            .set('authorization', `Basic ${process.env.crawlerToken}`)
+            .set('Accept', 'application/json')
+            .end((err, res) => {
+                if(err){
+                    log.error({
+                        message: 'Failed to filter urls',
+                        error: err
+                    });
+                    return reject(err);
+                }
+                resolve(res.body?.urls);
+            });
+        })
+
+    }
+}

server/crawlers/classic.crawler.js

@@ -0,0 +1,137 @@
+const puppeteer = require('puppeteer');
+const genericVinParserFactory = require('../processors/generics/generic-vin-parser');
+const superagent = require('superagent');
+const {log} = require('clew-logger');
+
+const selectors = {
+    listingInternalLink: "h3>a.typography-body1",
+    vinSpan: "h2>span.flex-1",
+    listingExternalLink: "#vehicle_gotoAuction>a",
+    totalPageIndicator: ".mx-2>span.typography-subtitle1:nth-child(3)"
+}
+const baseUrl = 'https://www.classic.com/m/amc/amx'
+
+module.exports = {
+    cronString: '00 6 * * 0',
+    run: async function () {
+
+
+        const startingPoint = baseUrl;
+        const browser = await puppeteer.launch({
+            headless: true,
+            args: ['--no-sandbox']
+        });
+        const page = await browser.newPage();
+        await page.setViewport({
+            width: 1200,
+            height: 800
+        });
+        await page.setUserAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36");
+        log.info('Loading page...');
+        await page.goto(startingPoint, { timeout: 60000 });
+        log.info('page loaded');
+        
+        //get total page count
+        const visiblePageNumbers = await Promise.all((await page.$$(selectors.totalPageIndicator)).map(async element => await page.evaluate(el => el.textContent, element)))
+        const highestPage = Math.max(...visiblePageNumbers.map(num => parseInt(num)));
+        for(let pageNumber = 1;pageNumber < highestPage;pageNumber++) {
+            log.info(`loading page # ${pageNumber}`)
+            await page.goto(`${startingPoint}?page=${pageNumber}`, {timeout: 60000});
+            const cars = await module.exports.processPage(page);
+            log.info(cars);
+            cars.forEach(car => {
+                superagent.post(`${process.env.parentUrl}/lead/createFromCrawler`)
+                    .send({
+                        url: car.url,
+                        vinNumber: car.vin
+                    })
+                    .set('authorization', `Basic ${process.env.crawlerToken}`)
+                    .end((err, res) => {
+                        if(err){
+                            log.error({
+                                message: 'Failed to send lead', 
+                                error: err
+                            });
+                        }
+                    });
+            })
+        }
+        await browser.close();
+
+    },
+    processPage: async function(page) {
+        const candidateLeads = [];
+        const vinParser = genericVinParserFactory({
+            vinElementSelector: selectors.vinSpan,
+            vinRegex: /(?<vin>A\d\w397\w\d{6})/i
+        })
+
+        //start with this page, pull all the carname elements for their links
+        let links = await page.$$(selectors.listingInternalLink);
+        links = await Promise.all(links.map(async element => (await page.evaluate(el => el.href, element))));
+
+        links = await this.filterCompletedLinks(links);
+
+
+        log.info(`Found ${links.length} unexplored links...`);
+
+
+
+        for(let i = 0;i < links.length;i++){
+            let link = links[i];
+            const newTab = await page.browser().newPage();
+            await newTab.setViewport({
+                width: 1200,
+                height: 800
+            });
+            await newTab.setUserAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36");
+            log.info('Loading page...');
+            await newTab.goto(link, { timeout: 60000 });
+            log.info({
+                message: 'loaded new tab',
+                link
+            });
+            const possibleVin = await vinParser(newTab);
+            if(possibleVin){
+
+                //get the external link
+                const externalAnchor = await newTab.$(selectors.listingExternalLink);
+                if(externalAnchor) {
+                    //candidate found
+
+                    const externalLink = await page.evaluate(el => el.href, externalAnchor);
+                    candidateLeads.push({
+                        vin: possibleVin, 
+                        url: externalLink
+                    });
+                }
+
+                
+            }
+            await newTab.close();
+
+        }
+        return candidateLeads;
+    },
+    filterCompletedLinks: function(links){
+        return new Promise((resolve, reject) => {
+            superagent.post(`${process.env.parentUrl}/lead/crawler/filterUrls`)
+            .send({
+                urls: links
+            })
+            .set('authorization', `Basic ${process.env.crawlerToken}`)
+            .set('Accept', 'application/json')
+            .end((err, res) => {
+                if(err){
+                    log.error({
+                        message: 'Failed to filter urls',
+                        error: err
+                    });
+                    return reject(err);
+                }
+                resolve(res.body?.urls);
+            });
+        })
+
+    }
+}

server/crawlers/topclassiccarsforsale.crawler.js

@@ -1,12 +1,14 @@
 const puppeteer = require('puppeteer');
 const genericVinParserFactory = require('../processors/generics/generic-vin-parser');
 const superagent = require('superagent');
+const {log} = require('clew-logger');
 module.exports = {
-    cronString: '0 22 * * *',
+    cronString: '15 23 * * 0',
     run: async function () {
         const startingPoint = 'https://topclassiccarsforsale.com/amc';
         const browser = await puppeteer.launch({
-            headless: true
+            headless: true,
+            args: ['--no-sandbox']
         });
         const page = await browser.newPage();
         await page.setViewport({
@@ -14,29 +16,36 @@ module.exports = {
             height: 800
         });
         await page.setUserAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36");
-        console.log('Loading page...');
+        log.info('Loading page...');
         await page.goto(startingPoint, { timeout: 60000 });
-        console.log('page loaded');
+        log.info('page loaded');
         
         //get total page count
         const visiblePageNumbers = await Promise.all((await page.$$('span.pgs > a')).map(async element => await page.evaluate(el => el.textContent, element)))
         const highestPage = Math.max(...visiblePageNumbers.map(num => parseInt(num)));
         for(let pageNumber = 1;pageNumber < highestPage;pageNumber++) {
-            console.log('loading page #', pageNumber)
+            log.info(`loading page # ${pageNumber}`)
             await page.goto(`${startingPoint}/page/${pageNumber}/`, {timeout: 60000});
             const cars = await module.exports.processPage(page);
-            console.log(cars);
+            log.info(cars);
             cars.forEach(car => {
-                superagent.post('http://localhost:3000/lead/createFromCrawler')
+                superagent.post(`${process.env.parentUrl}/lead/createFromCrawler`)
                     .send({
-                        url: car.url
+                        url: car.url,
-                    }).end((err, res) => {
+                        vinNumber: car.vin
+                    })
+                    .set('authorization', `Basic ${process.env.crawlerToken}`)
+                    .end((err, res) => {
                         if(err){
-                            console.error('Failed to send lead', err);
+                            log.error({
+                                message: 'Failed to send Lead',
+                                error: err
+                            });
                         }
                     });
             })
         }
+        await browser.close();
 
     },
     processPage: async function(page) {
@@ -49,6 +58,14 @@ module.exports = {
         //start with this page, pull all the carname elements for their links
         let links = await page.$$('.carname');
         links = await Promise.all(links.map(async element => await page.evaluate(el => el.href, element)));
+
+        links = await this.filterCompletedLinks(links);
+
+
+        log.info(`Found ${links.length} unexplored links...`);
+
+
+
         for(let i = 0;i < links.length;i++){
             let link = links[i];
             const newTab = await page.browser().newPage();
@@ -57,9 +74,12 @@ module.exports = {
                 height: 800
             });
             await newTab.setUserAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36");
-            console.log('Loading page...');
+            log.info('Loading page...');
             await newTab.goto(link, { timeout: 60000 });
-            console.log('loaded new tab', link);
+            log.info({
+                message: 'loaded new tab',
+                link
+            });
             const possibleVin = await vinParser(newTab);
             if(possibleVin){
                 //candidate found
@@ -72,5 +92,26 @@ module.exports = {
 
         }
         return candidateLeads;
+    },
+    filterCompletedLinks: function(links){
+        return new Promise((resolve, reject) => {
+            superagent.post(`${process.env.parentUrl}/lead/crawler/filterUrls`)
+            .send({
+                urls: links
+            })
+            .set('authorization', `Basic ${process.env.crawlerToken}`)
+            .set('Accept', 'application/json')
+            .end((err, res) => {
+                if(err){
+                    log.error({
+                        message: 'Failed to filter urls',
+                        error: err
+                    });
+                    return reject(err);
+                }
+                resolve(res.body?.urls);
+            });
+        })
+
     }
 }

server/index.js

@@ -0,0 +1,163 @@
+const puppeteer = require('puppeteer')
+const superagent = require('superagent');
+const path = require('path');
+const express = require('express');
+const glob = require('glob');
+const _ = require('lodash');
+const app = express();
+var bodyParser = require('body-parser')
+const CronJob = require('cron').CronJob;
+const clew = require('clew-logger');
+clew.create({
+  prod: true,
+  prodHost: 'http://tonkatown.docker:3100',
+  appTag: 'listingExtractor',
+
+})
+const log = clew.log;
+// console.log(log);
+app.use(clew.context.attachContext('listingExtractor'));
+app.use(clew.requestLogger);
+// setTimeout(() => console.log(clew.log), 5000)
+
+const processors = [];
+//load the processor files
+glob(path.resolve(__dirname, 'processors/*.processor.js'), (error, matches) => {
+  clew.context.assignRequestId(clew.context.init('listingExtractor'), () => {
+    _.forEach(matches, file => {
+      const processor = require(path.resolve(__dirname, file));
+      processors.push(processor);
+    })
+    log.info(`${matches.length} processors loaded`);
+  }, {
+    process: 'processor loader'
+  })
+
+})
+//load the crawler files
+glob(path.resolve(__dirname, 'crawlers/*.crawler.js'), (error, matches) => {
+  clew.context.assignRequestId(clew.context.init('listingExtractor'), () => {
+
+    _.forEach(matches, file => {
+      const crawler = require(path.resolve(__dirname, file));
+      const cronJob = new CronJob(crawler.cronString, () => {
+        const ctx = clew.context.init('listingExtractor');
+        clew.context.assignRequestId(ctx, crawler.run, {
+          crawler: file
+        });
+      });
+      cronJob.start();
+
+    })
+    log.info(`${matches.length} crawlers loaded`);
+  }, {
+    process: 'crawler loader'
+  })
+})
+app.use(bodyParser.json())
+app.post('/convertGalleryToHar', async (req, res) => {
+  const url = req.body.url;
+  log.info(url);
+
+  // get processor
+  const searchableHostname = (new URL(url)).hostname.replace(/^www\./i, '');
+  log.info(`Searching for hostname: ${searchableHostname}`)
+  const processor = _.find(processors, (processor) => (searchableHostname) === (processor.baseUrl.replace(/^www\./i, '')));
+  if (!processor) {
+    return res.status(400).json({
+      message: 'Could not find processor for url'
+    })
+  }
+  log.info(`Processor found ${processor.baseUrl}`);
+  try {
+    const payloads = await run(url, processor);
+    res.status(200).json({
+      vin: payloads.vin,
+      mileage: payloads.mileage,
+      log: {
+        entries: payloads.payloads
+      }
+      // payloads
+    })
+  } catch (error) {
+    log.info({ message: error.message });
+    res.status(500).json(error);
+  }
+
+})
+app.get('/supportedSites', async (req, res) => {
+  res.status(200).json(processors.map(p => p.baseUrl));
+})
+app.listen(2667);
+
+async function run(url, processor) {
+  const browser = await puppeteer.launch({
+    headless: true,
+    args: ['--no-sandbox']
+  });
+
+  const page = await browser.newPage();
+  await page.setViewport({
+    width: 1200,
+    height: 800
+  });
+  await page.setUserAgent("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36");
+  log.info('Loading page...');
+  await page.goto(url, { timeout: 60000 });
+  log.info('page loaded');
+  log.info('attempting to parse fields');
+  const vin = await processor.parseVIN(page);
+  log.info(`parsed VIN: ${vin}`);
+  const mileage = await processor.parseMileage(page);
+  log.info(`parsed Mileage: ${mileage}`);
+  const galleryUrls = await processor.execute(page);
+  await page.close();
+
+  log.info(`Done collecting URLS (${galleryUrls.length})`);
+  const payloads = await Promise.all(galleryUrls.map(image => new Promise(async (resolve, reject) => {
+    if (image.url) {
+      superagent.get(image.url).responseType('blob').timeout(30000).then(function (response) {
+        if (response.statusCode == 200) {
+          log.info(`Resolving: ${image.url}`)
+          return resolve({
+            response: {
+              content: {
+                mimeType: response.headers["content-type"],
+                encoding: 'base64',
+                text: response.body.toString('base64')
+              }
+            }
+          });
+        } else {
+          log.info(`Invalid status code ${response.statusCode} for ${image.url}`);
+          resolve({})
+
+        }
+      }).catch(error => {
+        log.error(error.message)
+        resolve({})
+      });
+    } else if (image.base64) {
+      return resolve({
+        response: {
+          content: {
+            mimeType: image.contentType,
+            encoding: 'base64',
+            text: image.base64
+          }
+        }
+      });
+    } else {
+      resolve({})
+    }
+  })))
+  log.info('URLS done downloading')
+  await browser.close();
+  return {
+    vin,
+    payloads,
+    mileage
+  };
+}
+
+// run();

server/processors/barrettjackson.processor.js

@@ -14,7 +14,6 @@ module.exports = {
         const thumbnailNodes = await page.$$(thumbnailSelector);
         const sourcesFromThumbnails = await Promise.all(thumbnailNodes.map(async carouselItem => {
             const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
-            // console.log(src);
             return { url: convertThumbnailUrlToFullSize(src) };
         }));
         const sources = sourcesFromThumbnails;

server/processors/classiccarsbay.processor.js

@@ -1,15 +1,17 @@
 const genericVinParserFactory = require("./generics/generic-vin-parser");
+const {log} = require('clew-logger');
+
 module.exports = {
-    baseUrl: 'survivor-cars.com',
+    baseUrl: 'classiccarsbay.com',
     async execute(page) {
-        const gallerySelector = '.show-car-thumbs'
+        const gallerySelector = '.swiper-wrapper'
-        const imageSelector = '.show-car-thumbs > a';
+        const imageSelector = '.swiper-image';
         await page.waitForSelector(gallerySelector);
         const images = await page.$$(imageSelector);
         const sources = await Promise.all(images.map(async carouselItem => {
-            const src = await page.evaluate(el => el.getAttribute('href'), carouselItem);
+            const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
-            console.log(src);
+            log.info(src);
-            return { url: src };
+            return { url: `https://${module.exports.baseUrl}${src}` };
         }));
         return sources;
     },

server/processors/cruisinclassicsinc.processor.js

@@ -0,0 +1,14 @@
+const _ = require('lodash');
+const genericVinParserFactory = require("./generics/generic-vin-parser");
+const genericShowCarThumbs = require('./generics/generic-showCarThumbs');
+module.exports = {
+    baseUrl: 'cruisinclassicsinc.com',
+    execute: genericShowCarThumbs({}),
+    parseVIN: async function (page) {
+        return null;
+    },
+    parseMileage: async function (page) {
+        return null;
+    }
+   
+}

server/processors/customclassics.processor.js

@@ -0,0 +1,15 @@
+
+const _ = require('lodash');
+const genericVinParserFactory = require("./generics/generic-vin-parser");
+const genericShowCarThumbs = require('./generics/generic-showCarThumbs');
+module.exports = {
+    baseUrl: 'customclassics.net',
+    execute: genericShowCarThumbs({}),
+    parseVIN: async function (page) {
+        return null;
+    },
+    parseMileage: async function (page) {
+        return null;
+    }
+   
+}

server/processors/davidsclassiccars.processor.js

@@ -1,5 +1,6 @@
 const _ = require('lodash');
 const genericVinParserFactory = require("./generics/generic-vin-parser");
+const {log} = require('clew-logger');
 
 module.exports = {
     baseUrl: 'davidsclassiccars.com',
@@ -8,10 +9,9 @@ module.exports = {
         await page.waitForSelector(pageLoadIndicator);
         const imageSelector = '.carimage > img';
         const images = await page.$$(imageSelector);
-        console.log(`Found ${images.length} images...`)
+        log.info(`Found ${images.length} images...`)
         const sources = await Promise.all(images.map(async carouselItem => {
             const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
-            // console.log(src);
             return { url: this.baseUrl + src };
         }));
         return sources;

server/processors/ebay.processor.js

@@ -1,18 +1,34 @@
 
 const genericVinParserFactory = require("./generics/generic-vin-parser");
+const {log} = require('clew-logger');
+
 module.exports = {
     baseUrl: 'ebay.com',
     execute: async function (page) {
         const thumbnailGallerySelector = '.ux-thumb-image-carousel';
         const thumbnailGalleryItemSelector = '.ux-image-filmstrip-carousel-item > img:nth-child(1)';
-        const sellerSelector = '.ux-seller-section__item--seller > a:nth-child(1) > span:nth-child(1)'
+        const sellerSelector = '.ux-seller-section__item--seller > a:nth-child(1) > span:nth-child(1)';
+        const alternateSellerSelector = 'html.font-marketsans body.vi-body.en-US.gh-flex div.vi-evo div.main-container div.vim.x-vi-evo-main-container.template-evo-avip div#CenterPanel.center-panel-container.vi-mast div.vi-grid div.vi-mast__grid div#RightSummaryPanel.right-summary-panel-container.vi-mast__col-right div#mainContent.vim.x-evo-atf-right-river div.vim.d-vi-evo-region div.vim.x-sellercard-atf_main.mar-t-12 div.vim.x-sellercard-atf div.x-sellercard-atf__info div.x-sellercard-atf__info__about-seller a.ux-action span.ux-textspans.ux-textspans--BOLD';
         const descriptionSelectr = '#desc_ifr';
+
+        const filmstripSelector = '.filmstrip';
+        const filmstripThumbnailSelector = '.ux-image-grid-item > img';
         //page is considered loaded
         //get the seller to see if we can tell what layout its using
 
         const sellerElement = await page.$(sellerSelector);
-        const seller = await page.evaluate(el => el.textContent, sellerElement);
+        let seller = null;
-        console.log(`User is '${seller}'`);
+        try {
+            seller = await page.evaluate(el => el.textContent, sellerElement);
+        } catch(error) {
+            log.info('Failed default seller selector, trying alternative...');
+            try {
+                seller = await page.evaluate(el => el.textContent, await page.$(alternateSellerSelector));
+            } catch(error){
+                log.info('Failed alternative seller selector, defaulting to default strategy...');
+            }
+        }
+        log.info(`User is '${seller}'`);
         if (seller === 'classicautomall') {
             await page.waitForSelector(descriptionSelectr);
 
@@ -28,26 +44,39 @@ module.exports = {
             if (hasThumbnailGallery) {
 
                 const carouselItems = await page.$$(thumbnailGalleryItemSelector);
-                // console.log(carouselItems)
                 const sourcesFromThumbnailGallery = await Promise.all(carouselItems.map(async carouselItem => {
                     const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
-                    // console.log(src);
                     return { url: convertThumbnailUrlToFullSize(src) };
                 }));
                 sources = sourcesFromThumbnailGallery;
             }
-            console.log('Performing autoscroll')
+            log.info(`Found ${sources.length} images through thumbnails location as backup....`);
+            log.info('Checking filmstrip...');
+            const hasFilmstrip = !!(await page.$(filmstripSelector));
+            if(hasFilmstrip) {
+                const carouselItems = await page.$$(filmstripThumbnailSelector);
+                const sourcesFromThumbnailGallery = await Promise.all(carouselItems.map(async carouselItem => {
+                    const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
+                    return { url: convertThumbnailUrlToFullSize(src) };
+                }));
+                sources.push(...sourcesFromThumbnailGallery.filter((o => !!o.url)));
+            }
+            log.info(`Found ${sources.length} images through filmstrip location as backup....`);
+
+
+
+            log.info('Performing autoscroll')
             await autoScroll(page); //auto scroll to trigger loading of description iframe
-            console.log('autoscroll complete');
+            log.info('autoscroll complete');
             
             const descriptionIframe = await page.$('#desc_ifr');
-            console.log('has description iframe?', !!descriptionIframe);
+            log.info(`has description iframe? ${!!descriptionIframe}`);
             const descriptionUrl = await page.evaluate(el => el.getAttribute('src'), descriptionIframe)
-            console.log('has description URL?', descriptionUrl);
+            log.info(`has description URL? ${descriptionUrl}`);
             await page.goto(descriptionUrl);
             //check the description if it has viewall button
             const hasViewAllButton = !!(await page.$('.thumbnail-list-wrapper > a.cgg-btn:nth-child(6)'))
-            console.log('hasViewALl', hasViewAllButton)
+            log.info(`hasViewALl ${hasViewAllButton}`)
             if (hasViewAllButton) {
                 const openFullGallerySelector = '.thumbnail-list-wrapper > a.cgg-btn:nth-child(6)';
                 const nextUrl = await page.evaluate(el => el.getAttribute('href'), await page.$(openFullGallerySelector));
@@ -56,20 +85,18 @@ module.exports = {
                 const images = await page.$$('.photo');
                 sources = await Promise.all(images.map(async carouselItem => {
                     const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
-                    // console.log(src);
                     return { url: src };
                 }));
                 return sources;
             } 
             const hasSectionedImages = !!(await page.$('main.content'));
-            console.log(hasSectionedImages);
+            log.info(`hasSectionedImages: ${hasSectionedImages}`);
             if(hasSectionedImages) {
                 const imageSelector = 'main.content > div > div > div > a > img';
                 const images = await page.$$(imageSelector);
-                console.log(images.length)
+                log.info(images.length)
                 sources = await Promise.all(images.map(async carouselItem => {
                     const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
-                    // console.log(src);
                     return { url: src };
                 }));
                 return sources;
@@ -86,6 +113,7 @@ module.exports = {
     }
 }
 function convertThumbnailUrlToFullSize(url) {
+    if(!url) return null;
     const baseName = url.substring(url.lastIndexOf('/') + 1)
     const basePath = url.replace(baseName, '');
     return `${basePath}s-l1600.jpg`;
@@ -100,7 +128,6 @@ async function classicautomall(page) {
 
     const sources = await Promise.all(items.map(async carouselItem => {
         const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
-        // console.log(src);
         return { url: src };
     }));
     return sources;
@@ -116,7 +143,6 @@ async function gatewayclassiccars(page) {
     const images = await page.$$('.photo');
     const sources = await Promise.all(images.map(async carouselItem => {
         const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
-        // console.log(src);
         return { url: src };
     }));
     return sources;
@@ -130,7 +156,6 @@ async function autoScroll(page){
                 var scrollHeight = document.body.scrollHeight;
                 window.scrollBy(0, distance);
                 totalHeight += distance;
-                console.log(totalHeight);
                 if(totalHeight >= scrollHeight - window.innerHeight){
                     clearInterval(timer);
                     resolve();

server/processors/generics/generic-bootstrap.js

@@ -1,4 +1,6 @@
 const uuid = require('uuid');
+const {log} = require('clew-logger');
+
 module.exports = function (processorConfig) {
 
     downloadGallery = async (vin) => {
@@ -7,7 +9,6 @@ module.exports = function (processorConfig) {
             var imgWrap;
             do {
                 imgWrap = document.elementFromPoint(100, 100);
-                console.log(imgWrap.classList);
                 await delay(50);
             } while (imgWrap.classList.contains('pswp__img--placeholder'));
 
@@ -19,13 +20,20 @@ module.exports = function (processorConfig) {
 
         // Find the active image, surround it in an anchor tag, then click it.
         async function downloadImage(id) {
-            const imgWrap = document.elementFromPoint(300, 300);
+            const imgWrap = document.elementFromPoint(window.innerWidth / 2, 300);
             const children = imgWrap.querySelectorAll('.pswp__img');
             const img = children[children.length - 1];
-            console.log(img, imgWrap);
             // debugger;
             // Full image hasn't loaded yet
             const src = imgWrap.src.split('?')[0]// get rid of querystring
+            //check for a duplicated src
+            for(var i = 0;i < galleryUrls.length;i++) {
+                if(src === galleryUrls[i].url && externalCounter > 1) {
+                    //we're using the fallback counter, reset it to 1 to trigger the exit.
+                    externalCounter = 0;
+                    return;
+                }
+            }
             return downloadSrc(src, id);
         }
 
@@ -37,11 +45,16 @@ module.exports = function (processorConfig) {
         }
 
         function nextImage() {
-            document.querySelector('.pswp__button.pswp__button--arrow--right').click();
+            //get the navigation button element
+            const navigation = document.querySelector('.pswp__button.pswp__button--arrow--right') || document.querySelector('.pswp__button.pswp__button--arrow--next')
+            navigation.click();
+            externalCounter += 1;
         }
-
+        var externalCounter = 1;
         function getCounterValue() {
-            const [position, total] = document.querySelector('.pswp__counter').textContent.split('/');
+            const internalCounterElement = document.querySelector('.pswp__counter');
+            if(!internalCounterElement) return externalCounter;
+            const [position, total] = internalCounterElement.textContent.split('/');
             return parseInt(position.trim(), 10);
         }
 
@@ -66,7 +79,7 @@ module.exports = function (processorConfig) {
     }
 
     return async function (page) {
-        console.log('Running generic boostrap extractor')
+        log.info('Running generic boostrap extractor')
 
         const allResultsSelector = processorConfig.pageLoadIndicator;
         await page.waitForSelector(allResultsSelector);
@@ -77,15 +90,18 @@ module.exports = function (processorConfig) {
             let element = await page.$(vinSelector)
             vin = await page.evaluate(el => el.textContent, element);
         }).catch(error => {
-            console.error('Unable to grab VIN, falling back to UUID');
+            log.error({
+                message: 'Unable to grab VIN, falling back to UUID',
+                error: error
+            });
             vin = uuid.v4();
         });
-        console.log(vin);
+        log.info(vin);
 
         const firstImageLinkSelector = processorConfig.carouselTrigger;
         await page.click(firstImageLinkSelector);
         await page.waitForSelector('.pswp__img')
-        console.log('Gallery is loaded, fetching URLS')
+        log.info('Gallery is loaded, fetching URLS')
         const galleryUrls = await page.evaluate(downloadGallery, vin);
         
 

server/processors/generics/generic-showCarThumbs.js

@@ -0,0 +1,22 @@
+const _ = require('lodash');
+const {log} = require('clew-logger');
+
+module.exports = function (config) {
+  return async function (page) {
+        const pageLoadIndicator = '.show-car-thumbs';
+        await page.waitForSelector(pageLoadIndicator);
+        const imageSelector = '.show-car-thumbs > a';
+        const images = await page.$$(imageSelector);
+        log.info(`Found ${images.length} images...`)
+        const sources = await Promise.all(images.map(async carouselItem => {
+            const src = await page.evaluate(el => el.getAttribute('data-original'), carouselItem);
+            if(!src) {
+                return {}
+            }
+            return { url: src };
+        }));
+        return sources;
+
+    }
+   
+}

server/processors/hemmings.processor.js

@@ -10,9 +10,9 @@ module.exports = {
         carouselTrigger: '.bg-center'
       }),
     baseUrl :'hemmings.com',
-    parseVIN: genericVinParserFactory({
+    parseVIN: async function(page) {
-      vinElementSelector: `div.py-4.px-4.flex.flex-col.lg:flex-row.flex-wrap div.w-full.lg:w-1/2.my-2 div.flex.flex-col div.text-lg.lg:text-xl.whitespace-normal`
+      return null;
-    }),
+  },
     parseMileage: async function (page) {
       return null;
     }

server/processors/mecum.processor.js

@@ -0,0 +1,20 @@
+const genericBootstrapFactory = require("./generics/generic-bootstrap");
+const genericVinParserFactory = require("./generics/generic-vin-parser");
+
+
+module.exports = {
+    execute: genericBootstrapFactory({
+        baseUrl: 'mecum.com',
+        pageLoadIndicator: 'html body div#__next main#page-content div.Container_containerW__RTJwy.Container_paddingBtm__RIfcP article.innerWrap section.LotHeader_lotHeader__9tt91 div.LotHeader_gallerySection__LZQOt',
+        vinSelector: 'html body div#__next main#page-content div.Container_containerW__RTJwy.Container_paddingBtm__RIfcP article.innerWrap section.LotHeader_lotHeader__9tt91 div.LotHeader_headerBottom__F8FRR div.LotHeader_meta__3S0lZ div.LotHeader_odometerSerial___fuHb div p',
+        carouselTrigger: 'html body div#__next main#page-content div.Container_containerW__RTJwy.Container_paddingBtm__RIfcP article.innerWrap section.LotHeader_lotHeader__9tt91 div.LotHeader_gallerySection__LZQOt section.ImageGallery_imageGallery__qITSq.LotHeader_gallery__OrqMX.image-gallery.ImageGallery_standard__a65TU div.ImageGallery_mainImage__GFBTp button.ImageGallery_imageButton__ld6Eg'
+      }),
+    baseUrl :'mecum.com',
+    parseVIN: genericVinParserFactory({
+      vinElementSelector: `html body div#__next main#page-content div.Container_containerW__RTJwy.Container_paddingBtm__RIfcP article.innerWrap section.LotHeader_lotHeader__9tt91 div.LotHeader_headerBottom__F8FRR div.LotHeader_meta__3S0lZ div.LotHeader_odometerSerial___fuHb div p`
+    }),
+    parseMileage: async function (page) {
+      return null;
+    }
+}
+

server/processors/superstockamx.processor.js

@@ -0,0 +1,46 @@
+const genericVinParserFactory = require("./generics/generic-vin-parser");
+const {log} = require('clew-logger');
+
+module.exports = {
+    baseUrl: 'superstockamx.com',
+    async execute(page) {
+        const gallerySelector = '.main-bg'
+        const imageSelector = 'img';
+        await page.waitForSelector(gallerySelector);
+        const images = await page.$$(imageSelector);
+        const sources = await Promise.all(images.map(async carouselItem => {
+            const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
+
+            if(src.includes('base64')) {
+                //base 64 pasted image
+                log.info('Found base64 image.')
+                
+                const regex = /^data:(?<contentType>[^;]+);base64,(?<data>.+)/g;
+                const matches = src.matchAll(regex);
+                const groups = Array.from(matches, m => m.groups)[0]
+
+
+
+                return {base64: groups.data,
+                        contentType: groups.contentType !== '<' ? groups.contentType : 'image/jpeg'};
+            } else if(src.startsWith('http')) {
+                return { url: src };
+                
+            } else { 
+                return {}
+                // anything hosted seperately on his site is not an actual car pic
+                // return {
+                //     url: module.exports.baseUrl + '/' + src
+                // }
+            }
+            
+        }));
+        return sources;
+    },
+    parseVIN: async function (page) {
+        return null;
+    },
+    parseMileage: async function (page) {
+        return null;
+    }
+}

server/processors/survivor-cars.processor.js

@@ -0,0 +1,13 @@
+const genericVinParserFactory = require("./generics/generic-vin-parser");
+const genericShowCarThumbs = require('./generics/generic-showCarThumbs');
+
+module.exports = {
+    baseUrl: 'survivor-cars.com',
+    execute: genericShowCarThumbs({}),
+    parseVIN: async function(page) {
+        return null;
+    },
+    parseMileage: async function (page) {
+        return null;
+    }
+}

server/processors/topclassiccarsforsale.processor.js

@@ -1,4 +1,6 @@
 const genericVinParserFactory = require("./generics/generic-vin-parser");
+const {log} = require('clew-logger');
+
 module.exports = {
     baseUrl: 'topclassiccarsforsale.com',
     async execute(page) {
@@ -8,7 +10,7 @@ module.exports = {
         const images = await page.$$(imageSelector);
         const sources = await Promise.all(images.map(async carouselItem => {
             const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
-            console.log(src);
+            log.info(src);
             return { url: this.baseUrl + src };
         }));
         return sources;

server/processors/vanguardmotorsales.processor.js

@@ -0,0 +1,15 @@
+const _ = require('lodash');
+const genericVinParserFactory = require("./generics/generic-vin-parser");
+const genericShowCarThumbs = require('./generics/generic-showCarThumbs');
+
+module.exports = {
+    baseUrl: 'vanguardmotorsales.com',
+    execute: genericShowCarThumbs({}),
+    parseVIN: async function (page) {
+        return null;
+    },
+    parseMileage: async function (page) {
+        return null;
+    }
+   
+}