ListingExtractor/processors/ebay.processor.js

const _ = require('lodash');
const path = require('path');
module.exports = {
    baseUrl: 'ebay.com',
    execute: async function (page) {
        const thumbnailGallerySelector = '.ux-thumb-image-carousel';
        const thumbnailGalleryItemSelector = '.ux-image-filmstrip-carousel-item > img:nth-child(1)';
        const sellerSelector = '.ux-seller-section__item--seller > a:nth-child(1) > span:nth-child(1)'
        const descriptionSelectr = '#desc_ifr';
        //page is considered loaded
        //get the seller to see if we can tell what layout its using

        const sellerElement = await page.$(sellerSelector);
        const seller = await page.evaluate(el => el.textContent, sellerElement);
        console.log(`User is '${seller}'`);
        if (seller === 'classicautomall') {
            await page.waitForSelector(descriptionSelectr);

            return await classicautomall(page);
        } else if (seller === 'gateway-classic-cars') {
            await page.waitForSelector(descriptionSelectr);

            return await gatewayclassiccars(page);
        } else {
            var sources = [];
            const hasThumbnailGallery = !!(await page.$(thumbnailGallerySelector));
            // const hasViewAllButton = !!(await page.$('.thumbnail-list-wrapper > a.cgg-btn:nth-child(6)'))
            if (hasThumbnailGallery) {

                const carouselItems = await page.$$(thumbnailGalleryItemSelector);
                // console.log(carouselItems)
                const sourcesFromThumbnailGallery = await Promise.all(carouselItems.map(async carouselItem => {
                    const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
                    // console.log(src);
                    return { url: convertThumbnailUrlToFullSize(src) };
                }));
                sources = sourcesFromThumbnailGallery;
            }
            console.log('Performing autoscroll')
            await autoScroll(page); //auto scroll to trigger loading of description iframe
            console.log('autoscroll complete');
            
            const descriptionIframe = await page.$('#desc_ifr');
            console.log('has description iframe?', !!descriptionIframe);
            const descriptionUrl = await page.evaluate(el => el.getAttribute('src'), descriptionIframe)
            console.log('has description URL?', descriptionUrl);
            await page.goto(descriptionUrl);
            //check the description if it has viewall button
            const hasViewAllButton = !!(await page.$('.thumbnail-list-wrapper > a.cgg-btn:nth-child(6)'))
            console.log('hasViewALl', hasViewAllButton)
            if (hasViewAllButton) {
                const openFullGallerySelector = '.thumbnail-list-wrapper > a.cgg-btn:nth-child(6)';
                const nextUrl = await page.evaluate(el => el.getAttribute('href'), await page.$(openFullGallerySelector));
                await page.goto(nextUrl);
                await page.waitForSelector('.photo');
                const images = await page.$$('.photo');
                sources = await Promise.all(images.map(async carouselItem => {
                    const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
                    // console.log(src);
                    return { url: src };
                }));
                return sources;
            } 
            const hasSectionedImages = !!(await page.$('main.content'));
            console.log(hasSectionedImages);
            if(hasSectionedImages) {
                const imageSelector = 'main.content > div > div > div > a > img';
                const images = await page.$$(imageSelector);
                console.log(images.length)
                sources = await Promise.all(images.map(async carouselItem => {
                    const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
                    // console.log(src);
                    return { url: src };
                }));
                return sources;
            }
            
            return sources;
        }
    }
}
function convertThumbnailUrlToFullSize(url) {
    const baseName = url.substring(url.lastIndexOf('/') + 1)
    const basePath = url.replace(baseName, '');
    return `${basePath}s-l1600.jpg`;
}
async function classicautomall(page) {
    const descriptionIframe = await page.$('#desc_ifr');
    const descriptionUrl = await page.evaluate(el => el.getAttribute('src'), descriptionIframe)
    await page.goto(descriptionUrl);
    // const iframe = descriptionIframe.contentFrame();
    // await .waitForSelector('section.image-gallery')
    const items = await page.$$('.lg-gallery > img:nth-child(1)');

    const sources = await Promise.all(items.map(async carouselItem => {
        const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
        // console.log(src);
        return { url: src };
    }));
    return sources;
}
async function gatewayclassiccars(page) {
    const descriptionIframe = await page.$('#desc_ifr');
    const descriptionUrl = await page.evaluate(el => el.getAttribute('src'), descriptionIframe)
    await page.goto(descriptionUrl);
    const openFullGallerySelector = '.thumbnail-list-wrapper > a.cgg-btn:nth-child(6)';
    const nextUrl = await page.evaluate(el => el.getAttribute('href'), await page.$(openFullGallerySelector));
    await page.goto(nextUrl);
    await page.waitForSelector('.photo');
    const images = await page.$$('.photo');
    const sources = await Promise.all(images.map(async carouselItem => {
        const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
        // console.log(src);
        return { url: src };
    }));
    return sources;
}
async function autoScroll(page){
    await page.evaluate(async () => {
        await new Promise((resolve) => {
            var totalHeight = 0;
            var distance = 100;
            var timer = setInterval(() => {
                var scrollHeight = document.body.scrollHeight;
                window.scrollBy(0, distance);
                totalHeight += distance;
                console.log(totalHeight);
                if(totalHeight >= scrollHeight - window.innerHeight){
                    clearInterval(timer);
                    resolve();
                }
            }, 100);
        });
    });
}