ListingExtractor/server/processors/ebay.processor.js

const genericVinParserFactory = require("./generics/generic-vin-parser");
const {log} = require('clew-logger');

module.exports = {
    baseUrl: 'ebay.com',
    execute: async function (page) {
        const thumbnailGallerySelector = '.ux-thumb-image-carousel';
        const thumbnailGalleryItemSelector = '.ux-image-filmstrip-carousel-item > img:nth-child(1)';
        const sellerSelector = '.ux-seller-section__item--seller > a:nth-child(1) > span:nth-child(1)';
        const alternateSellerSelector = 'html.font-marketsans body.vi-body.en-US.gh-flex div.vi-evo div.main-container div.vim.x-vi-evo-main-container.template-evo-avip div#CenterPanel.center-panel-container.vi-mast div.vi-grid div.vi-mast__grid div#RightSummaryPanel.right-summary-panel-container.vi-mast__col-right div#mainContent.vim.x-evo-atf-right-river div.vim.d-vi-evo-region div.vim.x-sellercard-atf_main.mar-t-12 div.vim.x-sellercard-atf div.x-sellercard-atf__info div.x-sellercard-atf__info__about-seller a.ux-action span.ux-textspans.ux-textspans--BOLD';
        const descriptionSelectr = '#desc_ifr';
        //page is considered loaded
        //get the seller to see if we can tell what layout its using

        const sellerElement = await page.$(sellerSelector);
        let seller = null;
        try {
            seller = await page.evaluate(el => el.textContent, sellerElement);
        } catch(error) {
            log.info('Failed default seller selector, trying alternative...');
            try {
                seller = await page.evaluate(el => el.textContent, await page.$(alternateSellerSelector));
            } catch(error){
                log.info('Failed alternative seller selector, defaulting to default strategy...');
            }
        }
        log.info(`User is '${seller}'`);
        if (seller === 'classicautomall') {
            await page.waitForSelector(descriptionSelectr);

            return await classicautomall(page);
        } else if (seller === 'gateway-classic-cars') {
            await page.waitForSelector(descriptionSelectr);

            return await gatewayclassiccars(page);
        } else {
            var sources = [];
            const hasThumbnailGallery = !!(await page.$(thumbnailGallerySelector));
            // const hasViewAllButton = !!(await page.$('.thumbnail-list-wrapper > a.cgg-btn:nth-child(6)'))
            if (hasThumbnailGallery) {

                const carouselItems = await page.$$(thumbnailGalleryItemSelector);
                const sourcesFromThumbnailGallery = await Promise.all(carouselItems.map(async carouselItem => {
                    const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
                    return { url: convertThumbnailUrlToFullSize(src) };
                }));
                sources = sourcesFromThumbnailGallery;
            }
            log.info('Performing autoscroll')
            await autoScroll(page); //auto scroll to trigger loading of description iframe
            log.info('autoscroll complete');
            
            const descriptionIframe = await page.$('#desc_ifr');
            log.info(`has description iframe? ${!!descriptionIframe}`);
            const descriptionUrl = await page.evaluate(el => el.getAttribute('src'), descriptionIframe)
            log.info(`has description URL? ${descriptionUrl}`);
            await page.goto(descriptionUrl);
            //check the description if it has viewall button
            const hasViewAllButton = !!(await page.$('.thumbnail-list-wrapper > a.cgg-btn:nth-child(6)'))
            log.info(`hasViewALl ${hasViewAllButton}`)
            if (hasViewAllButton) {
                const openFullGallerySelector = '.thumbnail-list-wrapper > a.cgg-btn:nth-child(6)';
                const nextUrl = await page.evaluate(el => el.getAttribute('href'), await page.$(openFullGallerySelector));
                await page.goto(nextUrl);
                await page.waitForSelector('.photo');
                const images = await page.$$('.photo');
                sources = await Promise.all(images.map(async carouselItem => {
                    const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
                    return { url: src };
                }));
                return sources;
            } 
            const hasSectionedImages = !!(await page.$('main.content'));
            log.info(hasSectionedImages);
            if(hasSectionedImages) {
                const imageSelector = 'main.content > div > div > div > a > img';
                const images = await page.$$(imageSelector);
                log.info(images.length)
                sources = await Promise.all(images.map(async carouselItem => {
                    const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
                    return { url: src };
                }));
                return sources;
            }
            
            return sources;
        }
    },
    parseVIN: genericVinParserFactory({
        vinElementSelector: `div.ux-layout-section__row div.ux-labels-values__values div.ux-labels-values__values-content div span.ux-textspans`
    }),
    parseMileage: async function (page) {
        return null;
    }
}
function convertThumbnailUrlToFullSize(url) {
    const baseName = url.substring(url.lastIndexOf('/') + 1)
    const basePath = url.replace(baseName, '');
    return `${basePath}s-l1600.jpg`;
}
async function classicautomall(page) {
    const descriptionIframe = await page.$('#desc_ifr');
    const descriptionUrl = await page.evaluate(el => el.getAttribute('src'), descriptionIframe)
    await page.goto(descriptionUrl);
    // const iframe = descriptionIframe.contentFrame();
    // await .waitForSelector('section.image-gallery')
    const items = await page.$$('.lg-gallery > img:nth-child(1)');

    const sources = await Promise.all(items.map(async carouselItem => {
        const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
        return { url: src };
    }));
    return sources;
}
async function gatewayclassiccars(page) {
    const descriptionIframe = await page.$('#desc_ifr');
    const descriptionUrl = await page.evaluate(el => el.getAttribute('src'), descriptionIframe)
    await page.goto(descriptionUrl);
    const openFullGallerySelector = '.thumbnail-list-wrapper > a.cgg-btn:nth-child(6)';
    const nextUrl = await page.evaluate(el => el.getAttribute('href'), await page.$(openFullGallerySelector));
    await page.goto(nextUrl);
    await page.waitForSelector('.photo');
    const images = await page.$$('.photo');
    const sources = await Promise.all(images.map(async carouselItem => {
        const src = await page.evaluate(el => el.getAttribute('src'), carouselItem);
        return { url: src };
    }));
    return sources;
}
async function autoScroll(page){
    await page.evaluate(async () => {
        await new Promise((resolve) => {
            var totalHeight = 0;
            var distance = 100;
            var timer = setInterval(() => {
                var scrollHeight = document.body.scrollHeight;
                window.scrollBy(0, distance);
                totalHeight += distance;
                if(totalHeight >= scrollHeight - window.innerHeight){
                    clearInterval(timer);
                    resolve();
                }
            }, 100);
        });
    });
}