From cbfb0d07a9d608b16dd89ad132d88c9e1ff3e499 Mon Sep 17 00:00:00 2001 From: Edward Peterson Date: Tue, 28 Nov 2023 20:12:52 -0500 Subject: [PATCH] dockerized --- .gitignore | 1 + Dockerfile | 7 +++++++ README.md | 10 ++++++++++ docker-compose.yml | 15 +++++++++++++++ .../crawlers}/topclassiccarsforsale.crawler.js | 7 ++++--- index.js => server/index.js | 3 ++- .../processors}/barrettjackson.processor.js | 0 .../processors}/bringatrailer.processor.js | 0 .../processors}/classiccars.processor.js | 0 .../processors}/classiccarsbay.processor.js | 0 .../processors}/cruisinclassicsinc.processor.js | 0 .../processors}/customclassics.processor.js | 0 .../processors}/davidsclassiccars.processor.js | 0 .../processors}/ebay.processor.js | 0 .../processors}/generics/generic-bootstrap.js | 0 .../processors}/generics/generic-showCarThumbs.js | 0 .../processors}/generics/generic-vin-parser.js | 0 .../processors}/hemmings.processor.js | 0 .../processors}/mecum.processor.js | 0 .../processors}/superstockamx.processor.js | 0 .../processors}/survivor-cars.processor.js | 0 .../topclassiccarsforsale.processor.js | 0 .../processors}/vanguardmotorsales.processor.js | 0 23 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml rename {crawlers => server/crawlers}/topclassiccarsforsale.crawler.js (94%) rename index.js => server/index.js (98%) rename {processors => server/processors}/barrettjackson.processor.js (100%) rename {processors => server/processors}/bringatrailer.processor.js (100%) rename {processors => server/processors}/classiccars.processor.js (100%) rename {processors => server/processors}/classiccarsbay.processor.js (100%) rename {processors => server/processors}/cruisinclassicsinc.processor.js (100%) rename {processors => server/processors}/customclassics.processor.js (100%) rename {processors => server/processors}/davidsclassiccars.processor.js (100%) rename {processors => server/processors}/ebay.processor.js (100%) rename {processors => server/processors}/generics/generic-bootstrap.js (100%) rename {processors => server/processors}/generics/generic-showCarThumbs.js (100%) rename {processors => server/processors}/generics/generic-vin-parser.js (100%) rename {processors => server/processors}/hemmings.processor.js (100%) rename {processors => server/processors}/mecum.processor.js (100%) rename {processors => server/processors}/superstockamx.processor.js (100%) rename {processors => server/processors}/survivor-cars.processor.js (100%) rename {processors => server/processors}/topclassiccarsforsale.processor.js (100%) rename {processors => server/processors}/vanguardmotorsales.processor.js (100%) diff --git a/.gitignore b/.gitignore index b6d7547..d5c05a9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules/ images/ +stack.env \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..fc01e89 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM ghcr.io/puppeteer/puppeteer:21.5.2 +WORKDIR /app + +COPY /server ./ +COPY /package.json ./package.json +RUN npm install +CMD ["node", "index.js"] \ No newline at end of file diff --git a/README.md b/README.md index 9e8ed04..425fed4 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,12 @@ # bringatrailerbot + +Deploy instructions: +- Uncomment the `pull_policy` line from `docker-compose.yml` +- Update `docker-compose.yml` with new version numbers. +- Run `docker login git.edwardpeterson.dev` +- Run `docker-compose build --pull` from the root directory +- Run `docker-compose push` next to push the newest image to gitea. +- Comment back out the `pull_policy` line from `docker-compose.yml` +- Commit and push changes. + \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..f505a0d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,15 @@ +version: '3.7' + +services: + + amxregistry-bundler: + image: git.edwardpeterson.dev/cubemaster21/amxregistry-bundler:v1.0 + # container_name: amxregistry-client + # pull_policy: build + restart: always + build: + no_cache: true + context: . + dockerfile: ./Dockerfile + ports: + - 2667:2667 \ No newline at end of file diff --git a/crawlers/topclassiccarsforsale.crawler.js b/server/crawlers/topclassiccarsforsale.crawler.js similarity index 94% rename from crawlers/topclassiccarsforsale.crawler.js rename to server/crawlers/topclassiccarsforsale.crawler.js index 79e2e58..f59f46e 100644 --- a/crawlers/topclassiccarsforsale.crawler.js +++ b/server/crawlers/topclassiccarsforsale.crawler.js @@ -7,7 +7,8 @@ module.exports = { run: async function () { const startingPoint = 'https://topclassiccarsforsale.com/amc'; const browser = await puppeteer.launch({ - headless: true + headless: true, + args: ['--no-sandbox'] }); const page = await browser.newPage(); await page.setViewport({ @@ -28,7 +29,7 @@ module.exports = { const cars = await module.exports.processPage(page); log.info(cars); cars.forEach(car => { - superagent.post('http://localhost:3000/lead/createFromCrawler') + superagent.post(`${process.env.parentUrl}/lead/createFromCrawler`) .send({ url: car.url, vinNumber: car.vin @@ -90,7 +91,7 @@ module.exports = { }, filterCompletedLinks: function(links){ return new Promise((resolve, reject) => { - superagent.post('http://localhost:3000/lead/crawler/filterUrls') + superagent.post(`${process.env.parentUrl}/lead/crawler/filterUrls`) .send({ urls: links }) diff --git a/index.js b/server/index.js similarity index 98% rename from index.js rename to server/index.js index 4ddf657..1ebc1b9 100644 --- a/index.js +++ b/server/index.js @@ -92,7 +92,8 @@ app.listen(2667); async function run(url, processor) { const browser = await puppeteer.launch({ - headless: true + headless: true, + args: ['--no-sandbox'] }); const page = await browser.newPage(); diff --git a/processors/barrettjackson.processor.js b/server/processors/barrettjackson.processor.js similarity index 100% rename from processors/barrettjackson.processor.js rename to server/processors/barrettjackson.processor.js diff --git a/processors/bringatrailer.processor.js b/server/processors/bringatrailer.processor.js similarity index 100% rename from processors/bringatrailer.processor.js rename to server/processors/bringatrailer.processor.js diff --git a/processors/classiccars.processor.js b/server/processors/classiccars.processor.js similarity index 100% rename from processors/classiccars.processor.js rename to server/processors/classiccars.processor.js diff --git a/processors/classiccarsbay.processor.js b/server/processors/classiccarsbay.processor.js similarity index 100% rename from processors/classiccarsbay.processor.js rename to server/processors/classiccarsbay.processor.js diff --git a/processors/cruisinclassicsinc.processor.js b/server/processors/cruisinclassicsinc.processor.js similarity index 100% rename from processors/cruisinclassicsinc.processor.js rename to server/processors/cruisinclassicsinc.processor.js diff --git a/processors/customclassics.processor.js b/server/processors/customclassics.processor.js similarity index 100% rename from processors/customclassics.processor.js rename to server/processors/customclassics.processor.js diff --git a/processors/davidsclassiccars.processor.js b/server/processors/davidsclassiccars.processor.js similarity index 100% rename from processors/davidsclassiccars.processor.js rename to server/processors/davidsclassiccars.processor.js diff --git a/processors/ebay.processor.js b/server/processors/ebay.processor.js similarity index 100% rename from processors/ebay.processor.js rename to server/processors/ebay.processor.js diff --git a/processors/generics/generic-bootstrap.js b/server/processors/generics/generic-bootstrap.js similarity index 100% rename from processors/generics/generic-bootstrap.js rename to server/processors/generics/generic-bootstrap.js diff --git a/processors/generics/generic-showCarThumbs.js b/server/processors/generics/generic-showCarThumbs.js similarity index 100% rename from processors/generics/generic-showCarThumbs.js rename to server/processors/generics/generic-showCarThumbs.js diff --git a/processors/generics/generic-vin-parser.js b/server/processors/generics/generic-vin-parser.js similarity index 100% rename from processors/generics/generic-vin-parser.js rename to server/processors/generics/generic-vin-parser.js diff --git a/processors/hemmings.processor.js b/server/processors/hemmings.processor.js similarity index 100% rename from processors/hemmings.processor.js rename to server/processors/hemmings.processor.js diff --git a/processors/mecum.processor.js b/server/processors/mecum.processor.js similarity index 100% rename from processors/mecum.processor.js rename to server/processors/mecum.processor.js diff --git a/processors/superstockamx.processor.js b/server/processors/superstockamx.processor.js similarity index 100% rename from processors/superstockamx.processor.js rename to server/processors/superstockamx.processor.js diff --git a/processors/survivor-cars.processor.js b/server/processors/survivor-cars.processor.js similarity index 100% rename from processors/survivor-cars.processor.js rename to server/processors/survivor-cars.processor.js diff --git a/processors/topclassiccarsforsale.processor.js b/server/processors/topclassiccarsforsale.processor.js similarity index 100% rename from processors/topclassiccarsforsale.processor.js rename to server/processors/topclassiccarsforsale.processor.js diff --git a/processors/vanguardmotorsales.processor.js b/server/processors/vanguardmotorsales.processor.js similarity index 100% rename from processors/vanguardmotorsales.processor.js rename to server/processors/vanguardmotorsales.processor.js