Save bad pages, with cUrl ''verification''; start to treat dealerships differently

shaperilio · shaperilio · commit 3d7bf645576a · 2020-05-25T08:22:19.000-07:00
diff --git a/.gitignore b/.gitignore
@@ -11,4 +11,5 @@ unmatched-jeep.json
 subaru.json
 page.html
 cache
-install.sh
+install.sh
+crap
diff --git a/dealer-common.js b/dealer-common.js
@@ -1,10 +1,25 @@
 const fetcher = require('./fetch-with-cache.js');
 const cheerio = require('cheerio');
 const fs = require('fs');
+const childProcess = require('child_process');
 
 const windowStickerUrl =  'https://window-sticker-services.pse.dealer.com/windowsticker/MAKE?vin=VIN'
 
+function getQueryByDealer(dealerUrl, make) {
+    if (make === 'subaru') {
+        return 'new-inventory/index.htm?search=&model=Outback&trim=Onyx+Edition+XT';
+    }
+
+    if (dealerUrl.includes('fremontcdjr')) {
+        return 'new-vehicles/#action=im_ajax_call&perform=get_results&model=Wrangler&page=1';
+    }
+
+    return 'new-inventory/index.htm?search=&model=Wrangler';
+}
+
 async function fetchFromDealer(dealerUrl, make, query) {
+    if (!query)
+        query = getQueryByDealer(dealerUrl, make);
     const url = `${dealerUrl}${query}`;
     const body = await fetcher.getHtml(url);
     let result = await parseResults(body, dealerUrl, make, url);
@@ -47,9 +62,13 @@ async function parseResults(body, dealer, make, pageUrl) {
 
     if (carList.length === 0) {
         console.error(`No cars found at ${pageUrl}`);
-        // const dealerForFile = dealer.match(/https?\:\/\/(www\.)?(?<dealer>\w*)\./).groups.dealer;
-        // fs.writeFileSync(`crap/page_${dealerForFile}.html`, body);
-        // console.error('Page saved for inspection');
+        const dealerForFile = dealer.match(/https?\:\/\/(www\.)?(?<dealer>\w*)\./).groups.dealer;
+        fs.writeFileSync(`crap/page_${dealerForFile}.html`, body);
+        childProcess.exec(`curl -L "${pageUrl}" -o crap/curl_${dealerForFile}.html`);
+        console.error('Page saved for inspection');
+        if (content('*').text().toUpperCase().includes('CAPTCHA')) {
+            console.error(`Captcha request at ${pageUrl}`);
+        }
     }
     let dealerName = content('.org').text().trim();
     if (!dealerName) {
@@ -127,4 +146,4 @@ async function parseResults(body, dealer, make, pageUrl) {
     return { cars, reportedCars };
 }
 
-module.exports = { fetchFromDealer }
+module.exports = { fetchFromDealer, getQueryByDealer }
diff --git a/fetch-with-cache.js b/fetch-with-cache.js
@@ -2,6 +2,7 @@ const fetch = require('node-fetch');
 const moment = require('moment');
 const md5 = require('md5');
 const fs = require('fs');
+const childProcess = require('child_process');
 
 function makeCacheFilename(url) {
     const key = `${url}${moment().format('YYYY-MM-DD_HH')}`
@@ -12,9 +13,12 @@ async function getHtml(url) {
     const filename = `${makeCacheFilename(url)}.html`;
     if (fs.existsSync(filename)) {
         // console.debug(`Cache hit ${filename} => ${url}`);
-        return fs.readFileSync(filename);
+        return String(fs.readFileSync(filename));
     }
 
+    // childProcess.execSync(`curl -L "${url}" -o ${filename}`);
+    // return String(fs.readFileSync(filename));
+
     const response = await fetch(url);
     const data = await response.text();
     fs.writeFile(filename, data, err => {
diff --git a/jeep-mfg.js b/jeep-mfg.js
@@ -1,7 +1,7 @@
 const fetcher = require('./fetch-with-cache.js');
 const moment = require('moment');
 const fs = require('fs');
-const { fetchFromDealer } = require('./dealer-common.js');
+const { fetchFromDealer, getQueryByDealer } = require('./dealer-common.js');
  
 function formatCurrency(numberString) {
     let result = ''
@@ -212,9 +212,7 @@ async function getCarsFromDealers() {
     const dealerUrls = [... new Set(factoryCars.map(a => a.dealer.website))];
     console.log(`Factory query resulted in ${factoryCars.length} cars from ${dealerUrls.length} dealers.`);
 
-    const query = 'new-inventory/index.htm?search=&model=Wrangler';
-
-    const carsByDealer = await Promise.all(dealerUrls.map(url => fetchFromDealer(url, 'jeep', query)));
+    const carsByDealer = await Promise.all(dealerUrls.map(url => fetchFromDealer(url, 'jeep')));
     const allCars = [];
     carsByDealer.map(cars => allCars.push(...cars)); // these are all the cars in dealerships that have at least one car we want. 
 
@@ -223,6 +221,7 @@ async function getCarsFromDealers() {
     for (const factoryCar of factoryCars) {
         const matches = allCars.filter(a => a.vin.trim() === factoryCar.vin.trim());
         if (!matches || matches.length === 0) {
+            const query = getQueryByDealer(factoryCar.dealer.website, 'jeep');
             console.error(`Could not find car ${factoryCar.vin} at ${factoryCar.dealer.website}${query}`)
             unmatchedCars.push({vin: factoryCar.vin, url:`${factoryCar.dealer.website}${query}`});
             continue;
diff --git a/subaru.js b/subaru.js
@@ -2,10 +2,6 @@ const fs = require('fs');
 const moment = require('moment');
 const { fetchFromDealer } = require('./dealer-common.js');
 
-async function fetchCars(dealer) {
-    const query = 'new-inventory/index.htm?search=&model=Outback&trim=Onyx+Edition+XT';
-    return await fetchFromDealer(dealer, 'subaru', query);
-}
 
 async function getCarsFromDealers() {
     const dealers = [
@@ -27,7 +23,7 @@ async function getCarsFromDealers() {
         'https://www.superiorsubaruofhouston.com/'
     ]
 
-    const carsByDealer = await Promise.all(dealers.map(dealer => fetchCars(dealer)));
+    const carsByDealer = await Promise.all(dealers.map(dealer => fetchFromDealer(dealer, 'subaru')));
     const allCars = [];
     carsByDealer.map(cars => allCars.push(...cars));
 
