Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit a82c3a0

Browse files
shaperilioshaperilio
committed
Parse jeep dealership pages just like subaru
1 parent 83a7473 commit a82c3a0

6 files changed

Lines changed: 185 additions & 59 deletions

File tree

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,5 @@ run.sh
66
node_modules
77
index.html
88
jeep.json
9-
subaru.json
9+
subaru.json
10+
page.html

‎dealer-common.js‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
const cheerio = require('cheerio');
2+
const fs = require('fs');
3+
4+
const windowStickerUrl = 'https://window-sticker-services.pse.dealer.com/windowsticker/MAKE?vin=VIN'
5+
6+
function parseResults(body, dealer, make, pageUrl) {
7+
console.log(pageUrl);
8+
const cars = [];
9+
const content = cheerio.load(body);
10+
fs.writeFileSync('page.html', body)
11+
const numCars = content('.vehicle-count').last().text();
12+
const apparentNumCars = content('.hproduct', '.bd').length;
13+
console.log(`numCars = ${numCars}; apparentNumCars = ${apparentNumCars}`);
14+
if (!numCars || numCars === '0' || apparentNumCars === 0)
15+
return cars;
16+
17+
let dealerName = content('.org').text().trim();
18+
if (!dealerName) {
19+
dealerName = 'noname';
20+
}
21+
if (!dealerName) {
22+
dealerName = 'noname';
23+
}
24+
if (!dealerName) {
25+
dealerName = 'noname';
26+
}
27+
if (!dealerName) {
28+
dealerName = 'noname';
29+
}
30+
const dealerAddress = `${content('.street-address').text().trim()}, ${content('.locality').text().trim()}, ${content('.region').text().trim()}, ${content('.postal-code').text().trim()}`;
31+
const dealerCityState = `${content('.locality').text().trim()}, ${content('.region').text().trim()}`;
32+
content('.hproduct', '.bd').each(
33+
(i, car) => {
34+
const name = content('.url', car).text().trim();
35+
const url = `${dealer}${content('.url', car).attr('href')}`;
36+
const imgUrl = content('img', content('.media', car)).attr('src');
37+
const pricing = content('.pricing', car);
38+
let msrp = content('li', pricing).find('.msrp').find('.value').text();
39+
if (!msrp) {
40+
msrp = content('.an-msrp .price', pricing).text();
41+
}
42+
if (!msrp) {
43+
msrp = content('span', pricing).first().next().text();
44+
}
45+
if (!msrp) {
46+
msrp = content('.value', content('.salePrice', pricing)).text();
47+
}
48+
if (!msrp) {
49+
msrp = '0'
50+
}
51+
let finalPrice = content('li', pricing).find('.final-price').find('.value').text();
52+
if (!finalPrice) {
53+
finalPrice = content('.an-final-price .price', pricing).text();
54+
}
55+
if (!finalPrice) {
56+
finalPrice = '0';
57+
}
58+
if (!finalPrice) {
59+
finalPrice = '0';
60+
}
61+
if (!finalPrice) {
62+
finalPrice = '0';
63+
}
64+
const internetPrice = content('li', pricing).find('.internetPrice').find('.value').text();
65+
let vin = content('.vin dd', car).text();
66+
const engine = content('.description dt:contains("Engine:")', car).next().text().replace(',', '');
67+
const theCar = {
68+
dealerName,
69+
dealerAddress,
70+
dealerCityState,
71+
name,
72+
url,
73+
imgUrl,
74+
engine,
75+
vin,
76+
windowSticker: windowStickerUrl.replace('MAKE', make).replace('VIN', vin),
77+
msrp,
78+
internetPrice,
79+
finalPrice,
80+
prices: []
81+
};
82+
content('li', pricing).each(
83+
(i, price) => {
84+
const number = content('.value', price).text();
85+
const label = content('.label', price).text();
86+
if (label)
87+
theCar.prices.push({ label, number });
88+
})
89+
cars.push(theCar);
90+
}
91+
);
92+
if (cars.length !== parseInt(numCars)) {
93+
console.error(`${dealerName} website reports ${numCars} but we retrieved ${cars.length}!`);
94+
}
95+
return cars;
96+
}
97+
98+
module.exports = {parseResults}

‎jeep-mfg.js‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ function makeImageUrl(urlParams, width, interior=false) {
2626
return `${imageUrl}${urlParams}${renderSettings}`;
2727
}
2828

29-
async function getDealers(zipCode) {
29+
async function getDealers(zipCode, radiusMiles) {
3030
const url = 'https://www.jeep.com/bdlws/MDLSDealerLocator?zipCode=ZIPCODE&func=SALES&radius=RADIUSMILES&brandCode=J&resultsPerPage=999'
3131
const requestUrl = url
3232
.replace('ZIPCODE', zipCode)
@@ -46,7 +46,8 @@ async function getDealers(zipCode) {
4646
code: dealer.dealerCode,
4747
name: dealer.dealerName,
4848
address: `${dealerAddress}, ${dealer.dealerCity}, ${dealer.dealerState}`,
49-
cityState: `${dealer.dealerCity}, ${dealer.dealerState}`
49+
cityState: `${dealer.dealerCity}, ${dealer.dealerState}`,
50+
website: dealer.website + '/'
5051
})
5152
}
5253

@@ -171,7 +172,7 @@ async function run() {
171172

172173
let promises = [];
173174
for (const zipCode of zipCodes) {
174-
promises.push(getDealers(zipCode));
175+
promises.push(getDealers(zipCode, radiusMiles));
175176
}
176177

177178
const dealersByZip = await Promise.all(promises);
@@ -221,4 +222,4 @@ async function run() {
221222
return allCars;
222223
}
223224

224-
module.exports = {run};
225+
module.exports = {run, getDealers};

‎jeep.js‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
const fetch = require('node-fetch');
2+
const fs = require('fs');
3+
const moment = require('moment');
4+
const { getDealers } = require('./jeep-mfg.js')
5+
const { parseResults } = require('./dealer-common.js');
6+
7+
async function fetchFromDealer(dealer) {
8+
const query = 'new-inventory/index.htm?search=&model=Wrangler&gvOption=Distance+Pacing+Cruise+Control&gvOption=Heated+Seats';
9+
const url = `${dealer}${query}`;
10+
const response = await fetch(url);
11+
const body = await response.text();
12+
const cars = parseResults(body, dealer, 'jeep', url);
13+
console.log(`${cars.length} car(s) found at ${url}`);
14+
return cars;
15+
}
16+
17+
async function run() {
18+
19+
const zipCodes = [
20+
77478,
21+
94610
22+
]
23+
24+
const radiusMiles = 50;
25+
26+
let promises = [];
27+
for (const zipCode of zipCodes) {
28+
promises.push(getDealers(zipCode, radiusMiles));
29+
}
30+
31+
const dealersByZip = await Promise.all(promises);
32+
const dealers = [];
33+
for (const d of dealersByZip) {
34+
dealers.push(...d.map(a => a.website));
35+
}
36+
37+
const allCars = [];
38+
39+
promises = [];
40+
for (const dealer of dealers) {
41+
promises.push(fetchFromDealer(dealer));
42+
}
43+
44+
const dealerCars = await Promise.all(promises);
45+
for (const cars of dealerCars) {
46+
allCars.push(...cars);
47+
}
48+
49+
allCars.sort(function (a, b) {
50+
if (a.finalPrice < b.finalPrice) return -1;
51+
if (a.finalPrice > b.finalPrice) return 1;
52+
return 0;
53+
});
54+
55+
const archive = `archive/jeep_${moment().format('YYYY-MM-DD_HH-mm-ss')}.json`;
56+
fs.writeFileSync(archive, JSON.stringify(allCars, null, 2), err => {
57+
console.error(err);
58+
});
59+
60+
fs.writeFileSync('jeep.json', JSON.stringify(allCars, null, 2), err => {
61+
console.error(err);
62+
});
63+
64+
return allCars;
65+
}
66+
67+
module.exports = {run};

‎main.js‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
const subaru = require('./subaru.js');
2-
const jeep = require('./jeep-mfg.js');
2+
const jeep = require('./jeep.js');
33
const fs = require('fs');
44
const moment = require('moment');
55

6-
async function makeSubaruTable() {
7-
const allCars = await subaru.run();
6+
function makeDealerTable(allCars) {
87
let html = `
98
<table cellpadding="5px" border="1px">
109
<tr>
@@ -20,7 +19,9 @@ async function makeSubaruTable() {
2019
<td><img src="${car.imgUrl}" height="120px"></td>
2120
<td>${car.msrp}</td>
2221
<td><a href="${car.url}" target="_blank">${car.finalPrice}</a></td>
23-
<td><a href="${googleMapsLink}" target="_blank">${car.dealerName} - ${car.dealerCityState}</a></td>
22+
<td><a href="${googleMapsLink}" target="_blank">${car.dealerName} - ${car.dealerCityState}</a><br />
23+
<a href="${car.windowSticker}" target="_blank">${car.vin}</a><br />
24+
${car.engine}</td>
2425
</tr>`
2526
console.log(`${car.finalPrice} - ${car.name} (${car.url}).`);
2627
}
@@ -30,7 +31,7 @@ async function makeSubaruTable() {
3031
return html;
3132
}
3233

33-
async function makeJeepTable() {
34+
async function makeJeepMfgTable() {
3435
const allCars = await jeep.run();
3536
let html = `
3637
<table cellpadding="5px" border="1px">
@@ -67,12 +68,15 @@ async function makePage() {
6768
<p>Updated ${moment().format('YYYY-MM-DD HH:mm:ss')}</p>
6869
<table><tr><td valign="top">`;
6970

70-
html += await makeSubaruTable();
71+
let cars = await subaru.run();
72+
html += makeDealerTable(cars);
7173

7274
html += `</td>
7375
<td valign="top">`;
7476

75-
html += await makeJeepTable();
77+
cars = await jeep.run();
78+
html += makeDealerTable(cars);
79+
// html += await makeJeepMfgTable();
7680

7781
html += `
7882
</td></tr></table>

‎subaru.js‎

Lines changed: 2 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,59 +1,14 @@
11
const fetch = require('node-fetch');
2-
const cheerio = require('cheerio');
32
const fs = require('fs');
43
const moment = require('moment');
5-
6-
function parseResults(body, dealer) {
7-
const cars = [];
8-
const content = cheerio.load(body);
9-
const dealerName = content('.org').text().trim();
10-
const dealerAddress = `${content('.street-address').text().trim()}, ${content('.locality').text().trim()}, ${content('.region').text().trim()}, ${content('.postal-code').text().trim()}`;
11-
const dealerCityState = `${content('.locality').text().trim()}, ${content('.region').text().trim()}`;
12-
const numCars = content('.vehicle-count').last().text();
13-
if (!numCars || numCars === '0') return cars;
14-
content('.hproduct', '.bd').each(
15-
(i, car) => {
16-
const name = content('.url', car).text().trim();
17-
const url = `${dealer}${content('.url', car).attr('href')}`;
18-
const imgUrl = content('img', content('.media', car)).attr('src');
19-
const pricing = content('.pricing', car);
20-
const msrp = content('li', pricing).find('.msrp').find('.value').text();
21-
const internetPrice = content('li', pricing).find('.internetPrice').find('.value').text();
22-
const finalPrice = content('li', pricing).find('.final-price').find('.value').text();
23-
const theCar = {
24-
dealerName,
25-
dealerAddress,
26-
dealerCityState,
27-
name,
28-
url,
29-
imgUrl,
30-
msrp,
31-
internetPrice,
32-
finalPrice,
33-
prices: []
34-
};
35-
content('li', pricing).each(
36-
(j, price) => {
37-
const number = content('.value', price).text();
38-
const label = content('.label', price).text();
39-
if (label)
40-
theCar.prices.push({ label, number });
41-
})
42-
cars.push(theCar);
43-
}
44-
);
45-
if (cars.length !== parseInt(numCars)) {
46-
console.error(`${dealerName} website reports ${numCars} but we retrieved ${cars.length}!`);
47-
}
48-
return cars;
49-
}
4+
const { parseResults } = require('./dealer-common.js');
505

516
async function fetchFromDealer(dealer) {
527
const query = 'new-inventory/index.htm?search=&model=Outback&trim=Onyx+Edition+XT';
538
const url = `${dealer}${query}`;
549
const response = await fetch(url);
5510
const body = await response.text();
56-
const cars = parseResults(body, dealer);
11+
const cars = parseResults(body, dealer, 'subaru', url);
5712
console.log(`${cars.length} car(s) found at ${url}`);
5813
return cars;
5914
}

0 commit comments

Comments
 (0)