Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 0f83943

Browse files
shaperilioshaperilio
committed
Handle pagination
1 parent 376ce07 commit 0f83943

5 files changed

Lines changed: 44 additions & 38 deletions

File tree

‎dealer-common.js‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,40 @@
1+
const fetch = require('node-fetch');
12
const cheerio = require('cheerio');
23
const fs = require('fs');
34

45
const windowStickerUrl = 'https://window-sticker-services.pse.dealer.com/windowsticker/MAKE?vin=VIN'
56

7+
async function fetchFromDealer(dealerUrl, make, query) {
8+
const url = `${dealerUrl}${query}`;
9+
const response = await fetch(url);
10+
const body = await response.text();
11+
let result = parseResults(body, dealerUrl, make, url);
12+
const cars = result.cars;
13+
console.log(`${cars.length} car(s) found at ${url}`);
14+
while (result.reportedCars > cars.length) {
15+
const response = await fetch(url.replace('search=', `start=${cars.length}`));
16+
const body = await response.text();
17+
result = parseResults(body, dealerUrl, make, url);
18+
cars.push(...result.cars);
19+
}
20+
return cars;
21+
}
22+
623
function parseResults(body, dealer, make, pageUrl) {
7-
console.log(pageUrl);
824
const cars = [];
925
const content = cheerio.load(body);
1026
fs.writeFileSync('page.html', body)
1127
const numCars = content('.vehicle-count').last().text();
12-
const apparentNumCars = content('.hproduct', '.bd').length;
13-
console.log(`numCars = ${numCars}; apparentNumCars = ${apparentNumCars}`);
14-
if (!numCars || numCars === '0' || apparentNumCars === 0)
15-
return cars;
28+
const carList = content('.hproduct', '.bd');
29+
console.log(`numCars = ${numCars}; carList.length = ${carList.length} at ${pageUrl}`);
1630

1731
let dealerName = content('.org').text().trim();
1832
if (!dealerName) {
1933
dealerName = 'Unkown dealer name';
2034
}
2135
const dealerAddress = `${content('.street-address').text().trim()}, ${content('.locality').text().trim()}, ${content('.region').text().trim()}, ${content('.postal-code').text().trim()}`;
2236
const dealerCityState = `${content('.locality').text().trim()}, ${content('.region').text().trim()}`;
23-
content('.hproduct', '.bd').each(
37+
carList.each(
2438
(i, car) => {
2539
const name = content('.url', car).text().trim();
2640
const url = `${dealer}${content('.url', car).attr('href')}`;
@@ -50,6 +64,7 @@ function parseResults(body, dealer, make, pageUrl) {
5064
let vin = content('.vin dd', car).text();
5165
const engine = content('.description dt:contains("Engine:")', car).next().text().replace(',', '');
5266
const theCar = {
67+
pageUrl,
5368
dealerName,
5469
dealerAddress,
5570
dealerCityState,
@@ -74,10 +89,13 @@ function parseResults(body, dealer, make, pageUrl) {
7489
cars.push(theCar);
7590
}
7691
);
77-
if (cars.length !== parseInt(numCars)) {
78-
console.error(`${dealerName} website reports ${numCars} but we retrieved ${cars.length}!`);
92+
93+
let reportedCars = 0;
94+
if (!isNaN(parseInt(numCars)) && carList.length > 0) {
95+
reportedCars = parseInt(numCars)
7996
}
80-
return cars;
97+
98+
return { cars, reportedCars };
8199
}
82100

83-
module.exports = {parseResults}
101+
module.exports = { fetchFromDealer }

‎jeep-mfg.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -210,12 +210,12 @@ async function run() {
210210
return 0;
211211
});
212212

213-
const archive = `archive/jeep_${moment().format('YYYY-MM-DD_HH-mm-ss')}.json`;
213+
const archive = `archive/jeep-mfg_${moment().format('YYYY-MM-DD_HH-mm-ss')}.json`;
214214
fs.writeFileSync(archive, JSON.stringify(allCars, null, 2), err => {
215215
console.error(err);
216216
});
217217

218-
fs.writeFileSync('jeep.json', JSON.stringify(allCars, null, 2), err => {
218+
fs.writeFileSync('jeep-mfg.json', JSON.stringify(allCars, null, 2), err => {
219219
console.error(err);
220220
});
221221

‎jeep.js‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,14 @@
1-
const fetch = require('node-fetch');
21
const fs = require('fs');
32
const moment = require('moment');
43
const { getDealers } = require('./jeep-mfg.js')
5-
const { parseResults } = require('./dealer-common.js');
4+
const { fetchFromDealer } = require('./dealer-common.js');
65

7-
async function fetchFromDealer(dealer) {
6+
async function fetchCars(dealer) {
87
const query = 'new-inventory/index.htm?search=&model=Wrangler&gvOption=Distance+Pacing+Cruise+Control&gvOption=Heated+Seats';
9-
const url = `${dealer}${query}`;
10-
const response = await fetch(url);
11-
const body = await response.text();
12-
const cars = parseResults(body, dealer, 'jeep', url);
13-
console.log(`${cars.length} car(s) found at ${url}`);
14-
return cars;
8+
return await fetchFromDealer(dealer, 'jeep', query);
159
}
1610

17-
async function run() {
11+
async function getCarsFromDealers() {
1812

1913
const zipCodes = [
2014
77478,
@@ -38,7 +32,7 @@ async function run() {
3832

3933
promises = [];
4034
for (const dealer of dealers) {
41-
promises.push(fetchFromDealer(dealer));
35+
promises.push(fetchCars(dealer));
4236
}
4337

4438
const dealerCars = await Promise.all(promises);
@@ -64,4 +58,4 @@ async function run() {
6458
return allCars;
6559
}
6660

67-
module.exports = {run};
61+
module.exports = {getCarsFromDealers};

‎main.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,13 +68,13 @@ async function makePage() {
6868
<p>Updated ${moment().format('YYYY-MM-DD HH:mm:ss')}</p>
6969
<table><tr><td valign="top">`;
7070

71-
let cars = await subaru.run();
71+
let cars = await subaru.getCarsFromDealers();
7272
html += makeDealerTable(cars);
7373

7474
html += `</td>
7575
<td valign="top">`;
7676

77-
cars = await jeep.run();
77+
cars = await jeep.getCarsFromDealers();
7878
html += makeDealerTable(cars);
7979
// html += await makeJeepMfgTable();
8080

‎subaru.js‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,13 @@
1-
const fetch = require('node-fetch');
21
const fs = require('fs');
32
const moment = require('moment');
4-
const { parseResults } = require('./dealer-common.js');
3+
const { fetchFromDealer } = require('./dealer-common.js');
54

6-
async function fetchFromDealer(dealer) {
5+
async function fetchCars(dealer) {
76
const query = 'new-inventory/index.htm?search=&model=Outback&trim=Onyx+Edition+XT';
8-
const url = `${dealer}${query}`;
9-
const response = await fetch(url);
10-
const body = await response.text();
11-
const cars = parseResults(body, dealer, 'subaru', url);
12-
console.log(`${cars.length} car(s) found at ${url}`);
13-
return cars;
7+
return await fetchFromDealer(dealer, 'subaru', query);
148
}
159

16-
async function run() {
10+
async function getCarsFromDealers() {
1711
const dealers = [
1812
'https://www.fairfieldsubaru.com/',
1913
'https://www.diablosubaru.com/',
@@ -37,7 +31,7 @@ async function run() {
3731

3832
const promises = [];
3933
for (const dealer of dealers) {
40-
promises.push(fetchFromDealer(dealer));
34+
promises.push(fetchCars(dealer));
4135
}
4236

4337
const dealerCars = await Promise.all(promises);
@@ -63,4 +57,4 @@ async function run() {
6357
return allCars;
6458
}
6559

66-
module.exports = {run};
60+
module.exports = {getCarsFromDealers};

0 commit comments

Comments
 (0)