Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit c39db53

Browse files
shaperilioshaperilio
committed
Cleanup logging; save missing Jeeps
1 parent f4743cd commit c39db53

5 files changed

Lines changed: 54 additions & 21 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ node_modules
77
index.html
88
jeep.json
99
jeep-mfg.json
10+
unmatched-jeep.json
1011
subaru.json
1112
page.html
1213
cache

‎dealer-common.js‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,13 @@ async function fetchFromDealer(dealerUrl, make, query) {
99
const body = await fetcher.getHtml(url);
1010
let result = await parseResults(body, dealerUrl, make, url);
1111
const cars = result.cars;
12-
console.log(`${cars.length} car(s) found at ${url}`);
1312
while (result.reportedCars > cars.length) {
1413
const paginatedUrl = url.replace('search=', `start=${cars.length}`)
1514
const body = await fetcher.getHtml(paginatedUrl);
1615
result = await parseResults(body, dealerUrl, make, paginatedUrl);
1716
cars.push(...result.cars);
1817
}
18+
console.log(`${cars.length} car(s) parsed from ${url}`);
1919
return cars;
2020
}
2121

@@ -43,8 +43,14 @@ async function parseResults(body, dealer, make, pageUrl) {
4343
fs.writeFileSync('page.html', body)
4444
const numCars = content('.vehicle-count').last().text();
4545
const carList = content('.hproduct', '.bd');
46-
console.log(`numCars = ${numCars}; carList.length = ${carList.length} at ${pageUrl}`);
46+
// console.log(`numCars = ${numCars}; carList.length = ${carList.length} at ${pageUrl}`);
4747

48+
if (carList.length === 0) {
49+
console.error(`No cars found at ${pageUrl}`);
50+
// const dealerForFile = dealer.match(/https?\:\/\/(www\.)?(?<dealer>\w*)\./).groups.dealer;
51+
// fs.writeFileSync(`crap/page_${dealerForFile}.html`, body);
52+
// console.error('Page saved for inspection');
53+
}
4854
let dealerName = content('.org').text().trim();
4955
if (!dealerName) {
5056
dealerName = 'Unkown dealer name';

‎fetch-with-cache.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ function makeCacheFilename(url) {
1111
async function getHtml(url) {
1212
const filename = `${makeCacheFilename(url)}.html`;
1313
if (fs.existsSync(filename)) {
14+
// console.debug(`Cache hit ${filename} => ${url}`);
1415
return fs.readFileSync(filename);
1516
}
1617

@@ -27,6 +28,7 @@ async function getHtml(url) {
2728
async function getJson(url) {
2829
const filename = `${makeCacheFilename(url)}.json`;
2930
if (fs.existsSync(filename)) {
31+
// console.debug(`Cache hit ${filename} => ${url}`);
3032
return JSON.parse(fs.readFileSync(filename));
3133
}
3234

‎jeep-mfg.js‎

Lines changed: 43 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -113,10 +113,6 @@ async function getCars(filters, zipCode, dealers, radiusMiles) {
113113
for (const car of cars) {
114114
if (filters && !isCarValid(car, filters)) continue;
115115

116-
if (car.incentives) {
117-
console.log(car.incentives);
118-
}
119-
120116
const engine = car.engineDesc;
121117
const tranny = car.transmissionDesc;
122118
const model = car.vehicleDesc;
@@ -211,8 +207,7 @@ async function getCarsFromDealers() {
211207
// Get cars by filtering the factory inventory search, then looking for them in the dealerships'
212208
// website and matching by VIN.
213209

214-
const dealerCars = [];
215-
const factoryCars = await getCarsFromFactory();
210+
const factoryCars = await getCarsFromFactory(); // contains the cars we actually want.
216211

217212
const dealerUrls = [... new Set(factoryCars.map(a => a.dealer.website))];
218213
console.log(`Factory query resulted in ${factoryCars.length} cars from ${dealerUrls.length} dealers.`);
@@ -221,23 +216,54 @@ async function getCarsFromDealers() {
221216

222217
const carsByDealer = await Promise.all(dealerUrls.map(url => fetchFromDealer(url, 'jeep', query)));
223218
const allCars = [];
224-
carsByDealer.map(cars => allCars.push(...cars));
219+
carsByDealer.map(cars => allCars.push(...cars)); // these are all the cars in dealerships that have at least one car we want.
225220

221+
const dealerCars = [];
222+
const unmatchedCars = []
226223
for (const factoryCar of factoryCars) {
227-
let found = false;
228-
const dealerUrl = factoryCar.dealer.website;
229-
for (const car of allCars) {
230-
if (car.vin.trim() === factoryCar.vin.trim()) {
231-
dealerCars.push(car);
232-
found = true;
233-
break;
234-
}
224+
const matches = allCars.filter(a => a.vin.trim() === factoryCar.vin.trim());
225+
if (!matches || matches.length === 0) {
226+
console.error(`Could not find car ${factoryCar.vin} at ${factoryCar.dealer.website}${query}`)
227+
unmatchedCars.push({vin: factoryCar.vin, url:`${factoryCar.dealer.website}${query}`});
228+
continue;
235229
}
236-
if (!found) {
237-
console.error(`Could not find car ${factoryCar.vin} at ${dealerUrl}${query}`)
230+
if (matches.length > 1) {
231+
console.error(`Found ${matches.length} cars with ${factoryCar.vin}!`);
238232
}
233+
const car = matches[0];
234+
dealerCars.push(car)
239235
}
240236

237+
dealerCars.sort(function (a, b) {
238+
if (a.finalPrice < b.finalPrice) return -1;
239+
if (a.finalPrice > b.finalPrice) return 1;
240+
return 0;
241+
});
242+
243+
const archive = `archive/jeep_${moment().format('YYYY-MM-DD_HH-mm-ss')}.json`;
244+
fs.writeFileSync(archive, JSON.stringify(dealerCars, null, 2), err => {
245+
console.error(err);
246+
});
247+
248+
fs.writeFileSync('jeep.json', JSON.stringify(dealerCars, null, 2), err => {
249+
console.error(err);
250+
});
251+
252+
unmatchedCars.sort(function (a, b) {
253+
if (a.url < b.url) return -1;
254+
if (a.url > b.url) return 1;
255+
return 0;
256+
});
257+
258+
fs.writeFileSync(archive.replace('jeep_', 'unmatched-jeep_'), JSON.stringify(unmatchedCars, null, 2), err => {
259+
console.error(err);
260+
});
261+
262+
fs.writeFileSync('unmatched-jeep.json', JSON.stringify(unmatchedCars, null, 2), err => {
263+
console.error(err);
264+
});
265+
266+
console.log(`Found ${dealerCars.length} dealer cars of ${factoryCars.length} factory cars.`);
241267
return dealerCars;
242268
}
243269

‎main.js‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ function makeDealerTable(allCars) {
2424
<a href="${car.windowSticker}" target="_blank">${car.vin}</a><br />
2525
${car.engine}</td>
2626
</tr>`
27-
console.log(`${car.finalPrice} - ${car.name} (${car.url}).`);
2827
}
2928

3029
html += `</table>`;
@@ -55,7 +54,6 @@ async function makeJeepMfgTable() {
5554
<td><a href="${googleMapsLink}" target="_blank">${car.dealer.name} - ${car.dealer.cityState}</a><br />
5655
<a href="${car.windowSticker}" target="_blank">${car.vin}</a></td>
5756
</tr>`
58-
console.log(`${car.finalPrice} - ${car.model} (${car.url}).`);
5957
}
6058

6159
html += `</table>`;

0 commit comments

Comments
 (0)