From f30ce56826b4ff8baca49411664c46587830b519 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 6 Sep 2026 08:14:47 +0000 Subject: [PATCH 1/2] Find the car when the two databases call it different things, and stop waiting on a map server Two things the live deployment showed that the local one could not. vPIC decodes a VIN to "Outback". The EPA lists "Outback AWD" and "Outback AWD Turbo" for the same year, so an exact match found nothing and a 2019 Outback came back with no mpg, no engine and no fuel type. When the exact name misses, ask the EPA what it calls that make's models that year and take the closest: ours-plus-a-qualifier first, then ours-minus-one, then the loose match, then nothing rather than the wrong car. The answer says which name it matched on. And a cold Overpass mirror turned a three-second profile into a forty-nine second one. Inside a profile the map now gets nine seconds and two mirrors, because the other five sections are already answered and nobody should wait on repair-shop names to be told their airbag is recalled. Overpass is told the same budget as its own server-side timeout, so it stops working on an answer no one is waiting for. /mechanics on its own is still patient, and either way the tile is cached for a week: one caller waits, nobody after them does. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012BzJdwiBGU9hmMGE5fBxjA --- apps/web/src/lib/automotive.js | 101 +++++++++++++++++++++++++++++---- test/automotive.test.js | 15 +++++ 2 files changed, 104 insertions(+), 12 deletions(-) diff --git a/apps/web/src/lib/automotive.js b/apps/web/src/lib/automotive.js index b0208fd..ab70521 100644 --- a/apps/web/src/lib/automotive.js +++ b/apps/web/src/lib/automotive.js @@ -341,14 +341,60 @@ export async function ratingFor({ year, make, model }) { }; } +const menuItems = (res) => { + const raw = res?.menuItem; + return (Array.isArray(raw) ? raw : raw ? [raw] : []).filter(Boolean); +}; + +/** + * The two databases do not agree on what a car is called. + * + * vPIC decodes a VIN to "Outback"; the EPA lists "Outback AWD" and + * "Outback AWD Turbo" for the same year. An exact match therefore finds + * nothing for a great many real cars, so when it misses, ask the EPA what it + * calls that make's models that year and take the closest name. + */ +export function closestModel(wanted, candidates) { + const want = String(wanted ?? '') + .trim() + .toLowerCase(); + if (!want) return null; + const names = candidates.map((c) => String(c).trim()); + const exact = names.find((n) => n.toLowerCase() === want); + if (exact) return exact; + // "Outback" → "Outback AWD": the EPA name usually adds a qualifier. + const prefixed = names.filter((n) => n.toLowerCase().startsWith(want)); + if (prefixed.length) return prefixed.sort((a, b) => a.length - b.length)[0]; + // "Civic Hatchback" → "Civic": ours is the longer name. + const contained = names.filter((n) => want.startsWith(n.toLowerCase())); + if (contained.length) return contained.sort((a, b) => b.length - a.length)[0]; + const loose = names.filter( + (n) => n.toLowerCase().includes(want) || want.includes(n.toLowerCase()), + ); + return loose.sort((a, b) => a.length - b.length)[0] ?? null; +} + export async function economyFor({ year, make, model }) { if (!year || !make || !model) return null; - const menu = await getJson( - `${FE}/vehicle/menu/options?year=${year}&make=${encodeURIComponent(make)}&model=${encodeURIComponent(model)}`, - { headers: { accept: 'application/json' } }, - ).catch(() => null); - const raw = menu?.menuItem; - const trims = (Array.isArray(raw) ? raw : raw ? [raw] : []).filter(Boolean); + const options = (name) => + getJson( + `${FE}/vehicle/menu/options?year=${year}&make=${encodeURIComponent(make)}&model=${encodeURIComponent(name)}`, + { headers: { accept: 'application/json' } }, + ).catch(() => null); + + let trims = menuItems(await options(model)); + let matched = model; + if (!trims.length) { + const models = menuItems( + await getJson(`${FE}/vehicle/menu/model?year=${year}&make=${encodeURIComponent(make)}`, { + headers: { accept: 'application/json' }, + }).catch(() => null), + ).map((m) => String(m.value)); + const best = closestModel(model, models); + if (!best) return null; + trims = menuItems(await options(best)); + matched = best; + } if (!trims.length) return null; const detail = await getJson(`${FE}/vehicle/${encodeURIComponent(trims[0].value)}`, { headers: { accept: 'application/json' }, @@ -358,6 +404,8 @@ export async function economyFor({ year, make, model }) { return Number.isFinite(x) && x !== -1 ? x : null; }; return { + // Said out loud when it is not the name that was asked for. + matchedModel: matched === model ? null : matched, trims: trims.map((t) => ({ id: String(t.value), name: t.text })), fuel: detail?.fuelType ?? null, cylinders: n(detail?.cylinders), @@ -590,8 +638,22 @@ export function milesBetween(a, b) { * Tuesday — so this tries the mirrors in turn and caches per rounded tile. * A miss returns an empty list with a reason rather than failing the profile * around it: a VIN lookup should not 500 because a map server is busy. + * + * It should not take a minute either. Asked from inside a vehicle profile the + * budget is short and only the first mirrors are tried, because the rest of + * the answer is already waiting; asked on its own, `/mechanics` can afford to + * be patient. Either way the tile is cached for a week, so the slow call + * happens to one caller and no one after them. */ -export async function placesNear({ lat, lon, radiusMiles = 10, kind = 'car_repair', limit = 25 }) { +export async function placesNear({ + lat, + lon, + radiusMiles = 10, + kind = 'car_repair', + limit = 25, + timeoutMs = 25_000, + maxMirrors = OVERPASS.length, +}) { if (typeof lat !== 'number' || typeof lon !== 'number' || Number.isNaN(lat) || Number.isNaN(lon)) return { places: [], error: 'A latitude and longitude are needed.' }; const filter = OSM_KINDS[kind]; @@ -603,14 +665,17 @@ export async function placesNear({ lat, lon, radiusMiles = 10, kind = 'car_repai const cached = await auto.getPlaces(key, 7 * 24 * 3600).catch(() => null); if (cached) return { places: cached.slice(0, limit), cached: true, attribution: OSM_ATTRIBUTION }; - const query = `[out:json][timeout:20];nwr[${filter.split('=')[0]}=${filter.split('=')[1]}](around:${radiusM},${lat},${lon});out center ${Math.min(limit * 2, 60)};`; - for (const endpoint of OVERPASS) { + // Overpass's own server-side timeout is told the same budget, so it gives up + // when we would have anyway rather than working on an answer nobody waits for. + const serverSeconds = Math.max(5, Math.round(timeoutMs / 1000) - 2); + const query = `[out:json][timeout:${serverSeconds}];nwr[${filter.split('=')[0]}=${filter.split('=')[1]}](around:${radiusM},${lat},${lon});out center ${Math.min(limit * 2, 60)};`; + for (const endpoint of OVERPASS.slice(0, Math.max(1, maxMirrors))) { try { const res = await fetch(endpoint, { method: 'POST', headers: { 'user-agent': UA, 'content-type': 'application/x-www-form-urlencoded' }, body: `data=${encodeURIComponent(query)}`, - signal: AbortSignal.timeout(25_000), + signal: AbortSignal.timeout(timeoutMs), }); if (!res.ok) continue; const json = await res.json(); @@ -634,6 +699,14 @@ export async function placesNear({ lat, lon, radiusMiles = 10, kind = 'car_repai export const OSM_ATTRIBUTION = '© OpenStreetMap contributors, ODbL'; +/** + * What a vehicle profile will wait for a map server. Short on purpose: the + * other five sections are already answered, and a cold Overpass mirror can + * otherwise turn a three-second profile into a fifty-second one. The tile is + * cached for a week, so the next caller in that town waits for nothing. + */ +const PLACE_BUDGET = { timeoutMs: 9_000, maxMirrors: 2 }; + /* ---------------------------------------------------------------- profile -- */ /** @@ -682,10 +755,14 @@ export async function vehicleProfile({ }) .catch(() => []), wantPlaces - ? placesNear({ lat, lon, radiusMiles, kind: 'car_repair' }).catch(() => ({ places: [] })) + ? placesNear({ ...PLACE_BUDGET, lat, lon, radiusMiles, kind: 'car_repair' }).catch(() => ({ + places: [], + })) : Promise.resolve(null), wantPlaces - ? placesNear({ lat, lon, radiusMiles, kind: 'car_parts' }).catch(() => ({ places: [] })) + ? placesNear({ ...PLACE_BUDGET, lat, lon, radiusMiles, kind: 'car_parts' }).catch(() => ({ + places: [], + })) : Promise.resolve(null), ]); diff --git a/test/automotive.test.js b/test/automotive.test.js index ff313a4..9270797 100644 --- a/test/automotive.test.js +++ b/test/automotive.test.js @@ -8,6 +8,7 @@ process.env.SITE_URL ??= 'https://nichedb.test'; const { checkDigit, checkDigitOk, + closestModel, compactDecode, maintenanceSchedule, milesBetween, @@ -129,6 +130,20 @@ describe('parts and places', () => { expect(partsSearches({ year: 2014, make: 'Honda' })).toEqual([]); }); + test('the EPA and vPIC disagree on names, so the closest one wins', () => { + // vPIC decodes "Outback"; the EPA lists it with a drivetrain qualifier. + expect(closestModel('Outback', ['Outback AWD', 'Outback AWD Turbo', 'Impreza AWD'])).toBe( + 'Outback AWD', + ); + // Ours can be the longer name instead. + expect(closestModel('Civic Hatchback', ['Civic', 'Civic Si', 'Accord'])).toBe('Civic'); + // An exact name is never second-guessed. + expect(closestModel('Accord', ['Accord', 'Accord Hybrid'])).toBe('Accord'); + // Nothing close is null rather than a wrong car. + expect(closestModel('Outback', ['F-150', 'Mustang'])).toBeNull(); + expect(closestModel('', ['Accord'])).toBeNull(); + }); + test('distance is in miles and survives a missing coordinate', () => { // San Francisco to Oakland, about 10 miles. const miles = milesBetween({ lat: 37.7749, lon: -122.4194 }, { lat: 37.8044, lon: -122.2712 }); From d0fa1440760b75817072dccee37d0befaf8d1545 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Sun, 6 Sep 2026 08:26:24 +0000 Subject: [PATCH 2/2] NHTSA answers 403, not 429, so the crash-test walk goes slowly on purpose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ratings cost three nested calls per vehicle — the make's models, that model's variants, then each variant's full record — which makes this by far the hungriest walk here. Raising the per-run budget to fill the catalogue faster pushed it over whatever line NHTSA draws, and the answer came back 403. A 403 is not 429, so the client's retry-after backoff never saw it and the source recorded a failure instead of waiting. So this one now goes deliberately slowly: sixty lookups a run with a short pause between calls, both configurable. Nothing about a crash test from a past model year needs to arrive quickly. And a throttle on the very first call of a run — listing the year's makes — now ends the run quietly with the cursor untouched, so the next one resumes rather than recording the whole source broken. The registry test caught that the new options were not declared as config fields, which is exactly what it is for. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012BzJdwiBGU9hmMGE5fBxjA --- packages/adapters/src/nhtsa.js | 44 +++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/packages/adapters/src/nhtsa.js b/packages/adapters/src/nhtsa.js index f769444..cefd4b1 100644 --- a/packages/adapters/src/nhtsa.js +++ b/packages/adapters/src/nhtsa.js @@ -353,12 +353,37 @@ export const nhtsaRatings = defineAdapter({ type: 'number', help: 'NCAP publishes by model year; 5 covers the current market.', }, + { + key: 'perRun', + label: 'Lookups per run', + type: 'number', + help: 'Three calls per vehicle, so this is the hungriest walk here. NHTSA answers 403 rather than 429 when pushed.', + }, + { + key: 'paceMs', + label: 'Pause between calls', + type: 'number', + help: 'Milliseconds. A short wait is what keeps NHTSA answering.', + }, ], - defaults: { yearsBack: 5 }, + defaults: { yearsBack: 5, perRun: 60, paceMs: 150 }, defaultSources: [ - { slug: 'nhtsa-safety-ratings', name: 'NHTSA: crash-test ratings', config: { yearsBack: 5 } }, + { + slug: 'nhtsa-safety-ratings', + name: 'NHTSA: crash-test ratings', + config: { yearsBack: 5, perRun: 60, paceMs: 150 }, + }, ], async pull({ config, cursor, http, log, budget, deadline }) { + // Ratings cost three nested calls per vehicle — the make's models, that + // model's variants, then each variant's full record — so this is by far + // the hungriest walk here. Asked at full speed, NHTSA answers 403 rather + // than 429, which no retry-after backoff catches. So it goes deliberately + // slowly: a smaller run and a pause between calls. Nothing about a crash + // test from a past model year needs to arrive quickly. + const cap = Math.min(Math.max(Number(config.perRun) || 60, 5), budget); + const paceMs = Math.min(Math.max(Number(config.paceMs) || 0, 0), 2000); + const pace = () => (paceMs ? Bun.sleep(paceMs) : Promise.resolve()); const years = yearsFor(config); let yearIdx = Number(cursor.yearIdx) || 0; if (yearIdx >= years.length) yearIdx = 0; @@ -367,7 +392,11 @@ export const nhtsaRatings = defineAdapter({ let makes = Array.isArray(cursor.makes) && cursor.cursorYear === year ? cursor.makes : null; if (!makes) { - const res = await http.json(`${RATINGS}/modelyear/${year}`); + const res = await http.json(`${RATINGS}/modelyear/${year}`).catch((err) => { + log(`could not list ${year} makes: ${err.message}`); + return null; + }); + if (!res) return { items: [], cursor, note: `${year}: upstream busy, will resume` }; makes = [...new Set((res.Results ?? []).map((r) => r.Make).filter(Boolean))].sort(); makeIdx = 0; log(`${makes.length} makes rated for ${year}`); @@ -375,14 +404,16 @@ export const nhtsaRatings = defineAdapter({ const items = []; let spent = 0; - while (makeIdx < makes.length && spent < budget && Date.now() < deadline) { + while (makeIdx < makes.length && spent < cap && Date.now() < deadline) { const make = makes[makeIdx]; + await pace(); const models = await http .json(`${RATINGS}/modelyear/${year}/make/${encodeURIComponent(make)}`) .catch(() => ({ Results: [] })); spent++; for (const m of models.Results ?? []) { - if (spent >= budget || Date.now() >= deadline) break; + if (spent >= cap || Date.now() >= deadline) break; + await pace(); const detail = await http .json( `${RATINGS}/modelyear/${year}/make/${encodeURIComponent(make)}/model/${encodeURIComponent(m.Model)}`, @@ -390,7 +421,8 @@ export const nhtsaRatings = defineAdapter({ .catch(() => ({ Results: [] })); spent++; for (const v of detail.Results ?? []) { - if (!v.VehicleId || spent >= budget) continue; + if (!v.VehicleId || spent >= cap || Date.now() >= deadline) continue; + await pace(); const full = await http.json(`${RATINGS}/VehicleId/${v.VehicleId}`).catch(() => null); spent++; items.push(ratingToItem(full?.Results?.[0] ?? v, { year, make, model: m.Model }));