From c32a273a73cab26741ba5c907f9743e2fd76115f Mon Sep 17 00:00:00 2001 From: Mike Gallegos Date: Sat, 12 Sep 2026 17:19:09 -0600 Subject: [PATCH] fix(icims): stop doubling the careers- prefix on iCIMS board hosts scan-ats-full.mjs built every iCIMS board as careers-.icims.com, but the public dataset stores many tenants as the full portal subdomain already ("careers-acme", "uscareers-acme", "acmecareers-west"). Those entries became hosts like careers-careers-acme.icims.com, answered 404, and were recorded as dead boards, so a large share of live iCIMS boards were never scanned. icimsHostCandidates() now returns the likelier host first and the other shape as a fallback: an entry already starting with careers- is used as-is; one that contains "careers" is tried as-is first; any other entry keeps the canonical careers- host first. icims.fetch() tries a fallback only when the previous host answers 404 on its first page. Any other failure is rethrown unchanged, so dead-board tracking still reads throttles and timeouts as unknown. Entries without fallback_urls (every portals.yml entry) behave as before. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012krT4azNNRJQDdavJBuzfp --- docs/SUPPORTED_JOB_BOARDS.md | 2 +- providers/icims.mjs | 83 +++++++++--- scan-ats-full.mjs | 30 ++++- tests/providers/icims-host-fallback.test.mjs | 128 +++++++++++++++++++ tests/scan-ats-full-icims-hosts.test.mjs | 75 +++++++++++ 5 files changed, 293 insertions(+), 25 deletions(-) create mode 100644 tests/providers/icims-host-fallback.test.mjs create mode 100644 tests/scan-ats-full-icims-hosts.test.mjs diff --git a/docs/SUPPORTED_JOB_BOARDS.md b/docs/SUPPORTED_JOB_BOARDS.md index bf8b260a92..13893573b2 100644 --- a/docs/SUPPORTED_JOB_BOARDS.md +++ b/docs/SUPPORTED_JOB_BOARDS.md @@ -43,7 +43,7 @@ are shared helpers and are not loaded as providers. | HigherEdJobs | RSS | Reads the public `https://www.higheredjobs.com/rss/categoryFeed.cfm?catID={catID}` feed and parses it in-process. Configure with `provider: higheredjobs` and optional `cat_id` (default 68 = Higher Education). Not auto-detected — requires explicit `provider:` config. | | Himalayas | API | Reads the board-wide `https://himalayas.app/jobs/api?limit=50` JSON remote-jobs feed. Configure with `provider: himalayas` in a `job_boards:` entry. | | IBM Careers | API | Posts to IBM's public careers search API and supports optional IBM facet filters in the portal entry. | -| iCIMS | Parser | Auto-detects any `*.icims.com` HTTPS host from `careers_url`/`api` (canonical form `https://careers-.icims.com/jobs/search?ss=1`) and scrapes the public hosted-portal search pages. List pages carry no posted date; `enrichDate()` fetches the JD detail page's JSON-LD `datePosted` for jobs that already passed title/location filters. Paginates up to a fixed 30-page cap, warning if a tenant's postings exceed it. | +| iCIMS | Parser | Auto-detects any `*.icims.com` HTTPS host from `careers_url`/`api` (canonical form `https://careers-.icims.com/jobs/search?ss=1`) and scrapes the public hosted-portal search pages. In the reverse sweep, a dataset entry that is already a portal subdomain (`careers-acme`, `uscareers-acme`) is used as-is, and each entry carries the other host shape as a fallback that is tried only when the first host answers 404 on its first page. List pages carry no posted date; `enrichDate()` fetches the JD detail page's JSON-LD `datePosted` for jobs that already passed title/location filters. Paginates up to a fixed 30-page cap, warning if a tenant's postings exceed it. | | Interamt.de | Parser | Playwright-driven scanner for Germany's federal/state/municipal public-sector job portal. Interamt runs on Apache Wicket (stateful) with no REST API, so `scan-interamt.mjs` drives a real browser session instead of an HTTP client; run directly with `npm run scan:interamt`. Reads `interamt_searches` from `portals.yml`, falling back to a generic set of German IT keywords if that section is absent. | | ITviec | Parser | Reads Vietnam's largest IT job board `https://itviec.com/it-jobs` — the project's first Vietnamese source. Configure with `provider: itviec`; optional `searchKeywords` and `searchLocation` (Ho Chi Minh / Hanoi / Da Nang) narrow the listing via the path segments the board's own search form generates. Paginates `?page=N` (10 pages by default, raise with `max_pages` up to 50), reads title, company, city and the relative "Posted … ago" label from the per-card Stimulus attributes. The board serves its listing pages fully server-rendered over plain HTTPS; `robots.txt` disallows only `/subscriptions/new`, which this parser never requests. Pacing between pages is 750ms — throttling (HTTP 429) was observed under back-to-back sweeps, so this is measured politeness. If the FIRST listing page still contains job cards but parses to none, the provider throws, so a markup change surfaces as a broken board instead of an empty one; a later page that parses to nothing simply ends the pagination. | | JibeApply | API | Auto-detects `https://.jibeapply.com/jobs` careers URLs (rewriting `/jobs` to the public `/api/jobs` endpoint); paginates `?page=N` up to `max_pages` (default 50), warning if a tenant's postings exceed the cap. Also supports branded/iCIMS-hosted sites at their own `/jobs` path via an explicit `provider: jibeapply` + `api:` URL. | diff --git a/providers/icims.mjs b/providers/icims.mjs index 067ed018d6..528b559c63 100644 --- a/providers/icims.mjs +++ b/providers/icims.mjs @@ -100,6 +100,41 @@ export function parseIcimsSearchPage(html, origin, companyName) { return jobs; } +/** + * Walk one host's search pages for `entry`. Split out of fetch() so a + * fallback host runs exactly the same pagination and truncation rules. + */ +async function fetchPortal(origin, entry, ctx) { + const all = []; + let prevFirstUrl = null; + // Distinguishes "walked the whole board" from "stopped at the page cap". + // Exhausting the cap silently would drop every later posting and look + // identical to a complete board — the same failure mode the Workday + // truncation tag exists to prevent. + let reachedEnd = false; + for (let pageNum = 0; pageNum < ICIMS_MAX_PAGES; pageNum++) { + if (pageNum > 0) await sleep(INTER_PAGE_DELAY_MS, ctx); + let html; + try { + html = await ctx.fetchText(searchUrl(origin, pageNum), { headers: HEADERS, redirect: 'error' }); + } catch (err) { + // Only a first-page failure says anything about whether this host + // has a board at all; fetch() uses the mark to decide on a fallback. + if (pageNum === 0 && err && typeof err === 'object') err.firstPage = true; + throw err; + } + const pageJobs = parseIcimsSearchPage(html, origin, entry.name); + if (pageJobs.length === 0) { reachedEnd = true; break; } // past the last page + // Some tenants serve the last real page again for an out-of-range pr + // instead of an empty one — a repeated first URL means we're looping. + if (pageJobs[0].url === prevFirstUrl) { reachedEnd = true; break; } + prevFirstUrl = pageJobs[0].url; + all.push(...pageJobs); + } + if (!reachedEnd) all.icimsTruncated = true; + return all; +} + /** @type {Provider} */ export default { id: 'icims', @@ -109,29 +144,35 @@ export default { return origin ? { url: searchUrl(origin, 0) } : null; }, + /** + * Walk a tenant's search pages. An entry may carry `fallback_urls`: other + * hosts the same tenant could be served from (scan-ats-full.mjs builds them, + * because the public dataset stores some tenants bare and some as a full + * portal subdomain). A fallback is tried only when the previous host answers + * 404 on its FIRST page, the one response that means "no board here". Any + * other failure (throttle, timeout, DNS, a later-page 404) is rethrown as-is, + * so dead-board tracking still reads it as "unknown", never "dead". + */ async fetch(entry, ctx) { - const origin = resolveOrigin(entry); - if (!origin) throw new Error(`icims: cannot derive portal origin for ${entry.name}`); - const all = []; - let prevFirstUrl = null; - // Distinguishes "walked the whole board" from "stopped at the page cap". - // Exhausting the cap silently would drop every later posting and look - // identical to a complete board — the same failure mode the Workday - // truncation tag exists to prevent. - let reachedEnd = false; - for (let pageNum = 0; pageNum < ICIMS_MAX_PAGES; pageNum++) { - if (pageNum > 0) await sleep(INTER_PAGE_DELAY_MS, ctx); - const html = await ctx.fetchText(searchUrl(origin, pageNum), { headers: HEADERS, redirect: 'error' }); - const pageJobs = parseIcimsSearchPage(html, origin, entry.name); - if (pageJobs.length === 0) { reachedEnd = true; break; } // past the last page - // Some tenants serve the last real page again for an out-of-range pr - // instead of an empty one — a repeated first URL means we're looping. - if (pageJobs[0].url === prevFirstUrl) { reachedEnd = true; break; } - prevFirstUrl = pageJobs[0].url; - all.push(...pageJobs); + const primary = resolveOrigin(entry); + if (!primary) throw new Error(`icims: cannot derive portal origin for ${entry.name}`); + const origins = [primary]; + for (const raw of Array.isArray(entry.fallback_urls) ? entry.fallback_urls : []) { + // Same https + *.icims.com gate as the primary: a fallback can never + // point the scanner at another host. + const origin = resolveOrigin({ careers_url: raw }); + if (origin && !origins.includes(origin)) origins.push(origin); + } + let notFound; + for (const origin of origins) { + try { + return await fetchPortal(origin, entry, ctx); + } catch (err) { + if (err?.status !== 404 || !err.firstPage) throw err; + notFound = err; + } } - if (!reachedEnd) all.icimsTruncated = true; - return all; + throw notFound; }, /** diff --git a/scan-ats-full.mjs b/scan-ats-full.mjs index 754d0a3e4a..ee129fb0f9 100644 --- a/scan-ats-full.mjs +++ b/scan-ats-full.mjs @@ -204,6 +204,24 @@ export function entryOnHost(name, careersUrl, isCanonicalHost) { return isCanonicalHost(hostname) ? { name, careers_url: careersUrl } : null; } +// The public iCIMS dataset is not consistent about what an entry is. Most are a +// bare tenant ("acmefreight", served at careers-acmefreight.icims.com), but +// thousands are already the full portal subdomain: "careers-acmefreight", +// "uscareers-acme", "acmecareers-west". Prefixing every entry with "careers-" +// built hosts like careers-careers-acmefreight.icims.com that do not exist, so +// those boards answered 404 and were recorded as dead. Neither reading is safe +// on its own (a bare tenant can contain a hyphen, and some bare tenants are +// served without the prefix), so return the likelier host first and the other +// shape as a fallback that icims.fetch() tries only on a first-page 404. +export function icimsHostCandidates(slug) { + const s = String(slug ?? '').toLowerCase().replace(/^-+/, ''); + if (!s) return []; + const asIs = `${s}.icims.com`; + const prefixed = `careers-${s}.icims.com`; + if (s.startsWith('careers-')) return [asIs]; + return s.includes('careers') ? [asIs, prefixed] : [prefixed, asIs]; +} + // Each source: the provider module that does the fetching, plus how to turn a // dataset entry into a synthetic PortalEntry the provider can detect/fetch. export const SOURCES = { @@ -251,9 +269,15 @@ export const SOURCES = { icims: { provider: icims, dataset: `${DATASET_BASE}/icims_companies.json`, - toEntry: (slug) => SLUG_RE.test(String(slug)) - ? entryOnHost(String(slug), `https://careers-${slug}.icims.com/jobs/search?ss=1&in_iframe=1`, h => h === `careers-${String(slug).toLowerCase()}.icims.com`) - : null, + toEntry: (slug) => { + if (!SLUG_RE.test(String(slug))) return null; + const hosts = icimsHostCandidates(slug); + if (hosts.length === 0) return null; + const [primary, ...fallbacks] = hosts.map((h) => `https://${h}/jobs/search?ss=1&in_iframe=1`); + const entry = entryOnHost(String(slug), primary, (h) => h === hosts[0]); + if (entry && fallbacks.length) entry.fallback_urls = fallbacks; + return entry; + }, }, }; diff --git a/tests/providers/icims-host-fallback.test.mjs b/tests/providers/icims-host-fallback.test.mjs new file mode 100644 index 0000000000..71d61b1401 --- /dev/null +++ b/tests/providers/icims-host-fallback.test.mjs @@ -0,0 +1,128 @@ +// tests/providers/icims-host-fallback.test.mjs — fetch() moves to a fallback +// host only when the previous host answers 404 on its first page. +import { join } from 'path'; +import { pathToFileURL } from 'url'; +import { pass, fail, ROOT } from '../helpers.mjs'; + +console.log('\nProvider — icims host fallback'); + +const icims = (await import(pathToFileURL(join(ROOT, 'providers/icims.mjs')).href)).default; + +const PRIMARY = 'https://careers-acmefreight.icims.com'; +const FALLBACK = 'https://acmefreight.icims.com'; +const card = (origin, id, title) => `
  • +

    ${title}

    • `; +const page = (...cards) => `
      ${cards.join('')}
    `; +const httpError = (status) => Object.assign(new Error(`HTTP ${status}`), { status }); + +// boards maps an origin to its pages (array), a per-page function, or an Error +// thrown for every page. An origin with no entry answers 404. +function mkCtx(boards) { + const calls = []; + return { + calls, + transport: 'http', + sleep: async () => {}, + fetchJson: async () => { throw new Error('fetchJson should not be called'); }, + fetchText: async (url) => { + const u = new URL(url); + const pr = Number(u.searchParams.get('pr')); + calls.push(`${u.origin}#${pr}`); + const board = boards[u.origin]; + if (board === undefined) throw httpError(404); + if (board instanceof Error) throw board; + if (typeof board === 'function') return board(pr); + return board[pr] ?? page(); + }, + }; +} + +const entry = { + name: 'acmefreight', + careers_url: `${PRIMARY}/jobs/search?ss=1&in_iframe=1`, + fallback_urls: [`${FALLBACK}/jobs/search?ss=1&in_iframe=1`], +}; +const onlyPrimary = (ctx) => ctx.calls.length > 0 && ctx.calls.every((c) => c.startsWith(`${PRIMARY}#`)); + +// Primary answers 404 on its first page: the fallback host is walked instead. +{ + const ctx = mkCtx({ [FALLBACK]: [page(card(FALLBACK, 1, 'Role A'))] }); + const jobs = await icims.fetch(entry, ctx); + if (jobs.length === 1 && jobs[0].url.startsWith(`${FALLBACK}/jobs/1/`)) pass('first-page 404 on the primary host falls back to the next host'); + else fail(`fallback: jobs=${JSON.stringify(jobs)} calls=${ctx.calls.join(',')}`); +} + +// A live primary never requests the fallback. +{ + const ctx = mkCtx({ + [PRIMARY]: [page(card(PRIMARY, 2, 'Role B'))], + [FALLBACK]: new Error('fallback must not be requested'), + }); + const jobs = await icims.fetch(entry, ctx); + if (jobs.length === 1 && onlyPrimary(ctx)) pass('a live primary host never requests the fallback'); + else fail(`live primary: jobs=${jobs.length} calls=${ctx.calls.join(',')}`); +} + +// Every host answers 404: the 404 surfaces, so dead-board tracking counts a miss. +{ + const ctx = mkCtx({}); + try { + await icims.fetch(entry, ctx); + fail('fetch resolved with no live host'); + } catch (err) { + if (err.status === 404 && ctx.calls.join(',') === `${PRIMARY}#0,${FALLBACK}#0`) pass('all hosts 404: each tried once, then the 404 is thrown'); + else fail(`all 404: status=${err.status} calls=${ctx.calls.join(',')}`); + } +} + +// A throttle is not "no board here": rethrown without trying the fallback. +{ + const ctx = mkCtx({ [PRIMARY]: httpError(429), [FALLBACK]: [page(card(FALLBACK, 3, 'Role C'))] }); + try { + await icims.fetch(entry, ctx); + fail('fetch swallowed a 429'); + } catch (err) { + if (err.status === 429 && onlyPrimary(ctx)) pass('non-404 failure is rethrown without falling back'); + else fail(`429: status=${err.status} calls=${ctx.calls.join(',')}`); + } +} + +// A 404 after the first page is a problem on a real board, not a missing board. +{ + const ctx = mkCtx({ + [PRIMARY]: (pr) => { if (pr === 0) return page(card(PRIMARY, 4, 'Role D')); throw httpError(404); }, + [FALLBACK]: [page(card(FALLBACK, 5, 'Role E'))], + }); + try { + await icims.fetch(entry, ctx); + fail('fetch resolved despite a later-page 404'); + } catch (err) { + if (err.status === 404 && onlyPrimary(ctx)) pass('a later-page 404 does not switch hosts'); + else fail(`later-page 404: status=${err.status} calls=${ctx.calls.join(',')}`); + } +} + +// Fallback URLs that are not https *.icims.com are ignored. +{ + const ctx = mkCtx({ 'https://evil.example': [page(card('https://evil.example', 6, 'Role F'))] }); + const hostile = { ...entry, fallback_urls: ['https://evil.example/jobs/search', 'http://acmefreight.icims.com/jobs/search'] }; + try { + await icims.fetch(hostile, ctx); + fail('fetch followed an off-host fallback'); + } catch (err) { + if (err.status === 404 && onlyPrimary(ctx)) pass('fallback URLs off https *.icims.com are ignored'); + else fail(`hostile fallback: status=${err.status} calls=${ctx.calls.join(',')}`); + } +} + +// An entry with no fallback behaves exactly as before: the 404 is thrown. +{ + const ctx = mkCtx({}); + try { + await icims.fetch({ name: 'acmefreight', careers_url: `${PRIMARY}/jobs/search?ss=1` }, ctx); + fail('fetch resolved for a missing board without fallbacks'); + } catch (err) { + if (err.status === 404 && ctx.calls.length === 1) pass('entry without fallback_urls: 404 thrown after one request'); + else fail(`no fallback: status=${err.status} calls=${ctx.calls.join(',')}`); + } +} diff --git a/tests/scan-ats-full-icims-hosts.test.mjs b/tests/scan-ats-full-icims-hosts.test.mjs new file mode 100644 index 0000000000..668b6ff7d5 --- /dev/null +++ b/tests/scan-ats-full-icims-hosts.test.mjs @@ -0,0 +1,75 @@ +// tests/scan-ats-full-icims-hosts.test.mjs — how iCIMS dataset entries become +// board hosts. The public dataset stores some tenants bare ("acmefreight") and +// some as a full portal subdomain ("careers-acmefreight"); prefixing both with +// "careers-" produced careers-careers-* hosts that do not exist. +import { join } from 'path'; +import { pathToFileURL } from 'url'; +import { pass, fail, ROOT } from './helpers.mjs'; + +console.log('\nscan-ats-full — iCIMS host candidates'); + +const { SOURCES, icimsHostCandidates } = await import(pathToFileURL(join(ROOT, 'scan-ats-full.mjs')).href); +const url = (host) => `https://${host}/jobs/search?ss=1&in_iframe=1`; +const same = (a, b) => JSON.stringify(a) === JSON.stringify(b); + +// A bare tenant keeps the canonical careers- host first, with the bare host as fallback. +{ + const e = SOURCES.icims.toEntry('acmefreight'); + if (e?.careers_url === url('careers-acmefreight.icims.com') && same(e.fallback_urls, [url('acmefreight.icims.com')])) { + pass('bare tenant: careers- host first, bare host as fallback'); + } else { + fail(`bare tenant: ${JSON.stringify(e)}`); + } +} + +// An entry that is already a careers- subdomain is used as-is and never doubled. +{ + const e = SOURCES.icims.toEntry('careers-acmefreight'); + if (e?.careers_url === url('careers-acmefreight.icims.com') && e.fallback_urls === undefined) { + pass('careers- entry used as-is, no careers-careers- host'); + } else { + fail(`careers- entry: ${JSON.stringify(e)}`); + } +} + +// Other portal subdomains that contain "careers" are tried as-is first. +{ + const e = SOURCES.icims.toEntry('uscareers-acme'); + if (e?.careers_url === url('uscareers-acme.icims.com') && same(e.fallback_urls, [url('careers-uscareers-acme.icims.com')])) { + pass('portal subdomain containing "careers" tried as-is first'); + } else { + fail(`uscareers- entry: ${JSON.stringify(e)}`); + } +} + +// A hyphenated entry without "careers" could be either shape; the prefixed host stays first. +{ + const e = SOURCES.icims.toEntry('jobs-acme'); + if (e?.careers_url === url('careers-jobs-acme.icims.com') && same(e.fallback_urls, [url('jobs-acme.icims.com')])) { + pass('hyphenated entry without "careers": prefixed host first, as-is fallback'); + } else { + fail(`jobs- entry: ${JSON.stringify(e)}`); + } +} + +// Normalization: the dataset carries one entry with a stray leading dash. +{ + if (same(icimsHostCandidates('-careers-acme'), ['careers-acme.icims.com'])) pass('leading dash stripped before building the host'); + else fail(`leading dash: ${JSON.stringify(icimsHostCandidates('-careers-acme'))}`); + + if (same(icimsHostCandidates('AcmeFreight'), ['careers-acmefreight.icims.com', 'acmefreight.icims.com'])) pass('host lowercased'); + else fail(`uppercase: ${JSON.stringify(icimsHostCandidates('AcmeFreight'))}`); + + if (same(icimsHostCandidates(''), [])) pass('empty entry yields no host'); + else fail(`empty: ${JSON.stringify(icimsHostCandidates(''))}`); +} + +// Every candidate stays on icims.com, and hostile input is still rejected. +{ + const hosts = ['acme', 'careers-acme', 'jobs-acme', 'acmecareers-west', 'a.b'].flatMap(icimsHostCandidates); + if (hosts.length > 0 && hosts.every((h) => h.endsWith('.icims.com'))) pass('all candidate hosts stay on icims.com'); + else fail(`off-host candidate: ${JSON.stringify(hosts)}`); + + if (SOURCES.icims.toEntry('evil/..%2f') === null) pass('toEntry still rejects non-slug input'); + else fail('toEntry accepted a hostile slug'); +}