fix(icims): stop doubling the careers- prefix on iCIMS board hosts

docs/SUPPORTED_JOB_BOARDS.md

@@ -43,7 +43,7 @@ are shared helpers and are not loaded as providers.
 | HigherEdJobs | RSS | Reads the public `https://www.higheredjobs.com/rss/categoryFeed.cfm?catID={catID}` feed and parses it in-process. Configure with `provider: higheredjobs` and optional `cat_id` (default 68 = Higher Education). Not auto-detected — requires explicit `provider:` config. |
 | Himalayas | API | Reads the board-wide `https://himalayas.app/jobs/api?limit=50` JSON remote-jobs feed. Configure with `provider: himalayas` in a `job_boards:` entry. |
 | IBM Careers | API | Posts to IBM's public careers search API and supports optional IBM facet filters in the portal entry. |
-| iCIMS | Parser | Auto-detects any `*.icims.com` HTTPS host from `careers_url`/`api` (canonical form `https://careers-<tenant>.icims.com/jobs/search?ss=1`) and scrapes the public hosted-portal search pages. List pages carry no posted date; `enrichDate()` fetches the JD detail page's JSON-LD `datePosted` for jobs that already passed title/location filters. Paginates up to a fixed 30-page cap, warning if a tenant's postings exceed it. |
+| iCIMS | Parser | Auto-detects any `*.icims.com` HTTPS host from `careers_url`/`api` (canonical form `https://careers-<tenant>.icims.com/jobs/search?ss=1`) and scrapes the public hosted-portal search pages. In the reverse sweep, a dataset entry that is already a portal subdomain (`careers-acme`, `uscareers-acme`) is used as-is, and each entry carries the other host shape as a fallback that is tried only when the first host answers 404 on its first page. List pages carry no posted date; `enrichDate()` fetches the JD detail page's JSON-LD `datePosted` for jobs that already passed title/location filters. Paginates up to a fixed 30-page cap, warning if a tenant's postings exceed it. |
 | Interamt.de | Parser | Playwright-driven scanner for Germany's federal/state/municipal public-sector job portal. Interamt runs on Apache Wicket (stateful) with no REST API, so `scan-interamt.mjs` drives a real browser session instead of an HTTP client; run directly with `npm run scan:interamt`. Reads `interamt_searches` from `portals.yml`, falling back to a generic set of German IT keywords if that section is absent. |
 | ITviec | Parser | Reads Vietnam's largest IT job board `https://itviec.com/it-jobs` — the project's first Vietnamese source. Configure with `provider: itviec`; optional `searchKeywords` and `searchLocation` (Ho Chi Minh / Hanoi / Da Nang) narrow the listing via the path segments the board's own search form generates. Paginates `?page=N` (10 pages by default, raise with `max_pages` up to 50), reads title, company, city and the relative "Posted … ago" label from the per-card Stimulus attributes. The board serves its listing pages fully server-rendered over plain HTTPS; `robots.txt` disallows only `/subscriptions/new`, which this parser never requests. Pacing between pages is 750ms — throttling (HTTP 429) was observed under back-to-back sweeps, so this is measured politeness. If the FIRST listing page still contains job cards but parses to none, the provider throws, so a markup change surfaces as a broken board instead of an empty one; a later page that parses to nothing simply ends the pagination. |
 | JibeApply | API | Auto-detects `https://<slug>.jibeapply.com/jobs` careers URLs (rewriting `/jobs` to the public `/api/jobs` endpoint); paginates `?page=N` up to `max_pages` (default 50), warning if a tenant's postings exceed the cap. Also supports branded/iCIMS-hosted sites at their own `/jobs` path via an explicit `provider: jibeapply` + `api:` URL. |

providers/icims.mjs

@@ -100,6 +100,41 @@ export function parseIcimsSearchPage(html, origin, companyName) {
   return jobs;
 }
 
+/**
+ * Walk one host's search pages for `entry`. Split out of fetch() so a
+ * fallback host runs exactly the same pagination and truncation rules.
+ */
+async function fetchPortal(origin, entry, ctx) {
+  const all = [];
+  let prevFirstUrl = null;
+  // Distinguishes "walked the whole board" from "stopped at the page cap".
+  // Exhausting the cap silently would drop every later posting and look
+  // identical to a complete board — the same failure mode the Workday
+  // truncation tag exists to prevent.
+  let reachedEnd = false;
+  for (let pageNum = 0; pageNum < ICIMS_MAX_PAGES; pageNum++) {
+    if (pageNum > 0) await sleep(INTER_PAGE_DELAY_MS, ctx);
+    let html;
+    try {
+      html = await ctx.fetchText(searchUrl(origin, pageNum), { headers: HEADERS, redirect: 'error' });
+    } catch (err) {
+      // Only a first-page failure says anything about whether this host
+      // has a board at all; fetch() uses the mark to decide on a fallback.
+      if (pageNum === 0 && err && typeof err === 'object') err.firstPage = true;
+      throw err;
+    }
+    const pageJobs = parseIcimsSearchPage(html, origin, entry.name);
+    if (pageJobs.length === 0) { reachedEnd = true; break; } // past the last page
+    // Some tenants serve the last real page again for an out-of-range pr
+    // instead of an empty one — a repeated first URL means we're looping.
+    if (pageJobs[0].url === prevFirstUrl) { reachedEnd = true; break; }
+    prevFirstUrl = pageJobs[0].url;
+    all.push(...pageJobs);
+  }
+  if (!reachedEnd) all.icimsTruncated = true;
+  return all;
+}
+
 /** @type {Provider} */
 export default {
   id: 'icims',
@@ -109,29 +144,35 @@ export default {
     return origin ? { url: searchUrl(origin, 0) } : null;
   },
 
+  /**
+   * Walk a tenant's search pages. An entry may carry `fallback_urls`: other
+   * hosts the same tenant could be served from (scan-ats-full.mjs builds them,
+   * because the public dataset stores some tenants bare and some as a full
+   * portal subdomain). A fallback is tried only when the previous host answers
+   * 404 on its FIRST page, the one response that means "no board here". Any
+   * other failure (throttle, timeout, DNS, a later-page 404) is rethrown as-is,
+   * so dead-board tracking still reads it as "unknown", never "dead".
+   */
   async fetch(entry, ctx) {
-    const origin = resolveOrigin(entry);
-    if (!origin) throw new Error(`icims: cannot derive portal origin for ${entry.name}`);
-    const all = [];
-    let prevFirstUrl = null;
-    // Distinguishes "walked the whole board" from "stopped at the page cap".
-    // Exhausting the cap silently would drop every later posting and look
-    // identical to a complete board — the same failure mode the Workday
-    // truncation tag exists to prevent.
-    let reachedEnd = false;
-    for (let pageNum = 0; pageNum < ICIMS_MAX_PAGES; pageNum++) {
-      if (pageNum > 0) await sleep(INTER_PAGE_DELAY_MS, ctx);
-      const html = await ctx.fetchText(searchUrl(origin, pageNum), { headers: HEADERS, redirect: 'error' });
-      const pageJobs = parseIcimsSearchPage(html, origin, entry.name);
-      if (pageJobs.length === 0) { reachedEnd = true; break; } // past the last page
-      // Some tenants serve the last real page again for an out-of-range pr
-      // instead of an empty one — a repeated first URL means we're looping.
-      if (pageJobs[0].url === prevFirstUrl) { reachedEnd = true; break; }
-      prevFirstUrl = pageJobs[0].url;
-      all.push(...pageJobs);
+    const primary = resolveOrigin(entry);
+    if (!primary) throw new Error(`icims: cannot derive portal origin for ${entry.name}`);
+    const origins = [primary];
+    for (const raw of Array.isArray(entry.fallback_urls) ? entry.fallback_urls : []) {
+      // Same https + *.icims.com gate as the primary: a fallback can never
+      // point the scanner at another host.
+      const origin = resolveOrigin({ careers_url: raw });
+      if (origin && !origins.includes(origin)) origins.push(origin);
+    }
+    let notFound;
+    for (const origin of origins) {
+      try {
+        return await fetchPortal(origin, entry, ctx);
+      } catch (err) {
+        if (err?.status !== 404 || !err.firstPage) throw err;
+        notFound = err;
+      }
     }
-    if (!reachedEnd) all.icimsTruncated = true;
-    return all;
+    throw notFound;
   },
 
   /**

scan-ats-full.mjs

@@ -204,6 +204,24 @@ export function entryOnHost(name, careersUrl, isCanonicalHost) {
   return isCanonicalHost(hostname) ? { name, careers_url: careersUrl } : null;
 }
 
+// The public iCIMS dataset is not consistent about what an entry is. Most are a
+// bare tenant ("acmefreight", served at careers-acmefreight.icims.com), but
+// thousands are already the full portal subdomain: "careers-acmefreight",
+// "uscareers-acme", "acmecareers-west". Prefixing every entry with "careers-"
+// built hosts like careers-careers-acmefreight.icims.com that do not exist, so
+// those boards answered 404 and were recorded as dead. Neither reading is safe
+// on its own (a bare tenant can contain a hyphen, and some bare tenants are
+// served without the prefix), so return the likelier host first and the other
+// shape as a fallback that icims.fetch() tries only on a first-page 404.
+export function icimsHostCandidates(slug) {
+  const s = String(slug ?? '').toLowerCase().replace(/^-+/, '');
+  if (!s) return [];
+  const asIs = `${s}.icims.com`;
+  const prefixed = `careers-${s}.icims.com`;
+  if (s.startsWith('careers-')) return [asIs];
+  return s.includes('careers') ? [asIs, prefixed] : [prefixed, asIs];
+}
+
 // Each source: the provider module that does the fetching, plus how to turn a
 // dataset entry into a synthetic PortalEntry the provider can detect/fetch.
 export const SOURCES = {
@@ -251,9 +269,15 @@ export const SOURCES = {
   icims: {
     provider: icims,
     dataset: `${DATASET_BASE}/icims_companies.json`,
-    toEntry: (slug) => SLUG_RE.test(String(slug))
-      ? entryOnHost(String(slug), `https://careers-${slug}.icims.com/jobs/search?ss=1&in_iframe=1`, h => h === `careers-${String(slug).toLowerCase()}.icims.com`)
-      : null,
+    toEntry: (slug) => {
+      if (!SLUG_RE.test(String(slug))) return null;
+      const hosts = icimsHostCandidates(slug);
+      if (hosts.length === 0) return null;
+      const [primary, ...fallbacks] = hosts.map((h) => `https://${h}/jobs/search?ss=1&in_iframe=1`);
+      const entry = entryOnHost(String(slug), primary, (h) => h === hosts[0]);
+      if (entry && fallbacks.length) entry.fallback_urls = fallbacks;
+      return entry;
+    },
   },
 };
 

tests/providers/icims-host-fallback.test.mjs

@@ -0,0 +1,128 @@
+// tests/providers/icims-host-fallback.test.mjs — fetch() moves to a fallback
+// host only when the previous host answers 404 on its first page.
+import { join } from 'path';
+import { pathToFileURL } from 'url';
+import { pass, fail, ROOT } from '../helpers.mjs';
+
+console.log('\nProvider — icims host fallback');
+
+const icims = (await import(pathToFileURL(join(ROOT, 'providers/icims.mjs')).href)).default;
+
+const PRIMARY = 'https://careers-acmefreight.icims.com';
+const FALLBACK = 'https://acmefreight.icims.com';
+const card = (origin, id, title) => `<li class="iCIMS_JobCardItem"><div class="col-xs-12 title">
+<a href="${origin}/jobs/${id}/role-${id}/job?in_iframe=1" class="iCIMS_Anchor"><h3>${title}</h3></a></div></li>`;
+const page = (...cards) => `<ul class="iCIMS_JobsTable">${cards.join('')}</ul>`;
+const httpError = (status) => Object.assign(new Error(`HTTP ${status}`), { status });
+
+// boards maps an origin to its pages (array), a per-page function, or an Error
+// thrown for every page. An origin with no entry answers 404.
+function mkCtx(boards) {
+  const calls = [];
+  return {
+    calls,
+    transport: 'http',
+    sleep: async () => {},
+    fetchJson: async () => { throw new Error('fetchJson should not be called'); },
+    fetchText: async (url) => {
+      const u = new URL(url);
+      const pr = Number(u.searchParams.get('pr'));
+      calls.push(`${u.origin}#${pr}`);
+      const board = boards[u.origin];
+      if (board === undefined) throw httpError(404);
+      if (board instanceof Error) throw board;
+      if (typeof board === 'function') return board(pr);
+      return board[pr] ?? page();
+    },
+  };
+}
+
+const entry = {
+  name: 'acmefreight',
+  careers_url: `${PRIMARY}/jobs/search?ss=1&in_iframe=1`,
+  fallback_urls: [`${FALLBACK}/jobs/search?ss=1&in_iframe=1`],
+};
+const onlyPrimary = (ctx) => ctx.calls.length > 0 && ctx.calls.every((c) => c.startsWith(`${PRIMARY}#`));
+
+// Primary answers 404 on its first page: the fallback host is walked instead.
+{
+  const ctx = mkCtx({ [FALLBACK]: [page(card(FALLBACK, 1, 'Role A'))] });
+  const jobs = await icims.fetch(entry, ctx);
+  if (jobs.length === 1 && jobs[0].url.startsWith(`${FALLBACK}/jobs/1/`)) pass('first-page 404 on the primary host falls back to the next host');
+  else fail(`fallback: jobs=${JSON.stringify(jobs)} calls=${ctx.calls.join(',')}`);
+}
+
+// A live primary never requests the fallback.
+{
+  const ctx = mkCtx({
+    [PRIMARY]: [page(card(PRIMARY, 2, 'Role B'))],
+    [FALLBACK]: new Error('fallback must not be requested'),
+  });
+  const jobs = await icims.fetch(entry, ctx);
+  if (jobs.length === 1 && onlyPrimary(ctx)) pass('a live primary host never requests the fallback');
+  else fail(`live primary: jobs=${jobs.length} calls=${ctx.calls.join(',')}`);
+}
+
+// Every host answers 404: the 404 surfaces, so dead-board tracking counts a miss.
+{
+  const ctx = mkCtx({});
+  try {
+    await icims.fetch(entry, ctx);
+    fail('fetch resolved with no live host');
+  } catch (err) {
+    if (err.status === 404 && ctx.calls.join(',') === `${PRIMARY}#0,${FALLBACK}#0`) pass('all hosts 404: each tried once, then the 404 is thrown');
+    else fail(`all 404: status=${err.status} calls=${ctx.calls.join(',')}`);
+  }
+}
+
+// A throttle is not "no board here": rethrown without trying the fallback.
+{
+  const ctx = mkCtx({ [PRIMARY]: httpError(429), [FALLBACK]: [page(card(FALLBACK, 3, 'Role C'))] });
+  try {
+    await icims.fetch(entry, ctx);
+    fail('fetch swallowed a 429');
+  } catch (err) {
+    if (err.status === 429 && onlyPrimary(ctx)) pass('non-404 failure is rethrown without falling back');
+    else fail(`429: status=${err.status} calls=${ctx.calls.join(',')}`);
+  }
+}
+
+// A 404 after the first page is a problem on a real board, not a missing board.
+{
+  const ctx = mkCtx({
+    [PRIMARY]: (pr) => { if (pr === 0) return page(card(PRIMARY, 4, 'Role D')); throw httpError(404); },
+    [FALLBACK]: [page(card(FALLBACK, 5, 'Role E'))],
+  });
+  try {
+    await icims.fetch(entry, ctx);
+    fail('fetch resolved despite a later-page 404');
+  } catch (err) {
+    if (err.status === 404 && onlyPrimary(ctx)) pass('a later-page 404 does not switch hosts');
+    else fail(`later-page 404: status=${err.status} calls=${ctx.calls.join(',')}`);
+  }
+}
+
+// Fallback URLs that are not https *.icims.com are ignored.
+{
+  const ctx = mkCtx({ 'https://evil.example': [page(card('https://evil.example', 6, 'Role F'))] });
+  const hostile = { ...entry, fallback_urls: ['https://evil.example/jobs/search', 'http://acmefreight.icims.com/jobs/search'] };
+  try {
+    await icims.fetch(hostile, ctx);
+    fail('fetch followed an off-host fallback');
+  } catch (err) {
+    if (err.status === 404 && onlyPrimary(ctx)) pass('fallback URLs off https *.icims.com are ignored');
+    else fail(`hostile fallback: status=${err.status} calls=${ctx.calls.join(',')}`);
+  }
+}
+
+// An entry with no fallback behaves exactly as before: the 404 is thrown.
+{
+  const ctx = mkCtx({});
+  try {
+    await icims.fetch({ name: 'acmefreight', careers_url: `${PRIMARY}/jobs/search?ss=1` }, ctx);
+    fail('fetch resolved for a missing board without fallbacks');
+  } catch (err) {
+    if (err.status === 404 && ctx.calls.length === 1) pass('entry without fallback_urls: 404 thrown after one request');
+    else fail(`no fallback: status=${err.status} calls=${ctx.calls.join(',')}`);
+  }
+}

tests/scan-ats-full-icims-hosts.test.mjs

@@ -0,0 +1,75 @@
+// tests/scan-ats-full-icims-hosts.test.mjs — how iCIMS dataset entries become
+// board hosts. The public dataset stores some tenants bare ("acmefreight") and
+// some as a full portal subdomain ("careers-acmefreight"); prefixing both with
+// "careers-" produced careers-careers-* hosts that do not exist.
+import { join } from 'path';
+import { pathToFileURL } from 'url';
+import { pass, fail, ROOT } from './helpers.mjs';
+
+console.log('\nscan-ats-full — iCIMS host candidates');
+
+const { SOURCES, icimsHostCandidates } = await import(pathToFileURL(join(ROOT, 'scan-ats-full.mjs')).href);
+const url = (host) => `https://${host}/jobs/search?ss=1&in_iframe=1`;
+const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
+
+// A bare tenant keeps the canonical careers- host first, with the bare host as fallback.
+{
+  const e = SOURCES.icims.toEntry('acmefreight');
+  if (e?.careers_url === url('careers-acmefreight.icims.com') && same(e.fallback_urls, [url('acmefreight.icims.com')])) {
+    pass('bare tenant: careers- host first, bare host as fallback');
+  } else {
+    fail(`bare tenant: ${JSON.stringify(e)}`);
+  }
+}
+
+// An entry that is already a careers- subdomain is used as-is and never doubled.
+{
+  const e = SOURCES.icims.toEntry('careers-acmefreight');
+  if (e?.careers_url === url('careers-acmefreight.icims.com') && e.fallback_urls === undefined) {
+    pass('careers- entry used as-is, no careers-careers- host');
+  } else {
+    fail(`careers- entry: ${JSON.stringify(e)}`);
+  }
+}
+
+// Other portal subdomains that contain "careers" are tried as-is first.
+{
+  const e = SOURCES.icims.toEntry('uscareers-acme');
+  if (e?.careers_url === url('uscareers-acme.icims.com') && same(e.fallback_urls, [url('careers-uscareers-acme.icims.com')])) {
+    pass('portal subdomain containing "careers" tried as-is first');
+  } else {
+    fail(`uscareers- entry: ${JSON.stringify(e)}`);
+  }
+}
+
+// A hyphenated entry without "careers" could be either shape; the prefixed host stays first.
+{
+  const e = SOURCES.icims.toEntry('jobs-acme');
+  if (e?.careers_url === url('careers-jobs-acme.icims.com') && same(e.fallback_urls, [url('jobs-acme.icims.com')])) {
+    pass('hyphenated entry without "careers": prefixed host first, as-is fallback');
+  } else {
+    fail(`jobs- entry: ${JSON.stringify(e)}`);
+  }
+}
+
+// Normalization: the dataset carries one entry with a stray leading dash.
+{
+  if (same(icimsHostCandidates('-careers-acme'), ['careers-acme.icims.com'])) pass('leading dash stripped before building the host');
+  else fail(`leading dash: ${JSON.stringify(icimsHostCandidates('-careers-acme'))}`);
+
+  if (same(icimsHostCandidates('AcmeFreight'), ['careers-acmefreight.icims.com', 'acmefreight.icims.com'])) pass('host lowercased');
+  else fail(`uppercase: ${JSON.stringify(icimsHostCandidates('AcmeFreight'))}`);
+
+  if (same(icimsHostCandidates(''), [])) pass('empty entry yields no host');
+  else fail(`empty: ${JSON.stringify(icimsHostCandidates(''))}`);
+}
+
+// Every candidate stays on icims.com, and hostile input is still rejected.
+{
+  const hosts = ['acme', 'careers-acme', 'jobs-acme', 'acmecareers-west', 'a.b'].flatMap(icimsHostCandidates);
+  if (hosts.length > 0 && hosts.every((h) => h.endsWith('.icims.com'))) pass('all candidate hosts stay on icims.com');
+  else fail(`off-host candidate: ${JSON.stringify(hosts)}`);
+
+  if (SOURCES.icims.toEntry('evil/..%2f') === null) pass('toEntry still rejects non-slug input');
+  else fail('toEntry accepted a hostile slug');
+}