From bd3f7517b3dc6d51937c24938ba6a28587e940b3 Mon Sep 17 00:00:00 2001 From: cuidong233 Date: Mon, 14 Sep 2026 10:53:58 +0800 Subject: [PATCH] fix(scan): honor blacklist domain scope --- DATA_CONTRACT.md | 3 +-- docs/SCRIPTS.md | 2 +- scan-ats-full.mjs | 5 ++-- scan.mjs | 43 +++++++++++++++++++++++++++++++--- templates/blacklist.example.md | 7 ++++++ test-all.mjs | 40 ++++++++++++++++++++++--------- 6 files changed, 80 insertions(+), 20 deletions(-) diff --git a/DATA_CONTRACT.md b/DATA_CONTRACT.md index 31c979ef3e..1a02c13c71 100644 --- a/DATA_CONTRACT.md +++ b/DATA_CONTRACT.md @@ -46,7 +46,7 @@ These files contain your personal data, customizations, and work product. Update | `data/salary-observations.tsv` | Your append-only compensation observation log: `{tracker#}\t{date}\t{desired\|advertised\|actual}\t{amount}\t{currency}\t{source}\t{note}`. Written by interactive modes when a figure is stated/confirmed; never edited in place. Advertised figures come from reports' `advertised_comp` instead — reports are themselves observation sources. Read by `salary-gap.mjs` | | `status-log.tsv` (sibling of the active tracker file — `data/status-log.tsv` in the default layout) | Your append-only status transition ledger: `{tracker#}\t{date}\t{from}\t{to}\t{source}\t{note}`. Appended by `set-status.mjs` next to wherever the tracker lives, on every real status change (the tracker stays the source of truth for *state*; the ledger records *when* transitions happened); never edited in place — corrections are new `correction`-source lines. An unknown from- or to-state is the sentinel `-`, never an empty cell; the two columns then diverge, with a from of `-` parsing to null (no prior state) and a to of `-` preserved as the literal unknown-target sentinel, while an empty cell is rejected as `unknown from-state ""` or `unknown to-state ""` for its own column. The source column is a closed set whose members are `VALID_SOURCES` in `funnel-velocity.mjs` — that declaration is the authority, so this contract points at it rather than restating a list that goes stale the next time a writer is added. Any value outside the set parses but is counted as an unknown source and excluded from the funnel, so per-writer detail belongs in the note column rather than namespaced onto the source. Read by `funnel-velocity.mjs` and `company-history.mjs` | | `data/upskill/*` | Your skill-gap analysis reports (written by the `upskill` mode) | -| `data/blacklist.md` | Your do-not-apply company list (opt-in — absence = no filtering; never auto-populated: only you, or the agent on your explicit instruction, write to it. Respected by `scan.mjs` and the `auto-pipeline`/`oferta`/`apply` gates; never a scoring input) | +| `data/blacklist.md` | Your do-not-apply list (opt-in — absence = no filtering; never auto-populated: only you, or the agent on your explicit instruction, write to it. Its `Scope` is `company` (default: normalized feed company name) or `domain` (the Company cell is a hostname suffix matched to the posting URL); no parent/subsidiary relationship is inferred. Respected by `scan.mjs` and the `auto-pipeline`/`oferta`/`apply` gates; never a scoring input) | | `data/assessments.tsv` | Your append-only skills-assessment log: `{date}\t{company}\t{report#\|-}\t{platform}\t{subject}\t{threshold%\|-}\t{score%\|-}\t{stale_note}`. Appended by `node assessment-log.mjs add`; never edited in place. Empty stale_note = no staleness observed. Read by `assessment-log.mjs` | | `data/contacts.tsv` | Your job-search phonebook (third-party PII — gitignored): `{name}\t{company}\t{type}\t{title}\t{phone}\t{email}\t{linkedin}\t{tracker#\|-}\t{notes}`. `type` optional; when present must be one of the enum (recruiter\|hiring-manager\|peer\|interviewer\|other), else flagged in `quality`. Written by the `contacto` mode only after you confirm; lines are updated in place when a contact's details change (unlike the append-only salary log). Read by `contacts.mjs` | | `data/Connections.csv` | Your LinkedIn connections export (third-party PII — gitignored, never committed, never touched by the updater). Placed here by you: LinkedIn → Settings → Data Privacy → Get a copy of your data → Connections. Read fresh on every run by `linkedin-join.mjs`, which writes no cache, index or sidecar state, so the file is disposable — delete it after use and re-export when you need it again. Never enters a prompt and never leaves the machine; only rows you paste by hand reach `data/contacts.tsv` | @@ -193,4 +193,3 @@ When resolved, all User Layer files/directories (e.g. `cv.md`, `config/profile.y - **Read Resolution:** If no tracker override is set, reading resolves to `{DATA_ROOT}/data/applications.md` if it exists; otherwise falls back to `{DATA_ROOT}/applications.md`. - **Write Resolution:** All writes (including merge operations and first-run creation) target the canonical location `{DATA_ROOT}/data/applications.md`. - diff --git a/docs/SCRIPTS.md b/docs/SCRIPTS.md index 3562bbfa57..a833473d80 100644 --- a/docs/SCRIPTS.md +++ b/docs/SCRIPTS.md @@ -632,7 +632,7 @@ If a parser writes full extraction artifacts for debugging or audit, store them When the ATS provider's list API returns a description, each new offer is fingerprinted for cross-listing detection. See [Cross-listing detection](#cross-listing-detection) under `scan:full` for details. -**Company blacklist (#1742):** if `data/blacklist.md` exists (user layer, opt-in — see `templates/blacklist.example.md`), postings from listed companies are skipped, matched case- and punctuation-insensitively with the same company normalization the tracker scripts share. Skips are never silent: the run summary reports `N skipped (blacklist)` and the count is persisted to `data/scan-runs.tsv` as `filtered_blacklist`. Pass `--include-blacklisted` to bypass the filter for auditing — matching postings flow through annotated (`note: blacklisted: {reason}` in `data/pipeline.md`). No blacklist file = no filtering; nothing ever adds a company to the list automatically. +**Company blacklist (#1742):** if `data/blacklist.md` exists (user layer, opt-in — see `templates/blacklist.example.md`), postings from listed companies are skipped. `Scope: company` is the default and matches the feed-provided company label case- and punctuation-insensitively with the same normalization the tracker scripts share. `Scope: domain` makes the Company cell a hostname suffix: `ibm.com` matches a posting at `jobs.ibm.com`, not `notibm.com`. It is explicit rather than inferred — the scanner never guesses parent/subsidiary ownership from a URL. Skips are never silent: the run summary reports `N skipped (blacklist)` and the count is persisted to `data/scan-runs.tsv` as `filtered_blacklist`. Pass `--include-blacklisted` to bypass the filter for auditing — matching postings flow through annotated (`note: blacklisted: {reason}` in `data/pipeline.md`). No blacklist file = no filtering; nothing ever adds a company to the list automatically. ```bash npm run scan diff --git a/scan-ats-full.mjs b/scan-ats-full.mjs index 754d0a3e4a..183fd2d694 100644 --- a/scan-ats-full.mjs +++ b/scan-ats-full.mjs @@ -54,11 +54,10 @@ import lever from './providers/lever.mjs'; import ashby from './providers/ashby.mjs'; import workday from './providers/workday.mjs'; import icims from './providers/icims.mjs'; -import { buildTitleFilter, buildTitleFilterOverrides, buildTitleFilterWithOverrides, buildLocationFilter, buildContentFilter, matchedTitleKeywords, loadSeenUrls, normalizeUrlForDedup, appendToPipeline, appendToScanHistory, loadBlacklist, parseSinceDays, PORTALS_PATH, PIPELINE_PATH } from './scan.mjs'; +import { buildTitleFilter, buildTitleFilterOverrides, buildTitleFilterWithOverrides, buildLocationFilter, buildContentFilter, matchedTitleKeywords, loadSeenUrls, normalizeUrlForDedup, appendToPipeline, appendToScanHistory, findBlacklistEntry, loadBlacklist, parseSinceDays, PORTALS_PATH, PIPELINE_PATH } from './scan.mjs'; import { localToday } from './lib/local-today.mjs'; import { printScanSummaryHeader } from './lib/scan-summary-marker.mjs'; import { SEED_SOURCES, toPortalEntry } from './seeds/vc-portfolios.mjs'; -import { normalizeCompany } from './tracker-utils.mjs'; import { validateFlags } from './lib/cli-flags.mjs'; import { isMainModule } from './lib/is-main-module.mjs'; import { boardKey, loadDeadBoards, recordBoardResult, saveDeadBoards, shouldSkipDeadBoard } from './dead-boards.mjs'; @@ -409,7 +408,7 @@ export function filterBlacklistedOffers(offers, blacklist, { includeBlacklisted let annotatedBlacklisted = 0; for (const offer of offers) { - const entry = blacklist.get(normalizeCompany(offer.company || '')); + const entry = findBlacklistEntry(blacklist, offer.company || '', offer.url); if (!entry) { kept.push(offer); continue; diff --git a/scan.mjs b/scan.mjs index af02e9bea0..b2e1d4f3d2 100644 --- a/scan.mjs +++ b/scan.mjs @@ -2428,7 +2428,7 @@ const BLACKLIST_PATH = path.join(DATA_ROOT, 'data/blacklist.md'); * blacklist row "Acme Corp." still catches an ATS feed that says "acme corp". * * @param {string} text - Raw data/blacklist.md content. - * @returns {Map} + * @returns {Map} * Normalized company key → entry. First row wins on duplicate keys. */ export function parseBlacklist(text) { @@ -2441,16 +2441,53 @@ export function parseBlacklist(text) { if (company.toLowerCase() === 'company') continue; // header row const key = normalizeCompany(company); if (!key || entries.has(key)) continue; + const scope = (cells[3] || 'company').toLowerCase(); entries.set(key, { company, since: cells[2] || '', - scope: cells[3] || '', + // A blank or unsupported scope keeps the long-standing company-name + // behavior. Only the documented `domain` value enables host matching. + scope: scope === 'domain' ? 'domain' : 'company', reason: cells[4] || '', }); } return entries; } +/** + * Find the blacklist entry that applies to one posting. + * + * `company` is the established default: compare the feed's company label with + * the normalized table value. `domain` is opt-in: the table's Company cell is + * a hostname suffix, so `ibm.com` matches `jobs.ibm.com` but not `notibm.com`. + * This deliberately does not infer parent/subsidiary ownership from a URL. + * + * @param {Map} blacklist + * @param {string} company - Feed-provided company label. + * @param {string} url - Posting URL. + * @returns {{company: string, since: string, scope?: string, reason: string}|null} + */ +export function findBlacklistEntry(blacklist, company, url) { + if (!blacklist || blacklist.size === 0) return null; + + const companyEntry = blacklist.get(normalizeCompany(company || '')); + if (companyEntry && companyEntry.scope !== 'domain') return companyEntry; + + let hostname; + try { + hostname = new URL(url).hostname.toLowerCase().replace(/\.$/, ''); + } catch { + return null; + } + + for (const entry of blacklist.values()) { + if (entry.scope !== 'domain') continue; + const suffix = String(entry.company || '').trim().toLowerCase().replace(/\.$/, ''); + if (suffix && (hostname === suffix || hostname.endsWith(`.${suffix}`))) return entry; + } + return null; +} + /** * Load data/blacklist.md if the user opted in. Absent file = empty Map = no * filtering anywhere — the scan stays byte-identical to a pre-#1742 run. @@ -3099,7 +3136,7 @@ async function main() { // silent: skips are counted and reported in the run summary, and // --include-blacklisted lets the posting through annotated instead. if (blacklist.size > 0) { - const blEntry = blacklist.get(normalizeCompany(job.company || company.name || '')); + const blEntry = findBlacklistEntry(blacklist, job.company || company.name || '', job.url); if (blEntry) { if (!includeBlacklisted) { totalFilteredBlacklist++; diff --git a/templates/blacklist.example.md b/templates/blacklist.example.md index 5e8950f7f7..f6e62b29d8 100644 --- a/templates/blacklist.example.md +++ b/templates/blacklist.example.md @@ -8,7 +8,14 @@ When `data/blacklist.md` exists: - The `auto-pipeline`, `oferta`, and `apply` modes stop on a match, quote your recorded reason, and ask for an explicit override before proceeding. Your call always wins. - A blacklist entry never changes any score anywhere — it is a gate, not a signal. +`Scope` is either `company` (the default: match the feed-provided company label, +case- and punctuation-insensitively) or `domain` (match the posting URL's hostname +against the Company cell as a suffix). For `domain`, enter a hostname suffix without +a scheme or path, such as `ibm.com`: it matches `jobs.ibm.com`, not `notibm.com`. +Domain scope is explicit; the scanner never infers parent/subsidiary relationships. + | Company | Since | Scope | Reason | |---------|-------|-------|--------| | Acme Corp | 2026-01-15 | company | example: post-interview process signals | | Globex | 2026-02-01 | company | example: repeated applications, zero conversion | +| ibm.com | 2026-03-01 | domain | example: avoid postings on IBM-owned ATS hosts | diff --git a/test-all.mjs b/test-all.mjs index ad0585d6e1..9988dfa51a 100755 --- a/test-all.mjs +++ b/test-all.mjs @@ -4705,7 +4705,7 @@ try { // is case- and punctuation-insensitive; loadBlacklist on an absent file is a // no-op (empty Map — the scan filter never fires). try { - const { parseBlacklist, loadBlacklist } = await import(pathToFileURL(join(ROOT, 'scan.mjs')).href); + const { parseBlacklist, loadBlacklist, findBlacklistEntry } = await import(pathToFileURL(join(ROOT, 'scan.mjs')).href); const bl = parseBlacklist([ '# Company Blacklist', '', @@ -4713,14 +4713,15 @@ try { '|---------|-------|-------|--------|', '| Acme Corp. | 2026-01-15 | company | post-interview process signals |', '| Globex | 2026-02-01 | company | zero conversion |', + '| ibm.com | 2026-03-01 | domain | avoid parent-company ATS hosts |', ].join('\n')); const exact = bl.get('acmecorp'); if ( - bl.size === 2 && + bl.size === 3 && exact && exact.reason === 'post-interview process signals' && exact.since === '2026-01-15' && - bl.has('globex') && !bl.has('company') + bl.has('globex') && bl.get('ibmcom')?.scope === 'domain' && !bl.has('company') ) { - pass('scan.mjs parseBlacklist parses the table and keys by normalized company (#1742)'); + pass('scan.mjs parseBlacklist parses normalized company and domain-scope rows (#1742, #4139)'); } else { fail(`scan.mjs parseBlacklist wrong: size=${bl.size} keys=${[...bl.keys()].join(',')}`); } @@ -4734,6 +4735,16 @@ try { fail('scan.mjs blacklist matching misses case/punctuation company variants'); } + const domain = bl.get('ibmcom'); + const domainMatch = findBlacklistEntry(bl, 'Confluent', 'https://jobs.ibm.com/engineering/123'); + const boundaryMiss = findBlacklistEntry(bl, 'Confluent', 'https://notibm.com/engineering/123'); + const legacyMatch = findBlacklistEntry(bl, 'ACME-CORP', 'https://example.com/jobs/123'); + if (domainMatch === domain && boundaryMiss === null && legacyMatch === exact) { + pass('scan.mjs blacklist domain scope matches host suffixes without weakening company matching (#4139)'); + } else { + fail('scan.mjs blacklist domain scope does not preserve host boundaries and legacy company matching (#4139)'); + } + const fixtureRoot = mkdtempSync(join(tmpdir(), 'career-ops-blacklist-')); try { const absent = loadBlacklist(join(fixtureRoot, 'data', 'blacklist.md')); @@ -4761,6 +4772,7 @@ try { // scan-runs.tsv by header name, and --include-blacklisted bypasses the filter. if ( scanScript.includes("args.includes('--include-blacklisted')") && + scanScript.includes('findBlacklistEntry(blacklist') && scanScript.includes('totalFilteredBlacklist') && scanScript.includes('skipped (blacklist)') && scanScript.includes('filtered_blacklist') @@ -5000,11 +5012,14 @@ try { try { const { filterBlacklistedOffers } = await import(pathToFileURL(join(ROOT, 'scan-ats-full.mjs')).href); const blacklist = new Map([ - ['acmecorp', { company: 'Acme Corp', reason: 'example reason' }], + ['acmecorp', { company: 'Acme Corp', scope: 'company', reason: 'example reason' }], + ['ibmcom', { company: 'ibm.com', scope: 'domain', reason: 'parent ATS host' }], ]); const offers = [ { company: 'Acme Corp.', title: 'Software Engineer', url: 'https://example.com/acme' }, { company: 'Globex', title: 'Software Engineer', url: 'https://example.com/globex' }, + { company: 'Confluent', title: 'Software Engineer', url: 'https://jobs.ibm.com/confluent' }, + { company: 'Not IBM', title: 'Software Engineer', url: 'https://notibm.com/role' }, ]; const skipped = typeof filterBlacklistedOffers === 'function' ? filterBlacklistedOffers(offers, blacklist, { includeBlacklisted: false }) @@ -5013,16 +5028,19 @@ try { ? filterBlacklistedOffers(offers, blacklist, { includeBlacklisted: true }) : null; const ok = - skipped?.filteredBlacklist === 1 && - skipped.offers.length === 1 && + skipped?.filteredBlacklist === 2 && + skipped.offers.length === 2 && skipped.offers[0].company === 'Globex' && - audited?.annotatedBlacklisted === 1 && - audited.offers.length === 2 && + skipped.offers[1].company === 'Not IBM' && + audited?.annotatedBlacklisted === 2 && + audited.offers.length === 4 && audited.offers[0].blacklisted === true && audited.offers[0].note.includes('blacklisted: example reason') && + audited.offers[2].blacklisted === true && + audited.offers[2].note.includes('blacklisted: parent ATS host') && offers[0].blacklisted === undefined; - if (ok) pass('scan-ats-full filters data/blacklist.md matches by default and annotates them under --include-blacklisted (#1911)'); - else fail('scan-ats-full missing blacklist filter/audit semantics (#1911)'); + if (ok) pass('scan-ats-full applies company and domain blacklist scopes in default and audit modes (#1911, #4139)'); + else fail('scan-ats-full missing company/domain blacklist filter/audit semantics (#1911, #4139)'); } catch (e) { fail(`scan-ats-full blacklist test crashed: ${e.message}`); }