diff --git a/README.md b/README.md index ed4cb9e..062ce80 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ The [skill](skills/convert-documents-to-markdown/SKILL.md) teaches the agent to ```bash npx @firecrawl/anydoc report.docx # Markdown to stdout npx @firecrawl/anydoc slides.pptx -o slides.md # or to a file +npx @firecrawl/anydoc --batch ./docs -o ./out # convert a directory tree npx @firecrawl/anydoc - --format csv < data.csv # read stdin ``` diff --git a/node/README.md b/node/README.md index 5263150..e850175 100644 --- a/node/README.md +++ b/node/README.md @@ -31,6 +31,7 @@ The package ships an `anydoc` command, so `npx` converts a document with no inst ```bash npx @firecrawl/anydoc report.docx # Markdown to stdout npx @firecrawl/anydoc slides.pptx -o slides.md # or to a file +npx @firecrawl/anydoc --batch ./docs -o ./out # convert a directory tree npx @firecrawl/anydoc - --format csv < data.csv # read stdin ``` diff --git a/node/cli.js b/node/cli.js index bfba762..f08dd0b 100644 --- a/node/cli.js +++ b/node/cli.js @@ -1,26 +1,42 @@ #!/usr/bin/env node 'use strict' -const { readFile, writeFile } = require('node:fs/promises') +const { copyFile, mkdir, readdir, readFile, stat, writeFile } = require('node:fs/promises') +const { dirname, extname, join, relative, resolve, sep } = require('node:path') const FORMATS = 'doc, docx, odt, pdf, ppt, pptx, rtf, epub, xlsx, ods, odp, csv' +// Plain-text extensions copied as-is in batch mode (names unchanged). +const PASSTHROUGH_EXTS = new Set(['.txt', '.json', '.md', '.py']) + const HELP = `anydoc: convert documents to GitHub-Flavored Markdown Usage: anydoc [options] + anydoc --batch -o [options] anydoc - [options] < file -Converts one document per invocation and writes the Markdown to stdout. -Pass - as the input to read the document from stdin. Never prompts; all -diagnostics go to stderr. +Converts one document per invocation and writes the Markdown to stdout, +or converts a directory tree in batch mode. Pass - as the input to read +the document from stdin. Never prompts; all diagnostics go to stderr. Options: - -o, --output Write the Markdown to instead of stdout + --batch Convert every supported file under + recursively into -o, preserving relative paths. + Requires -o. Converted files keep the original + name and gain a .md suffix (report.pdf becomes + report.pdf.md). Plain-text files (.txt, .json, + .md, .py) are copied with names unchanged. + Unsupported files are skipped silently. Existing + outputs are overwritten. A conversion failure on + one file is logged to stderr and the run continues; + the process exits 1 if any conversion failed. + -o, --output Write Markdown to (single-file), or the + output directory (required with --batch) -f, --format Name the input format instead of detecting it: ${FORMATS} (extension aliases like xls, docm, ppsx resolve - to these) + to these). Not valid with --batch. -h, --help Print this help and exit -V, --version Print the version and exit @@ -31,12 +47,14 @@ which anydoc does not do, and error as unsupported. Exit codes: 0 success - 1 the document could not be read or converted + 1 the document could not be read or converted (in batch mode: at least + one file failed to convert) 2 usage error: unknown option, missing input, or invalid --format Examples: anydoc report.docx anydoc slides.pptx -o slides.md + anydoc --batch ./docs -o ./out anydoc - --format csv < data.csv curl -s https://example.com/paper.pdf | anydoc - ` @@ -50,7 +68,7 @@ function fail(code, message) { } function parseArgs(argv) { - const args = { input: null, output: null, format: null } + const args = { input: null, output: null, format: null, batch: false } let positionalOnly = false for (let i = 0; i < argv.length; i++) { let arg = argv[i] @@ -87,6 +105,21 @@ function parseArgs(argv) { process.stdout.write(`${require('./package.json').version}\n`) process.exit(0) break + case '--batch': + // Allow --batch (value form) or --batch with a separate positional. + if (inline !== null) { + if (args.input !== null) { + fail(USAGE_ERROR, `one document per invocation: unexpected second input '${inline}'`) + } + args.input = inline + } else if (i + 1 < argv.length && !argv[i + 1].startsWith('-')) { + if (args.input !== null) { + fail(USAGE_ERROR, `one document per invocation: unexpected second input '${argv[i + 1]}'`) + } + args.input = argv[++i] + } + args.batch = true + break case '-o': case '--output': args.output = value() @@ -113,16 +146,93 @@ async function readStdin() { return Buffer.concat(chunks) } -async function main() { - const args = parseArgs(process.argv.slice(2)) - if (args.input === null) { - fail(USAGE_ERROR, 'missing input: pass a document path, or - for stdin (see anydoc --help)') +function isPassthrough(path) { + return PASSTHROUGH_EXTS.has(extname(path).toLowerCase()) +} + +/** Walk a directory tree depth-first; yield absolute file paths. */ +async function* walkFiles(root) { + const entries = await readdir(root, { withFileTypes: true }) + for (const entry of entries) { + const path = join(root, entry.name) + if (entry.isDirectory()) { + yield* walkFiles(path) + } else if (entry.isFile()) { + yield path + } } +} - // Loaded after argument handling so --help and --version work even where - // no native binding is available. - const { formatFromExtension, toMarkdown, toMarkdownBytes } = require('./index.js') +async function runBatch(args, { formatFromPath, toMarkdown }) { + if (args.input === '-') { + fail(USAGE_ERROR, '--batch does not read stdin; pass an input directory') + } + if (args.output === null) { + fail(USAGE_ERROR, '--batch requires -o/--output naming an output directory') + } + if (args.format !== null) { + fail(USAGE_ERROR, '--format is not valid with --batch (format is detected per file)') + } + + let inputStat + try { + inputStat = await stat(args.input) + } catch (error) { + fail(CONVERSION_ERROR, error.message) + } + if (!inputStat.isDirectory()) { + fail(USAGE_ERROR, `--batch expects a directory, got '${args.input}'`) + } + + const inputRoot = resolve(args.input) + const outputRoot = resolve(args.output) + + try { + await mkdir(outputRoot, { recursive: true }) + } catch (error) { + fail(CONVERSION_ERROR, error.message) + } + + let failed = 0 + for await (const absPath of walkFiles(inputRoot)) { + const rel = relative(inputRoot, absPath) + // Guard against path escape on odd relative() results. + if (rel.startsWith(`..${sep}`) || rel === '..') continue + if (isPassthrough(absPath)) { + const outPath = join(outputRoot, rel) + try { + await mkdir(dirname(outPath), { recursive: true }) + await copyFile(absPath, outPath) + } catch (error) { + process.stderr.write(`anydoc: ${rel}: ${error.message}\n`) + failed++ + } + continue + } + + if (formatFromPath(absPath) === null) { + // Unsupported (images, videos, archives, unknown extensions): skip. + continue + } + + const outPath = join(outputRoot, `${rel}.md`) + try { + const markdown = await toMarkdown(absPath) + await mkdir(dirname(outPath), { recursive: true }) + await writeFile(outPath, markdown) + } catch (error) { + process.stderr.write(`anydoc: ${rel}: ${error.message}\n`) + failed++ + } + } + + if (failed > 0) { + process.exit(CONVERSION_ERROR) + } +} + +async function runSingle(args, { formatFromExtension, toMarkdown, toMarkdownBytes }) { let format if (args.format !== null) { format = formatFromExtension(args.format) @@ -160,4 +270,26 @@ async function main() { } } +async function main() { + const args = parseArgs(process.argv.slice(2)) + if (args.input === null) { + fail( + USAGE_ERROR, + args.batch + ? 'missing input: pass a directory with --batch (see anydoc --help)' + : 'missing input: pass a document path, or - for stdin (see anydoc --help)', + ) + } + + // Loaded after argument handling so --help and --version work even where + // no native binding is available. + const bindings = require('./index.js') + + if (args.batch) { + await runBatch(args, bindings) + } else { + await runSingle(args, bindings) + } +} + main() diff --git a/node/test.mjs b/node/test.mjs index ec98a3c..95f6e34 100644 --- a/node/test.mjs +++ b/node/test.mjs @@ -1,7 +1,7 @@ // Smoke test: the bindings load and every entry point round-trips a fixture. import assert from 'node:assert/strict' import { execFile } from 'node:child_process' -import { mkdtemp, readFile, rm } from 'node:fs/promises' +import { copyFile, mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { fileURLToPath } from 'node:url' @@ -136,6 +136,64 @@ test('cli help and version go to stdout and exit 0', async () => { const help = await runCli(['--help']) assert.equal(help.code, undefined) assert.match(help.stdout, /Exit codes:/) + assert.match(help.stdout, /--batch/) const version = await runCli(['--version']) assert.match(version.stdout.trim(), /^\d+\.\d+\.\d+/) }) + +test('cli --batch converts a tree and copies passthrough files', async () => { + const dir = await mkdtemp(join(tmpdir(), 'anydoc-batch-')) + const input = join(dir, 'in') + const output = join(dir, 'out') + try { + await mkdir(join(input, 'nested'), { recursive: true }) + await copyFile(OUTLINE, join(input, 'nested', 'handbook.docx')) + await copyFile(CSV, join(input, 'sheet.csv')) + await writeFile(join(input, 'notes.txt'), 'plain text\n') + await writeFile(join(input, 'nested', 'skip.png'), Buffer.from([0x89, 0x50, 0x4e, 0x47])) + + const { code, stdout, stderr } = await runCli(['--batch', input, '-o', output]) + assert.equal(code, undefined) + assert.equal(stdout, '') + assert.equal(stderr, '') + + assert.match(await readFile(join(output, 'nested', 'handbook.docx.md'), 'utf8'), /^# /m) + assert.match(await readFile(join(output, 'sheet.csv.md'), 'utf8'), /\| --- \|/) + assert.equal(await readFile(join(output, 'notes.txt'), 'utf8'), 'plain text\n') + await assert.rejects(stat(join(output, 'nested', 'skip.png'))) + await assert.rejects(stat(join(output, 'nested', 'skip.png.md'))) + } finally { + await rm(dir, { recursive: true, force: true }) + } +}) + +test('cli --batch continues after a conversion failure and exits 1', async () => { + const dir = await mkdtemp(join(tmpdir(), 'anydoc-batch-fail-')) + const input = join(dir, 'in') + const output = join(dir, 'out') + try { + await mkdir(input, { recursive: true }) + await copyFile(OUTLINE, join(input, 'ok.docx')) + await copyFile(ENCRYPTED, join(input, 'bad.odt')) + + const { code, stderr } = await runCli(['--batch', input, '-o', output]) + assert.equal(code, 1) + assert.match(stderr, /bad\.odt/) + assert.match(await readFile(join(output, 'ok.docx.md'), 'utf8'), /^# /m) + } finally { + await rm(dir, { recursive: true, force: true }) + } +}) + +test('cli --batch usage errors exit 2', async () => { + for (const args of [ + ['--batch', CSV, '-o', 'out'], + ['--batch', '-o', 'out'], + ['--batch', OUTLINE], + ['--batch', OUTLINE, '-o', 'out', '--format', 'docx'], + ]) { + const { code, stderr } = await runCli(args) + assert.equal(code, 2, `expected usage error for ${JSON.stringify(args)}`) + assert.match(stderr, /^anydoc: /) + } +})