Skip to content
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/remote-device/device.ts
Original file line number Diff line number Diff line change
Expand Up @@ -267,8 +267,8 @@ export class MCPDevice {
console.debug('[DEBUG] Tool call received, device_id:', device_id, 'this.deviceId:', this.deviceId);

// Only process jobs for this device
if (device_id && device_id !== this.deviceId) {
console.debug('[DEBUG] Ignoring tool call for different device');
if (!device_id || device_id !== this.deviceId) {
console.debug('[DEBUG] Ignoring tool call not explicitly targeted to this device');
return;
}

Expand Down
7 changes: 6 additions & 1 deletion src/remote-device/remote-channel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,12 @@ export class RemoteChannel {
filter: `user_id=eq.${this.user.id}`
},
(payload: any) => {
console.debug('[DEBUG] Realtime event received, payload:', payload?.new?.id);
const payloadDeviceId = payload?.new?.device_id;
console.debug('[DEBUG] Realtime event received, payload:', payload?.new?.id, 'device_id:', payloadDeviceId);
if (!payloadDeviceId || payloadDeviceId !== this.deviceId) {
console.debug('[DEBUG] Ignoring remote call not explicitly targeted to this device');
return;
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
if (this.onToolCall) {
this.onToolCall(payload);
}
Expand Down
220 changes: 207 additions & 13 deletions src/tools/filesystem.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ import fs from "fs/promises";
import path from "path";
import os from 'os';
import fetch from 'cross-fetch';
import dns from 'dns/promises';
import net from 'net';
import { Agent as HttpsAgent } from 'https';
import { execFile } from 'child_process';
import { promisify } from 'util';
import { capture } from '../utils/capture.js';
Expand All @@ -10,7 +13,7 @@ import { configManager } from '../config-manager.js';
import { getFileHandler, TextFileHandler } from '../utils/files/index.js';
import type { ReadOptions, FileResult, PdfPageItem } from '../utils/files/base.js';
import { isPdfFile } from "./mime-types.js";
import { parsePdfToMarkdown, editPdf, PdfOperations, PdfMetadata, parseMarkdownToPdf } from './pdf/index.js';
import { parsePdfBufferToMarkdown, editPdf, PdfOperations, PdfMetadata, parseMarkdownToPdf } from './pdf/index.js';
import { isBinaryFile } from 'isbinaryfile';

// CONSTANTS SECTION - Consolidate all timeouts and thresholds
Expand All @@ -32,7 +35,169 @@ const FILE_SIZE_LIMITS = {
LINE_COUNT_LIMIT: 10 * 1024 * 1024, // 10MB for line counting
} as const;

const URL_SECURITY_LIMITS = {
MAX_REDIRECTS: 5,
} as const;

// UTILITY FUNCTIONS - Eliminate duplication
type ValidatedRemoteReadTarget = {
url: URL;
allowedAddresses: string[];
};

function isBlockedHostname(hostname: string): boolean {
const normalized = hostname.toLowerCase();
return normalized === 'localhost' || normalized.endsWith('.localhost');
}

function isPrivateIpAddress(rawIp: string): boolean {
const ip = rawIp.toLowerCase().split('%')[0];
const ipVersion = net.isIP(ip);

// IPv4 private/local ranges
if (ipVersion === 4) {
const octets = ip.split('.').map((part) => Number.parseInt(part, 10));
if (octets.length !== 4 || octets.some((o) => Number.isNaN(o))) {
return true;
}
const [a, b] = octets;
return (
a === 0 || // "this network"
a === 10 ||
a === 127 ||
(a === 100 && b >= 64 && b <= 127) || // carrier-grade NAT
(a === 169 && b === 254) ||
(a === 172 && b >= 16 && b <= 31) ||
(a === 192 && b === 168)
);
}

// IPv6 local/loopback/IPv4-mapped ranges
if (ipVersion === 6) {
if (ip === '::1') {
return true;
}
Comment thread
AMeloDragon marked this conversation as resolved.
if (ip.startsWith('fc') || ip.startsWith('fd')) {
return true; // unique local address space (fc00::/7)
}
if (ip.startsWith('fe8') || ip.startsWith('fe9') || ip.startsWith('fea') || ip.startsWith('feb')) {
return true; // link-local (fe80::/10)
}
if (ip.startsWith('::ffff:')) {
return isPrivateIpAddress(ip.slice('::ffff:'.length));
}
}

return false;
}

async function validateRemoteReadUrl(urlString: string): Promise<ValidatedRemoteReadTarget> {
let parsedUrl: URL;
try {
parsedUrl = new URL(urlString);
} catch {
throw new Error(`Invalid URL: ${urlString}`);
}

if (parsedUrl.protocol !== 'https:') {
throw new Error(`Only HTTPS URLs are allowed: ${urlString}`);
}

const hostname = parsedUrl.hostname;
if (!hostname) {
throw new Error(`URL must include a hostname: ${urlString}`);
}

if (isBlockedHostname(hostname)) {
throw new Error(`Blocked URL hostname: ${hostname}`);
}

if (net.isIP(hostname)) {
if (isPrivateIpAddress(hostname)) {
throw new Error(`Blocked private or loopback IP address: ${hostname}`);
}
return { url: parsedUrl, allowedAddresses: [hostname] };
}

let resolvedAddresses: Array<{ address: string }>;
try {
resolvedAddresses = await dns.lookup(hostname, { all: true, verbatim: true });
} catch (error) {
throw new Error(`Failed to resolve URL hostname "${hostname}": ${error instanceof Error ? error.message : String(error)}`);
}

if (resolvedAddresses.length === 0) {
throw new Error(`URL hostname did not resolve to an address: ${hostname}`);
}

for (const resolvedAddress of resolvedAddresses) {
if (isPrivateIpAddress(resolvedAddress.address)) {
throw new Error(`Blocked URL hostname "${hostname}" because it resolves to a private or loopback address`);
}
}

const allowedAddresses = Array.from(new Set(resolvedAddresses.map((addr) => addr.address)));
return { url: parsedUrl, allowedAddresses };
}

function isRedirectStatus(statusCode: number): boolean {
return statusCode === 301 || statusCode === 302 || statusCode === 303 || statusCode === 307 || statusCode === 308;
}

function createPinnedLookup(allowedAddresses: string[]) {
return (
_hostname: string,
options: any,
callback: any
): void => {
const requestedFamily = typeof options === 'number' ? options : options?.family;
const returnAll = typeof options === 'object' && options?.all === true;

const candidates = allowedAddresses.filter((address) => {
if (!requestedFamily) {
return true;
}
return net.isIP(address) === requestedFamily;
});

if (candidates.length === 0) {
callback(new Error('No validated addresses available for requested address family') as NodeJS.ErrnoException, '');
return;
}

if (returnAll) {
callback(null, candidates.map((address) => ({ address, family: net.isIP(address) })));
return;
}

const selectedAddress = candidates[0];
callback(null, selectedAddress, net.isIP(selectedAddress));
};
}

function createPinnedHttpsAgent(validatedTarget: ValidatedRemoteReadTarget): HttpsAgent {
return new HttpsAgent({
keepAlive: false,
servername: validatedTarget.url.hostname,
lookup: createPinnedLookup(validatedTarget.allowedAddresses) as any
});
}

async function releaseResponseBody(response: Awaited<ReturnType<typeof fetch>>): Promise<void> {
const responseBody = response.body as unknown as { cancel?: () => Promise<void>; destroy?: () => void } | null;
if (!responseBody) {
return;
}

if (typeof responseBody.cancel === 'function') {
await responseBody.cancel();
return;
}

if (typeof responseBody.destroy === 'function') {
responseBody.destroy();
}
}

/**
* Get MIME type information for a file
Expand Down Expand Up @@ -364,26 +529,55 @@ export async function readFileFromUrl(url: string): Promise<FileResult> {
const timeoutId = setTimeout(() => controller.abort(), FILE_OPERATION_TIMEOUTS.URL_FETCH);

try {
const response = await fetch(url, {
signal: controller.signal
});
let currentTarget = await validateRemoteReadUrl(url);
let response: Awaited<ReturnType<typeof fetch>> | null = null;

for (let redirectCount = 0; redirectCount <= URL_SECURITY_LIMITS.MAX_REDIRECTS; redirectCount++) {
const pinnedAgent = createPinnedHttpsAgent(currentTarget);
const requestOptions: RequestInit & { agent: HttpsAgent } = {
signal: controller.signal,
redirect: 'manual',
agent: pinnedAgent
};
response = await fetch(currentTarget.url.toString(), requestOptions as RequestInit);

// Clear the timeout since fetch completed
clearTimeout(timeoutId);
if (!isRedirectStatus(response.status)) {
break;
}

if (redirectCount === URL_SECURITY_LIMITS.MAX_REDIRECTS) {
await releaseResponseBody(response);
throw new Error(`URL redirected too many times (max ${URL_SECURITY_LIMITS.MAX_REDIRECTS})`);
}

const redirectLocation = response.headers.get('location');
if (!redirectLocation) {
await releaseResponseBody(response);
throw new Error(`Redirect response missing location header for URL: ${currentTarget.url.toString()}`);
}

const redirectedUrl = new URL(redirectLocation, currentTarget.url).toString();
await releaseResponseBody(response);
currentTarget = await validateRemoteReadUrl(redirectedUrl);
}

if (!response) {
throw new Error(`Failed to fetch URL: ${url}`);
}

if (!response.ok) {
await releaseResponseBody(response);
throw new Error(`HTTP error! Status: ${response.status}`);
}

// Get MIME type from Content-Type header or infer from URL
const contentType = response.headers.get('content-type') || 'text/plain';
const isImage = isImageFile(contentType);
const isPdf = isPdfFile(contentType) || url.toLowerCase().endsWith('.pdf');
const isPdf = isPdfFile(contentType) || currentTarget.url.pathname.toLowerCase().endsWith('.pdf');

// NEW: Add PDF handling before image check
if (isPdf) {
// Use URL directly - pdfreader handles URL downloads internally
const pdfResult = await parsePdfToMarkdown(url);
const pdfBuffer = Buffer.from(await response.arrayBuffer());
const pdfResult = await parsePdfBufferToMarkdown(pdfBuffer);
Comment thread
AMeloDragon marked this conversation as resolved.
Outdated

return {
content: "",
Expand Down Expand Up @@ -411,15 +605,15 @@ export async function readFileFromUrl(url: string): Promise<FileResult> {
return { content, mimeType: contentType, metadata: { isImage } };
}
} catch (error) {
// Clear the timeout to prevent memory leaks
clearTimeout(timeoutId);

// Return error information instead of throwing
const errorMessage = error instanceof DOMException && error.name === 'AbortError'
? `URL fetch timed out after ${FILE_OPERATION_TIMEOUTS.URL_FETCH}ms: ${url}`
: `Failed to fetch URL: ${error instanceof Error ? error.message : String(error)}`;

throw new Error(errorMessage);
} finally {
// Clear the timeout to prevent memory leaks
clearTimeout(timeoutId);
}
}

Expand Down
3 changes: 1 addition & 2 deletions src/tools/pdf/index.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
export { editPdf } from './manipulations.js';
export type { PdfOperations, PdfInsertOperation, PdfDeleteOperation } from './manipulations.js';
export { parsePdfToMarkdown, parseMarkdownToPdf } from './markdown.js';
export { parsePdfToMarkdown, parsePdfBufferToMarkdown, parseMarkdownToPdf } from './markdown.js';
export type { PdfMetadata, PdfPageItem } from './lib/pdf2md.js';
export { extractImagesFromPdf } from './extract-images.js';
export type { ImageInfo, PageImages } from './extract-images.js';

12 changes: 9 additions & 3 deletions src/tools/pdf/markdown.ts
Original file line number Diff line number Diff line change
Expand Up @@ -274,16 +274,22 @@ async function loadPdfToBuffer(source: string): Promise<Buffer | ArrayBuffer> {
export async function parsePdfToMarkdown(source: string, pageNumbers: number[] | PageRange = []): Promise<PdfParseResult> {
try {
const data = await loadPdfToBuffer(source);

// @ts-ignore: Type definition mismatch for ESM usage
return await pdf2md(new Uint8Array(data), pageNumbers);
return await parsePdfBufferToMarkdown(data, pageNumbers);

} catch (error) {
console.error("Error converting PDF to Markdown (v3):", error);
throw error;
}
}

export async function parsePdfBufferToMarkdown(
data: Buffer | ArrayBuffer | Uint8Array,
pageNumbers: number[] | PageRange = []
): Promise<PdfParseResult> {
// @ts-ignore: Type definition mismatch for ESM usage
return await pdf2md(data instanceof Uint8Array ? data : new Uint8Array(data), pageNumbers);
}

export async function parseMarkdownToPdf(markdown: string, options: any = {}): Promise<Buffer> {
try {
// Find Chrome: puppeteer cache -> system Chrome -> install
Expand Down