diff --git a/convex/githubImport.test.ts b/convex/githubImport.test.ts index 7fe977925..44a515da9 100644 --- a/convex/githubImport.test.ts +++ b/convex/githubImport.test.ts @@ -1,5 +1,6 @@ /* @vitest-environment node */ import { generateKeyPairSync } from "node:crypto"; +import { zipSync } from "fflate"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { internal } from "./_generated/api"; import { __test } from "./githubImport"; @@ -86,6 +87,15 @@ describe("githubImport", () => { ]); }); + it("rejects oversized files so they are not inflated past import limits", () => { + const zip = zipSync({ + "demo-repo/skill/SKILL.md": new TextEncoder().encode("# Demo\n"), + "demo-repo/skill/model.bin": new Uint8Array(10 * 1024 * 1024 + 1), + }); + + expect(() => __test.unzipToEntries(zip)).toThrow(/file that is too large/i); + }); + it("rejects a public repo owned by another GitHub account before repo lookup", async () => { const ctx = { runQuery: vi.fn().mockResolvedValue("123"), diff --git a/convex/githubImport.ts b/convex/githubImport.ts index dc534d1e6..7f409c62c 100644 --- a/convex/githubImport.ts +++ b/convex/githubImport.ts @@ -1,5 +1,5 @@ import { ConvexError, v } from "convex/values"; -import { unzipSync } from "fflate"; +import { unzipSync, type UnzipFileInfo } from "fflate"; import semver from "semver"; import { api, internal } from "./_generated/api"; import type { Id } from "./_generated/dataModel"; @@ -892,17 +892,17 @@ function normalizeRepoSearchQuery(query: string) { } function unzipToEntries(zipBytes: Uint8Array) { - const entries = unzipSync(zipBytes); + const limits = createZipEntryLimitFilter(); + const entries = unzipSync(zipBytes, { + filter: (file) => limits.accept(file), + }); const out: Record = {}; - const rawPaths = Object.keys(entries); - if (rawPaths.length > MAX_FILE_COUNT) throw new ConvexError("Repo archive has too many files"); let totalBytes = 0; for (const [rawPath, bytes] of Object.entries(entries)) { const normalizedPath = normalizeZipPath(rawPath); if (!normalizedPath) continue; if (isMacJunkPath(normalizedPath)) continue; if (!bytes) continue; - if (bytes.byteLength > MAX_SINGLE_FILE_BYTES) continue; totalBytes += bytes.byteLength; if (totalBytes > MAX_UNZIPPED_BYTES) throw new ConvexError("Repo archive is too large"); out[normalizedPath] = bytes; @@ -910,6 +910,29 @@ function unzipToEntries(zipBytes: Uint8Array) { return out; } +function createZipEntryLimitFilter() { + let fileCount = 0; + let totalBytes = 0; + return { + accept(file: UnzipFileInfo) { + fileCount += 1; + if (fileCount > MAX_FILE_COUNT) throw new ConvexError("Repo archive has too many files"); + if (file.name.endsWith("/")) return false; + + const normalizedPath = normalizeZipPath(file.name); + if (!normalizedPath) return false; + if (isMacJunkPath(normalizedPath)) return false; + + if (file.originalSize > MAX_SINGLE_FILE_BYTES) { + throw new ConvexError("Repo archive contains a file that is too large"); + } + totalBytes += file.originalSize; + if (totalBytes > MAX_UNZIPPED_BYTES) throw new ConvexError("Repo archive is too large"); + return true; + }, + }; +} + function isCandidateUnderResolvedPath(candidatePath: string, resolvedPath: string) { const root = normalizeRepoPath(resolvedPath); if (!root) return true;