Repository navigation
Expand file tree
/
Copy pathtypes.ts
More file actions
157 lines (143 loc) · 3.86 KB
/
Copy pathtypes.ts
File metadata and controls
157 lines (143 loc) · 3.86 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
export interface RunConfig {
url: string;
name?: string;
outputDir: string;
lang: string;
excludePaths: string[];
forceMap: boolean;
maxPages?: number;
smart?: string;
filter?: string;
condense: boolean;
condenseModel: string;
condenseConcurrency: number;
fabricPattern?: string;
topN: number;
jinaApiKey?: string;
firecrawlApiKey?: string;
llmApiKey?: string;
llmBaseUrl?: string;
}
export interface DiscoveryResult {
urls: string[];
method: DiscoveryMethod;
fullContent?: string;
sitemapUrls?: string[];
/** True whenever any rung declined to explore something it saw — not only on cascade fall-through. */
partial?: boolean;
/** Links a rung retained but never followed for their own links (e.g. the link-crawl frontier cap). Their subtrees are undiscovered. */
unfollowedUrls?: string[];
metadata: Record<string, string>;
}
/** What discovery saw and declined, persisted in run.json so `resume` keeps the same denominator. */
export interface DiscoverySnapshot {
partial: boolean;
sitemapUrls?: string[];
unfollowedUrls?: string[];
}
export type DiscoveryMethod =
| "llms-full-txt"
| "llms-txt"
| "content-negotiation"
| "md-suffix"
| "rtd-sources"
| "sitemap"
| "link-crawl"
| "firecrawl-map";
export interface PageResult {
url: string;
status: "ok" | "error" | "skipped";
fetchMethod?: FetchMethod;
rawPath?: string;
cleanPath?: string;
error?: string;
wordsBefore?: number;
wordsAfter?: number;
platform?: string;
fetchDurationMs?: number;
flags?: string[];
}
export type FetchMethod =
| "llms-full-txt"
| "content-negotiation"
| "md-suffix"
| "rtd-sources"
| "local-html"
| "jina";
export interface PlatformDetection {
platform: string;
confidence: number;
evidence: string[];
}
export interface CleanResult {
content: string;
removed: string[];
qualityGateFailed: boolean;
}
export interface ValidationReport {
cleanliness: CleanlinessReport;
fidelity: FidelityReport;
coverage: CoverageReport;
}
export interface CleanlinessReport {
totalPages: number;
flaggedPages: number;
flaggedPercent: number;
pages: Array<{
url: string;
noiseHits: number;
fingerprints: string[];
}>;
}
export interface FidelityReport {
totalPages: number;
overStripped: number;
pages: Array<{
url: string;
wordsBefore: number;
wordsAfter: number;
retentionPercent: number;
missingHeadings: string[];
}>;
}
export interface CoverageReport {
discoveredUrls: number;
/** Widest denominator any discovery rung observed: discovered ∪ sitemap ∪ unfollowed. */
observedUrls: number;
fetchedPages: number;
/** fetchedPages / observedUrls — drops when discovery lost pages, unlike fetchOfDiscoveredPercent. */
fetchPercent: number;
/** fetchedPages / discoveredUrls — the fetch-stage number only ("did I fetch what I found?"). */
fetchOfDiscoveredPercent: number;
sitemapUrls?: number;
sitemapCoverage?: number;
/** Links seen but never followed for their own links; their subtrees are absent from the mirror. */
unfollowedUrls?: number;
gaps: Array<{ url: string; reason: string }>;
}
export interface RunManifest {
version: string;
url: string;
name: string;
startedAt: string;
completedAt?: string;
discoveryMethod: DiscoveryMethod;
discovery?: DiscoverySnapshot;
platform: PlatformDetection;
pages: PageResult[];
validation?: ValidationReport;
condenseStats?: { total: number; condensed: number; fallback: number; errors: number; avgReductionPct: number };
tokenEstimate?: number;
qualitySummary?: QualitySummary;
config: Omit<RunConfig, "jinaApiKey" | "firecrawlApiKey" | "llmApiKey" | "llmBaseUrl">;
}
export interface QualitySummary {
total: number;
clean: number;
cleanPct: number;
flagCounts: Record<string, number>;
}
export interface ExclusionSummary {
count: number;
reasons: Record<string, number>;
}