From 119445bcfb3ff6b5aa85dba1e8f62d287c17c80a Mon Sep 17 00:00:00 2001 From: mantisfury Date: Wed, 14 Jan 2026 18:00:07 -0700 Subject: [PATCH 1/4] Feature: Add Regex Search, Document Deduplication, and Graph Pathfinding Group 1 Feature Parity Implementation: 1. Regex Search (Search Shard) - New RegexSearchEngine with PostgreSQL native regex (~, ~*) for scalability - Pattern presets for common patterns (email, phone, SSN, credit card, etc.) - Pattern validation with performance estimation (fast/moderate/slow/dangerous) - UI panel with detected patterns integration from Patterns shard - Fixed sentence chunking to preserve emails, URLs, and abbreviations 2. Document Deduplication (Documents Shard) - New DeduplicationService with exact hash and fuzzy similarity matching - SimHash-based near-duplicate detection with configurable thresholds - API endpoints for finding duplicates, grouping, and merging - UI panel in DocumentViewer for duplicate management 3. Graph Pathfinding (Graph Shard) - New PathfindingEngine with BFS, DFS, Dijkstra, and A* algorithms - Shortest path and all paths discovery - Path analysis with relationship type filtering - API endpoints for path queries UI Improvements: - RegexSearchPanel with dark theme CSS styling - DeduplicationPanel with match highlighting - Detected patterns section showing patterns from Patterns shard Bug Fix: - Fixed sentence chunking regex that was splitting on periods in email addresses (e.g., "agent.smith@cia.gov" was becoming "agent smith@cia gov") Co-Authored-By: Claude Opus 4.5 --- migrations/001_consolidation.sql | 4 +- .../arkham_shard_documents/api.py | 258 +++++++ .../services/__init__.py | 5 + .../services/deduplication.py | 701 ++++++++++++++++++ .../arkham_shard_documents/shard.py | 73 +- .../arkham_shard_graph/algorithms.py | 311 ++++++++ .../arkham_shard_graph/api.py | 235 ++++++ .../arkham_shard_parse/chunker.py | 20 +- .../arkham_shard_search/api.py | 614 ++++++++++++++- .../arkham_shard_search/engines/__init__.py | 3 +- .../arkham_shard_search/engines/regex.py | 458 ++++++++++++ .../arkham_shard_search/models.py | 62 ++ .../arkham_shard_search/shard.py | 238 +++++- .../pages/documents/DeduplicationPanel.css | 261 +++++++ .../pages/documents/DeduplicationPanel.tsx | 289 ++++++++ .../src/pages/documents/DocumentViewer.tsx | 28 +- .../src/pages/documents/api.ts | 153 ++++ .../src/pages/search/RegexSearchPanel.css | 447 +++++++++++ .../src/pages/search/RegexSearchPanel.tsx | 398 ++++++++++ .../src/pages/search/SearchPage.tsx | 26 +- .../src/pages/search/api.ts | 259 +++++++ .../src/pages/search/types.ts | 95 ++- 22 files changed, 4919 insertions(+), 19 deletions(-) create mode 100644 packages/arkham-shard-documents/arkham_shard_documents/services/__init__.py create mode 100644 packages/arkham-shard-documents/arkham_shard_documents/services/deduplication.py create mode 100644 packages/arkham-shard-search/arkham_shard_search/engines/regex.py create mode 100644 packages/arkham-shard-shell/src/pages/documents/DeduplicationPanel.css create mode 100644 packages/arkham-shard-shell/src/pages/documents/DeduplicationPanel.tsx create mode 100644 packages/arkham-shard-shell/src/pages/search/RegexSearchPanel.css create mode 100644 packages/arkham-shard-shell/src/pages/search/RegexSearchPanel.tsx diff --git a/migrations/001_consolidation.sql b/migrations/001_consolidation.sql index 71e92ea..a40ae2a 100644 --- a/migrations/001_consolidation.sql +++ b/migrations/001_consolidation.sql @@ -40,10 +40,12 @@ CREATE TABLE IF NOT EXISTS arkham_vectors.collections ( ); -- Main embeddings table +-- Using untyped vector allows variable dimensions per collection/model +-- Each collection tracks its own vector_size in arkham_vectors.collections CREATE TABLE IF NOT EXISTS arkham_vectors.embeddings ( id VARCHAR(36) PRIMARY KEY, collection VARCHAR(100) NOT NULL, - embedding vector(2000), -- Max pgvector dimension with index + embedding vector, -- Untyped: supports 384 (MiniLM), 768 (mpnet), 1024 (bge-m3), etc. payload JSONB DEFAULT '{}', created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP diff --git a/packages/arkham-shard-documents/arkham_shard_documents/api.py b/packages/arkham-shard-documents/arkham_shard_documents/api.py index be06235..3d9d832 100644 --- a/packages/arkham-shard-documents/arkham_shard_documents/api.py +++ b/packages/arkham-shard-documents/arkham_shard_documents/api.py @@ -662,3 +662,261 @@ async def ai_junior_analyst(request: Request, body: AIJuniorAnalystRequest): "X-Accel-Buffering": "no", }, ) + + +# --- Deduplication Endpoints --- + + +class DuplicateMatchResponse(BaseModel): + """A potential duplicate match.""" + document_id: str + title: str + similarity_score: float + hamming_distance: int + match_type: str + + +class DuplicateGroupResponse(BaseModel): + """A group of duplicate documents.""" + group_id: str + primary_document_id: str + duplicate_ids: List[str] + similarity_threshold: float + detection_method: str + + +class MergeRequest(BaseModel): + """Request to merge duplicate documents.""" + primary_id: str + duplicate_ids: List[str] + strategy: str = "keep_primary" + preserve_references: bool = True + cleanup_action: str = "soft_delete" + + +class MergeResponse(BaseModel): + """Response from merge operation.""" + primary_id: str + merged_count: int + references_updated: int + documents_cleaned: int + cleanup_action: str + + +class DeduplicationStats(BaseModel): + """Deduplication statistics.""" + total_documents: int + documents_with_hash: int + unique_content_hashes: int + potential_duplicates: int + + +@router.post("/{document_id}/compute-hash") +async def compute_document_hash(document_id: str, request: Request): + """ + Compute and store content hashes for a document. + + Computes MD5, SHA256, and SimHash for deduplication. + """ + shard = get_shard(request) + + if not shard.deduplication: + raise HTTPException(status_code=503, detail="Deduplication service not available") + + # Get document content + content = await shard.get_document_content(document_id) + if not content: + raise HTTPException(status_code=404, detail=f"Document content not found: {document_id}") + + # Compute hashes + result = await shard.deduplication.compute_hash( + document_id=document_id, + text=content["content"], + store=True, + ) + + return { + "document_id": document_id, + "content_md5": result["content_md5"], + "content_sha256": result["content_sha256"], + "simhash": result["simhash"], + "text_length": result["text_length"], + } + + +@router.get("/{document_id}/duplicates/exact", response_model=List[DuplicateMatchResponse]) +async def find_exact_duplicates( + document_id: str, + request: Request, + project_id: Optional[str] = Query(None, description="Limit to project"), +): + """ + Find exact duplicates of a document. + + Uses content SHA256 hash for exact matching. + """ + shard = get_shard(request) + + if not shard.deduplication: + raise HTTPException(status_code=503, detail="Deduplication service not available") + + matches = await shard.deduplication.find_exact_duplicates( + document_id=document_id, + project_id=project_id, + ) + + return [ + DuplicateMatchResponse( + document_id=m.document_id, + title=m.title, + similarity_score=m.similarity_score, + hamming_distance=m.hamming_distance, + match_type=m.match_type, + ) + for m in matches + ] + + +@router.get("/{document_id}/duplicates/similar", response_model=List[DuplicateMatchResponse]) +async def find_similar_documents( + document_id: str, + request: Request, + threshold: float = Query(0.85, ge=0.0, le=1.0, description="Similarity threshold"), + project_id: Optional[str] = Query(None, description="Limit to project"), + limit: int = Query(20, ge=1, le=100, description="Maximum results"), +): + """ + Find similar documents using SimHash. + + Returns documents with similarity above threshold. + """ + shard = get_shard(request) + + if not shard.deduplication: + raise HTTPException(status_code=503, detail="Deduplication service not available") + + matches = await shard.deduplication.find_similar_documents( + document_id=document_id, + threshold=threshold, + project_id=project_id, + limit=limit, + ) + + return [ + DuplicateMatchResponse( + document_id=m.document_id, + title=m.title, + similarity_score=m.similarity_score, + hamming_distance=m.hamming_distance, + match_type=m.match_type, + ) + for m in matches + ] + + +@router.post("/deduplication/scan", response_model=List[DuplicateGroupResponse]) +async def scan_project_duplicates( + request: Request, + project_id: str = Query(..., description="Project to scan"), + threshold: float = Query(0.85, ge=0.0, le=1.0, description="Similarity threshold"), +): + """ + Scan entire project for duplicate groups. + + Returns groups of documents that appear to be duplicates. + """ + shard = get_shard(request) + + if not shard.deduplication: + raise HTTPException(status_code=503, detail="Deduplication service not available") + + groups = await shard.deduplication.scan_project_duplicates( + project_id=project_id, + threshold=threshold, + ) + + return [ + DuplicateGroupResponse( + group_id=g.group_id, + primary_document_id=g.primary_document_id, + duplicate_ids=g.duplicate_ids, + similarity_threshold=g.similarity_threshold, + detection_method=g.detection_method, + ) + for g in groups + ] + + +@router.post("/deduplication/merge", response_model=MergeResponse) +async def merge_duplicates(body: MergeRequest, request: Request): + """ + Merge duplicate documents into a primary document. + + Cleanup actions: + - soft_delete: Mark duplicates as merged (recommended) + - archive: Move to archive status + - hard_delete: Permanently delete (irreversible) + - keep: Only update references, keep duplicates + """ + shard = get_shard(request) + + if not shard.deduplication: + raise HTTPException(status_code=503, detail="Deduplication service not available") + + # Validate primary document exists + primary_doc = await shard.get_document(body.primary_id) + if not primary_doc: + raise HTTPException(status_code=404, detail=f"Primary document not found: {body.primary_id}") + + result = await shard.deduplication.merge_documents( + primary_id=body.primary_id, + duplicate_ids=body.duplicate_ids, + strategy=body.strategy, + preserve_references=body.preserve_references, + cleanup_action=body.cleanup_action, + ) + + # Emit event + if shard._events: + await shard._events.emit( + "documents.duplicates.merged", + { + "primary_id": result.primary_id, + "merged_count": result.merged_count, + "cleanup_action": result.cleanup_action, + }, + source="documents-shard", + ) + + return MergeResponse( + primary_id=result.primary_id, + merged_count=result.merged_count, + references_updated=result.references_updated, + documents_cleaned=result.documents_cleaned, + cleanup_action=result.cleanup_action, + ) + + +@router.get("/deduplication/stats", response_model=DeduplicationStats) +async def get_deduplication_stats( + request: Request, + project_id: Optional[str] = Query(None, description="Filter by project"), +): + """ + Get deduplication statistics. + + Returns counts of documents, unique hashes, and potential duplicates. + """ + shard = get_shard(request) + + if not shard.deduplication: + raise HTTPException(status_code=503, detail="Deduplication service not available") + + stats = await shard.deduplication.get_deduplication_stats(project_id=project_id) + + return DeduplicationStats( + total_documents=stats.get("total_documents", 0), + documents_with_hash=stats.get("documents_with_hash", 0), + unique_content_hashes=stats.get("unique_content_hashes", 0), + potential_duplicates=stats.get("potential_duplicates", 0), + ) diff --git a/packages/arkham-shard-documents/arkham_shard_documents/services/__init__.py b/packages/arkham-shard-documents/arkham_shard_documents/services/__init__.py new file mode 100644 index 0000000..c08c974 --- /dev/null +++ b/packages/arkham-shard-documents/arkham_shard_documents/services/__init__.py @@ -0,0 +1,5 @@ +"""Documents shard services.""" + +from .deduplication import DeduplicationService + +__all__ = ["DeduplicationService"] diff --git a/packages/arkham-shard-documents/arkham_shard_documents/services/deduplication.py b/packages/arkham-shard-documents/arkham_shard_documents/services/deduplication.py new file mode 100644 index 0000000..3717043 --- /dev/null +++ b/packages/arkham-shard-documents/arkham_shard_documents/services/deduplication.py @@ -0,0 +1,701 @@ +"""Document deduplication service using SimHash for similarity detection.""" + +import hashlib +import json +import logging +import re +import uuid +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any + +logger = logging.getLogger(__name__) + + +@dataclass +class DuplicateMatch: + """A potential duplicate match.""" + document_id: str + title: str + similarity_score: float + hamming_distance: int + match_type: str # exact, near, content_similar + shared_entities: int = 0 + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class DuplicateGroup: + """A group of duplicate documents.""" + group_id: str + primary_document_id: str + duplicate_ids: list[str] + similarity_threshold: float + detection_method: str + created_at: datetime = field(default_factory=datetime.utcnow) + + +@dataclass +class MergeResult: + """Result of merging duplicate documents.""" + primary_id: str + merged_count: int + references_updated: int + documents_cleaned: int + cleanup_action: str # soft_delete, archive, hard_delete, keep + + +class SimHash: + """ + SimHash implementation for text similarity detection. + + SimHash produces a 64-bit hash where similar documents have similar hashes. + Hamming distance between hashes indicates similarity. + """ + + def __init__(self, hash_bits: int = 64): + """ + Initialize SimHash. + + Args: + hash_bits: Number of bits in hash (default: 64) + """ + self.hash_bits = hash_bits + + def _tokenize(self, text: str) -> list[str]: + """ + Tokenize text into shingles (n-grams). + + Args: + text: Input text + + Returns: + List of token strings + """ + # Normalize text + text = text.lower() + text = re.sub(r'[^\w\s]', '', text) + + # Generate word shingles (3-grams) + words = text.split() + shingles = [] + + # Unigrams + shingles.extend(words) + + # Bigrams + for i in range(len(words) - 1): + shingles.append(f"{words[i]} {words[i+1]}") + + # Trigrams + for i in range(len(words) - 2): + shingles.append(f"{words[i]} {words[i+1]} {words[i+2]}") + + return shingles + + def _hash_token(self, token: str) -> int: + """ + Hash a single token to a 64-bit integer. + + Args: + token: Token string + + Returns: + 64-bit hash integer + """ + # Use MD5 and take first 8 bytes (64 bits) + h = hashlib.md5(token.encode('utf-8')).digest() + return int.from_bytes(h[:8], byteorder='big') + + def compute(self, text: str) -> int: + """ + Compute SimHash for a text. + + Args: + text: Input text + + Returns: + 64-bit SimHash integer + """ + if not text or not text.strip(): + return 0 + + tokens = self._tokenize(text) + if not tokens: + return 0 + + # Initialize bit vector + v = [0] * self.hash_bits + + # Accumulate weighted hash bits + for token in tokens: + token_hash = self._hash_token(token) + + # Add/subtract based on each bit + for i in range(self.hash_bits): + bit = (token_hash >> (self.hash_bits - 1 - i)) & 1 + if bit == 1: + v[i] += 1 + else: + v[i] -= 1 + + # Generate final hash from accumulated values + simhash = 0 + for i in range(self.hash_bits): + if v[i] > 0: + simhash |= (1 << (self.hash_bits - 1 - i)) + + return simhash + + @staticmethod + def hamming_distance(hash1: int, hash2: int) -> int: + """ + Calculate Hamming distance between two hashes. + + Args: + hash1: First hash + hash2: Second hash + + Returns: + Number of differing bits (0-64) + """ + xor = hash1 ^ hash2 + distance = 0 + while xor: + distance += xor & 1 + xor >>= 1 + return distance + + @staticmethod + def similarity_score(hash1: int, hash2: int, hash_bits: int = 64) -> float: + """ + Calculate similarity score from Hamming distance. + + Args: + hash1: First hash + hash2: Second hash + hash_bits: Number of bits in hash + + Returns: + Similarity score (0.0-1.0) + """ + distance = SimHash.hamming_distance(hash1, hash2) + return 1.0 - (distance / hash_bits) + + +class DeduplicationService: + """ + Document deduplication service. + + Provides: + - SimHash-based similarity detection + - Exact match detection (MD5/SHA256) + - Duplicate grouping + - Merge operations with cleanup options + """ + + def __init__(self, database_service, config: dict | None = None): + """ + Initialize deduplication service. + + Args: + database_service: Frame database service + config: Optional configuration dict + """ + self._db = database_service + self._config = config or {} + self._simhash = SimHash(hash_bits=64) + + # Configuration + self._similarity_threshold = self._config.get("similarity_threshold", 0.85) + self._hamming_threshold = self._config.get("hamming_threshold", 10) + + async def compute_hash( + self, + document_id: str, + text: str, + store: bool = True, + ) -> dict[str, Any]: + """ + Compute and store content hashes for a document. + + Args: + document_id: Document ID + text: Document text content + store: Whether to store hashes in database + + Returns: + Dict with hash values + """ + # Compute hashes + content_md5 = hashlib.md5(text.encode('utf-8')).hexdigest() + content_sha256 = hashlib.sha256(text.encode('utf-8')).hexdigest() + simhash = self._simhash.compute(text) + + result = { + "document_id": document_id, + "content_md5": content_md5, + "content_sha256": content_sha256, + "simhash": simhash, + "text_length": len(text), + } + + if store and self._db: + try: + # Upsert into content_hashes table + await self._db.execute( + """INSERT INTO arkham_documents.content_hashes + (id, document_id, content_md5, content_sha256, simhash, text_length, created_at) + VALUES (:id, :doc_id, :md5, :sha256, :simhash, :text_len, CURRENT_TIMESTAMP) + ON CONFLICT (document_id) DO UPDATE SET + content_md5 = EXCLUDED.content_md5, + content_sha256 = EXCLUDED.content_sha256, + simhash = EXCLUDED.simhash, + text_length = EXCLUDED.text_length, + created_at = CURRENT_TIMESTAMP""", + { + "id": str(uuid.uuid4()), + "doc_id": document_id, + "md5": content_md5, + "sha256": content_sha256, + "simhash": simhash, + "text_len": len(text), + } + ) + except Exception as e: + logger.warning(f"Failed to store content hashes: {e}") + + return result + + async def find_exact_duplicates( + self, + document_id: str, + project_id: str | None = None, + ) -> list[DuplicateMatch]: + """ + Find exact duplicates using content hash. + + Args: + document_id: Source document ID + project_id: Optional project scope + + Returns: + List of exact duplicate matches + """ + if not self._db: + return [] + + # Get source document's hash + source_row = await self._db.fetch_one( + "SELECT * FROM arkham_documents.content_hashes WHERE document_id = :doc_id", + {"doc_id": document_id} + ) + + if not source_row: + return [] + + source_hash = source_row["content_sha256"] + + # Find documents with matching hash + query = """ + SELECT ch.document_id, ch.content_sha256, d.filename as title + FROM arkham_documents.content_hashes ch + JOIN arkham_frame.documents d ON ch.document_id = d.id + WHERE ch.content_sha256 = :hash + AND ch.document_id != :source_id + """ + params: dict[str, Any] = {"hash": source_hash, "source_id": document_id} + + if project_id: + query += " AND d.project_id = :project_id" + params["project_id"] = project_id + + rows = await self._db.fetch_all(query, params) + + return [ + DuplicateMatch( + document_id=row["document_id"], + title=row["title"] or "", + similarity_score=1.0, + hamming_distance=0, + match_type="exact", + ) + for row in rows + ] + + async def find_similar_documents( + self, + document_id: str, + threshold: float | None = None, + project_id: str | None = None, + limit: int = 20, + ) -> list[DuplicateMatch]: + """ + Find similar documents using SimHash. + + Args: + document_id: Source document ID + threshold: Similarity threshold (0.0-1.0) + project_id: Optional project scope + limit: Maximum results + + Returns: + List of similar document matches + """ + if not self._db: + return [] + + threshold = threshold or self._similarity_threshold + + # Get source document's simhash + source_row = await self._db.fetch_one( + "SELECT * FROM arkham_documents.content_hashes WHERE document_id = :doc_id", + {"doc_id": document_id} + ) + + if not source_row or source_row["simhash"] is None: + return [] + + source_simhash = source_row["simhash"] + + # Get all other documents' simhashes (in batches for large corpora) + query = """ + SELECT ch.document_id, ch.simhash, d.filename as title + FROM arkham_documents.content_hashes ch + JOIN arkham_frame.documents d ON ch.document_id = d.id + WHERE ch.document_id != :source_id + AND ch.simhash IS NOT NULL + """ + params: dict[str, Any] = {"source_id": document_id} + + if project_id: + query += " AND d.project_id = :project_id" + params["project_id"] = project_id + + rows = await self._db.fetch_all(query, params) + + # Calculate similarity for each + similar = [] + for row in rows: + other_simhash = row["simhash"] + if other_simhash is None: + continue + + similarity = SimHash.similarity_score(source_simhash, other_simhash) + hamming = SimHash.hamming_distance(source_simhash, other_simhash) + + if similarity >= threshold: + match_type = "exact" if hamming == 0 else "near" if hamming <= 5 else "content_similar" + + similar.append(DuplicateMatch( + document_id=row["document_id"], + title=row["title"] or "", + similarity_score=round(similarity, 4), + hamming_distance=hamming, + match_type=match_type, + )) + + # Sort by similarity descending + similar.sort(key=lambda x: x.similarity_score, reverse=True) + + return similar[:limit] + + async def scan_project_duplicates( + self, + project_id: str, + threshold: float | None = None, + ) -> list[DuplicateGroup]: + """ + Scan entire project for duplicate groups. + + Uses efficient pairwise comparison with early termination. + + Args: + project_id: Project to scan + threshold: Similarity threshold + + Returns: + List of duplicate groups found + """ + if not self._db: + return [] + + threshold = threshold or self._similarity_threshold + + # Get all documents with simhashes in project + rows = await self._db.fetch_all( + """SELECT ch.document_id, ch.simhash, d.filename as title + FROM arkham_documents.content_hashes ch + JOIN arkham_frame.documents d ON ch.document_id = d.id + WHERE d.project_id = :project_id + AND ch.simhash IS NOT NULL + ORDER BY d.created_at""", + {"project_id": project_id} + ) + + if len(rows) < 2: + return [] + + # Build duplicate groups using union-find + groups: dict[str, set[str]] = {} # primary_id -> set of duplicate ids + assigned: dict[str, str] = {} # doc_id -> group primary_id + + for i, row1 in enumerate(rows): + doc1_id = row1["document_id"] + simhash1 = row1["simhash"] + + for j, row2 in enumerate(rows): + if j <= i: + continue + + doc2_id = row2["document_id"] + simhash2 = row2["simhash"] + + similarity = SimHash.similarity_score(simhash1, simhash2) + + if similarity >= threshold: + # Determine which group these belong to + group1 = assigned.get(doc1_id) + group2 = assigned.get(doc2_id) + + if group1 is None and group2 is None: + # Create new group with doc1 as primary + groups[doc1_id] = {doc1_id, doc2_id} + assigned[doc1_id] = doc1_id + assigned[doc2_id] = doc1_id + elif group1 is not None and group2 is None: + # Add doc2 to group1 + groups[group1].add(doc2_id) + assigned[doc2_id] = group1 + elif group1 is None and group2 is not None: + # Add doc1 to group2 + groups[group2].add(doc1_id) + assigned[doc1_id] = group2 + elif group1 != group2: + # Merge groups (keep smaller primary_id as primary) + primary = min(group1, group2) + secondary = max(group1, group2) + groups[primary].update(groups[secondary]) + for doc_id in groups[secondary]: + assigned[doc_id] = primary + del groups[secondary] + + # Convert to DuplicateGroup objects + result = [] + for primary_id, members in groups.items(): + if len(members) > 1: + duplicate_ids = [m for m in members if m != primary_id] + result.append(DuplicateGroup( + group_id=str(uuid.uuid4())[:8], + primary_document_id=primary_id, + duplicate_ids=duplicate_ids, + similarity_threshold=threshold, + detection_method="simhash", + )) + + return result + + async def merge_documents( + self, + primary_id: str, + duplicate_ids: list[str], + strategy: str = "keep_primary", + preserve_references: bool = True, + cleanup_action: str = "soft_delete", + ) -> MergeResult: + """ + Merge duplicate documents into a primary document. + + Args: + primary_id: Document to keep + duplicate_ids: Documents to merge into primary + strategy: Merge strategy (keep_primary, merge_metadata, keep_longest) + preserve_references: Update references from duplicates to primary + cleanup_action: What to do with duplicate documents after merge: + - "soft_delete": Mark as deleted, set merged_into_id (default, audit-safe) + - "archive": Move to archive status, keep all data + - "hard_delete": Permanently delete (data loss, not recommended) + - "keep": Leave duplicates in place (just update references) + + Returns: + MergeResult with details + + Cleanup Actions Explained: + ------------------------- + 1. soft_delete (DEFAULT - Recommended for audit trails): + - Sets status = 'merged' on duplicate documents + - Adds merged_into_id = primary_id for traceability + - Adds merged_at timestamp + - Document remains in database but excluded from normal queries + - Can be fully restored if merge was incorrect + + 2. archive: + - Sets status = 'archived' on duplicate documents + - Document remains fully accessible via archive views + - Best for legal/compliance scenarios requiring data retention + + 3. hard_delete: + - Permanently removes duplicate documents from database + - Also removes associated chunks, embeddings, hashes + - IRREVERSIBLE - only use when storage is critical + - Merge history still preserved for audit + + 4. keep: + - Only updates references, leaves duplicates unchanged + - Useful for testing or when duplicates serve different purposes + """ + references_updated = 0 + documents_cleaned = 0 + + if preserve_references: + # Update entity mentions + for dup_id in duplicate_ids: + try: + result = await self._db.execute( + """UPDATE arkham_entity_mentions + SET document_id = :primary_id + WHERE document_id = :dup_id""", + {"primary_id": primary_id, "dup_id": dup_id} + ) + references_updated += getattr(result, 'rowcount', 0) + except Exception as e: + logger.warning(f"Failed to update entity mentions: {e}") + + # Update claims + try: + result = await self._db.execute( + """UPDATE arkham_claims + SET document_id = :primary_id + WHERE document_id = :dup_id""", + {"primary_id": primary_id, "dup_id": dup_id} + ) + references_updated += getattr(result, 'rowcount', 0) + except Exception as e: + logger.warning(f"Failed to update claims: {e}") + + # ===================================================== + # CLEANUP: Handle duplicate documents based on cleanup_action + # ===================================================== + for dup_id in duplicate_ids: + try: + if cleanup_action == "soft_delete": + # Recommended: Soft delete with merge pointer for audit trail + await self._db.execute( + """UPDATE arkham_frame.documents + SET status = 'merged', + metadata = jsonb_set( + COALESCE(metadata::jsonb, '{}'), + '{merged_into_id}', + :primary_json + ), + updated_at = CURRENT_TIMESTAMP + WHERE id = :dup_id""", + {"primary_json": json.dumps(primary_id), "dup_id": dup_id} + ) + documents_cleaned += 1 + + elif cleanup_action == "archive": + # Archive: Keep fully accessible but mark as archived + await self._db.execute( + """UPDATE arkham_frame.documents + SET status = 'archived', + metadata = jsonb_set( + COALESCE(metadata::jsonb, '{}'), + '{archived_reason}', + :reason_json + ), + updated_at = CURRENT_TIMESTAMP + WHERE id = :dup_id""", + {"reason_json": json.dumps(f"duplicate_of_{primary_id}"), "dup_id": dup_id} + ) + documents_cleaned += 1 + + elif cleanup_action == "hard_delete": + # DANGER: Permanent deletion - use with caution + # Delete in order: chunks -> embeddings -> hashes -> document + await self._db.execute( + "DELETE FROM arkham_frame.chunks WHERE document_id = :dup_id", + {"dup_id": dup_id} + ) + await self._db.execute( + "DELETE FROM arkham_documents.content_hashes WHERE document_id = :dup_id", + {"dup_id": dup_id} + ) + await self._db.execute( + "DELETE FROM arkham_frame.documents WHERE id = :dup_id", + {"dup_id": dup_id} + ) + documents_cleaned += 1 + + elif cleanup_action == "keep": + # Do nothing - just updated references above + pass + + except Exception as e: + logger.warning(f"Failed to cleanup duplicate {dup_id}: {e}") + + # Record merge in history (always, regardless of cleanup action) + await self._db.execute( + """INSERT INTO arkham_documents.merge_history + (id, primary_document_id, merged_document_ids, strategy, cleanup_action, + references_updated, documents_cleaned, merged_at) + VALUES (:id, :primary_id, :dup_ids, :strategy, :cleanup, :refs, :cleaned, CURRENT_TIMESTAMP)""", + { + "id": str(uuid.uuid4()), + "primary_id": primary_id, + "dup_ids": json.dumps(duplicate_ids), + "strategy": strategy, + "cleanup": cleanup_action, + "refs": references_updated, + "cleaned": documents_cleaned, + } + ) + + return MergeResult( + primary_id=primary_id, + merged_count=len(duplicate_ids), + references_updated=references_updated, + documents_cleaned=documents_cleaned, + cleanup_action=cleanup_action, + ) + + async def get_deduplication_stats(self, project_id: str | None = None) -> dict[str, Any]: + """ + Get deduplication statistics. + + Args: + project_id: Optional project scope + + Returns: + Dict with statistics + """ + if not self._db: + return {} + + query = """ + SELECT COUNT(*) as total_documents, + COUNT(ch.id) as documents_with_hash, + COUNT(DISTINCT ch.content_sha256) as unique_content_hashes + FROM arkham_frame.documents d + LEFT JOIN arkham_documents.content_hashes ch ON d.id = ch.document_id + """ + params: dict[str, Any] = {} + + if project_id: + query += " WHERE d.project_id = :project_id" + params["project_id"] = project_id + + row = await self._db.fetch_one(query, params) + + stats = { + "total_documents": row["total_documents"] if row else 0, + "documents_with_hash": row["documents_with_hash"] if row else 0, + "unique_content_hashes": row["unique_content_hashes"] if row else 0, + "potential_duplicates": 0, + } + + # Calculate potential duplicates + if row and row["documents_with_hash"] > row["unique_content_hashes"]: + stats["potential_duplicates"] = row["documents_with_hash"] - row["unique_content_hashes"] + + return stats diff --git a/packages/arkham-shard-documents/arkham_shard_documents/shard.py b/packages/arkham-shard-documents/arkham_shard_documents/shard.py index 68f0479..b43b3dc 100644 --- a/packages/arkham-shard-documents/arkham_shard_documents/shard.py +++ b/packages/arkham-shard-documents/arkham_shard_documents/shard.py @@ -9,6 +9,7 @@ from .api import router from .models import DocumentRecord, DocumentStatus +from .services import DeduplicationService logger = logging.getLogger(__name__) @@ -49,6 +50,7 @@ def __init__(self): self._events = None self._storage = None self._document_service = None + self._deduplication_service = None async def initialize(self, frame) -> None: """ @@ -78,6 +80,10 @@ async def initialize(self, frame) -> None: if not self._document_service: logger.warning("Document service not available - using database directly") + # Initialize deduplication service + self._deduplication_service = DeduplicationService(self._db) + logger.info("Deduplication service initialized") + # Create database schema await self._create_schema() @@ -108,6 +114,7 @@ async def shutdown(self) -> None: self._events = None self._storage = None self._document_service = None + self._deduplication_service = None logger.info("Documents Shard shutdown complete") @@ -115,6 +122,11 @@ def get_routes(self): """Return FastAPI router for this shard.""" return router + @property + def deduplication(self) -> DeduplicationService | None: + """Get deduplication service.""" + return self._deduplication_service + async def _create_schema(self) -> None: """ Create database schema for documents shard. @@ -262,6 +274,63 @@ async def _create_schema(self) -> None: ON arkham_document_views(tenant_id) """) + # =========================================== + # Deduplication Tables (arkham_documents schema) + # =========================================== + await self._db.execute("CREATE SCHEMA IF NOT EXISTS arkham_documents") + + # Content hashes for deduplication + await self._db.execute(""" + CREATE TABLE IF NOT EXISTS arkham_documents.content_hashes ( + id TEXT PRIMARY KEY, + document_id TEXT NOT NULL UNIQUE, + content_md5 TEXT, + content_sha256 TEXT, + simhash BIGINT, + text_length INTEGER, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + tenant_id UUID + ) + """) + + # Merge history for audit trail + await self._db.execute(""" + CREATE TABLE IF NOT EXISTS arkham_documents.merge_history ( + id TEXT PRIMARY KEY, + primary_document_id TEXT NOT NULL, + merged_document_ids JSONB NOT NULL, + strategy TEXT, + cleanup_action TEXT, + references_updated INTEGER DEFAULT 0, + documents_cleaned INTEGER DEFAULT 0, + merged_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + merged_by TEXT, + tenant_id UUID + ) + """) + + # Indexes for deduplication + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_content_hashes_doc + ON arkham_documents.content_hashes(document_id) + """) + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_content_hashes_md5 + ON arkham_documents.content_hashes(content_md5) + """) + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_content_hashes_sha256 + ON arkham_documents.content_hashes(content_sha256) + """) + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_content_hashes_simhash + ON arkham_documents.content_hashes(simhash) + """) + await self._db.execute(""" + CREATE INDEX IF NOT EXISTS idx_merge_history_primary + ON arkham_documents.merge_history(primary_document_id) + """) + logger.info("Documents shard database schema created") # --- Helper Methods --- @@ -324,8 +393,8 @@ def _frame_row_to_document(self, row: Dict[str, Any], entity_count: int = 0) -> id=row["id"], title=row.get("filename", ""), # Frame uses filename, no separate title filename=row.get("filename", ""), - file_type=row.get("mime_type", ""), - file_size=row.get("file_size", 0), + file_type=row.get("mime_type") or "", # Handle None values + file_size=row.get("file_size") or 0, status=DocumentStatus(row.get("status", "pending")), page_count=row.get("page_count", 0), chunk_count=row.get("chunk_count", 0), diff --git a/packages/arkham-shard-graph/arkham_shard_graph/algorithms.py b/packages/arkham-shard-graph/arkham_shard_graph/algorithms.py index c16d785..eb80cd6 100644 --- a/packages/arkham-shard-graph/arkham_shard_graph/algorithms.py +++ b/packages/arkham-shard-graph/arkham_shard_graph/algorithms.py @@ -84,6 +84,317 @@ def find_shortest_path( return None + def find_all_paths( + self, + graph: Graph, + source_entity_id: str, + target_entity_id: str, + max_depth: int = 6, + max_paths: int = 10, + ) -> list[GraphPath]: + """ + Find all paths between two entities up to max_depth. + + Uses DFS with backtracking to enumerate paths. + + Args: + graph: Graph to search + source_entity_id: Source entity ID + target_entity_id: Target entity ID + max_depth: Maximum path length + max_paths: Maximum number of paths to return + + Returns: + List of GraphPath objects, shortest first + """ + adjacency = self._build_adjacency_dict(graph.edges) + all_paths: list[GraphPath] = [] + + def dfs(current: str, target: str, path: list[str], visited: set[str]): + if len(all_paths) >= max_paths: + return + + if len(path) > max_depth + 1: + return + + if current == target: + # Found a path + edges = self._get_path_edges(graph.edges, path) + total_weight = sum(e.weight for e in edges) + all_paths.append(GraphPath( + source_entity_id=source_entity_id, + target_entity_id=target_entity_id, + path=path.copy(), + edges=edges, + total_weight=total_weight, + path_length=len(path) - 1, + )) + return + + for neighbor in adjacency.get(current, []): + if neighbor not in visited: + visited.add(neighbor) + path.append(neighbor) + dfs(neighbor, target, path, visited) + path.pop() + visited.remove(neighbor) + + # Start DFS + dfs(source_entity_id, target_entity_id, [source_entity_id], {source_entity_id}) + + # Sort by path length (shortest first) + all_paths.sort(key=lambda p: p.path_length) + + return all_paths + + def find_weighted_path( + self, + graph: Graph, + source_entity_id: str, + target_entity_id: str, + max_depth: int = 10, + use_max_weight: bool = True, + ) -> GraphPath | None: + """ + Find optimal weighted path using Dijkstra's algorithm. + + Can find either maximum weight path (strongest connections) + or minimum weight path (weakest connections). + + Args: + graph: Graph to search + source_entity_id: Source entity ID + target_entity_id: Target entity ID + max_depth: Maximum path length + use_max_weight: If True, find path with maximum total weight + + Returns: + GraphPath if found, None otherwise + """ + import heapq + + # Build weighted adjacency + adjacency = self._build_weighted_adjacency(graph.edges) + + # For max weight, we negate weights and use min-heap + # For min weight, we use positive weights + def get_edge_cost(weight: float) -> float: + if use_max_weight: + return -weight # Negate for max-heap behavior + return weight + + # Dijkstra's algorithm + # heap entries: (cost, path_length, node_id, path) + heap = [(0.0, 0, source_entity_id, [source_entity_id])] + visited: dict[str, float] = {} + + while heap: + cost, path_len, current, path = heapq.heappop(heap) + + if path_len > max_depth: + continue + + if current in visited: + continue + visited[current] = cost + + if current == target_entity_id: + # Found target - reconstruct path + edges = self._get_path_edges(graph.edges, path) + total_weight = sum(e.weight for e in edges) + return GraphPath( + source_entity_id=source_entity_id, + target_entity_id=target_entity_id, + path=path, + edges=edges, + total_weight=total_weight, + path_length=len(path) - 1, + ) + + for neighbor, weight in adjacency.get(current, []): + if neighbor not in visited: + new_cost = cost + get_edge_cost(weight) + heapq.heappush( + heap, + (new_cost, path_len + 1, neighbor, path + [neighbor]) + ) + + return None + + def find_constrained_path( + self, + graph: Graph, + source_entity_id: str, + target_entity_id: str, + required_entities: list[str] | None = None, + excluded_entities: list[str] | None = None, + required_relationship_types: list[str] | None = None, + min_edge_weight: float = 0.0, + max_depth: int = 8, + ) -> GraphPath | None: + """ + Find path with constraints on nodes and edges. + + Args: + graph: Graph to search + source_entity_id: Source entity ID + target_entity_id: Target entity ID + required_entities: Entities that MUST be on the path + excluded_entities: Entities that MUST NOT be on the path + required_relationship_types: Only traverse these relationship types + min_edge_weight: Minimum weight for edges to traverse + max_depth: Maximum path length + + Returns: + GraphPath if found, None otherwise + """ + required = set(required_entities or []) + excluded = set(excluded_entities or []) + required_types = set(required_relationship_types) if required_relationship_types else None + + # Build filtered adjacency + adjacency: dict[str, list[tuple[str, float, str]]] = defaultdict(list) + for edge in graph.edges: + # Filter by relationship type + if required_types and edge.relationship_type not in required_types: + continue + # Filter by weight + if edge.weight < min_edge_weight: + continue + + adjacency[edge.source].append((edge.target, edge.weight, edge.relationship_type)) + adjacency[edge.target].append((edge.source, edge.weight, edge.relationship_type)) + + # Use BFS with required entity tracking + # State: (current_node, path, required_visited) + from collections import deque + + initial_required = required & {source_entity_id} + queue = deque([(source_entity_id, [source_entity_id], initial_required)]) + visited: dict[tuple[str, frozenset], bool] = {(source_entity_id, frozenset(initial_required)): True} + + while queue: + current, path, req_visited = queue.popleft() + + if len(path) > max_depth + 1: + continue + + # Check if we found the target with all required nodes visited + if current == target_entity_id and req_visited >= required: + edges = self._get_path_edges(graph.edges, path) + total_weight = sum(e.weight for e in edges) + return GraphPath( + source_entity_id=source_entity_id, + target_entity_id=target_entity_id, + path=path, + edges=edges, + total_weight=total_weight, + path_length=len(path) - 1, + ) + + for neighbor, weight, rel_type in adjacency.get(current, []): + # Skip excluded entities + if neighbor in excluded: + continue + + # Skip already visited in this path + if neighbor in path: + continue + + # Update required visited + new_req = req_visited | ({neighbor} & required) + state_key = (neighbor, frozenset(new_req)) + + if state_key not in visited: + visited[state_key] = True + queue.append((neighbor, path + [neighbor], new_req)) + + return None + + def find_paths_through( + self, + graph: Graph, + intermediate_entity_id: str, + max_sources: int = 5, + max_targets: int = 5, + max_depth: int = 3, + ) -> list[GraphPath]: + """ + Find paths that pass through a specific entity. + + Useful for finding what connections an entity bridges. + + Args: + graph: Graph to search + intermediate_entity_id: Entity that paths must pass through + max_sources: Maximum source entities to consider + max_targets: Maximum target entities to consider + max_depth: Maximum hops on each side of intermediate + + Returns: + List of paths passing through the intermediate entity + """ + adjacency = self._build_adjacency_dict(graph.edges) + + # Find entities reachable from intermediate (potential targets) + reachable_from: dict[str, int] = {} # entity_id -> distance + queue = deque([(intermediate_entity_id, 0)]) + visited = {intermediate_entity_id} + + while queue: + current, dist = queue.popleft() + if dist > max_depth: + continue + if current != intermediate_entity_id: + reachable_from[current] = dist + + for neighbor in adjacency.get(current, []): + if neighbor not in visited: + visited.add(neighbor) + queue.append((neighbor, dist + 1)) + + # Find entities that can reach intermediate (potential sources) + # (Same search since graph is undirected) + can_reach: dict[str, int] = reachable_from.copy() + + # Select top sources and targets by degree (most connected) + node_degrees = {n.id: n.degree for n in graph.nodes} + + source_candidates = sorted( + can_reach.keys(), + key=lambda x: node_degrees.get(x, 0), + reverse=True + )[:max_sources] + + target_candidates = sorted( + reachable_from.keys(), + key=lambda x: node_degrees.get(x, 0), + reverse=True + )[:max_targets] + + # Find paths between sources and targets that pass through intermediate + paths = [] + for source in source_candidates: + for target in target_candidates: + if source == target: + continue + + # Find path requiring intermediate + path = self.find_constrained_path( + graph, + source, + target, + required_entities=[intermediate_entity_id], + max_depth=max_depth * 2 + 1, + ) + if path: + paths.append(path) + + # Sort by total weight descending + paths.sort(key=lambda p: p.total_weight, reverse=True) + + return paths + def calculate_degree_centrality( self, graph: Graph, limit: int = 50 ) -> list[CentralityResult]: diff --git a/packages/arkham-shard-graph/arkham_shard_graph/api.py b/packages/arkham-shard-graph/arkham_shard_graph/api.py index 0d4f4c2..020be01 100644 --- a/packages/arkham-shard-graph/arkham_shard_graph/api.py +++ b/packages/arkham-shard-graph/arkham_shard_graph/api.py @@ -1949,6 +1949,241 @@ async def find_path(request: PathRequest) -> PathResponse: raise HTTPException(status_code=500, detail=str(e)) +# --- Advanced Pathfinding Endpoints --- + + +class AllPathsRequest(BaseModel): + """Request for finding all paths between entities.""" + project_id: str + source_entity_id: str + target_entity_id: str + max_depth: int = 6 + max_paths: int = 10 + + +class AllPathsResponse(BaseModel): + """Response for all paths query.""" + paths_found: int + paths: list[dict[str, Any]] + + +class WeightedPathRequest(BaseModel): + """Request for weighted path finding.""" + project_id: str + source_entity_id: str + target_entity_id: str + max_depth: int = 10 + use_max_weight: bool = True # True for strongest path, False for weakest + + +class ConstrainedPathRequest(BaseModel): + """Request for constrained path finding.""" + project_id: str + source_entity_id: str + target_entity_id: str + required_entities: list[str] | None = None + excluded_entities: list[str] | None = None + required_relationship_types: list[str] | None = None + min_edge_weight: float = 0.0 + max_depth: int = 8 + + +class PathsThroughRequest(BaseModel): + """Request for finding paths through an entity.""" + project_id: str + intermediate_entity_id: str + max_sources: int = 5 + max_targets: int = 5 + max_depth: int = 3 + + +@router.post("/paths/all") +async def find_all_paths(request: AllPathsRequest) -> AllPathsResponse: + """ + Find all paths between two entities. + + Returns multiple paths up to max_paths, sorted by length. + Useful for discovering all connection routes. + """ + if not _algorithms: + raise HTTPException(status_code=503, detail="Graph algorithms not available") + + try: + # Get graph + if _storage: + graph = await _storage.load_graph(request.project_id) + elif _builder: + graph = await _builder.build_graph(project_id=request.project_id) + else: + raise HTTPException(status_code=503, detail="Graph service not available") + + # Find all paths + paths = _algorithms.find_all_paths( + graph=graph, + source_entity_id=request.source_entity_id, + target_entity_id=request.target_entity_id, + max_depth=request.max_depth, + max_paths=request.max_paths, + ) + + return AllPathsResponse( + paths_found=len(paths), + paths=[p.to_dict() for p in paths], + ) + + except Exception as e: + logger.error(f"Error finding all paths: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/paths/weighted") +async def find_weighted_path(request: WeightedPathRequest) -> PathResponse: + """ + Find optimal weighted path between entities. + + Uses Dijkstra's algorithm to find the path with: + - Maximum total weight (strongest connections) when use_max_weight=True + - Minimum total weight (weakest connections) when use_max_weight=False + """ + if not _algorithms: + raise HTTPException(status_code=503, detail="Graph algorithms not available") + + try: + # Get graph + if _storage: + graph = await _storage.load_graph(request.project_id) + elif _builder: + graph = await _builder.build_graph(project_id=request.project_id) + else: + raise HTTPException(status_code=503, detail="Graph service not available") + + # Find weighted path + path = _algorithms.find_weighted_path( + graph=graph, + source_entity_id=request.source_entity_id, + target_entity_id=request.target_entity_id, + max_depth=request.max_depth, + use_max_weight=request.use_max_weight, + ) + + if not path: + return PathResponse( + path_found=False, + path_length=0, + path=[], + edges=[], + total_weight=0.0, + ) + + return PathResponse( + path_found=True, + path_length=path.path_length, + path=path.path, + edges=[e.to_dict() for e in path.edges], + total_weight=path.total_weight, + ) + + except Exception as e: + logger.error(f"Error finding weighted path: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/paths/constrained") +async def find_constrained_path(request: ConstrainedPathRequest) -> PathResponse: + """ + Find path with constraints on nodes and edges. + + Supports: + - Required entities (must pass through) + - Excluded entities (must avoid) + - Relationship type filtering + - Minimum edge weight + """ + if not _algorithms: + raise HTTPException(status_code=503, detail="Graph algorithms not available") + + try: + # Get graph + if _storage: + graph = await _storage.load_graph(request.project_id) + elif _builder: + graph = await _builder.build_graph(project_id=request.project_id) + else: + raise HTTPException(status_code=503, detail="Graph service not available") + + # Find constrained path + path = _algorithms.find_constrained_path( + graph=graph, + source_entity_id=request.source_entity_id, + target_entity_id=request.target_entity_id, + required_entities=request.required_entities, + excluded_entities=request.excluded_entities, + required_relationship_types=request.required_relationship_types, + min_edge_weight=request.min_edge_weight, + max_depth=request.max_depth, + ) + + if not path: + return PathResponse( + path_found=False, + path_length=0, + path=[], + edges=[], + total_weight=0.0, + ) + + return PathResponse( + path_found=True, + path_length=path.path_length, + path=path.path, + edges=[e.to_dict() for e in path.edges], + total_weight=path.total_weight, + ) + + except Exception as e: + logger.error(f"Error finding constrained path: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + +@router.post("/paths/through") +async def find_paths_through_entity(request: PathsThroughRequest) -> AllPathsResponse: + """ + Find paths that pass through a specific entity. + + Useful for identifying what connections an entity bridges + and understanding their position in the network. + """ + if not _algorithms: + raise HTTPException(status_code=503, detail="Graph algorithms not available") + + try: + # Get graph + if _storage: + graph = await _storage.load_graph(request.project_id) + elif _builder: + graph = await _builder.build_graph(project_id=request.project_id) + else: + raise HTTPException(status_code=503, detail="Graph service not available") + + # Find paths through entity + paths = _algorithms.find_paths_through( + graph=graph, + intermediate_entity_id=request.intermediate_entity_id, + max_sources=request.max_sources, + max_targets=request.max_targets, + max_depth=request.max_depth, + ) + + return AllPathsResponse( + paths_found=len(paths), + paths=[p.to_dict() for p in paths], + ) + + except Exception as e: + logger.error(f"Error finding paths through entity: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=str(e)) + + @router.get("/centrality/{project_id}") async def calculate_centrality( project_id: str, diff --git a/packages/arkham-shard-parse/arkham_shard_parse/chunker.py b/packages/arkham-shard-parse/arkham_shard_parse/chunker.py index 763c7b7..96d23d1 100644 --- a/packages/arkham-shard-parse/arkham_shard_parse/chunker.py +++ b/packages/arkham-shard-parse/arkham_shard_parse/chunker.py @@ -126,9 +126,19 @@ def _chunk_by_sentences( Returns: List of text chunks """ - # Simple sentence splitting on period, exclamation, question mark + # Sentence splitting that preserves emails, URLs, and abbreviations + # Only splits on punctuation that looks like a sentence boundary: + # - Period/!/? followed by whitespace and then an uppercase letter + # - Period/!/? followed by newline + # Does NOT split on periods in emails, URLs, abbreviations, or decimals import re - sentences = re.split(r'[.!?]+', text) + + # Split on sentence-ending punctuation followed by whitespace and capital letter, + # or followed by newline/end of text + # This preserves: emails (agent.smith@cia.gov), URLs, abbreviations, decimals + sentence_pattern = r'(?<=[.!?])\s+(?=[A-Z])|(?<=[.!?])\s*(?=\n)' + + sentences = re.split(sentence_pattern, text) chunks = [] chunk_index = 0 @@ -210,8 +220,10 @@ def _chunk_semantic( import re import numpy as np - # Split into sentences - sentences = re.split(r'(?<=[.!?])\s+', text) + # Split into sentences - preserve emails, URLs, abbreviations + # Only split on punctuation followed by whitespace and capital letter + sentence_pattern = r'(?<=[.!?])\s+(?=[A-Z])|(?<=[.!?])\s*(?=\n)' + sentences = re.split(sentence_pattern, text) sentences = [s.strip() for s in sentences if s.strip()] if len(sentences) < 3: diff --git a/packages/arkham-shard-search/arkham_shard_search/api.py b/packages/arkham-shard-search/arkham_shard_search/api.py index c2e2d58..d0f7346 100644 --- a/packages/arkham-shard-search/arkham_shard_search/api.py +++ b/packages/arkham-shard-search/arkham_shard_search/api.py @@ -32,16 +32,18 @@ _semantic_engine = None _keyword_engine = None _hybrid_engine = None +_regex_engine = None _filter_optimizer = None _event_bus = None -def init_api(semantic_engine, keyword_engine, hybrid_engine, filter_optimizer, event_bus): +def init_api(semantic_engine, keyword_engine, hybrid_engine, regex_engine, filter_optimizer, event_bus): """Initialize API with shard dependencies.""" - global _semantic_engine, _keyword_engine, _hybrid_engine, _filter_optimizer, _event_bus + global _semantic_engine, _keyword_engine, _hybrid_engine, _regex_engine, _filter_optimizer, _event_bus _semantic_engine = semantic_engine _keyword_engine = keyword_engine _hybrid_engine = hybrid_engine + _regex_engine = regex_engine _filter_optimizer = filter_optimizer _event_bus = event_bus @@ -727,3 +729,611 @@ async def submit_ai_feedback(request: Request, body: AIFeedbackRequest): logger.warning(f"AI Feedback not stored (DB unavailable): session={body.session_id}") return AIFeedbackResponse(success=True, feedback_id=feedback_id) + + +# --- Regex Search --- + + +class RegexSearchRequest(BaseModel): + """Request for regex search.""" + pattern: str + flags: list[str] = [] + project_id: str | None = None + document_ids: list[str] | None = None + limit: int = 100 + offset: int = 0 + highlight: bool = True + context_chars: int = 100 + + +class RegexMatchResponse(BaseModel): + """Individual regex match.""" + document_id: str + document_title: str + page_number: int | None + chunk_id: str | None + match_text: str + context: str + start_offset: int + end_offset: int + line_number: int | None + + +class RegexSearchResponse(BaseModel): + """Response for regex search.""" + pattern: str + matches: list[RegexMatchResponse] + total_matches: int + total_chunks_with_matches: int + documents_searched: int + duration_ms: float + error: str | None = None + + +class ValidatePatternRequest(BaseModel): + """Request to validate regex pattern.""" + pattern: str + + +class ValidatePatternResponse(BaseModel): + """Response for pattern validation.""" + valid: bool + error: str | None = None + estimated_performance: str + + +class RegexPresetResponse(BaseModel): + """Regex preset.""" + id: str + name: str + pattern: str + description: str + category: str + flags: list[str] = [] + is_system: bool = True + + +class CreatePresetRequest(BaseModel): + """Request to create custom preset.""" + name: str + pattern: str + description: str = "" + category: str = "custom" + flags: list[str] = [] + + +@router.post("/regex", response_model=RegexSearchResponse) +async def search_regex(request: Request, body: RegexSearchRequest): + """ + Search using regex pattern across document content. + + Supports flags: case_insensitive, multiline, dotall + + The search uses PostgreSQL native regex matching for scalability, + with pattern matching happening in the database. + """ + if not _regex_engine: + raise HTTPException(status_code=503, detail="Regex search not initialized") + + # Validate pattern first + valid, error, perf = _regex_engine.validate_pattern(body.pattern) + if not valid: + raise HTTPException(status_code=400, detail=f"Invalid regex pattern: {error}") + if perf == "dangerous": + raise HTTPException( + status_code=400, + detail="Pattern may cause performance issues (catastrophic backtracking)" + ) + + from .models import RegexSearchQuery + + query = RegexSearchQuery( + pattern=body.pattern, + flags=body.flags, + project_id=body.project_id, + document_ids=body.document_ids, + limit=body.limit, + offset=body.offset, + highlight=body.highlight, + context_chars=body.context_chars, + ) + + try: + result = await _regex_engine.search(query) + except Exception as e: + logger.error(f"Regex search failed: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=f"Search failed: {str(e)}") + + # Emit event (following existing pattern) + if _event_bus: + await _event_bus.emit( + "search.regex.executed", + { + "pattern": body.pattern, + "flags": body.flags, + "matches_count": result.total_matches, + "documents_searched": result.documents_searched, + "duration_ms": result.duration_ms, + }, + source="search-shard", + ) + + return RegexSearchResponse( + pattern=result.pattern, + matches=[ + RegexMatchResponse( + document_id=m.document_id, + document_title=m.document_title, + page_number=m.page_number, + chunk_id=m.chunk_id, + match_text=m.match_text, + context=m.context, + start_offset=m.start_offset, + end_offset=m.end_offset, + line_number=m.line_number, + ) + for m in result.matches + ], + total_matches=result.total_matches, + total_chunks_with_matches=result.total_chunks_with_matches, + documents_searched=result.documents_searched, + duration_ms=result.duration_ms, + error=result.error, + ) + + +@router.post("/regex/validate", response_model=ValidatePatternResponse) +async def validate_regex_pattern(body: ValidatePatternRequest): + """ + Validate a regex pattern and estimate performance. + + Returns validation status, error message (if invalid), + and performance estimate (fast/moderate/slow/dangerous). + """ + if not _regex_engine: + raise HTTPException(status_code=503, detail="Regex search not initialized") + + valid, error, perf = _regex_engine.validate_pattern(body.pattern) + + return ValidatePatternResponse( + valid=valid, + error=error, + estimated_performance=perf, + ) + + +@router.get("/regex/presets", response_model=list[RegexPresetResponse]) +async def get_regex_presets(category: str | None = None): + """ + Get available regex presets. + + Categories: pii, contact, financial, technical, temporal, custom + + Returns both system presets and custom user-defined presets. + """ + if not _regex_engine: + raise HTTPException(status_code=503, detail="Regex search not initialized") + + presets = await _regex_engine.get_presets(category=category) + + return [ + RegexPresetResponse( + id=p["id"], + name=p["name"], + pattern=p["pattern"], + description=p.get("description", ""), + category=p["category"], + flags=p.get("flags", []), + is_system=p.get("is_system", True), + ) + for p in presets + ] + + +@router.post("/regex/presets", response_model=RegexPresetResponse) +async def create_regex_preset(request: Request, body: CreatePresetRequest): + """ + Create a custom regex preset. + + Custom presets are stored in the database and can be + retrieved alongside system presets. + """ + if not _regex_engine: + raise HTTPException(status_code=503, detail="Regex search not initialized") + + # Validate pattern + valid, error, _ = _regex_engine.validate_pattern(body.pattern) + if not valid: + raise HTTPException(status_code=400, detail=f"Invalid pattern: {error}") + + # Get tenant_id if available (following codebase pattern) + shard = get_shard(request) + tenant_id = shard.get_tenant_id_or_none() if hasattr(shard, 'get_tenant_id_or_none') else None + + try: + preset = await _regex_engine.save_custom_preset( + name=body.name, + pattern=body.pattern, + description=body.description, + category=body.category, + flags=body.flags, + tenant_id=str(tenant_id) if tenant_id else None, + ) + except Exception as e: + logger.error(f"Failed to create preset: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=f"Failed to create preset: {str(e)}") + + # Emit event + if _event_bus: + await _event_bus.emit( + "search.regex.preset_created", + {"preset_id": preset["id"], "name": preset["name"]}, + source="search-shard", + ) + + return RegexPresetResponse(**preset) + + +@router.delete("/regex/presets/{preset_id}") +async def delete_regex_preset(request: Request, preset_id: str): + """ + Delete a custom regex preset. + + System presets cannot be deleted. + """ + if not _regex_engine: + raise HTTPException(status_code=503, detail="Regex search not initialized") + + # Get tenant_id if available + shard = get_shard(request) + tenant_id = shard.get_tenant_id_or_none() if hasattr(shard, 'get_tenant_id_or_none') else None + + try: + deleted = await _regex_engine.delete_custom_preset( + preset_id=preset_id, + tenant_id=str(tenant_id) if tenant_id else None, + ) + except Exception as e: + logger.error(f"Failed to delete preset: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=f"Failed to delete preset: {str(e)}") + + if not deleted: + raise HTTPException(status_code=404, detail="Preset not found or is a system preset") + + return {"success": True, "preset_id": preset_id} + + +# --- Pattern Extractions API --- + + +class PatternExtractionResponse(BaseModel): + """Individual pattern extraction.""" + id: str + document_id: str + preset_id: str + preset_name: str | None + category: str | None + match_text: str + context: str | None + page_number: int | None + chunk_id: str | None + start_offset: int | None + end_offset: int | None + line_number: int | None + extracted_at: str | None + + +class ExtractionsListResponse(BaseModel): + """Response for pattern extractions list.""" + extractions: list[PatternExtractionResponse] + total: int + document_id: str | None = None + preset_id: str | None = None + category: str | None = None + + +class ExtractionStatsResponse(BaseModel): + """Response for extraction statistics.""" + total_extractions: int + documents_with_patterns: int + by_category: dict[str, int] + by_preset: dict[str, int] + + +@router.get("/regex/extractions", response_model=ExtractionsListResponse) +async def get_pattern_extractions( + request: Request, + document_id: str | None = None, + preset_id: str | None = None, + category: str | None = None, + limit: int = 100, + offset: int = 0, +): + """ + Get auto-extracted pattern matches. + + These are patterns automatically extracted when documents are parsed. + Filter by document_id, preset_id, or category. + """ + shard = get_shard(request) + + if not shard._db: + raise HTTPException(status_code=503, detail="Database not available") + + query = """ + SELECT id, document_id, preset_id, preset_name, category, match_text, + context, page_number, chunk_id, start_offset, end_offset, + line_number, extracted_at + FROM arkham_search.pattern_extractions + WHERE 1=1 + """ + params: dict = {} + + if document_id: + query += " AND document_id = :document_id" + params["document_id"] = document_id + + if preset_id: + query += " AND preset_id = :preset_id" + params["preset_id"] = preset_id + + if category: + query += " AND category = :category" + params["category"] = category + + # Get total count + count_query = """ + SELECT COUNT(*) as total FROM arkham_search.pattern_extractions WHERE 1=1 + """ + if document_id: + count_query += " AND document_id = :document_id" + if preset_id: + count_query += " AND preset_id = :preset_id" + if category: + count_query += " AND category = :category" + + try: + count_row = await shard._db.fetch_one(count_query, params) + total = count_row["total"] if count_row else 0 + except Exception: + total = 0 + + query += " ORDER BY extracted_at DESC LIMIT :limit OFFSET :offset" + params["limit"] = limit + params["offset"] = offset + + try: + rows = await shard._db.fetch_all(query, params) + except Exception as e: + logger.error(f"Failed to fetch extractions: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=f"Failed to fetch extractions: {str(e)}") + + extractions = [ + PatternExtractionResponse( + id=row["id"], + document_id=row["document_id"], + preset_id=row["preset_id"], + preset_name=row["preset_name"], + category=row["category"], + match_text=row["match_text"], + context=row["context"], + page_number=row["page_number"], + chunk_id=row["chunk_id"], + start_offset=row["start_offset"], + end_offset=row["end_offset"], + line_number=row["line_number"], + extracted_at=str(row["extracted_at"]) if row["extracted_at"] else None, + ) + for row in rows + ] + + return ExtractionsListResponse( + extractions=extractions, + total=total, + document_id=document_id, + preset_id=preset_id, + category=category, + ) + + +@router.get("/regex/extractions/stats", response_model=ExtractionStatsResponse) +async def get_extraction_stats( + request: Request, + project_id: str | None = None, +): + """ + Get statistics on pattern extractions. + + Returns counts by category and preset. + """ + shard = get_shard(request) + + if not shard._db: + raise HTTPException(status_code=503, detail="Database not available") + + try: + # Total extractions + total_row = await shard._db.fetch_one( + "SELECT COUNT(*) as total FROM arkham_search.pattern_extractions" + ) + total = total_row["total"] if total_row else 0 + + # Documents with patterns + doc_row = await shard._db.fetch_one( + "SELECT COUNT(DISTINCT document_id) as cnt FROM arkham_search.pattern_extractions" + ) + docs_with_patterns = doc_row["cnt"] if doc_row else 0 + + # By category + cat_rows = await shard._db.fetch_all( + "SELECT category, COUNT(*) as cnt FROM arkham_search.pattern_extractions GROUP BY category" + ) + by_category = {row["category"]: row["cnt"] for row in cat_rows} if cat_rows else {} + + # By preset + preset_rows = await shard._db.fetch_all( + "SELECT preset_id, COUNT(*) as cnt FROM arkham_search.pattern_extractions GROUP BY preset_id" + ) + by_preset = {row["preset_id"]: row["cnt"] for row in preset_rows} if preset_rows else {} + + except Exception as e: + logger.error(f"Failed to fetch extraction stats: {e}", exc_info=True) + raise HTTPException(status_code=500, detail=f"Failed to fetch stats: {str(e)}") + + return ExtractionStatsResponse( + total_extractions=total, + documents_with_patterns=docs_with_patterns, + by_category=by_category, + by_preset=by_preset, + ) + + +@router.post("/regex/extract/{document_id}") +async def trigger_pattern_extraction( + request: Request, + document_id: str, + preset_ids: list[str] | None = None, +): + """ + Manually trigger pattern extraction for a specific document. + + By default runs all system presets. Optionally specify preset_ids + to run only specific presets. + """ + import re + import uuid + + shard = get_shard(request) + + if not shard._db: + raise HTTPException(status_code=503, detail="Database not available") + + # Check document exists + doc_row = await shard._db.fetch_one( + "SELECT id FROM arkham_frame.documents WHERE id = :doc_id", + {"doc_id": document_id} + ) + if not doc_row: + raise HTTPException(status_code=404, detail="Document not found") + + # Fetch chunks + chunks = await shard._db.fetch_all( + """ + SELECT id, text, chunk_index, page_number + FROM arkham_frame.chunks + WHERE document_id = :doc_id + ORDER BY chunk_index + """, + {"doc_id": document_id} + ) + + if not chunks: + return {"document_id": document_id, "extractions": 0, "message": "No chunks found"} + + # Get presets to use + from .engines.regex import REGEX_PRESETS + + if preset_ids: + presets = [p for p in REGEX_PRESETS if p["id"] in preset_ids] + # Also get custom presets from DB + if shard._db: + custom_query = """ + SELECT id, name, pattern, category FROM arkham_search.regex_presets + WHERE id = ANY(:ids) + """ + custom_rows = await shard._db.fetch_all(custom_query, {"ids": preset_ids}) + for row in custom_rows: + presets.append({ + "id": row["id"], + "name": row["name"], + "pattern": row["pattern"], + "category": row["category"], + }) + else: + presets = list(REGEX_PRESETS) + + # Clear existing extractions for this document (re-extract) + await shard._db.execute( + "DELETE FROM arkham_search.pattern_extractions WHERE document_id = :doc_id", + {"doc_id": document_id} + ) + + # Extract patterns + extractions = [] + for preset in presets: + preset_id = preset["id"] + preset_name = preset["name"] + category = preset["category"] + pattern = preset["pattern"] + + try: + compiled = re.compile(pattern, re.IGNORECASE) + except re.error: + continue + + for chunk in chunks: + text = chunk["text"] or "" + chunk_id = chunk["id"] + page_number = chunk["page_number"] + + for match in compiled.finditer(text): + match_text = match.group(0) + match_start = match.start() + match_end = match.end() + + ctx_start = max(0, match_start - 50) + ctx_end = min(len(text), match_end + 50) + context = text[ctx_start:ctx_end] + if ctx_start > 0: + context = "..." + context + if ctx_end < len(text): + context = context + "..." + + line_number = text[:match_start].count('\n') + 1 + + extractions.append({ + "id": str(uuid.uuid4())[:12], + "document_id": document_id, + "preset_id": preset_id, + "preset_name": preset_name, + "category": category, + "match_text": match_text, + "context": context, + "page_number": page_number, + "chunk_id": chunk_id, + "start_offset": match_start, + "end_offset": match_end, + "line_number": line_number, + }) + + # Store extractions + for ext in extractions: + await shard._db.execute( + """ + INSERT INTO arkham_search.pattern_extractions + (id, document_id, preset_id, preset_name, category, match_text, + context, page_number, chunk_id, start_offset, end_offset, line_number) + VALUES (:id, :document_id, :preset_id, :preset_name, :category, :match_text, + :context, :page_number, :chunk_id, :start_offset, :end_offset, :line_number) + ON CONFLICT (id) DO NOTHING + """, + ext + ) + + # Emit event + if _event_bus: + await _event_bus.emit( + "search.patterns.extracted", + { + "document_id": document_id, + "total_matches": len(extractions), + "manual_trigger": True, + }, + source="search-shard", + ) + + return { + "document_id": document_id, + "extractions": len(extractions), + "presets_used": [p["id"] for p in presets], + } diff --git a/packages/arkham-shard-search/arkham_shard_search/engines/__init__.py b/packages/arkham-shard-search/arkham_shard_search/engines/__init__.py index 182440d..7b4de8e 100644 --- a/packages/arkham-shard-search/arkham_shard_search/engines/__init__.py +++ b/packages/arkham-shard-search/arkham_shard_search/engines/__init__.py @@ -3,5 +3,6 @@ from .semantic import SemanticSearchEngine from .keyword import KeywordSearchEngine from .hybrid import HybridSearchEngine +from .regex import RegexSearchEngine -__all__ = ["SemanticSearchEngine", "KeywordSearchEngine", "HybridSearchEngine"] +__all__ = ["SemanticSearchEngine", "KeywordSearchEngine", "HybridSearchEngine", "RegexSearchEngine"] diff --git a/packages/arkham-shard-search/arkham_shard_search/engines/regex.py b/packages/arkham-shard-search/arkham_shard_search/engines/regex.py new file mode 100644 index 0000000..a0fcdf3 --- /dev/null +++ b/packages/arkham-shard-search/arkham_shard_search/engines/regex.py @@ -0,0 +1,458 @@ +"""Regex search engine for pattern matching across documents.""" + +import re +import logging +import time +from typing import Any + +from ..models import RegexMatch, RegexSearchQuery, RegexSearchResult + +logger = logging.getLogger(__name__) + + +# Built-in presets (following codebase constant pattern) +REGEX_PRESETS = [ + { + "id": "email", + "name": "Email Addresses", + "pattern": r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", + "description": "Match email addresses", + "category": "contact", + }, + { + "id": "phone_us", + "name": "US Phone Numbers", + "pattern": r"\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}", + "description": "Match US phone number formats", + "category": "contact", + }, + { + "id": "ssn", + "name": "Social Security Numbers", + "pattern": r"\d{3}-\d{2}-\d{4}", + "description": "Match SSN format (XXX-XX-XXXX)", + "category": "pii", + }, + { + "id": "credit_card", + "name": "Credit Card Numbers", + "pattern": r"\d{4}[-\s]?\d{4}[-\s]?\d{4}[-\s]?\d{4}", + "description": "Match credit card number format", + "category": "financial", + }, + { + "id": "ip_address", + "name": "IP Addresses", + "pattern": r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", + "description": "Match IPv4 addresses", + "category": "technical", + }, + { + "id": "date_mdy", + "name": "Dates (MM/DD/YYYY)", + "pattern": r"\d{1,2}/\d{1,2}/\d{2,4}", + "description": "Match dates in MM/DD/YYYY format", + "category": "temporal", + }, + { + "id": "url", + "name": "URLs", + "pattern": r"https?://[^\s]+", + "description": "Match HTTP/HTTPS URLs", + "category": "technical", + }, + { + "id": "money_usd", + "name": "USD Amounts", + "pattern": r"\$[\d,]+\.?\d*", + "description": "Match US dollar amounts", + "category": "financial", + }, +] + + +class RegexSearchEngine: + """ + Regex search engine that searches across document content. + + Follows the same pattern as SemanticSearchEngine and KeywordSearchEngine. + + IMPORTANT: Uses PostgreSQL native regex (~, ~*) for scalability. + Pattern matching happens in the database, not in Python memory. + This prevents memory blowup on large corpora. + """ + + def __init__(self, database_service, documents_service=None, config=None): + """ + Initialize regex search engine. + + Args: + database_service: Frame database service (required) + documents_service: Frame documents service (optional) + config: Configuration dict (optional) + """ + self._db = database_service + self._documents_service = documents_service + self._config = config or {} + self._max_results = self._config.get("max_results", 1000) + self._context_chars = self._config.get("context_chars", 100) + self._timeout_ms = self._config.get("timeout_ms", 30000) + + def validate_pattern(self, pattern: str) -> tuple[bool, str | None, str]: + """ + Validate regex pattern and estimate performance. + + Args: + pattern: Regex pattern to validate + + Returns: + Tuple of (is_valid, error_message, performance_estimate) + """ + try: + re.compile(pattern) + except re.error as e: + return False, str(e), "invalid" + + # Check for dangerous patterns (catastrophic backtracking) + dangerous_patterns = [ + r'(.+)+', # Nested quantifiers + r'(.*)*', # Nested stars + r'(a|a)+', # Exponential alternatives + r'([^"]*)*', # Nested negated character class with star + ] + + for dp in dangerous_patterns: + if dp in pattern: + return True, None, "dangerous" + + # Estimate performance based on pattern complexity + if len(pattern) < 10 and not any(c in pattern for c in '*+?{}'): + return True, None, "fast" + elif len(pattern) < 50: + return True, None, "moderate" + else: + return True, None, "slow" + + async def search(self, query: RegexSearchQuery) -> RegexSearchResult: + """ + Search for regex pattern across document content. + + IMPORTANT: Uses PostgreSQL native regex (~, ~*) for scalability. + Pattern matching happens in the database, not in Python memory. + This prevents memory blowup on large corpora. + + Args: + query: RegexSearchQuery with pattern and filters + + Returns: + RegexSearchResult with matches + """ + start_time = time.time() + + # Validate pattern syntax using Python regex (for better error messages) + try: + re.compile(query.pattern) + except re.error as e: + logger.error(f"Invalid regex pattern: {e}") + return RegexSearchResult( + pattern=query.pattern, + matches=[], + total_matches=0, + documents_searched=0, + duration_ms=(time.time() - start_time) * 1000, + error=f"Invalid regex: {e}", + ) + + # Build PostgreSQL regex operator based on flags + # ~ = case-sensitive match + # ~* = case-insensitive match + case_insensitive = any(f in ("case_insensitive", "i") for f in query.flags) + regex_op = "~*" if case_insensitive else "~" + + # PostgreSQL regex uses different syntax for some features: + # - DOTALL (s flag) not directly supported, but newlines match . in bracket expressions + # - MULTILINE (m flag) is default behavior in PostgreSQL + pg_pattern = query.pattern + + # First, get total count (for pagination info) - uses COUNT with regex + count_sql = f""" + SELECT COUNT(DISTINCT c.id) as match_count, + COUNT(DISTINCT c.document_id) as doc_count + FROM arkham_frame.chunks c + JOIN arkham_frame.documents d ON c.document_id = d.id + WHERE c.text {regex_op} :pattern + """ + params: dict[str, Any] = {"pattern": pg_pattern} + + if query.project_id: + count_sql += " AND d.project_id = :project_id" + params["project_id"] = query.project_id + + if query.document_ids: + count_sql += " AND d.id = ANY(:document_ids)" + params["document_ids"] = query.document_ids + + try: + count_row = await self._db.fetch_one(count_sql, params) + total_chunks_with_matches = count_row["match_count"] if count_row else 0 + documents_searched = count_row["doc_count"] if count_row else 0 + except Exception as e: + logger.error(f"Regex count query failed: {e}", exc_info=True) + return RegexSearchResult( + pattern=query.pattern, + matches=[], + total_matches=0, + documents_searched=0, + duration_ms=(time.time() - start_time) * 1000, + error=f"Database error: {e}", + ) + + # Now fetch actual matches with pagination + # Use subquery for matching chunks, then extract match positions in Python + # (PostgreSQL regexp_matches with 'g' flag returns multiple rows per match, + # which can be complex to handle in SQL - we'll extract in Python) + search_sql = f""" + SELECT c.id as chunk_id, c.document_id, c.text, c.chunk_index, + d.filename as document_title, c.page_number + FROM arkham_frame.chunks c + JOIN arkham_frame.documents d ON c.document_id = d.id + WHERE c.text {regex_op} :pattern + """ + + if query.project_id: + search_sql += " AND d.project_id = :project_id" + + if query.document_ids: + search_sql += " AND d.id = ANY(:document_ids)" + + search_sql += """ + ORDER BY d.id, c.chunk_index + LIMIT :chunk_limit + """ + + params["chunk_limit"] = min(1000, self._max_results) # Cap chunks to prevent memory issues + + matches = [] + try: + rows = await self._db.fetch_all(search_sql, params) + + # Build Python regex for match extraction with flags + py_flags = 0 + if case_insensitive: + py_flags |= re.IGNORECASE + if "multiline" in query.flags: + py_flags |= re.MULTILINE + if "dotall" in query.flags: + py_flags |= re.DOTALL + + compiled_pattern = re.compile(query.pattern, py_flags) + + for row in rows: + text = row["text"] or "" + document_id = row["document_id"] + document_title = row["document_title"] or "" + page_number = row["page_number"] + chunk_id = row["chunk_id"] + + # Find all matches in this chunk's text + for match in compiled_pattern.finditer(text): + match_text = match.group(0) + match_start = match.start() + match_end = match.end() + + # Extract context around match + context_chars = query.context_chars + ctx_start = max(0, match_start - context_chars) + ctx_end = min(len(text), match_end + context_chars) + context = text[ctx_start:ctx_end] + + # Add ellipsis if truncated + if ctx_start > 0: + context = "..." + context + if ctx_end < len(text): + context = context + "..." + + # Calculate line number + line_number = text[:match_start].count('\n') + 1 + + matches.append(RegexMatch( + document_id=document_id, + document_title=document_title, + page_number=page_number, + chunk_id=chunk_id, + match_text=match_text, + context=context, + start_offset=match_start, + end_offset=match_end, + line_number=line_number, + )) + + if len(matches) >= self._max_results: + break + + if len(matches) >= self._max_results: + break + + except Exception as e: + logger.error(f"Regex search failed: {e}", exc_info=True) + return RegexSearchResult( + pattern=query.pattern, + matches=[], + total_matches=0, + documents_searched=0, + duration_ms=(time.time() - start_time) * 1000, + error=f"Search failed: {e}", + ) + + # Apply pagination to results + paginated_matches = matches[query.offset:query.offset + query.limit] + + duration_ms = (time.time() - start_time) * 1000 + + logger.info( + f"Regex search '{query.pattern}' found {len(matches)} matches " + f"in {total_chunks_with_matches} chunks across {documents_searched} documents " + f"({duration_ms:.1f}ms)" + ) + + return RegexSearchResult( + pattern=query.pattern, + matches=paginated_matches, + total_matches=len(matches), # Total from this batch + total_chunks_with_matches=total_chunks_with_matches, # True total + documents_searched=documents_searched, + duration_ms=duration_ms, + ) + + async def get_presets(self, category: str | None = None) -> list[dict]: + """ + Get available regex presets (system + custom from database). + + Args: + category: Optional category filter + + Returns: + List of preset dictionaries + """ + # Start with system presets + presets = list(REGEX_PRESETS) + + # Add custom presets from database + if self._db: + try: + query = """ + SELECT id, name, pattern, description, category, flags, is_system + FROM arkham_search.regex_presets + WHERE 1=1 + """ + params: dict[str, Any] = {} + + if category: + query += " AND category = :category" + params["category"] = category + + query += " ORDER BY name" + + rows = await self._db.fetch_all(query, params) + for row in rows: + import json + presets.append({ + "id": row["id"], + "name": row["name"], + "pattern": row["pattern"], + "description": row.get("description", ""), + "category": row["category"], + "flags": json.loads(row["flags"]) if isinstance(row["flags"], str) else (row["flags"] or []), + "is_system": row.get("is_system", False), + }) + except Exception as e: + logger.warning(f"Failed to fetch custom presets: {e}") + + # Filter system presets by category if specified + if category: + presets = [p for p in presets if p["category"] == category] + + return presets + + async def save_custom_preset( + self, + name: str, + pattern: str, + description: str = "", + category: str = "custom", + flags: list[str] | None = None, + tenant_id: str | None = None, + ) -> dict: + """ + Save a custom regex preset to the database. + + Args: + name: Preset name + pattern: Regex pattern + description: Optional description + category: Category (default: custom) + flags: Optional regex flags + tenant_id: Optional tenant ID for multi-tenancy + + Returns: + Created preset dictionary + """ + import uuid + import json + + preset_id = str(uuid.uuid4())[:8] + + await self._db.execute( + """ + INSERT INTO arkham_search.regex_presets + (id, tenant_id, name, pattern, description, category, flags, is_system, created_at) + VALUES (:id, :tenant_id, :name, :pattern, :description, :category, :flags, FALSE, CURRENT_TIMESTAMP) + """, + { + "id": preset_id, + "tenant_id": tenant_id, + "name": name, + "pattern": pattern, + "description": description, + "category": category, + "flags": json.dumps(flags or []), + } + ) + + return { + "id": preset_id, + "name": name, + "pattern": pattern, + "description": description, + "category": category, + "flags": flags or [], + "is_system": False, + } + + async def delete_custom_preset(self, preset_id: str, tenant_id: str | None = None) -> bool: + """ + Delete a custom regex preset. + + Args: + preset_id: Preset ID to delete + tenant_id: Optional tenant ID for multi-tenancy + + Returns: + True if deleted successfully + """ + try: + query = """ + DELETE FROM arkham_search.regex_presets + WHERE id = :id AND is_system = FALSE + """ + params: dict[str, Any] = {"id": preset_id} + + if tenant_id: + query += " AND tenant_id = :tenant_id" + params["tenant_id"] = tenant_id + + await self._db.execute(query, params) + return True + except Exception as e: + logger.error(f"Failed to delete preset: {e}") + return False diff --git a/packages/arkham-shard-search/arkham_shard_search/models.py b/packages/arkham-shard-search/arkham_shard_search/models.py index 518670a..049e1cc 100644 --- a/packages/arkham-shard-search/arkham_shard_search/models.py +++ b/packages/arkham-shard-search/arkham_shard_search/models.py @@ -11,6 +11,14 @@ class SearchMode(Enum): HYBRID = "hybrid" SEMANTIC = "semantic" KEYWORD = "keyword" + REGEX = "regex" + + +class RegexFlag(Enum): + """Regex flags for pattern matching.""" + CASE_INSENSITIVE = "case_insensitive" + MULTILINE = "multiline" + DOTALL = "dotall" class SortBy(Enum): @@ -119,3 +127,57 @@ class SimilarityRequest: limit: int = 10 min_similarity: float = 0.5 filters: SearchFilters | None = None + + +# --- Regex Search Models --- + + +@dataclass +class RegexMatch: + """Individual regex match result.""" + document_id: str + document_title: str + page_number: int | None + chunk_id: str | None + match_text: str + context: str + start_offset: int + end_offset: int + line_number: int | None = None + + +@dataclass +class RegexSearchQuery: + """Regex search query parameters.""" + pattern: str + flags: list[str] = field(default_factory=list) + project_id: str | None = None + document_ids: list[str] | None = None + limit: int = 100 + offset: int = 0 + highlight: bool = True + context_chars: int = 100 + + +@dataclass +class RegexSearchResult: + """Regex search result container.""" + pattern: str + matches: list[RegexMatch] = field(default_factory=list) + total_matches: int = 0 + total_chunks_with_matches: int = 0 + documents_searched: int = 0 + duration_ms: float = 0.0 + error: str | None = None + + +@dataclass +class RegexPreset: + """Predefined regex pattern.""" + id: str + name: str + pattern: str + description: str + category: str # pii, contact, financial, technical, custom + flags: list[str] = field(default_factory=list) + is_system: bool = True diff --git a/packages/arkham-shard-search/arkham_shard_search/shard.py b/packages/arkham-shard-search/arkham_shard_search/shard.py index 8b80f77..8997d60 100644 --- a/packages/arkham-shard-search/arkham_shard_search/shard.py +++ b/packages/arkham-shard-search/arkham_shard_search/shard.py @@ -5,7 +5,7 @@ from arkham_frame.shard_interface import ArkhamShard from .api import init_api, router -from .engines import SemanticSearchEngine, KeywordSearchEngine, HybridSearchEngine +from .engines import SemanticSearchEngine, KeywordSearchEngine, HybridSearchEngine, RegexSearchEngine from .filters import FilterOptimizer logger = logging.getLogger(__name__) @@ -33,6 +33,7 @@ def __init__(self): self.semantic_engine = None self.keyword_engine = None self.hybrid_engine = None + self.regex_engine = None self.filter_optimizer = None self._db = None @@ -95,6 +96,14 @@ async def initialize(self, frame) -> None: if database_service: self.filter_optimizer = FilterOptimizer(database_service) + # Initialize regex search engine + if database_service: + self.regex_engine = RegexSearchEngine( + database_service=database_service, + documents_service=documents_service, + ) + logger.info("Regex search engine initialized") + # Create search schema and tables if database_service: await self._create_schema(database_service) @@ -104,6 +113,7 @@ async def initialize(self, frame) -> None: semantic_engine=self.semantic_engine, keyword_engine=self.keyword_engine, hybrid_engine=self.hybrid_engine, + regex_engine=self.regex_engine, filter_optimizer=self.filter_optimizer, event_bus=event_bus, ) @@ -112,6 +122,8 @@ async def initialize(self, frame) -> None: if event_bus: await event_bus.subscribe("documents.indexed", self._on_document_indexed) await event_bus.subscribe("documents.deleted", self._on_document_deleted) + # Subscribe to parse completion for auto-extraction of regex patterns + await event_bus.subscribe("parse.document.completed", self._on_parse_completed) # Register self in app state for API access if hasattr(frame, "app") and frame.app: @@ -129,10 +141,12 @@ async def shutdown(self) -> None: if event_bus: await event_bus.unsubscribe("documents.indexed", self._on_document_indexed) await event_bus.unsubscribe("documents.deleted", self._on_document_deleted) + await event_bus.unsubscribe("parse.document.completed", self._on_parse_completed) self.semantic_engine = None self.keyword_engine = None self.hybrid_engine = None + self.regex_engine = None self.filter_optimizer = None logger.info("Search Shard shutdown complete") @@ -163,7 +177,7 @@ async def _create_schema(self, db) -> None: ) """) - # Create indexes + # Create indexes for AI feedback await db.execute(""" CREATE INDEX IF NOT EXISTS idx_ai_feedback_session ON arkham_search.ai_feedback(session_id) @@ -177,6 +191,90 @@ async def _create_schema(self, db) -> None: ON arkham_search.ai_feedback(created_at DESC) """) + # Regex presets table for custom patterns + await db.execute(""" + CREATE TABLE IF NOT EXISTS arkham_search.regex_presets ( + id TEXT PRIMARY KEY, + tenant_id UUID, + name TEXT NOT NULL, + pattern TEXT NOT NULL, + description TEXT, + category TEXT DEFAULT 'custom', + flags JSONB DEFAULT '[]', + is_system BOOLEAN DEFAULT FALSE, + created_by TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + usage_count INTEGER DEFAULT 0 + ) + """) + + # Regex search history for analytics + await db.execute(""" + CREATE TABLE IF NOT EXISTS arkham_search.regex_history ( + id TEXT PRIMARY KEY, + tenant_id UUID, + pattern TEXT NOT NULL, + flags JSONB DEFAULT '[]', + matches_found INTEGER DEFAULT 0, + documents_searched INTEGER DEFAULT 0, + search_time_ms REAL, + searched_by TEXT, + searched_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + + # Indexes for regex tables + await db.execute(""" + CREATE INDEX IF NOT EXISTS idx_regex_presets_tenant + ON arkham_search.regex_presets(tenant_id) + """) + await db.execute(""" + CREATE INDEX IF NOT EXISTS idx_regex_presets_category + ON arkham_search.regex_presets(category) + """) + await db.execute(""" + CREATE INDEX IF NOT EXISTS idx_regex_history_tenant + ON arkham_search.regex_history(tenant_id) + """) + await db.execute(""" + CREATE INDEX IF NOT EXISTS idx_regex_history_searched_at + ON arkham_search.regex_history(searched_at DESC) + """) + + # Auto-extracted pattern matches table + await db.execute(""" + CREATE TABLE IF NOT EXISTS arkham_search.pattern_extractions ( + id TEXT PRIMARY KEY, + document_id TEXT NOT NULL, + preset_id TEXT NOT NULL, + preset_name TEXT, + category TEXT, + match_text TEXT NOT NULL, + context TEXT, + page_number INTEGER, + chunk_id TEXT, + start_offset INTEGER, + end_offset INTEGER, + line_number INTEGER, + extracted_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + tenant_id UUID + ) + """) + + # Indexes for pattern extractions + await db.execute(""" + CREATE INDEX IF NOT EXISTS idx_pattern_extractions_doc + ON arkham_search.pattern_extractions(document_id) + """) + await db.execute(""" + CREATE INDEX IF NOT EXISTS idx_pattern_extractions_preset + ON arkham_search.pattern_extractions(preset_id) + """) + await db.execute(""" + CREATE INDEX IF NOT EXISTS idx_pattern_extractions_category + ON arkham_search.pattern_extractions(category) + """) + logger.info("Search schema and tables created") except Exception as e: logger.warning(f"Failed to create search schema: {e}") @@ -252,10 +350,142 @@ async def _on_document_deleted(self, event: dict) -> None: - Remove from search indexes - Clean up cache entries """ - doc_id = event.get("doc_id") + doc_id = event.get("doc_id") or event.get("document_id") logger.debug(f"Document deleted: {doc_id}") - # TODO: Remove from any search caches + # Clean up pattern extractions for deleted document + if self._db and doc_id: + try: + await self._db.execute( + "DELETE FROM arkham_search.pattern_extractions WHERE document_id = :doc_id", + {"doc_id": doc_id} + ) + except Exception as e: + logger.warning(f"Failed to delete pattern extractions for {doc_id}: {e}") + + async def _on_parse_completed(self, event: dict) -> None: + """ + Handle parse completion event - auto-extract regex patterns from document. + + Runs all system presets against the newly parsed document and stores + the matches for quick retrieval. + """ + import re + import uuid + + # Handle both wrapped and unwrapped event payloads + payload = event.get("payload", event) + doc_id = payload.get("document_id") + + if not doc_id: + logger.debug("No document_id in parse.document.completed event") + return + + if not self._db or not self.regex_engine: + logger.debug("Database or regex engine not available for pattern extraction") + return + + logger.info(f"Auto-extracting regex patterns for document {doc_id}") + + try: + # Fetch all chunks for this document + chunks = await self._db.fetch_all( + """ + SELECT id, text, chunk_index, page_number + FROM arkham_frame.chunks + WHERE document_id = :doc_id + ORDER BY chunk_index + """, + {"doc_id": doc_id} + ) + + if not chunks: + logger.debug(f"No chunks found for document {doc_id}") + return + + # Get all system presets + from .engines.regex import REGEX_PRESETS + + extractions = [] + for preset in REGEX_PRESETS: + preset_id = preset["id"] + preset_name = preset["name"] + category = preset["category"] + pattern = preset["pattern"] + + try: + compiled = re.compile(pattern, re.IGNORECASE) + except re.error: + continue + + for chunk in chunks: + text = chunk["text"] or "" + chunk_id = chunk["id"] + page_number = chunk["page_number"] + + for match in compiled.finditer(text): + match_text = match.group(0) + match_start = match.start() + match_end = match.end() + + # Extract context + ctx_start = max(0, match_start - 50) + ctx_end = min(len(text), match_end + 50) + context = text[ctx_start:ctx_end] + if ctx_start > 0: + context = "..." + context + if ctx_end < len(text): + context = context + "..." + + line_number = text[:match_start].count('\n') + 1 + + extractions.append({ + "id": str(uuid.uuid4())[:12], + "document_id": doc_id, + "preset_id": preset_id, + "preset_name": preset_name, + "category": category, + "match_text": match_text, + "context": context, + "page_number": page_number, + "chunk_id": chunk_id, + "start_offset": match_start, + "end_offset": match_end, + "line_number": line_number, + }) + + # Store extractions in batch + if extractions: + for ext in extractions: + await self._db.execute( + """ + INSERT INTO arkham_search.pattern_extractions + (id, document_id, preset_id, preset_name, category, match_text, + context, page_number, chunk_id, start_offset, end_offset, line_number) + VALUES (:id, :document_id, :preset_id, :preset_name, :category, :match_text, + :context, :page_number, :chunk_id, :start_offset, :end_offset, :line_number) + ON CONFLICT (id) DO NOTHING + """, + ext + ) + + logger.info(f"Auto-extracted {len(extractions)} pattern matches from document {doc_id}") + + # Emit event for pattern extraction completion + event_bus = self.frame.get_service("events") if self.frame else None + if event_bus: + await event_bus.emit( + "search.patterns.extracted", + { + "document_id": doc_id, + "total_matches": len(extractions), + "categories": list(set(e["category"] for e in extractions)), + }, + source="search-shard", + ) + + except Exception as e: + logger.error(f"Failed to extract patterns for document {doc_id}: {e}", exc_info=True) # --- Public API for other shards --- diff --git a/packages/arkham-shard-shell/src/pages/documents/DeduplicationPanel.css b/packages/arkham-shard-shell/src/pages/documents/DeduplicationPanel.css new file mode 100644 index 0000000..e8ef4e3 --- /dev/null +++ b/packages/arkham-shard-shell/src/pages/documents/DeduplicationPanel.css @@ -0,0 +1,261 @@ +/* DeduplicationPanel Styles */ + +.dedup-panel { + display: flex; + flex-direction: column; + background: var(--surface-primary); + border: 1px solid var(--border-primary); + border-radius: 0.5rem; + overflow: hidden; +} + +/* Header */ +.dedup-header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 1rem; + background: var(--surface-secondary); + border-bottom: 1px solid var(--border-primary); + gap: 1rem; +} + +.dedup-header-title { + display: flex; + align-items: center; + gap: 0.5rem; + min-width: 0; +} + +.dedup-header-title h3 { + margin: 0; + font-size: 0.9375rem; + font-weight: 600; + color: var(--text-primary); +} + +/* Badges */ +.dedup-badge { + padding: 0.125rem 0.5rem; + border-radius: 0.25rem; + font-size: 0.75rem; + font-weight: 500; +} + +.dedup-badge-warning { + background: rgba(245, 158, 11, 0.15); + color: #f59e0b; +} + +.dedup-badge-error { + background: rgba(239, 68, 68, 0.15); + color: #ef4444; +} + +/* Alert */ +.dedup-alert { + display: flex; + align-items: center; + gap: 0.5rem; + padding: 0.75rem 1rem; + font-size: 0.875rem; +} + +.dedup-alert-error { + background: rgba(239, 68, 68, 0.1); + border-bottom: 1px solid rgba(239, 68, 68, 0.2); + color: #ef4444; +} + +/* Sections */ +.dedup-section { + border-bottom: 1px solid var(--border-primary); +} + +.dedup-section:last-of-type { + border-bottom: none; +} + +.dedup-section-header { + width: 100%; + display: flex; + justify-content: space-between; + align-items: center; + padding: 0.75rem 1rem; + background: transparent; + border: none; + cursor: pointer; + transition: background 0.15s; +} + +.dedup-section-header:hover { + background: var(--surface-secondary); +} + +.dedup-section-title { + display: flex; + align-items: center; + gap: 0.5rem; + font-size: 0.875rem; + font-weight: 500; + color: var(--text-primary); +} + +/* Loading/Error/Empty states */ +.dedup-loading, +.dedup-error, +.dedup-empty { + display: flex; + align-items: center; + gap: 0.5rem; + padding: 1rem; + font-size: 0.875rem; + color: var(--text-secondary); +} + +.dedup-error { + color: #ef4444; +} + +.dedup-empty svg { + color: #10b981; +} + +/* Duplicate list */ +.dedup-list { + border-top: 1px solid var(--border-primary); +} + +.dedup-item { + display: flex; + align-items: center; + gap: 0.75rem; + padding: 0.75rem 1rem; + border-bottom: 1px solid var(--border-primary); + transition: background 0.15s; +} + +.dedup-item:last-child { + border-bottom: none; +} + +.dedup-item:hover { + background: var(--surface-secondary); +} + +.dedup-checkbox { + width: 1rem; + height: 1rem; + accent-color: var(--color-primary); + cursor: pointer; +} + +.dedup-item-info { + flex: 1; + min-width: 0; +} + +.dedup-item-title { + font-size: 0.875rem; + font-weight: 500; + color: var(--text-primary); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.dedup-item-meta { + font-size: 0.75rem; + margin-top: 0.125rem; +} + +.dedup-exact { + color: #ef4444; +} + +.dedup-similar { + color: #f59e0b; +} + +/* Threshold control */ +.dedup-threshold { + padding: 0.75rem 1rem; + background: var(--surface-secondary); + border-top: 1px solid var(--border-primary); +} + +.dedup-threshold label { + display: block; + font-size: 0.75rem; + color: var(--text-tertiary); + margin-bottom: 0.375rem; +} + +.dedup-threshold-control { + display: flex; + align-items: center; + gap: 0.75rem; +} + +.dedup-threshold-control input[type="range"] { + flex: 1; + height: 4px; + accent-color: var(--color-primary); +} + +.dedup-threshold-control span { + font-size: 0.875rem; + font-weight: 500; + color: var(--text-primary); + min-width: 2.5rem; + text-align: right; +} + +/* Merge actions */ +.dedup-merge { + display: flex; + align-items: center; + gap: 0.75rem; + padding: 1rem; + background: var(--surface-secondary); + border-top: 1px solid var(--border-primary); + flex-wrap: wrap; +} + +.dedup-merge-count { + flex: 1; + font-size: 0.875rem; + color: var(--text-secondary); + min-width: 120px; +} + +.dedup-select { + padding: 0.375rem 0.75rem; + background: var(--surface-primary); + border: 1px solid var(--border-primary); + border-radius: 0.375rem; + color: var(--text-primary); + font-size: 0.875rem; + cursor: pointer; +} + +.dedup-select:focus { + outline: none; + border-color: var(--color-primary); +} + +/* Reuse btn styles from DocumentViewer.css - btn-primary addition */ +.btn-primary { + background: var(--color-primary); + color: white; + border: none; +} + +.btn-primary:hover { + background: var(--color-primary-hover, #2563eb); +} + +.btn-primary:disabled { + opacity: 0.5; + cursor: not-allowed; +} diff --git a/packages/arkham-shard-shell/src/pages/documents/DeduplicationPanel.tsx b/packages/arkham-shard-shell/src/pages/documents/DeduplicationPanel.tsx new file mode 100644 index 0000000..6c25edc --- /dev/null +++ b/packages/arkham-shard-shell/src/pages/documents/DeduplicationPanel.tsx @@ -0,0 +1,289 @@ +/** + * Deduplication Panel + * + * Shows duplicate detection status and actions for a document. + */ + +import { useState, useCallback } from 'react'; +import { Icon } from '../../components/common/Icon'; +import { + useExactDuplicates, + useSimilarDuplicates, + useComputeHash, + useMergeDuplicates, +} from './api'; +import type { DuplicateResult, MergeRequest } from './api'; +import './DeduplicationPanel.css'; + +interface DeduplicationPanelProps { + documentId: string; + onViewDocument?: (docId: string) => void; + onRefresh?: () => void; +} + +export function DeduplicationPanel({ + documentId, + onViewDocument, + onRefresh, +}: DeduplicationPanelProps) { + const [showExact, setShowExact] = useState(true); + const [showSimilar, setShowSimilar] = useState(true); + const [similarThreshold, setSimilarThreshold] = useState(0.8); + const [selectedDuplicates, setSelectedDuplicates] = useState>(new Set()); + const [mergeStrategy, setMergeStrategy] = useState('soft_delete'); + + const { computeHash, loading: hashLoading, error: hashError } = useComputeHash(); + const { merge, loading: mergeLoading } = useMergeDuplicates(); + + const { + data: exactDuplicates, + loading: exactLoading, + error: exactError, + refetch: refetchExact, + } = useExactDuplicates(documentId); + + const { + data: similarDuplicates, + loading: similarLoading, + error: similarError, + refetch: refetchSimilar, + } = useSimilarDuplicates(documentId, similarThreshold); + + const handleComputeHash = useCallback(async () => { + try { + await computeHash(documentId); + refetchExact(); + refetchSimilar(); + onRefresh?.(); + } catch { + // Error handled by hook + } + }, [documentId, computeHash, refetchExact, refetchSimilar, onRefresh]); + + const toggleDuplicateSelection = useCallback((dupId: string) => { + setSelectedDuplicates((prev) => { + const next = new Set(prev); + if (next.has(dupId)) { + next.delete(dupId); + } else { + next.add(dupId); + } + return next; + }); + }, []); + + const handleMerge = useCallback(async () => { + if (selectedDuplicates.size === 0) return; + + try { + await merge({ + source_ids: Array.from(selectedDuplicates), + target_id: documentId, + strategy: mergeStrategy, + }); + setSelectedDuplicates(new Set()); + refetchExact(); + refetchSimilar(); + onRefresh?.(); + } catch { + // Error handled by hook + } + }, [selectedDuplicates, documentId, mergeStrategy, merge, refetchExact, refetchSimilar, onRefresh]); + + const renderDuplicateList = ( + duplicates: DuplicateResult[] | undefined, + loading: boolean, + error: Error | null, + type: 'exact' | 'similar' + ) => { + if (loading) { + return ( +
+ + Finding {type} duplicates... +
+ ); + } + + if (error) { + return ( +
+ + {error.message} +
+ ); + } + + if (!duplicates || duplicates.length === 0) { + return ( +
+ + No {type} duplicates found +
+ ); + } + + return ( +
+ {duplicates.map((dup) => ( +
+ toggleDuplicateSelection(dup.document_id)} + className="dedup-checkbox" + /> + +
+
{dup.title}
+
+ {type === 'exact' ? ( + Exact match (100%) + ) : ( + + {(dup.similarity_score * 100).toFixed(1)}% similar + (hamming: {dup.hamming_distance}) + + )} +
+
+ +
+ ))} +
+ ); + }; + + const totalDuplicates = + (exactDuplicates?.length || 0) + (similarDuplicates?.length || 0); + + return ( +
+ {/* Header */} +
+
+ +

Deduplication

+ {totalDuplicates > 0 && ( + + {totalDuplicates} found + + )} +
+ +
+ + {hashError && ( +
+ + {hashError.message} +
+ )} + + {/* Exact Duplicates Section */} +
+ + {showExact && renderDuplicateList(exactDuplicates ?? undefined, exactLoading, exactError, 'exact')} +
+ + {/* Similar Duplicates Section */} +
+ + {showSimilar && ( + <> +
+ +
+ setSimilarThreshold(parseFloat(e.target.value))} + /> + {(similarThreshold * 100).toFixed(0)}% +
+
+ {renderDuplicateList(similarDuplicates ?? undefined, similarLoading, similarError, 'similar')} + + )} +
+ + {/* Merge Actions */} + {selectedDuplicates.size > 0 && ( +
+ + {selectedDuplicates.size} selected for merge + + + +
+ )} +
+ ); +} + +export default DeduplicationPanel; diff --git a/packages/arkham-shard-shell/src/pages/documents/DocumentViewer.tsx b/packages/arkham-shard-shell/src/pages/documents/DocumentViewer.tsx index 4c7664e..17305b8 100644 --- a/packages/arkham-shard-shell/src/pages/documents/DocumentViewer.tsx +++ b/packages/arkham-shard-shell/src/pages/documents/DocumentViewer.tsx @@ -14,6 +14,7 @@ import { type DocumentChunk, type DocumentEntity, } from './api'; +import { DeduplicationPanel } from './DeduplicationPanel'; import './DocumentViewer.css'; interface DocumentViewerProps { @@ -21,7 +22,7 @@ interface DocumentViewerProps { onClose: () => void; } -type ViewerTab = 'content' | 'chunks' | 'entities' | 'metadata'; +type ViewerTab = 'content' | 'chunks' | 'entities' | 'metadata' | 'duplicates'; const ENTITY_TYPE_COLORS: Record = { PERSON: '#3b82f6', @@ -130,11 +131,29 @@ export function DocumentViewer({ documentId, onClose }: DocumentViewerProps) { return renderEntitiesTab(); case 'metadata': return renderMetadataTab(); + case 'duplicates': + return renderDuplicatesTab(); default: return null; } }; + const renderDuplicatesTab = () => { + return ( + { + // Could navigate to the duplicate document + console.log('View duplicate document:', docId); + }} + onRefresh={() => { + // Refresh document data after merge + refetchContent(); + }} + /> + ); + }; + const renderContentTab = () => { if (loadingContent) { return ( @@ -510,6 +529,13 @@ export function DocumentViewer({ documentId, onClose }: DocumentViewerProps) { Metadata +
{renderTabContent()}
diff --git a/packages/arkham-shard-shell/src/pages/documents/api.ts b/packages/arkham-shard-shell/src/pages/documents/api.ts index aa22678..44a4370 100644 --- a/packages/arkham-shard-shell/src/pages/documents/api.ts +++ b/packages/arkham-shard-shell/src/pages/documents/api.ts @@ -90,6 +90,37 @@ export interface BatchResult { details?: Array<{ id: string; success?: boolean; error?: string }>; } +// --- Deduplication Types --- + +export interface DocumentHash { + document_id: string; + content_md5: string; + content_sha256: string; + simhash: number; + text_length: number; +} + +export interface DuplicateResult { + document_id: string; + title: string; + similarity_score: number; + hamming_distance: number; + match_type: 'exact' | 'similar'; +} + +export interface DeduplicationStats { + total_documents: number; + documents_with_hash: number; + unique_content_hashes: number; + potential_duplicates: number; +} + +export interface MergeRequest { + source_ids: string[]; + target_id: string; + strategy: 'soft_delete' | 'archive' | 'permanent_delete'; +} + // --- API Functions --- async function fetchAPI(endpoint: string, options?: RequestInit): Promise { @@ -187,6 +218,51 @@ export async function batchDeleteDocuments(documentIds: string[]): Promise { + return fetchAPI(`/${documentId}/compute-hash`, { + method: 'POST', + }); +} + +export async function findExactDuplicates(documentId: string): Promise { + return fetchAPI(`/${documentId}/duplicates/exact`); +} + +export async function findSimilarDuplicates( + documentId: string, + threshold: number = 0.8 +): Promise { + return fetchAPI(`/${documentId}/duplicates/similar?threshold=${threshold}`); +} + +export async function getDeduplicationStats(): Promise { + return fetchAPI('/deduplication/stats'); +} + +export async function scanForDuplicates( + projectId: string, + similarityThreshold: number = 0.8 +): Promise { + return fetchAPI( + `/deduplication/scan?project_id=${projectId}`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ similarity_threshold: similarityThreshold }), + } + ); +} + +export async function mergeDuplicates(request: MergeRequest): Promise { + return fetchAPI('/deduplication/merge', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(request), + }); +} + // --- Hooks --- /** @@ -319,3 +395,80 @@ export function useBatchDelete() { return { deleteDocuments, loading, error }; } + +// --- Deduplication Hooks --- + +/** + * Hook for fetching deduplication stats + */ +export function useDeduplicationStats() { + return useFetch(`${API_PREFIX}/deduplication/stats`); +} + +/** + * Hook for finding exact duplicates of a document + */ +export function useExactDuplicates(documentId: string | null) { + const url = documentId ? `${API_PREFIX}/${documentId}/duplicates/exact` : null; + return useFetch(url); +} + +/** + * Hook for finding similar duplicates of a document + */ +export function useSimilarDuplicates(documentId: string | null, threshold: number = 0.8) { + const url = documentId + ? `${API_PREFIX}/${documentId}/duplicates/similar?threshold=${threshold}` + : null; + return useFetch(url); +} + +/** + * Hook for computing document hash + */ +export function useComputeHash() { + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + const computeHash = useCallback(async (documentId: string) => { + setLoading(true); + setError(null); + try { + const result = await computeDocumentHash(documentId); + return result; + } catch (err) { + const error = err instanceof Error ? err : new Error('Hash computation failed'); + setError(error); + throw error; + } finally { + setLoading(false); + } + }, []); + + return { computeHash, loading, error }; +} + +/** + * Hook for merging duplicates + */ +export function useMergeDuplicates() { + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + const merge = useCallback(async (request: MergeRequest) => { + setLoading(true); + setError(null); + try { + const result = await mergeDuplicates(request); + return result; + } catch (err) { + const error = err instanceof Error ? err : new Error('Merge failed'); + setError(error); + throw error; + } finally { + setLoading(false); + } + }, []); + + return { merge, loading, error }; +} diff --git a/packages/arkham-shard-shell/src/pages/search/RegexSearchPanel.css b/packages/arkham-shard-shell/src/pages/search/RegexSearchPanel.css new file mode 100644 index 0000000..bc94c57 --- /dev/null +++ b/packages/arkham-shard-shell/src/pages/search/RegexSearchPanel.css @@ -0,0 +1,447 @@ +/* RegexSearchPanel Styles */ + +.regex-panel { + display: flex; + flex-direction: column; + gap: 1rem; + height: 100%; +} + +/* Input Section */ +.regex-input-section { + display: flex; + flex-direction: column; + gap: 0.75rem; + padding: 1rem; + background: var(--surface-secondary); + border: 1px solid var(--border-primary); + border-radius: 0.5rem; +} + +.regex-input-row { + display: flex; + gap: 0.75rem; + align-items: center; +} + +.regex-input-wrapper { + flex: 1; + position: relative; + display: flex; + align-items: center; +} + +.regex-input { + width: 100%; + padding: 0.625rem 2.5rem 0.625rem 0.75rem; + background: var(--surface-primary); + border: 1px solid var(--border-primary); + border-radius: 0.375rem; + color: var(--text-primary); + font-family: monospace; + font-size: 0.875rem; +} + +.regex-input:focus { + outline: none; + border-color: var(--color-primary); +} + +.regex-input::placeholder { + color: var(--text-tertiary); +} + +.regex-input-icon { + position: absolute; + right: 0.75rem; + display: flex; + align-items: center; +} + +/* Validation Feedback */ +.regex-validation { + padding: 0.5rem 0; +} + +.regex-validation-msg { + display: flex; + align-items: center; + gap: 0.5rem; + font-size: 0.8125rem; +} + +.regex-validation-warning { + color: var(--text-tertiary); + font-size: 0.75rem; +} + +/* Performance indicators */ +.regex-perf-fast { + color: #10b981; +} + +.regex-perf-moderate { + color: #f59e0b; +} + +.regex-perf-slow { + color: #f97316; +} + +.regex-perf-dangerous { + color: #ef4444; +} + +.regex-perf-invalid { + color: #ef4444; +} + +/* Flags */ +.regex-flags { + display: flex; + align-items: center; + gap: 0.5rem; +} + +.regex-flags-label { + font-size: 0.8125rem; + color: var(--text-secondary); +} + +.regex-flag-btn { + padding: 0.25rem 0.625rem; + background: var(--surface-primary); + border: 1px solid var(--border-primary); + border-radius: 0.25rem; + color: var(--text-secondary); + font-family: monospace; + font-size: 0.8125rem; + font-weight: 500; + cursor: pointer; + transition: all 0.15s; +} + +.regex-flag-btn:hover { + background: var(--surface-secondary); + border-color: var(--text-tertiary); +} + +.regex-flag-btn.active { + background: var(--color-primary); + border-color: var(--color-primary); + color: white; +} + +/* Presets Section */ +.regex-presets-section { + background: var(--surface-secondary); + border: 1px solid var(--border-primary); + border-radius: 0.5rem; + overflow: hidden; +} + +.regex-presets-toggle { + width: 100%; + display: flex; + justify-content: space-between; + align-items: center; + padding: 0.75rem 1rem; + background: transparent; + border: none; + color: var(--text-primary); + font-size: 0.875rem; + font-weight: 500; + cursor: pointer; + transition: background 0.15s; +} + +.regex-presets-toggle:hover { + background: var(--surface-primary); +} + +.regex-presets-content { + padding: 0.75rem 1rem; + border-top: 1px solid var(--border-primary); + display: flex; + flex-direction: column; + gap: 1rem; +} + +.regex-presets-loading { + padding: 1rem; + text-align: center; + color: var(--text-secondary); + font-size: 0.875rem; +} + +.regex-preset-category { + display: flex; + flex-direction: column; + gap: 0.5rem; +} + +.regex-preset-category-header { + display: flex; + align-items: center; + gap: 0.375rem; + font-size: 0.75rem; + font-weight: 500; + color: var(--text-tertiary); + text-transform: uppercase; + letter-spacing: 0.025em; +} + +.regex-preset-list { + display: flex; + flex-wrap: wrap; + gap: 0.375rem; +} + +.regex-preset-btn { + padding: 0.375rem 0.75rem; + background: var(--surface-primary); + border: 1px solid var(--border-primary); + border-radius: 0.375rem; + color: var(--text-primary); + font-size: 0.8125rem; + cursor: pointer; + transition: all 0.15s; +} + +.regex-preset-btn:hover { + background: var(--color-primary); + border-color: var(--color-primary); + color: white; +} + +/* Results Section */ +.regex-results { + flex: 1; + display: flex; + flex-direction: column; + background: var(--surface-secondary); + border: 1px solid var(--border-primary); + border-radius: 0.5rem; + overflow: hidden; + min-height: 200px; +} + +.regex-results-loading { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 0.75rem; + padding: 2rem; + color: var(--text-secondary); +} + +.regex-results-error { + display: flex; + align-items: center; + gap: 0.5rem; + padding: 1rem; + color: #ef4444; + font-size: 0.875rem; +} + +.regex-results-header { + display: flex; + justify-content: space-between; + align-items: center; + padding: 0.75rem 1rem; + background: var(--surface-primary); + border-bottom: 1px solid var(--border-primary); + font-size: 0.875rem; + color: var(--text-secondary); +} + +.regex-results-header strong { + color: var(--text-primary); +} + +.regex-results-empty { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 0.5rem; + padding: 2rem; + color: var(--text-tertiary); + text-align: center; +} + +.regex-results-empty p { + margin: 0; + font-size: 0.875rem; +} + +.regex-results-hint { + font-size: 0.75rem !important; + color: var(--text-tertiary); +} + +/* Match List */ +.regex-match-list { + flex: 1; + overflow-y: auto; +} + +.regex-match-item { + display: flex; + gap: 0.75rem; + padding: 0.75rem 1rem; + border-bottom: 1px solid var(--border-primary); + cursor: pointer; + transition: background 0.15s; +} + +.regex-match-item:last-child { + border-bottom: none; +} + +.regex-match-item:hover { + background: var(--surface-primary); +} + +.regex-match-item > svg { + flex-shrink: 0; + color: var(--text-tertiary); + margin-top: 0.125rem; +} + +.regex-match-content { + flex: 1; + min-width: 0; +} + +.regex-match-header { + display: flex; + align-items: center; + gap: 0.5rem; + flex-wrap: wrap; + margin-bottom: 0.375rem; +} + +.regex-match-title { + font-size: 0.875rem; + font-weight: 500; + color: var(--text-primary); +} + +.regex-match-badge { + padding: 0.125rem 0.375rem; + background: var(--surface-secondary); + border-radius: 0.25rem; + font-size: 0.6875rem; + color: var(--text-tertiary); +} + +.regex-match-context { + font-family: monospace; + font-size: 0.8125rem; + color: var(--text-secondary); + line-height: 1.5; + word-break: break-word; +} + +.regex-context-text { + color: var(--text-secondary); +} + +.regex-match-highlight { + background: rgba(59, 130, 246, 0.3); + color: #60a5fa; + padding: 0.0625rem 0.125rem; + border-radius: 0.125rem; +} + +/* Spinner animation */ +.spin { + animation: spin 1s linear infinite; +} + +@keyframes spin { + from { + transform: rotate(0deg); + } + to { + transform: rotate(360deg); + } +} + +/* Button styles - reusing common patterns */ +.btn { + display: inline-flex; + align-items: center; + gap: 0.375rem; + padding: 0.5rem 1rem; + border-radius: 0.375rem; + font-size: 0.875rem; + font-weight: 500; + cursor: pointer; + transition: all 0.15s; +} + +.btn:disabled { + opacity: 0.5; + cursor: not-allowed; +} + +.btn-primary { + background: var(--color-primary); + border: none; + color: white; +} + +.btn-primary:hover:not(:disabled) { + background: var(--color-primary-hover, #2563eb); +} + +/* Detected Patterns Section */ +.regex-detected-badge { + padding: 0.125rem 0.5rem; + background: var(--color-primary); + border-radius: 0.75rem; + font-size: 0.75rem; + font-weight: 500; + color: white; + margin-left: auto; + margin-right: 0.5rem; +} + +.regex-detected-empty { + display: flex; + align-items: center; + gap: 0.5rem; + padding: 0.75rem; + color: var(--text-tertiary); + font-size: 0.8125rem; +} + +.regex-detected-count { + margin-left: 0.375rem; + padding: 0.0625rem 0.375rem; + background: rgba(255, 255, 255, 0.15); + border-radius: 0.25rem; + font-size: 0.6875rem; +} + +/* Status indicators for detected patterns */ +.regex-detected-status-detected { + border-left: 2px solid #f59e0b; +} + +.regex-detected-status-confirmed { + border-left: 2px solid #10b981; +} + +.regex-detected-status-dismissed { + opacity: 0.5; + border-left: 2px solid var(--border-primary); +} + +.regex-detected-status-archived { + opacity: 0.6; + border-left: 2px solid var(--text-tertiary); +} diff --git a/packages/arkham-shard-shell/src/pages/search/RegexSearchPanel.tsx b/packages/arkham-shard-shell/src/pages/search/RegexSearchPanel.tsx new file mode 100644 index 0000000..2eb5107 --- /dev/null +++ b/packages/arkham-shard-shell/src/pages/search/RegexSearchPanel.tsx @@ -0,0 +1,398 @@ +/** + * Regex Search Panel + * + * A dedicated panel for regex pattern searching with preset support. + */ + +import React, { useState, useEffect, useCallback } from 'react'; +import { Icon } from '../../components/common/Icon'; +import { useRegexSearch, useRegexPresets, useDetectedPatterns, validatePattern } from './api'; +import type { RegexPreset, RegexMatch, PatternValidation, DetectedPattern } from './types'; +import './RegexSearchPanel.css'; + +interface RegexSearchPanelProps { + projectId?: string; + onMatchClick?: (match: RegexMatch) => void; +} + +const CATEGORY_ICONS: Record = { + contact: 'Mail', + pii: 'Shield', + financial: 'DollarSign', + technical: 'Globe', + temporal: 'Calendar', + custom: 'Hash', +}; + +const PATTERN_TYPE_ICONS: Record = { + recurring_theme: 'Repeat', + behavioral: 'Activity', + temporal: 'Clock', + correlation: 'Link', + linguistic: 'Type', + structural: 'Layout', + custom: 'Settings', +}; + +export default function RegexSearchPanel({ projectId, onMatchClick }: RegexSearchPanelProps) { + const [pattern, setPattern] = useState(''); + const [searchPattern, setSearchPattern] = useState(''); + const [flags, setFlags] = useState([]); + const [showPresets, setShowPresets] = useState(true); + const [showDetected, setShowDetected] = useState(true); + const [validation, setValidation] = useState(null); + const [validating, setValidating] = useState(false); + + const { data: presets, loading: presetsLoading } = useRegexPresets(); + const { data: detectedPatterns, loading: detectedLoading } = useDetectedPatterns({ limit: 50 }); + const { + data: searchResults, + loading: searchLoading, + error: searchError, + } = useRegexSearch({ + pattern: searchPattern, + request: { flags, project_id: projectId, limit: 100 }, + enabled: !!searchPattern, // Allow search even without validation - backend validates + }); + + // Validate pattern on change (debounced) + useEffect(() => { + if (!pattern) { + setValidation(null); + return; + } + + const timer = setTimeout(async () => { + setValidating(true); + try { + const result = await validatePattern(pattern); + setValidation(result); + } catch { + setValidation({ valid: false, error: 'Validation failed', estimated_performance: 'invalid' }); + } finally { + setValidating(false); + } + }, 300); + + return () => clearTimeout(timer); + }, [pattern]); + + const handleSearch = useCallback(() => { + // Allow search even if validation hasn't completed yet - backend will validate + if (pattern.trim()) { + setSearchPattern(pattern); + } + }, [pattern]); + + const handlePresetClick = useCallback((preset: RegexPreset) => { + setPattern(preset.pattern); + setFlags(preset.flags || []); + }, []); + + const handleDetectedPatternClick = useCallback((detected: DetectedPattern) => { + // Use the first regex pattern from criteria if available + if (detected.criteria?.regex_patterns?.length) { + setPattern(detected.criteria.regex_patterns[0]); + setFlags([]); + } else if (detected.criteria?.keywords?.length) { + // Fall back to keyword as a simple pattern + setPattern(detected.criteria.keywords.join('|')); + setFlags(['case_insensitive']); + } + }, []); + + const handleKeyDown = useCallback( + (e: React.KeyboardEvent) => { + if (e.key === 'Enter' && pattern.trim()) { + handleSearch(); + } + }, + [handleSearch, pattern] + ); + + const toggleFlag = useCallback((flag: string) => { + setFlags((prev) => (prev.includes(flag) ? prev.filter((f) => f !== flag) : [...prev, flag])); + }, []); + + // Group presets by category + const presetsByCategory = presets.reduce((acc, preset) => { + const cat = preset.category || 'custom'; + if (!acc[cat]) acc[cat] = []; + acc[cat].push(preset); + return acc; + }, {} as Record); + + // Filter and group detected patterns that have searchable criteria + const searchableDetectedPatterns = detectedPatterns.filter( + (p) => p.criteria?.regex_patterns?.length || p.criteria?.keywords?.length + ); + + // Group detected patterns by type + const detectedByType = searchableDetectedPatterns.reduce((acc, pattern) => { + const type = pattern.pattern_type || 'custom'; + if (!acc[type]) acc[type] = []; + acc[type].push(pattern); + return acc; + }, {} as Record); + + const getPerformanceClass = (perf: string) => { + switch (perf) { + case 'fast': return 'regex-perf-fast'; + case 'moderate': return 'regex-perf-moderate'; + case 'slow': return 'regex-perf-slow'; + case 'dangerous': return 'regex-perf-dangerous'; + default: return 'regex-perf-invalid'; + } + }; + + return ( +
+ {/* Pattern Input */} +
+
+
+ setPattern(e.target.value)} + onKeyDown={handleKeyDown} + placeholder="Enter regex pattern (e.g., \d{3}-\d{4})" + className="regex-input" + /> + {validating && ( +
+ +
+ )} + {!validating && validation && ( +
+ {validation.valid ? ( + + ) : ( + + )} +
+ )} +
+ +
+ + {/* Validation Feedback */} + {validation && ( +
+ {validation.valid ? ( +
+ + Performance: {validation.estimated_performance} + {validation.estimated_performance === 'dangerous' && ( + + (may cause slowdowns on large datasets) + + )} +
+ ) : ( +
+ + {validation.error} +
+ )} +
+ )} + + {/* Regex Flags */} +
+ Flags: + {[ + { id: 'case_insensitive', label: 'Case Insensitive', short: 'i' }, + { id: 'multiline', label: 'Multiline', short: 'm' }, + { id: 'dotall', label: 'Dot All', short: 's' }, + ].map((flag) => ( + + ))} +
+
+ + {/* Presets Section */} +
+ + + {showPresets && ( +
+ {presetsLoading ? ( +
Loading presets...
+ ) : ( + Object.entries(presetsByCategory).map(([category, categoryPresets]) => ( +
+
+ + {category} +
+
+ {categoryPresets.map((preset) => ( + + ))} +
+
+ )) + )} +
+ )} +
+ + {/* Detected Patterns from Ingest Section */} +
+ + + {showDetected && ( +
+ {detectedLoading ? ( +
Loading detected patterns...
+ ) : searchableDetectedPatterns.length === 0 ? ( +
+ + No patterns detected yet. Process documents to auto-detect patterns. +
+ ) : ( + Object.entries(detectedByType).map(([type, typePatterns]) => ( +
+
+ + {type.replace(/_/g, ' ')} +
+
+ {typePatterns.map((detected) => ( + + ))} +
+
+ )) + )} +
+ )} +
+ + {/* Results Section */} +
+ {searchLoading && ( +
+ + Searching... +
+ )} + + {searchError && ( +
+ + {searchError.message} +
+ )} + + {searchResults && !searchLoading && ( + <> + {/* Results Header */} +
+ + Found {searchResults.total_matches} matches + in {searchResults.documents_searched} documents + + {searchResults.duration_ms.toFixed(1)}ms +
+ + {/* Match List */} +
+ {searchResults.matches.length === 0 ? ( +
+ +

No matches found for this pattern

+
+ ) : ( + searchResults.matches.map((match, index) => ( +
onMatchClick?.(match)} + > + +
+
+ {match.document_title} + {match.page_number && ( + Page {match.page_number} + )} + Line {match.line_number} +
+
+ {match.context.split(match.match_text).map((part, i, arr) => ( + + {part} + {i < arr.length - 1 && ( + {match.match_text} + )} + + ))} +
+
+
+ )) + )} +
+ + )} + + {!searchResults && !searchLoading && !searchError && ( +
+ +

Enter a regex pattern and click Search

+

Or select a preset pattern above

+
+ )} +
+
+ ); +} diff --git a/packages/arkham-shard-shell/src/pages/search/SearchPage.tsx b/packages/arkham-shard-shell/src/pages/search/SearchPage.tsx index 17fa5e0..f39bf3e 100644 --- a/packages/arkham-shard-shell/src/pages/search/SearchPage.tsx +++ b/packages/arkham-shard-shell/src/pages/search/SearchPage.tsx @@ -11,8 +11,9 @@ import { AIAnalystButton } from '../../components/AIAnalyst'; import { useToast } from '../../context/ToastContext'; import { SearchResultCard } from './SearchResultCard'; import { RAGChatPanel } from './RAGChatPanel'; +import RegexSearchPanel from './RegexSearchPanel'; import { useSearch, findSimilar } from './api'; -import type { SearchMode, SearchFilters, SearchResultItem } from './types'; +import type { SearchMode, SearchFilters, SearchResultItem, RegexMatch } from './types'; export function SearchPage() { const { toast } = useToast(); @@ -215,6 +216,13 @@ export function SearchPage() { Keyword + {/* Filters Toggle */} @@ -344,8 +352,20 @@ export function SearchPage() { )} - {/* Results Section */} - {queryFromUrl && ( + {/* Regex Search Panel - Shown when regex mode is selected */} + {searchMode === 'regex' && ( +
+ { + navigate(`/documents/${match.document_id}`); + }} + /> +
+ )} + + {/* Results Section - Shown for non-regex modes */} + {searchMode !== 'regex' && queryFromUrl && (
{loading && (
diff --git a/packages/arkham-shard-shell/src/pages/search/api.ts b/packages/arkham-shard-shell/src/pages/search/api.ts index 8d057da..3a35276 100644 --- a/packages/arkham-shard-shell/src/pages/search/api.ts +++ b/packages/arkham-shard-shell/src/pages/search/api.ts @@ -11,6 +11,13 @@ import type { Suggestion, SimilarResponse, AvailableFilters, + RegexPreset, + RegexSearchRequest, + RegexSearchResponse, + PatternValidation, + PatternExtraction, + ExtractionStats, + DetectedPattern, } from './types'; const API_PREFIX = '/api/search'; @@ -107,6 +114,112 @@ export async function getAvailableFilters(query?: string): Promise { + return fetchAPI('/regex', { + method: 'POST', + body: JSON.stringify(request), + }); +} + +export async function validatePattern(pattern: string): Promise { + return fetchAPI('/regex/validate', { + method: 'POST', + body: JSON.stringify({ pattern }), + }); +} + +export async function getRegexPresets(category?: string): Promise { + const params = new URLSearchParams(); + if (category) params.set('category', category); + return fetchAPI(`/regex/presets?${params.toString()}`); +} + +export async function createRegexPreset(preset: { + name: string; + pattern: string; + description?: string; + category?: string; + flags?: string[]; +}): Promise { + return fetchAPI('/regex/presets', { + method: 'POST', + body: JSON.stringify(preset), + }); +} + +export async function deleteRegexPreset(presetId: string): Promise<{ success: boolean }> { + return fetchAPI<{ success: boolean }>(`/regex/presets/${presetId}`, { + method: 'DELETE', + }); +} + +export async function getPatternExtractions(params: { + document_id?: string; + preset_id?: string; + category?: string; + limit?: number; + offset?: number; +}): Promise<{ extractions: PatternExtraction[]; total: number }> { + const searchParams = new URLSearchParams(); + if (params.document_id) searchParams.set('document_id', params.document_id); + if (params.preset_id) searchParams.set('preset_id', params.preset_id); + if (params.category) searchParams.set('category', params.category); + if (params.limit) searchParams.set('limit', params.limit.toString()); + if (params.offset) searchParams.set('offset', params.offset.toString()); + return fetchAPI<{ extractions: PatternExtraction[]; total: number }>( + `/regex/extractions?${searchParams.toString()}` + ); +} + +export async function getExtractionStats(): Promise { + return fetchAPI('/regex/extractions/stats'); +} + +export async function triggerPatternExtraction( + documentId: string, + presetIds?: string[] +): Promise<{ document_id: string; extractions: number; presets_used: string[] }> { + const body = presetIds ? JSON.stringify({ preset_ids: presetIds }) : '{}'; + return fetchAPI<{ document_id: string; extractions: number; presets_used: string[] }>( + `/regex/extract/${documentId}`, + { + method: 'POST', + body, + } + ); +} + +// ============================================ +// Detected Patterns from Patterns Shard +// ============================================ + +export async function getDetectedPatterns(params?: { + status?: string; + pattern_type?: string; + has_regex?: boolean; + limit?: number; +}): Promise<{ items: DetectedPattern[]; total: number }> { + const searchParams = new URLSearchParams(); + if (params?.status) searchParams.set('status', params.status); + if (params?.pattern_type) searchParams.set('type', params.pattern_type); + if (params?.limit) searchParams.set('page_size', params.limit.toString()); + + const response = await fetch(`/api/patterns/?${searchParams.toString()}`, { + headers: { 'Content-Type': 'application/json' }, + }); + + if (!response.ok) { + const error = await response.json().catch(() => ({ detail: response.statusText })); + throw new Error(error.detail || `HTTP ${response.status}`); + } + + return response.json(); +} + // ============================================ // React Hooks // ============================================ @@ -251,6 +364,117 @@ export function useKeyword(options: UseSearchOptions): UseSearchResult { return { data, loading, error, refetch }; } +// ============================================ +// Regex Search Hooks +// ============================================ + +interface UseRegexSearchOptions { + pattern: string; + request?: Partial; + enabled?: boolean; +} + +interface UseRegexSearchResult { + data: RegexSearchResponse | null; + loading: boolean; + error: Error | null; + refetch: () => void; +} + +export function useRegexSearch(options: UseRegexSearchOptions): UseRegexSearchResult { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + const requestJson = JSON.stringify(options.request || {}); + + const fetchData = useCallback(async () => { + if (!options.pattern || options.enabled === false) { + setLoading(false); + return; + } + + setLoading(true); + setError(null); + + try { + const request = JSON.parse(requestJson); + const result = await performRegexSearch({ + pattern: options.pattern, + ...request, + }); + setData(result); + setError(null); + } catch (err) { + setError(err instanceof Error ? err : new Error('Unknown error')); + } finally { + setLoading(false); + } + }, [options.pattern, requestJson, options.enabled]); + + useEffect(() => { + fetchData(); + }, [fetchData]); + + const refetch = useCallback(() => { + fetchData(); + }, [fetchData]); + + return { data, loading, error, refetch }; +} + +export function useRegexPresets(category?: string) { + const [data, setData] = useState([]); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + const fetchData = useCallback(async () => { + setLoading(true); + setError(null); + + try { + const result = await getRegexPresets(category); + setData(result); + } catch (err) { + setError(err instanceof Error ? err : new Error('Unknown error')); + } finally { + setLoading(false); + } + }, [category]); + + useEffect(() => { + fetchData(); + }, [fetchData]); + + return { data, loading, error, refetch: fetchData }; +} + +export function useExtractionStats() { + const [data, setData] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + const fetchData = useCallback(async () => { + setLoading(true); + setError(null); + + try { + const result = await getExtractionStats(); + setData(result); + } catch (err) { + setError(err instanceof Error ? err : new Error('Unknown error')); + } finally { + setLoading(false); + } + }, []); + + useEffect(() => { + fetchData(); + }, [fetchData]); + + return { data, loading, error, refetch: fetchData }; +} + interface UseSimilarResult { data: SimilarResponse | null; loading: boolean; @@ -293,3 +517,38 @@ export function useSimilar(docId: string | null, limit = 10): UseSimilarResult { return { data, loading, error, refetch }; } + +export function useDetectedPatterns(params?: { + status?: string; + pattern_type?: string; + has_regex?: boolean; + limit?: number; +}) { + const [data, setData] = useState([]); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + const paramsJson = JSON.stringify(params || {}); + + const fetchData = useCallback(async () => { + setLoading(true); + setError(null); + + try { + const parsedParams = JSON.parse(paramsJson); + const result = await getDetectedPatterns(parsedParams); + setData(result.items || []); + } catch (err) { + setError(err instanceof Error ? err : new Error('Unknown error')); + setData([]); + } finally { + setLoading(false); + } + }, [paramsJson]); + + useEffect(() => { + fetchData(); + }, [fetchData]); + + return { data, loading, error, refetch: fetchData }; +} diff --git a/packages/arkham-shard-shell/src/pages/search/types.ts b/packages/arkham-shard-shell/src/pages/search/types.ts index d578e98..ebe0827 100644 --- a/packages/arkham-shard-shell/src/pages/search/types.ts +++ b/packages/arkham-shard-shell/src/pages/search/types.ts @@ -5,7 +5,7 @@ */ // Search modes -export type SearchMode = 'hybrid' | 'semantic' | 'keyword'; +export type SearchMode = 'hybrid' | 'semantic' | 'keyword' | 'regex'; // Sort options export type SortBy = 'relevance' | 'date' | 'title'; @@ -83,3 +83,96 @@ export interface AvailableFilters { projects?: { value: string; label: string; count: number }[]; date_range?: { min: string; max: string }; } + +// Regex search types +export interface RegexPreset { + id: string; + name: string; + pattern: string; + description: string; + category: string; + flags: string[]; + is_system: boolean; +} + +export interface RegexMatch { + document_id: string; + document_title: string; + page_number: number | null; + chunk_id: string; + match_text: string; + context: string; + start_offset: number; + end_offset: number; + line_number: number; +} + +export interface RegexSearchRequest { + pattern: string; + flags?: string[]; + project_id?: string; + document_ids?: string[]; + limit?: number; + offset?: number; + context_chars?: number; +} + +export interface RegexSearchResponse { + pattern: string; + matches: RegexMatch[]; + total_matches: number; + total_chunks_with_matches: number; + documents_searched: number; + duration_ms: number; + error: string | null; +} + +export interface PatternValidation { + valid: boolean; + error: string | null; + estimated_performance: 'fast' | 'moderate' | 'slow' | 'dangerous' | 'invalid'; +} + +export interface PatternExtraction { + id: string; + document_id: string; + preset_id: string; + preset_name: string | null; + category: string | null; + match_text: string; + context: string | null; + page_number: number | null; + chunk_id: string | null; + start_offset: number | null; + end_offset: number | null; + line_number: number | null; + extracted_at: string | null; +} + +export interface ExtractionStats { + total_extractions: number; + documents_with_patterns: number; + by_category: Record; + by_preset: Record; +} + +// Detected pattern from patterns shard (auto-extracted during ingest) +export interface DetectedPattern { + id: string; + name: string; + description: string | null; + pattern_type: string; + status: 'detected' | 'confirmed' | 'dismissed' | 'archived'; + confidence: number; + match_count: number; + document_count: number; + first_detected: string | null; + last_matched: string | null; + detection_method: string; + criteria: { + keywords?: string[]; + regex_patterns?: string[]; + entity_types?: string[]; + min_occurrences?: number; + } | null; +} From 89a6bc6863003c34598a8ccdd30056af617d0814 Mon Sep 17 00:00:00 2001 From: mantisfury Date: Wed, 14 Jan 2026 18:28:39 -0700 Subject: [PATCH 2/4] Fix: Deduplication merge API and add MERGED document status - Fix frontend MergeRequest to match backend API schema: - source_ids -> duplicate_ids - target_id -> primary_id - strategy -> cleanup_action - Add MERGED status to DocumentStatus enums in both frame and documents shard - Update DeduplicationPanel dropdown options to match backend cleanup_action values Co-Authored-By: Claude Opus 4.5 --- .../arkham_frame/services/documents.py | 21 ++++++++++++------- .../arkham_shard_documents/models.py | 1 + .../pages/documents/DeduplicationPanel.tsx | 14 +++++++------ .../src/pages/documents/api.ts | 8 ++++--- 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/packages/arkham-frame/arkham_frame/services/documents.py b/packages/arkham-frame/arkham_frame/services/documents.py index 0ede91d..63d9d85 100644 --- a/packages/arkham-frame/arkham_frame/services/documents.py +++ b/packages/arkham-frame/arkham_frame/services/documents.py @@ -37,6 +37,7 @@ class DocumentStatus(str, Enum): COMPLETED = "completed" FAILED = "failed" ARCHIVED = "archived" + MERGED = "merged" # Document merged into another @dataclass @@ -897,26 +898,32 @@ async def search( if filters: vector_filter.update(filters) - # Perform vector search - results = await self.vectors.search( - collection="documents", - query_text=query, + # Perform vector search using text query + # search_text handles embedding internally + results = await self.vectors.search_text( + collection="arkham_documents", # Use correct collection name + text=query, limit=limit, filter=vector_filter if vector_filter else None, ) search_results = [] for result in results: - doc_id = result.get("document_id") + # SearchResult is a dataclass with .payload attribute + payload = result.payload if hasattr(result, 'payload') else result + doc_id = payload.get("document_id") if isinstance(payload, dict) else None if doc_id: doc = await self.get_document(doc_id) if doc: + # Extract score from SearchResult dataclass + score = result.score if hasattr(result, 'score') else (payload.get("score", 0.0) if isinstance(payload, dict) else 0.0) + highlights = payload.get("highlights", []) if isinstance(payload, dict) else [] search_results.append( SearchResult( document=doc, chunk=None, # Could load chunk if needed - score=result.get("score", 0.0), - highlights=result.get("highlights", []), + score=score, + highlights=highlights, ) ) diff --git a/packages/arkham-shard-documents/arkham_shard_documents/models.py b/packages/arkham-shard-documents/arkham_shard_documents/models.py index b385974..8bcc7af 100644 --- a/packages/arkham-shard-documents/arkham_shard_documents/models.py +++ b/packages/arkham-shard-documents/arkham_shard_documents/models.py @@ -13,6 +13,7 @@ class DocumentStatus(Enum): PROCESSED = "processed" FAILED = "failed" ARCHIVED = "archived" + MERGED = "merged" class ViewMode(Enum): diff --git a/packages/arkham-shard-shell/src/pages/documents/DeduplicationPanel.tsx b/packages/arkham-shard-shell/src/pages/documents/DeduplicationPanel.tsx index 6c25edc..8b99c02 100644 --- a/packages/arkham-shard-shell/src/pages/documents/DeduplicationPanel.tsx +++ b/packages/arkham-shard-shell/src/pages/documents/DeduplicationPanel.tsx @@ -30,7 +30,7 @@ export function DeduplicationPanel({ const [showSimilar, setShowSimilar] = useState(true); const [similarThreshold, setSimilarThreshold] = useState(0.8); const [selectedDuplicates, setSelectedDuplicates] = useState>(new Set()); - const [mergeStrategy, setMergeStrategy] = useState('soft_delete'); + const [mergeStrategy, setMergeStrategy] = useState('soft_delete'); const { computeHash, loading: hashLoading, error: hashError } = useComputeHash(); const { merge, loading: mergeLoading } = useMergeDuplicates(); @@ -77,9 +77,10 @@ export function DeduplicationPanel({ try { await merge({ - source_ids: Array.from(selectedDuplicates), - target_id: documentId, - strategy: mergeStrategy, + primary_id: documentId, + duplicate_ids: Array.from(selectedDuplicates), + cleanup_action: mergeStrategy, + preserve_references: true, }); setSelectedDuplicates(new Set()); refetchExact(); @@ -261,12 +262,13 @@ export function DeduplicationPanel({ + +
+ + {/* Tab Content */} + {activeTab === 'list' ? ( + + ) : ( + + )} + + ); } // ============================================ @@ -197,31 +238,25 @@ function AnomaliesListView() { }; if (loading && anomalies.length === 0) { - return ( -
- -
- ); + return ; } if (error) { return ( -
-
- -

Failed to load anomalies

-

{error}

- -
+
+ +

Failed to load anomalies

+

{error}

+
); } return ( -
+ <>
@@ -566,7 +601,7 @@ function AnomaliesListView() { onCancel={() => setShowDetectDialog(false)} /> )} -
+ ); } diff --git a/packages/arkham-shard-shell/src/pages/anomalies/HiddenContentTab.tsx b/packages/arkham-shard-shell/src/pages/anomalies/HiddenContentTab.tsx new file mode 100644 index 0000000..81c3f11 --- /dev/null +++ b/packages/arkham-shard-shell/src/pages/anomalies/HiddenContentTab.tsx @@ -0,0 +1,413 @@ +/** + * HiddenContentTab - Steganography and Hidden Data Detection + * + * Provides UI for detecting hidden content in documents: + * - Entropy analysis (encrypted/compressed data detection) + * - LSB pattern analysis (image steganography) + * - File type mismatch detection + * - Chi-square statistical tests + */ + +import { useState, useCallback } from 'react'; +import { Icon } from '../../components/common/Icon'; +import { useToast } from '../../context/ToastContext'; +import { useFetch } from '../../hooks/useFetch'; +import * as api from './api'; +import type { + HiddenContentScan, + HiddenContentStats, + EntropyRegion, + StegoIndicator, +} from './api'; + +// Props +interface HiddenContentTabProps { + onScanComplete?: () => void; +} + +export function HiddenContentTab({ onScanComplete }: HiddenContentTabProps) { + const { toast } = useToast(); + + // State + const [scanning, setScanning] = useState(false); + const [selectedDocId, setSelectedDocId] = useState(''); + const [selectedScan, setSelectedScan] = useState(null); + const [recentScans, setRecentScans] = useState([]); + + // Fetch stats + const { data: statsData, refetch: refetchStats } = useFetch<{ stats: HiddenContentStats }>( + '/api/anomalies/hidden-content/stats' + ); + const stats = statsData?.stats; + + // Fetch documents for selection + const { data: documentsData } = useFetch<{ items: Array<{ id: string; filename: string; file_type: string }> }>( + '/api/documents/items?limit=100' + ); + const documents = documentsData?.items || []; + + // Scan a document + const handleScan = useCallback(async () => { + if (!selectedDocId) { + toast.error('Please select a document to scan'); + return; + } + + setScanning(true); + try { + const result = await api.scanHiddenContent({ doc_id: selectedDocId }); + setSelectedScan(result.scan); + setRecentScans(prev => [result.scan, ...prev.slice(0, 9)]); + refetchStats(); + + if (result.anomaly_created) { + toast.warning('Hidden content detected! Anomaly created.'); + onScanComplete?.(); + } else if (result.scan.stego_confidence > 0.5) { + toast.warning(`Suspicious patterns detected (${(result.scan.stego_confidence * 100).toFixed(0)}% confidence)`); + } else { + toast.success('Scan complete - no hidden content detected'); + } + } catch (err) { + toast.error(err instanceof Error ? err.message : 'Scan failed'); + } finally { + setScanning(false); + } + }, [selectedDocId, toast, refetchStats, onScanComplete]); + + // Quick scan multiple documents + const handleQuickScan = useCallback(async () => { + const docIds = documents.slice(0, 20).map(d => d.id); + if (docIds.length === 0) { + toast.error('No documents available for scanning'); + return; + } + + setScanning(true); + try { + const result = await api.quickScanHiddenContent(docIds); + toast.success(`Scanned ${result.scanned} documents. ${result.requires_full_scan_count} need full analysis.`); + + if (result.requires_full_scan_count > 0) { + toast.warning(`High entropy detected in: ${result.requires_full_scan.join(', ')}`); + } + } catch (err) { + toast.error(err instanceof Error ? err.message : 'Quick scan failed'); + } finally { + setScanning(false); + } + }, [documents, toast]); + + return ( +
+ {/* Stats Overview */} +
+
+ +
{stats?.total_scans ?? 0}
+
Total Scans
+
+
+ +
{stats?.documents_with_findings ?? 0}
+
With Findings
+
+
+ +
{stats?.high_entropy_files ?? 0}
+
High Entropy
+
+
+ +
{stats?.stego_candidates ?? 0}
+
Stego Candidates
+
+
+ + {/* Scan Controls */} +
+
+ + +
+ +
+ + +
+
+ + {/* Scan Results */} + {selectedScan && ( +
+

+ + Scan Results +

+ + {/* Summary */} +
+
+ + {(selectedScan.stego_confidence * 100).toFixed(0)}% + + Stego Confidence +
+ +
+
+ Global Entropy: + 7 ? 'high' : selectedScan.entropy_global > 6 ? 'medium' : ''}`}> + {selectedScan.entropy_global.toFixed(3)} / 8.0 + +
+
+ File Type Match: + + {selectedScan.file_mismatch ? 'MISMATCH' : 'OK'} + +
+
+ Indicators Found: + {selectedScan.stego_indicators.length} +
+
+
+ + {/* Findings */} + {selectedScan.findings.length > 0 && ( +
+

+ + Findings +

+
    + {selectedScan.findings.map((finding, i) => ( +
  • {finding}
    • + ))} +
+
+ )} + + {/* Entropy Regions */} + {selectedScan.entropy_regions.length > 0 && ( +
+

+ + Entropy Analysis by Region +

+
+ {selectedScan.entropy_regions.map((region, i) => ( + + ))} +
+
+ )} + + {/* Stego Indicators */} + {selectedScan.stego_indicators.length > 0 && ( +
+

+ + Steganography Indicators +

+
+ {selectedScan.stego_indicators.map((indicator, i) => ( + + ))} +
+
+ )} + + {/* LSB Analysis */} + {selectedScan.lsb_result && ( +
+

+ + LSB (Least Significant Bit) Analysis +

+
+
+ Bit Ratio: + {(selectedScan.lsb_result.bit_ratio * 100).toFixed(2)}% +
+
+ Chi-Square Value: + {selectedScan.lsb_result.chi_square_value.toFixed(4)} +
+
+ P-Value: + + {selectedScan.lsb_result.chi_square_p_value.toFixed(6)} + +
+
+ Suspicious: + + {selectedScan.lsb_result.is_suspicious ? 'YES' : 'NO'} + +
+
+ Confidence: + {(selectedScan.lsb_result.confidence * 100).toFixed(0)}% +
+
+
+ )} + + {/* Magic Bytes */} +
+

+ + File Type Detection +

+
+
+ Expected Type: + {selectedScan.magic_expected || 'Unknown'} +
+
+ Actual Type: + {selectedScan.magic_actual || 'Unknown'} +
+ {selectedScan.file_mismatch && ( +
+ + File extension does not match actual file type! +
+ )} +
+
+
+ )} + + {/* Recent Scans */} + {recentScans.length > 0 && ( +
+

Recent Scans

+
+ {recentScans.map(scan => ( +
setSelectedScan(scan)} + > +
{scan.doc_id.slice(0, 8)}...
+
+ {(scan.stego_confidence * 100).toFixed(0)}% +
+
+ {new Date(scan.created_at).toLocaleTimeString()} +
+
+ ))} +
+
+ )} +
+ ); +} + +// Helper Components + +function EntropyRegionBar({ region, index }: { region: EntropyRegion; index: number }) { + const percentage = (region.entropy_value / 8) * 100; + const colorClass = region.is_anomalous ? 'anomalous' : percentage > 87.5 ? 'high' : percentage > 75 ? 'medium' : 'normal'; + + return ( +
+
+ Region {index + 1} + + ({region.start_offset} - {region.end_offset}) + +
+
+
+ {region.entropy_value.toFixed(3)} +
+ {region.description && ( +
+ {region.description} +
+ )} +
+ ); +} + +function StegoIndicatorCard({ indicator }: { indicator: StegoIndicator }) { + const iconMap: Record = { + entropy_spike: 'TrendingUp', + high_global_entropy: 'Thermometer', + lsb_anomaly: 'Binary', + histogram_anomaly: 'BarChart', + file_mismatch: 'FileWarning', + }; + + return ( +
0.7 ? 'high-conf' : ''}`}> +
+ +
+
+
+ {indicator.indicator_type.replace(/_/g, ' ')} +
+
+ Location: {indicator.location} +
+
+ {(indicator.confidence * 100).toFixed(0)}% confidence +
+
+
+ ); +} + +// Helper Functions + +function getConfidenceClass(confidence: number): string { + if (confidence >= 0.8) return 'critical'; + if (confidence >= 0.6) return 'high'; + if (confidence >= 0.4) return 'medium'; + return 'low'; +} + +export default HiddenContentTab; diff --git a/packages/arkham-shard-shell/src/pages/anomalies/api.ts b/packages/arkham-shard-shell/src/pages/anomalies/api.ts index 0420231..040d15d 100644 --- a/packages/arkham-shard-shell/src/pages/anomalies/api.ts +++ b/packages/arkham-shard-shell/src/pages/anomalies/api.ts @@ -172,3 +172,122 @@ export async function bulkUpdateStatus( }), }); } + +// ============================================ +// Hidden Content Detection Operations +// ============================================ + +export interface HiddenContentScanRequest { + doc_id: string; + scan_type?: string; + config?: Record; +} + +export interface EntropyRegion { + start_offset: number; + end_offset: number; + entropy_value: number; + is_anomalous: boolean; + description: string; +} + +export interface LSBAnalysisResult { + bit_ratio: number; + chi_square_value: number; + chi_square_p_value: number; + is_suspicious: boolean; + confidence: number; + sample_size: number; +} + +export interface StegoIndicator { + indicator_type: string; + confidence: number; + location: string; + details: Record; +} + +export interface HiddenContentScan { + id: string; + doc_id: string; + scan_type: string; + scan_status: string; + entropy_global: number; + entropy_regions: EntropyRegion[]; + magic_expected: string; + magic_actual: string; + file_mismatch: boolean; + lsb_result: LSBAnalysisResult | null; + stego_indicators: StegoIndicator[]; + stego_confidence: number; + findings: string[]; + created_at: string; + completed_at: string | null; +} + +export interface HiddenContentScanResponse { + scan: HiddenContentScan; + anomaly_created: boolean; +} + +export interface HiddenContentQuickScanRequest { + doc_ids: string[]; +} + +export interface QuickScanResult { + doc_id: string; + entropy_global?: number; + requires_full_scan?: boolean; + error?: string; +} + +export interface HiddenContentQuickScanResponse { + scanned: number; + results: QuickScanResult[]; + requires_full_scan_count: number; + requires_full_scan: string[]; +} + +export interface HiddenContentStats { + total_scans: number; + scans_by_type: Record; + documents_with_findings: number; + high_entropy_files: number; + stego_candidates: number; +} + +export async function scanHiddenContent( + request: HiddenContentScanRequest +): Promise { + return fetchAPI('/hidden-content/scan', { + method: 'POST', + body: JSON.stringify(request), + }); +} + +export async function quickScanHiddenContent( + docIds: string[] +): Promise { + return fetchAPI('/hidden-content/quick-scan', { + method: 'POST', + body: JSON.stringify({ doc_ids: docIds }), + }); +} + +export async function getHiddenContentScan( + scanId: string +): Promise<{ scan: HiddenContentScan }> { + return fetchAPI<{ scan: HiddenContentScan }>(`/hidden-content/${scanId}`); +} + +export async function getDocumentHiddenScans( + docId: string +): Promise<{ scans: HiddenContentScan[]; total: number }> { + return fetchAPI<{ scans: HiddenContentScan[]; total: number }>( + `/hidden-content/document/${docId}` + ); +} + +export async function getHiddenContentStats(): Promise<{ stats: HiddenContentStats }> { + return fetchAPI<{ stats: HiddenContentStats }>('/hidden-content/stats'); +} diff --git a/packages/arkham-shard-shell/src/pages/provenance/ForensicsTab.tsx b/packages/arkham-shard-shell/src/pages/provenance/ForensicsTab.tsx new file mode 100644 index 0000000..870e722 --- /dev/null +++ b/packages/arkham-shard-shell/src/pages/provenance/ForensicsTab.tsx @@ -0,0 +1,776 @@ +/** + * ForensicsTab - Metadata Forensics Analysis + * + * Provides UI for forensic metadata analysis: + * - EXIF extraction (camera info, GPS, timestamps) + * - PDF metadata extraction + * - Office document metadata + * - Timeline reconstruction + * - Document comparison + */ + +import { useState, useCallback } from 'react'; +import { Icon } from '../../components/common/Icon'; +import { useToast } from '../../context/ToastContext'; +import { useFetch } from '../../hooks/useFetch'; + +// Types +interface ExifData { + make?: string; + model?: string; + serial_number?: string; + datetime_original?: string; + datetime_digitized?: string; + datetime_modified?: string; + gps_latitude?: number; + gps_longitude?: number; + gps_altitude?: number; + software?: string; + width?: number; + height?: number; +} + +interface PdfMetadata { + title?: string; + author?: string; + subject?: string; + creator?: string; + producer?: string; + creation_date?: string; + modification_date?: string; + keywords?: string[]; + page_count?: number; + pdf_version?: string; + is_encrypted?: boolean; +} + +interface OfficeMetadata { + title?: string; + author?: string; + subject?: string; + company?: string; + manager?: string; + created?: string; + modified?: string; + last_modified_by?: string; + revision?: number; + keywords?: string[]; + category?: string; +} + +interface ForensicFinding { + finding_type: string; + severity: string; + description: string; + evidence: Record; + confidence: number; +} + +interface TimelineEvent { + id: string; + doc_id: string; + event_type: string; + event_timestamp?: string; + event_source: string; + event_actor?: string; + event_details: Record; + confidence: number; + is_estimated: boolean; +} + +interface ForensicScan { + id: string; + doc_id: string; + scan_status: string; + file_hash_md5?: string; + file_hash_sha256?: string; + file_hash_sha512?: string; + file_size?: number; + exif_data?: ExifData; + pdf_metadata?: PdfMetadata; + office_metadata?: OfficeMetadata; + findings: ForensicFinding[]; + integrity_status: string; + confidence_score: number; + timeline_events: TimelineEvent[]; + scanned_at?: string; +} + +interface ForensicStats { + total_scans: number; + scans_by_status: Record; + documents_with_findings: number; + integrity_status_counts: Record; + average_confidence: number; +} + +interface ForensicsTabProps { + onScanComplete?: () => void; +} + +export function ForensicsTab({ onScanComplete }: ForensicsTabProps) { + const { toast } = useToast(); + + // State + const [scanning, setScanning] = useState(false); + const [selectedDocId, setSelectedDocId] = useState(''); + const [selectedScan, setSelectedScan] = useState(null); + const [recentScans, setRecentScans] = useState([]); + const [comparing, setComparing] = useState(false); + const [compareSourceId, setCompareSourceId] = useState(''); + const [compareTargetId, setCompareTargetId] = useState(''); + const [comparisonResult, setComparisonResult] = useState(null); + + // Fetch stats + const { data: statsData, refetch: refetchStats } = useFetch<{ stats: ForensicStats }>( + '/api/provenance/forensics/stats' + ); + const stats = statsData?.stats; + + // Fetch documents for selection + const { data: documentsData } = useFetch<{ items: Array<{ id: string; filename: string; file_type: string }> }>( + '/api/documents/items?limit=100' + ); + const documents = documentsData?.items || []; + + // Scan a document + const handleScan = useCallback(async () => { + if (!selectedDocId) { + toast.error('Please select a document to scan'); + return; + } + + setScanning(true); + try { + const response = await fetch('/api/provenance/forensics/scan', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ doc_id: selectedDocId }), + }); + + if (!response.ok) { + const error = await response.json(); + throw new Error(error.detail || 'Scan failed'); + } + + const result = await response.json(); + setSelectedScan(result.scan); + setRecentScans(prev => [result.scan, ...prev.slice(0, 9)]); + refetchStats(); + + if (result.scan.findings.length > 0) { + toast.warning(`Forensic analysis complete. ${result.scan.findings.length} findings detected.`); + } else { + toast.success('Forensic analysis complete - no issues detected'); + } + onScanComplete?.(); + } catch (err) { + toast.error(err instanceof Error ? err.message : 'Scan failed'); + } finally { + setScanning(false); + } + }, [selectedDocId, toast, refetchStats, onScanComplete]); + + // Compare documents + const handleCompare = useCallback(async () => { + if (!compareSourceId || !compareTargetId) { + toast.error('Please select both source and target documents'); + return; + } + + setComparing(true); + try { + const response = await fetch('/api/provenance/forensics/compare', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + source_doc_id: compareSourceId, + target_doc_id: compareTargetId, + }), + }); + + if (!response.ok) { + const error = await response.json(); + throw new Error(error.detail || 'Comparison failed'); + } + + const result = await response.json(); + setComparisonResult(result.comparison); + toast.success('Document comparison complete'); + } catch (err) { + toast.error(err instanceof Error ? err.message : 'Comparison failed'); + } finally { + setComparing(false); + } + }, [compareSourceId, compareTargetId, toast]); + + const formatDate = (dateStr?: string) => { + if (!dateStr) return 'N/A'; + try { + return new Date(dateStr).toLocaleString(); + } catch { + return dateStr; + } + }; + + const getIntegrityBadgeClass = (status: string): string => { + switch (status) { + case 'pristine': return 'integrity-pristine'; + case 'modified': return 'integrity-modified'; + case 'suspicious': return 'integrity-suspicious'; + case 'compromised': return 'integrity-compromised'; + default: return 'integrity-unknown'; + } + }; + + const getSeverityClass = (severity: string): string => { + switch (severity) { + case 'critical': return 'severity-critical'; + case 'high': return 'severity-high'; + case 'medium': return 'severity-medium'; + case 'low': return 'severity-low'; + default: return 'severity-info'; + } + }; + + return ( +
+ {/* Stats Overview */} +
+
+ +
{stats?.total_scans ?? 0}
+
Total Scans
+
+
+ +
{stats?.documents_with_findings ?? 0}
+
With Findings
+
+
+ +
{stats?.integrity_status_counts?.pristine ?? 0}
+
Pristine
+
+
+ +
+ {stats?.average_confidence ? `${(stats.average_confidence * 100).toFixed(0)}%` : '0%'} +
+
Avg Confidence
+
+
+ + {/* Scan Controls */} +
+
+ + +
+ +
+ +
+
+ + {/* Scan Results */} + {selectedScan && ( +
+

+ + Forensic Analysis Results +

+ + {/* Summary */} +
+
+ {selectedScan.integrity_status} + Integrity Status +
+ +
+
+ Confidence: + {(selectedScan.confidence_score * 100).toFixed(0)}% +
+
+ Findings: + 0 ? 'warning' : ''}`}> + {selectedScan.findings.length} + +
+
+ Timeline Events: + {selectedScan.timeline_events.length} +
+
+ File Size: + + {selectedScan.file_size ? `${(selectedScan.file_size / 1024).toFixed(1)} KB` : 'N/A'} + +
+
+
+ + {/* File Hashes */} + {(selectedScan.file_hash_md5 || selectedScan.file_hash_sha256) && ( +
+

+ + File Hashes +

+
+ {selectedScan.file_hash_md5 && ( +
+ MD5: + {selectedScan.file_hash_md5} +
+ )} + {selectedScan.file_hash_sha256 && ( +
+ SHA256: + {selectedScan.file_hash_sha256.substring(0, 32)}... +
+ )} +
+
+ )} + + {/* EXIF Data */} + {selectedScan.exif_data && Object.values(selectedScan.exif_data).some(v => v !== null && v !== undefined) && ( +
+

+ + EXIF Data (Image Metadata) +

+
+ {selectedScan.exif_data.make && ( +
+ Camera Make: + {selectedScan.exif_data.make} +
+ )} + {selectedScan.exif_data.model && ( +
+ Camera Model: + {selectedScan.exif_data.model} +
+ )} + {selectedScan.exif_data.serial_number && ( +
+ Serial Number: + {selectedScan.exif_data.serial_number} +
+ )} + {selectedScan.exif_data.software && ( +
+ Software: + {selectedScan.exif_data.software} +
+ )} + {selectedScan.exif_data.datetime_original && ( +
+ Date Taken: + {formatDate(selectedScan.exif_data.datetime_original)} +
+ )} + {(selectedScan.exif_data.gps_latitude || selectedScan.exif_data.gps_longitude) && ( +
+ GPS Location: + + {selectedScan.exif_data.gps_latitude?.toFixed(6)}, {selectedScan.exif_data.gps_longitude?.toFixed(6)} + +
+ )} + {(selectedScan.exif_data.width && selectedScan.exif_data.height) && ( +
+ Dimensions: + {selectedScan.exif_data.width} x {selectedScan.exif_data.height} +
+ )} +
+
+ )} + + {/* PDF Metadata */} + {selectedScan.pdf_metadata && Object.values(selectedScan.pdf_metadata).some(v => v !== null && v !== undefined) && ( +
+

+ + PDF Metadata +

+
+ {selectedScan.pdf_metadata.title && ( +
+ Title: + {selectedScan.pdf_metadata.title} +
+ )} + {selectedScan.pdf_metadata.author && ( +
+ Author: + {selectedScan.pdf_metadata.author} +
+ )} + {selectedScan.pdf_metadata.creator && ( +
+ Creator: + {selectedScan.pdf_metadata.creator} +
+ )} + {selectedScan.pdf_metadata.producer && ( +
+ Producer: + {selectedScan.pdf_metadata.producer} +
+ )} + {selectedScan.pdf_metadata.creation_date && ( +
+ Created: + {formatDate(selectedScan.pdf_metadata.creation_date)} +
+ )} + {selectedScan.pdf_metadata.modification_date && ( +
+ Modified: + {formatDate(selectedScan.pdf_metadata.modification_date)} +
+ )} + {selectedScan.pdf_metadata.page_count && ( +
+ Pages: + {selectedScan.pdf_metadata.page_count} +
+ )} + {selectedScan.pdf_metadata.pdf_version && ( +
+ PDF Version: + {selectedScan.pdf_metadata.pdf_version} +
+ )} + {selectedScan.pdf_metadata.is_encrypted !== undefined && ( +
+ Encrypted: + + {selectedScan.pdf_metadata.is_encrypted ? 'Yes' : 'No'} + +
+ )} + {selectedScan.pdf_metadata.keywords && selectedScan.pdf_metadata.keywords.length > 0 && ( +
+ Keywords: + {selectedScan.pdf_metadata.keywords.join(', ')} +
+ )} +
+
+ )} + + {/* Office Metadata */} + {selectedScan.office_metadata && Object.values(selectedScan.office_metadata).some(v => v !== null && v !== undefined) && ( +
+

+ + Office Document Metadata +

+
+ {selectedScan.office_metadata.title && ( +
+ Title: + {selectedScan.office_metadata.title} +
+ )} + {selectedScan.office_metadata.author && ( +
+ Author: + {selectedScan.office_metadata.author} +
+ )} + {selectedScan.office_metadata.company && ( +
+ Company: + {selectedScan.office_metadata.company} +
+ )} + {selectedScan.office_metadata.last_modified_by && ( +
+ Last Modified By: + {selectedScan.office_metadata.last_modified_by} +
+ )} + {selectedScan.office_metadata.created && ( +
+ Created: + {formatDate(selectedScan.office_metadata.created)} +
+ )} + {selectedScan.office_metadata.modified && ( +
+ Modified: + {formatDate(selectedScan.office_metadata.modified)} +
+ )} + {selectedScan.office_metadata.revision && ( +
+ Revision: + {selectedScan.office_metadata.revision} +
+ )} + {selectedScan.office_metadata.subject && ( +
+ Subject: + {selectedScan.office_metadata.subject} +
+ )} + {selectedScan.office_metadata.keywords && selectedScan.office_metadata.keywords.length > 0 && ( +
+ Keywords: + {selectedScan.office_metadata.keywords.join(', ')} +
+ )} +
+
+ )} + + {/* Findings */} + {selectedScan.findings.length > 0 && ( +
+

+ + Findings ({selectedScan.findings.length}) +

+
+ {selectedScan.findings.map((finding, i) => ( +
+
+ {finding.finding_type.replace(/_/g, ' ')} + + {finding.severity} + +
+

{finding.description}

+
+ {(finding.confidence * 100).toFixed(0)}% confidence +
+
+ ))} +
+
+ )} + + {/* Timeline Events */} + {selectedScan.timeline_events.length > 0 && ( +
+

+ + Timeline Reconstruction ({selectedScan.timeline_events.length} events) +

+
+ {selectedScan.timeline_events + .sort((a, b) => { + if (!a.event_timestamp) return 1; + if (!b.event_timestamp) return -1; + return new Date(a.event_timestamp).getTime() - new Date(b.event_timestamp).getTime(); + }) + .map((event, i) => ( +
+
+ +
+
+
+ {event.event_type.replace(/_/g, ' ')} + {event.is_estimated && Estimated} +
+
+ {event.event_timestamp ? formatDate(event.event_timestamp) : 'Unknown time'} +
+ {event.event_actor && ( +
+ + {event.event_actor} +
+ )} +
+ Source: {event.event_source} +
+
+ {(event.confidence * 100).toFixed(0)}% confidence +
+
+
+ ))} +
+
+ )} +
+ )} + + {/* Document Comparison */} +
+

+ + Document Comparison +

+
+
+ + +
+
+ +
+
+ + +
+ +
+ + {/* Comparison Result */} + {comparisonResult && ( +
+
+
+ {(comparisonResult.match_score * 100).toFixed(0)}% + Match Score +
+
+ Relationship: + + {comparisonResult.relationship_type?.replace(/_/g, ' ') || 'Unknown'} + +
+
+ Confidence: + {(comparisonResult.confidence * 100).toFixed(0)}% +
+
+ + {comparisonResult.similarities && comparisonResult.similarities.length > 0 && ( +
+

+ + Similarities ({comparisonResult.similarities.length}) +

+
    + {comparisonResult.similarities.map((item: string, i: number) => ( +
  • {item}
    • + ))} +
+
+ )} + + {comparisonResult.differences && comparisonResult.differences.length > 0 && ( +
+

+ + Differences ({comparisonResult.differences.length}) +

+
    + {comparisonResult.differences.map((item: string, i: number) => ( +
  • {item}
    • + ))} +
+
+ )} +
+ )} +
+ + {/* Recent Scans */} + {recentScans.length > 0 && ( +
+

Recent Scans

+
+ {recentScans.map(scan => ( +
setSelectedScan(scan)} + > +
{scan.doc_id.slice(0, 8)}...
+
+ {scan.integrity_status} +
+
+ {scan.findings.length} findings +
+
+ {scan.scanned_at ? new Date(scan.scanned_at).toLocaleTimeString() : 'N/A'} +
+
+ ))} +
+
+ )} +
+ ); +} + +export default ForensicsTab; diff --git a/packages/arkham-shard-shell/src/pages/provenance/ProvenancePage.css b/packages/arkham-shard-shell/src/pages/provenance/ProvenancePage.css index c71a6a5..7726d04 100644 --- a/packages/arkham-shard-shell/src/pages/provenance/ProvenancePage.css +++ b/packages/arkham-shard-shell/src/pages/provenance/ProvenancePage.css @@ -757,3 +757,809 @@ transform: rotate(360deg); } } + +/* ============================================ + Forensics Tab + ============================================ */ + +.forensics-tab { + display: flex; + flex-direction: column; + gap: 24px; + padding: 24px; + overflow-y: auto; + height: 100%; +} + +/* Stats Grid */ +.forensics-stats-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); + gap: 16px; +} + +.forensics-stat-card { + display: flex; + flex-direction: column; + align-items: center; + gap: 8px; + padding: 20px; + background: var(--color-surface); + border: 1px solid var(--color-border); + border-radius: 12px; + text-align: center; +} + +.forensics-stat-card svg { + color: var(--color-primary); + opacity: 0.8; +} + +.forensics-stat-value { + font-size: 28px; + font-weight: 700; + color: var(--color-text); +} + +.forensics-stat-label { + font-size: 12px; + color: var(--color-text-secondary); + text-transform: uppercase; + letter-spacing: 0.5px; +} + +/* Scan Controls */ +.forensics-scan-controls { + display: flex; + gap: 16px; + padding: 20px; + background: var(--color-surface); + border: 1px solid var(--color-border); + border-radius: 12px; + flex-wrap: wrap; + align-items: flex-end; +} + +.forensics-scan-form { + display: flex; + flex-direction: column; + gap: 8px; + flex: 1; + min-width: 250px; +} + +.forensics-scan-form label { + font-size: 14px; + font-weight: 500; + color: var(--color-text-secondary); +} + +.forensics-scan-form select { + padding: 10px 12px; + border: 1px solid var(--color-border); + border-radius: 6px; + background: var(--color-background); + color: var(--color-text); + font-size: 14px; +} + +.forensics-scan-form select:focus { + outline: none; + border-color: var(--color-primary); +} + +.forensics-scan-actions { + display: flex; + gap: 8px; +} + +/* Scan Results */ +.forensics-scan-results { + padding: 24px; + background: var(--color-surface); + border: 1px solid var(--color-border); + border-radius: 12px; +} + +.forensics-scan-results h3 { + display: flex; + align-items: center; + gap: 8px; + margin: 0 0 20px 0; + font-size: 18px; + font-weight: 600; + color: var(--color-text); +} + +.forensics-scan-results h4 { + display: flex; + align-items: center; + gap: 8px; + margin: 0 0 12px 0; + font-size: 15px; + font-weight: 600; + color: var(--color-text); +} + +/* Result Summary */ +.forensics-result-summary { + display: flex; + gap: 24px; + padding: 16px; + background: var(--color-background); + border-radius: 8px; + margin-bottom: 24px; + flex-wrap: wrap; + align-items: center; +} + +.forensics-integrity-badge { + display: flex; + flex-direction: column; + align-items: center; + padding: 16px 24px; + border-radius: 8px; + min-width: 120px; +} + +.forensics-integrity-badge .integrity-status { + font-size: 20px; + font-weight: 700; + text-transform: capitalize; +} + +.forensics-integrity-badge .integrity-label { + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.5px; + opacity: 0.9; +} + +.integrity-pristine { + background: rgba(74, 222, 128, 0.15); + color: var(--color-success); +} + +.integrity-modified { + background: rgba(251, 191, 36, 0.15); + color: var(--color-warning, #fbbf24); +} + +.integrity-suspicious { + background: rgba(251, 146, 60, 0.15); + color: #fb923c; +} + +.integrity-compromised { + background: rgba(248, 113, 113, 0.15); + color: var(--color-danger); +} + +.integrity-unknown { + background: var(--color-surface); + color: var(--color-text-secondary); +} + +.forensics-summary-stats { + display: flex; + flex-direction: column; + gap: 8px; + flex: 1; +} + +.forensics-summary-stats .summary-item { + display: flex; + justify-content: space-between; + padding: 6px 0; + border-bottom: 1px solid var(--color-border); +} + +.forensics-summary-stats .summary-item:last-child { + border-bottom: none; +} + +.forensics-summary-stats .label { + font-size: 14px; + color: var(--color-text-secondary); +} + +.forensics-summary-stats .value { + font-size: 14px; + font-weight: 600; + color: var(--color-text); +} + +.forensics-summary-stats .value.warning { + color: var(--color-warning, #fbbf24); +} + +/* File Hashes */ +.forensics-hashes { + margin-bottom: 24px; +} + +.hash-grid { + display: flex; + flex-direction: column; + gap: 8px; + padding: 12px; + background: var(--color-background); + border-radius: 6px; +} + +.hash-item { + display: flex; + gap: 12px; + align-items: center; +} + +.hash-label { + font-size: 13px; + font-weight: 500; + color: var(--color-text-secondary); + min-width: 60px; +} + +.hash-value { + font-family: 'Courier New', monospace; + font-size: 12px; + padding: 4px 8px; + background: var(--color-surface); + border-radius: 4px; + color: var(--color-text); + word-break: break-all; +} + +/* Metadata Sections */ +.forensics-exif, +.forensics-pdf, +.forensics-office { + margin-bottom: 24px; +} + +.metadata-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(200px, 1fr)); + gap: 12px; + padding: 16px; + background: var(--color-background); + border-radius: 8px; +} + +.metadata-item { + display: flex; + flex-direction: column; + gap: 4px; +} + +.metadata-item.full-width { + grid-column: 1 / -1; +} + +.metadata-item .label { + font-size: 12px; + color: var(--color-text-secondary); + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.metadata-item .value { + font-size: 14px; + color: var(--color-text); + word-break: break-word; +} + +.metadata-item .value.warning { + color: var(--color-warning, #fbbf24); +} + +/* Findings */ +.forensics-findings { + margin-bottom: 24px; +} + +.findings-list { + display: flex; + flex-direction: column; + gap: 12px; +} + +.finding-card { + padding: 16px; + background: var(--color-background); + border: 1px solid var(--color-border); + border-radius: 8px; + border-left: 4px solid; +} + +.finding-card.severity-critical { + border-left-color: var(--color-danger); +} + +.finding-card.severity-high { + border-left-color: #fb923c; +} + +.finding-card.severity-medium { + border-left-color: var(--color-warning, #fbbf24); +} + +.finding-card.severity-low { + border-left-color: var(--color-text-secondary); +} + +.finding-card.severity-info { + border-left-color: var(--color-primary); +} + +.finding-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 8px; +} + +.finding-type { + font-weight: 600; + color: var(--color-text); + text-transform: capitalize; +} + +.finding-severity { + padding: 4px 8px; + border-radius: 4px; + font-size: 12px; + font-weight: 500; + text-transform: uppercase; +} + +.finding-severity.severity-critical { + background: rgba(248, 113, 113, 0.15); + color: var(--color-danger); +} + +.finding-severity.severity-high { + background: rgba(251, 146, 60, 0.15); + color: #fb923c; +} + +.finding-severity.severity-medium { + background: rgba(251, 191, 36, 0.15); + color: var(--color-warning, #fbbf24); +} + +.finding-severity.severity-low { + background: var(--color-surface); + color: var(--color-text-secondary); +} + +.finding-description { + margin: 0 0 8px 0; + font-size: 14px; + color: var(--color-text); + line-height: 1.5; +} + +.finding-confidence { + font-size: 12px; + color: var(--color-text-secondary); +} + +/* Timeline */ +.forensics-timeline { + margin-bottom: 24px; +} + +.timeline-list { + display: flex; + flex-direction: column; + gap: 0; + position: relative; + padding-left: 24px; +} + +.timeline-list::before { + content: ''; + position: absolute; + left: 4px; + top: 8px; + bottom: 8px; + width: 2px; + background: var(--color-border); +} + +.timeline-event { + display: flex; + gap: 16px; + padding: 12px 0; + position: relative; +} + +.timeline-event.estimated { + opacity: 0.7; +} + +.event-marker { + position: absolute; + left: -24px; + top: 14px; + width: 10px; + height: 10px; + background: var(--color-primary); + border-radius: 50%; + border: 2px solid var(--color-background); +} + +.event-content { + display: flex; + flex-direction: column; + gap: 4px; + flex: 1; +} + +.event-header { + display: flex; + align-items: center; + gap: 8px; +} + +.event-type { + font-weight: 600; + font-size: 14px; + color: var(--color-text); + text-transform: capitalize; +} + +.event-estimated { + font-size: 11px; + padding: 2px 6px; + background: var(--color-surface); + border-radius: 4px; + color: var(--color-text-secondary); +} + +.event-timestamp { + font-size: 13px; + color: var(--color-text); +} + +.event-actor { + display: flex; + align-items: center; + gap: 4px; + font-size: 12px; + color: var(--color-text-secondary); +} + +.event-source { + font-size: 12px; + color: var(--color-text-secondary); +} + +.event-confidence { + font-size: 11px; + color: var(--color-text-secondary); +} + +/* Document Comparison */ +.forensics-comparison { + padding: 24px; + background: var(--color-surface); + border: 1px solid var(--color-border); + border-radius: 12px; +} + +.forensics-comparison h3 { + display: flex; + align-items: center; + gap: 8px; + margin: 0 0 20px 0; + font-size: 18px; + font-weight: 600; + color: var(--color-text); +} + +.comparison-controls { + display: flex; + gap: 12px; + align-items: flex-end; + flex-wrap: wrap; +} + +.comparison-select { + display: flex; + flex-direction: column; + gap: 6px; + flex: 1; + min-width: 180px; +} + +.comparison-select label { + font-size: 13px; + color: var(--color-text-secondary); +} + +.comparison-select select { + padding: 10px 12px; + border: 1px solid var(--color-border); + border-radius: 6px; + background: var(--color-background); + color: var(--color-text); + font-size: 14px; +} + +.comparison-arrow { + display: flex; + align-items: center; + justify-content: center; + padding: 10px; + color: var(--color-text-secondary); +} + +/* Comparison Result */ +.comparison-result { + margin-top: 24px; + padding: 16px; + background: var(--color-background); + border-radius: 8px; +} + +.comparison-header { + display: flex; + gap: 24px; + align-items: center; + flex-wrap: wrap; + padding-bottom: 16px; + border-bottom: 1px solid var(--color-border); + margin-bottom: 16px; +} + +.match-score { + display: flex; + flex-direction: column; + align-items: center; +} + +.match-score .score-value { + font-size: 32px; + font-weight: 700; + color: var(--color-primary); +} + +.match-score .score-label { + font-size: 11px; + text-transform: uppercase; + color: var(--color-text-secondary); +} + +.relationship-type { + display: flex; + flex-direction: column; + gap: 4px; +} + +.relationship-label { + font-size: 11px; + text-transform: uppercase; + color: var(--color-text-secondary); +} + +.relationship-value { + font-size: 16px; + font-weight: 600; + text-transform: capitalize; + color: var(--color-text); +} + +.relationship-value.copy { + color: var(--color-danger); +} + +.relationship-value.same_source { + color: #fb923c; +} + +.relationship-value.same_author { + color: var(--color-warning, #fbbf24); +} + +.comparison-confidence { + display: flex; + flex-direction: column; + gap: 4px; +} + +.confidence-label { + font-size: 11px; + text-transform: uppercase; + color: var(--color-text-secondary); +} + +.confidence-value { + font-size: 16px; + font-weight: 600; + color: var(--color-text); +} + +.comparison-section { + margin-bottom: 16px; +} + +.comparison-section:last-child { + margin-bottom: 0; +} + +.comparison-section h4 { + display: flex; + align-items: center; + gap: 8px; + font-size: 14px; + font-weight: 600; + margin: 0 0 8px 0; + color: var(--color-text); +} + +.comparison-list { + margin: 0; + padding-left: 24px; +} + +.comparison-list li { + font-size: 14px; + color: var(--color-text); + margin-bottom: 4px; + line-height: 1.5; +} + +.comparison-list.similarities li::marker { + color: var(--color-success); +} + +.comparison-list.differences li::marker { + color: var(--color-danger); +} + +/* Recent Scans */ +.forensics-recent-scans { + padding: 20px; + background: var(--color-surface); + border: 1px solid var(--color-border); + border-radius: 12px; +} + +.forensics-recent-scans h3 { + margin: 0 0 16px 0; + font-size: 16px; + font-weight: 600; + color: var(--color-text); +} + +.recent-scans-list { + display: flex; + flex-direction: column; + gap: 8px; +} + +.recent-scan-item { + display: flex; + align-items: center; + gap: 16px; + padding: 10px 14px; + background: var(--color-background); + border: 1px solid var(--color-border); + border-radius: 6px; + cursor: pointer; + transition: all 0.2s; +} + +.recent-scan-item:hover { + border-color: var(--color-primary); +} + +.recent-scan-item.selected { + border-color: var(--color-primary); + background: rgba(var(--color-primary-rgb), 0.05); +} + +.recent-scan-item .scan-doc { + font-size: 14px; + font-family: 'Courier New', monospace; + color: var(--color-text); + flex: 1; +} + +.recent-scan-item .scan-integrity { + font-size: 12px; + font-weight: 500; + padding: 4px 8px; + border-radius: 4px; + text-transform: capitalize; +} + +.scan-integrity.integrity-pristine { + background: rgba(74, 222, 128, 0.15); + color: var(--color-success); +} + +.scan-integrity.integrity-modified { + background: rgba(251, 191, 36, 0.15); + color: var(--color-warning, #fbbf24); +} + +.scan-integrity.integrity-suspicious { + background: rgba(251, 146, 60, 0.15); + color: #fb923c; +} + +.scan-integrity.integrity-compromised { + background: rgba(248, 113, 113, 0.15); + color: var(--color-danger); +} + +.recent-scan-item .scan-findings { + font-size: 12px; + color: var(--color-text-secondary); +} + +.recent-scan-item .scan-date { + font-size: 12px; + color: var(--color-text-secondary); +} + +/* Responsive */ +@media (max-width: 768px) { + .forensics-tab { + padding: 16px; + } + + .forensics-stats-grid { + grid-template-columns: repeat(2, 1fr); + } + + .forensics-scan-controls { + flex-direction: column; + } + + .forensics-scan-form { + min-width: 100%; + } + + .forensics-scan-actions { + width: 100%; + } + + .forensics-scan-actions button { + flex: 1; + } + + .forensics-result-summary { + flex-direction: column; + align-items: stretch; + } + + .forensics-integrity-badge { + align-self: center; + } + + .metadata-grid { + grid-template-columns: 1fr; + } + + .comparison-controls { + flex-direction: column; + } + + .comparison-select { + min-width: 100%; + } + + .comparison-arrow { + transform: rotate(90deg); + } + + .comparison-header { + flex-direction: column; + align-items: stretch; + text-align: center; + } +} diff --git a/packages/arkham-shard-shell/src/pages/provenance/ProvenancePage.tsx b/packages/arkham-shard-shell/src/pages/provenance/ProvenancePage.tsx index c0754d5..75a514d 100644 --- a/packages/arkham-shard-shell/src/pages/provenance/ProvenancePage.tsx +++ b/packages/arkham-shard-shell/src/pages/provenance/ProvenancePage.tsx @@ -10,6 +10,7 @@ import { Icon } from '../../components/common/Icon'; import { AIAnalystButton } from '../../components/AIAnalyst'; import { useToast } from '../../context/ToastContext'; import { useFetch } from '../../hooks/useFetch'; +import { ForensicsTab } from './ForensicsTab'; import './ProvenancePage.css'; // Types @@ -81,7 +82,7 @@ interface LineageGraph { descendant_count: number; } -type MainTab = 'artifacts' | 'chains' | 'lineage'; +type MainTab = 'artifacts' | 'chains' | 'lineage' | 'forensics'; export function ProvenancePage() { const { toast } = useToast(); @@ -694,6 +695,13 @@ export function ProvenancePage() { Lineage +
{/* Tab Content */} @@ -701,6 +709,7 @@ export function ProvenancePage() { {mainTab === 'artifacts' && renderArtifactsTab()} {mainTab === 'chains' && renderChainsTab()} {mainTab === 'lineage' && renderLineageTab()} + {mainTab === 'forensics' && } {/* Create Chain Dialog */} From c65390946b7e45f0f5a8e58577fd988136c87b98 Mon Sep 17 00:00:00 2001 From: mantisfury Date: Thu, 15 Jan 2026 01:43:26 -0700 Subject: [PATCH 4/4] Fix: Improve Deception Analysis Impact and Embedding Statistics Credibility Shard - Deception Analysis Improvements: - Increase default credibility_weight from 0.3 to 0.7 for meaningful impact - Add severity override system: critical deception (>=80%) caps credibility at 15, high (>=60%) caps at 30, moderate (>=40%) caps at 50 - Auto-link deception assessments to existing credibility assessments - Fix DeceptionPanel to pass linked_assessment_id when creating assessments - Fix DeceptionPanel to reset state when switching between documents Embed Shard - Statistics Fix: - Replace in-memory cache stats with actual database metrics - Show total embeddings, pending chunks, coverage rate, and document counts - Stats now reflect real embedding coverage instead of always showing zeros - Update UI labels: "Embeddings", "Pending", "Coverage", "Docs / Chunks" Co-Authored-By: Claude Opus 4.5 --- .../arkham_shard_credibility/api.py | 2 +- .../arkham_shard_credibility/models.py | 27 ++++++-- .../arkham_shard_credibility/shard.py | 69 +++++++++++++++++-- .../arkham_shard_embed/api.py | 60 ++++++++++++++-- .../credibility/components/DeceptionPanel.tsx | 6 +- .../src/pages/embed/EmbedPage.tsx | 22 +++--- 6 files changed, 159 insertions(+), 27 deletions(-) diff --git a/packages/arkham-shard-credibility/arkham_shard_credibility/api.py b/packages/arkham-shard-credibility/arkham_shard_credibility/api.py index af5a40b..78abd15 100644 --- a/packages/arkham-shard-credibility/arkham_shard_credibility/api.py +++ b/packages/arkham-shard-credibility/arkham_shard_credibility/api.py @@ -516,7 +516,7 @@ class DeceptionAssessmentCreate(BaseModel): source_name: Optional[str] = Field(None, description="Human-readable source name") linked_assessment_id: Optional[str] = None affects_credibility: bool = True - credibility_weight: float = Field(0.3, ge=0.0, le=1.0) + credibility_weight: float = Field(0.7, ge=0.0, le=1.0, description="Weight in credibility calc (higher = more impact)") class DeceptionChecklistUpdate(BaseModel): diff --git a/packages/arkham-shard-credibility/arkham_shard_credibility/models.py b/packages/arkham-shard-credibility/arkham_shard_credibility/models.py index cf38522..cba643d 100644 --- a/packages/arkham-shard-credibility/arkham_shard_credibility/models.py +++ b/packages/arkham-shard-credibility/arkham_shard_credibility/models.py @@ -352,7 +352,16 @@ def to_dict(self) -> Dict[str, Any]: } def calculate_score(self) -> int: - """Calculate checklist score from indicators.""" + """ + Calculate checklist score from indicators. + + Only assessed indicators (strength != NONE) contribute to the score. + This prevents unassessed indicators from diluting the deception signal. + + Returns 0-100 where: + - 0 = no deception indicators found + - 100 = conclusive deception indicators across all assessed questions + """ if not self.indicators: return 0 @@ -364,11 +373,21 @@ def calculate_score(self) -> int: IndicatorStrength.CONCLUSIVE: 100, } + # Only count indicators that have been assessed (not NONE) + assessed_indicators = [ + ind for ind in self.indicators + if ind.strength != IndicatorStrength.NONE + ] + + if not assessed_indicators: + return 0 + + # Calculate score based only on assessed indicators total = sum( strength_scores.get(ind.strength, 0) * ind.confidence - for ind in self.indicators + for ind in assessed_indicators ) - max_possible = len(self.indicators) * 100 + max_possible = len(assessed_indicators) * 100 return int((total / max_possible) * 100) if max_possible > 0 else 0 @@ -399,7 +418,7 @@ class DeceptionAssessment: # Integration with main credibility linked_assessment_id: Optional[str] = None # Link to CredibilityAssessment affects_credibility: bool = True # Whether to factor into credibility score - credibility_weight: float = 0.3 # Weight in credibility calculation + credibility_weight: float = 0.7 # Weight in credibility calculation (increased for meaningful impact) # Assessment metadata assessed_by: AssessmentMethod = AssessmentMethod.MANUAL diff --git a/packages/arkham-shard-credibility/arkham_shard_credibility/shard.py b/packages/arkham-shard-credibility/arkham_shard_credibility/shard.py index 215c9d8..57d50d6 100644 --- a/packages/arkham-shard-credibility/arkham_shard_credibility/shard.py +++ b/packages/arkham-shard-credibility/arkham_shard_credibility/shard.py @@ -946,6 +946,33 @@ def _parse_llm_assessment(self, response: str) -> Dict[str, Any]: return {"score": 50, "factors": []} + async def _find_existing_credibility_assessment( + self, + source_type: SourceType, + source_id: str, + ) -> Optional[CredibilityAssessment]: + """Find an existing credibility assessment for a source (most recent).""" + if not self._db: + return None + + query = """ + SELECT * FROM arkham_credibility_assessments + WHERE source_type = :source_type AND source_id = :source_id + """ + params = {"source_type": source_type.value, "source_id": source_id} + + tenant_id = self.get_tenant_id_or_none() + if tenant_id: + query += " AND tenant_id = :tenant_id" + params["tenant_id"] = str(tenant_id) + + query += " ORDER BY created_at DESC LIMIT 1" + + row = await self._db.fetch_one(query, params) + if row: + return self._row_to_assessment(row) + return None + # ========================================================================= # DECEPTION DETECTION (MOM/POP/MOSES/EVE) METHODS # ========================================================================= @@ -957,12 +984,19 @@ async def create_deception_assessment( source_name: Optional[str] = None, linked_assessment_id: Optional[str] = None, affects_credibility: bool = True, - credibility_weight: float = 0.3, + credibility_weight: float = 0.7, ) -> DeceptionAssessment: """Create a new deception detection assessment with empty checklists.""" assessment_id = str(uuid4()) now = datetime.utcnow() + # Auto-link to existing credibility assessment if none provided + if not linked_assessment_id and affects_credibility: + existing = await self._find_existing_credibility_assessment(source_type, source_id) + if existing: + linked_assessment_id = existing.id + logger.info(f"Auto-linked deception assessment to existing credibility {existing.id}") + # Create empty checklists with standard indicators mom_checklist = create_empty_checklist(DeceptionChecklistType.MOM) pop_checklist = create_empty_checklist(DeceptionChecklistType.POP) @@ -1361,7 +1395,7 @@ async def _sync_deception_to_credibility(self, deception: DeceptionAssessment) - - 100 = strong deception indicators Credibility score is inverted: high deception = low credibility - The deception score is weighted by credibility_weight (default 0.3). + The deception score is weighted by credibility_weight (default 0.7). """ if not deception.affects_credibility: return @@ -1377,8 +1411,14 @@ async def _sync_deception_to_credibility(self, deception: DeceptionAssessment) - deception_credibility = 100 - deception.overall_score # Create credibility factors from completed checklists + # Use the same weights as the deception assessment calculation + checklist_weights = { + "MOM Analysis": 0.35, # MOM most heavily weighted + "EVE Analysis": 0.25, # Evidence evaluation second + "MOSES Analysis": 0.25, # Manipulability third + "POP Analysis": 0.15, # Past practices least (often unknown) + } factors = [] - checklist_weight = 0.25 # Each checklist gets equal weight for checklist, name in [ (deception.mom_checklist, "MOM Analysis"), @@ -1391,7 +1431,7 @@ async def _sync_deception_to_credibility(self, deception: DeceptionAssessment) - checklist_credibility = 100 - checklist.overall_score factors.append(CredibilityFactor( factor_type=f"deception_{name.lower().replace(' ', '_')}", - weight=checklist_weight, + weight=checklist_weights[name], score=checklist_credibility, notes=f"{name}: {checklist.risk_level} risk ({checklist.overall_score}% deception indicators)", )) @@ -1406,6 +1446,25 @@ async def _sync_deception_to_credibility(self, deception: DeceptionAssessment) - else: final_score = deception_credibility + # SEVERITY OVERRIDE: High deception scores force low credibility + # This ensures that conclusive/strong deception evidence dramatically impacts credibility + # regardless of how other factors might average out + if deception.overall_score >= 80: # Critical risk + # Conclusive deception: cap credibility at 15 (Unreliable) + final_score = min(final_score, 15) + logger.info(f"Deception severity override: critical risk ({deception.overall_score}%) " + f"capped credibility at {final_score}") + elif deception.overall_score >= 60: # High risk + # Strong deception: cap credibility at 30 (Low) + final_score = min(final_score, 30) + logger.info(f"Deception severity override: high risk ({deception.overall_score}%) " + f"capped credibility at {final_score}") + elif deception.overall_score >= 40: # Moderate risk + # Moderate deception: cap credibility at 50 (Medium) + final_score = min(final_score, 50) + logger.info(f"Deception severity override: moderate risk ({deception.overall_score}%) " + f"capped credibility at {final_score}") + # Confidence based on how many checklists were completed confidence = deception.confidence if deception.confidence > 0 else 0.5 @@ -1573,7 +1632,7 @@ def parse_checklist(data_str: str, checklist_type: DeceptionChecklistType) -> Op confidence=row.get("confidence", 0.0), linked_assessment_id=row.get("linked_assessment_id"), affects_credibility=bool(row.get("affects_credibility", 1)), - credibility_weight=row.get("credibility_weight", 0.3), + credibility_weight=row.get("credibility_weight", 0.7), assessed_by=AssessmentMethod(row.get("assessed_by", "manual")), assessor_id=row.get("assessor_id"), summary=row.get("summary"), diff --git a/packages/arkham-shard-embed/arkham_shard_embed/api.py b/packages/arkham-shard-embed/arkham_shard_embed/api.py index 15b1a14..6a63ea0 100644 --- a/packages/arkham-shard-embed/arkham_shard_embed/api.py +++ b/packages/arkham-shard-embed/arkham_shard_embed/api.py @@ -709,13 +709,63 @@ async def update_config(request: ConfigUpdateRequest): @router.get("/cache/stats") async def get_cache_stats(): - """Get embedding cache statistics.""" - if not _embedding_manager: - raise HTTPException(status_code=503, detail="Embedding service not initialized") + """ + Get embedding statistics. + Returns actual embedding counts from the vector store database, + which is more meaningful than in-memory cache stats since most + embeddings are processed through the worker pool. + """ try: - cache_info = _embedding_manager.get_cache_info() - return cache_info + # Get in-memory cache info if available + cache_info = {} + if _embedding_manager: + cache_info = _embedding_manager.get_cache_info() + + # Get actual embedding stats from database + total_embeddings = 0 + total_documents = 0 + total_chunks = 0 + + if _db_service: + try: + # Count total embeddings in vector store + embed_count = await _db_service.fetch_one( + "SELECT COUNT(*) as count FROM arkham_vectors.embeddings" + ) + total_embeddings = embed_count.get("count", 0) if embed_count else 0 + + # Count unique documents with embeddings + doc_count = await _db_service.fetch_one( + """SELECT COUNT(DISTINCT payload->>'document_id') as count + FROM arkham_vectors.embeddings + WHERE payload->>'document_id' IS NOT NULL""" + ) + total_documents = doc_count.get("count", 0) if doc_count else 0 + + # Count total chunks in system + chunk_count = await _db_service.fetch_one( + "SELECT COUNT(*) as count FROM arkham_frame.chunks" + ) + total_chunks = chunk_count.get("count", 0) if chunk_count else 0 + except Exception as db_err: + logger.debug(f"Could not get embedding counts from DB: {db_err}") + + # Calculate coverage rate (embeddings vs chunks) + coverage_rate = total_embeddings / total_chunks if total_chunks > 0 else 0.0 + + return { + "enabled": cache_info.get("enabled", False), + "hits": total_embeddings, # Total embeddings stored + "misses": max(0, total_chunks - total_embeddings), # Chunks not yet embedded + "size": total_documents, # Documents with embeddings + "max_size": total_chunks, # Total chunks available + "hit_rate": coverage_rate, # Embedding coverage percentage + # Additional context + "total_embeddings": total_embeddings, + "total_documents": total_documents, + "total_chunks": total_chunks, + } except Exception as e: logger.error(f"Failed to get cache stats: {e}", exc_info=True) raise HTTPException(status_code=500, detail=f"Failed to get cache stats: {str(e)}") diff --git a/packages/arkham-shard-shell/src/pages/credibility/components/DeceptionPanel.tsx b/packages/arkham-shard-shell/src/pages/credibility/components/DeceptionPanel.tsx index 135f44b..3784c19 100644 --- a/packages/arkham-shard-shell/src/pages/credibility/components/DeceptionPanel.tsx +++ b/packages/arkham-shard-shell/src/pages/credibility/components/DeceptionPanel.tsx @@ -42,6 +42,7 @@ const CHECKLIST_TYPES: DeceptionChecklistType[] = ['mom', 'pop', 'moses', 'eve'] export function DeceptionPanel({ sourceType, sourceId, + credibilityAssessmentId, onRiskChange, }: DeceptionPanelProps) { const { toast } = useToast(); @@ -110,6 +111,8 @@ export function DeceptionPanel({ useEffect(() => { const fetchData = async () => { setLoading(true); + // Reset assessment state when source changes to avoid showing stale data + setAssessment(null); try { // Fetch source info for display await fetchSourceInfo(); @@ -161,9 +164,10 @@ export function DeceptionPanel({ source_type: sourceType, source_id: sourceId, source_name: sourceInfo?.name, + linked_assessment_id: credibilityAssessmentId, assessed_by: 'manual', affects_credibility: true, - credibility_weight: 0.3, + credibility_weight: 0.7, // Higher weight for meaningful impact }), }); diff --git a/packages/arkham-shard-shell/src/pages/embed/EmbedPage.tsx b/packages/arkham-shard-shell/src/pages/embed/EmbedPage.tsx index 09bed89..4c3ee5c 100644 --- a/packages/arkham-shard-shell/src/pages/embed/EmbedPage.tsx +++ b/packages/arkham-shard-shell/src/pages/embed/EmbedPage.tsx @@ -691,12 +691,12 @@ export function EmbedPage() { )}
- {/* Cache Statistics */} + {/* Embedding Statistics */}

- Cache Statistics + Embedding Statistics

) : cacheStats ? (
- +
{formatNumber(cacheStats.hits)}
-
Hits
+
Embeddings
- +
{formatNumber(cacheStats.misses)}
-
Misses
+
Pending
@@ -744,24 +744,24 @@ export function EmbedPage() {
{(cacheStats.hit_rate * 100).toFixed(1)}%
-
Hit Rate
+
Coverage
- +
{cacheStats.size}/{cacheStats.max_size}
-
Cache Size
+
Docs / Chunks
) : (
-

No cache statistics available

+

No statistics available

)}