ai-appsec-analyzer.py

#!/usr/bin/env python3

import dspy
import litellm 
import json
import os
import argparse
from typing import List, Dict, Any, Optional, Tuple
from collections import defaultdict
import fnmatch # For ignore pattern matching

# --- Configuration ---
DEFAULT_MODEL_NAME = ""
DEFAULT_API_BASE_URL = ""
# API Key should be set via --api_key argument or API_KEY environment variable.

DEFAULT_CHUNK_SIZE = 12000
DEFAULT_CHUNK_OVERLAP = 500
DEFAULT_MAX_CHARS_NO_CHUNK = 600000 
DEFAULT_MAX_CONSOLIDATOR_INPUT_CHARS = 750000

RECURSIVE_MIN_CHUNKS_TO_SUBDIVIDE = 4 
RECURSIVE_MAX_DEPTH = 3             
RECURSIVE_MAX_CHUNKS_FOR_LEAF_ANALYSIS = 5 
RECURSIVE_SUBDIVISION_FACTOR = 2 

# max_tokens will now be set to the highest needed value and used for all calls
DEFAULT_MAX_OUTPUT_TOKENS = 60000 


# --- 1. File Reader, Content Type Detection, and Initial Chunker ---

def read_file_content(file_path: str) -> Optional[str]:
    """Reads file content with debugging."""
    absolute_path = os.path.abspath(os.path.expanduser(os.path.expandvars(file_path)))
    # print(f"Attempting to read file from resolved absolute path: {absolute_path}") # Verbose

    if not os.path.exists(absolute_path):
        print(f"Error: Path does not exist: {absolute_path}")
        print(f"Current working directory: {os.getcwd()}")
        return None
    if not os.path.isfile(absolute_path):
        print(f"Error: Path is not a file: {absolute_path}")
        return None
    if not os.access(absolute_path, os.R_OK):
        print(f"Error: No read permission for file: {absolute_path}")
        return None
    try:
        with open(absolute_path, 'r', encoding='utf-8') as f:
            content = f.read()
        # print(f"Successfully read file: {absolute_path} (UTF-8)") # Verbose
        return content
    except UnicodeDecodeError: 
        print(f"Warning: UTF-8 decoding failed for {absolute_path}. Trying latin-1.")
        try:
            with open(absolute_path, 'r', encoding='latin-1') as f:
                content = f.read()
            # print(f"Successfully read file: {absolute_path} (latin-1)") # Verbose
            return content
        except Exception as e_latin1:
            print(f"Error: Could not read file {absolute_path} even with latin-1. Error: {e_latin1}")
            return None
    except Exception as e:
        print(f"An unexpected error occurred while reading file {absolute_path}: {e}")
        return None

def detect_content_type(file_path: str, content_sample: str = "") -> str:
    """Simple heuristic to detect content type."""
    _, extension = os.path.splitext(file_path)
    extension = extension.lower()
    if extension in ['.py', '.python']: return "Python Source Code"
    if extension in ['.java']: return "Java Source Code"
    if extension in ['.js', '.ts', '.jsx', '.tsx']: return "JavaScript/TypeScript Source Code"
    if extension in ['.c', '.cpp', '.h', '.hpp']: return "C/C++ Source Code"
    if extension in ['.cs']: return "C# Source Code"
    if extension in ['.go']: return "Go Source Code"
    if extension in ['.rb']: return "Ruby Source Code"
    if extension in ['.php']: return "PHP Source Code"
    if extension in ['.swift']: return "Swift Source Code"
    if extension in ['.kt', '.kts']: return "Kotlin Source Code"
    if extension in ['.json']:
        if '"info":' in content_sample[:1000] and '"item":' in content_sample[:1000]: return "Postman Collection (JSON)"
        if '"openapi":' in content_sample[:1000] or '"swagger":' in content_sample[:1000]: return "OpenAPI/Swagger Specification (JSON)"
        return "JSON Data/Configuration"
    if extension in ['.yaml', '.yml']:
        if 'openapi:' in content_sample[:1000] or 'swagger:' in content_sample[:1000]: return "OpenAPI/Swagger Specification (YAML)"
        return "YAML Configuration"
    if extension in ['.xml']:
        if '<soap:Envelope' in content_sample[:1000] or '<soapenv:Envelope' in content_sample[:1000]: return "SOAP XML"
        return "XML Data/Configuration"
    if extension in ['.conf', '.cfg', '.ini', '.properties']: return "Generic Configuration File"
    if extension in ['.tf', '.tfvars']: return "Terraform HCL"
    if extension in ['.md', '.txt', '.rst', '.adoc']: return "Documentation/Text File"
    if extension in ['.sh', '.bash']: return "Shell Script"
    if extension in ['.sql']: return "SQL Script"
    if content_sample.strip().startswith("{") or content_sample.strip().startswith("["): return "Likely JSON Data"
    if content_sample.strip().startswith("<"): return "Likely XML Data"
    return "Generic Text File"


def initial_chunk_file_content(text: str, chunk_size: int, chunk_overlap: int) -> List[str]:
    """Splits text into initial chunks if it's very large."""
    if not text: return []
    chunks = []
    start_index = 0
    text_len = len(text)
    while start_index < text_len:
        end_index = min(start_index + chunk_size, text_len)
        chunks.append(text[start_index:end_index])
        if end_index == text_len: break
        start_index += (chunk_size - chunk_overlap)
        if start_index >= text_len - chunk_overlap and start_index < text_len:
            if text_len - start_index > chunk_overlap / 2: 
                chunks.append(text[start_index:])
            break
        elif start_index >= text_len: break
    return chunks

# --- 2. DSPy Signatures and Modules ---

class ProduceGistForSecurityContext(dspy.Signature):
    """
    You are a text summarizer AI. Your task is to create a concise, factual gist of a given text chunk, focusing on what it IS, not what it does wrong. This gist provides context for other expert agents.

    **Instructions:**
    1.  **Identify Content Type:** State the most likely type of content (e.g., Python function, JSON configuration, API route definition, technical documentation, user story).
    2.  **Describe Purpose:** Summarize the main purpose, functionality, or topic described in the chunk in 1-2 sentences.
    3.  **List Key Entities:** Note any key technologies, libraries, function names, or services explicitly mentioned.
    4.  **Do NOT analyze for security vulnerabilities.** Do not list findings or suggest improvements. Stick to summarizing the provided context.
    5.  Do NOT infer functionality beyond what is explicitly stated in the chunk.
    """
    toc_path_context: str = dspy.InputField(desc="The file path or table of contents entry for context.")
    chunk_content: str = dspy.InputField(desc="The raw text content of the code, config, or documentation chunk.")
    security_gist: str = dspy.OutputField(desc="A detailed yet concise gist (target 2-4 sentences) describing the chunk's content and purpose for contextual understanding.")

class GeneralOverviewSignature(dspy.Signature):
    """
    You are the General Analyzer AI agent for static application security analysis.
    Your role is to process the provided text_content. Provide a high-level overview of what the content appears to be
    (e.g., source code, API spec, config, docs) and identify any immediately obvious, major security concerns or areas
    that warrant deeper inspection by specialized modules. Do not perform a deep dive yourself, but rather set the stage.
    """
    text_content = dspy.InputField(desc="The text content (full file or a chunk) to be initially assessed.")
    file_context_description = dspy.InputField(desc="A brief description of the source file (e.g., 'File: example.py', 'Chunk 1 of large_api_spec.json').")
    preliminary_assessment_markdown = dspy.OutputField(desc="A brief Markdown summary of the content type and any high-level security observations or areas of interest for specialized analysis. If no immediate major concerns, state that.")

class CodeAnalysisSignature(dspy.Signature):
    """
    You are the Code Module of the General Analyzer. Your role is to analyze source code for security vulnerabilities,
    bugs, and violations of best practices. Identify the programming language and frameworks used if possible.
    Look for issues such as improper input validation, insecure data handling (PII, secrets), weak authentication/authorization,
    injection flaws (SQLi, XSS, Command Injection), insecure cryptographic usage, and other common coding errors.
    Provide detailed explanations and suggest remediation steps in Markdown. Ensure findings clearly indicate the specific code area.
    """
    code_snippet = dspy.InputField(desc="A snippet or full body of source code.")
    language_context = dspy.InputField(desc="Context about the file, including detected language if known (e.g., 'Python Source Code from user_auth.py', 'Chunk of Java Spring Boot controller').")
    code_findings_markdown = dspy.OutputField(desc="Markdown list of security findings in the code. For each: describe the issue, affected lines/area (if inferable), impact, and remediation. If none, state 'No specific code vulnerabilities identified in this snippet.'")

class ApiAnalysisSignature(dspy.Signature):
    """
    You are the API Module of the General Analyzer. Analyze API specifications (e.g., Postman collections, OpenAPI/Swagger)
    or inferred API interactions from code/logs for security issues. Focus on OWASP API Security Top 10.
    Report findings with severity and remediation in Markdown. Ensure findings clearly indicate the specific API element or area.
    """
    api_description_or_spec_chunk = dspy.InputField(desc="A chunk of an API specification or text describing API interactions.")
    api_context = dspy.InputField(desc="Context about the API content (e.g., 'Postman collection request body', 'OpenAPI path definition', 'Chunk of API documentation').")
    api_findings_markdown = dspy.OutputField(desc="Markdown list of API security findings. For each: describe the issue, relevant OWASP API category, impact, and remediation. If none, state 'No specific API security issues identified in this segment.'")

class ConfigAnalysisSignature(dspy.Signature):
    """
    You are the Configuration Module of the General Analyzer. Analyze configuration files for security misconfigurations.
    Identify settings that could lead to vulnerabilities, such as default credentials, weak encryption, exposed sensitive information,
    or insecure access controls. Provide detailed explanations and suggest secure settings in Markdown. Ensure findings specify the configuration item.
    """
    config_content_chunk = dspy.InputField(desc="A chunk of a configuration file.")
    config_context = dspy.InputField(desc="Context about the configuration file (e.g., 'Kubernetes deployment YAML chunk', 'JSON application settings', 'Apache httpd.conf snippet').")
    config_findings_markdown = dspy.OutputField(desc="Markdown list of configuration security findings. For each: describe the misconfiguration, potential impact, and recommended secure setting. If none, state 'No specific configuration issues identified in this segment.'")

class DocsAnalysisSignature(dspy.Signature):
    """
    You are the Documentation Module of the General Analyzer. Analyze documentation for adherence to security best practices.
    Check for missing security considerations, such as lack of guidance on secure coding, API security, or data protection.
    Provide suggestions for improving the documentation to better address security concerns in Markdown.
    """
    documentation_chunk = dspy.InputField(desc="A chunk of documentation text.")
    docs_context = dspy.InputField(desc="Context about the documentation (e.g., 'Section from API usage guide', 'README.md for a service', 'Security policy document chunk').")
    docs_findings_markdown = dspy.OutputField(desc="Markdown list of documentation security gaps or improvement suggestions. For each: describe the missing information or weakness and suggest content to add/improve. If none, state 'Documentation segment appears adequate regarding security mentions for its scope.'")

class ThreatModelingSignature(dspy.Signature):
    """
    You are the Threat Modeler AI agent. Based on the provided text_content (which could be code, API specs, configs, or docs),
    identify potential threats, attack vectors, and vulnerable assets. Briefly outline a threat model perspective.
    Focus on what could go wrong and who might attack it.
    Output should be a concise Markdown summary of potential threats, trying to link them to the content provided.
    """
    text_content = dspy.InputField(desc="A chunk of application-related text (code, API, config, docs).")
    context_description = dspy.InputField(desc="Context about the content to aid threat modeling.")
    threat_model_summary_markdown = dspy.OutputField(desc="Markdown summary of potential threats, attack vectors, and vulnerable assets identified from the text. If difficult to infer, state that. Example: '- Threat: Unauthorized data access via API. Vector: Missing auth on /data endpoint. Asset: User PII.'")

class ComplianceCheckSignature(dspy.Signature):
    """
    You are the Compliance Checker AI agent. Review the provided text_content against common security standards
    like OWASP Top 10 (Web), OWASP API Security Top 10, and general principles from standards like PCI-DSS or HIPAA if applicable
    (e.g., handling of sensitive data, access controls).
    Identify potential deviations or areas where compliance might be a concern.
    Output a concise Markdown list of potential compliance issues, linking to parts of the text if possible.
    """
    text_content = dspy.InputField(desc="A chunk of application-related text.")
    context_description = dspy.InputField(desc="Context about the content to aid compliance checking.")
    compliance_issues_markdown = dspy.OutputField(desc="Markdown list of potential compliance issues or areas misaligned with common security standards (OWASP Top 10, OWASP API Top 10, general PII/PCI principles). If none apparent, state 'No specific compliance concerns readily identifiable from this segment.'")

class ConsolidateExpertFindings(dspy.Signature):
    """
    You are an expert technical writer specializing in cybersecurity reports.
    You are given a collection of security findings and assessments from a specific security expert (e.g., Code Analyzer, Threat Modeler)
    that were generated by analyzing different parts/chunks of one or more source files within a project/directory.
    Your task is to synthesize these potentially repetitive or fragmented findings into a single, cohesive, well-organized,
    and de-duplicated Markdown section for that expert's analysis of the entire project/directory.

    - Identify recurring themes or types of vulnerabilities/observations across all provided contexts.
    - Group similar findings. Summarize if multiple instances of the exact same minor issue occur across different contexts/files,
      but detail unique significant findings. For recurring issues, you can mention the pattern and then list a few example contexts/files where it was observed.
    - Remove redundant phrasing while preserving all unique critical information.
    - The input `raw_findings_with_context_str` will contain findings, each prefixed by its original context (e.g., "Context: File.py / Part 1 / Chunk X...").
      Use this contextual information to inform your synthesis. For example:
      "Input validation issues were prevalent. Specifically:
        - In `user_controller.py` (related to original context '...Chunk X'), user input for 'username' is not sanitized.
        - Similar lack of sanitization was observed for 'email' in `profile_handler.py` (related to original context '...Chunk Y')."
    - Ensure the final output is a clear, readable, and actionable Markdown section.
    - If the raw findings mostly consist of errors or "no findings", reflect that appropriately in a brief summary for this expert.
    """
    expert_name = dspy.InputField(desc="The name of the security expert whose findings are being consolidated.")
    overall_project_context = dspy.InputField(desc="General context of the project/directory being analyzed (e.g., 'Directory: /src/auth_service, primarily Python and JSON files').")
    raw_findings_with_context_str = dspy.InputField(desc="A single string containing all raw findings from this expert across multiple files/chunks, where each finding is prefixed by its original file and chunk/part context.")
    consolidated_markdown_section = dspy.OutputField(desc="A single, cohesive, de-duplicated, and well-organized Markdown section representing the expert's complete analysis for the entire project/directory.")


# Expert Modules - now inherit the global LLM from dspy.settings
class GeneralOverviewModule(dspy.Module):
    def __init__(self): super().__init__(); self.generate_overview = dspy.ChainOfThought(GeneralOverviewSignature)
    def forward(self, text_content: str, file_context_description: str) -> dspy.Prediction: return self.generate_overview(text_content=text_content, file_context_description=file_context_description)
class CodeAnalysisModule(dspy.Module):
    def __init__(self): super().__init__(); self.analyze_code = dspy.ChainOfThought(CodeAnalysisSignature)
    def forward(self, code_snippet: str, language_context: str) -> dspy.Prediction: return self.analyze_code(code_snippet=code_snippet, language_context=language_context)
class ApiAnalysisModule(dspy.Module):
    def __init__(self): super().__init__(); self.analyze_api = dspy.ChainOfThought(ApiAnalysisSignature)
    def forward(self, api_description_or_spec_chunk: str, api_context: str) -> dspy.Prediction: return self.analyze_api(api_description_or_spec_chunk=api_description_or_spec_chunk, api_context=api_context)
class ConfigAnalysisModule(dspy.Module):
    def __init__(self): super().__init__(); self.analyze_config = dspy.ChainOfThought(ConfigAnalysisSignature)
    def forward(self, config_content_chunk: str, config_context: str) -> dspy.Prediction: return self.analyze_config(config_content_chunk=config_content_chunk, config_context=config_context)
class DocsAnalysisModule(dspy.Module):
    def __init__(self): super().__init__(); self.analyze_docs = dspy.ChainOfThought(DocsAnalysisSignature)
    def forward(self, documentation_chunk: str, docs_context: str) -> dspy.Prediction: return self.analyze_docs(documentation_chunk=documentation_chunk, docs_context=docs_context)
class ThreatModelingModule(dspy.Module):
    def __init__(self): super().__init__(); self.model_threats = dspy.ChainOfThought(ThreatModelingSignature)
    def forward(self, text_content: str, context_description: str) -> dspy.Prediction: return self.model_threats(text_content=text_content, context_description=context_description)
class ComplianceCheckingModule(dspy.Module):
    def __init__(self): super().__init__(); self.check_compliance = dspy.ChainOfThought(ComplianceCheckSignature)
    def forward(self, text_content: str, context_description: str) -> dspy.Prediction: return self.check_compliance(text_content=text_content, context_description=context_description)
class FindingsConsolidatorModule(dspy.Module):
    def __init__(self): super().__init__(); self.consolidate = dspy.ChainOfThought(ConsolidateExpertFindings)
    def forward(self, expert_name: str, overall_project_context: str, raw_findings_with_context_str: str) -> dspy.Prediction: return self.consolidate(expert_name=expert_name, overall_project_context=overall_project_context, raw_findings_with_context_str=raw_findings_with_context_str)

gist_producer_module = None 

def initialize_structuring_modules():
    global gist_producer_module
    gist_producer_module = dspy.Predict(ProduceGistForSecurityContext)


# --- 3. Recursive Analysis and Reporting Logic ---

def _apply_expert_analysis_to_leaf_content(
    content_to_analyze: str,
    current_leaf_context_str: str, 
    original_file_path_for_leaf: str, 
    detected_content_type: str,
    expert_modules: Dict[str, dspy.Module],
    aggregated_findings_collector: Dict[str, List[Tuple[str, str, Optional[str]]]], 
    is_aggregated_content: bool 
) -> Dict[str, Any]: 
    expert_json_outputs_for_this_leaf = []
    active_experts_to_run = {
        "General Overview": expert_modules["General Overview"],
        "Threat Modeler": expert_modules["Threat Modeler"],
        "Compliance Checker": expert_modules["Compliance Checker"]
    }
    if "Code" in detected_content_type: active_experts_to_run["Code Module"] = expert_modules["Code Module"]
    if "API" in detected_content_type or "JSON" in detected_content_type or "XML" in detected_content_type or "Postman" in detected_content_type or "OpenAPI" in detected_content_type:
        active_experts_to_run["API Module"] = expert_modules["API Module"]
    if "Configuration" in detected_content_type or "Config" in detected_content_type or ".conf" in original_file_path_for_leaf.lower() or ".cfg" in original_file_path_for_leaf.lower() or ".ini" in original_file_path_for_leaf.lower() or "YAML" in detected_content_type or "HCL" in detected_content_type:
        active_experts_to_run["Configuration Module"] = expert_modules["Configuration Module"]
    if "Documentation" in detected_content_type or ".md" in original_file_path_for_leaf.lower() or ".txt" in original_file_path_for_leaf.lower() or ".rst" in original_file_path_for_leaf.lower():
        active_experts_to_run["Documentation Module"] = expert_modules["Documentation Module"]

    contextual_gist_for_experts = ""
    if is_aggregated_content and gist_producer_module and len(content_to_analyze) > 500: 
        try:
            print(f"      Generating context gist for aggregated content: {current_leaf_context_str}")
            gist_input_content = content_to_analyze
            if len(gist_input_content) > 8000: 
                gist_input_content = content_to_analyze[:4000] + "\n...\n" + content_to_analyze[-4000:]
            gist_pred = gist_producer_module(toc_path_context=current_leaf_context_str, chunk_content=gist_input_content)
            contextual_gist_for_experts = gist_pred.security_gist
        except Exception as e_gist:
            print(f"      Warning: Could not generate context gist for aggregated content: {e_gist}")

    for expert_name, expert_module_instance in active_experts_to_run.items():
        print(f"      Invoking {expert_name} for: {current_leaf_context_str}...")
        output_md, rationale_str, analysis_obj, error_msg = "", None, None, None
        content_for_this_expert = content_to_analyze
        context_for_expert_prompt = f"{current_leaf_context_str} (Original File Type: {detected_content_type})"
        if contextual_gist_for_experts and expert_name != "General Overview": 
            context_for_expert_prompt += f"\nBrief Gist of this content block being analyzed: {contextual_gist_for_experts}"
        try:
            if not content_to_analyze.strip():
                output_md = "- Skipped analysis as content was empty or whitespace."
            else:
                if expert_name == "General Overview":
                    analysis_obj = expert_module_instance.forward(text_content=content_for_this_expert, file_context_description=context_for_expert_prompt)
                    output_md = analysis_obj.preliminary_assessment_markdown
                elif expert_name == "Code Module":
                    analysis_obj = expert_module_instance.forward(code_snippet=content_for_this_expert, language_context=context_for_expert_prompt)
                    output_md = analysis_obj.code_findings_markdown
                elif expert_name == "API Module":
                    analysis_obj = expert_module_instance.forward(api_description_or_spec_chunk=content_for_this_expert, api_context=context_for_expert_prompt)
                    output_md = analysis_obj.api_findings_markdown
                elif expert_name == "Configuration Module":
                    analysis_obj = expert_module_instance.forward(config_content_chunk=content_for_this_expert, config_context=context_for_expert_prompt)
                    output_md = analysis_obj.config_findings_markdown
                elif expert_name == "Documentation Module":
                    analysis_obj = expert_module_instance.forward(documentation_chunk=content_for_this_expert, docs_context=context_for_expert_prompt)
                    output_md = analysis_obj.docs_findings_markdown
                elif expert_name == "Threat Modeler":
                    analysis_obj = expert_module_instance.forward(text_content=content_for_this_expert, context_description=context_for_expert_prompt)
                    output_md = analysis_obj.threat_model_summary_markdown
                elif expert_name == "Compliance Checker":
                    analysis_obj = expert_module_instance.forward(text_content=content_for_this_expert, context_description=context_for_expert_prompt)
                    output_md = analysis_obj.compliance_issues_markdown
            
            if analysis_obj and hasattr(analysis_obj, 'rationale') and analysis_obj.rationale:
                rationale_str = str(analysis_obj.rationale).strip()
            
            current_finding_md = output_md if output_md else "- No specific findings or assessment reported by this expert."
            aggregated_findings_collector[expert_name].append((current_leaf_context_str, current_finding_md, rationale_str))
            expert_json_outputs_for_this_leaf.append({"expert": expert_name, "output_markdown": current_finding_md, "rationale": rationale_str, "error": None})
        except Exception as e:
            print(f"        Error during {expert_name} analysis for '{current_leaf_context_str}': {e}")
            error_msg = str(e)
            aggregated_findings_collector[expert_name].append((current_leaf_context_str, f"- Error during {expert_name} analysis: {e}\n", None))
            expert_json_outputs_for_this_leaf.append({"expert": expert_name, "output_markdown": None, "rationale": None, "error": error_msg})
    return {"expert_analyses_for_leaf": expert_json_outputs_for_this_leaf}


def recursive_structured_analysis(
    current_toc_path: List[str],
    chunks_for_this_level: List[str],
    original_file_path_for_reports: str,
    detected_content_type: str,
    all_expert_modules: Dict[str, dspy.Module],
    aggregated_findings_collector: Dict[str, List[Tuple[str, str, Optional[str]]]], 
    max_recursion_depth: int,
    min_chunks_for_subdivision: int,
    max_chunks_for_leaf: int,
    subdivision_factor: int
) -> Dict[str, Any]: 
    current_toc_path_str = " / ".join(current_toc_path) 
    print(f"  Recursively analyzing for ToC path: '{current_toc_path_str}' with {len(chunks_for_this_level)} chunks.")
    is_aggregated_leaf_content = False

    if len(chunks_for_this_level) < min_chunks_for_subdivision or len(current_toc_path) >= max_recursion_depth:
        print(f"    Base case reached for '{current_toc_path_str}'. Analyzing content directly.")
        leaf_json_nodes = [] 
        concat_char_count = sum(len(c) for c in chunks_for_this_level[:max_chunks_for_leaf])
        
        if len(chunks_for_this_level) <= max_chunks_for_leaf and concat_char_count <= DEFAULT_MAX_CHARS_NO_CHUNK :
            content_block = "\n\n--- (Original Chunk Separator) ---\n\n".join(chunks_for_this_level)
            context_for_leaf = f"{current_toc_path_str} (Aggregated {len(chunks_for_this_level)} original chunks)"
            is_aggregated_leaf_content = len(chunks_for_this_level) > 1
            print(f"    Analyzing aggregated content for '{context_for_leaf}' (Length: {len(content_block)} chars)")
            
            expert_json_output_node = _apply_expert_analysis_to_leaf_content(
                content_block, context_for_leaf, original_file_path_for_reports, 
                detected_content_type, all_expert_modules, aggregated_findings_collector,
                is_aggregated_content=is_aggregated_leaf_content
            )
            leaf_json_nodes.append({
                "toc_title": current_toc_path[-1], "toc_path": current_toc_path, "context": context_for_leaf,
                "type": "aggregated_leaf_analysis", **expert_json_output_node
            })
        else: 
            print(f"    Leaf node '{current_toc_path_str}' has {len(chunks_for_this_level)} chunks, processing individually (or exceeds max_chunks_for_leaf / total char limit).")
            for idx, chunk_content in enumerate(chunks_for_this_level):
                sub_part_context = f"{current_toc_path_str} - Original Chunk {idx+1} of {len(chunks_for_this_level)}"
                print(f"    Analyzing individual original chunk: '{sub_part_context}' (Length: {len(chunk_content)} chars)")
                expert_json_output_node = _apply_expert_analysis_to_leaf_content(
                    chunk_content, sub_part_context, original_file_path_for_reports, 
                    detected_content_type, all_expert_modules, aggregated_findings_collector,
                    is_aggregated_content=False 
                )
                leaf_json_nodes.append({
                    "toc_title": f"{current_toc_path[-1]} - Original Chunk {idx+1}", "toc_path": current_toc_path + [f"Original Chunk {idx+1}"],
                    "context": sub_part_context, "type": "individual_leaf_part_analysis", **expert_json_output_node
                })
        return {"title": current_toc_path[-1], "path": current_toc_path, "leaf_analyses": leaf_json_nodes, "type": "leaf_node"}

    print(f"    Recursive step for '{current_toc_path_str}'. Structurally subdividing {len(chunks_for_this_level)} chunks.")
    
    num_sub_parts = subdivision_factor
    if len(chunks_for_this_level) < num_sub_parts * 2 and len(chunks_for_this_level) > 1 : 
        num_sub_parts = max(1, len(chunks_for_this_level) // min_chunks_for_subdivision if min_chunks_for_subdivision > 0 else 1)
        num_sub_parts = max(1, min(num_sub_parts, len(chunks_for_this_level))) 
    if num_sub_parts <= 1 and len(chunks_for_this_level) >= min_chunks_for_subdivision : 
        num_sub_parts = 2 if len(chunks_for_this_level) > 1 else 1

    chunk_sublists = [[] for _ in range(num_sub_parts)]
    if chunks_for_this_level: 
        for i, chunk in enumerate(chunks_for_this_level):
            chunk_sublists[i % num_sub_parts].append(chunk)
    chunk_sublists = [sublist for sublist in chunk_sublists if sublist] 

    if not chunk_sublists or (len(chunk_sublists) <=1 and len(chunks_for_this_level) >= min_chunks_for_subdivision) : 
         print(f"    Subdivision resulted in <=1 part for '{current_toc_path_str}' with {len(chunks_for_this_level)} chunks. Forcing leaf node analysis.")
         return recursive_structured_analysis( current_toc_path, chunks_for_this_level, original_file_path_for_reports, detected_content_type, all_expert_modules, aggregated_findings_collector, 0, 0, max_chunks_for_leaf, subdivision_factor )

    section_json_outputs = []
    for i, sub_chunk_list in enumerate(chunk_sublists):
        if sub_chunk_list:
            sub_section_toc_path = current_toc_path + [f"Section {i+1} of {len(chunk_sublists)}"] 
            print(f"    Recursing into: '{' / '.join(sub_section_toc_path)}' with {len(sub_chunk_list)} chunks.")
            sub_section_json = recursive_structured_analysis(
                sub_section_toc_path, sub_chunk_list, original_file_path_for_reports,
                detected_content_type, all_expert_modules, aggregated_findings_collector, 
                max_recursion_depth, min_chunks_for_subdivision, max_chunks_for_leaf, subdivision_factor
            )
            section_json_outputs.append(sub_section_json)
        else:
            print(f"    No chunks in sub-part for '{current_toc_path_str} - Part {i+1}'. Skipping.")
    
    return {"title": current_toc_path[-1] if current_toc_path else "Root Analysis", 
            "path": current_toc_path, 
            "sub_sections": section_json_outputs,
            "type": "branch_node_structural"}


def analyze_single_file(
    file_path: str,
    expert_modules: Dict[str, dspy.Module], 
    max_chars_no_chunk: int,
    initial_chunk_size: int,
    initial_chunk_overlap: int,
    max_recursion_depth: int,
    min_chunks_for_subdivision: int,
    max_chunks_for_leaf: int,
    subdivision_factor: int
) -> Tuple[Optional[Dict[str, List[Tuple[str, str, Optional[str]]]]], Optional[Dict[str, Any]], Optional[str]]:
    """
    Analyzes a single file and returns its aggregated findings and JSON structure.
    """
    print(f"\n--- Analyzing Single File: {file_path} ---")
    file_content = read_file_content(file_path)

    if file_content is None:
        return None, {"file_path": file_path, "error": "File not readable or accessible."}, None

    file_basename = os.path.basename(file_path)
    detected_content_type = detect_content_type(file_path, file_content[:2000])
    print(f"Detected content type for {file_basename}: {detected_content_type}")

    initial_chunks: List[str]
    if len(file_content) <= max_chars_no_chunk:
        initial_chunks = [file_content]
    else:
        initial_chunks = initial_chunk_file_content(file_content, initial_chunk_size, initial_chunk_overlap)

    if not initial_chunks:
        return None, {"file_path": file_path, "status": "No content after initial processing"}, detected_content_type

    current_file_aggregated_findings: Dict[str, List[Tuple[str, str, Optional[str]]]] = defaultdict(list)
    report_title_for_toc = f"File: {file_basename}" 

    hierarchical_json_structure = recursive_structured_analysis(
        current_toc_path=[report_title_for_toc], 
        chunks_for_this_level=initial_chunks,
        original_file_path_for_reports=file_path, 
        detected_content_type=detected_content_type,
        all_expert_modules=expert_modules,
        aggregated_findings_collector=current_file_aggregated_findings, 
        max_recursion_depth=max_recursion_depth,
        min_chunks_for_subdivision=min_chunks_for_subdivision,
        max_chunks_for_leaf=max_chunks_for_leaf,
        subdivision_factor=subdivision_factor
    )
    
    single_file_json_output = {
        "file_analyzed": file_path, 
        "detected_content_type_overall": detected_content_type,
        "analysis_structure_tree": hierarchical_json_structure, 
        "raw_aggregated_findings_by_expert_for_file": dict(current_file_aggregated_findings) 
    }
    return current_file_aggregated_findings, single_file_json_output, detected_content_type


def generate_consolidated_report_markdown(
    report_title: str,
    overall_context_str: str,
    aggregated_findings: Dict[str, List[Tuple[str, str, Optional[str]]]],
    findings_consolidator: FindingsConsolidatorModule,
    max_consolidator_input_chars: int,
    is_directory_report: bool
) -> str:
    """
    Generates a final, consolidated Markdown report string from aggregated findings.
    """
    markdown_report_parts = [f"# {report_title}\n"]
    markdown_report_parts.append(f"{overall_context_str}\n")
    
    if is_directory_report:
        # Add a note about the nature of the report
        markdown_report_parts.append("**Note:** This is a consolidated report for an entire directory. Findings from multiple files have been synthesized under each expert's section.\n")

    expert_order = ["General Overview", "Code Module", "API Module", "Configuration Module", 
                    "Documentation Module", "Threat Modeler", "Compliance Checker"]

    for expert_name in expert_order:
        if expert_name in aggregated_findings and aggregated_findings[expert_name]:
            markdown_report_parts.append(f"\n## {expert_name} - Consolidated Analysis\n")
            raw_findings_str_parts = []
            # Sort findings by context to maintain a logical order
            sorted_findings = sorted(aggregated_findings[expert_name], key=lambda x: x[0])
            for context_str, finding_md, _ in sorted_findings:
                raw_findings_str_parts.append(f"**Original Finding Context:** `{context_str}`\n{finding_md}\n---")
            
            raw_findings_input_for_llm = "\n".join(raw_findings_str_parts)

            if not raw_findings_input_for_llm.strip():
                markdown_report_parts.append("- No specific findings were reported by this expert for any analyzed part.\n")
                continue

            print(f"  Consolidating findings for {expert_name}...")
            try:
                if len(raw_findings_input_for_llm) > max_consolidator_input_chars:
                    print(f"    Warning: Raw findings for {expert_name} ({len(raw_findings_input_for_llm):,} chars) too long for consolidation, truncating to {max_consolidator_input_chars:,} chars.")
                    raw_findings_input_for_llm = raw_findings_input_for_llm[:max_consolidator_input_chars] + "\n... (TRUNCATED FOR CONSOLIDATION)"
                
                consolidation_result = findings_consolidator.forward(
                    expert_name=expert_name,
                    overall_project_context=overall_context_str, # Use the overall context here
                    raw_findings_with_context_str=raw_findings_input_for_llm
                )
                markdown_report_parts.append(consolidation_result.consolidated_markdown_section)
            except Exception as e_consolidate:
                print(f"    Error during findings consolidation for {expert_name}: {e_consolidate}")
                markdown_report_parts.append(f"**Error consolidating findings for {expert_name}. Raw findings from parts follow:**\n")
                for context_str, finding_md, _ in sorted_findings:
                    markdown_report_parts.append(f"\n**From Context:** `{context_str}`\n{finding_md}\n")
        else:
            # Add a section even if no findings, for report consistency
            markdown_report_parts.append(f"\n## {expert_name} - Consolidated Analysis\n")
            markdown_report_parts.append("- No findings were reported by this expert.\n")
    
    return "\n".join(markdown_report_parts)


def process_single_file_and_generate_reports(
    file_path: str,
    expert_modules: Dict[str, dspy.Module],
    findings_consolidator: FindingsConsolidatorModule,
    max_chars_no_chunk: int, initial_chunk_size: int, initial_chunk_overlap: int,
    max_recursion_depth: int, min_chunks_for_subdivision: int, max_chunks_for_leaf: int,
    subdivision_factor: int,
    output_report_filename: str, output_json_filename: str,
    output_dir_path: Optional[str],
    max_consolidator_input_chars: int
) -> None:
    """Wrapper for single file processing that also handles report saving."""
    aggregated_findings_for_file, single_file_json_structure, detected_content_type_for_file = analyze_single_file(
        file_path, expert_modules,
        max_chars_no_chunk, initial_chunk_size, initial_chunk_overlap,
        max_recursion_depth, min_chunks_for_subdivision, max_chunks_for_leaf,
        subdivision_factor
    )

    if aggregated_findings_for_file is None or detected_content_type_for_file is None: 
        print(f"Skipping report generation for {file_path} due to read error.")
        error_json_path = os.path.join(output_dir_path, output_json_filename) if output_dir_path else output_json_filename
        if output_dir_path: os.makedirs(output_dir_path, exist_ok=True)
        try:
            with open(error_json_path, 'w', encoding='utf-8') as f: json.dump(single_file_json_structure or {"file_path": file_path, "error": "File processing failed before JSON structure creation."}, f, indent=4)
            print(f"Error JSON results for '{os.path.basename(file_path)}' saved to: {error_json_path}")
        except IOError as e: print(f"Error saving error JSON for '{os.path.basename(file_path)}': {e}")
        return

    file_basename = os.path.basename(file_path)
    report_title = f"Security Analysis Report for: {file_basename}"
    report_context_str = f"**Detected Overall Content Type:** `{detected_content_type_for_file}`"
    
    final_markdown_report = generate_consolidated_report_markdown(
        report_title, report_context_str, aggregated_findings_for_file,
        findings_consolidator, max_consolidator_input_chars, is_directory_report=False
    )
    
    md_output_path = os.path.join(output_dir_path, output_report_filename) if output_dir_path else output_report_filename
    json_output_path = os.path.join(output_dir_path, output_json_filename) if output_dir_path else output_json_filename
    if output_dir_path: os.makedirs(output_dir_path, exist_ok=True)

    try:
        with open(md_output_path, 'w', encoding='utf-8') as f: f.write(final_markdown_report)
        print(f"\nMarkdown report for '{file_basename}' saved to: {md_output_path}")
    except IOError as e: print(f"Error saving Markdown report for '{file_basename}': {e}")
    
    try:
        with open(json_output_path, 'w', encoding='utf-8') as f: json.dump(single_file_json_structure, f, indent=4)
        print(f"Detailed JSON results for '{file_basename}' saved to: {json_output_path}")
    except IOError as e: print(f"Error saving JSON results for '{file_basename}': {e}")


def process_directory_and_consolidate_reports(
    directory_path: str,
    expert_modules: Dict[str, dspy.Module], 
    findings_consolidator: FindingsConsolidatorModule, 
    ignore_patterns: List[str],
    max_chars_no_chunk: int, initial_chunk_size: int, initial_chunk_overlap: int,
    max_recursion_depth: int, min_chunks_for_subdivision: int, max_chunks_for_leaf: int,
    subdivision_factor: int,
    output_report_filename: str, 
    output_json_filename: str,
    output_dir_path: Optional[str],
    max_consolidator_input_chars: int,
    append_detailed_reports_flag: bool
):
    print(f"--- Processing Directory for Consolidated Report: {directory_path} ---")
    print(f"Ignoring patterns: {ignore_patterns}")

    master_aggregated_findings_by_expert: Dict[str, List[Tuple[str, str, Optional[str]]]] = defaultdict(list)
    all_files_json_structures: List[Dict[str, Any]] = [] 
    per_file_raw_findings: List[Tuple[str, Dict, str]] = [] # (file_path, findings_dict, content_type)
    processed_file_count = 0
    analyzed_file_paths: List[str] = []

    for root, dirs, files in os.walk(directory_path, topdown=True):
        dirs[:] = [d for d in dirs if not any(fnmatch.fnmatch(os.path.join(os.path.relpath(root, directory_path), d), pattern) for pattern in ignore_patterns)]
        
        for filename in files:
            file_path_abs = os.path.join(root, filename)
            file_path_rel = os.path.relpath(file_path_abs, directory_path)
            
            should_ignore = False
            for pattern in ignore_patterns:
                if fnmatch.fnmatch(file_path_rel, pattern) or fnmatch.fnmatch(filename, pattern):
                    should_ignore = True
                    break
            if should_ignore:
                print(f"  Ignoring: {file_path_rel}")
                continue

            file_findings, file_json_structure, detected_content_type = analyze_single_file(
                file_path_abs, expert_modules,
                max_chars_no_chunk, initial_chunk_size, initial_chunk_overlap,
                max_recursion_depth, min_chunks_for_subdivision, max_chunks_for_leaf,
                subdivision_factor
            )
            
            if file_findings:
                processed_file_count +=1
                analyzed_file_paths.append(file_path_rel)
                per_file_raw_findings.append((file_path_rel, file_findings, detected_content_type))
                
                for expert_name, findings_list in file_findings.items():
                    for context_str, finding_md, rationale_str in findings_list:
                        master_context_for_dir_report = f"File: {file_path_rel} ({context_str.replace('File: '+os.path.basename(file_path_abs) + ' / ', '')})" 
                        master_aggregated_findings_by_expert[expert_name].append((master_context_for_dir_report, finding_md, rationale_str))
            if file_json_structure: 
                all_files_json_structures.append(file_json_structure)

    if processed_file_count == 0:
        print("No files found to analyze in the directory (after ignoring).")
        return

    # Generate the main consolidated directory report
    dir_report_title = f"Consolidated Security Analysis Report for Directory: {os.path.basename(directory_path)}"
    dir_context_str = f"**Total Files Analyzed:** {processed_file_count}\n\n"
    if analyzed_file_paths:
        dir_context_str += "**Analyzed Files List:**\n" + "\n".join([f"- `{p}`" for p in analyzed_file_paths])

    final_dir_markdown_report = generate_consolidated_report_markdown(
        dir_report_title, dir_context_str, master_aggregated_findings_by_expert,
        findings_consolidator, max_consolidator_input_chars, is_directory_report=True
    )
    
    # Append detailed per-file reports if requested
    if append_detailed_reports_flag:
        print("\n--- Generating and Appending Detailed Per-File Reports ---")
        detailed_report_parts = ["\n\n---\n\n# Appendix: Detailed Per-File Analysis"]
        for file_path_rel, findings_dict, content_type in per_file_raw_findings:
            file_report_title = f"Detailed Report for: {file_path_rel}"
            file_context_str = f"**Detected Content Type:** `{content_type}`"
            file_report_md = generate_consolidated_report_markdown(
                file_report_title, file_context_str, findings_dict,
                findings_consolidator, max_consolidator_input_chars, is_directory_report=False
            )
            detailed_report_parts.append(f"\n\n---\n\n{file_report_md}")
        final_dir_markdown_report += "\n".join(detailed_report_parts)

    # Save the final consolidated reports
    md_output_path = os.path.join(output_dir_path, output_report_filename) if output_dir_path else output_report_filename
    json_output_path = os.path.join(output_dir_path, output_json_filename) if output_dir_path else output_json_filename
    if output_dir_path: os.makedirs(output_dir_path, exist_ok=True)

    try:
        with open(md_output_path, 'w', encoding='utf-8') as f: f.write(final_dir_markdown_report)
        print(f"\nConsolidated Markdown report for directory '{directory_path}' saved to: {md_output_path}")
    except IOError as e: print(f"Error saving consolidated Markdown report for directory: {e}")
    
    consolidated_json_data = {
        "directory_analyzed": directory_path,
        "summary_of_files_processed": processed_file_count,
        "files_analyzed_list": analyzed_file_paths,
        "individual_file_analysis_trees": all_files_json_structures, 
        "raw_aggregated_findings_across_all_files_by_expert": dict(master_aggregated_findings_by_expert)
    }
    try:
        with open(json_output_path, 'w', encoding='utf-8') as f: json.dump(consolidated_json_data, f, indent=4)
        print(f"Consolidated JSON results for directory '{directory_path}' saved to: {json_output_path}")
    except IOError as e: print(f"Error saving consolidated JSON results for directory: {e}")


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Perform a recursive, multi-expert security analysis on files or directories.")
    parser.add_argument("input_path", help="Path to the file or directory to analyze.")
    parser.add_argument("--model", default=DEFAULT_MODEL_NAME, help=f"Model name (default: {DEFAULT_MODEL_NAME}).")
    parser.add_argument("--api_base_url", default=DEFAULT_API_BASE_URL, help=f"LLM API base URL (default: {DEFAULT_API_BASE_URL}).")
    parser.add_argument("--api_key", default=os.getenv("API_KEY"), help="LLM API key (or use API_KEY env var).")
    
    parser.add_argument("--initial_chunk_size", type=int, default=DEFAULT_CHUNK_SIZE)
    parser.add_argument("--initial_chunk_overlap", type=int, default=DEFAULT_CHUNK_OVERLAP)
    parser.add_argument("--max_chars_no_initial_chunk", type=int, default=DEFAULT_MAX_CHARS_NO_CHUNK)
    
    parser.add_argument("--rec_min_chunks_subdivide", type=int, default=RECURSIVE_MIN_CHUNKS_TO_SUBDIVIDE)
    parser.add_argument("--rec_max_depth", type=int, default=RECURSIVE_MAX_DEPTH)
    parser.add_argument("--rec_max_chunks_leaf", type=int, default=RECURSIVE_MAX_CHUNKS_FOR_LEAF_ANALYSIS)
    parser.add_argument("--rec_subdivision_factor", type=int, default=RECURSIVE_SUBDIVISION_FACTOR)
    
    parser.add_argument("--max_output_tokens", type=int, default=DEFAULT_MAX_OUTPUT_TOKENS, help=f"Max output tokens for LLM responses (default: {DEFAULT_MAX_OUTPUT_TOKENS}).")
    parser.add_argument("--max_consolidator_input_chars", type=int, default=DEFAULT_MAX_CONSOLIDATOR_INPUT_CHARS, help=f"Max characters for the input to the findings consolidator LLM (default: {DEFAULT_MAX_CONSOLIDATOR_INPUT_CHARS}).")

    parser.add_argument("--output_report_file", default="security_analysis_report.md", help="Base filename for Markdown reports.")
    parser.add_argument("--output_json_file", default="security_analysis_details.json", help="Base filename for detailed JSON results.")
    parser.add_argument("--output_dir", default=None, help="Optional directory to save all output reports. If not set, reports are saved in current dir.")
    parser.add_argument("-i", "--ignore_paths", nargs='*', default=[], help="List of file/directory names or glob patterns to ignore (e.g., *.log __pycache__ temp/*).")
    parser.add_argument("--append_detailed_reports", action="store_true", help="If processing a directory, append the full analysis for each file to the end of the consolidated report.")

    args = parser.parse_args()

    print("Initializing Multi-Expert Recursive APPSEC Analysis POC Script...")

    if not args.api_key:
        print(f"Error: API Key not provided. Set --api_key or API_KEY environment variable.")
        exit(1)

    try:
        model_identifier_for_dspy_lm = f"openai/{args.model}" 
        
        llm_instance = dspy.LM(
            model=model_identifier_for_dspy_lm,
            api_key=args.api_key,
            api_base=args.api_base_url,
            max_tokens=args.max_output_tokens 
        )
        
        dspy.settings.configure(lm=llm_instance)
        print(f"DSPy configured with a single global LLM instance (model: '{model_identifier_for_dspy_lm}', api_base: '{args.api_base_url}')")

    except Exception as e:
        print(f"Error initializing DSPy LLM instance with dspy.LM: {e}")
        print("Ensure your DSPy version and LiteLLM installation are compatible and can handle this configuration.")
        exit(1)

    expert_modules_collection = {
        "General Overview": GeneralOverviewModule(),
        "Code Module": CodeAnalysisModule(),
        "API Module": ApiAnalysisModule(),
        "Configuration Module": ConfigAnalysisModule(),
        "Documentation Module": DocsAnalysisModule(),
        "Threat Modeler": ThreatModelingModule(),
        "Compliance Checker": ComplianceCheckingModule()
    }
    findings_consolidator_module = FindingsConsolidatorModule()
    initialize_structuring_modules()

    print("Security expert and utility modules initialized.")

    if args.output_dir:
        os.makedirs(args.output_dir, exist_ok=True)
        print(f"Output directory set to: {os.path.abspath(args.output_dir)}")

    if os.path.isfile(args.input_path):
        process_single_file_and_generate_reports( 
            args.input_path, expert_modules_collection, findings_consolidator_module,
            args.max_chars_no_initial_chunk, args.initial_chunk_size, args.initial_chunk_overlap,
            args.rec_max_depth, args.rec_min_chunks_subdivide, args.rec_max_chunks_leaf,
            args.rec_subdivision_factor,
            args.output_report_file, args.output_json_file, args.output_dir,
            args.max_consolidator_input_chars
        )
    elif os.path.isdir(args.input_path):
        process_directory_and_consolidate_reports(
            args.input_path, expert_modules_collection, findings_consolidator_module, args.ignore_paths,
            args.max_chars_no_initial_chunk, args.initial_chunk_size, args.initial_chunk_overlap,
            args.rec_max_depth, args.rec_min_chunks_subdivide, args.rec_max_chunks_leaf,
            args.rec_subdivision_factor,
            args.output_report_file, args.output_json_file, args.output_dir,
            args.max_consolidator_input_chars,
            args.append_detailed_reports # Pass the new flag
        )
    else:
        print(f"Error: Input path '{args.input_path}' is not a valid file or directory.")
        exit(1)

    print("\nMulti-Expert Recursive Analysis POC script finished.")