Server rewrite. Focused semantic search with cached embedings. #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| push: | |
| branches: [main, develop] | |
| pull_request: | |
| branches: [main] | |
| schedule: | |
| # Run benchmarks daily at 2 AM UTC | |
| - cron: "0 2 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| benchmark_filter: | |
| description: "Benchmark filter (e.g., SearchBenchmarks)" | |
| required: false | |
| default: "" | |
| action: | |
| description: "Action to perform (run or clear)" | |
| required: false | |
| default: "run" | |
| type: choice | |
| options: | |
| - run | |
| - clear | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| run-benchmark: | |
| if: github.event.inputs.action != 'clear' | |
| # Use specific runner for consistent benchmark results | |
| runs-on: ubuntu-22.04 # 4 cores, 16GB RAM, consistent specs | |
| permissions: | |
| contents: read | |
| pages: write | |
| id-token: write | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deployment.outputs.page_url }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Need full history for ASV | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.12" | |
| - name: Install UV | |
| uses: astral-sh/setup-uv@v2 | |
| - name: Install ASV | |
| run: uv tool install asv | |
| - name: Download previous benchmark results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: asv-results | |
| path: . | |
| continue-on-error: true | |
| - name: Setup ASV machine | |
| run: | | |
| # Get actual system specifications from the runner | |
| CPU_INFO=$(lscpu | grep "Model name" | cut -d: -f2 | sed 's/^[ \t]*//' | head -1) | |
| CPU_COUNT=$(nproc) | |
| RAM_MB=$(free -m | grep "Mem:" | awk '{print $2}') | |
| RAM_GB=$((RAM_MB / 1024)) | |
| ARCH=$(uname -m) | |
| # Get OS details | |
| OS_VERSION=$(lsb_release -ds 2>/dev/null | tr -d '"' || echo "Unknown") | |
| # Create machine name from key specs to detect hardware changes | |
| # Format: {arch}-{cores}c-{ram}gb | |
| MACHINE_NAME="${ARCH}-${CPU_COUNT}c-${RAM_GB}gb" | |
| echo "Machine name: $MACHINE_NAME" | |
| # Configure machine with detected specs | |
| asv machine \ | |
| --machine "$MACHINE_NAME" \ | |
| --os "$OS_VERSION" \ | |
| --arch "$ARCH" \ | |
| --cpu "$CPU_INFO" \ | |
| --num_cpu "$CPU_COUNT" \ | |
| --ram "${RAM_GB}GB" \ | |
| --yes | |
| # Store machine name for benchmark run | |
| echo "MACHINE_NAME=$MACHINE_NAME" >> $GITHUB_ENV | |
| - name: Run benchmarks | |
| run: | | |
| # Run benchmarks | |
| if [ -n "${{ github.event.inputs.benchmark_filter }}" ]; then | |
| asv run --python=3.12 --machine "$MACHINE_NAME" -b "${{ github.event.inputs.benchmark_filter }}" --verbose | |
| else | |
| asv run --python=3.12 --machine "$MACHINE_NAME" HEAD^! --verbose | |
| fi | |
| - name: Show benchmark results on failure | |
| if: failure() | |
| run: | | |
| echo "Benchmark run failed. Checking for logs..." | |
| find .asv -name "*.log" -exec echo "=== {} ===" \; -exec cat {} \; || true | |
| echo "Checking ASV results directory..." | |
| ls -la .asv/ || true | |
| echo "Checking if schema files exist..." | |
| ls -la imas_mcp/resources/schemas/ || true | |
| - name: Generate HTML report | |
| run: asv publish | |
| - name: Upload benchmark results | |
| uses: actions/upload-artifact@v4 | |
| if: success() | |
| with: | |
| name: asv-results | |
| path: | | |
| .asv/results/ | |
| .asv/html/ | |
| .asv/machine.json | |
| retention-days: 90 | |
| if-no-files-found: warn | |
| include-hidden-files: true | |
| - name: Setup Pages | |
| if: github.ref == 'refs/heads/main' | |
| uses: actions/configure-pages@v5 | |
| - name: Prepare benchmarks for deployment | |
| if: github.ref == 'refs/heads/main' | |
| run: | | |
| # Create benchmark-specific subdirectory that won't conflict with docs | |
| mkdir -p deploy/benchmarks | |
| cp -r .asv/html/* deploy/benchmarks/ | |
| # Create a simple landing page for benchmarks only | |
| echo '<!DOCTYPE html><html><head><title>IMAS MCP Benchmarks</title></head><body><h1>Performance Benchmarks</h1><p><a href="./benchmarks/">View Benchmark Reports</a></p></body></html>' > deploy/index.html | |
| - name: Upload Pages artifact | |
| if: github.ref == 'refs/heads/main' | |
| uses: actions/upload-pages-artifact@v3 | |
| with: | |
| path: deploy | |
| - name: Deploy to GitHub Pages | |
| if: github.ref == 'refs/heads/main' | |
| id: deployment | |
| uses: actions/deploy-pages@v4 | |
| - name: Comment PR with benchmark results | |
| if: github.event_name == 'pull_request' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| const path = '.asv/html/index.json'; | |
| if (fs.existsSync(path)) { | |
| const results = JSON.parse(fs.readFileSync(path, 'utf8')); | |
| const benchmarkUrl = `https://${{ github.repository_owner }}.github.io/${{ github.event.repository.name }}/benchmarks/`; | |
| const body = `## 📊 Benchmark Results | |
| Performance benchmarks have been run for this PR. | |
| 🔗 **[View full benchmark report](${benchmarkUrl})** | |
| > Results are compared against the main branch. Significant changes will be highlighted in the full report. | |
| `; | |
| github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: body | |
| }); | |
| } | |
| clear-benchmarks: | |
| if: github.event.inputs.action == 'clear' | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| contents: read | |
| pages: write | |
| id-token: write | |
| environment: | |
| name: github-pages | |
| url: ${{ steps.deployment.outputs.page_url }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Upload empty benchmark results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: asv-results | |
| path: | | |
| # Empty path to clear the artifact | |
| if-no-files-found: ignore | |
| overwrite: true | |
| - name: Setup Pages | |
| if: github.ref == 'refs/heads/main' | |
| uses: actions/configure-pages@v5 | |
| - name: Prepare empty deployment | |
| if: github.ref == 'refs/heads/main' | |
| run: | | |
| # Create empty deploy directory (this will effectively clear benchmarks) | |
| mkdir -p deploy | |
| echo '<!DOCTYPE html><html><head><title>Benchmarks Cleared</title></head><body><h1>Benchmarks have been cleared</h1><p>The benchmark reports have been removed from this site.</p></body></html>' > deploy/index.html | |
| - name: Upload Pages artifact | |
| if: github.ref == 'refs/heads/main' | |
| uses: actions/upload-pages-artifact@v3 | |
| with: | |
| path: deploy | |
| - name: Deploy to GitHub Pages | |
| if: github.ref == 'refs/heads/main' | |
| id: deployment | |
| uses: actions/deploy-pages@v4 |