fix: move secrets check from if-expression to run block in container CI #290
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| env: | |
| FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true | |
| on: | |
| push: | |
| tags: ["v*"] | |
| workflow_dispatch: | |
| inputs: | |
| graph_tag: | |
| description: "Graph version tag to test" | |
| required: false | |
| default: "latest" | |
| registry: | |
| description: "GHCR registry (default: ghcr.io/iterorganization)" | |
| required: false | |
| default: "ghcr.io/iterorganization" | |
| benchmark_filter: | |
| description: "Benchmark filter (e.g., SearchToolBenchmarks)" | |
| required: false | |
| default: "" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| run-benchmark: | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 60 | |
| permissions: | |
| contents: write | |
| packages: read | |
| services: | |
| neo4j: | |
| image: neo4j:2026.01.4-community | |
| ports: | |
| - 7474:7474 | |
| - 7687:7687 | |
| env: | |
| NEO4J_AUTH: neo4j/imas-codex | |
| NEO4J_PLUGINS: '["apoc"]' | |
| NEO4J_server_memory_heap_initial__size: 512m | |
| NEO4J_server_memory_heap_max__size: 1G | |
| options: >- | |
| --health-cmd "wget -q --spider http://localhost:7474/ || exit 1" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 10 | |
| --health-start-period 30s | |
| steps: | |
| # ── Setup ────────────────────────────────────────────────────────── | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Ensure main branch exists | |
| run: | | |
| # On tag checkouts, 'main' doesn't exist as a local branch | |
| # ASV needs it for publish | |
| if ! git rev-parse --verify main >/dev/null 2>&1; then | |
| git branch main origin/main | |
| fi | |
| - name: Checkout gh-pages for benchmark history | |
| run: | | |
| if git ls-remote --exit-code origin gh-pages >/dev/null 2>&1; then | |
| git fetch origin gh-pages:gh-pages | |
| git worktree add gh-pages-data gh-pages | |
| else | |
| mkdir -p gh-pages-data | |
| fi | |
| continue-on-error: true | |
| - name: Restore previous benchmark results | |
| run: | | |
| if [ -d "gh-pages-data/.asv" ]; then | |
| mkdir -p .asv | |
| cp -r gh-pages-data/.asv/* .asv/ || true | |
| fi | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| - name: Free disk space | |
| run: | | |
| sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/share/boost | |
| df -h | |
| - name: Install UV + ASV | |
| uses: astral-sh/setup-uv@v5 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Install tools | |
| run: | | |
| uv tool install asv | |
| ORAS_VERSION="1.2.0" | |
| curl -sLO "https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_amd64.tar.gz" | |
| tar -xzf "oras_${ORAS_VERSION}_linux_amd64.tar.gz" oras | |
| sudo mv oras /usr/local/bin/ | |
| - name: Configure pip for CPU-only PyTorch | |
| run: | | |
| mkdir -p ~/.config/pip | |
| cat > ~/.config/pip/pip.conf << 'EOF' | |
| [global] | |
| extra-index-url = https://download.pytorch.org/whl/cpu | |
| EOF | |
| # ── Load Graph Data ──────────────────────────────────────────────── | |
| # Packages live in the org registry (iterorganization). | |
| # GHCR_TOKEN (PAT with read:packages) is required — packages are private. | |
| - name: Login to GHCR | |
| run: | | |
| echo "${{ secrets.GHCR_TOKEN }}" | oras login ghcr.io -u token --password-stdin | |
| - name: Resolve graph tag | |
| id: graph-tag | |
| run: | | |
| REGISTRY="${{ github.event.inputs.registry || 'ghcr.io/iterorganization' }}" | |
| PACKAGE="imas-codex-graph" | |
| REQUESTED="${{ github.event.inputs.graph_tag || 'latest' }}" | |
| if [ "$REQUESTED" != "latest" ]; then | |
| echo "tag=${REQUESTED}" >> $GITHUB_OUTPUT | |
| echo "registry=${REGISTRY}" >> $GITHUB_OUTPUT | |
| exit 0 | |
| fi | |
| # Resolve 'latest': check if tag exists, otherwise pick most recent | |
| TAGS=$(oras repo tags "${REGISTRY}/${PACKAGE}" 2>&1) || true | |
| CLEAN_TAGS=$(echo "$TAGS" | grep -E '^[0-9v]' || true) | |
| if echo "$CLEAN_TAGS" | grep -qx "latest"; then | |
| echo "tag=latest" >> $GITHUB_OUTPUT | |
| elif [ -n "$CLEAN_TAGS" ]; then | |
| BEST=$(echo "$CLEAN_TAGS" | tail -1) | |
| echo "No 'latest' tag. Using most recent: $BEST" | |
| echo "tag=${BEST}" >> $GITHUB_OUTPUT | |
| else | |
| echo "::error::No graph tags found in ${REGISTRY}/${PACKAGE}. Ensure GHCR_TOKEN secret has read:packages scope." | |
| exit 1 | |
| fi | |
| echo "registry=${REGISTRY}" >> $GITHUB_OUTPUT | |
| - name: Pull graph dump from GHCR | |
| run: | | |
| REGISTRY="${{ steps.graph-tag.outputs.registry }}" | |
| PACKAGE="imas-codex-graph" | |
| TAG="${{ steps.graph-tag.outputs.tag }}" | |
| echo "Pulling: ${REGISTRY}/${PACKAGE}:${TAG}" | |
| mkdir -p /tmp/graph-dump | |
| oras pull "${REGISTRY}/${PACKAGE}:${TAG}" -o /tmp/graph-dump | |
| ls -la /tmp/graph-dump/ | |
| - name: Stop Neo4j for dump load | |
| run: | | |
| docker stop $(docker ps -q --filter "ancestor=neo4j:2026.01.4-community") || true | |
| sleep 3 | |
| - name: Load graph dump | |
| run: | | |
| set -euo pipefail | |
| # Handle both raw .dump files and .tar.gz archives from GHCR | |
| DUMP_FILE=$(find /tmp/graph-dump -name "*.dump" | head -1) | |
| if [ -z "${DUMP_FILE}" ]; then | |
| # Try tar.gz archive format | |
| ARCHIVE=$(ls /tmp/graph-dump/*.tar.gz 2>/dev/null | head -1) | |
| if [ -n "${ARCHIVE}" ]; then | |
| mkdir -p /tmp/graph-extracted | |
| tar -xzf "${ARCHIVE}" -C /tmp/graph-extracted | |
| DUMP_FILE=$(find /tmp/graph-extracted -name "*.dump" | head -1) | |
| fi | |
| fi | |
| if [ -z "${DUMP_FILE}" ]; then | |
| echo "ERROR: No .dump file found" | |
| ls -laR /tmp/graph-dump/ | |
| exit 1 | |
| fi | |
| echo "Loading dump: ${DUMP_FILE}" | |
| # neo4j-admin load expects the dump file named as <database>.dump | |
| DUMP_DIR=$(mktemp -d) | |
| cp "${DUMP_FILE}" "${DUMP_DIR}/neo4j.dump" | |
| chmod -R 777 "${DUMP_DIR}" | |
| NEO4J_CONTAINER=$(docker ps -aq --filter "ancestor=neo4j:2026.01.4-community" | head -1) | |
| NEO4J_DATA_VOLUME=$(docker inspect "${NEO4J_CONTAINER}" --format '{{range .Mounts}}{{if eq .Destination "/data"}}{{.Name}}{{end}}{{end}}') | |
| docker run --rm --user root \ | |
| -v "${NEO4J_DATA_VOLUME}:/data" \ | |
| -v "${DUMP_DIR}:/dump" \ | |
| neo4j:2026.01.4-community \ | |
| neo4j-admin database load neo4j \ | |
| --from-path=/dump \ | |
| --overwrite-destination=true \ | |
| --verbose | |
| - name: Restart Neo4j with loaded data | |
| run: | | |
| NEO4J_CONTAINER=$(docker ps -aq --filter "ancestor=neo4j:2026.01.4-community" | head -1) | |
| docker start "${NEO4J_CONTAINER}" | |
| for i in $(seq 1 60); do | |
| if curl -sf http://localhost:7474/ > /dev/null 2>&1; then | |
| echo "Neo4j ready with loaded graph" | |
| break | |
| fi | |
| if [ $i -eq 60 ]; then | |
| echo "ERROR: Neo4j did not restart in time" | |
| docker logs "${NEO4J_CONTAINER}" --tail 50 | |
| exit 1 | |
| fi | |
| sleep 2 | |
| done | |
| - name: Reset Neo4j password | |
| run: | | |
| NEO4J_CONTAINER=$(docker ps -q --filter "ancestor=neo4j:2026.01.4-community" | head -1) | |
| docker exec "${NEO4J_CONTAINER}" neo4j-admin dbms set-initial-password imas-codex 2>/dev/null || true | |
| - name: Verify graph loaded | |
| run: | | |
| NEO4J_CONTAINER=$(docker ps -q --filter "ancestor=neo4j:2026.01.4-community" | head -1) | |
| docker exec "${NEO4J_CONTAINER}" cypher-shell \ | |
| -u neo4j -p imas-codex \ | |
| "MATCH (n) RETURN count(n) AS nodes, labels(n)[0] AS label ORDER BY nodes DESC LIMIT 10" | |
| - name: Create vector and fulltext indexes | |
| run: | | |
| # Dimension must match pyproject.toml [tool.imas-codex.embedding] dimension | |
| DIM=256 | |
| NEO4J_CONTAINER=$(docker ps -q --filter "ancestor=neo4j:2026.01.4-community" | head -1) | |
| # Drop any stale vector indexes from the dump (may have wrong dimension) | |
| echo "Dropping stale vector indexes..." | |
| docker exec "${NEO4J_CONTAINER}" cypher-shell \ | |
| -u neo4j -p imas-codex \ | |
| "SHOW INDEXES YIELD name, type WHERE type = 'VECTOR' RETURN name" \ | |
| --format plain 2>/dev/null | tail -n +2 | tr -d ' "' | while read -r IDX; do | |
| [ -z "$IDX" ] && continue | |
| echo " Dropping $IDX" | |
| docker exec "${NEO4J_CONTAINER}" cypher-shell \ | |
| -u neo4j -p imas-codex "DROP INDEX $IDX IF EXISTS" 2>/dev/null || true | |
| done | |
| echo "Creating vector indexes with dimension $DIM..." | |
| # Vector indexes — matches schema declarations in imas_codex/schemas/ | |
| declare -A VECTOR_INDEXES=( | |
| ["ids_embedding"]="IDS:embedding" | |
| ["imas_node_embedding"]="IMASNode:embedding" | |
| ["cluster_embedding"]="IMASSemanticCluster:embedding" | |
| ["cluster_label_embedding"]="IMASSemanticCluster:label_embedding" | |
| ["cluster_description_embedding"]="IMASSemanticCluster:description_embedding" | |
| ["facility_path_desc_embedding"]="FacilityPath:embedding" | |
| ["signal_node_desc_embedding"]="SignalNode:embedding" | |
| ["signal_source_desc_embedding"]="SignalSource:embedding" | |
| ["facility_signal_desc_embedding"]="FacilitySignal:embedding" | |
| ["code_example_desc_embedding"]="CodeExample:embedding" | |
| ["document_desc_embedding"]="Document:embedding" | |
| ["code_chunk_embedding"]="CodeChunk:embedding" | |
| ["wiki_chunk_embedding"]="WikiChunk:embedding" | |
| ["image_desc_embedding"]="Image:embedding" | |
| ["identifier_schema_embedding"]="IMASIdentifier:embedding" | |
| ) | |
| for IDX_NAME in "${!VECTOR_INDEXES[@]}"; do | |
| IFS=':' read -r LABEL PROP <<< "${VECTOR_INDEXES[$IDX_NAME]}" | |
| echo " Creating $IDX_NAME on $LABEL.$PROP (dim=$DIM)" | |
| docker exec "${NEO4J_CONTAINER}" cypher-shell \ | |
| -u neo4j -p imas-codex \ | |
| "CREATE VECTOR INDEX $IDX_NAME IF NOT EXISTS | |
| FOR (n:$LABEL) ON (n.$PROP) | |
| OPTIONS {indexConfig: {\`vector.dimensions\`: $DIM, \`vector.similarity_function\`: 'cosine'}}" \ | |
| 2>/dev/null || echo " Warning: $IDX_NAME failed" | |
| done | |
| # Fulltext indexes | |
| echo "Creating fulltext indexes..." | |
| docker exec "${NEO4J_CONTAINER}" cypher-shell \ | |
| -u neo4j -p imas-codex \ | |
| "CREATE FULLTEXT INDEX imas_node_text IF NOT EXISTS FOR (n:IMASNode) ON EACH [n.path, n.description, n.documentation]" \ | |
| 2>/dev/null || echo " Warning: imas_node_text already exists" | |
| # Wait for indexes to come online | |
| echo "Waiting for indexes to populate..." | |
| sleep 10 | |
| # Verify indexes | |
| docker exec "${NEO4J_CONTAINER}" cypher-shell \ | |
| -u neo4j -p imas-codex \ | |
| "SHOW INDEXES YIELD name, type, state WHERE type IN ['VECTOR', 'FULLTEXT'] RETURN name, type, state ORDER BY name" | |
| # ── Run Benchmarks ───────────────────────────────────────────────── | |
| - name: Setup ASV machine | |
| run: | | |
| CPU_INFO=$(lscpu | grep "Model name" | cut -d: -f2 | sed 's/^[ \t]*//' | head -1) | |
| CPU_COUNT=$(nproc) | |
| RAM_GB=$(($(free -m | grep "Mem:" | awk '{print $2}') / 1024)) | |
| ARCH=$(uname -m) | |
| OS_VERSION=$(lsb_release -ds 2>/dev/null | tr -d '"' || echo "Unknown") | |
| MACHINE_NAME="ghactions-${ARCH}-${CPU_COUNT}c-${RAM_GB}gb" | |
| asv machine --machine "$MACHINE_NAME" --os "$OS_VERSION" --arch "$ARCH" \ | |
| --cpu "$CPU_INFO" --num_cpu "$CPU_COUNT" --ram "${RAM_GB}GB" --yes | |
| echo "MACHINE_NAME=$MACHINE_NAME" >> $GITHUB_ENV | |
| - name: Run benchmarks | |
| env: | |
| NEO4J_URI: bolt://localhost:7687 | |
| NEO4J_USERNAME: neo4j | |
| NEO4J_PASSWORD: imas-codex | |
| IMAS_CODEX_EMBEDDING_LOCATION: local | |
| IMAS_CODEX_GRAPH_LOCATION: local | |
| run: | | |
| FILTER="${{ github.event.inputs.benchmark_filter }}" | |
| if [ -n "$FILTER" ]; then | |
| asv run --python=3.12 --machine "$MACHINE_NAME" -b "$FILTER" --verbose --show-stderr | |
| else | |
| # asv run returns exit code 2 when it cannot find prior results | |
| # for the parent commit (common on tag-triggered runs). This is | |
| # not a real failure — treat exit code 2 as success. | |
| asv run --python=3.12 --machine "$MACHINE_NAME" HEAD^! --verbose --show-stderr || { | |
| EC=$? | |
| if [ $EC -eq 2 ]; then | |
| echo "::warning::asv exited with code 2 (no prior baseline) — treating as success" | |
| else | |
| exit $EC | |
| fi | |
| } | |
| fi | |
| # ── Regression Detection ─────────────────────────────────────────── | |
| - name: Check for regressions | |
| continue-on-error: true | |
| run: | | |
| PREV_TAG=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || echo "") | |
| if [ -n "$PREV_TAG" ]; then | |
| echo "Comparing against previous release: $PREV_TAG" | |
| asv compare "$PREV_TAG" HEAD --factor 1.5 --split | |
| else | |
| echo "No previous tag found — skipping comparison" | |
| fi | |
| # ── Publish ──────────────────────────────────────────────────────── | |
| - name: Generate HTML report | |
| run: asv publish | |
| - name: Deploy to GitHub Pages | |
| run: | | |
| git config --global user.name "github-actions[bot]" | |
| git config --global user.email "github-actions[bot]@users.noreply.github.com" | |
| mkdir -p gh-pages-deploy | |
| cp -r .asv/html/* gh-pages-deploy/ | |
| mkdir -p gh-pages-deploy/.asv/results | |
| cp -r .asv/results/* gh-pages-deploy/.asv/results/ || true | |
| cp .asv/machine.json gh-pages-deploy/.asv/ || true | |
| touch gh-pages-deploy/.nojekyll | |
| cd gh-pages-deploy | |
| git init && git checkout -b gh-pages | |
| git add -f .asv/ && git add -A | |
| git commit -m "benchmark: ${{ github.ref_name }}" | |
| git remote add origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git | |
| git push -f origin gh-pages |