diff --git a/.github/workflows/platform-backend-ci.yml b/.github/workflows/platform-backend-ci.yml index 22d1e91ead98..eeb3dbe97d2a 100644 --- a/.github/workflows/platform-backend-ci.yml +++ b/.github/workflows/platform-backend-ci.yml @@ -5,12 +5,14 @@ on: branches: [master, dev, ci-test*] paths: - ".github/workflows/platform-backend-ci.yml" + - ".github/workflows/scripts/get_package_version_from_lockfile.py" - "autogpt_platform/backend/**" - "autogpt_platform/autogpt_libs/**" pull_request: branches: [master, dev, release-*] paths: - ".github/workflows/platform-backend-ci.yml" + - ".github/workflows/scripts/get_package_version_from_lockfile.py" - "autogpt_platform/backend/**" - "autogpt_platform/autogpt_libs/**" merge_group: diff --git a/.github/workflows/platform-frontend-ci.yml b/.github/workflows/platform-frontend-ci.yml index 3f35896ddac1..d5946e328f86 100644 --- a/.github/workflows/platform-frontend-ci.yml +++ b/.github/workflows/platform-frontend-ci.yml @@ -120,175 +120,6 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} exitOnceUploaded: true - e2e_test: - name: end-to-end tests - runs-on: big-boi - - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - submodules: recursive - - - name: Set up Platform - Copy default supabase .env - run: | - cp ../.env.default ../.env - - - name: Set up Platform - Copy backend .env and set OpenAI API key - run: | - cp ../backend/.env.default ../backend/.env - echo "OPENAI_INTERNAL_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> ../backend/.env - env: - # Used by E2E test data script to generate embeddings for approved store agents - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - - - name: Set up Platform - Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - driver: docker-container - driver-opts: network=host - - - name: Set up Platform - Expose GHA cache to docker buildx CLI - uses: crazy-max/ghaction-github-runtime@v4 - - - name: Set up Platform - Build Docker images (with cache) - working-directory: autogpt_platform - run: | - pip install pyyaml - - # Resolve extends and generate a flat compose file that bake can understand - docker compose -f docker-compose.yml config > docker-compose.resolved.yml - - # Add cache configuration to the resolved compose file - python ../.github/workflows/scripts/docker-ci-fix-compose-build-cache.py \ - --source docker-compose.resolved.yml \ - --cache-from "type=gha" \ - --cache-to "type=gha,mode=max" \ - --backend-hash "${{ hashFiles('autogpt_platform/backend/Dockerfile', 'autogpt_platform/backend/poetry.lock', 'autogpt_platform/backend/backend') }}" \ - --frontend-hash "${{ hashFiles('autogpt_platform/frontend/Dockerfile', 'autogpt_platform/frontend/pnpm-lock.yaml', 'autogpt_platform/frontend/src') }}" \ - --git-ref "${{ github.ref }}" - - # Build with bake using the resolved compose file (now includes cache config) - docker buildx bake --allow=fs.read=.. -f docker-compose.resolved.yml --load - env: - NEXT_PUBLIC_PW_TEST: true - - - name: Set up tests - Cache E2E test data - id: e2e-data-cache - uses: actions/cache@v5 - with: - path: /tmp/e2e_test_data.sql - key: e2e-test-data-${{ hashFiles('autogpt_platform/backend/test/e2e_test_data.py', 'autogpt_platform/backend/migrations/**', '.github/workflows/platform-frontend-ci.yml') }} - - - name: Set up Platform - Start Supabase DB + Auth - run: | - docker compose -f ../docker-compose.resolved.yml up -d db auth --no-build - echo "Waiting for database to be ready..." - timeout 60 sh -c 'until docker compose -f ../docker-compose.resolved.yml exec -T db pg_isready -U postgres 2>/dev/null; do sleep 2; done' - echo "Waiting for auth service to be ready..." - timeout 60 sh -c 'until docker compose -f ../docker-compose.resolved.yml exec -T db psql -U postgres -d postgres -c "SELECT 1 FROM auth.users LIMIT 1" 2>/dev/null; do sleep 2; done' || echo "Auth schema check timeout, continuing..." - - - name: Set up Platform - Run migrations - run: | - echo "Running migrations..." - docker compose -f ../docker-compose.resolved.yml run --rm migrate - echo "✅ Migrations completed" - env: - NEXT_PUBLIC_PW_TEST: true - - - name: Set up tests - Load cached E2E test data - if: steps.e2e-data-cache.outputs.cache-hit == 'true' - run: | - echo "✅ Found cached E2E test data, restoring..." - { - echo "SET session_replication_role = 'replica';" - cat /tmp/e2e_test_data.sql - echo "SET session_replication_role = 'origin';" - } | docker compose -f ../docker-compose.resolved.yml exec -T db psql -U postgres -d postgres -b - # Refresh materialized views after restore - docker compose -f ../docker-compose.resolved.yml exec -T db \ - psql -U postgres -d postgres -b -c "SET search_path TO platform; SELECT refresh_store_materialized_views();" || true - - echo "✅ E2E test data restored from cache" - - - name: Set up Platform - Start (all other services) - run: | - docker compose -f ../docker-compose.resolved.yml up -d --no-build - echo "Waiting for rest_server to be ready..." - timeout 60 sh -c 'until curl -f http://localhost:8006/health 2>/dev/null; do sleep 2; done' || echo "Rest server health check timeout, continuing..." - env: - NEXT_PUBLIC_PW_TEST: true - - - name: Set up tests - Create E2E test data - if: steps.e2e-data-cache.outputs.cache-hit != 'true' - run: | - echo "Creating E2E test data..." - docker cp ../backend/test/e2e_test_data.py $(docker compose -f ../docker-compose.resolved.yml ps -q rest_server):/tmp/e2e_test_data.py - docker compose -f ../docker-compose.resolved.yml exec -T rest_server sh -c "cd /app/autogpt_platform && python /tmp/e2e_test_data.py" || { - echo "❌ E2E test data creation failed!" - docker compose -f ../docker-compose.resolved.yml logs --tail=50 rest_server - exit 1 - } - - # Dump auth.users + platform schema for cache (two separate dumps) - echo "Dumping database for cache..." - { - docker compose -f ../docker-compose.resolved.yml exec -T db \ - pg_dump -U postgres --data-only --column-inserts \ - --table='auth.users' postgres - docker compose -f ../docker-compose.resolved.yml exec -T db \ - pg_dump -U postgres --data-only --column-inserts \ - --schema=platform \ - --exclude-table='platform._prisma_migrations' \ - --exclude-table='platform.apscheduler_jobs' \ - --exclude-table='platform.apscheduler_jobs_batched_notifications' \ - postgres - } > /tmp/e2e_test_data.sql - - echo "✅ Database dump created for caching ($(wc -l < /tmp/e2e_test_data.sql) lines)" - - - name: Set up tests - Enable corepack - run: corepack enable - - - name: Set up tests - Set up Node - uses: actions/setup-node@v6 - with: - node-version: "22.18.0" - cache: "pnpm" - cache-dependency-path: autogpt_platform/frontend/pnpm-lock.yaml - - - name: Set up tests - Install dependencies - run: pnpm install --frozen-lockfile - - - name: Set up tests - Install browser 'chromium' - run: pnpm playwright install --with-deps chromium - - - name: Run Playwright tests - run: pnpm test:no-build - continue-on-error: false - - - name: Upload Playwright report - if: always() - uses: actions/upload-artifact@v4 - with: - name: playwright-report - path: playwright-report - if-no-files-found: ignore - retention-days: 3 - - - name: Upload Playwright test results - if: always() - uses: actions/upload-artifact@v4 - with: - name: playwright-test-results - path: test-results - if-no-files-found: ignore - retention-days: 3 - - - name: Print Final Docker Compose logs - if: always() - run: docker compose -f ../docker-compose.resolved.yml logs - integration_test: runs-on: ubuntu-latest needs: setup diff --git a/.github/workflows/platform-fullstack-ci.yml b/.github/workflows/platform-fullstack-ci.yml index b4724245dc78..a67f80e414af 100644 --- a/.github/workflows/platform-fullstack-ci.yml +++ b/.github/workflows/platform-fullstack-ci.yml @@ -1,14 +1,18 @@ -name: AutoGPT Platform - Frontend CI +name: AutoGPT Platform - Full-stack CI on: push: branches: [master, dev] paths: - ".github/workflows/platform-fullstack-ci.yml" + - ".github/workflows/scripts/docker-ci-fix-compose-build-cache.py" + - ".github/workflows/scripts/get_package_version_from_lockfile.py" - "autogpt_platform/**" pull_request: paths: - ".github/workflows/platform-fullstack-ci.yml" + - ".github/workflows/scripts/docker-ci-fix-compose-build-cache.py" + - ".github/workflows/scripts/get_package_version_from_lockfile.py" - "autogpt_platform/**" merge_group: @@ -24,42 +28,28 @@ defaults: jobs: setup: runs-on: ubuntu-latest - outputs: - cache-key: ${{ steps.cache-key.outputs.key }} steps: - name: Checkout repository uses: actions/checkout@v6 - - name: Set up Node.js - uses: actions/setup-node@v6 - with: - node-version: "22.18.0" - - name: Enable corepack run: corepack enable - - name: Generate cache key - id: cache-key - run: echo "key=${{ runner.os }}-pnpm-${{ hashFiles('autogpt_platform/frontend/pnpm-lock.yaml', 'autogpt_platform/frontend/package.json') }}" >> $GITHUB_OUTPUT - - - name: Cache dependencies - uses: actions/cache@v5 + - name: Set up Node + uses: actions/setup-node@v6 with: - path: ~/.pnpm-store - key: ${{ steps.cache-key.outputs.key }} - restore-keys: | - ${{ runner.os }}-pnpm-${{ hashFiles('autogpt_platform/frontend/pnpm-lock.yaml') }} - ${{ runner.os }}-pnpm- + node-version: "22.18.0" + cache: "pnpm" + cache-dependency-path: autogpt_platform/frontend/pnpm-lock.yaml - - name: Install dependencies + - name: Install dependencies to populate cache run: pnpm install --frozen-lockfile - types: - runs-on: big-boi + check-api-types: + name: check API types + runs-on: ubuntu-latest needs: setup - strategy: - fail-fast: false steps: - name: Checkout repository @@ -67,70 +57,256 @@ jobs: with: submodules: recursive - - name: Set up Node.js - uses: actions/setup-node@v6 - with: - node-version: "22.18.0" - - - name: Enable corepack - run: corepack enable - - - name: Copy default supabase .env - run: | - cp ../.env.default ../.env + # ------------------------ Backend setup ------------------------ - - name: Copy backend .env - run: | - cp ../backend/.env.default ../backend/.env + - name: Set up Backend - Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" - - name: Run docker compose + - name: Set up Backend - Install Poetry + working-directory: autogpt_platform/backend run: | - docker compose -f ../docker-compose.yml --profile local up -d deps_backend + POETRY_VERSION=$(python ../../.github/workflows/scripts/get_package_version_from_lockfile.py poetry) + echo "Installing Poetry version ${POETRY_VERSION}" + curl -sSL https://install.python-poetry.org | POETRY_VERSION=$POETRY_VERSION python3 - - - name: Restore dependencies cache + - name: Set up Backend - Set up dependency cache uses: actions/cache@v5 with: - path: ~/.pnpm-store - key: ${{ needs.setup.outputs.cache-key }} - restore-keys: | - ${{ runner.os }}-pnpm- + path: ~/.cache/pypoetry + key: poetry-${{ runner.os }}-${{ hashFiles('autogpt_platform/backend/poetry.lock') }} - - name: Install dependencies - run: pnpm install --frozen-lockfile + - name: Set up Backend - Install dependencies + working-directory: autogpt_platform/backend + run: poetry install - - name: Setup .env - run: cp .env.default .env + - name: Set up Backend - Generate Prisma client + working-directory: autogpt_platform/backend + run: poetry run prisma generate && poetry run gen-prisma-stub - - name: Wait for services to be ready - run: | - echo "Waiting for rest_server to be ready..." - timeout 60 sh -c 'until curl -f http://localhost:8006/health 2>/dev/null; do sleep 2; done' || echo "Rest server health check timeout, continuing..." - echo "Waiting for database to be ready..." - timeout 60 sh -c 'until docker compose -f ../docker-compose.yml exec -T db pg_isready -U postgres 2>/dev/null; do sleep 2; done' || echo "Database ready check timeout, continuing..." + - name: Set up Frontend - Export OpenAPI schema from Backend + working-directory: autogpt_platform/backend + run: poetry run export-api-schema --output ../frontend/src/app/api/openapi.json + + # ------------------------ Frontend setup ------------------------ + + - name: Set up Frontend - Enable corepack + run: corepack enable + + - name: Set up Frontend - Set up Node + uses: actions/setup-node@v6 + with: + node-version: "22.18.0" + cache: "pnpm" + cache-dependency-path: autogpt_platform/frontend/pnpm-lock.yaml + + - name: Set up Frontend - Install dependencies + run: pnpm install --frozen-lockfile - - name: Generate API queries - run: pnpm generate:api:force + - name: Set up Frontend - Format OpenAPI schema + id: format-schema + run: pnpm prettier --write ./src/app/api/openapi.json - name: Check for API schema changes run: | if ! git diff --exit-code src/app/api/openapi.json; then echo "❌ API schema changes detected in src/app/api/openapi.json" echo "" - echo "The openapi.json file has been modified after running 'pnpm generate:api-all'." + echo "The openapi.json file has been modified after exporting the API schema." echo "This usually means changes have been made in the BE endpoints without updating the Frontend." echo "The API schema is now out of sync with the Front-end queries." echo "" echo "To fix this:" - echo "1. Pull the backend 'docker compose pull && docker compose up -d --build --force-recreate'" - echo "2. Run 'pnpm generate:api' locally" - echo "3. Run 'pnpm types' locally" - echo "4. Fix any TypeScript errors that may have been introduced" - echo "5. Commit and push your changes" + echo "\nIn the backend directory:" + echo "1. Run 'poetry run export-api-schema --output ../frontend/src/app/api/openapi.json'" + echo "\nIn the frontend directory:" + echo "2. Run 'pnpm prettier --write src/app/api/openapi.json'" + echo "3. Run 'pnpm generate:api'" + echo "4. Run 'pnpm types'" + echo "5. Fix any TypeScript errors that may have been introduced" + echo "6. Commit and push your changes" echo "" exit 1 else echo "✅ No API schema changes detected" fi - - name: Run Typescript checks + - name: Set up Frontend - Generate API client + id: generate-api-client + run: pnpm orval --config ./orval.config.ts + # Continue with type generation & check even if there are schema changes + if: success() || (steps.format-schema.outcome == 'success') + + - name: Check for TypeScript errors run: pnpm types + if: success() || (steps.generate-api-client.outcome == 'success') + + e2e_test: + name: end-to-end tests + runs-on: big-boi + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + submodules: recursive + + - name: Set up Platform - Copy default supabase .env + run: | + cp ../.env.default ../.env + + - name: Set up Platform - Copy backend .env and set OpenAI API key + run: | + cp ../backend/.env.default ../backend/.env + echo "OPENAI_INTERNAL_API_KEY=${{ secrets.OPENAI_API_KEY }}" >> ../backend/.env + env: + # Used by E2E test data script to generate embeddings for approved store agents + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + - name: Set up Platform - Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + driver: docker-container + driver-opts: network=host + + - name: Set up Platform - Expose GHA cache to docker buildx CLI + uses: crazy-max/ghaction-github-runtime@v4 + + - name: Set up Platform - Build Docker images (with cache) + working-directory: autogpt_platform + run: | + pip install pyyaml + + # Resolve extends and generate a flat compose file that bake can understand + docker compose -f docker-compose.yml config > docker-compose.resolved.yml + + # Add cache configuration to the resolved compose file + python ../.github/workflows/scripts/docker-ci-fix-compose-build-cache.py \ + --source docker-compose.resolved.yml \ + --cache-from "type=gha" \ + --cache-to "type=gha,mode=max" \ + --backend-hash "${{ hashFiles('autogpt_platform/backend/Dockerfile', 'autogpt_platform/backend/poetry.lock', 'autogpt_platform/backend/backend/**') }}" \ + --frontend-hash "${{ hashFiles('autogpt_platform/frontend/Dockerfile', 'autogpt_platform/frontend/pnpm-lock.yaml', 'autogpt_platform/frontend/src/**') }}" \ + --git-ref "${{ github.ref }}" + + # Build with bake using the resolved compose file (now includes cache config) + docker buildx bake --allow=fs.read=.. -f docker-compose.resolved.yml --load + env: + NEXT_PUBLIC_PW_TEST: true + + - name: Set up tests - Cache E2E test data + id: e2e-data-cache + uses: actions/cache@v5 + with: + path: /tmp/e2e_test_data.sql + key: e2e-test-data-${{ hashFiles('autogpt_platform/backend/test/e2e_test_data.py', 'autogpt_platform/backend/migrations/**', '.github/workflows/platform-fullstack-ci.yml') }} + + - name: Set up Platform - Start Supabase DB + Auth + run: | + docker compose -f ../docker-compose.resolved.yml up -d db auth --no-build + echo "Waiting for database to be ready..." + timeout 60 sh -c 'until docker compose -f ../docker-compose.resolved.yml exec -T db pg_isready -U postgres 2>/dev/null; do sleep 2; done' + echo "Waiting for auth service to be ready..." + timeout 60 sh -c 'until docker compose -f ../docker-compose.resolved.yml exec -T db psql -U postgres -d postgres -c "SELECT 1 FROM auth.users LIMIT 1" 2>/dev/null; do sleep 2; done' || echo "Auth schema check timeout, continuing..." + + - name: Set up Platform - Run migrations + run: | + echo "Running migrations..." + docker compose -f ../docker-compose.resolved.yml run --rm migrate + echo "✅ Migrations completed" + env: + NEXT_PUBLIC_PW_TEST: true + + - name: Set up tests - Load cached E2E test data + if: steps.e2e-data-cache.outputs.cache-hit == 'true' + run: | + echo "✅ Found cached E2E test data, restoring..." + { + echo "SET session_replication_role = 'replica';" + cat /tmp/e2e_test_data.sql + echo "SET session_replication_role = 'origin';" + } | docker compose -f ../docker-compose.resolved.yml exec -T db psql -U postgres -d postgres -b + # Refresh materialized views after restore + docker compose -f ../docker-compose.resolved.yml exec -T db \ + psql -U postgres -d postgres -b -c "SET search_path TO platform; SELECT refresh_store_materialized_views();" || true + + echo "✅ E2E test data restored from cache" + + - name: Set up Platform - Start (all other services) + run: | + docker compose -f ../docker-compose.resolved.yml up -d --no-build + echo "Waiting for rest_server to be ready..." + timeout 60 sh -c 'until curl -f http://localhost:8006/health 2>/dev/null; do sleep 2; done' || echo "Rest server health check timeout, continuing..." + env: + NEXT_PUBLIC_PW_TEST: true + + - name: Set up tests - Create E2E test data + if: steps.e2e-data-cache.outputs.cache-hit != 'true' + run: | + echo "Creating E2E test data..." + docker cp ../backend/test/e2e_test_data.py $(docker compose -f ../docker-compose.resolved.yml ps -q rest_server):/tmp/e2e_test_data.py + docker compose -f ../docker-compose.resolved.yml exec -T rest_server sh -c "cd /app/autogpt_platform && python /tmp/e2e_test_data.py" || { + echo "❌ E2E test data creation failed!" + docker compose -f ../docker-compose.resolved.yml logs --tail=50 rest_server + exit 1 + } + + # Dump auth.users + platform schema for cache (two separate dumps) + echo "Dumping database for cache..." + { + docker compose -f ../docker-compose.resolved.yml exec -T db \ + pg_dump -U postgres --data-only --column-inserts \ + --table='auth.users' postgres + docker compose -f ../docker-compose.resolved.yml exec -T db \ + pg_dump -U postgres --data-only --column-inserts \ + --schema=platform \ + --exclude-table='platform._prisma_migrations' \ + --exclude-table='platform.apscheduler_jobs' \ + --exclude-table='platform.apscheduler_jobs_batched_notifications' \ + postgres + } > /tmp/e2e_test_data.sql + + echo "✅ Database dump created for caching ($(wc -l < /tmp/e2e_test_data.sql) lines)" + + - name: Set up tests - Enable corepack + run: corepack enable + + - name: Set up tests - Set up Node + uses: actions/setup-node@v6 + with: + node-version: "22.18.0" + cache: "pnpm" + cache-dependency-path: autogpt_platform/frontend/pnpm-lock.yaml + + - name: Set up tests - Install dependencies + run: pnpm install --frozen-lockfile + + - name: Set up tests - Install browser 'chromium' + run: pnpm playwright install --with-deps chromium + + - name: Run Playwright tests + run: pnpm test:no-build + continue-on-error: false + + - name: Upload Playwright report + if: always() + uses: actions/upload-artifact@v4 + with: + name: playwright-report + path: playwright-report + if-no-files-found: ignore + retention-days: 3 + + - name: Upload Playwright test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: playwright-test-results + path: test-results + if-no-files-found: ignore + retention-days: 3 + + - name: Print Final Docker Compose logs + if: always() + run: docker compose -f ../docker-compose.resolved.yml logs diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py index b587479fee29..a0f26fdab9d9 100644 --- a/autogpt_platform/backend/backend/copilot/baseline/service.py +++ b/autogpt_platform/backend/backend/copilot/baseline/service.py @@ -40,7 +40,7 @@ from backend.copilot.service import ( _build_system_prompt, _generate_session_title, - client, + _get_openai_client, config, ) from backend.copilot.tools import execute_tool, get_available_tools @@ -89,7 +89,7 @@ async def _compress_session_messages( result = await compress_context( messages=messages_dict, model=config.model, - client=client, + client=_get_openai_client(), ) except Exception as e: logger.warning("[Baseline] Context compression with LLM failed: %s", e) @@ -235,7 +235,7 @@ async def stream_chat_completion_baseline( ) if tools: create_kwargs["tools"] = tools - response = await client.chat.completions.create(**create_kwargs) # type: ignore[arg-type] # dynamic kwargs + response = await _get_openai_client().chat.completions.create(**create_kwargs) # type: ignore[arg-type] # dynamic kwargs # Accumulate streamed response (text + tool calls) round_text = "" diff --git a/autogpt_platform/backend/backend/copilot/service.py b/autogpt_platform/backend/backend/copilot/service.py index 7f728ae1c1a4..5058341e6896 100644 --- a/autogpt_platform/backend/backend/copilot/service.py +++ b/autogpt_platform/backend/backend/copilot/service.py @@ -28,10 +28,24 @@ config = ChatConfig() settings = Settings() -client = LangfuseAsyncOpenAI(api_key=config.api_key, base_url=config.base_url) +_client: LangfuseAsyncOpenAI | None = None +_langfuse = None + + +def _get_openai_client() -> LangfuseAsyncOpenAI: + global _client + if _client is None: + _client = LangfuseAsyncOpenAI(api_key=config.api_key, base_url=config.base_url) + return _client + + +def _get_langfuse(): + global _langfuse + if _langfuse is None: + _langfuse = get_client() + return _langfuse -langfuse = get_client() # Default system prompt used when Langfuse is not configured # Provides minimal baseline tone and personality - all workflow, tools, and @@ -84,7 +98,7 @@ async def _get_system_prompt_template(context: str) -> str: else "latest" ) prompt = await asyncio.to_thread( - langfuse.get_prompt, + _get_langfuse().get_prompt, config.langfuse_prompt_name, label=label, cache_ttl_seconds=config.langfuse_prompt_cache_ttl, @@ -158,7 +172,7 @@ async def _generate_session_title( "environment": settings.config.app_env.value, } - response = await client.chat.completions.create( + response = await _get_openai_client().chat.completions.create( model=config.title_model, messages=[ {