diff --git a/.env.example b/.env.example index 9aedec5eed..3c3ea8ed6f 100644 --- a/.env.example +++ b/.env.example @@ -623,6 +623,12 @@ VICTORIA_LOGS_TENANT_ID= VERCEL_API_TOKEN= VERCEL_TEAM_ID= +# Neon (project, compute, operations, and Object Storage diagnostics) +NEON_API_KEY= +NEON_PROJECT_ID= +# Optional; defaults to the project's default branch +NEON_BRANCH_ID= + # PostHog (REST credentials) POSTHOG_PROJECT_ID= POSTHOG_PERSONAL_API_KEY= diff --git a/.github/ci/test_scope_rules.py b/.github/ci/test_scope_rules.py index 54df9957df..f3abba5eae 100644 --- a/.github/ci/test_scope_rules.py +++ b/.github/ci/test_scope_rules.py @@ -306,6 +306,10 @@ class PathRule: "tests/tools/test_sentry_search_issues_tool.py", ), ), + PathRule( + "integrations/neon/", + ("tests/integrations/neon/",), + ), PathRule( "integrations/supabase/", ("tests/integrations/test_supabase.py",), diff --git a/README.md b/README.md index cca03b6ffd..fbdc755e34 100644 --- a/README.md +++ b/README.md @@ -215,7 +215,7 @@ OpenSRE connects to 60+ tools and services across the modern cloud stack, from L | **AI / LLM Providers** | Anthropic · OpenAI · OpenAI Codex · Ollama · Google Gemini · OpenRouter · TrustedRouter · NVIDIA NIM · Bedrock | | | **Observability** | Grafana (Loki · Mimir · Tempo · annotations) · Datadog · Honeycomb · Coralogix · groundcover · CloudWatch · Sentry · Elasticsearch · Better Stack · Splunk · Victoria Logs · SignOz · OpenObserve · OpenSearch · Azure Monitor · New Relic | | | **Infrastructure** | Kubernetes · AWS (S3 · Lambda · EKS · EC2 · CloudTrail · Bedrock) · Azure · ArgoCD · Helm | | -| **Database** | MongoDB · ClickHouse · PostgreSQL · MySQL · MariaDB · MongoDB Atlas · Azure SQL · Snowflake · Redis · RDS · Supabase | | +| **Database** | MongoDB · ClickHouse · PostgreSQL · MySQL · MariaDB · MongoDB Atlas · Azure SQL · Snowflake · Redis · RDS · Supabase · Neon | | | **Data Platform** | Apache Airflow · Apache Kafka · Prefect · RabbitMQ · Dagster · Temporal | | | **Dev Tools** | GitHub · GitHub MCP · Bitbucket · GitLab · Jenkins | | | **Incident Management** | PagerDuty · Jira · Alertmanager · incident.io · ServiceNow · Trello (client helpers only) | [Linear](https://github.com/Tracer-Cloud/opensre/issues/124) | diff --git a/README.zh-CN.md b/README.zh-CN.md index 6bd7b27b6e..12f436d7e6 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -233,7 +233,7 @@ OpenSRE 可连接现代云技术栈中的 60 多种工具与服务，从 LLM 提 | **AI / LLM 提供商** | Anthropic · OpenAI · OpenAI Codex · Ollama · Google Gemini · OpenRouter · TrustedRouter · NVIDIA NIM · Bedrock | | | **可观测性** | Grafana (Loki · Mimir · Tempo · annotations) · Datadog · Honeycomb · Coralogix · groundcover · CloudWatch · Sentry · Elasticsearch · Better Stack · Splunk · Victoria Logs · SignOz · OpenObserve · OpenSearch · Azure Monitor · New Relic | | | **基础设施** | Kubernetes · AWS (S3 · Lambda · EKS · EC2 · CloudTrail · Bedrock) · Azure · ArgoCD · Helm | | -| **数据库** | MongoDB · ClickHouse · PostgreSQL · MySQL · MariaDB · MongoDB Atlas · Azure SQL · Snowflake · Redis · RDS · Supabase | | +| **数据库** | MongoDB · ClickHouse · PostgreSQL · MySQL · MariaDB · MongoDB Atlas · Azure SQL · Snowflake · Redis · RDS · Supabase · Neon | | | **数据平台** | Apache Airflow · Apache Kafka · Prefect · RabbitMQ · Dagster · Temporal | | | **开发工具** | GitHub · GitHub MCP · Bitbucket · GitLab · Jenkins | | | **故障管理** | PagerDuty · Jira · Alertmanager · incident.io · ServiceNow · Trello（仅提供客户端辅助功能） | [Linear](https://github.com/Tracer-Cloud/opensre/issues/124) | diff --git a/config/constants/__init__.py b/config/constants/__init__.py index af10234a3b..76b9d503ef 100644 --- a/config/constants/__init__.py +++ b/config/constants/__init__.py @@ -953,6 +953,18 @@ from config.constants.mysql import ( MYSQL_USERNAME_ENV as MYSQL_USERNAME_ENV, ) + from config.constants.neon import ( + NEON_API_BASE_URL as NEON_API_BASE_URL, + ) + from config.constants.neon import ( + NEON_API_KEY_ENV as NEON_API_KEY_ENV, + ) + from config.constants.neon import ( + NEON_BRANCH_ID_ENV as NEON_BRANCH_ID_ENV, + ) + from config.constants.neon import ( + NEON_PROJECT_ID_ENV as NEON_PROJECT_ID_ENV, + ) from config.constants.new_relic import ( NEW_RELIC_ACCOUNT_ID_ENV as NEW_RELIC_ACCOUNT_ID_ENV, ) diff --git a/config/constants/exports.py b/config/constants/exports.py index 1964508f6d..184f5fdb57 100644 --- a/config/constants/exports.py +++ b/config/constants/exports.py @@ -425,6 +425,11 @@ "MYSQL_PORT_ENV": "mysql", "MYSQL_SSL_MODE_ENV": "mysql", "MYSQL_USERNAME_ENV": "mysql", + # neon + "NEON_API_BASE_URL": "neon", + "NEON_API_KEY_ENV": "neon", + "NEON_BRANCH_ID_ENV": "neon", + "NEON_PROJECT_ID_ENV": "neon", # new_relic "NEW_RELIC_ACCOUNT_ID_ENV": "new_relic", "NEW_RELIC_ALLOWED_BASE_URLS": "new_relic", diff --git a/config/constants/neon.py b/config/constants/neon.py new file mode 100644 index 0000000000..6a14be156f --- /dev/null +++ b/config/constants/neon.py @@ -0,0 +1,21 @@ +"""Neon environment variable names and shared static constants. + +``NEON_API_BASE_URL`` is the Neon control-plane API (projects, branches, +computes, operations, and Object Storage buckets). Postgres connections and +the S3-compatible Object Storage data plane use per-branch hosts that the API +reports, so neither has a fixed constant here. +""" + +from __future__ import annotations + +NEON_API_KEY_ENV = "NEON_API_KEY" +NEON_PROJECT_ID_ENV = "NEON_PROJECT_ID" +NEON_BRANCH_ID_ENV = "NEON_BRANCH_ID" +NEON_API_BASE_URL = "https://console.neon.tech/api/v2" + +__all__ = [ + "NEON_API_BASE_URL", + "NEON_API_KEY_ENV", + "NEON_BRANCH_ID_ENV", + "NEON_PROJECT_ID_ENV", +] diff --git a/docs/docs.json b/docs/docs.json index de65c84b68..045723d5f9 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -219,6 +219,7 @@ "integrations/databases/mongodb", "integrations/databases/mongodb-atlas", "integrations/databases/mysql", + "integrations/databases/neon", "integrations/databases/opensearch", "integrations/databases/postgresql", "integrations/databases/rds", diff --git a/docs/install/environment-variables.mdx b/docs/install/environment-variables.mdx index fffe8d9a03..a06f1da970 100644 --- a/docs/install/environment-variables.mdx +++ b/docs/install/environment-variables.mdx @@ -412,12 +412,12 @@ Mirror conversation history and memory to a user-owned object store. Details: [R | Variable | Default | Description | | --- | --- | --- | | `OPENSRE_REMOTE_SYNC` | | Enable remote sync (`1`/`true`) | -| `OPENSRE_REMOTE_SYNC_PROVIDER` | `aws` | Store backend (`aws`, `gcs`, `vercel`, `azure`, or `s3compat`) | +| `OPENSRE_REMOTE_SYNC_PROVIDER` | `aws` | Store backend (`aws`, `gcs`, `vercel`, `azure`, `s3compat`, or `neon`) | | `OPENSRE_REMOTE_SYNC_BUCKET` | | Bucket / Blob store name (required when sync is on) | | `OPENSRE_REMOTE_SYNC_PREFIX` | `opensre` | Key prefix inside the store | | `OPENSRE_REMOTE_SYNC_REGION` | | Region override | | `OPENSRE_REMOTE_SYNC_PROFILE` | | Named credentials profile | -| `OPENSRE_REMOTE_SYNC_ENDPOINT_URL` | | Custom S3 endpoint when provider is `s3compat` | +| `OPENSRE_REMOTE_SYNC_ENDPOINT_URL` | | Custom S3 endpoint when provider is `s3compat` or `neon` | | `OPENSRE_REMOTE_SYNC_EXCLUDE` | | Comma-separated globs to exclude from sync | | `OPENSRE_REMOTE_SYNC_EXCLUDE_OFF` | | Ignore configured exclusions for one run | | `BLOB_READ_WRITE_TOKEN` | | Vercel Blob token (when provider is `vercel`) | @@ -640,6 +640,9 @@ Mirror conversation history and memory to a user-owned object store. Details: [R | `VERCEL_API_TOKEN` | | Vercel API token | | `VERCEL_TEAM_ID` | | Vercel team ID | | `VERCEL_RUNTIME_LOGS_READ_TIMEOUT` | `600` | Seconds to wait per Vercel runtime-log read (minimum `30`) | +| `NEON_API_KEY` | | Neon API key | +| `NEON_PROJECT_ID` | | Neon project the Neon tools read | +| `NEON_BRANCH_ID` | | Branch for the Neon Object Storage tools (defaults to the project default branch) | | `GOOGLE_CREDENTIALS_FILE` | | Path to Google service-account JSON for Docs export | | `GOOGLE_DRIVE_FOLDER_ID` | | Google Drive folder for Docs export | diff --git a/docs/install/remote-sync.mdx b/docs/install/remote-sync.mdx index 6ffb9ed153..8d60f72425 100644 --- a/docs/install/remote-sync.mdx +++ b/docs/install/remote-sync.mdx @@ -1,6 +1,6 @@ --- title: "Sync to your own object store" -description: "Copy sessions and memory between machines using an S3, GCS, Azure, or Vercel Blob store you own." +description: "Copy sessions and memory between machines using an S3, GCS, Azure, Vercel Blob, or Neon Object Storage store you own." --- By default OpenSRE keeps everything on your machine. Turn on remote sync to copy **conversations** and **memory** to a private store you own, so another laptop can continue where you left off. @@ -70,7 +70,7 @@ Or set env vars yourself: export OPENSRE_REMOTE_SYNC=1 export OPENSRE_REMOTE_SYNC_BUCKET=my-opensre-bucket # optional: -# export OPENSRE_REMOTE_SYNC_PROVIDER=aws # default; also gcs, vercel, azure +# export OPENSRE_REMOTE_SYNC_PROVIDER=aws # default; also gcs, vercel, azure, s3compat, neon # export OPENSRE_REMOTE_SYNC_PREFIX=opensre # default ``` @@ -79,12 +79,12 @@ A bucket name alone does nothing — you must also set `OPENSRE_REMOTE_SYNC=1` ( | Variable | What it does | | --- | --- | | `OPENSRE_REMOTE_SYNC` | `1` / `true` turns sync on | -| `OPENSRE_REMOTE_SYNC_PROVIDER` | `aws` (default), `gcs`, `vercel`, `azure`, or `s3compat` | +| `OPENSRE_REMOTE_SYNC_PROVIDER` | `aws` (default), `gcs`, `vercel`, `azure`, `s3compat`, or `neon` | | `OPENSRE_REMOTE_SYNC_BUCKET` | Bucket, container, or Blob store name (required when on) | | `OPENSRE_REMOTE_SYNC_PREFIX` | Key prefix (default `opensre`). Same prefix = shared history | | `OPENSRE_REMOTE_SYNC_REGION` | AWS region (AWS only) | | `OPENSRE_REMOTE_SYNC_PROFILE` | AWS profile name, or Azure storage account name when `provider=azure` | -| `OPENSRE_REMOTE_SYNC_ENDPOINT_URL` | Custom S3 endpoint when `provider=s3compat` (MinIO / R2 / Spaces) | +| `OPENSRE_REMOTE_SYNC_ENDPOINT_URL` | Custom S3 endpoint when `provider=s3compat` (MinIO / R2 / Spaces) or `provider=neon` | | `BLOB_READ_WRITE_TOKEN` | Vercel Blob token (Vercel only) | | `OPENSRE_REMOTE_SYNC_EXCLUDE` | Comma-separated paths to skip (see below) | | `OPENSRE_REMOTE_SYNC_EXCLUDE_OFF` | `1` ignores exclusions for one run | @@ -130,6 +130,8 @@ Remote sync is for a **personal machine**. Org-bound Slack/Discord turns are ref **S3-compatible (MinIO / R2 / Spaces)** — set `OPENSRE_REMOTE_SYNC_PROVIDER=s3compat` and `OPENSRE_REMOTE_SYNC_ENDPOINT_URL` to the custom endpoint (or `AWS_ENDPOINT_URL_S3`). Path-style addressing is enabled for MinIO. Use the same ambient S3 credentials as AWS. +**Neon Object Storage**: private bucket on a Neon branch. Set `OPENSRE_REMOTE_SYNC_PROVIDER=neon` and run `neon env pull` in the shell (or `.env`) OpenSRE uses, which sets `AWS_ENDPOINT_URL_S3`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_REGION` for the linked branch. Without the CLI, set `OPENSRE_REMOTE_SYNC_ENDPOINT_URL` to the branch's `s3_endpoint` and use a key from `neon credentials create` with the `storage:read` and `storage:write` scopes. The endpoint is required: unlike `s3compat`, the `neon` provider refuses to start without one so it never falls back to AWS. Buckets belong to a branch, so every machine must use the same branch endpoint. `status` reports the bucket as public when its access level is `public_read`. + ## Skip some paths By default every session and memory file syncs. To hold some back: diff --git a/docs/integrations/databases/neon.mdx b/docs/integrations/databases/neon.mdx new file mode 100644 index 0000000000..46a3b22052 --- /dev/null +++ b/docs/integrations/databases/neon.mdx @@ -0,0 +1,152 @@ +--- +title: "Neon" +description: "Connect Neon so OpenSRE can check compute state, control-plane operations, and Object Storage buckets during incidents" +--- + +## Overview + +OpenSRE reads your Neon project through the Neon API to answer the questions that come up first in a Postgres incident: is the compute running or scaled to zero, did a restart or branch operation fail, which branch is the app pointed at, and did the expected files land in Neon Object Storage. + +The integration is read-only. It never starts, suspends, resets, or deletes anything. + +Neon also plugs into two other parts of OpenSRE: + +- **Postgres diagnostics** (active queries, locks, slow queries, table stats) come from the [PostgreSQL integration](/integrations/databases/postgresql) pointed at your Neon compute. See [Postgres diagnostics on Neon](#postgres-diagnostics-on-neon). +- **Remote sync** can mirror sessions and memory to a Neon Object Storage bucket. See [Remote sync to Neon Object Storage](#remote-sync-to-neon-object-storage). + +## Prerequisites + +- A Neon project +- A Neon API key that can read that project +- The project ID (for example `little-lab-10567887`) + +## Setup + +### Option 1: Interactive CLI + +```bash +opensre integrations setup neon +``` + +Provide the API key, the project ID, and optionally a branch ID when prompted. Setup verifies the key against the project before saving. + +### Option 2: Environment variables + +Add to your `.env`: + +```bash +NEON_API_KEY=your-neon-api-key +NEON_PROJECT_ID=little-lab-10567887 +NEON_BRANCH_ID=br-summer-art-bdbz1xg2 # optional +``` + +| Variable | Default | Description | +| --- | --- | --- | +| `NEON_API_KEY` | | **Required.** Neon API key | +| `NEON_PROJECT_ID` | | **Required.** The project OpenSRE investigates | +| `NEON_BRANCH_ID` | project default branch | Branch the Object Storage tools read when a call does not name one | + +### Option 3: Persistent store + +```json +{ + "version": 1, + "integrations": [ + { + "id": "neon-prod", + "service": "neon", + "status": "active", + "credentials": { + "api_key": "your-neon-api-key", + "project_id": "little-lab-10567887", + "branch_id": "" + } + } + ] +} +``` + +## Credentials + +1. In the Neon Console, open **Organization settings** → **API keys**. +2. Create a key and, under the project option, limit it to the project OpenSRE should read. +3. Copy the key. It is shown once. +4. Copy the project ID from **Project settings** → **General**, or from the URL of the project page. + +A personal API key (**Account settings** → **API keys**) also works but can read every project you can. + + +The key is injected into each tool call by OpenSRE and hidden from the model, and the project ID is fixed to the configured one. The model can pick a branch inside that project but cannot point the key at another project. + + +OpenSRE calls the Neon API at `https://console.neon.tech/api/v2`. + +## Tools + +| Tool | What it does | +| --- | --- | +| `get_neon_project_status` | Region, Postgres version, IP allowlist, branches and their state, compute endpoints (active or idle, autoscaling CU limits, scale-to-zero timeout, last active time), and recent operations with failed and in-progress ones listed separately. Optional `branch_id` narrows computes and operations to one branch. | +| `get_neon_storage_buckets` | Whether Object Storage is enabled on a branch, its S3 endpoint and region, and every bucket with its access level. Public buckets are listed separately. | +| `list_neon_storage_objects` | One page of objects (key, size, last modified, ETag) in a bucket, with optional `prefix`. Folders are collapsed on `/` unless `recursive` is true. Returns `next_cursor` when more objects exist. | + +The tools are available in both chat and investigation turns. + +## Postgres diagnostics on Neon + +Configure the [PostgreSQL integration](/integrations/databases/postgresql) with your Neon connection details to get active queries, locks, replication, server status, slow queries, and table stats: + +```bash +POSTGRESQL_HOST=ep-sparkling-lake-bd31vdlv.c-8.us-east-2.aws.neon.tech +POSTGRESQL_DATABASE=neondb +POSTGRESQL_USERNAME=neondb_owner +POSTGRESQL_PASSWORD=your-password +POSTGRESQL_SSL_MODE=require +``` + +- Use the **direct** compute host. The pooled host (the one with `-pooler` in its name) rejects the connection, because OpenSRE sets a statement timeout at connect time and Neon's connection pooler does not accept that startup parameter. +- Set `POSTGRESQL_SSL_MODE=require`. Neon only accepts TLS connections. +- For slow queries, run `CREATE EXTENSION pg_stat_statements;` once in the database. Neon already preloads the library. +- A compute that has scaled to zero starts on the first connection, so the first diagnostic call after an idle period takes a moment longer. + +## Remote sync to Neon Object Storage + +[Remote sync](/install/remote-sync) can mirror your sessions and memory to a Neon Object Storage bucket: + +```bash +neon env pull # writes AWS_ENDPOINT_URL_S3, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION +opensre remote-sync setup --provider neon --bucket opensre-sync +opensre remote-sync status +``` + +Keep the bucket private. `opensre remote-sync status` warns when the bucket's access level is `public_read`. + +## Verify + +```bash +opensre integrations verify neon +``` + +Expected output: + +``` +SERVICE │ SOURCE │ STATUS │ DETAIL +neon │ local env │ ✓ passed │ Connected to Neon project opensre-neon-test (little-lab-10567887, aws-us-east-2). +``` + +Inside the REPL: `/integrations verify neon` or `/verify neon`. + +## Troubleshooting + +| Symptom | Fix | +| --- | --- | +| **Neon rejected the API key (HTTP 401)** | The key was revoked or mistyped. Create a new one. | +| **Neon resource not found (HTTP 404)** | Check `NEON_PROJECT_ID`. A project-scoped key also returns 404 for any other project. | +| **Object Storage is not enabled on this branch** | Create a bucket on that branch first (Console or `neon buckets create`). | +| **Bucket not found** | Buckets belong to a branch. Pass the branch the app uses, or set `NEON_BRANCH_ID`. | +| **PostgreSQL tools fail with "unsupported startup parameter"** | `POSTGRESQL_HOST` points at the `-pooler` host. Use the direct host. | + +## Security + +- Prefer a project-scoped API key so OpenSRE can only read the project it investigates. +- Store the key in `.env` or the OpenSRE credentials store, not in source code. +- Rotate the key periodically and revoke it when you remove the integration. diff --git a/docs/integrations/index.mdx b/docs/integrations/index.mdx index e6f4bca2d2..4572a958cb 100644 --- a/docs/integrations/index.mdx +++ b/docs/integrations/index.mdx @@ -56,7 +56,7 @@ configured separately under [LLM providers](/platform/llm-providers). | **Code and CI** | [Bitbucket](/integrations/code/bitbucket), [GitHub](/integrations/code/github), [GitHub Actions](/integrations/code/github-actions), [GitLab](/integrations/code/gitlab), [Jenkins](/integrations/code/jenkins) | | **Collaboration** | [Google Docs](/integrations/collaboration/google-docs), [Jira](/integrations/collaboration/jira), [Trello](/integrations/collaboration/trello) | | **Messaging** | [Slack](/integrations/messaging/slack), [Discord](/integrations/messaging/discord), [Telegram](/integrations/messaging/telegram), [Rocket.Chat](/integrations/messaging/rocketchat), [Buzz](/integrations/messaging/buzz), [WhatsApp](/integrations/messaging/whatsapp), [Twilio SMS](/integrations/messaging/twilio-sms), [SMTP](/integrations/messaging/smtp) | -| **Databases** | [Azure SQL](/integrations/databases/azure-sql), [ClickHouse](/integrations/databases/clickhouse), [Elasticsearch](/integrations/databases/elasticsearch), [MariaDB](/integrations/databases/mariadb), [MongoDB](/integrations/databases/mongodb), [MongoDB Atlas](/integrations/databases/mongodb-atlas), [MySQL](/integrations/databases/mysql), [OpenSearch](/integrations/databases/opensearch), [PostgreSQL](/integrations/databases/postgresql), [RDS](/integrations/databases/rds), [Redis](/integrations/databases/redis), [Snowflake](/integrations/databases/snowflake), [Supabase](/integrations/databases/supabase) | +| **Databases** | [Azure SQL](/integrations/databases/azure-sql), [ClickHouse](/integrations/databases/clickhouse), [Elasticsearch](/integrations/databases/elasticsearch), [MariaDB](/integrations/databases/mariadb), [MongoDB](/integrations/databases/mongodb), [MongoDB Atlas](/integrations/databases/mongodb-atlas), [MySQL](/integrations/databases/mysql), [Neon](/integrations/databases/neon), [OpenSearch](/integrations/databases/opensearch), [PostgreSQL](/integrations/databases/postgresql), [RDS](/integrations/databases/rds), [Redis](/integrations/databases/redis), [Snowflake](/integrations/databases/snowflake), [Supabase](/integrations/databases/supabase) | | **Workflows** | [Airflow](/integrations/workflows/airflow), [Dagster](/integrations/workflows/dagster), [Kafka](/integrations/workflows/kafka), [Prefect](/integrations/workflows/prefect), [RabbitMQ](/integrations/workflows/rabbitmq), [Temporal](/integrations/workflows/temporal) | ## Hosted (OpenSRE Cloud) diff --git a/docs/styles/config/vocabularies/Mintlify/accept.txt b/docs/styles/config/vocabularies/Mintlify/accept.txt index 9da6f03c51..592bd5b682 100644 --- a/docs/styles/config/vocabularies/Mintlify/accept.txt +++ b/docs/styles/config/vocabularies/Mintlify/accept.txt @@ -124,6 +124,7 @@ pg_stat_statements prefetch replayable Supabase +Neon Trendshift Trello repos diff --git a/infrastructure/filestorage/enums.py b/infrastructure/filestorage/enums.py index bc507200cc..02ce809a77 100644 --- a/infrastructure/filestorage/enums.py +++ b/infrastructure/filestorage/enums.py @@ -37,6 +37,7 @@ class BuiltInProvider(StrEnum): VERCEL = "vercel" AZURE = "azure" S3COMPAT = "s3compat" + NEON = "neon" class RemoteSyncSubcommand(StrEnum): diff --git a/infrastructure/filestorage/providers/neon.py b/infrastructure/filestorage/providers/neon.py new file mode 100644 index 0000000000..e0e46c89df --- /dev/null +++ b/infrastructure/filestorage/providers/neon.py @@ -0,0 +1,106 @@ +"""Neon Object Storage backend for remote sync. + +Neon Object Storage is S3-compatible and branch-scoped: each Neon branch has its +own path-style endpoint (``https://.storage...aws.neon.tech``) +and its own buckets. This provider is the S3-compatible store with two Neon +rules on top: + +- An endpoint is required. Without one, a plain S3 client would quietly talk to + AWS instead, so the store fails closed with a pointer to ``neon env pull``. +- The endpoint and keys come from the environment ``neon env pull`` writes + (``AWS_ENDPOINT_URL_S3``, ``AWS_ACCESS_KEY_ID``, ``AWS_SECRET_ACCESS_KEY``, + ``AWS_REGION``), or ``OPENSRE_REMOTE_SYNC_ENDPOINT_URL`` and a credential from + ``neon credentials create``. opensre never stores them. + +Bucket exposure comes from ``GetBucketPolicyStatus``, which Neon answers from +the bucket's access level (``public_read`` reports public). +""" + +from __future__ import annotations + +from typing import Any + +from infrastructure.filestorage.config import RemoteSyncConfig +from infrastructure.filestorage.enums import BucketExposure, BuiltInProvider, RemoteSyncField +from infrastructure.filestorage.errors import RemoteSyncConfigError +from infrastructure.filestorage.exposure import PublicAccessStatus +from infrastructure.filestorage.providers._s3_shared import s3_check_public_access +from infrastructure.filestorage.providers.registry import SetupExtraField, register_object_store +from infrastructure.filestorage.providers.s3compat import ( + S3CompatObjectStore, + resolve_endpoint_url, +) + +PROVIDER_NAME = BuiltInProvider.NEON +CREDENTIAL_HINT = ( + "Neon Object Storage credentials come from the environment: run `neon env pull` " + "(AWS_ENDPOINT_URL_S3, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY), or set " + "OPENSRE_REMOTE_SYNC_ENDPOINT_URL plus a key from `neon credentials create`." +) +MISSING_ENDPOINT_MESSAGE = ( + "Neon remote sync needs the branch's storage endpoint. Run `neon env pull` to set " + "AWS_ENDPOINT_URL_S3, or set OPENSRE_REMOTE_SYNC_ENDPOINT_URL to the s3_endpoint " + "shown by `neon api /projects//branches//storage`." +) +EXTRA_FIELDS = (SetupExtraField(RemoteSyncField.REGION, "Region (blank to use AWS_REGION)"),) +# Same ceiling as the generic S3-compatible store: boto3 retries throttled writes. +MAX_PARALLEL_UPLOADS = 16 +_UNSUPPORTED_POLICY_STATUS_CODES = frozenset({"NotImplemented", "MethodNotAllowed"}) + + +class NeonObjectStore(S3CompatObjectStore): + """Reads and writes objects under one Neon Object Storage bucket and prefix.""" + + def __init__( + self, + config: RemoteSyncConfig, + *, + endpoint_url: str | None = None, + client: Any | None = None, + ) -> None: + resolved = resolve_endpoint_url(endpoint_url) + if not resolved: + raise RemoteSyncConfigError(MISSING_ENDPOINT_MESSAGE) + super().__init__(config, endpoint_url=resolved, client=client) + + def describe(self) -> str: + return f"neon://{self._config.bucket}/{self._config.prefix}" + + +def _factory(config: RemoteSyncConfig) -> NeonObjectStore: + return NeonObjectStore(config) + + +def check_public_access( + config: RemoteSyncConfig, *, client: Any | None = None +) -> PublicAccessStatus: + """Ask Neon whether ``config.bucket`` is publicly readable (``public_read``).""" + if client is None and not resolve_endpoint_url(): + return PublicAccessStatus(BucketExposure.UNKNOWN, "no Neon storage endpoint configured") + return s3_check_public_access( + config, + build_client=lambda cfg: NeonObjectStore(cfg)._client, + client=client, + unsupported_codes=_UNSUPPORTED_POLICY_STATUS_CODES, + no_policy_status=PublicAccessStatus(BucketExposure.PRIVATE), + ) + + +register_object_store( + PROVIDER_NAME, + _factory, + credential_hint=CREDENTIAL_HINT, + extra_fields=EXTRA_FIELDS, + public_access_checker=check_public_access, + max_parallel_uploads=MAX_PARALLEL_UPLOADS, +) + +__all__ = [ + "CREDENTIAL_HINT", + "EXTRA_FIELDS", + "MAX_PARALLEL_UPLOADS", + "MISSING_ENDPOINT_MESSAGE", + "PROVIDER_NAME", + "NeonObjectStore", + "check_public_access", +] diff --git a/infrastructure/filestorage/providers/registry.py b/infrastructure/filestorage/providers/registry.py index 8c52599f7e..45b760b53f 100644 --- a/infrastructure/filestorage/providers/registry.py +++ b/infrastructure/filestorage/providers/registry.py @@ -71,6 +71,7 @@ class SetupExtraField: BuiltInProvider.VERCEL.value: "infrastructure.filestorage.providers.vercel", BuiltInProvider.AZURE.value: "infrastructure.filestorage.providers.azure", BuiltInProvider.S3COMPAT.value: "infrastructure.filestorage.providers.s3compat", + BuiltInProvider.NEON.value: "infrastructure.filestorage.providers.neon", } diff --git a/infrastructure/filestorage/providers/s3compat.py b/infrastructure/filestorage/providers/s3compat.py index 5a766e4b94..812648fefe 100644 --- a/infrastructure/filestorage/providers/s3compat.py +++ b/infrastructure/filestorage/providers/s3compat.py @@ -50,7 +50,7 @@ def __init__( client: Any | None = None, ) -> None: self._config = config - self._endpoint_url = _resolve_endpoint_url(endpoint_url) + self._endpoint_url = resolve_endpoint_url(endpoint_url) self._client = client if client is not None else _build_client(config, self._endpoint_url) def describe(self) -> str: @@ -67,7 +67,7 @@ def put_object(self, key: str, data: bytes) -> None: raise RemoteSyncUnavailableError(f"cannot write {key} — {s3_reason(exc)}") from exc -def _resolve_endpoint_url(explicit: str | None = None) -> str | None: +def resolve_endpoint_url(explicit: str | None = None) -> str | None: """The endpoint URL to use, else ``None`` for the built-in AWS endpoint. Checked in order: the ``explicit`` argument (tests / direct callers), the @@ -135,7 +135,7 @@ def check_public_access( """ return s3_check_public_access( config, - build_client=lambda cfg: _build_client(cfg, _resolve_endpoint_url()), + build_client=lambda cfg: _build_client(cfg, resolve_endpoint_url()), client=client, unsupported_codes=_UNSUPPORTED_POLICY_STATUS_CODES, no_policy_status=PublicAccessStatus(BucketExposure.UNKNOWN, "no bucket policy present"), @@ -160,4 +160,5 @@ def check_public_access( "PROVIDER_NAME", "S3CompatObjectStore", "check_public_access", + "resolve_endpoint_url", ] diff --git a/integrations/_catalog_impl.py b/integrations/_catalog_impl.py index f1625e2dd0..fa35d2a34a 100644 --- a/integrations/_catalog_impl.py +++ b/integrations/_catalog_impl.py @@ -146,6 +146,7 @@ MYSQL_SSL_MODE_ENV, MYSQL_USERNAME_ENV, ) +from config.constants.neon import NEON_API_KEY_ENV, NEON_BRANCH_ID_ENV, NEON_PROJECT_ID_ENV from config.constants.new_relic import ( NEW_RELIC_ACCOUNT_ID_ENV, NEW_RELIC_API_KEY_ENV, @@ -309,6 +310,8 @@ from integrations.mongodb_atlas import classify as _classify_mongodb_atlas from integrations.mysql import build_mysql_config from integrations.mysql import classify as _classify_mysql +from integrations.neon import build_neon_config +from integrations.neon import classify as _classify_neon from integrations.new_relic import classify as _classify_new_relic from integrations.new_relic.config import NewRelicIntegrationConfig from integrations.openobserve import classify as _classify_openobserve @@ -494,6 +497,7 @@ def classify_integrations(integrations: list[dict[str, Any]]) -> dict[str, Any]: "mongodb_atlas": _classify_mongodb_atlas, "mariadb": _classify_mariadb, "vercel": _classify_vercel, + "neon": _classify_neon, "pagerduty": _classify_pagerduty, "pipedream": _classify_pipedream, "incident_io": _classify_incident_io, @@ -1072,6 +1076,27 @@ def load_env_integrations() -> list[dict[str, Any]]: ) ) + neon_api_key = resolve_env_credential(NEON_API_KEY_ENV) + neon_project_id = os.getenv(NEON_PROJECT_ID_ENV, "").strip() + if neon_api_key and neon_project_id: + try: + neon_config = build_neon_config( + { + "api_key": neon_api_key, + "project_id": neon_project_id, + "branch_id": os.getenv(NEON_BRANCH_ID_ENV, "").strip(), + } + ) + except Exception as exc: + _report_env_loader_failure(exc, integration="neon") + else: + integrations.append( + _active_env_record( + "neon", + neon_config.model_dump(include={"api_key", "project_id", "branch_id"}), + ) + ) + pagerduty_api_key = resolve_env_credential(PAGERDUTY_API_KEY_ENV) if pagerduty_api_key: try: diff --git a/integrations/alert_source_catalog.py b/integrations/alert_source_catalog.py index 33cc9fb06f..80c3a05389 100644 --- a/integrations/alert_source_catalog.py +++ b/integrations/alert_source_catalog.py @@ -53,6 +53,7 @@ "dagster": routing(("dagster",), ("dagster",)), "rabbitmq": routing(("rabbitmq",), ("rabbitmq",)), "supabase": routing(("supabase",), ("supabase",)), + "neon": routing(("neon", "postgresql"), ("neon", "postgresql")), "opensearch": routing(("opensearch",), ("opensearch",)), "openobserve": routing(("openobserve",), ("openobserve",)), "betterstack": routing(("betterstack",), ("betterstack",)), @@ -108,6 +109,7 @@ "kafka": ("kafka",), "rabbitmq": ("rabbitmq", "amqp"), "supabase": ("supabase",), + "neon": ("neon", "neon.tech", "neon postgres", "neon branch", "neon compute"), "opensearch": ("opensearch", "elasticsearch"), "openobserve": ("openobserve",), "betterstack": ("betterstack", "better stack"), diff --git a/integrations/catalog.py b/integrations/catalog.py index 24cc158b5a..87b76eaafb 100644 --- a/integrations/catalog.py +++ b/integrations/catalog.py @@ -155,6 +155,7 @@ def add(service: str, configured: bool) -> None: or _all_env("RAILWAY_PROJECT", "RAILWAY_SERVICE", "RAILWAY_ENVIRONMENT"), ) add("vercel", _env_is_set("VERCEL_API_TOKEN")) + add("neon", _all_env("NEON_API_KEY", "NEON_PROJECT_ID")) add("pagerduty", _env_is_set("PAGERDUTY_API_KEY")) add("incident_io", _env_is_set("INCIDENT_IO_API_KEY")) add("jira", _all_env("JIRA_BASE_URL", "JIRA_EMAIL", "JIRA_API_TOKEN")) diff --git a/integrations/cli.py b/integrations/cli.py index e07118877b..f86efa69f4 100644 --- a/integrations/cli.py +++ b/integrations/cli.py @@ -221,6 +221,12 @@ def _setup_vercel() -> None: _run_spec_setup(VERCEL_SETUP) +def _setup_neon() -> None: + from integrations.neon.setup import NEON_SETUP + + _run_spec_setup(NEON_SETUP) + + def _setup_railway() -> None: from integrations.railway.setup import RAILWAY_SETUP @@ -504,6 +510,7 @@ def _setup_google_docs() -> None: "tracer": _setup_tracer, "vercel": _setup_vercel, "railway": _setup_railway, + "neon": _setup_neon, "github": setup_github, "gitlab": _setup_gitlab, "sentry": _setup_sentry, diff --git a/integrations/effective_models.py b/integrations/effective_models.py index b64a839bd4..39e7dec1b5 100644 --- a/integrations/effective_models.py +++ b/integrations/effective_models.py @@ -48,6 +48,7 @@ class EffectiveIntegrations(StrictConfigModel): google_docs: EffectiveIntegrationEntry | None = None gitlab: EffectiveIntegrationEntry | None = None vercel: EffectiveIntegrationEntry | None = None + neon: EffectiveIntegrationEntry | None = None railway: EffectiveIntegrationEntry | None = None jira: EffectiveIntegrationEntry | None = None servicenow: EffectiveIntegrationEntry | None = None diff --git a/integrations/neon/__init__.py b/integrations/neon/__init__.py new file mode 100644 index 0000000000..d6e1befa54 --- /dev/null +++ b/integrations/neon/__init__.py @@ -0,0 +1,17 @@ +"""Neon integration: project, compute, operations, and Object Storage diagnostics.""" + +from __future__ import annotations + +from integrations.neon.config import ( + NeonConfig, + build_neon_config, + classify, + neon_config_from_env, +) + +__all__ = [ + "NeonConfig", + "build_neon_config", + "classify", + "neon_config_from_env", +] diff --git a/integrations/neon/client.py b/integrations/neon/client.py new file mode 100644 index 0000000000..4ce5e6ce19 --- /dev/null +++ b/integrations/neon/client.py @@ -0,0 +1,214 @@ +"""Read-only client for the Neon control-plane API. + +Covers the endpoints an incident investigation reads: the project, its +branches, compute endpoints, recent operations, and the Object Storage state, +buckets, and object listings of one branch. Every method is a GET; nothing here +changes a project. +""" + +from __future__ import annotations + +import logging +import re +from http import HTTPStatus +from typing import Any + +import httpx + +from infrastructure.delivery.notifications.redaction import redact_token +from integrations.neon.config import NeonConfig +from integrations.probes import ProbeResult + +logger = logging.getLogger(__name__) + +_MAX_PATH_SEGMENT_LEN = 128 +# Neon project/branch IDs and bucket names are lowercase slugs: letters, +# digits, hyphens (bucket names may also hold dots). Reject anything else so a +# model-supplied value can never add path segments or traversal. +_PATH_SEGMENT_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +_ERROR_DETAIL_LIMIT = 300 + + +class NeonAPIError(RuntimeError): + """A Neon API call that did not return a usable 2xx response.""" + + def __init__( + self, message: str, *, status_code: int | None = None, invalid_input: bool = False + ) -> None: + super().__init__(message) + self.status_code = status_code + self.invalid_input = invalid_input + + +def safe_path_segment(raw: str, *, label: str) -> str: + """Return ``raw`` stripped, or raise when it is not a plain Neon identifier.""" + cleaned = (raw or "").strip() + if ( + not cleaned + or len(cleaned) > _MAX_PATH_SEGMENT_LEN + or ".." in cleaned + or not _PATH_SEGMENT_RE.fullmatch(cleaned) + ): + raise NeonAPIError( + f"Invalid Neon {label}: {cleaned[:_MAX_PATH_SEGMENT_LEN]!r}.", invalid_input=True + ) + return cleaned + + +def _status_message(status_code: int, body: Any) -> str: + """A short, secret-free reason for a non-2xx Neon response.""" + detail = "" + if isinstance(body, dict): + detail = str(body.get("message") or body.get("error") or "").strip() + if status_code == HTTPStatus.UNAUTHORIZED: + return "Neon rejected the API key (HTTP 401)." + if status_code == HTTPStatus.FORBIDDEN: + reason = f": {detail}" if detail else "." + return f"The Neon API key cannot access this resource (HTTP 403){reason}" + if status_code == HTTPStatus.NOT_FOUND: + reason = f": {detail}" if detail else "." + return f"Neon resource not found (HTTP 404){reason}" + if status_code == HTTPStatus.TOO_MANY_REQUESTS: + return "Neon API rate limit reached (HTTP 429). Retry shortly." + if status_code >= HTTPStatus.INTERNAL_SERVER_ERROR: + return f"Neon API is unavailable (HTTP {status_code}). Retry shortly." + reason = f": {detail}" if detail else "." + return f"Neon API returned HTTP {status_code}{reason}" + + +class NeonClient: + """Synchronous client for the Neon API, scoped to one configured project.""" + + def __init__(self, config: NeonConfig, *, transport: httpx.BaseTransport | None = None) -> None: + self.config = config + self._transport = transport + self._client: httpx.Client | None = None + + def _http(self) -> httpx.Client: + if self._client is None: + self._client = httpx.Client( + base_url=self.config.base_url, + headers=self.config.headers, + timeout=self.config.timeout_seconds, + transport=self._transport, + ) + return self._client + + def close(self) -> None: + """Close the underlying HTTP connection pool.""" + if self._client is not None: + self._client.close() + self._client = None + + def __enter__(self) -> NeonClient: + return self + + def __exit__(self, *_: object) -> None: + self.close() + + @property + def project_id(self) -> str: + return safe_path_segment(self.config.project_id, label="project ID") + + def _get(self, path: str, params: dict[str, Any] | None = None) -> dict[str, Any]: + query = {key: value for key, value in (params or {}).items() if value not in ("", None)} + try: + response = self._http().get(path, params=query) + except httpx.TimeoutException as exc: + raise NeonAPIError("Neon API request timed out.") from exc + except httpx.HTTPError as exc: + reason = redact_token(str(exc), self.config.api_key)[:_ERROR_DETAIL_LIMIT] + raise NeonAPIError(f"Neon API request failed: {reason}") from exc + try: + body: Any = response.json() + except ValueError: + body = None + if not response.is_success: + raise NeonAPIError( + redact_token(_status_message(response.status_code, body), self.config.api_key), + status_code=response.status_code, + ) + if not isinstance(body, dict): + raise NeonAPIError("Neon API returned a response that is not a JSON object.") + return body + + def get_project(self) -> dict[str, Any]: + """``GET /projects/{project_id}`` — the project record.""" + project = self._get(f"/projects/{self.project_id}").get("project") + return project if isinstance(project, dict) else {} + + def list_branches(self) -> list[dict[str, Any]]: + """``GET /projects/{project_id}/branches``.""" + branches = self._get(f"/projects/{self.project_id}/branches").get("branches") + return [item for item in branches or [] if isinstance(item, dict)] + + def list_endpoints(self) -> list[dict[str, Any]]: + """``GET /projects/{project_id}/endpoints`` — every compute in the project.""" + endpoints = self._get(f"/projects/{self.project_id}/endpoints").get("endpoints") + return [item for item in endpoints or [] if isinstance(item, dict)] + + def list_operations(self, limit: int) -> list[dict[str, Any]]: + """``GET /projects/{project_id}/operations`` — newest first.""" + operations = self._get( + f"/projects/{self.project_id}/operations", {"limit": max(1, min(limit, 1000))} + ).get("operations") + return [item for item in operations or [] if isinstance(item, dict)] + + def resolve_branch_id(self, branch_id: str = "") -> str: + """The requested branch, else the configured one, else the project default.""" + requested = (branch_id or "").strip() or self.config.branch_id + if requested: + return safe_path_segment(requested, label="branch ID") + for branch in self.list_branches(): + if branch.get("default"): + return safe_path_segment(str(branch.get("id", "")), label="branch ID") + raise NeonAPIError("The Neon project has no default branch; pass a branch ID.") + + def get_branch_storage(self, branch_id: str) -> dict[str, Any]: + """``GET /projects/{project_id}/branches/{branch_id}/storage``.""" + branch = safe_path_segment(branch_id, label="branch ID") + return self._get(f"/projects/{self.project_id}/branches/{branch}/storage") + + def list_buckets(self, branch_id: str) -> list[dict[str, Any]]: + """``GET /projects/{project_id}/branches/{branch_id}/buckets``.""" + branch = safe_path_segment(branch_id, label="branch ID") + buckets = self._get(f"/projects/{self.project_id}/branches/{branch}/buckets").get("buckets") + return [item for item in buckets or [] if isinstance(item, dict)] + + def list_objects( + self, + branch_id: str, + bucket: str, + *, + prefix: str = "", + delimiter: str = "", + limit: int = 50, + cursor: str = "", + ) -> dict[str, Any]: + """``GET .../buckets/{bucket}/objects`` — one page of objects and folders.""" + branch = safe_path_segment(branch_id, label="branch ID") + bucket_name = safe_path_segment(bucket, label="bucket name") + return self._get( + f"/projects/{self.project_id}/branches/{branch}/buckets/{bucket_name}/objects", + {"prefix": prefix, "delimiter": delimiter, "limit": limit, "cursor": cursor}, + ) + + def probe_access(self) -> ProbeResult: + """Validate the key and project ID by reading the project.""" + if not self.config.api_key: + return ProbeResult.missing("Missing Neon API key.") + if not self.config.project_id: + return ProbeResult.missing("Missing Neon project ID.") + try: + with self: + project = self.get_project() + except NeonAPIError as exc: + return ProbeResult.failed(str(exc)) + name = str(project.get("name") or self.config.project_id) + region = str(project.get("region_id") or "unknown region") + return ProbeResult.passed( + f"Connected to Neon project {name} ({self.config.project_id}, {region})." + ) + + +__all__ = ["NeonAPIError", "NeonClient", "safe_path_segment"] diff --git a/integrations/neon/config.py b/integrations/neon/config.py new file mode 100644 index 0000000000..1ad5c654c0 --- /dev/null +++ b/integrations/neon/config.py @@ -0,0 +1,116 @@ +"""Neon integration config, env loading, and store classification. + +One record scopes OpenSRE to a single Neon project. ``branch_id`` is optional: +when it is empty, the tools resolve the project's default branch at call time, +so a promoted branch is followed without editing the integration. +""" + +from __future__ import annotations + +import logging +import os +from typing import Any + +from pydantic import Field, field_validator + +from config.constants.neon import ( + NEON_API_BASE_URL, + NEON_API_KEY_ENV, + NEON_BRANCH_ID_ENV, + NEON_PROJECT_ID_ENV, +) +from config.llm_credentials import resolve_env_credential +from config.strict_config import StrictConfigModel +from integrations._validation_helpers import report_classify_failure + +logger = logging.getLogger(__name__) + +DEFAULT_NEON_TIMEOUT_SECONDS = 15.0 +DEFAULT_NEON_MAX_RESULTS = 50 + + +class NeonConfig(StrictConfigModel): + """Normalized Neon API settings for one project.""" + + api_key: str = "" + project_id: str = "" + branch_id: str = "" + base_url: str = NEON_API_BASE_URL + timeout_seconds: float = Field(default=DEFAULT_NEON_TIMEOUT_SECONDS, gt=0) + max_results: int = Field(default=DEFAULT_NEON_MAX_RESULTS, gt=0, le=200) + integration_id: str = "" + + @field_validator("api_key", "project_id", "branch_id", "integration_id", mode="before") + @classmethod + def _strip(cls, value: Any) -> str: + return str(value or "").strip() + + @field_validator("base_url", mode="before") + @classmethod + def _normalize_base_url(cls, value: Any) -> str: + normalized = str(value or "").strip().rstrip("/") + return normalized or NEON_API_BASE_URL + + @property + def is_configured(self) -> bool: + return bool(self.api_key and self.project_id) + + @property + def headers(self) -> dict[str, str]: + return { + "Authorization": f"Bearer {self.api_key}", + "Accept": "application/json", + } + + +def build_neon_config(raw: dict[str, Any] | None) -> NeonConfig: + """Build a normalized Neon config from env/store data.""" + return NeonConfig.model_validate(raw or {}) + + +def neon_config_from_env() -> NeonConfig | None: + """Load a Neon config from env vars, or ``None`` when the pair is incomplete. + + The API key goes through ``resolve_env_credential`` (env, then the local + credentials file); the project and branch IDs are plain env reads. + """ + api_key = resolve_env_credential(NEON_API_KEY_ENV) + project_id = os.getenv(NEON_PROJECT_ID_ENV, "").strip() + if not api_key or not project_id: + return None + return build_neon_config( + { + "api_key": api_key, + "project_id": project_id, + "branch_id": os.getenv(NEON_BRANCH_ID_ENV, ""), + } + ) + + +def classify(credentials: dict[str, Any], record_id: str) -> tuple[NeonConfig | None, str | None]: + """Turn a stored or env record into a resolved ``neon`` source.""" + try: + config = build_neon_config( + { + "api_key": credentials.get("api_key", ""), + "project_id": credentials.get("project_id", ""), + "branch_id": credentials.get("branch_id", ""), + "integration_id": record_id, + } + ) + except Exception as exc: + report_classify_failure(exc, logger=logger, integration="neon", record_id=record_id) + return None, None + if config.is_configured: + return config, "neon" + return None, None + + +__all__ = [ + "DEFAULT_NEON_MAX_RESULTS", + "DEFAULT_NEON_TIMEOUT_SECONDS", + "NeonConfig", + "build_neon_config", + "classify", + "neon_config_from_env", +] diff --git a/integrations/neon/diagnostics.py b/integrations/neon/diagnostics.py new file mode 100644 index 0000000000..917667b5cb --- /dev/null +++ b/integrations/neon/diagnostics.py @@ -0,0 +1,271 @@ +"""Turn Neon API payloads into compact, agent-facing incident summaries. + +Each function takes a :class:`NeonClient`, never raises for an expected API +failure, and returns either a result dict or the shared unavailable envelope +with the right failure state (missing config, denied, no match, transport). +""" + +from __future__ import annotations + +from http import HTTPStatus +from typing import Any + +from core.tool_framework.utils import ( + tool_access_denied, + tool_missing_config, + tool_no_match, + tool_transport_failure, +) +from integrations.neon.client import NeonAPIError, NeonClient + +SOURCE = "neon" +DEFAULT_OPERATIONS_LIMIT = 20 +MAX_OPERATIONS_LIMIT = 100 +# Neon operation statuses that mean the control plane gave up on a change. +FAILED_OPERATION_STATUSES = frozenset({"failed", "error"}) +# Statuses that mean work is still queued or running against the project. +PENDING_OPERATION_STATUSES = frozenset({"running", "scheduling", "cancelling"}) + + +def _failure(exc: NeonAPIError) -> dict[str, Any]: + status = exc.status_code + if status in (HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN): + return tool_access_denied(SOURCE, str(exc)) + if exc.invalid_input or status == HTTPStatus.NOT_FOUND: + return tool_no_match(SOURCE, str(exc)) + return tool_transport_failure(SOURCE, str(exc)) + + +def _not_configured(client: NeonClient) -> dict[str, Any] | None: + if not client.config.is_configured: + return tool_missing_config( + SOURCE, "Neon is not configured. Set NEON_API_KEY and NEON_PROJECT_ID." + ) + return None + + +def _clamp(value: int, *, low: int, high: int) -> int: + return max(low, min(int(value), high)) + + +def _branch_summary(branch: dict[str, Any]) -> dict[str, Any]: + return { + "id": branch.get("id", ""), + "name": branch.get("name", ""), + "default": bool(branch.get("default", False)), + "protected": bool(branch.get("protected", False)), + "current_state": branch.get("current_state", ""), + "pending_state": branch.get("pending_state"), + "parent_id": branch.get("parent_id"), + "logical_size_bytes": branch.get("logical_size"), + "expires_at": branch.get("expires_at"), + "updated_at": branch.get("updated_at", ""), + } + + +def _endpoint_summary(endpoint: dict[str, Any]) -> dict[str, Any]: + return { + "id": endpoint.get("id", ""), + "branch_id": endpoint.get("branch_id", ""), + "type": endpoint.get("type", ""), + "host": endpoint.get("host", ""), + "current_state": endpoint.get("current_state", ""), + "pending_state": endpoint.get("pending_state"), + "disabled": bool(endpoint.get("disabled", False)), + "autoscaling_limit_min_cu": endpoint.get("autoscaling_limit_min_cu"), + "autoscaling_limit_max_cu": endpoint.get("autoscaling_limit_max_cu"), + "suspend_timeout_seconds": endpoint.get("suspend_timeout_seconds"), + "pooler_enabled": bool(endpoint.get("pooler_enabled", False)), + "last_active": endpoint.get("last_active"), + "started_at": endpoint.get("started_at"), + "suspended_at": endpoint.get("suspended_at"), + } + + +def _operation_summary(operation: dict[str, Any]) -> dict[str, Any]: + summary = { + "id": operation.get("id", ""), + "action": operation.get("action", ""), + "status": operation.get("status", ""), + "branch_id": operation.get("branch_id"), + "endpoint_id": operation.get("endpoint_id"), + "failures_count": operation.get("failures_count", 0), + "created_at": operation.get("created_at", ""), + "updated_at": operation.get("updated_at", ""), + "total_duration_ms": operation.get("total_duration_ms"), + } + if operation.get("error"): + summary["error"] = str(operation["error"]) + return summary + + +def get_project_status( + client: NeonClient, + *, + branch_id: str = "", + operations_limit: int = DEFAULT_OPERATIONS_LIMIT, +) -> dict[str, Any]: + """Project settings, branches, computes, and recent control-plane operations. + + ``branch_id`` narrows computes and operations to one branch; the branch + list always covers the whole project so a missing or renamed branch shows. + """ + if (missing := _not_configured(client)) is not None: + return missing + limit = _clamp(operations_limit, low=1, high=MAX_OPERATIONS_LIMIT) + branch_filter = (branch_id or "").strip() + max_results = client.config.max_results + try: + with client: + project = client.get_project() + branches = client.list_branches() + endpoints = client.list_endpoints() + operations = client.list_operations(limit) + except NeonAPIError as exc: + return _failure(exc) + + if branch_filter: + endpoints = [item for item in endpoints if item.get("branch_id") == branch_filter] + operations = [item for item in operations if item.get("branch_id") == branch_filter] + + endpoint_rows = [_endpoint_summary(item) for item in endpoints] + operation_rows = [_operation_summary(item) for item in operations] + raw_settings = project.get("settings") + settings: dict[str, Any] = raw_settings if isinstance(raw_settings, dict) else {} + raw_allowed_ips = settings.get("allowed_ips") + allowed_ips = raw_allowed_ips.get("ips", []) if isinstance(raw_allowed_ips, dict) else [] + return { + "source": SOURCE, + "available": True, + "project": { + "id": project.get("id", client.config.project_id), + "name": project.get("name", ""), + "region_id": project.get("region_id", ""), + "pg_version": project.get("pg_version"), + "created_at": project.get("created_at", ""), + "compute_last_active_at": project.get("compute_last_active_at"), + "history_retention_seconds": project.get("history_retention_seconds"), + "allowed_ips": allowed_ips, + "block_public_connections": settings.get("block_public_connections"), + "maintenance_window": settings.get("maintenance_window"), + }, + "branch_filter": branch_filter or None, + "total_branches": len(branches), + "branches": [_branch_summary(item) for item in branches[:max_results]], + "branches_truncated": len(branches) > max_results, + "total_endpoints": len(endpoint_rows), + "endpoints": endpoint_rows[:max_results], + "endpoints_truncated": len(endpoint_rows) > max_results, + "active_endpoints": [ + row["id"] for row in endpoint_rows if row["current_state"] == "active" + ], + "disabled_endpoints": [row["id"] for row in endpoint_rows if row["disabled"]], + "operations": operation_rows, + "failed_operations": [ + row for row in operation_rows if row["status"] in FAILED_OPERATION_STATUSES + ], + "pending_operations": [ + row for row in operation_rows if row["status"] in PENDING_OPERATION_STATUSES + ], + } + + +def get_storage_buckets(client: NeonClient, *, branch_id: str = "") -> dict[str, Any]: + """Object Storage state and buckets for one branch (default: the configured one).""" + if (missing := _not_configured(client)) is not None: + return missing + max_results = client.config.max_results + try: + with client: + branch = client.resolve_branch_id(branch_id) + storage = client.get_branch_storage(branch) + buckets = client.list_buckets(branch) if storage.get("enabled", True) else [] + except NeonAPIError as exc: + return _failure(exc) + rows = [ + { + "name": item.get("name", ""), + "access_level": item.get("access_level", ""), + "created_at": item.get("created_at", ""), + } + for item in buckets + ] + return { + "source": SOURCE, + "available": True, + "project_id": client.config.project_id, + "branch_id": branch, + "storage_enabled": bool(storage.get("enabled", False)), + "s3_endpoint": storage.get("s3_endpoint", ""), + "region": storage.get("region", ""), + "force_path_style": storage.get("force_path_style"), + "total_buckets": len(rows), + "buckets": rows[:max_results], + "truncated": len(rows) > max_results, + "public_buckets": [row["name"] for row in rows if row["access_level"] != "private"], + } + + +def list_storage_objects( + client: NeonClient, + *, + bucket: str, + branch_id: str = "", + prefix: str = "", + recursive: bool = False, + limit: int = 0, + cursor: str = "", +) -> dict[str, Any]: + """One page of objects (and collapsed folders unless ``recursive``) in a bucket.""" + if (missing := _not_configured(client)) is not None: + return missing + if not (bucket or "").strip(): + return tool_missing_config(SOURCE, "A bucket name is required.") + page_size = _clamp(limit or client.config.max_results, low=1, high=200) + try: + with client: + branch = client.resolve_branch_id(branch_id) + page = client.list_objects( + branch, + bucket, + prefix=prefix, + delimiter="" if recursive else "/", + limit=page_size, + cursor=cursor, + ) + except NeonAPIError as exc: + return _failure(exc) + objects = [ + { + "key": item.get("key", ""), + "size": item.get("size"), + "last_modified": item.get("last_modified", ""), + "etag": item.get("etag", ""), + } + for item in page.get("objects") or [] + if isinstance(item, dict) + ] + folders = [str(item) for item in page.get("folders") or []] + return { + "source": SOURCE, + "available": True, + "branch_id": branch, + "bucket": bucket.strip(), + "prefix": page.get("prefix", prefix), + "folders": folders, + "objects": objects, + "returned": len(objects) + len(folders), + "truncated": bool(page.get("is_truncated", False)), + "next_cursor": page.get("next_cursor") or None, + } + + +__all__ = [ + "DEFAULT_OPERATIONS_LIMIT", + "FAILED_OPERATION_STATUSES", + "MAX_OPERATIONS_LIMIT", + "PENDING_OPERATION_STATUSES", + "get_project_status", + "get_storage_buckets", + "list_storage_objects", +] diff --git a/integrations/neon/setup.py b/integrations/neon/setup.py new file mode 100644 index 0000000000..34f0dbb12c --- /dev/null +++ b/integrations/neon/setup.py @@ -0,0 +1,49 @@ +"""What Neon needs before it is considered configured. + +An API key and a project ID are required. ``branch_id`` is optional: left +blank, the tools follow the project's default branch. +""" + +from __future__ import annotations + +from config.constants.neon import NEON_API_KEY_ENV, NEON_BRANCH_ID_ENV, NEON_PROJECT_ID_ENV +from integrations.neon.verifier import verify_neon +from integrations.setup_flow import IntegrationSetupSpec, SetupField + +API_KEY_FIELD = "api_key" +PROJECT_ID_FIELD = "project_id" +BRANCH_ID_FIELD = "branch_id" + +NEON_SETUP = IntegrationSetupSpec( + service="neon", + fields=( + SetupField( + name=API_KEY_FIELD, + label="Neon API key", + prompt="Neon API key (personal, organization, or project-scoped)", + env_var=NEON_API_KEY_ENV, + secret=True, + ), + SetupField( + name=PROJECT_ID_FIELD, + label="Neon project ID", + prompt="Neon project ID (e.g. little-lab-10567887)", + env_var=NEON_PROJECT_ID_ENV, + ), + SetupField( + name=BRANCH_ID_FIELD, + label="Neon branch ID", + prompt="Branch ID for storage tools (optional, defaults to the project's default branch)", + env_var=NEON_BRANCH_ID_ENV, + required=False, + ), + ), + verify=verify_neon, +) + +__all__ = [ + "API_KEY_FIELD", + "BRANCH_ID_FIELD", + "NEON_SETUP", + "PROJECT_ID_FIELD", +] diff --git a/integrations/neon/tools/__init__.py b/integrations/neon/tools/__init__.py new file mode 100644 index 0000000000..1faa0e1a16 --- /dev/null +++ b/integrations/neon/tools/__init__.py @@ -0,0 +1 @@ +"""Agent tools for the Neon integration (all read-only).""" diff --git a/integrations/neon/tools/_params.py b/integrations/neon/tools/_params.py new file mode 100644 index 0000000000..f83db1534e --- /dev/null +++ b/integrations/neon/tools/_params.py @@ -0,0 +1,65 @@ +"""Credential and scope plumbing shared by the Neon tools. + +The API key, project ID, and API base URL are injected from the resolved +``neon`` integration and hidden from the model, so a tool call can never point +the key at another project or host. ``branch_id`` is a default the model may +override to look at another branch of the same project. +""" + +from __future__ import annotations + +from typing import Any + +from integrations.neon.client import NeonClient +from integrations.neon.config import build_neon_config + +INJECTED_PARAMS = ("api_key", "project_id", "base_url") + +INJECTED_SCHEMA_PROPERTIES: dict[str, dict[str, Any]] = { + "api_key": {"type": "string", "description": "Neon API key (injected)."}, + "project_id": {"type": "string", "description": "Neon project ID (injected)."}, + "base_url": {"type": "string", "description": "Neon API base URL (injected)."}, +} + +BRANCH_ID_PROPERTY: dict[str, Any] = { + "type": "string", + "default": "", + "description": ( + "Neon branch ID (br-...). Defaults to the configured branch, else the " + "project's default branch." + ), +} + + +def neon_is_available(sources: dict[str, Any]) -> bool: + """True when a resolved Neon integration carries a key and a project.""" + neon = sources.get("neon") or {} + return bool(neon.get("api_key") and neon.get("project_id")) + + +def neon_extract_params(sources: dict[str, Any]) -> dict[str, Any]: + """Injected credentials plus the configured default branch.""" + neon = sources.get("neon") or {} + return { + "api_key": str(neon.get("api_key", "")), + "project_id": str(neon.get("project_id", "")), + "base_url": str(neon.get("base_url", "")), + "branch_id": str(neon.get("branch_id", "")), + } + + +def neon_client(api_key: str, project_id: str, base_url: str = "") -> NeonClient: + """A client for the injected project.""" + return NeonClient( + build_neon_config({"api_key": api_key, "project_id": project_id, "base_url": base_url}) + ) + + +__all__ = [ + "BRANCH_ID_PROPERTY", + "INJECTED_PARAMS", + "INJECTED_SCHEMA_PROPERTIES", + "neon_client", + "neon_extract_params", + "neon_is_available", +] diff --git a/integrations/neon/tools/neon_project_status_tool/__init__.py b/integrations/neon/tools/neon_project_status_tool/__init__.py new file mode 100644 index 0000000000..4610f065a1 --- /dev/null +++ b/integrations/neon/tools/neon_project_status_tool/__init__.py @@ -0,0 +1,12 @@ +"""Registry entrypoint for the Neon project status tool.""" + +from __future__ import annotations + +from integrations.neon.tools.neon_project_status_tool.tool import ( + NeonProjectStatusTool, + get_neon_project_status, +) + +TOOL_MODULES = ("tool",) + +__all__ = ["TOOL_MODULES", "NeonProjectStatusTool", "get_neon_project_status"] diff --git a/integrations/neon/tools/neon_project_status_tool/tool.py b/integrations/neon/tools/neon_project_status_tool/tool.py new file mode 100644 index 0000000000..9da84a784f --- /dev/null +++ b/integrations/neon/tools/neon_project_status_tool/tool.py @@ -0,0 +1,137 @@ +"""Neon project status: branches, computes, and recent control-plane operations.""" + +from __future__ import annotations + +from typing import Any + +from core.domain.types.evidence import record_evidence_entry +from core.domain.types.tools import ToolSurface +from core.tool import BaseTool, SideEffectLevel +from integrations.neon.diagnostics import ( + DEFAULT_OPERATIONS_LIMIT, + MAX_OPERATIONS_LIMIT, + get_project_status, +) +from integrations.neon.tools._params import ( + BRANCH_ID_PROPERTY, + INJECTED_PARAMS, + INJECTED_SCHEMA_PROPERTIES, + neon_client, + neon_extract_params, + neon_is_available, +) + + +def map_neon_project_status( + evidence: dict[str, Any], output: dict[str, Any], _tool_input: dict[str, Any] +) -> None: + """Cite compute states and any failed or pending operations.""" + if not output.get("available"): + return + project = output.get("project") or {} + # Both counts cover every compute: ``endpoints`` is capped at max_results. + total = output.get("total_endpoints", len(output.get("endpoints") or [])) + active = output.get("active_endpoints") or [] + parts = [ + f"{len(active)}/{total} computes active", + f"{output.get('total_branches', 0)} branches", + ] + failed = output.get("failed_operations") or [] + if failed: + actions = ", ".join(sorted({str(item.get("action", "")) for item in failed})) + parts.append(f"{len(failed)} failed operations ({actions})") + pending = output.get("pending_operations") or [] + if pending: + parts.append(f"{len(pending)} operations in progress") + disabled = output.get("disabled_endpoints") or [] + if disabled: + parts.append(f"disabled computes: {', '.join(disabled)}") + record_evidence_entry( + evidence, + source="get_neon_project_status", + label=f"Neon project {project.get('name') or project.get('id', '')}".strip(), + summary=", ".join(parts), + ) + + +class NeonProjectStatusTool(BaseTool): + """Summarize a Neon project's branches, computes, and recent operations.""" + + name = "get_neon_project_status" + source = "neon" + surfaces = (ToolSurface.CHAT, ToolSurface.ACTION) + side_effect_level = SideEffectLevel.READ_ONLY + concurrency_safe = True + evidence_mapper = map_neon_project_status + description = ( + "Read the configured Neon Postgres project: region and Postgres version, branches " + "and their state, compute endpoints (active/idle, autoscaling CU limits, " + "scale-to-zero timeout, last active time), and recent control-plane operations " + "with failed and in-progress ones listed separately." + ) + use_cases = [ + "Checking whether a Neon compute is suspended, disabled, or stuck starting during a connection incident", + "Finding failed or long-running Neon operations (compute restarts, branch resets) near an incident", + "Confirming autoscaling limits and scale-to-zero settings when latency spikes on cold starts", + "Listing branches to spot a reset, deleted, or unexpectedly restored branch", + ] + requires = ["api_key", "project_id"] + injected_params = INJECTED_PARAMS + input_schema = { + "type": "object", + "properties": { + **INJECTED_SCHEMA_PROPERTIES, + "branch_id": { + **BRANCH_ID_PROPERTY, + "description": ( + "Optional Neon branch ID to narrow computes and operations to one branch. " + "Leave empty for the whole project." + ), + }, + "operations_limit": { + "type": "integer", + "default": DEFAULT_OPERATIONS_LIMIT, + "minimum": 1, + "maximum": MAX_OPERATIONS_LIMIT, + "description": "How many recent operations to read (newest first).", + }, + }, + "required": ["api_key", "project_id"], + "additionalProperties": False, + } + outputs = { + "project": "Project region, Postgres version, IP allowlist, and maintenance window", + "branches": "Branches with state, default/protected flags, and parent", + "total_endpoints": "Number of compute endpoints before the list is capped", + "endpoints": "Compute endpoints with state, CU limits, and suspend timeout", + "failed_operations": "Operations whose status is failed or error", + "pending_operations": "Operations still running, scheduling, or cancelling", + } + + def is_available(self, sources: dict[str, Any]) -> bool: + return neon_is_available(sources) + + def extract_params(self, sources: dict[str, Any]) -> dict[str, Any]: + params = neon_extract_params(sources) + # Status covers the whole project unless the model narrows it. + params["branch_id"] = "" + params["operations_limit"] = DEFAULT_OPERATIONS_LIMIT + return params + + def run( + self, + api_key: str, + project_id: str, + base_url: str = "", + branch_id: str = "", + operations_limit: int = DEFAULT_OPERATIONS_LIMIT, + **_kwargs: Any, + ) -> dict[str, Any]: + return get_project_status( + neon_client(api_key, project_id, base_url), + branch_id=branch_id, + operations_limit=operations_limit, + ) + + +get_neon_project_status = NeonProjectStatusTool() diff --git a/integrations/neon/tools/neon_storage_buckets_tool/__init__.py b/integrations/neon/tools/neon_storage_buckets_tool/__init__.py new file mode 100644 index 0000000000..3db2c0cb8c --- /dev/null +++ b/integrations/neon/tools/neon_storage_buckets_tool/__init__.py @@ -0,0 +1,12 @@ +"""Registry entrypoint for the Neon Object Storage buckets tool.""" + +from __future__ import annotations + +from integrations.neon.tools.neon_storage_buckets_tool.tool import ( + NeonStorageBucketsTool, + get_neon_storage_buckets, +) + +TOOL_MODULES = ("tool",) + +__all__ = ["TOOL_MODULES", "NeonStorageBucketsTool", "get_neon_storage_buckets"] diff --git a/integrations/neon/tools/neon_storage_buckets_tool/tool.py b/integrations/neon/tools/neon_storage_buckets_tool/tool.py new file mode 100644 index 0000000000..f093e82eee --- /dev/null +++ b/integrations/neon/tools/neon_storage_buckets_tool/tool.py @@ -0,0 +1,97 @@ +"""Neon Object Storage buckets for one branch.""" + +from __future__ import annotations + +from typing import Any + +from core.domain.types.evidence import record_evidence_entry +from core.domain.types.tools import ToolSurface +from core.tool import BaseTool, SideEffectLevel +from integrations.neon.diagnostics import get_storage_buckets +from integrations.neon.tools._params import ( + BRANCH_ID_PROPERTY, + INJECTED_PARAMS, + INJECTED_SCHEMA_PROPERTIES, + neon_client, + neon_extract_params, + neon_is_available, +) + + +def map_neon_storage_buckets( + evidence: dict[str, Any], output: dict[str, Any], _tool_input: dict[str, Any] +) -> None: + """Cite whether storage is on, the bucket count, and any public buckets.""" + if not output.get("available"): + return + if not output.get("storage_enabled"): + summary = "Object Storage is not enabled on this branch" + else: + summary = f"{output.get('total_buckets', 0)} buckets" + public = output.get("public_buckets") or [] + if public: + summary += f", public: {', '.join(public)}" + record_evidence_entry( + evidence, + source="get_neon_storage_buckets", + label=f"Neon Object Storage ({output.get('branch_id', '')})", + summary=summary, + url=output.get("s3_endpoint") or None, + ) + + +class NeonStorageBucketsTool(BaseTool): + """List Neon Object Storage buckets and the branch's S3 endpoint.""" + + name = "get_neon_storage_buckets" + source = "neon" + surfaces = (ToolSurface.CHAT, ToolSurface.ACTION) + side_effect_level = SideEffectLevel.READ_ONLY + concurrency_safe = True + evidence_mapper = map_neon_storage_buckets + description = ( + "List the Neon Object Storage buckets on a branch with their access level " + "(private or public_read), plus whether storage is enabled and the branch's " + "S3-compatible endpoint and region." + ) + use_cases = [ + "Checking whether a bucket an app uploads to exists on the branch it is deployed against", + "Auditing which Neon buckets are publicly readable after an access or data-exposure report", + "Getting the branch's S3 endpoint to compare with an app's AWS_ENDPOINT_URL_S3", + ] + requires = ["api_key", "project_id"] + injected_params = INJECTED_PARAMS + input_schema = { + "type": "object", + "properties": { + **INJECTED_SCHEMA_PROPERTIES, + "branch_id": BRANCH_ID_PROPERTY, + }, + "required": ["api_key", "project_id"], + "additionalProperties": False, + } + outputs = { + "storage_enabled": "Whether Object Storage is enabled on the branch", + "s3_endpoint": "The branch's S3-compatible endpoint (path-style)", + "buckets": "Buckets with access level and creation time", + "public_buckets": "Names of buckets that are not private", + } + + def is_available(self, sources: dict[str, Any]) -> bool: + return neon_is_available(sources) + + def extract_params(self, sources: dict[str, Any]) -> dict[str, Any]: + return neon_extract_params(sources) + + def run( + self, + api_key: str, + project_id: str, + base_url: str = "", + branch_id: str = "", + **_kwargs: Any, + ) -> dict[str, Any]: + return get_storage_buckets(neon_client(api_key, project_id, base_url), branch_id=branch_id) + + +get_neon_storage_buckets = NeonStorageBucketsTool() diff --git a/integrations/neon/tools/neon_storage_objects_tool/__init__.py b/integrations/neon/tools/neon_storage_objects_tool/__init__.py new file mode 100644 index 0000000000..c6b2d79695 --- /dev/null +++ b/integrations/neon/tools/neon_storage_objects_tool/__init__.py @@ -0,0 +1,12 @@ +"""Registry entrypoint for the Neon Object Storage object listing tool.""" + +from __future__ import annotations + +from integrations.neon.tools.neon_storage_objects_tool.tool import ( + NeonStorageObjectsTool, + list_neon_storage_objects, +) + +TOOL_MODULES = ("tool",) + +__all__ = ["TOOL_MODULES", "NeonStorageObjectsTool", "list_neon_storage_objects"] diff --git a/integrations/neon/tools/neon_storage_objects_tool/tool.py b/integrations/neon/tools/neon_storage_objects_tool/tool.py new file mode 100644 index 0000000000..e2e4b6ef21 --- /dev/null +++ b/integrations/neon/tools/neon_storage_objects_tool/tool.py @@ -0,0 +1,136 @@ +"""List objects in a Neon Object Storage bucket.""" + +from __future__ import annotations + +from typing import Any + +from core.domain.types.evidence import record_evidence_entry +from core.domain.types.tools import ToolSurface +from core.tool import BaseTool, SideEffectLevel +from integrations.neon.diagnostics import list_storage_objects +from integrations.neon.tools._params import ( + BRANCH_ID_PROPERTY, + INJECTED_PARAMS, + INJECTED_SCHEMA_PROPERTIES, + neon_client, + neon_extract_params, + neon_is_available, +) + + +def map_neon_storage_objects( + evidence: dict[str, Any], output: dict[str, Any], _tool_input: dict[str, Any] +) -> None: + """Cite the listed location and how much it holds.""" + if not output.get("available"): + return + objects = output.get("objects") or [] + folders = output.get("folders") or [] + summary = f"{len(objects)} objects" + if folders: + summary += f", {len(folders)} folders" + if output.get("truncated"): + summary += " (more available)" + newest = max((str(item.get("last_modified", "")) for item in objects), default="") + if newest: + summary += f", newest {newest}" + location = f"{output.get('bucket', '')}/{output.get('prefix', '')}" + record_evidence_entry( + evidence, + source="list_neon_storage_objects", + label=f"Neon bucket {location}", + summary=summary, + ) + + +class NeonStorageObjectsTool(BaseTool): + """List one page of objects in a Neon Object Storage bucket.""" + + name = "list_neon_storage_objects" + source = "neon" + surfaces = (ToolSurface.CHAT, ToolSurface.ACTION) + side_effect_level = SideEffectLevel.READ_ONLY + concurrency_safe = True + evidence_mapper = map_neon_storage_objects + description = ( + "List objects (key, size, last modified, ETag) in a Neon Object Storage bucket, " + "optionally under a key prefix. Folders are collapsed on '/' unless recursive is " + "true. Returns a next_cursor when more results exist." + ) + use_cases = [ + "Checking whether an expected upload, export, or backup object landed in a Neon bucket", + "Finding the newest objects under a prefix to correlate with an incident timeline", + "Browsing a bucket folder by folder to locate artifacts referenced in logs", + ] + requires = ["api_key", "project_id"] + injected_params = INJECTED_PARAMS + input_schema = { + "type": "object", + "properties": { + **INJECTED_SCHEMA_PROPERTIES, + "bucket": {"type": "string", "description": "Bucket name on the branch."}, + "branch_id": BRANCH_ID_PROPERTY, + "prefix": { + "type": "string", + "default": "", + "description": "Only list keys that start with this prefix (e.g. 'logs/2026-11-05/').", + }, + "recursive": { + "type": "boolean", + "default": False, + "description": "List every key under the prefix instead of collapsing folders.", + }, + "limit": { + "type": "integer", + "default": 50, + "minimum": 1, + "maximum": 200, + "description": "Maximum objects plus folders to return.", + }, + "cursor": { + "type": "string", + "default": "", + "description": "next_cursor from a previous call, to read the next page.", + }, + }, + "required": ["api_key", "project_id", "bucket"], + "additionalProperties": False, + } + outputs = { + "folders": "Collapsed key prefixes ending in '/' (empty when recursive)", + "objects": "Objects with key, size, last_modified, and etag", + "truncated": "True when another page exists", + "next_cursor": "Cursor for the next page, or null", + } + + def is_available(self, sources: dict[str, Any]) -> bool: + return neon_is_available(sources) + + def extract_params(self, sources: dict[str, Any]) -> dict[str, Any]: + return neon_extract_params(sources) + + def run( + self, + api_key: str, + project_id: str, + bucket: str = "", + base_url: str = "", + branch_id: str = "", + prefix: str = "", + recursive: bool = False, + limit: int = 50, + cursor: str = "", + **_kwargs: Any, + ) -> dict[str, Any]: + return list_storage_objects( + neon_client(api_key, project_id, base_url), + bucket=bucket, + branch_id=branch_id, + prefix=prefix, + recursive=recursive, + limit=limit, + cursor=cursor, + ) + + +list_neon_storage_objects = NeonStorageObjectsTool() diff --git a/integrations/neon/verifier.py b/integrations/neon/verifier.py new file mode 100644 index 0000000000..d14d202d9e --- /dev/null +++ b/integrations/neon/verifier.py @@ -0,0 +1,13 @@ +"""Neon integration verifier: reads the configured project with the API key.""" + +from __future__ import annotations + +from integrations.neon.client import NeonClient +from integrations.neon.config import build_neon_config +from integrations.verification import register_probe_verifier + +verify_neon = register_probe_verifier( + "neon", + config=build_neon_config, + client=NeonClient, +) diff --git a/integrations/registry.py b/integrations/registry.py index 0a2a66264e..a101543f6e 100644 --- a/integrations/registry.py +++ b/integrations/registry.py @@ -385,6 +385,13 @@ class IntegrationSpec: ), IntegrationSpec(service="trello"), IntegrationSpec(service="rds", setup_order=11), + IntegrationSpec( + service="neon", + has_verifier=True, + direct_effective=True, + setup_order=60, + verify_order=60, + ), IntegrationSpec( service="supabase", has_verifier=True, diff --git a/tests/filestorage/test_neon_provider.py b/tests/filestorage/test_neon_provider.py new file mode 100644 index 0000000000..2bec838328 --- /dev/null +++ b/tests/filestorage/test_neon_provider.py @@ -0,0 +1,120 @@ +"""Neon Object Storage remote-sync provider. + +Covers what differs from the generic S3-compatible store: the endpoint is +mandatory (no silent fallback to AWS), every endpoint source is honored in the +documented order, and exposure comes from ``GetBucketPolicyStatus``, which Neon +answers from the bucket's access level. No live calls. +""" + +from __future__ import annotations + +from typing import Any + +import pytest +from botocore.exceptions import ClientError + +from config.constants.filestorage import REMOTE_SYNC_ENDPOINT_URL_ENV +from infrastructure.filestorage.config import RemoteSyncConfig +from infrastructure.filestorage.enums import BucketExposure, BuiltInProvider +from infrastructure.filestorage.errors import RemoteSyncConfigError +from infrastructure.filestorage.providers.neon import ( + MISSING_ENDPOINT_MESSAGE, + NeonObjectStore, + check_public_access, +) +from infrastructure.filestorage.providers.registry import ( + build_object_store, + builtin_providers, + credential_hint_for_provider, + max_parallel_uploads_for_provider, +) + +NEON_ENDPOINT = "https://br-summer-art-bdbz1xg2.storage.c-8.us-east-2.aws.neon.tech" +_ENDPOINT_ENVS = (REMOTE_SYNC_ENDPOINT_URL_ENV, "AWS_ENDPOINT_URL_S3", "AWS_ENDPOINT_URL") + + +@pytest.fixture(autouse=True) +def _no_ambient_endpoint(monkeypatch: pytest.MonkeyPatch, tmp_path: Any) -> None: + for name in _ENDPOINT_ENVS: + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("OPENSRE_HOME", str(tmp_path)) + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "nak_live_test") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret") + + +def _config(bucket: str = "incident-artifacts") -> RemoteSyncConfig: + return RemoteSyncConfig(bucket=bucket, provider=BuiltInProvider.NEON, prefix="opensre") + + +class _PolicyClient: + def __init__(self, *, is_public: bool | None = None, error_code: str = "") -> None: + self.is_public = is_public + self.error_code = error_code + + def get_bucket_policy_status(self, Bucket: str) -> dict[str, Any]: # noqa: ARG002, N803 + if self.error_code: + raise ClientError({"Error": {"Code": self.error_code}}, "GetBucketPolicyStatus") + return {"PolicyStatus": {"IsPublic": self.is_public}} + + +def test_neon_is_a_builtin_provider_with_its_own_hint() -> None: + assert "neon" in builtin_providers() + assert "neon env pull" in credential_hint_for_provider("neon") + assert max_parallel_uploads_for_provider("neon") == 16 + + +def test_missing_endpoint_fails_closed_instead_of_falling_back_to_aws() -> None: + with pytest.raises(RemoteSyncConfigError) as err: + build_object_store(_config()) + assert str(err.value) == MISSING_ENDPOINT_MESSAGE + + +@pytest.mark.parametrize( + ("env", "expected"), + [ + ({"AWS_ENDPOINT_URL_S3": NEON_ENDPOINT}, NEON_ENDPOINT), + ({"AWS_ENDPOINT_URL": NEON_ENDPOINT}, NEON_ENDPOINT), + ( + {REMOTE_SYNC_ENDPOINT_URL_ENV: NEON_ENDPOINT, "AWS_ENDPOINT_URL_S3": "https://other"}, + NEON_ENDPOINT, + ), + ], +) +def test_endpoint_sources_follow_documented_precedence( + monkeypatch: pytest.MonkeyPatch, env: dict[str, str], expected: str +) -> None: + for name, value in env.items(): + monkeypatch.setenv(name, value) + store = build_object_store(_config()) + assert isinstance(store, NeonObjectStore) + client = store._client + assert client.meta.endpoint_url == expected + assert client.meta.config.s3["addressing_style"] == "path" + + +def test_explicit_endpoint_argument_wins_and_describe_names_neon( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("AWS_ENDPOINT_URL_S3", "https://ignored.example") + store = NeonObjectStore(_config(), endpoint_url=NEON_ENDPOINT) + assert store._client.meta.endpoint_url == NEON_ENDPOINT + assert store.describe() == "neon://incident-artifacts/opensre" + + +@pytest.mark.parametrize( + ("client", "exposure"), + [ + (_PolicyClient(is_public=True), BucketExposure.PUBLIC), + (_PolicyClient(is_public=False), BucketExposure.PRIVATE), + (_PolicyClient(error_code="AccessDenied"), BucketExposure.UNKNOWN), + (_PolicyClient(error_code="NotImplemented"), BucketExposure.UNKNOWN), + ], +) +def test_exposure_reflects_neon_access_level(client: _PolicyClient, exposure: str) -> None: + assert check_public_access(_config(), client=client).exposure == exposure + + +def test_exposure_without_endpoint_is_unknown_not_an_aws_call() -> None: + status = check_public_access(_config()) + assert status.exposure == BucketExposure.UNKNOWN + assert "endpoint" in status.detail diff --git a/tests/integrations/neon/__init__.py b/tests/integrations/neon/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integrations/neon/fixtures.py b/tests/integrations/neon/fixtures.py new file mode 100644 index 0000000000..227e6f6963 --- /dev/null +++ b/tests/integrations/neon/fixtures.py @@ -0,0 +1,213 @@ +"""Neon API payloads captured from a live project (IDs kept, secrets absent). + +Shapes match ``GET /projects/{id}``, ``/branches``, ``/endpoints``, +``/operations``, ``/branches/{id}/storage``, ``/buckets`` and +``/buckets/{name}/objects`` as returned on 2026-11-05. A failed operation and a +second, non-default branch were added to cover the incident-relevant paths. +""" + +from __future__ import annotations + +from typing import Any + +PROJECT_ID = "little-lab-10567887" +MAIN_BRANCH = "br-summer-art-bdbz1xg2" +PREVIEW_BRANCH = "br-quiet-moon-a1b2c3d4" +ENDPOINT_ID = "ep-sparkling-lake-bd31vdlv" + +PROJECT: dict[str, Any] = { + "project": { + "id": PROJECT_ID, + "name": "opensre-neon-test", + "region_id": "aws-us-east-2", + "pg_version": 18, + "created_at": "2026-11-05T10:26:04Z", + "compute_last_active_at": "2026-11-05T10:26:07Z", + "history_retention_seconds": 86400, + "settings": { + "allowed_ips": {"ips": [], "protected_branches_only": False}, + "enable_logical_replication": False, + "maintenance_window": {"weekdays": [1], "start_time": "05:00", "end_time": "06:00"}, + "block_public_connections": False, + "block_vpc_connections": False, + "hipaa": False, + }, + } +} + +BRANCHES: dict[str, Any] = { + "branches": [ + { + "id": MAIN_BRANCH, + "project_id": PROJECT_ID, + "name": "main", + "current_state": "ready", + "state_changed_at": "2026-11-05T10:26:06Z", + "logical_size": 31686656, + "primary": True, + "default": True, + "protected": False, + "created_at": "2026-11-05T10:26:04Z", + "updated_at": "2026-11-05T10:31:15Z", + }, + { + "id": PREVIEW_BRANCH, + "project_id": PROJECT_ID, + "parent_id": MAIN_BRANCH, + "name": "preview/checkout-fix", + "current_state": "init", + "pending_state": "ready", + "default": False, + "protected": False, + "created_at": "2026-11-05T11:00:00Z", + "updated_at": "2026-11-05T11:00:01Z", + }, + ] +} + +ENDPOINTS: dict[str, Any] = { + "endpoints": [ + { + "host": f"{ENDPOINT_ID}.c-8.us-east-2.aws.neon.tech", + "id": ENDPOINT_ID, + "project_id": PROJECT_ID, + "branch_id": MAIN_BRANCH, + "autoscaling_limit_min_cu": 1, + "autoscaling_limit_max_cu": 1, + "region_id": "aws-us-east-2", + "type": "read_write", + "current_state": "idle", + "pooler_enabled": False, + "pooler_mode": "transaction", + "disabled": False, + "passwordless_access": True, + "last_active": "2026-11-05T10:26:07Z", + "suspended_at": "2026-11-05T10:31:15Z", + "suspend_timeout_seconds": 0, + "group": {"size": {"min": 1, "max": 1}, "computes": []}, + }, + { + "host": "ep-young-sun-q9w8e7r6.c-8.us-east-2.aws.neon.tech", + "id": "ep-young-sun-q9w8e7r6", + "project_id": PROJECT_ID, + "branch_id": PREVIEW_BRANCH, + "autoscaling_limit_min_cu": 0.25, + "autoscaling_limit_max_cu": 2, + "type": "read_write", + "current_state": "init", + "pending_state": "active", + "disabled": True, + "suspend_timeout_seconds": 300, + }, + ] +} + +OPERATIONS: dict[str, Any] = { + "operations": [ + { + "id": "9f0e1d2c-0000-4000-8000-000000000001", + "project_id": PROJECT_ID, + "branch_id": PREVIEW_BRANCH, + "endpoint_id": "ep-young-sun-q9w8e7r6", + "action": "start_compute", + "status": "failed", + "error": "compute quota exceeded", + "failures_count": 3, + "created_at": "2026-11-05T11:00:02Z", + "updated_at": "2026-11-05T11:00:40Z", + "total_duration_ms": 38000, + }, + { + "id": "9f0e1d2c-0000-4000-8000-000000000002", + "project_id": PROJECT_ID, + "branch_id": PREVIEW_BRANCH, + "action": "create_timeline", + "status": "running", + "failures_count": 0, + "created_at": "2026-11-05T11:00:00Z", + "updated_at": "2026-11-05T11:00:01Z", + }, + { + "id": "5c878edb-d79a-46f2-b906-b4ea3285cb6a", + "project_id": PROJECT_ID, + "branch_id": MAIN_BRANCH, + "endpoint_id": ENDPOINT_ID, + "action": "suspend_compute", + "status": "finished", + "failures_count": 0, + "created_at": "2026-11-05T10:31:15Z", + "updated_at": "2026-11-05T10:31:15Z", + "total_duration_ms": 826, + }, + ], + "pagination": {"cursor": "2026-11-05T10:26:04.069482Z"}, +} + +STORAGE: dict[str, Any] = { + "enabled": True, + "s3_endpoint": f"https://{MAIN_BRANCH}.storage.c-8.us-east-2.aws.neon.tech", + "region": "us-east-2", + "force_path_style": True, +} + +BUCKETS: dict[str, Any] = { + "buckets": [ + { + "name": "incident-artifacts", + "access_level": "private", + "created_at": "2026-11-05T10:26:26Z", + }, + { + "name": "public-assets", + "access_level": "public_read", + "created_at": "2026-11-05T10:26:27Z", + }, + ] +} + +OBJECTS_FOLDED: dict[str, Any] = { + "folders": ["logs/"], + "objects": [ + { + "key": "root.txt", + "size": 7, + "last_modified": "2026-11-05T10:26:47Z", + "etag": "5c2ce561e1e263695dbd267271b86fb8", + } + ], + "prefix": "", + "is_truncated": False, +} + +OBJECTS_PAGE: dict[str, Any] = { + "folders": [], + "objects": [ + { + "key": "logs/2026-11-05/app.json", + "size": 20, + "last_modified": "2026-11-05T10:26:30Z", + "etag": "ba15e5bbca0e53abdfd9a730301d90b1", + }, + { + "key": "logs/2026-11-05/worker-1.txt", + "size": 7, + "last_modified": "2026-11-05T10:26:39Z", + "etag": "5c2ce561e1e263695dbd267271b86fb8", + }, + ], + "prefix": "logs/", + "next_cursor": "logs/2026-11-05/worker-1.txt", + "is_truncated": True, +} + +ROUTES: dict[str, dict[str, Any]] = { + f"/api/v2/projects/{PROJECT_ID}": PROJECT, + f"/api/v2/projects/{PROJECT_ID}/branches": BRANCHES, + f"/api/v2/projects/{PROJECT_ID}/endpoints": ENDPOINTS, + f"/api/v2/projects/{PROJECT_ID}/operations": OPERATIONS, + f"/api/v2/projects/{PROJECT_ID}/branches/{MAIN_BRANCH}/storage": STORAGE, + f"/api/v2/projects/{PROJECT_ID}/branches/{MAIN_BRANCH}/buckets": BUCKETS, + f"/api/v2/projects/{PROJECT_ID}/branches/{MAIN_BRANCH}/buckets/incident-artifacts/objects": ( + OBJECTS_FOLDED + ), +} diff --git a/tests/integrations/neon/test_neon.py b/tests/integrations/neon/test_neon.py new file mode 100644 index 0000000000..92c16cb2b1 --- /dev/null +++ b/tests/integrations/neon/test_neon.py @@ -0,0 +1,384 @@ +"""Neon integration: config resolution, API client, diagnostics, tools, and wiring.""" + +from __future__ import annotations + +import json +from collections.abc import Iterator +from http import HTTPStatus +from typing import Any + +import httpx +import pytest + +from config.constants.neon import NEON_API_KEY_ENV, NEON_BRANCH_ID_ENV, NEON_PROJECT_ID_ENV +from core.domain.types.evidence import CATALOG_ENTRIES_KEY +from core.domain.types.tools import ToolSurface +from core.llm.types import ToolCall +from core.tool.execution import execute_tools +from integrations.neon import build_neon_config, classify +from integrations.neon.client import NeonAPIError, NeonClient +from integrations.neon.diagnostics import ( + get_project_status, + get_storage_buckets, + list_storage_objects, +) +from integrations.neon.tools.neon_project_status_tool.tool import map_neon_project_status +from integrations.neon.tools.neon_storage_buckets_tool.tool import map_neon_storage_buckets +from integrations.neon.tools.neon_storage_objects_tool.tool import map_neon_storage_objects +from tests.integrations.neon import fixtures as fx + +API_KEY = "napi_test_key_do_not_log" +NEON_TOOL_NAMES = { + "get_neon_project_status", + "get_neon_storage_buckets", + "list_neon_storage_objects", +} + + +class _FakeNeonAPI: + """httpx transport serving captured Neon payloads and recording requests.""" + + def __init__( + self, + routes: dict[str, dict[str, Any]] | None = None, + *, + status_code: int = HTTPStatus.OK, + body: Any = None, + raise_exc: Exception | None = None, + ) -> None: + self.routes = fx.ROUTES if routes is None else routes + self.status_code = status_code + self.body = body + self.raise_exc = raise_exc + self.requests: list[httpx.Request] = [] + + def handler(self, request: httpx.Request) -> httpx.Response: + self.requests.append(request) + if self.raise_exc is not None: + raise self.raise_exc + if self.status_code != HTTPStatus.OK: + return httpx.Response(self.status_code, json=self.body or {}) + if request.url.params.get("prefix") == "logs/": + return httpx.Response(HTTPStatus.OK, json=fx.OBJECTS_PAGE) + payload = self.routes.get(request.url.path) + if payload is None: + return httpx.Response(HTTPStatus.NOT_FOUND, json={"message": "not found"}) + return httpx.Response(HTTPStatus.OK, json=payload) + + @property + def transport(self) -> httpx.MockTransport: + return httpx.MockTransport(self.handler) + + +def _client(api: _FakeNeonAPI, **overrides: Any) -> NeonClient: + config = build_neon_config({"api_key": API_KEY, "project_id": fx.PROJECT_ID, **overrides}) + return NeonClient(config, transport=api.transport) + + +@pytest.fixture +def neon_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setenv("OPENSRE_DISABLE_KEYRING", "1") + monkeypatch.setenv(NEON_API_KEY_ENV, API_KEY) + monkeypatch.setenv(NEON_PROJECT_ID_ENV, fx.PROJECT_ID) + monkeypatch.delenv(NEON_BRANCH_ID_ENV, raising=False) + yield + + +# --------------------------------------------------------------------------- config + + +def test_classify_requires_key_and_project_and_normalizes() -> None: + config, service = classify( + {"api_key": f" {API_KEY} ", "project_id": f" {fx.PROJECT_ID}\n", "branch_id": ""}, + "neon-1", + ) + assert service == "neon" + assert config is not None + assert (config.api_key, config.project_id, config.integration_id) == ( + API_KEY, + fx.PROJECT_ID, + "neon-1", + ) + assert config.base_url == "https://console.neon.tech/api/v2" + assert classify({"api_key": API_KEY}, "neon-2") == (None, None) + assert classify({"project_id": fx.PROJECT_ID}, "neon-3") == (None, None) + + +def test_env_loader_and_detection_publish_neon( + neon_env: None, monkeypatch: pytest.MonkeyPatch +) -> None: + from integrations.catalog import ( + classify_integrations, + load_env_integration_services, + load_env_integrations, + ) + + monkeypatch.setenv(NEON_BRANCH_ID_ENV, fx.MAIN_BRANCH) + records = [record for record in load_env_integrations() if record["service"] == "neon"] + assert len(records) == 1 + assert records[0]["credentials"] == { + "api_key": API_KEY, + "project_id": fx.PROJECT_ID, + "branch_id": fx.MAIN_BRANCH, + } + resolved = classify_integrations(records) + assert resolved["neon"].branch_id == fx.MAIN_BRANCH + assert "neon" in load_env_integration_services() + + monkeypatch.delenv(NEON_PROJECT_ID_ENV) + assert not [record for record in load_env_integrations() if record["service"] == "neon"] + + +# --------------------------------------------------------------------------- client + + +def test_client_sends_bearer_key_to_project_scoped_paths() -> None: + api = _FakeNeonAPI() + with _client(api) as client: + assert client.get_project()["name"] == "opensre-neon-test" + client.list_operations(5000) + first, second = api.requests + assert first.headers["Authorization"] == f"Bearer {API_KEY}" + assert first.url.path == f"/api/v2/projects/{fx.PROJECT_ID}" + assert second.url.params["limit"] == "1000" + + +@pytest.mark.parametrize( + "bad", + ["../other-project", "a/b", "bucket?x=1", "", "x" * 200, ".hidden"], +) +def test_untrusted_identifiers_never_reach_the_network(bad: str) -> None: + api = _FakeNeonAPI() + with _client(api) as client, pytest.raises(NeonAPIError) as err: + client.list_objects(fx.MAIN_BRANCH, bad) + assert err.value.invalid_input is True + assert api.requests == [] + + +def test_branch_resolution_prefers_explicit_then_configured_then_default() -> None: + api = _FakeNeonAPI() + assert _client(api).resolve_branch_id("br-explicit-1") == "br-explicit-1" + assert _client(api, branch_id="br-configured-1").resolve_branch_id() == "br-configured-1" + assert api.requests == [] + assert _client(api).resolve_branch_id() == fx.MAIN_BRANCH + assert api.requests[-1].url.path.endswith("/branches") + + +@pytest.mark.parametrize( + ("status", "failure_state", "message"), + [ + (HTTPStatus.UNAUTHORIZED, "denied", "rejected the API key"), + (HTTPStatus.FORBIDDEN, "denied", "cannot access"), + (HTTPStatus.NOT_FOUND, "no_match", "not found"), + (HTTPStatus.TOO_MANY_REQUESTS, "transport_failure", "rate limit"), + (HTTPStatus.SERVICE_UNAVAILABLE, "transport_failure", "unavailable"), + ], +) +def test_api_errors_become_structured_failures( + status: HTTPStatus, failure_state: str, message: str +) -> None: + api = _FakeNeonAPI(status_code=status, body={"message": "details from neon"}) + result = get_project_status(_client(api)) + assert result["available"] is False + assert result["failure_state"] == failure_state + assert message in result["error"] + assert API_KEY not in json.dumps(result) + + +def test_transport_errors_are_redacted_and_reported() -> None: + api = _FakeNeonAPI(raise_exc=httpx.ConnectError(f"refused while sending {API_KEY}")) + result = get_storage_buckets(_client(api)) + assert result["failure_state"] == "transport_failure" + assert API_KEY not in result["error"] + assert "" in result["error"] + + +def test_probe_reports_project_or_missing_config() -> None: + probe = _client(_FakeNeonAPI()).probe_access() + assert probe.ok + assert fx.PROJECT_ID in probe.detail + assert not NeonClient(build_neon_config({"api_key": API_KEY})).probe_access().ok + + +# --------------------------------------------------------------------------- diagnostics + + +def test_project_status_surfaces_failed_pending_and_disabled_computes() -> None: + result = get_project_status(_client(_FakeNeonAPI())) + + assert result["project"]["pg_version"] == 18 + assert result["total_branches"] == 2 + assert [row["action"] for row in result["failed_operations"]] == ["start_compute"] + assert result["failed_operations"][0]["error"] == "compute quota exceeded" + assert [row["action"] for row in result["pending_operations"]] == ["create_timeline"] + assert result["active_endpoints"] == [] + assert result["disabled_endpoints"] == ["ep-young-sun-q9w8e7r6"] + main = next(row for row in result["endpoints"] if row["id"] == fx.ENDPOINT_ID) + assert main["current_state"] == "idle" + assert main["suspended_at"] == "2026-11-05T10:31:15Z" + + +def test_project_status_branch_filter_narrows_computes_and_operations() -> None: + result = get_project_status(_client(_FakeNeonAPI()), branch_id=fx.MAIN_BRANCH) + assert {row["branch_id"] for row in result["endpoints"]} == {fx.MAIN_BRANCH} + assert [row["action"] for row in result["operations"]] == ["suspend_compute"] + assert result["failed_operations"] == [] + assert result["total_branches"] == 2 + + +def test_storage_buckets_flags_public_buckets_on_default_branch() -> None: + result = get_storage_buckets(_client(_FakeNeonAPI())) + assert result["branch_id"] == fx.MAIN_BRANCH + assert result["storage_enabled"] is True + assert result["s3_endpoint"] == fx.STORAGE["s3_endpoint"] + assert result["public_buckets"] == ["public-assets"] + + +def test_storage_disabled_branch_skips_bucket_listing() -> None: + routes = dict(fx.ROUTES) + routes[f"/api/v2/projects/{fx.PROJECT_ID}/branches/{fx.MAIN_BRANCH}/storage"] = { + "enabled": False + } + api = _FakeNeonAPI(routes) + result = get_storage_buckets(_client(api, branch_id=fx.MAIN_BRANCH)) + assert result["storage_enabled"] is False + assert result["buckets"] == [] + assert not any(request.url.path.endswith("/buckets") for request in api.requests) + + +def test_object_listing_folds_folders_and_paginates() -> None: + api = _FakeNeonAPI() + folded = list_storage_objects(_client(api), bucket="incident-artifacts") + assert folded["folders"] == ["logs/"] + assert [item["key"] for item in folded["objects"]] == ["root.txt"] + assert api.requests[-1].url.params["delimiter"] == "/" + + page = list_storage_objects( + _client(api), bucket="incident-artifacts", prefix="logs/", recursive=True, limit=2 + ) + assert "delimiter" not in api.requests[-1].url.params + assert api.requests[-1].url.params["limit"] == "2" + assert page["truncated"] is True + assert page["next_cursor"] == "logs/2026-11-05/worker-1.txt" + + +def test_unknown_bucket_is_a_no_match() -> None: + result = list_storage_objects(_client(_FakeNeonAPI()), bucket="missing-bucket") + assert result["failure_state"] == "no_match" + + +# --------------------------------------------------------------------------- evidence + + +def test_evidence_mappers_cite_incident_signals() -> None: + evidence: dict[str, Any] = {} + map_neon_project_status(evidence, get_project_status(_client(_FakeNeonAPI())), {}) + map_neon_storage_buckets(evidence, get_storage_buckets(_client(_FakeNeonAPI())), {}) + map_neon_storage_objects( + evidence, + list_storage_objects(_client(_FakeNeonAPI()), bucket="incident-artifacts"), + {}, + ) + entries = {entry["source"]: entry["summary"] for entry in evidence[CATALOG_ENTRIES_KEY]} + assert "1 failed operations (start_compute)" in entries["get_neon_project_status"] + assert "disabled computes: ep-young-sun-q9w8e7r6" in entries["get_neon_project_status"] + assert entries["get_neon_storage_buckets"] == "2 buckets, public: public-assets" + assert entries["list_neon_storage_objects"].startswith("1 objects, 1 folders") + + untouched: dict[str, Any] = {} + map_neon_project_status(untouched, {"available": False, "error": "x"}, {}) + assert untouched == {} + + +def test_compute_counts_cover_every_endpoint_when_the_list_is_capped() -> None: + routes = dict(fx.ROUTES) + routes[f"/api/v2/projects/{fx.PROJECT_ID}/endpoints"] = { + "endpoints": [ + {"id": f"ep-busy-{index}", "branch_id": fx.MAIN_BRANCH, "current_state": "active"} + for index in range(60) + ] + } + result = get_project_status(_client(_FakeNeonAPI(routes))) + assert len(result["endpoints"]) == 50 + assert result["endpoints_truncated"] is True + assert result["total_endpoints"] == 60 + + evidence: dict[str, Any] = {} + map_neon_project_status(evidence, result, {}) + assert evidence[CATALOG_ENTRIES_KEY][0]["summary"].startswith("60/60 computes active") + + +# --------------------------------------------------------------------------- tools + runtime + + +def test_tools_are_registered_on_chat_and_action_with_hidden_credentials() -> None: + from tools import registry + + for surface in (ToolSurface.CHAT, ToolSurface.ACTION): + names = {tool.name for tool in registry.get_registered_tools(surface)} + assert names >= NEON_TOOL_NAMES + by_name = {tool.name: tool for tool in registry.get_registered_tools()} + for name in NEON_TOOL_NAMES: + public = by_name[name].public_input_schema["properties"] + assert not {"api_key", "project_id", "base_url"} & set(public) + assert by_name[name].is_available({}) is False + + +def test_runtime_injects_credentials_and_refuses_model_overrides( + neon_env: None, monkeypatch: pytest.MonkeyPatch +) -> None: + from integrations.catalog import classify_integrations, load_env_integrations + from tools import registry + + api = _FakeNeonAPI() + monkeypatch.setattr( + "integrations.neon.tools._params.NeonClient", + lambda config: NeonClient(config, transport=api.transport), + ) + resolved = classify_integrations( + [record for record in load_env_integrations() if record["service"] == "neon"] + ) + tools = [tool for tool in registry.get_registered_tools() if tool.name in NEON_TOOL_NAMES] + + ok, refused = execute_tools( + [ + ToolCall(id="1", name="get_neon_storage_buckets", input={}), + ToolCall( + id="2", + name="list_neon_storage_objects", + input={"bucket": "incident-artifacts", "project_id": "someone-elses-project"}, + ), + ], + tools, + resolved, + ) + + assert ok["public_buckets"] == ["public-assets"] + assert "project_id" in refused["error"] + assert all( + request.url.path.startswith(f"/api/v2/projects/{fx.PROJECT_ID}") for request in api.requests + ) + assert all(request.headers["Authorization"] == f"Bearer {API_KEY}" for request in api.requests) + assert API_KEY not in json.dumps([ok, refused]) + + +# --------------------------------------------------------------------------- wiring + + +def test_neon_is_a_verifiable_setup_integration() -> None: + from integrations.cli import _HANDLERS + from integrations.neon.setup import NEON_SETUP + from integrations.registry import INTEGRATION_SPECS, SUPPORTED_SETUP_SERVICES + from integrations.verification import get_verifier + from integrations.verify import register_all_verifiers + + register_all_verifiers() + spec = next(spec for spec in INTEGRATION_SPECS if spec.service == "neon") + assert spec.has_verifier and spec.direct_effective + assert "neon" in SUPPORTED_SETUP_SERVICES + assert "neon" in _HANDLERS + assert get_verifier("neon") is not None + fields = {field.name: field for field in NEON_SETUP.fields} + assert fields["api_key"].secret and fields["api_key"].env_var == NEON_API_KEY_ENV + assert fields["project_id"].required + assert not fields["branch_id"].required diff --git a/tests/tools/test_telemetry.py b/tests/tools/test_telemetry.py index 2c7cdc9d86..0bb7dbed27 100644 --- a/tests/tools/test_telemetry.py +++ b/tests/tools/test_telemetry.py @@ -1300,6 +1300,8 @@ def _describe(_cluster: str, ng: str) -> dict[str, Any]: "get_mysql_server_status", "get_mysql_slow_queries", "get_mysql_table_stats", + "get_neon_project_status", + "get_neon_storage_buckets", "get_pods_on_node", "get_postgresql_current_queries", "get_postgresql_lock_status", @@ -1359,6 +1361,7 @@ def _describe(_cluster: str, ng: str) -> dict[str, Any]: "list_jenkins_builds", "list_jenkins_jobs", "list_jenkins_running_builds", + "list_neon_storage_objects", "list_s3_objects", "list_sentry_issue_events", "list_sentry_uptime_alerts", diff --git a/tools/registry_discovery.py b/tools/registry_discovery.py index f1547f9f86..95656d8c0e 100644 --- a/tools/registry_discovery.py +++ b/tools/registry_discovery.py @@ -61,6 +61,7 @@ "integrations.mongodb.tools", "integrations.mongodb_atlas.tools", "integrations.mysql.tools", + "integrations.neon.tools", "integrations.new_relic.tools", "integrations.openobserve.tools", "integrations.opensearch.tools",