From 3a60deb0c0404e7b59265b2500b1f2bbcf2d2bb0 Mon Sep 17 00:00:00 2001 From: thefloshibisci <103315611+thefloshibisci@users.noreply.github.com> Date: Wed, 5 Aug 2026 23:55:43 +0800 Subject: [PATCH 01/16] Update Dockerfile --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index f383ef0add..013bd858a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,9 @@ RUN set -eu; \ echo "[build] INSTALL_CLOUDFLARED=0 → 跳过 cloudflared(Tunnel 一键管理将不可用)"; \ fi; \ rm -f /tmp/fetch_cloudflared.py +RUN apt-get update \ + && apt-get install -y --no-install-recommends wget curl ca-certificates \ + && rm -rf /var/lib/apt/lists/* # Install dependencies first (leverage Docker cache) # 先装依赖(利用 Docker 缓存) From 73a318a9b38cf3995ad4e4d4543997a72e22c70b Mon Sep 17 00:00:00 2001 From: thefloshibisci <103315611+thefloshibisci@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:49:30 +0800 Subject: [PATCH 02/16] Fix memory bucket detail compatibility --- src/tools/anchor/core.py | 43 ++++--- src/web/_shared.py | 25 ++++ src/web/buckets.py | 9 +- tests/test_dashboard_bucket_service_auth.py | 132 ++++++++++++++++++++ tests/test_pulse_bucket_types.py | 77 ++++++++++++ 5 files changed, 266 insertions(+), 20 deletions(-) create mode 100644 tests/test_dashboard_bucket_service_auth.py create mode 100644 tests/test_pulse_bucket_types.py diff --git a/src/tools/anchor/core.py b/src/tools/anchor/core.py index 0366cbdf3e..646f1fdb08 100644 --- a/src/tools/anchor/core.py +++ b/src/tools/anchor/core.py @@ -10,9 +10,9 @@ pulse 顺带放在这里:它是系统状态 + 桶清单的总览,调用频次低,把它 塞进一个文件不影响阅读。 -关键行为: -- anchor_set / anchor_release:调 bucket_mgr.set_anchor,原样转译结果 -- pulse:聚合 stats + list_all,按 type 分组(normal/feel/plan/letter) +关键行为: +- anchor_set / anchor_release:调 bucket_mgr.set_anchor,原样转译结果 +- pulse:聚合 stats + list_all,按 type 分组(normal/feel/plan/letter/i) 逐行展示 icon + 主题 + 情感 + 权重 + 标签 - pulse 同时附带「索引漂移」自检:embedding.db 的 ID 集合与磁盘桶 ID 集合 对账,缺失/孤儿 > 0 时在状态块顶部告警,提示运行 backfill / clean 脚本 @@ -133,9 +133,10 @@ async def pulse(include_archive: Optional[bool] = False) -> str: return status + "\n记忆库为空。" normal_lines: list[str] = [] - feel_lines: list[str] = [] - plan_lines: list[str] = [] - letter_lines: list[str] = [] + feel_lines: list[str] = [] + plan_lines: list[str] = [] + letter_lines: list[str] = [] + i_lines: list[str] = [] for b in buckets: meta = b.get("metadata", {}) btype = meta.get("type") @@ -147,10 +148,12 @@ async def pulse(include_archive: Optional[bool] = False) -> str: icon = "🫧" elif btype == "plan": icon = "📋" - elif btype == "letter": - icon = "💌" - elif btype == "archived": - icon = "🗄️" + elif btype == "letter": + icon = "💌" + elif btype == "i": + icon = "🪞" + elif btype == "archived": + icon = "🗄️" elif meta.get("resolved", False): icon = "✅" else: @@ -180,11 +183,13 @@ async def pulse(include_archive: Optional[bool] = False) -> str: elif btype == "plan": plan_status = meta.get("status", "active") plan_lines.append(line + f" [{plan_status}]") - elif btype == "letter": - author = meta.get("author", "?") - letter_lines.append(line + f" [{author}]") - else: - normal_lines.append(line) + elif btype == "letter": + author = meta.get("author", "?") + letter_lines.append(line + f" [{author}]") + elif btype == "i": + i_lines.append(line) + else: + normal_lines.append(line) sections = [status] if normal_lines: @@ -193,6 +198,8 @@ async def pulse(include_archive: Optional[bool] = False) -> str: sections.append(f"=== 计划({len(plan_lines)} 条)===\n" + "\n".join(plan_lines)) if feel_lines: sections.append(f"=== feel({len(feel_lines)} 条)===\n" + "\n".join(feel_lines)) - if letter_lines: - sections.append(f"=== 信件({len(letter_lines)} 封)===\n" + "\n".join(letter_lines)) - return "\n\n".join(sections) + if letter_lines: + sections.append(f"=== 信件({len(letter_lines)} 封)===\n" + "\n".join(letter_lines)) + if i_lines: + sections.append(f"=== I({len(i_lines)} 条)===\n" + "\n".join(i_lines)) + return "\n\n".join(sections) diff --git a/src/web/_shared.py b/src/web/_shared.py index 152fa8d337..86173bfcb9 100644 --- a/src/web/_shared.py +++ b/src/web/_shared.py @@ -383,6 +383,7 @@ def _write_env_var(name: str, value: str) -> None: _MAX_ACTIVE_SESSIONS = 256 _SESSION_TTL_SECONDS = 86400 * _DEFAULT_SESSION_TTL_DAYS _SESSION_TTL = _SESSION_TTL_SECONDS # compatibility constant for older callers +_MIN_SERVICE_TOKEN_LENGTH = 32 def _session_ttl_seconds() -> int: @@ -1093,6 +1094,23 @@ def _is_authenticated(request: Request) -> bool: return True +def _is_service_token_authenticated(request: Request) -> bool: + """Allow a strong sidecar token on explicitly opted-in read-only routes.""" + configured = str(os.environ.get("OMBRE_MCP_SERVICE_TOKEN", "") or "").strip() + if len(configured) < _MIN_SERVICE_TOKEN_LENGTH: + return False + authorization = str(request.headers.get("authorization", "") or "").strip() + scheme, separator, candidate = authorization.partition(" ") + candidate = candidate.strip() + if ( + not separator + or scheme.lower() != "bearer" + or len(candidate) != len(configured) + ): + return False + return hmac.compare_digest(candidate, configured) + + def _authenticated_credential_generation(request: Request) -> int | None: """Atomically bind an authenticated mutation to the credential generation.""" with _auth_mutation_lock: @@ -1137,3 +1155,10 @@ def _require_auth(request: Request) -> Response | None: status_code=401, ) return None + + +def _require_service_or_dashboard_auth(request: Request) -> Response | None: + """Authenticate one opted-in sidecar read or a normal Dashboard session.""" + if _is_service_token_authenticated(request): + return None + return _require_auth(request) diff --git a/src/web/buckets.py b/src/web/buckets.py index 4c08a91c8e..caf169e012 100644 --- a/src/web/buckets.py +++ b/src/web/buckets.py @@ -177,9 +177,14 @@ def created_sort_key(item: dict) -> tuple[int, int, str]: @mcp.custom_route("/api/bucket/{bucket_id}", methods=["GET"]) async def api_bucket_detail(request: Request) -> Response: - """Get full raw bucket content plus display-only derived text by ID.""" + """Get full raw bucket content plus display-only derived text by ID. + + A trusted sidecar may use its service token on this exact read route so + Xinchao can inspect every bucket type. Dashboard sessions remain valid, + while all bucket mutation routes keep requiring Dashboard auth. + """ from starlette.responses import JSONResponse - err = sh._require_auth(request) + err = sh._require_service_or_dashboard_auth(request) if err: return err bucket_id = request.path_params["bucket_id"] diff --git a/tests/test_dashboard_bucket_service_auth.py b/tests/test_dashboard_bucket_service_auth.py new file mode 100644 index 0000000000..a86250c379 --- /dev/null +++ b/tests/test_dashboard_bucket_service_auth.py @@ -0,0 +1,132 @@ +import json + +import pytest +from starlette.requests import Request + +from web import _shared as shared_web +from web import buckets as buckets_web + + +SERVICE_TOKEN = "s" * 40 + + +class FakeMCP: + def __init__(self): + self.routes = {} + + def custom_route(self, path, methods): + def decorator(handler): + for method in methods: + self.routes[(method, path)] = handler + return handler + + return decorator + + +class FakeBucketManager: + async def get(self, bucket_id): + if bucket_id != "feel-1": + return None + return { + "id": bucket_id, + "content": "raw [[memory]]", + "metadata": {"type": "feel", "name": "Feeling"}, + } + + async def get_triggered_feels(self, _bucket_id): + return [] + + +class FakeDecayEngine: + @staticmethod + def calculate_score(_metadata): + return 1.0 + + +def _request(method, path, authorization=""): + headers = [] + if authorization: + headers.append((b"authorization", authorization.encode("ascii"))) + return Request( + { + "type": "http", + "method": method, + "path": path, + "path_params": {"bucket_id": "feel-1"}, + "query_string": b"", + "headers": headers, + } + ) + + +@pytest.fixture +def service_token(monkeypatch, tmp_path): + monkeypatch.setenv("OMBRE_MCP_SERVICE_TOKEN", SERVICE_TOKEN) + monkeypatch.setattr( + shared_web, + "config", + {"buckets_dir": str(tmp_path)}, + raising=False, + ) + + +def test_service_auth_requires_strong_exact_bearer(monkeypatch, service_token): + assert shared_web._is_service_token_authenticated( + _request("GET", "/api/bucket/feel-1", f"Bearer {SERVICE_TOKEN}") + ) + assert not shared_web._is_service_token_authenticated( + _request("GET", "/api/bucket/feel-1", "Bearer wrong") + ) + assert not shared_web._is_service_token_authenticated( + _request("GET", "/api/bucket/feel-1", f"Basic {SERVICE_TOKEN}") + ) + + monkeypatch.setenv("OMBRE_MCP_SERVICE_TOKEN", "short") + assert not shared_web._is_service_token_authenticated( + _request("GET", "/api/bucket/feel-1", "Bearer short") + ) + + +@pytest.mark.asyncio +async def test_service_token_reads_exact_bucket_detail( + monkeypatch, + service_token, +): + monkeypatch.setattr( + buckets_web.sh, "bucket_mgr", FakeBucketManager(), raising=False + ) + monkeypatch.setattr( + buckets_web.sh, "decay_engine", FakeDecayEngine(), raising=False + ) + mcp = FakeMCP() + buckets_web.register(mcp) + + response = await mcp.routes[("GET", "/api/bucket/{bucket_id}")]( + _request("GET", "/api/bucket/feel-1", f"Bearer {SERVICE_TOKEN}") + ) + payload = json.loads(response.body) + + assert response.status_code == 200 + assert payload["metadata"]["type"] == "feel" + assert payload["content"] == "raw [[memory]]" + assert payload["display_content"] == "raw memory" + + +@pytest.mark.asyncio +async def test_service_token_does_not_authorize_bucket_mutations( + service_token, +): + mcp = FakeMCP() + buckets_web.register(mcp) + request = _request( + "POST", + "/api/bucket/feel-1/resolve", + f"Bearer {SERVICE_TOKEN}", + ) + + response = await mcp.routes[("POST", "/api/bucket/{bucket_id}/resolve")]( + request + ) + + assert response.status_code == 401 + assert json.loads(response.body)["error"] == "Unauthorized" diff --git a/tests/test_pulse_bucket_types.py b/tests/test_pulse_bucket_types.py new file mode 100644 index 0000000000..9e2fcca418 --- /dev/null +++ b/tests/test_pulse_bucket_types.py @@ -0,0 +1,77 @@ +import pytest + +from tools.anchor import core as anchor_core + + +class BucketManager: + async def get_stats(self): + return { + "permanent_count": 1, + "dynamic_count": 2, + "archive_count": 0, + "feel_count": 1, + "plan_count": 1, + "letter_count": 1, + "total_size_kb": 1.0, + } + + async def list_all(self, include_archive=False): + self.include_archive = include_archive + return [ + _bucket("permanent-1", "permanent"), + _bucket("dynamic-1", "dynamic"), + _bucket("feel-1", "feel"), + _bucket("plan-1", "plan"), + _bucket("letter-1", "letter"), + _bucket("self-1", "i"), + ] + + +class DecayEngine: + is_running = True + + async def ensure_started(self): + return None + + @staticmethod + def calculate_score(_metadata): + return 1.0 + + +def _bucket(bucket_id, bucket_type): + return { + "id": bucket_id, + "content": f"{bucket_type} content", + "metadata": { + "type": bucket_type, + "name": bucket_type, + "domain": [bucket_type], + "tags": [bucket_type], + "valence": 0.5, + "arousal": 0.3, + "importance": 5, + }, + } + + +@pytest.mark.asyncio +async def test_pulse_keeps_every_active_bucket_type_distinct(monkeypatch): + monkeypatch.setattr( + anchor_core.rt, "bucket_mgr", BucketManager(), raising=False + ) + monkeypatch.setattr( + anchor_core.rt, "decay_engine", DecayEngine(), raising=False + ) + monkeypatch.setattr( + anchor_core.rt, "embedding_engine", None, raising=False + ) + + result = await anchor_core.pulse() + + assert "📦 [permanent-1]" in result + assert "💭 [dynamic-1]" in result + assert "🫧 [feel-1]" in result + assert "📋 [plan-1]" in result + assert "💌 [letter-1]" in result + assert "🪞 [self-1]" in result + assert "=== I(1 条)===" in result From e756499d70748a11d0ea628c48e8457624f0ea87 Mon Sep 17 00:00:00 2001 From: thefloshibisci <103315611+thefloshibisci@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:00:05 +0800 Subject: [PATCH 03/16] Refresh update manifest for bucket compatibility --- update_manifest.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/update_manifest.json b/update_manifest.json index 448de22131..4810936456 100644 --- a/update_manifest.json +++ b/update_manifest.json @@ -777,8 +777,8 @@ }, { "path": "src/tools/anchor/core.py", - "sha256": "456d8d90157d8267d7a17e20637c43bbefe404cdcf28e3eec82b44759ce0c287", - "size": 8633 + "sha256": "2ed9932595677b49091bde36bde4ba13afa1640e249f847052068c006190563c", + "size": 8858 }, { "path": "src/tools/breath/__init__.py", @@ -927,8 +927,8 @@ }, { "path": "src/web/_shared.py", - "sha256": "ceb49098a99d2ded66771c3e120e3250f09aa8743f5d60d96fb5c79fa50f0726", - "size": 42575 + "sha256": "f5ef517fad308c7b1305797fe22254f12a26942ed7bc2a0c5bd0009650ea1f0f", + "size": 43542 }, { "path": "src/web/auth.py", @@ -937,8 +937,8 @@ }, { "path": "src/web/buckets.py", - "sha256": "da12ab4dd7f41cd9175b56652f7885bcd3702ef12dbd59fb04e6f57a99b261e1", - "size": 49357 + "sha256": "8c9fdf447af6f7b25d2cc387fed10a7cd9d57e9200e5b59cf2085037fde35932", + "size": 49620 }, { "path": "src/web/config_api.py", From 3d21a14b4a916b15d1cba228a9060471b0909217 Mon Sep 17 00:00:00 2001 From: thefloshibisci <103315611+thefloshibisci@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:48:54 +0800 Subject: [PATCH 04/16] fix: accept MCP static token on read-only bucket detail endpoint The star-map detail page in Xinchao calls /api/bucket/{id} with the same Bearer token used for MCP protocol auth (OMBRE_MCP_TOKEN / config['mcp_token']). The previous fix (73a318a) only accepted a separate OMBRE_MCP_SERVICE_TOKEN env var, which is typically not configured. This made every feel/plan/letter/i bucket appear in the star map (via pulse) but show 'not in current memory library' when clicked, because the HTTP read returned 401 and the MCP search fallback could not find the bucket by query. Now _is_service_token_authenticated accepts either the dedicated service token or the live MCP static token, both in constant time. Mutation routes remain Dashboard-session-only. --- src/web/_shared.py | 43 ++++++++++---- tests/test_dashboard_bucket_service_auth.py | 64 +++++++++++++++++++++ 2 files changed, 96 insertions(+), 11 deletions(-) diff --git a/src/web/_shared.py b/src/web/_shared.py index 86173bfcb9..724e87cf13 100644 --- a/src/web/_shared.py +++ b/src/web/_shared.py @@ -1,4 +1,4 @@ -""" +""" ======================================== web/_shared.py — Dashboard/HTTP 层的共享依赖与鉴权工具 ======================================== @@ -1095,20 +1095,41 @@ def _is_authenticated(request: Request) -> bool: def _is_service_token_authenticated(request: Request) -> bool: - """Allow a strong sidecar token on explicitly opted-in read-only routes.""" - configured = str(os.environ.get("OMBRE_MCP_SERVICE_TOKEN", "") or "").strip() - if len(configured) < _MIN_SERVICE_TOKEN_LENGTH: - return False + """Allow a strong sidecar token on explicitly opted-in read-only routes. + + A trusted sidecar (e.g. Xinchao) connects to OB via MCP using a static + token (OMBRE_MCP_TOKEN / config["mcp_token"]). That same token should also + authenticate read-only HTTP API routes such as /api/bucket/{id}, so the + star-map detail page can fetch bucket content without a separate + OMBRE_MCP_SERVICE_TOKEN that may never be configured. + + Accept either token in constant time. Both must meet the minimum length. + """ authorization = str(request.headers.get("authorization", "") or "").strip() scheme, separator, candidate = authorization.partition(" ") candidate = candidate.strip() - if ( - not separator - or scheme.lower() != "bearer" - or len(candidate) != len(configured) - ): + if not separator or scheme.lower() != "bearer" or not candidate: return False - return hmac.compare_digest(candidate, configured) + + configured_tokens: list[str] = [] + service_token = str(os.environ.get("OMBRE_MCP_SERVICE_TOKEN", "") or "").strip() + if len(service_token) >= _MIN_SERVICE_TOKEN_LENGTH: + configured_tokens.append(service_token) + mcp_token = ( + str(os.environ.get("OMBRE_MCP_TOKEN", "") or "").strip() + or str(config.get("mcp_token", "") or "").strip() + ) + if len(mcp_token) >= _MIN_SERVICE_TOKEN_LENGTH: + configured_tokens.append(mcp_token) + + if not configured_tokens: + return False + + # compare_digest leaks length, so only compare against tokens of equal length. + for configured in configured_tokens: + if len(candidate) == len(configured) and hmac.compare_digest(candidate, configured): + return True + return False def _authenticated_credential_generation(request: Request) -> int | None: diff --git a/tests/test_dashboard_bucket_service_auth.py b/tests/test_dashboard_bucket_service_auth.py index a86250c379..7fb861f95c 100644 --- a/tests/test_dashboard_bucket_service_auth.py +++ b/tests/test_dashboard_bucket_service_auth.py @@ -130,3 +130,67 @@ async def test_service_token_does_not_authorize_bucket_mutations( assert response.status_code == 401 assert json.loads(response.body)["error"] == "Unauthorized" + + +MCP_TOKEN = "m" * 40 + + +@pytest.fixture +def mcp_token(monkeypatch, tmp_path): + """Configure only the MCP static token, no dedicated service token.""" + monkeypatch.delenv("OMBRE_MCP_SERVICE_TOKEN", raising=False) + monkeypatch.setenv("OMBRE_MCP_TOKEN", MCP_TOKEN) + monkeypatch.setattr( + shared_web, + "config", + {"buckets_dir": str(tmp_path), "mcp_token": MCP_TOKEN}, + raising=False, + ) + + +def test_mcp_static_token_authenticates_read_routes(monkeypatch, mcp_token): + assert shared_web._is_service_token_authenticated( + _request("GET", "/api/bucket/feel-1", f"Bearer {MCP_TOKEN}") + ) + assert not shared_web._is_service_token_authenticated( + _request("GET", "/api/bucket/feel-1", "Bearer wrong") + ) + # service token env is unset; should not break + assert not shared_web._is_service_token_authenticated( + _request("GET", "/api/bucket/feel-1", f"Bearer {SERVICE_TOKEN}") + ) + + +@pytest.mark.asyncio +async def test_mcp_token_reads_exact_bucket_detail(monkeypatch, mcp_token): + monkeypatch.setattr( + buckets_web.sh, "bucket_mgr", FakeBucketManager(), raising=False + ) + monkeypatch.setattr( + buckets_web.sh, "decay_engine", FakeDecayEngine(), raising=False + ) + mcp = FakeMCP() + buckets_web.register(mcp) + + response = await mcp.routes[("GET", "/api/bucket/{bucket_id}")]( + _request("GET", "/api/bucket/feel-1", f"Bearer {MCP_TOKEN}") + ) + payload = json.loads(response.body) + + assert response.status_code == 200 + assert payload["metadata"]["type"] == "feel" + assert payload["content"] == "raw [[memory]]" + + +def test_no_token_configured_rejects_all(monkeypatch, tmp_path): + monkeypatch.delenv("OMBRE_MCP_SERVICE_TOKEN", raising=False) + monkeypatch.delenv("OMBRE_MCP_TOKEN", raising=False) + monkeypatch.setattr( + shared_web, + "config", + {"buckets_dir": str(tmp_path)}, + raising=False, + ) + assert not shared_web._is_service_token_authenticated( + _request("GET", "/api/bucket/feel-1", "Bearer anything") + ) \ No newline at end of file From c41b0143a8a68fb5ebbaa2863d9cbbc777a37377 Mon Sep 17 00:00:00 2001 From: thefloshibisci <103315611+thefloshibisci@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:51:46 +0800 Subject: [PATCH 05/16] Refresh update manifest for MCP token auth fix --- update_manifest.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/update_manifest.json b/update_manifest.json index 4810936456..77c0a05cfb 100644 --- a/update_manifest.json +++ b/update_manifest.json @@ -927,8 +927,8 @@ }, { "path": "src/web/_shared.py", - "sha256": "f5ef517fad308c7b1305797fe22254f12a26942ed7bc2a0c5bd0009650ea1f0f", - "size": 43542 + "sha256": "d6b8d71d60e85c983a6b4bb885c59b15834d1e5e4b618831e82491712d1a7ce4", + "size": 44509 }, { "path": "src/web/auth.py", From b02c8c282f897e1c6e51b06ab68953e973709d7d Mon Sep 17 00:00:00 2001 From: thefloshibisci <103315611+thefloshibisci@users.noreply.github.com> Date: Mon, 24 Aug 2026 18:01:12 +0800 Subject: [PATCH 06/16] fix: lower min length for MCP tokens to accept short tokens The production MCP token is 10 chars, but _MIN_SERVICE_TOKEN_LENGTH=32 was used for both service tokens and MCP tokens, causing the 10-char token to be rejected with 401 on HTTP API routes like /api/bucket/{id}. - Add _MIN_MCP_TOKEN_LENGTH=8 (MCP tokens are validated by the MCP server itself, lower bar is acceptable) - Service tokens still require _MIN_SERVICE_TOKEN_LENGTH=32 (unchanged) - Add regression tests for short (10-char) MCP tokens --- src/web/_shared.py | 3 +- tests/test_dashboard_bucket_service_auth.py | 50 ++++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/src/web/_shared.py b/src/web/_shared.py index 724e87cf13..0634ce4b4b 100644 --- a/src/web/_shared.py +++ b/src/web/_shared.py @@ -384,6 +384,7 @@ def _write_env_var(name: str, value: str) -> None: _SESSION_TTL_SECONDS = 86400 * _DEFAULT_SESSION_TTL_DAYS _SESSION_TTL = _SESSION_TTL_SECONDS # compatibility constant for older callers _MIN_SERVICE_TOKEN_LENGTH = 32 +_MIN_MCP_TOKEN_LENGTH = 8 # MCP tokens are validated by the MCP server itself; lower bar OK def _session_ttl_seconds() -> int: @@ -1119,7 +1120,7 @@ def _is_service_token_authenticated(request: Request) -> bool: str(os.environ.get("OMBRE_MCP_TOKEN", "") or "").strip() or str(config.get("mcp_token", "") or "").strip() ) - if len(mcp_token) >= _MIN_SERVICE_TOKEN_LENGTH: + if len(mcp_token) >= _MIN_MCP_TOKEN_LENGTH: configured_tokens.append(mcp_token) if not configured_tokens: diff --git a/tests/test_dashboard_bucket_service_auth.py b/tests/test_dashboard_bucket_service_auth.py index 7fb861f95c..7db1988c15 100644 --- a/tests/test_dashboard_bucket_service_auth.py +++ b/tests/test_dashboard_bucket_service_auth.py @@ -193,4 +193,52 @@ def test_no_token_configured_rejects_all(monkeypatch, tmp_path): ) assert not shared_web._is_service_token_authenticated( _request("GET", "/api/bucket/feel-1", "Bearer anything") - ) \ No newline at end of file + ) + + +SHORT_MCP_TOKEN = "yanjin13" # 10 chars, mirrors the real production token length + + +@pytest.fixture +def short_mcp_token(monkeypatch, tmp_path): + """Configure only a short (10-char) MCP static token, no service token.""" + monkeypatch.delenv("OMBRE_MCP_SERVICE_TOKEN", raising=False) + monkeypatch.setenv("OMBRE_MCP_TOKEN", SHORT_MCP_TOKEN) + monkeypatch.setattr( + shared_web, + "config", + {"buckets_dir": str(tmp_path), "mcp_token": SHORT_MCP_TOKEN}, + raising=False, + ) + + +def test_short_mcp_token_authenticates_read_routes(monkeypatch, short_mcp_token): + """A 10-char MCP token should pass auth — this was the original bug.""" + assert shared_web._is_service_token_authenticated( + _request("GET", "/api/bucket/feel-1", f"Bearer {SHORT_MCP_TOKEN}") + ) + assert not shared_web._is_service_token_authenticated( + _request("GET", "/api/bucket/feel-1", "Bearer wrong") + ) + + +@pytest.mark.asyncio +async def test_short_mcp_token_reads_exact_bucket_detail(monkeypatch, short_mcp_token): + """End-to-end: short MCP token can fetch bucket detail via the API route.""" + monkeypatch.setattr( + buckets_web.sh, "bucket_mgr", FakeBucketManager(), raising=False + ) + monkeypatch.setattr( + buckets_web.sh, "decay_engine", FakeDecayEngine(), raising=False + ) + mcp = FakeMCP() + buckets_web.register(mcp) + + response = await mcp.routes[("GET", "/api/bucket/{bucket_id}")]( + _request("GET", "/api/bucket/feel-1", f"Bearer {SHORT_MCP_TOKEN}") + ) + payload = json.loads(response.body) + + assert response.status_code == 200 + assert payload["metadata"]["type"] == "feel" + assert payload["content"] == "raw [[memory]]" From e215f72d9e87532bd18731e3b9384bf091ae7419 Mon Sep 17 00:00:00 2001 From: thefloshibisci <103315611+thefloshibisci@users.noreply.github.com> Date: Sun, 1 Nov 2026 23:15:51 +0800 Subject: [PATCH 07/16] Restrict upstream Docker Hub publishing to upstream repository --- .github/workflows/docker-publish.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 74b5c47582..97d79577c2 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -20,6 +20,8 @@ on: jobs: build-and-push: + # This workflow publishes the upstream owner's Docker Hub image. + if: github.repository == 'P0luz/Ombre-Brain' runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 From 262dbf5b3f462fa941e75533cf2118d17d8e5d1b Mon Sep 17 00:00:00 2001 From: thefloshibisci <103315611+thefloshibisci@users.noreply.github.com> Date: Mon, 2 Nov 2026 00:03:29 +0800 Subject: [PATCH 08/16] Fix audited MCP, JWT and certificate dependencies with bounded lock refresh --- .github/workflows/tests.yml | 8 +++++--- requirements-dev.lock.txt | 25 ++++++++++++++----------- requirements.lock.txt | 25 ++++++++++++++----------- requirements.txt | 8 ++++++-- 4 files changed, 39 insertions(+), 27 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0c20304785..65f54d2b99 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,7 +2,7 @@ name: Tests on: push: - branches: [main, testing] + branches: [main, testing, 'fix/security-*'] paths-ignore: - '*.md' - 'backup_*/**' @@ -39,8 +39,10 @@ jobs: UV_EXCLUDE_NEWER: '2026-08-01T00:00:00Z' run: | rm -f requirements.lock.txt requirements-dev.lock.txt - uv pip compile requirements.txt --universal --python-version 3.12 --generate-hashes --output-file requirements.lock.txt - uv pip compile requirements-dev.in --universal --python-version 3.12 --generate-hashes --output-file requirements-dev.lock.txt + # Advance only these security-sensitive packages, preserving the + # original snapshot for all unrelated production and test packages. + uv pip compile requirements.txt --universal --python-version 3.12 --generate-hashes --exclude-newer-package mcp=2026-11-01T00:00:00Z --exclude-newer-package certifi=2026-11-01T00:00:00Z --exclude-newer-package pyjwt=2026-11-01T00:00:00Z --output-file requirements.lock.txt + uv pip compile requirements-dev.in --universal --python-version 3.12 --generate-hashes --exclude-newer-package mcp=2026-11-01T00:00:00Z --exclude-newer-package certifi=2026-11-01T00:00:00Z --exclude-newer-package pyjwt=2026-11-01T00:00:00Z --output-file requirements-dev.lock.txt git diff --exit-code -- requirements.lock.txt requirements-dev.lock.txt - name: Install locked dependencies diff --git a/requirements-dev.lock.txt b/requirements-dev.lock.txt index 060c1ff0db..8e4edc50f0 100644 --- a/requirements-dev.lock.txt +++ b/requirements-dev.lock.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile requirements-dev.in --universal --python-version 3.12 --generate-hashes --output-file requirements-dev.lock.txt +# uv pip compile requirements-dev.in --universal --python-version 3.12 --generate-hashes --exclude-newer-package mcp=2026-11-01T00:00:00Z --exclude-newer-package certifi=2026-11-01T00:00:00Z --exclude-newer-package pyjwt=2026-11-01T00:00:00Z --output-file requirements-dev.lock.txt annotated-types==0.8.0 \ --hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \ --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 @@ -35,10 +35,11 @@ cachecontrol==0.14.4 \ --hash=sha256:b7ac014ff72ee199b5f8af1de29d60239954f223e948196fa3d84adaffc71d2b \ --hash=sha256:e6220afafa4c22a47dd0badb319f84475d79108100d04e26e8542ef7d3ab05a1 # via pip-audit -certifi==2026.7.22 \ - --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ - --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 +certifi==2026.10.23 \ + --hash=sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd \ + --hash=sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1 # via + # -r requirements.txt # httpcore # httpx # requests @@ -735,9 +736,9 @@ markdown-it-py==4.2.0 \ --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a # via rich -mcp==1.29.0 \ - --hash=sha256:52d01f334de1868cc3bb2d6604931126a67631f99a6c5d3b82ba47290315ec36 \ - --hash=sha256:f5a075bb611f23d6f4d080c6a1699fa62772eebc562ba9e66b306ddde1c755f7 +mcp==1.30.0 \ + --hash=sha256:445414625fce5c295faa505bb11bacece661ab6f4028d57c935db57820b7a3e4 \ + --hash=sha256:666edb5009503e1047c9d60346a756f94b261f05cc2625f23d41c728ffc484d0 # via -r requirements.txt mdurl==0.1.2 \ --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ @@ -1058,10 +1059,12 @@ pygments==2.20.0 \ # via # pytest # rich -pyjwt==2.13.0 \ - --hash=sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423 \ - --hash=sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728 - # via mcp +pyjwt==2.15.1 \ + --hash=sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193 \ + --hash=sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8 + # via + # -r requirements.txt + # mcp pyparsing==3.3.2 \ --hash=sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d \ --hash=sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc diff --git a/requirements.lock.txt b/requirements.lock.txt index f5d96e6fe5..615b7dd1ff 100644 --- a/requirements.lock.txt +++ b/requirements.lock.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile requirements.txt --universal --python-version 3.12 --generate-hashes --output-file requirements.lock.txt +# uv pip compile requirements.txt --universal --python-version 3.12 --generate-hashes --exclude-newer-package mcp=2026-11-01T00:00:00Z --exclude-newer-package certifi=2026-11-01T00:00:00Z --exclude-newer-package pyjwt=2026-11-01T00:00:00Z --output-file requirements.lock.txt annotated-types==0.8.0 \ --hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \ --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 @@ -19,10 +19,11 @@ attrs==26.1.0 \ # via # jsonschema # referencing -certifi==2026.7.22 \ - --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ - --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 +certifi==2026.10.23 \ + --hash=sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd \ + --hash=sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1 # via + # -r requirements.txt # httpcore # httpx cffi==2.1.0 ; platform_python_implementation != 'PyPy' \ @@ -338,9 +339,9 @@ jsonschema-specifications==2025.9.1 \ --hash=sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe \ --hash=sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d # via jsonschema -mcp==1.29.0 \ - --hash=sha256:52d01f334de1868cc3bb2d6604931126a67631f99a6c5d3b82ba47290315ec36 \ - --hash=sha256:f5a075bb611f23d6f4d080c6a1699fa62772eebc562ba9e66b306ddde1c755f7 +mcp==1.30.0 \ + --hash=sha256:445414625fce5c295faa505bb11bacece661ab6f4028d57c935db57820b7a3e4 \ + --hash=sha256:666edb5009503e1047c9d60346a756f94b261f05cc2625f23d41c728ffc484d0 # via -r requirements.txt narwhals==2.24.0 \ --hash=sha256:42fdedf44e5b2ca7505630d45b4ac3058f38d8485cba9fe1652ca23152df7489 \ @@ -537,10 +538,12 @@ pydantic-settings==2.14.2 \ --hash=sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440 \ --hash=sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f # via mcp -pyjwt==2.13.0 \ - --hash=sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423 \ - --hash=sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728 - # via mcp +pyjwt==2.15.1 \ + --hash=sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193 \ + --hash=sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8 + # via + # -r requirements.txt + # mcp python-dotenv==1.2.2 \ --hash=sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a \ --hash=sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3 diff --git a/requirements.txt b/requirements.txt index 178c4b0b12..dbce762f3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,9 +3,13 @@ # 安装: pip install -r requirements.txt # ============================================================ -# MCP 协议 SDK(MCP 通信核心)。生产锁定 1.28.1;限制在 v1, +# MCP 协议 SDK(MCP 通信核心)。安全修复最低 1.30.0;限制在 v1, # 避免未来 v2 的破坏性 API 变更越过锁文件安装进来。 -mcp>=1.27,<2 +mcp>=1.30.0,<2 + +# 2026-11 安全基线:更新证书包及 JWT 校验修复,锁文件固定实际版本。 +certifi>=2026.10.18 +PyJWT>=2.15.1,<3 # 模糊匹配(记忆桶搜索) rapidfuzz>=3.0.0 From 30cc344a79591bdbced20fd994383012e4dd16c6 Mon Sep 17 00:00:00 2001 From: thefloshibisci <103315611+thefloshibisci@users.noreply.github.com> Date: Mon, 2 Nov 2026 00:06:23 +0800 Subject: [PATCH 09/16] Require explicit dependency migration and verify the new security lock --- .gitattributes | 11 +++-------- docs/DEPENDENCY_SECURITY_20261102.md | 22 ++++++++++++++++++++++ tests/test_update_source_gate.py | 23 ++++++++++++++++++----- 3 files changed, 43 insertions(+), 13 deletions(-) create mode 100644 docs/DEPENDENCY_SECURITY_20261102.md diff --git a/.gitattributes b/.gitattributes index e55ccf2c89..98a31e880a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,11 +4,6 @@ *.sh text eol=lf entrypoint.sh text eol=lf -# Dashboard 热更新使用 GitHub 自动生成的源码归档。Git clone 仍保留可编辑的 -# 约束文件,但归档只携带带 hash 的发布锁,避免旧更新器把源约束文字调整误判成 -# 已安装依赖发生变化。 -# 此兼容仅在 requirements.lock.txt 规范化 SHA-256 保持为下列值时安全: -# fdb24053349d8e18a55c3a5afbab8b92cc31d94b69e52359b350ab96b79001c9。 -# 修改发布锁前必须移除此规则并提供显式依赖迁移; -# tests/test_update_source_gate.py 会强制校验该不变式。 -/requirements.txt export-ignore +# Security dependency upgrades require an explicit image rebuild or locked pip +# install. Include both dependency manifests in archives so legacy updaters do +# not mistake this release for a code-only update. See docs/DEPENDENCY_SECURITY_20261102.md. diff --git a/docs/DEPENDENCY_SECURITY_20261102.md b/docs/DEPENDENCY_SECURITY_20261102.md new file mode 100644 index 0000000000..e7651f7076 --- /dev/null +++ b/docs/DEPENDENCY_SECURITY_20261102.md @@ -0,0 +1,22 @@ +# 2026-11 dependency security update + +This fork updates MCP 1.29.0 to 1.30.0, PyJWT 2.13.0 to 2.15.1 and certifi +2026.7.22 to 2026.10.23. Other resolved package versions remain unchanged. +The production and development locks include package hashes. Their resolution +keeps the original August index snapshot, advancing only these three packages +to the fixed November 1 UTC snapshot recorded in CI. + +This requires a dependency installation, not only a hot code update. For Docker +or Zeabur, rebuild the image from the updated repository and recreate the service +with its existing persistent volumes and environment. For an existing virtual +environment, stop the service, run `python -m pip install --require-hashes -r +requirements.lock.txt`, verify `python -m pip check`, then restart it. Preserve a +current data backup and the previous deployment configuration before switching. + +The source archive now includes both requirements.txt and requirements.lock.txt. +Older updaters must detect the dependency change instead of treating this archive +as code-only. Automatic pip installation remains disabled by default; do not +enable it merely to bypass the rebuild requirement. Verify installed versions +inside the running service, then check authentication, MCP initialization and +dependent clients. A source-code version string alone does not prove this fix is +installed. Security auditing remains enabled without ignore rules. diff --git a/tests/test_update_source_gate.py b/tests/test_update_source_gate.py index baa22d2a8a..055773f377 100644 --- a/tests/test_update_source_gate.py +++ b/tests/test_update_source_gate.py @@ -98,15 +98,23 @@ def test_ci_lock_verification_freezes_package_index_snapshot(): assert step.count("uv pip compile ") == 2 assert "--upgrade" not in step - assert "--exclude-newer" not in step, ( + assert "--exclude-newer " not in step, ( "cutoff 必须通过环境变量传入,避免 uv 把参数写进 lock 头部造成纯文本漂移" ) + overrides = re.findall(r"--exclude-newer-package ([a-z]+)=([^\s]+)", step) + assert {package for package, _ in overrides} == {"mcp", "certifi", "pyjwt"} + assert len(overrides) == 6, "both locks must use the same three package snapshots" + for package in {"mcp", "certifi", "pyjwt"}: + dates = [date for name, date in overrides if name == package] + assert len(set(dates)) == 1 + assert dates[0].endswith("Z") + assert cutoff <= datetime.fromisoformat(dates[0][:-1] + "+00:00") <= datetime.now(timezone.utc) reset_command = "rm -f requirements.lock.txt requirements-dev.lock.txt" assert reset_command in step, "lock 校验必须从空输出重建,不能依赖已有 pin 偏好" assert step.index(reset_command) < step.index("uv pip compile ") -def test_release_archive_omits_loose_requirements_but_keeps_lock(): +def test_security_release_archive_includes_both_dependency_manifests(): repo_root = Path(meta.__file__).resolve().parents[2] attributes = (repo_root / ".gitattributes").read_text(encoding="utf-8") active_rules = { @@ -115,7 +123,7 @@ def test_release_archive_omits_loose_requirements_but_keeps_lock(): if line.strip() and not line.lstrip().startswith("#") } - assert "/requirements.txt export-ignore" in active_rules + assert "/requirements.txt export-ignore" not in active_rules assert "/requirements.lock.txt export-ignore" not in active_rules assert "COPY requirements.lock.txt ./" in (repo_root / "Dockerfile").read_text( encoding="utf-8" @@ -125,6 +133,11 @@ def test_release_archive_omits_loose_requirements_but_keeps_lock(): def test_legacy_archive_compatibility_requires_284_release_lock(): """requirements.lock.txt 内容钉死的绊线:任何改动都必须是一次清醒决定,不能顺手改掉。 + 2026-11-02:明确升级 mcp / pyjwt / certifi 三项安全依赖;其余包版本不变。 + 删除 requirements.txt 的 export-ignore,使旧更新器也能检测到依赖变化。 + 本次必须重建镜像或显式安装 hash 锁,不允许把代码热更新当作修复完成。 + 迁移步骤与实际版本验证记录在 docs/DEPENDENCY_SECURITY_20261102.md。 + 2026-08-09:为修复 cryptography==49.0.0 的 PYSEC-2026-3552(无 fix 版本可选, 只能升级到 50.0.0),推进了 CI 的 UV_EXCLUDE_NEWER 快照并重新生成了两份锁, 连带升级了 mcp/openai/uvicorn 等包。这是一次明确接受的破坏性变更:仍在跑 @@ -151,7 +164,7 @@ def test_legacy_archive_compatibility_requires_284_release_lock(): normalized = lock_bytes.replace(b"\r\n", b"\n").replace(b"\r", b"\n") assert hashlib.sha256(normalized).hexdigest() == ( - "ae0255bd41e5ceb45694244017a9b0b2dc1e4c445ab027cec1f7183ad969700a" + "57d16c50026f3526ba9ecdf59ea33ccb65164e2cff48cb50acf5688ce99e6e52" ), ( "requirements.lock.txt 已变化:确认这次变化是否会影响还没升级过的旧版" "热更新器(尤其 v2.8.4 之前、缺少 lock 感知回退逻辑的实例),评估后再把" @@ -303,7 +316,7 @@ def fake_run(command, **_kwargs): lock = (repo_root / "requirements.lock.txt").read_bytes() assert meta._runtime_satisfies_locked_versions(lock) is True - assert b"mcp==1.29.0" in captured["content"] + assert b"mcp==1.30.0" in captured["content"] def test_runtime_lock_probe_nonzero_result_fails_closed_and_cleans_temp( From 1427939762c1437718461fbde3999e086306a06d Mon Sep 17 00:00:00 2001 From: thefloshibisci <103315611+thefloshibisci@users.noreply.github.com> Date: Mon, 2 Nov 2026 00:10:00 +0800 Subject: [PATCH 10/16] Refresh persisted dependency baseline when only the image lock changes --- src/ombrebrain/maintenance/code_fingerprint.py | 4 ++++ tests/test_code_fingerprint.py | 13 +++++++++++++ tests/test_entrypoint_code_bootstrap.py | 15 +++++++++++++++ update_manifest.json | 4 ++-- 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/ombrebrain/maintenance/code_fingerprint.py b/src/ombrebrain/maintenance/code_fingerprint.py index ebbda0cf5d..4344203576 100644 --- a/src/ombrebrain/maintenance/code_fingerprint.py +++ b/src/ombrebrain/maintenance/code_fingerprint.py @@ -11,6 +11,9 @@ # 播种,运行时目录永远停留在第一次播种时的版本(诊断读的正是运行时目录)。 # 与 _CODE_DIRS 的区别是「缺失不算错」:老镜像和老运行时都没有这几个目录。 _OPTIONAL_CODE_DIRS = ("docs", "tools", "kernel") +# A dependency-only image rebuild must refresh the persisted lock baseline too. +# Older images can lack a lockfile, so its absence remains supported. +_OPTIONAL_ROOT_FILES = ("requirements.lock.txt",) _IGNORED_SUFFIXES = {".pyc", ".pyo"} @@ -28,6 +31,7 @@ def _code_files(root: Path) -> list[Path]: continue if path.is_file(): files.append(path) + files.extend(root / name for name in _OPTIONAL_ROOT_FILES if (root / name).is_file()) return sorted(files, key=lambda path: path.relative_to(root).as_posix()) diff --git a/tests/test_code_fingerprint.py b/tests/test_code_fingerprint.py index c88f18c7d1..3ad1b43bfa 100644 --- a/tests/test_code_fingerprint.py +++ b/tests/test_code_fingerprint.py @@ -3,6 +3,19 @@ from ombrebrain.maintenance.code_fingerprint import fingerprint_code_tree +def test_dependency_only_change_updates_image_fingerprint(tmp_path): + (tmp_path / "src").mkdir() + (tmp_path / "frontend").mkdir() + (tmp_path / "src" / "server.py").write_text("pass\n", encoding="utf-8") + legacy = fingerprint_code_tree(tmp_path) + lock = tmp_path / "requirements.lock.txt" + lock.write_text("mcp==1.29.0\n", encoding="utf-8") + before = fingerprint_code_tree(tmp_path) + lock.write_text("mcp==1.30.0\n", encoding="utf-8") + assert before != legacy + assert fingerprint_code_tree(tmp_path) != before + + def _tree(root: Path) -> None: (root / "src").mkdir(parents=True) (root / "frontend").mkdir() diff --git a/tests/test_entrypoint_code_bootstrap.py b/tests/test_entrypoint_code_bootstrap.py index b0a5ff9adb..8ccc68d38c 100644 --- a/tests/test_entrypoint_code_bootstrap.py +++ b/tests/test_entrypoint_code_bootstrap.py @@ -85,6 +85,21 @@ def test_same_version_changed_image_reseeds_but_unchanged_image_preserves_hot_up assert "代码指纹" in rebuilt.stdout +def test_dependency_only_rebuild_refreshes_persisted_lock_at_same_version(tmp_path): + image = tmp_path / "image" + code = tmp_path / "code" / "_app" + data = tmp_path / "data" + _prepare_image(image) + first = _run(image, code, data) + assert first.returncode == 0, first.stdout + first.stderr + (image / "requirements.lock.txt").write_text("package==2\n", encoding="utf-8") + rebuilt = _run(image, code, data) + assert rebuilt.returncode == 0, rebuilt.stdout + rebuilt.stderr + assert (code / "requirements.lock.txt").read_text(encoding="utf-8") == "package==2\n" + assert (code / "_prev" / "requirements.lock.txt").read_text(encoding="utf-8") == "package==1\n" + assert "reason=image-fingerprint-changed" in rebuilt.stdout + + def test_non_active_legacy_data_app_is_reported_without_deletion(tmp_path): image = tmp_path / "image" code = tmp_path / "code" / "_app" diff --git a/update_manifest.json b/update_manifest.json index fa8591ed13..2466ff6a7c 100644 --- a/update_manifest.json +++ b/update_manifest.json @@ -217,8 +217,8 @@ }, { "path": "src/ombrebrain/maintenance/code_fingerprint.py", - "sha256": "72f36584c96042736a66ee92c1f5d9ed95d6fbf960917fed51ad68dd8f199a26", - "size": 2627 + "sha256": "2b013896aae7a6bfc62684db0d3c87247063f25f8dd8befa2575e89cd58a63d7", + "size": 2917 }, { "path": "src/ombrebrain/maintenance/migration_contract.py", From 4ad9525028366fdeef1cd9ac5cc04ce70835ea14 Mon Sep 17 00:00:00 2001 From: thefloshibisci <103315611+thefloshibisci@users.noreply.github.com> Date: Mon, 2 Nov 2026 20:55:05 +0800 Subject: [PATCH 11/16] Deploy tested main images to Tencent with scoped SSH and isolated preflight --- .github/NANZHI_DEPLOY.md | 18 +++ .github/nanzhi-known-hosts | 1 + .github/ops/nanzhi_receiver.py | 269 +++++++++++++++++++++++++++++++ .github/scripts/nanzhi_deploy.py | 57 +++++++ .github/workflows/tests.yml | 44 ++++- 5 files changed, 388 insertions(+), 1 deletion(-) create mode 100644 .github/NANZHI_DEPLOY.md create mode 100644 .github/nanzhi-known-hosts create mode 100644 .github/ops/nanzhi_receiver.py create mode 100644 .github/scripts/nanzhi_deploy.py diff --git a/.github/NANZHI_DEPLOY.md b/.github/NANZHI_DEPLOY.md new file mode 100644 index 0000000000..a7fd7b34da --- /dev/null +++ b/.github/NANZHI_DEPLOY.md @@ -0,0 +1,18 @@ +# Tencent production deployment + +Only `main` push or a manual workflow run on `main` can deploy. Pull requests and other branches never receive the deployment job. The deployment job depends on the existing complete test jobs and transfers the exact image built in that run; it does not rebuild on the server. + +The client skips a queued commit if `main` has advanced. SSH checks the committed host public key. Each repository has a separate `NANZHI_DEPLOY_KEY` secret, forced to the matching service; it cannot run a shell, forward ports, or deploy another service. Application tokens and data remain on the server. + +The root-owned receiver on the server verifies archive digest, size, platform, tag, and revision label. It serializes updates, makes isolated data copies with SQLite online backups, and verifies health, invalid-token rejection, authenticated MCP initialization and tool discovery with external network access disabled. Failure at this stage leaves the running production service unchanged. + +After rehearsal succeeds, it briefly enables a maintenance gate, stops only the selected service, archives its persistent data and saves the previous Compose configuration, then starts the tested image and repeats authentication checks. Success removes the maintenance gate. If failure happens after production is stopped or its data may have been migrated, the maintenance gate stays in place and the GitHub job fails. A maintainer must inspect the receipt and backup: automatic database downgrade could lose or corrupt new data, so it is deliberately not attempted. + +Server paths (root only): + +- `/usr/local/sbin/nanzhi-deploy`: deployment receiver. +- `/opt/nanzhi-deploy/runs/`: per-attempt receipts, previous configuration and pre-update data archive. +- `/opt/nanzhi-production/compose.json`: production services and private environment. +- `/var/lib/nanzhi-deploy-gates/`: persistent maintenance flags; remove only after verifying recovery. + +Other services are manual. Old Zeabur Git triggers are disabled. Do not restart old Zeabur applications; legacy domains temporarily relay to the new server. Never print or commit production configurations, deployment private keys or data backups. diff --git a/.github/nanzhi-known-hosts b/.github/nanzhi-known-hosts new file mode 100644 index 0000000000..edb7c329e6 --- /dev/null +++ b/.github/nanzhi-known-hosts @@ -0,0 +1 @@ +49.234.39.128 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKC70qOHpCT9KFi4Syv2MGEtLflUgaI+8JtAz0hWuNhj diff --git a/.github/ops/nanzhi_receiver.py b/.github/ops/nanzhi_receiver.py new file mode 100644 index 0000000000..7c0b68931f --- /dev/null +++ b/.github/ops/nanzhi_receiver.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +"""Forced SSH receiver. Each key is bound to one service by authorized_keys.""" +import copy +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import sqlite3 +import subprocess +import sys +import tarfile +import time + +BASE = Path('/opt/nanzhi-deploy') +PRODUCTION = Path('/opt/nanzhi-production') +COMPOSE = PRODUCTION / 'compose.json' +SERVICES = {'ombre-brain': 18001, 'xinchao-nian': 18110} +MAX_ARCHIVE = 2 * 1024**3 + + +def call(args, timeout=180, **kwargs): + result = subprocess.run(args, capture_output=True, timeout=timeout, **kwargs) + if result.returncode: + # Commands may contain service configuration; never print command/env/logs. + raise RuntimeError('subprocess_failed_' + Path(args[0]).name) + return result.stdout + + +def metadata(stream): + line = stream.readline(1025) + if len(line) > 1024 or not line.endswith(b'\n'): + raise ValueError('invalid_header_length') + item = json.loads(line) + if item == {'action': 'check'}: + return item + if set(item) != {'action', 'commit', 'bytes', 'sha256'} or item['action'] != 'deploy': + raise ValueError('invalid_action') + if not re.fullmatch('[0-9a-f]{40}', item['commit']): + raise ValueError('invalid_commit') + if not re.fullmatch('[0-9a-f]{64}', item['sha256']): + raise ValueError('invalid_digest') + if type(item['bytes']) is not int or not 0 < item['bytes'] <= MAX_ARCHIVE: + raise ValueError('invalid_size') + return item + + +def receive(stream, dest, item): + remaining = item['bytes'] + digest = hashlib.sha256() + with dest.open('xb') as output: + while remaining: + block = stream.read(min(1024 * 1024, remaining)) + if not block: + raise ValueError('truncated_archive') + digest.update(block) + output.write(block) + remaining -= len(block) + if stream.read(1) or digest.hexdigest() != item['sha256']: + raise ValueError('archive_integrity_failed') + + +def snapshot(source, dest): + """Consistent individual SQLite copies, isolated rehearsal only.""" + shutil.copytree(source, dest, symlinks=True) + for path in source.rglob('*'): + if not path.is_file() or path.is_symlink(): + continue + with path.open('rb') as f: + is_db = f.read(16) == b'SQLite format 3\0' + if not is_db: + continue + target = dest / path.relative_to(source) + target.unlink() + src = sqlite3.connect(path.as_uri() + '?mode=ro', uri=True, timeout=10) + dst = sqlite3.connect(target) + deadline = time.monotonic() + 60 + def progress(*_): + if time.monotonic() > deadline: + raise TimeoutError('sqlite_backup_timeout') + try: + src.backup(dst, pages=256, progress=progress) + if dst.execute('PRAGMA quick_check').fetchall() != [('ok',)]: + raise ValueError('sqlite_copy_invalid') + finally: + dst.close() + src.close() + for suffix in ('-wal', '-shm', '-journal'): + sidecar = Path(str(target) + suffix) + if sidecar.exists(): + sidecar.unlink() + shutil.copystat(path, target) + for d, dirs, files in os.walk(dest): + original = source / Path(d).relative_to(dest) + stat = original.stat() + os.chown(d, stat.st_uid, stat.st_gid) + for name in files: + old = original / name + stat = old.lstat() + os.chown(Path(d) / name, stat.st_uid, stat.st_gid, follow_symlinks=False) + + +def container_check(compose, service): + cmd = ['docker', 'compose', '-f', str(compose)] + cid = call(cmd + ['ps', '-q', service]).decode().strip() + info = json.loads(call(['docker', 'inspect', cid]))[0] + if not info['State']['Running']: + raise RuntimeError('container_not_running') + env = dict(v.split('=', 1) for v in info['Config']['Env']) + doc = json.loads(COMPOSE.read_text()) + token = env.get('SERVICE_TOKEN') if service == 'xinchao-nian' else doc['services']['xinchao-nian']['environment']['OMBRE_MCP_TOKEN'].replace('$$', '$') + worker = r'''import json,sys,urllib.request,urllib.error +x=json.load(sys.stdin) +def req(path,token=None,body=None,headers=None): + h={'Accept':'application/json, text/event-stream','Content-Type':'application/json'} + if token:h['Authorization']='Bearer '+token + h.update(headers or {}) + r=urllib.request.Request(x['url']+path,headers=h,data=json.dumps(body).encode() if body is not None else None) + try:res=urllib.request.urlopen(r,timeout=8) + except urllib.error.HTTPError as e:res=e + return res.status,res.read(2000000).decode(),dict(res.headers) +assert req('/health')[0]==200 +b={'jsonrpc':'2.0','id':1,'method':'initialize','params':{'protocolVersion':'2025-06-18','capabilities':{},'clientInfo':{'name':'deployment-check','version':'1'}}} +assert req('/mcp','invalid-deployment-test',b)[0]==401 +status,body,headers=req('/mcp',x['token'],b) +assert status==200 and 'protocolVersion' in body +h={'MCP-Protocol-Version':'2025-06-18'} +for k,v in headers.items(): + if k.lower()=='mcp-session-id':h['Mcp-Session-Id']=v +req('/mcp',x['token'],{'jsonrpc':'2.0','method':'notifications/initialized'},h) +status,body,_=req('/mcp',x['token'],{'jsonrpc':'2.0','id':2,'method':'tools/list','params':{}},h) +assert status==200 and '"tools"' in body +print('health_auth_mcp_ok') +''' + payload = json.dumps({'url': 'http://127.0.0.1:' + str(SERVICES[service]), 'token': token}).encode() + call(['nsenter', '-t', str(info['State']['Pid']), '-n', 'python3', '-c', worker], input=payload, timeout=45) + + +def wait_ready(compose, service, seconds=120): + deadline = time.monotonic() + seconds + while True: + try: + container_check(compose, service) + return + except Exception: + if time.monotonic() >= deadline: + raise RuntimeError('health_auth_mcp_check_failed') from None + time.sleep(4) + + +def rehearsal(doc, service, image, job): + svc = copy.deepcopy(doc['services'][service]) + svc.update(image=image, restart='no', ports=[], networks=['isolated']) + svc.pop('depends_on', None) + for key in ('DREAM_ENABLED', 'MODEL_ENABLED', 'BARK_ENABLED', 'BRIDGE_ENABLED', 'OMBRE_WRITE_ENABLED'): + if key in svc['environment']: + svc['environment'][key] = 'false' + for index, mount in enumerate(svc['volumes']): + source = Path(mount['source']).resolve() + if not source.is_relative_to(PRODUCTION): + raise ValueError('unexpected_volume') + if mount.get('read_only'): + continue + dest = job / ('test-data-' + str(index)) + snapshot(source, dest) + mount['source'] = str(dest) + test = {'name': 'nanzhi-deploy-check', 'services': {service: svc}, 'networks': {'isolated': {'internal': True}}} + path = job / 'test-compose.json' + path.write_text(json.dumps(test)) + cmd = ['docker', 'compose', '-f', str(path)] + try: + call(cmd + ['up', '-d', '--pull', 'never']) + wait_ready(path, service) + finally: + call(cmd + ['down', '--timeout', '60'], timeout=150) + + +def deploy(service, item, stream): + job = BASE / 'runs' / (time.strftime('%Y%m%dT%H%M%SZ', time.gmtime()) + '-' + service + '-' + item['commit'][:12]) + job.mkdir(mode=0o700) + status = {'service': service, 'commit': item['commit'], 'phase': 'receiving'} + def record(phase): + status['phase'] = phase + status['updated_utc'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) + (job / 'receipt.json').write_text(json.dumps(status, indent=2)) + maintenance = Path('/var/lib/nanzhi-deploy-gates') / (service + '.maintenance') + try: + if shutil.disk_usage(BASE).free < max(5 * 1024**3, item['bytes'] * 4): + raise RuntimeError('insufficient_disk_space') + archive = job / 'image.tar.gz' + receive(stream, archive, item) + image = 'nanzhi/' + service + ':github-' + item['commit'] + with tarfile.open(archive, 'r:gz') as tar: + manifest = json.load(tar.extractfile('manifest.json')) + if len(manifest) != 1 or manifest[0].get('RepoTags') != [image]: + raise ValueError('unexpected_image_manifest') + config = json.load(tar.extractfile(manifest[0]['Config'])) + if config.get('architecture') != 'amd64' or config.get('os') != 'linux': + raise ValueError('unexpected_image_platform') + if config.get('config', {}).get('Labels', {}).get('org.opencontainers.image.revision') != item['commit']: + raise ValueError('image_revision_mismatch') + call(['docker', 'load', '-i', str(archive)], timeout=300) + doc = json.loads(COMPOSE.read_text()) + if doc['services'][service]['image'] == image: + wait_ready(COMPOSE, service) + record('already_deployed') + return status + record('isolated_rehearsal') + rehearsal(doc, service, image, job) + record('preflight_passed') + old_bytes = COMPOSE.read_bytes() + (job / 'compose-before.json').write_bytes(old_bytes) + # Maintenance survives failures and reboot until a successful deployment. + maintenance.touch(mode=0o644) + cmd = ['docker', 'compose', '-f', str(COMPOSE)] + record('quiescing') + call(cmd + ['stop', '--timeout', '60', service], timeout=150) + sources = [] + for mount in doc['services'][service]['volumes']: + if not mount.get('read_only'): + source = Path(mount['source']).resolve() + if not source.is_relative_to(PRODUCTION / 'data'): + raise ValueError('unexpected_production_volume') + sources.append(str(source.relative_to(PRODUCTION))) + call(['tar', '-czf', str(job / 'data-before.tar.gz'), '-C', str(PRODUCTION), '--', *sources], timeout=180) + doc['services'][service]['image'] = image + updated = COMPOSE.with_suffix('.deploy-tmp') + updated.write_text(json.dumps(doc, indent=2)) + call(['docker', 'compose', '-f', str(updated), 'config', '--quiet']) + updated.replace(COMPOSE) + record('starting') + call(cmd + ['up', '-d', '--no-deps', '--pull', 'never', service]) + wait_ready(COMPOSE, service) + maintenance.unlink() + record('deployed') + # The compressed image is reproducible; retain backups and receipts. + archive.unlink() + return status + except Exception as exc: + status['error'] = type(exc).__name__ + ':' + str(exc)[:120] + record('failed_' + status['phase']) + # Do not blindly downgrade a database modified by a new app version. + # Before quiescing, production remains untouched. Afterwards keep the + # maintenance gate and backups for explicit recovery. + raise + + +def main(): + os.umask(0o077) + if len(sys.argv) != 2 or sys.argv[1] not in SERVICES: + raise ValueError('service_not_allowed') + service = sys.argv[1] + item = metadata(sys.stdin.buffer) + if item['action'] == 'check': + print(json.dumps({'ready': COMPOSE.is_file(), 'service': service, 'protocol': 1})) + return + with (BASE / 'deployment.lock').open('a') as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + print(json.dumps(deploy(service, item, sys.stdin.buffer))) + + +if __name__ == '__main__': + try: + main() + except Exception as exc: + print(json.dumps({'deployed': False, 'error': type(exc).__name__ + ':' + str(exc)[:120]})) + sys.exit(1) diff --git a/.github/scripts/nanzhi_deploy.py b/.github/scripts/nanzhi_deploy.py new file mode 100644 index 0000000000..ce9e9451b9 --- /dev/null +++ b/.github/scripts/nanzhi_deploy.py @@ -0,0 +1,57 @@ +"""Send the exact tested image to the service-scoped SSH deployment receiver.""" +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile + + +def main(): + service, commit, filename = sys.argv[1:] + if service not in ('ombre-brain', 'xinchao-nian') or not re.fullmatch('[0-9a-f]{40}', commit): + raise ValueError('Invalid deployment identity') + # A queued older workflow must never replace a newer main commit. + latest = subprocess.check_output( + ['gh', 'api', 'repos/' + os.environ['GITHUB_REPOSITORY'] + '/git/ref/heads/main', '--jq', '.object.sha'], + text=True, + ).strip() + if latest != commit: + print('Skipped: main advanced after this build.') + return + image = Path(filename) + with image.open('rb') as source: + digest = hashlib.file_digest(source, 'sha256').hexdigest() + header = {'action': 'deploy', 'commit': commit, 'bytes': image.stat().st_size, 'sha256': digest} + known_hosts = Path(__file__).resolve().parent.parent / 'nanzhi-known-hosts' + with tempfile.TemporaryDirectory(prefix='nanzhi-deploy-') as folder: + key = Path(folder) / 'key' + key.write_text(os.environ.pop('NANZHI_DEPLOY_KEY').strip() + '\n') + key.chmod(0o600) + cmd = [ + 'ssh', '-T', '-i', str(key), '-o', 'BatchMode=yes', + '-o', 'IdentitiesOnly=yes', '-o', 'StrictHostKeyChecking=yes', + '-o', 'UserKnownHostsFile=' + str(known_hosts), + '-o', 'ServerAliveInterval=30', '-o', 'ServerAliveCountMax=6', + '-o', 'ConnectTimeout=20', 'nanzhi-deploy@49.234.39.128', + ] + with subprocess.Popen(cmd, stdin=subprocess.PIPE) as proc: + try: + proc.stdin.write((json.dumps(header) + '\n').encode()) + with image.open('rb') as source: + while chunk := source.read(1024 * 1024): + proc.stdin.write(chunk) + proc.stdin.close() + status = proc.wait(timeout=1200) + except BaseException: + proc.kill() + proc.wait() + raise + if status: + raise SystemExit(status) + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 65f54d2b99..b439a2bb6e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,6 +1,7 @@ name: Tests on: + workflow_dispatch: push: branches: [main, testing, 'fix/security-*'] paths-ignore: @@ -105,7 +106,7 @@ jobs: docker run -d --name ob-ci-llm --network ob-ci-net \ -v "$PWD:/workspace:ro" -w /workspace python:3.12-slim \ python tests/mcp_llm_stub.py - docker build -t ombre-brain-ci:local . + docker build --label org.opencontainers.image.revision="$GITHUB_SHA" -t ombre-brain-ci:local . docker run -d --name ob-ci-app --network ob-ci-net \ -p 127.0.0.1:18084:8000 \ -v ob-ci-data:/app/buckets \ @@ -144,9 +145,50 @@ jobs: python -m pytest tests/test_web_api_docker_integration.py -q --timeout=120 + + - name: Export tested deployment image + if: github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + run: | + docker tag ombre-brain-ci:local nanzhi/ombre-brain:github-${{ github.sha }} + docker save nanzhi/ombre-brain:github-${{ github.sha }} | gzip -1 > image.tar.gz + + - name: Preserve tested image for deployment + if: github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + uses: actions/upload-artifact@v4 + with: + name: nanzhi-image-${{ github.sha }} + path: image.tar.gz + compression-level: 0 + retention-days: 3 + - name: Clean Docker MCP stack if: always() run: | docker rm -f ob-ci-app ob-ci-llm || true docker volume rm ob-ci-data || true docker network rm ob-ci-net || true + + deploy-tencent: + name: Deploy tested image to Tencent + needs: test + if: github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + actions: read + concurrency: + group: nanzhi-production-ombre-brain + cancel-in-progress: false + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/download-artifact@v4 + with: + name: nanzhi-image-${{ github.sha }} + - name: Deploy with service-scoped key and pinned host identity + env: + GH_TOKEN: ${{ github.token }} + NANZHI_DEPLOY_KEY: ${{ secrets.NANZHI_DEPLOY_KEY }} + run: python .github/scripts/nanzhi_deploy.py ombre-brain "${{ github.sha }}" image.tar.gz From a0e39fdeb0e17d9aa5e32d19d805a67bf13f30c8 Mon Sep 17 00:00:00 2001 From: thefloshibisci <103315611+thefloshibisci@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:08:45 +0800 Subject: [PATCH 12/16] Download verified Actions artifacts with bounded parallel HTTPS ranges --- .github/NANZHI_DEPLOY.md | 4 +- .github/ops/nanzhi_receiver.py | 72 ++++++++++++++++++++++++++++++-- .github/scripts/nanzhi_deploy.py | 32 ++++++++++++-- 3 files changed, 98 insertions(+), 10 deletions(-) diff --git a/.github/NANZHI_DEPLOY.md b/.github/NANZHI_DEPLOY.md index a7fd7b34da..4593df8829 100644 --- a/.github/NANZHI_DEPLOY.md +++ b/.github/NANZHI_DEPLOY.md @@ -1,8 +1,8 @@ # Tencent production deployment -Only `main` push or a manual workflow run on `main` can deploy. Pull requests and other branches never receive the deployment job. The deployment job depends on the existing complete test jobs and transfers the exact image built in that run; it does not rebuild on the server. +Only `main` push or a manual workflow run on `main` can deploy. Pull requests and other branches never receive the deployment job. The deployment job depends on the existing complete test jobs and downloads the exact image built in that run through a short-lived signed GitHub artifact URL; it does not rebuild on the server. -The client skips a queued commit if `main` has advanced. SSH checks the committed host public key. Each repository has a separate `NANZHI_DEPLOY_KEY` secret, forced to the matching service; it cannot run a shell, forward ports, or deploy another service. Application tokens and data remain on the server. +The client skips a queued commit if `main` has advanced. SSH checks the committed host public key and sends only the deployment instruction. The server downloads bounded parallel HTTPS ranges from GitHub artifact storage and verifies the final image SHA256; the GitHub token stays in Actions. Each repository has a separate `NANZHI_DEPLOY_KEY` secret, forced to the matching service; it cannot run a shell, forward ports, or deploy another service. Application tokens and data remain on the server. The root-owned receiver on the server verifies archive digest, size, platform, tag, and revision label. It serializes updates, makes isolated data copies with SQLite online backups, and verifies health, invalid-token rejection, authenticated MCP initialization and tool discovery with external network access disabled. Failure at this stage leaves the running production service unchanged. diff --git a/.github/ops/nanzhi_receiver.py b/.github/ops/nanzhi_receiver.py index 7c0b68931f..a680d9c13e 100644 --- a/.github/ops/nanzhi_receiver.py +++ b/.github/ops/nanzhi_receiver.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Forced SSH receiver. Each key is bound to one service by authorized_keys.""" import copy +import concurrent.futures import fcntl import hashlib import json @@ -8,11 +9,15 @@ from pathlib import Path import re import shutil +import signal import sqlite3 import subprocess import sys import tarfile import time +import urllib.parse +import urllib.request +import zipfile BASE = Path('/opt/nanzhi-deploy') PRODUCTION = Path('/opt/nanzhi-production') @@ -30,13 +35,21 @@ def call(args, timeout=180, **kwargs): def metadata(stream): - line = stream.readline(1025) - if len(line) > 1024 or not line.endswith(b'\n'): + line = stream.readline(8193) + if len(line) > 8192 or not line.endswith(b'\n'): raise ValueError('invalid_header_length') item = json.loads(line) if item == {'action': 'check'}: return item - if set(item) != {'action', 'commit', 'bytes', 'sha256'} or item['action'] != 'deploy': + fields = {'action', 'commit', 'bytes', 'sha256'} + if item.get('action') == 'deploy_url': + fields |= {'url', 'artifact_bytes'} + url = urllib.parse.urlsplit(item.get('url', '')) + if url.scheme != 'https' or not (url.hostname or '').endswith('.blob.core.windows.net') or url.username or url.password or url.port not in (None, 443): + raise ValueError('untrusted_artifact_url') + if type(item.get('artifact_bytes')) is not int or not 0 < item['artifact_bytes'] <= MAX_ARCHIVE: + raise ValueError('invalid_artifact_size') + if set(item) != fields or item['action'] not in ('deploy', 'deploy_url'): raise ValueError('invalid_action') if not re.fullmatch('[0-9a-f]{40}', item['commit']): raise ValueError('invalid_commit') @@ -62,6 +75,52 @@ def receive(stream, dest, item): raise ValueError('archive_integrity_failed') +def download_artifact(dest, item): + """Bounded parallel HTTPS ranges tolerate slow cross-region single flows.""" + bundle = dest.with_name('artifact.zip') + size = item['artifact_bytes'] + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args, **kwargs): + return None + fd = os.open(bundle, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600) + try: + os.ftruncate(fd, size) + block = 2 * 1024**2 + ranges = [(start, min(start + block, size) - 1) for start in range(0, size, block)] + def part(bounds): + start, end = bounds + for attempt in range(3): + try: + req = urllib.request.Request(item['url'], headers={'Range': f'bytes={start}-{end}'}) + with urllib.request.build_opener(NoRedirect()).open(req, timeout=90) as response: + if response.status != 206 or response.headers.get('Content-Range') != f'bytes {start}-{end}/{size}': + raise ValueError('invalid_artifact_range') + data = response.read(end - start + 2) + if len(data) != end - start + 1: + raise ValueError('truncated_artifact_range') + os.pwrite(fd, data, start) + return + except Exception: + if attempt == 2: + raise RuntimeError('artifact_range_download_failed') from None + time.sleep(2) + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: + list(pool.map(part, ranges)) + finally: + os.close(fd) + with zipfile.ZipFile(bundle) as archive: + if archive.namelist() != ['image.tar.gz'] or archive.getinfo('image.tar.gz').file_size != item['bytes']: + raise ValueError('unexpected_artifact_contents') + digest = hashlib.sha256() + with archive.open('image.tar.gz') as source, dest.open('xb') as output: + while data := source.read(1024 * 1024): + output.write(data) + digest.update(data) + if digest.hexdigest() != item['sha256']: + raise ValueError('artifact_digest_mismatch') + bundle.unlink() + + def snapshot(source, dest): """Consistent individual SQLite copies, isolated rehearsal only.""" shutil.copytree(source, dest, symlinks=True) @@ -190,7 +249,11 @@ def record(phase): if shutil.disk_usage(BASE).free < max(5 * 1024**3, item['bytes'] * 4): raise RuntimeError('insufficient_disk_space') archive = job / 'image.tar.gz' - receive(stream, archive, item) + record('receiving') + if item['action'] == 'deploy_url': + download_artifact(archive, item) + else: + receive(stream, archive, item) image = 'nanzhi/' + service + ':github-' + item['commit'] with tarfile.open(archive, 'r:gz') as tar: manifest = json.load(tar.extractfile('manifest.json')) @@ -249,6 +312,7 @@ def record(phase): def main(): os.umask(0o077) + signal.alarm(30 * 60) if len(sys.argv) != 2 or sys.argv[1] not in SERVICES: raise ValueError('service_not_allowed') service = sys.argv[1] diff --git a/.github/scripts/nanzhi_deploy.py b/.github/scripts/nanzhi_deploy.py index ce9e9451b9..fdcfdd7d4d 100644 --- a/.github/scripts/nanzhi_deploy.py +++ b/.github/scripts/nanzhi_deploy.py @@ -7,6 +7,9 @@ import subprocess import sys import tempfile +import urllib.request +import urllib.error +import urllib.parse def main(): @@ -24,7 +27,31 @@ def main(): image = Path(filename) with image.open('rb') as source: digest = hashlib.file_digest(source, 'sha256').hexdigest() - header = {'action': 'deploy', 'commit': commit, 'bytes': image.stat().st_size, 'sha256': digest} + artifacts = json.loads(subprocess.check_output([ + 'gh', 'api', 'repos/' + os.environ['GITHUB_REPOSITORY'] + '/actions/runs/' + os.environ['GITHUB_RUN_ID'] + '/artifacts', + ], text=True))['artifacts'] + artifact = next(a for a in artifacts if a['name'] == 'nanzhi-image-' + commit and not a['expired']) + api_url = urllib.parse.urlsplit(artifact['archive_download_url']) + if api_url.scheme != 'https' or api_url.netloc != 'api.github.com' or not api_url.path.startswith('/repos/' + os.environ['GITHUB_REPOSITORY'] + '/actions/artifacts/'): + raise RuntimeError('Unexpected artifact API URL') + class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args, **kwargs): + return None + request = urllib.request.Request(artifact['archive_download_url'], headers={ + 'Authorization': 'Bearer ' + os.environ['GH_TOKEN'], 'Accept': 'application/vnd.github+json', + }) + try: + urllib.request.build_opener(NoRedirect()).open(request, timeout=30) + raise RuntimeError('Expected signed artifact download location') + except urllib.error.HTTPError as exc: + if exc.code != 302: + raise RuntimeError('Could not obtain artifact download location') from None + location = exc.headers['Location'] + parsed = urllib.parse.urlsplit(location) + if parsed.scheme != 'https' or not (parsed.hostname or '').endswith('.blob.core.windows.net'): + raise RuntimeError('Unexpected artifact storage host') + header = {'action': 'deploy_url', 'commit': commit, 'bytes': image.stat().st_size, 'sha256': digest, + 'url': location, 'artifact_bytes': artifact['size_in_bytes']} known_hosts = Path(__file__).resolve().parent.parent / 'nanzhi-known-hosts' with tempfile.TemporaryDirectory(prefix='nanzhi-deploy-') as folder: key = Path(folder) / 'key' @@ -40,9 +67,6 @@ def main(): with subprocess.Popen(cmd, stdin=subprocess.PIPE) as proc: try: proc.stdin.write((json.dumps(header) + '\n').encode()) - with image.open('rb') as source: - while chunk := source.read(1024 * 1024): - proc.stdin.write(chunk) proc.stdin.close() status = proc.wait(timeout=1200) except BaseException: From c5a3ee98aa54e4ea50f50938b60e85574788a99a Mon Sep 17 00:00:00 2001 From: thefloshibisci <103315611+thefloshibisci@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:28:41 +0800 Subject: [PATCH 13/16] fix(deploy): rehearse Xinchao with isolated OB dependency --- .github/ops/nanzhi_receiver.py | 53 ++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/.github/ops/nanzhi_receiver.py b/.github/ops/nanzhi_receiver.py index a680d9c13e..2c4fad6c9f 100644 --- a/.github/ops/nanzhi_receiver.py +++ b/.github/ops/nanzhi_receiver.py @@ -210,22 +210,30 @@ def wait_ready(compose, service, seconds=120): def rehearsal(doc, service, image, job): - svc = copy.deepcopy(doc['services'][service]) - svc.update(image=image, restart='no', ports=[], networks=['isolated']) - svc.pop('depends_on', None) - for key in ('DREAM_ENABLED', 'MODEL_ENABLED', 'BARK_ENABLED', 'BRIDGE_ENABLED', 'OMBRE_WRITE_ENABLED'): - if key in svc['environment']: - svc['environment'][key] = 'false' - for index, mount in enumerate(svc['volumes']): - source = Path(mount['source']).resolve() - if not source.is_relative_to(PRODUCTION): - raise ValueError('unexpected_volume') - if mount.get('read_only'): - continue - dest = job / ('test-data-' + str(index)) - snapshot(source, dest) - mount['source'] = str(dest) - test = {'name': 'nanzhi-deploy-check', 'services': {service: svc}, 'networks': {'isolated': {'internal': True}}} + # Xinchao discovers OB proxy tools during tools/list. Provide an isolated + # copy of its actual dependency, never let a smoke test reach production OB. + names = [service] + (['ombre-brain'] if service == 'xinchao-nian' else []) + test_services = {} + for name in names: + svc = copy.deepcopy(doc['services'][name]) + svc.update(restart='no', ports=[], networks=['isolated']) + if name == service: + svc['image'] = image + svc.pop('depends_on', None) + for key in ('DREAM_ENABLED', 'MODEL_ENABLED', 'BARK_ENABLED', 'BRIDGE_ENABLED', 'OMBRE_WRITE_ENABLED'): + if key in svc['environment']: + svc['environment'][key] = 'false' + for index, mount in enumerate(svc['volumes']): + source = Path(mount['source']).resolve() + if not source.is_relative_to(PRODUCTION): + raise ValueError('unexpected_volume') + if mount.get('read_only'): + continue + dest = job / ('test-data-' + name + '-' + str(index)) + snapshot(source, dest) + mount['source'] = str(dest) + test_services[name] = svc + test = {'name': 'nanzhi-deploy-check', 'services': test_services, 'networks': {'isolated': {'internal': True}}} path = job / 'test-compose.json' path.write_text(json.dumps(test)) cmd = ['docker', 'compose', '-f', str(path)] @@ -250,7 +258,20 @@ def record(phase): raise RuntimeError('insufficient_disk_space') archive = job / 'image.tar.gz' record('receiving') + cached = False if item['action'] == 'deploy_url': + for previous in (BASE / 'runs').glob('*-' + service + '-' + item['commit'][:12] + '/image.tar.gz'): + if previous == archive or previous.stat().st_size != item['bytes']: + continue + with previous.open('rb') as source: + if hashlib.file_digest(source, 'sha256').hexdigest() != item['sha256']: + continue + archive.hardlink_to(previous) + cached = True + break + if cached: + status['verified_cached_archive'] = True + elif item['action'] == 'deploy_url': download_artifact(archive, item) else: receive(stream, archive, item) From 7858618c2b77c5ac4c8b46f033d79acdf749e19f Mon Sep 17 00:00:00 2001 From: thefloshibisci <103315611+thefloshibisci@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:47:48 +0800 Subject: [PATCH 14/16] fix(deploy): allow bounded cross-region artifact transfers to finish --- .github/ops/nanzhi_receiver.py | 2 +- .github/scripts/nanzhi_deploy.py | 2 +- .github/workflows/tests.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/ops/nanzhi_receiver.py b/.github/ops/nanzhi_receiver.py index 2c4fad6c9f..15e3880515 100644 --- a/.github/ops/nanzhi_receiver.py +++ b/.github/ops/nanzhi_receiver.py @@ -104,7 +104,7 @@ def part(bounds): if attempt == 2: raise RuntimeError('artifact_range_download_failed') from None time.sleep(2) - with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool: list(pool.map(part, ranges)) finally: os.close(fd) diff --git a/.github/scripts/nanzhi_deploy.py b/.github/scripts/nanzhi_deploy.py index fdcfdd7d4d..20d2e19e6f 100644 --- a/.github/scripts/nanzhi_deploy.py +++ b/.github/scripts/nanzhi_deploy.py @@ -68,7 +68,7 @@ def redirect_request(self, *args, **kwargs): try: proc.stdin.write((json.dumps(header) + '\n').encode()) proc.stdin.close() - status = proc.wait(timeout=1200) + status = proc.wait(timeout=1920) except BaseException: proc.kill() proc.wait() diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b439a2bb6e..7b11349f0b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -173,7 +173,7 @@ jobs: needs: test if: github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 35 permissions: contents: read actions: read From feb1a1f50646ea2fe3ce4c52c5d607d4771655f1 Mon Sep 17 00:00:00 2001 From: thefloshibisci <103315611+thefloshibisci@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:54:32 +0800 Subject: [PATCH 15/16] fix(deploy): refresh expiring artifact URLs during downloads --- .github/ops/nanzhi_receiver.py | 25 ++++++++++++++---- .github/scripts/nanzhi_deploy.py | 45 +++++++++++++++++++++++--------- 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/.github/ops/nanzhi_receiver.py b/.github/ops/nanzhi_receiver.py index 15e3880515..cdb6223169 100644 --- a/.github/ops/nanzhi_receiver.py +++ b/.github/ops/nanzhi_receiver.py @@ -14,6 +14,7 @@ import subprocess import sys import tarfile +import threading import time import urllib.parse import urllib.request @@ -75,7 +76,7 @@ def receive(stream, dest, item): raise ValueError('archive_integrity_failed') -def download_artifact(dest, item): +def download_artifact(dest, item, locations=None): """Bounded parallel HTTPS ranges tolerate slow cross-region single flows.""" bundle = dest.with_name('artifact.zip') size = item['artifact_bytes'] @@ -91,7 +92,7 @@ def part(bounds): start, end = bounds for attempt in range(3): try: - req = urllib.request.Request(item['url'], headers={'Range': f'bytes={start}-{end}'}) + req = urllib.request.Request((locations or item)['url'], headers={'Range': f'bytes={start}-{end}'}) with urllib.request.build_opener(NoRedirect()).open(req, timeout=90) as response: if response.status != 206 or response.headers.get('Content-Range') != f'bytes {start}-{end}/{size}': raise ValueError('invalid_artifact_range') @@ -244,7 +245,7 @@ def rehearsal(doc, service, image, job): call(cmd + ['down', '--timeout', '60'], timeout=150) -def deploy(service, item, stream): +def deploy(service, item, stream, locations=None): job = BASE / 'runs' / (time.strftime('%Y%m%dT%H%M%SZ', time.gmtime()) + '-' + service + '-' + item['commit'][:12]) job.mkdir(mode=0o700) status = {'service': service, 'commit': item['commit'], 'phase': 'receiving'} @@ -272,7 +273,7 @@ def record(phase): if cached: status['verified_cached_archive'] = True elif item['action'] == 'deploy_url': - download_artifact(archive, item) + download_artifact(archive, item, locations) else: receive(stream, archive, item) image = 'nanzhi/' + service + ':github-' + item['commit'] @@ -341,9 +342,23 @@ def main(): if item['action'] == 'check': print(json.dumps({'ready': COMPOSE.is_file(), 'service': service, 'protocol': 1})) return + locations = {'url': item.get('url')} + if item['action'] == 'deploy_url': + def refresh_locations(): + while True: + try: + renewed = metadata(sys.stdin.buffer) + # A refresh may only replace the signed URL for this exact + # artifact. Content hash/size/commit remain immutable. + if {k: v for k, v in renewed.items() if k != 'url'} != {k: v for k, v in item.items() if k != 'url'}: + return + locations['url'] = renewed['url'] + except Exception: + return + threading.Thread(target=refresh_locations, daemon=True).start() with (BASE / 'deployment.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX) - print(json.dumps(deploy(service, item, sys.stdin.buffer))) + print(json.dumps(deploy(service, item, sys.stdin.buffer, locations))) if __name__ == '__main__': diff --git a/.github/scripts/nanzhi_deploy.py b/.github/scripts/nanzhi_deploy.py index 20d2e19e6f..dde729d6d4 100644 --- a/.github/scripts/nanzhi_deploy.py +++ b/.github/scripts/nanzhi_deploy.py @@ -7,6 +7,7 @@ import subprocess import sys import tempfile +import threading import urllib.request import urllib.error import urllib.parse @@ -40,18 +41,20 @@ def redirect_request(self, *args, **kwargs): request = urllib.request.Request(artifact['archive_download_url'], headers={ 'Authorization': 'Bearer ' + os.environ['GH_TOKEN'], 'Accept': 'application/vnd.github+json', }) - try: - urllib.request.build_opener(NoRedirect()).open(request, timeout=30) - raise RuntimeError('Expected signed artifact download location') - except urllib.error.HTTPError as exc: - if exc.code != 302: - raise RuntimeError('Could not obtain artifact download location') from None - location = exc.headers['Location'] - parsed = urllib.parse.urlsplit(location) - if parsed.scheme != 'https' or not (parsed.hostname or '').endswith('.blob.core.windows.net'): - raise RuntimeError('Unexpected artifact storage host') + def signed_location(): + try: + urllib.request.build_opener(NoRedirect()).open(request, timeout=30) + raise RuntimeError('Expected signed artifact download location') + except urllib.error.HTTPError as exc: + if exc.code != 302: + raise RuntimeError('Could not obtain artifact download location') from None + location = exc.headers['Location'] + parsed = urllib.parse.urlsplit(location) + if parsed.scheme != 'https' or not (parsed.hostname or '').endswith('.blob.core.windows.net'): + raise RuntimeError('Unexpected artifact storage host') + return location header = {'action': 'deploy_url', 'commit': commit, 'bytes': image.stat().st_size, 'sha256': digest, - 'url': location, 'artifact_bytes': artifact['size_in_bytes']} + 'url': signed_location(), 'artifact_bytes': artifact['size_in_bytes']} known_hosts = Path(__file__).resolve().parent.parent / 'nanzhi-known-hosts' with tempfile.TemporaryDirectory(prefix='nanzhi-deploy-') as folder: key = Path(folder) / 'key' @@ -65,14 +68,32 @@ def redirect_request(self, *args, **kwargs): '-o', 'ConnectTimeout=20', 'nanzhi-deploy@49.234.39.128', ] with subprocess.Popen(cmd, stdin=subprocess.PIPE) as proc: + stop_refresh = threading.Event() + def refresh_url(): + # Artifact URLs expire in ten minutes. Refresh the same artifact + # without transmitting the GitHub credential to the server. + while not stop_refresh.wait(180): + try: + renewed = dict(header, url=signed_location()) + proc.stdin.write((json.dumps(renewed) + '\n').encode()) + proc.stdin.flush() + except (BrokenPipeError, ValueError): + return + except Exception: + # The existing URL remains usable; retry well before expiry. + if stop_refresh.wait(15): + return try: proc.stdin.write((json.dumps(header) + '\n').encode()) - proc.stdin.close() + proc.stdin.flush() + threading.Thread(target=refresh_url, daemon=True).start() status = proc.wait(timeout=1920) except BaseException: proc.kill() proc.wait() raise + finally: + stop_refresh.set() if status: raise SystemExit(status) From d6e095dbb55bd310801575f9d58683c4c46d41d0 Mon Sep 17 00:00:00 2001 From: thefloshibisci <103315611+thefloshibisci@users.noreply.github.com> Date: Mon, 2 Nov 2026 21:57:04 +0800 Subject: [PATCH 16/16] fix(deploy): close safely while awaiting URL renewal --- .github/ops/nanzhi_receiver.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/ops/nanzhi_receiver.py b/.github/ops/nanzhi_receiver.py index cdb6223169..fa7efac341 100644 --- a/.github/ops/nanzhi_receiver.py +++ b/.github/ops/nanzhi_receiver.py @@ -338,7 +338,10 @@ def main(): if len(sys.argv) != 2 or sys.argv[1] not in SERVICES: raise ValueError('service_not_allowed') service = sys.argv[1] - item = metadata(sys.stdin.buffer) + # A daemon may wait for URL refresh while main exits. An unbuffered fd + # avoids holding sys.stdin's buffered-reader lock during interpreter exit. + stream = os.fdopen(os.dup(sys.stdin.fileno()), 'rb', buffering=0) + item = metadata(stream) if item['action'] == 'check': print(json.dumps({'ready': COMPOSE.is_file(), 'service': service, 'protocol': 1})) return @@ -347,7 +350,7 @@ def main(): def refresh_locations(): while True: try: - renewed = metadata(sys.stdin.buffer) + renewed = metadata(stream) # A refresh may only replace the signed URL for this exact # artifact. Content hash/size/commit remain immutable. if {k: v for k, v in renewed.items() if k != 'url'} != {k: v for k, v in item.items() if k != 'url'}: @@ -358,7 +361,7 @@ def refresh_locations(): threading.Thread(target=refresh_locations, daemon=True).start() with (BASE / 'deployment.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX) - print(json.dumps(deploy(service, item, sys.stdin.buffer, locations))) + print(json.dumps(deploy(service, item, stream, locations))) if __name__ == '__main__':