From 516a58b7a1e3f6dd5c71c0c74b93f69922fcfc86 Mon Sep 17 00:00:00 2001 From: yh M Date: Sun, 23 Aug 2026 12:28:15 +0800 Subject: [PATCH 1/4] fix(docker): create compose env files and keep Windows compose paths relative Windows Docker reports a generic file-not-found when env_file targets are missing, or when compose paths are doubled. Make docker-start copy .env examples and invoke compose with filenames relative to docker/. Co-authored-by: Cursor --- scripts/docker.sh | 65 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 19 deletions(-) diff --git a/scripts/docker.sh b/scripts/docker.sh index a53052ecc87..0247dda5819 100755 --- a/scripts/docker.sh +++ b/scripts/docker.sh @@ -12,8 +12,46 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" DOCKER_DIR="$PROJECT_ROOT/docker" -# Docker Compose command with project name -COMPOSE_CMD="docker compose -p deer-flow-dev -f docker-compose-dev.yaml" +# Docker Compose command with project name. +# Use a filename relative to DOCKER_DIR (we always `cd` there) so Windows +# Docker Desktop does not receive a Git Bash `/c/...` path it cannot open. +# See https://github.com/bytedance/deer-flow/issues/2416 +COMPOSE_FILE="docker-compose-dev.yaml" +COMPOSE_CMD="docker compose -p deer-flow-dev -f $COMPOSE_FILE" + +ensure_from_example() { + local dest="$1" + local src="$2" + local label="$3" + + if [ -f "$dest" ]; then + return 0 + fi + if [ -f "$src" ]; then + cp "$src" "$dest" + echo -e "${BLUE}Created ${label} from $(basename "$src")${NC}" + return 0 + fi + echo -e "${YELLOW}✗ ${label} not found and no $(basename "$src") to copy from.${NC}" + echo "Create ${dest} before starting Docker." + exit 1 +} + +# Compose env_file entries fail closed on Windows when .env is missing +# ("The specified file cannot be found" / "Le fichier spécifique est introuvable"). +prepare_compose_env() { + if [ ! -f "$DOCKER_DIR/$COMPOSE_FILE" ]; then + echo -e "${YELLOW}✗ ${COMPOSE_FILE} not found at ${DOCKER_DIR}/${COMPOSE_FILE}${NC}" + echo "Run this from the DeerFlow repository root, e.g. 'make docker-start'." + echo "Do not run 'docker compose -f docker/${COMPOSE_FILE}' from inside docker/ — that resolves to docker/docker/${COMPOSE_FILE}." + exit 1 + fi + if [ -z "$DEER_FLOW_ROOT" ]; then + export DEER_FLOW_ROOT="$PROJECT_ROOT" + fi + ensure_from_example "$PROJECT_ROOT/.env" "$PROJECT_ROOT/.env.example" ".env" + ensure_from_example "$PROJECT_ROOT/frontend/.env" "$PROJECT_ROOT/frontend/.env.example" "frontend/.env" +} load_proxy_env_from_dotenv() { local env_file="$PROJECT_ROOT/.env" @@ -207,7 +245,7 @@ start() { exit 1 fi echo -e "${YELLOW}Mounting host Docker socket into gateway (DooD = host root-equivalent). See SECURITY.md.${NC}" - COMPOSE_CMD="$COMPOSE_CMD -f $DOCKER_DIR/docker-compose.dood.yaml" + COMPOSE_CMD="$COMPOSE_CMD -f docker-compose.dood.yaml" fi echo -e "${BLUE}Runtime: Gateway embedded agent runtime${NC}" @@ -260,6 +298,7 @@ start() { fi fi + prepare_compose_env load_proxy_env_from_dotenv echo "Building and starting containers..." @@ -283,12 +322,8 @@ start() { logs() { local service="" - # DEER_FLOW_ROOT is referenced in docker-compose-dev.yaml; set it before - # reading logs so Compose does not resolve mounted paths from an empty root. - if [ -z "$DEER_FLOW_ROOT" ]; then - export DEER_FLOW_ROOT="$PROJECT_ROOT" - fi - + prepare_compose_env + case "$1" in --frontend) service="frontend" @@ -325,11 +360,7 @@ logs() { # Stop Docker development environment stop() { - # DEER_FLOW_ROOT is referenced in docker-compose-dev.yaml; set it before - # running compose down to suppress "variable is not set" warnings. - if [ -z "$DEER_FLOW_ROOT" ]; then - export DEER_FLOW_ROOT="$PROJECT_ROOT" - fi + prepare_compose_env echo "Stopping Docker development services..." cd "$DOCKER_DIR" && $COMPOSE_CMD down echo "Cleaning up sandbox containers..." @@ -339,11 +370,7 @@ stop() { # Restart Docker development environment restart() { - # DEER_FLOW_ROOT is referenced in docker-compose-dev.yaml; set it before - # restarting services so Compose resolves mounted paths from this checkout. - if [ -z "$DEER_FLOW_ROOT" ]; then - export DEER_FLOW_ROOT="$PROJECT_ROOT" - fi + prepare_compose_env echo "========================================" echo " Restarting DeerFlow Docker Services" echo "========================================" From 5b4cbc7dc9a43cb5feb5bbce88874a2e25fe01a3 Mon Sep 17 00:00:00 2001 From: yh M Date: Mon, 24 Aug 2026 09:38:06 +0800 Subject: [PATCH 2/4] fix(docker): make dev compose env files optional and repair test fixture Address review feedback on #4956. [P1] prepare_compose_env aborted before the mocked COMPOSE_CMD in test_compose_commands_set_deer_flow_root_before_compose, because the temp root had no compose file or .env examples. Seed them in the fixture so the preflight reaches the mock. [P2] .env is gitignored, so a fresh clone has none and a direct `docker compose -f docker/docker-compose-dev.yaml up --build` aborts on Windows before scripts/docker.sh can help. Mark the dev env_file entries `required: false` so a missing .env is not fatal, and document that direct Compose must be run from the repository root. Co-authored-by: Cursor --- CONTRIBUTING.md | 4 ++++ .../tests/test_compose_default_bind_host.py | 13 +++++++++++ .../test_docker_sandbox_mode_detection.py | 15 ++++++++++-- docker/docker-compose-dev.yaml | 23 +++++++++++++++---- 4 files changed, 49 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9501df45c29..735ec84294f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,6 +43,10 @@ Docker provides a consistent, isolated environment with all dependencies pre-con ``` `make docker-start` reads `config.yaml` and starts `provisioner` only for provisioner/Kubernetes sandbox mode. + Prefer this wrapper over invoking Compose yourself. If you do run Compose + directly, do it from the repository root (`docker compose -f docker/docker-compose-dev.yaml`), + not from inside `docker/`. + All services will start with hot-reload enabled: - Frontend changes are automatically reloaded - Backend changes trigger automatic restart diff --git a/backend/tests/test_compose_default_bind_host.py b/backend/tests/test_compose_default_bind_host.py index 1064bb53ace..511fef03b06 100644 --- a/backend/tests/test_compose_default_bind_host.py +++ b/backend/tests/test_compose_default_bind_host.py @@ -110,3 +110,16 @@ def _bind_address(mapping: str) -> str | None: # ADDR:HOST:CONTAINER -> bound; HOST:CONTAINER or CONTAINER -> unbound. return segments[0] if len(segments) >= 3 else None + + +def test_dev_compose_env_files_are_optional(): + """Missing .env files must not fail `docker compose -f docker/docker-compose-dev.yaml`.""" + compose = yaml.safe_load(COMPOSE_PATHS["dev"].read_text(encoding="utf-8")) + expected = { + "provisioner": "../.env", + "frontend": "../frontend/.env", + "gateway": "../.env", + } + for service_name, path in expected.items(): + entries = compose["services"][service_name]["env_file"] + assert entries == [{"path": path, "required": False}], f"{service_name} env_file must be optional; got: {entries!r}" diff --git a/backend/tests/test_docker_sandbox_mode_detection.py b/backend/tests/test_docker_sandbox_mode_detection.py index c95f7430f08..b980fb73201 100644 --- a/backend/tests/test_docker_sandbox_mode_detection.py +++ b/backend/tests/test_docker_sandbox_mode_detection.py @@ -106,14 +106,25 @@ def test_detect_mode_unknown_provider_falls_back_to_local(): assert _detect_mode_with_config(config) == "local" +def _seed_compose_env(tmp_root: Path) -> None: + """Give prepare_compose_env the compose file and example env files it copies.""" + (tmp_root / "docker-compose-dev.yaml").write_text("services: {}\n", encoding="utf-8") + (tmp_root / ".env.example").write_text("# test\n", encoding="utf-8") + frontend = tmp_root / "frontend" + frontend.mkdir() + (frontend / ".env.example").write_text("# test\n", encoding="utf-8") + + @pytest.mark.parametrize("docker_command", ["logs --gateway", "restart"]) def test_compose_commands_set_deer_flow_root_before_compose(docker_command): """Log and restart commands should resolve mounts from the repository root.""" with tempfile.TemporaryDirectory() as tmpdir: + tmp_root = Path(tmpdir) + _seed_compose_env(tmp_root) command = f""" source '{SCRIPT_PATH}' -PROJECT_ROOT='{tmpdir}' -DOCKER_DIR='{tmpdir}' +PROJECT_ROOT='{tmp_root}' +DOCKER_DIR='{tmp_root}' COMPOSE_CMD=capture_compose capture_compose() {{ test "${{DEER_FLOW_ROOT:-}}" = "$PROJECT_ROOT"; }} unset DEER_FLOW_ROOT diff --git a/docker/docker-compose-dev.yaml b/docker/docker-compose-dev.yaml index c4dfa801e31..7f3d8622bce 100644 --- a/docker/docker-compose-dev.yaml +++ b/docker/docker-compose-dev.yaml @@ -1,5 +1,17 @@ # DeerFlow Development Environment -# Usage: docker-compose -f docker-compose-dev.yaml up --build +# +# Supported entry: from the repository root, run `make docker-start`. +# That wrapper creates missing .env files and invokes Compose from this +# directory with a relative filename. +# +# Direct Compose must also be run from the repository root: +# docker compose -f docker/docker-compose-dev.yaml up --build +# Do not reuse that -f path from inside docker/ — it resolves to +# docker/docker/docker-compose-dev.yaml and fails with file-not-found. +# +# env_file targets are optional so a missing ../.env or ../frontend/.env +# does not abort Compose on Windows ("file not found" / +# "Le fichier spécifique est introuvable"). # # Services: # - nginx: Reverse proxy (port 2026) @@ -74,7 +86,8 @@ services: # The same value must be set on the gateway side via config.yaml sandbox.provisioner_api_key. - PROVISIONER_API_KEY=${PROVISIONER_API_KEY:-} env_file: - - ../.env + - path: ../.env + required: false extra_hosts: - "host.docker.internal:host-gateway" networks: @@ -139,7 +152,8 @@ services: - CI=true - DEER_FLOW_INTERNAL_GATEWAY_BASE_URL=http://gateway:8001 env_file: - - ../frontend/.env + - path: ../frontend/.env + required: false networks: - deer-flow-dev restart: unless-stopped @@ -212,7 +226,8 @@ services: - NO_PROXY=${NO_PROXY:-}${NO_PROXY:+,}localhost,127.0.0.1,::1,gateway,frontend,nginx,provisioner,openviking,host.docker.internal - no_proxy=${no_proxy:-}${no_proxy:+,}localhost,127.0.0.1,::1,gateway,frontend,nginx,provisioner,openviking,host.docker.internal env_file: - - ../.env + - path: ../.env + required: false extra_hosts: # For Linux: map host.docker.internal to host gateway - "host.docker.internal:host-gateway" From 0bedc015b2f73f4e8cbcb72ba67027748c7ced04 Mon Sep 17 00:00:00 2001 From: yh M Date: Mon, 24 Aug 2026 13:59:31 +0800 Subject: [PATCH 3/4] fix(docker): declare Compose 2.24 floor and keep non-start commands read-only Address the second review round on #4956. - Document Compose >= 2.24 (CONTRIBUTING, README, compose header) and fail early from make docker-start with an actionable message; probe both `docker compose` and the hyphenated `docker-compose` binary. - Document DEER_FLOW_ROOT for direct Compose callers (bash + PowerShell); leave the variable without a $PWD fallback because PowerShell/cmd do not export it. - Split prepare_compose_env: compose_preflight is shared and read-only; ensure_env_files runs only from start. - Expand tests for version boundaries, hyphenated fallback, env-file creation, and read-only stop/logs/restart behavior. Co-authored-by: Cursor --- CONTRIBUTING.md | 32 ++- README.md | 4 + README_zh.md | 4 + .../test_docker_sandbox_mode_detection.py | 184 +++++++++++++++++- docker/docker-compose-dev.yaml | 36 +++- scripts/docker.sh | 116 +++++++++-- 6 files changed, 333 insertions(+), 43 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 735ec84294f..acb2a417e3c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -13,6 +13,12 @@ Docker provides a consistent, isolated environment with all dependencies pre-con #### Prerequisites - Docker Desktop or Docker Engine +- Docker Compose **v2.24 or newer** (check with `docker compose version`). The dev + Compose file marks its `env_file` entries optional using the long-form + `path`/`required` syntax; older clients reject it with + `services.gateway.env_file.0 must be a string`. `make docker-start` verifies the + version and tells you to upgrade — direct `docker compose` callers get that raw + message instead. - pnpm (for caching optimization) #### Setup Steps @@ -43,9 +49,29 @@ Docker provides a consistent, isolated environment with all dependencies pre-con ``` `make docker-start` reads `config.yaml` and starts `provisioner` only for provisioner/Kubernetes sandbox mode. - Prefer this wrapper over invoking Compose yourself. If you do run Compose - directly, do it from the repository root (`docker compose -f docker/docker-compose-dev.yaml`), - not from inside `docker/`. + Prefer this wrapper over invoking Compose yourself: it checks your Compose + version, creates the missing `.env` files, and exports `DEER_FLOW_ROOT`. + + If you do run Compose directly, run it **from the repository root** and set + `DEER_FLOW_ROOT` to the absolute path of your checkout. Compose interpolates + that variable into host-side paths (`DEER_FLOW_HOST_BASE_DIR`, + `THREADS_HOST_PATH`) that the AIO and provisioner sandbox modes bind-mount; + leaving it unset renders them as `/backend/.deer-flow`, so those mounts + silently miss your checkout instead of failing: + + ```bash + # macOS / Linux + DEER_FLOW_ROOT="$PWD" docker compose -f docker/docker-compose-dev.yaml up --build + ``` + + ```powershell + # Windows PowerShell + $env:DEER_FLOW_ROOT = (Get-Location).Path + docker compose -f docker/docker-compose-dev.yaml up --build + ``` + + Do not reuse that `-f` path from inside `docker/` — it resolves to + `docker/docker/docker-compose-dev.yaml` and fails with a file-not-found error. All services will start with hot-reload enabled: - Frontend changes are automatically reloaded diff --git a/README.md b/README.md index a8931e3643e..c3e20b86228 100644 --- a/README.md +++ b/README.md @@ -260,6 +260,10 @@ Use the table below as a practical starting point when choosing how to run DeerF #### Option 1: Docker (Recommended) +Requires Docker Desktop / Docker Engine and **Docker Compose v2.24+** +(`docker compose version`). Older Compose clients cannot parse the optional +`env_file` syntax in `docker/docker-compose-dev.yaml`. + **Development** (hot-reload, source mounts): ```bash diff --git a/README_zh.md b/README_zh.md index 9eee6394369..bfaba44cec0 100644 --- a/README_zh.md +++ b/README_zh.md @@ -246,6 +246,10 @@ DeerFlow 新近集成了 BytePlus 自研的智能搜索与抓取工具集——[ #### 方式一:Docker(推荐) +需要 Docker Desktop / Docker Engine,以及 **Docker Compose v2.24+** +(`docker compose version`)。更旧的 Compose 客户端无法解析 +`docker/docker-compose-dev.yaml` 里的可选 `env_file` 语法。 + **开发模式**(支持热更新,挂载源码): ```bash diff --git a/backend/tests/test_docker_sandbox_mode_detection.py b/backend/tests/test_docker_sandbox_mode_detection.py index b980fb73201..18bb84952c7 100644 --- a/backend/tests/test_docker_sandbox_mode_detection.py +++ b/backend/tests/test_docker_sandbox_mode_detection.py @@ -106,28 +106,192 @@ def test_detect_mode_unknown_provider_falls_back_to_local(): assert _detect_mode_with_config(config) == "local" -def _seed_compose_env(tmp_root: Path) -> None: - """Give prepare_compose_env the compose file and example env files it copies.""" +def _seed_compose_file(tmp_root: Path) -> None: + """Give require_compose_file the file it validates.""" (tmp_root / "docker-compose-dev.yaml").write_text("services: {}\n", encoding="utf-8") + + +def _seed_env_examples(tmp_root: Path) -> None: + """Provide the templates ensure_env_files copies from.""" (tmp_root / ".env.example").write_text("# test\n", encoding="utf-8") frontend = tmp_root / "frontend" - frontend.mkdir() + frontend.mkdir(exist_ok=True) (frontend / ".env.example").write_text("# test\n", encoding="utf-8") -@pytest.mark.parametrize("docker_command", ["logs --gateway", "restart"]) -def test_compose_commands_set_deer_flow_root_before_compose(docker_command): - """Log and restart commands should resolve mounts from the repository root.""" - with tempfile.TemporaryDirectory() as tmpdir: - tmp_root = Path(tmpdir) - _seed_compose_env(tmp_root) - command = f""" +def _run_docker_sh(tmp_root: Path, body: str) -> None: + """Run docker.sh against a temp checkout, stubbing the real Compose version probe. + + Keep SCRIPT_DIR at the real scripts/ directory so stop's cleanup-containers.sh + path still resolves; only PROJECT_ROOT / DOCKER_DIR are redirected. + """ + command = f""" source '{SCRIPT_PATH}' PROJECT_ROOT='{tmp_root}' DOCKER_DIR='{tmp_root}' +require_compose_version() {{ :; }} +{body} +""" + subprocess.check_call([BASH_EXECUTABLE, "-lc", command]) + + +@pytest.mark.parametrize("docker_command", ["logs --gateway", "stop", "restart"]) +def test_compose_commands_set_deer_flow_root_before_compose(docker_command): + """Read-only compose commands should resolve mounts from the repository root.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmp_root = Path(tmpdir) + _seed_compose_file(tmp_root) + _run_docker_sh( + tmp_root, + f""" COMPOSE_CMD=capture_compose capture_compose() {{ test "${{DEER_FLOW_ROOT:-}}" = "$PROJECT_ROOT"; }} unset DEER_FLOW_ROOT {docker_command} +""", + ) + + +@pytest.mark.parametrize("docker_command", ["logs --gateway", "stop", "restart"]) +def test_read_only_commands_do_not_create_env_files(docker_command): + """Only start may write configuration; logs/stop/restart must leave a checkout alone.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmp_root = Path(tmpdir) + _seed_compose_file(tmp_root) + _seed_env_examples(tmp_root) + + _run_docker_sh(tmp_root, f"COMPOSE_CMD=true\n{docker_command}") + + assert not (tmp_root / ".env").exists(), f"{docker_command} created .env" + assert not (tmp_root / "frontend" / ".env").exists(), f"{docker_command} created frontend/.env" + + +@pytest.mark.parametrize("docker_command", ["logs --gateway", "stop", "restart"]) +def test_read_only_commands_run_without_env_examples(docker_command): + """A missing .env.example must never block stopping or inspecting containers.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmp_root = Path(tmpdir) + _seed_compose_file(tmp_root) + + _run_docker_sh(tmp_root, f"COMPOSE_CMD=true\n{docker_command}") + + +def test_ensure_env_files_copies_from_examples(): + """start's env-file step should create .env files from their examples when missing.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmp_root = Path(tmpdir) + _seed_env_examples(tmp_root) + + command = f""" +source '{SCRIPT_PATH}' +PROJECT_ROOT='{tmp_root}' +ensure_env_files +""" + subprocess.check_call([BASH_EXECUTABLE, "-lc", command]) + + assert (tmp_root / ".env").is_file() + assert (tmp_root / "frontend" / ".env").is_file() + assert (tmp_root / ".env").read_text(encoding="utf-8") == "# test\n" + assert (tmp_root / "frontend" / ".env").read_text(encoding="utf-8") == "# test\n" + + +def test_ensure_env_files_leaves_existing_env_untouched(): + """ensure_env_files must not overwrite an already-present .env.""" + with tempfile.TemporaryDirectory() as tmpdir: + tmp_root = Path(tmpdir) + _seed_env_examples(tmp_root) + (tmp_root / ".env").write_text("KEEP=me\n", encoding="utf-8") + frontend = tmp_root / "frontend" + (frontend / ".env").write_text("KEEP=frontend\n", encoding="utf-8") + + command = f""" +source '{SCRIPT_PATH}' +PROJECT_ROOT='{tmp_root}' +ensure_env_files """ subprocess.check_call([BASH_EXECUTABLE, "-lc", command]) + + assert (tmp_root / ".env").read_text(encoding="utf-8") == "KEEP=me\n" + assert (frontend / ".env").read_text(encoding="utf-8") == "KEEP=frontend\n" + + +@pytest.mark.parametrize( + ("reported_version", "expected_returncode"), + [ + ("2.23.3", 1), + ("2.5.0", 1), + ("2.24.0", 0), + ("v2.40.2-desktop.1", 0), + ("3.0.1", 0), + ("", 0), # undetectable: warn, but do not block + ], +) +def test_require_compose_version_enforces_minimum(reported_version, expected_returncode): + """Old clients get our actionable message instead of a raw Compose parser error.""" + # Stub both binaries so an empty plugin probe cannot fall through to the + # real hyphenated docker-compose installed on the developer machine. + command = f""" +source '{SCRIPT_PATH}' +docker() {{ echo '{reported_version}'; }} +docker-compose() {{ echo '{reported_version}'; }} +require_compose_version +""" + result = subprocess.run( + [BASH_EXECUTABLE, "-lc", command], + capture_output=True, + text=True, + encoding="utf-8", + ) + + assert result.returncode == expected_returncode, result.stdout + result.stderr + if expected_returncode != 0: + assert "too old" in result.stdout + assert "docs.docker.com/compose/install" in result.stdout + + +def test_require_compose_version_falls_back_to_hyphenated_binary(): + """When the Compose plugin is missing, probe docker-compose (hyphenated).""" + command = f""" +source '{SCRIPT_PATH}' +docker() {{ + if [ "$1" = compose ]; then + echo "docker: unknown command" >&2 + return 1 + fi + command docker "$@" +}} +docker-compose() {{ echo '2.24.0'; }} +require_compose_version +""" + result = subprocess.run( + [BASH_EXECUTABLE, "-lc", command], + capture_output=True, + text=True, + encoding="utf-8", + ) + + assert result.returncode == 0, result.stdout + result.stderr + + +def test_require_compose_version_rejects_old_hyphenated_binary(): + """An old docker-compose binary must still fail the floor check.""" + command = f""" +source '{SCRIPT_PATH}' +docker() {{ + if [ "$1" = compose ]; then + return 1 + fi + command docker "$@" +}} +docker-compose() {{ echo '2.23.3'; }} +require_compose_version +""" + result = subprocess.run( + [BASH_EXECUTABLE, "-lc", command], + capture_output=True, + text=True, + encoding="utf-8", + ) + + assert result.returncode == 1, result.stdout + result.stderr + assert "too old" in result.stdout diff --git a/docker/docker-compose-dev.yaml b/docker/docker-compose-dev.yaml index 7f3d8622bce..799a88de794 100644 --- a/docker/docker-compose-dev.yaml +++ b/docker/docker-compose-dev.yaml @@ -1,17 +1,27 @@ # DeerFlow Development Environment # -# Supported entry: from the repository root, run `make docker-start`. -# That wrapper creates missing .env files and invokes Compose from this -# directory with a relative filename. +# Requires Docker Compose >= 2.24. The env_file entries below use the long-form +# path/required syntax so a missing ../.env or ../frontend/.env does not abort +# Compose on Windows ("file not found" / "Le fichier spécifique est +# introuvable"). Clients older than 2.24 do not understand that field and +# reject this file with: +# validating docker-compose-dev.yaml: services.gateway.env_file.0 must be a string +# make docker-start checks the version and says so; direct `docker compose` +# callers get the message above instead. # -# Direct Compose must also be run from the repository root: -# docker compose -f docker/docker-compose-dev.yaml up --build -# Do not reuse that -f path from inside docker/ — it resolves to -# docker/docker/docker-compose-dev.yaml and fails with file-not-found. +# Supported entry: from the repository root, run `make docker-start`. That +# wrapper checks the Compose version, creates missing .env files, exports +# DEER_FLOW_ROOT, and invokes Compose from this directory with a relative +# filename. # -# env_file targets are optional so a missing ../.env or ../frontend/.env -# does not abort Compose on Windows ("file not found" / -# "Le fichier spécifique est introuvable"). +# Running Compose yourself works too, but export DEER_FLOW_ROOT as the absolute +# checkout path first (see CONTRIBUTING.md for the Windows form). Unset, the +# DEER_FLOW_HOST_BASE_DIR and THREADS_HOST_PATH values below render as +# /backend/.deer-flow, so the provisioner bind-mounts the wrong host paths: +# DEER_FLOW_ROOT="$PWD" docker compose -f docker/docker-compose-dev.yaml up --build +# The -f path resolves against your shell's directory, not against this file: +# from the repository root use docker/docker-compose-dev.yaml; from inside +# docker/ use the bare docker-compose-dev.yaml, as make docker-start does. # # Services: # - nginx: Reverse proxy (port 2026) @@ -69,6 +79,12 @@ services: # On Docker Desktop/OrbStack, use your actual host paths like /Users/username/... # Set these in your shell before running docker-compose: # export DEER_FLOW_ROOT=/absolute/path/to/deer-flow + # Deliberately left without a ${DEER_FLOW_ROOT:-...} fallback. The obvious + # candidate, $PWD, is exported by POSIX shells but not by PowerShell or + # cmd, so the default would resolve to an empty string on Windows — the + # platform this variable exists to get right. A wrong-but-plausible host + # path mounts an empty directory instead of failing, so we require the + # caller to be explicit; make docker-start sets it for you. - THREADS_HOST_PATH=${DEER_FLOW_ROOT}/backend/.deer-flow/threads # Per-user data base directory for user-scoped skill mounts - DEER_FLOW_HOST_BASE_DIR=${DEER_FLOW_ROOT}/backend/.deer-flow diff --git a/scripts/docker.sh b/scripts/docker.sh index 0247dda5819..229f03b361b 100755 --- a/scripts/docker.sh +++ b/scripts/docker.sh @@ -15,10 +15,14 @@ DOCKER_DIR="$PROJECT_ROOT/docker" # Docker Compose command with project name. # Use a filename relative to DOCKER_DIR (we always `cd` there) so Windows # Docker Desktop does not receive a Git Bash `/c/...` path it cannot open. -# See https://github.com/bytedance/deer-flow/issues/2416 COMPOSE_FILE="docker-compose-dev.yaml" COMPOSE_CMD="docker compose -p deer-flow-dev -f $COMPOSE_FILE" +# docker-compose-dev.yaml marks its env_file entries optional with the long-form +# `- path: ... / required: false` syntax, understood by Compose v2.24.0 and up. +# Older clients abort while parsing the file, before any preflight below can run. +COMPOSE_MIN_VERSION="2.24.0" + ensure_from_example() { local dest="$1" local src="$2" @@ -37,18 +41,89 @@ ensure_from_example() { exit 1 } -# Compose env_file entries fail closed on Windows when .env is missing -# ("The specified file cannot be found" / "Le fichier spécifique est introuvable"). -prepare_compose_env() { - if [ ! -f "$DOCKER_DIR/$COMPOSE_FILE" ]; then - echo -e "${YELLOW}✗ ${COMPOSE_FILE} not found at ${DOCKER_DIR}/${COMPOSE_FILE}${NC}" - echo "Run this from the DeerFlow repository root, e.g. 'make docker-start'." - echo "Do not run 'docker compose -f docker/${COMPOSE_FILE}' from inside docker/ — that resolves to docker/docker/${COMPOSE_FILE}." - exit 1 +require_compose_file() { + if [ -f "$DOCKER_DIR/$COMPOSE_FILE" ]; then + return 0 + fi + echo -e "${YELLOW}✗ ${COMPOSE_FILE} not found at ${DOCKER_DIR}/${COMPOSE_FILE}${NC}" + echo "Run this from the DeerFlow repository root, e.g. 'make docker-start'." + echo "Do not run 'docker compose -f docker/${COMPOSE_FILE}' from inside docker/ — that resolves to docker/docker/${COMPOSE_FILE}." + exit 1 +} + +# Prefer the Compose V2 plugin (`docker compose`); fall back to the legacy +# hyphenated binary (`docker-compose`) when the plugin is missing. Direct +# callers of either binary get no such check: see CONTRIBUTING.md. +_compose_version_short() { + local out + + out="$(docker compose version --short 2>/dev/null || true)" + if [ -n "$out" ]; then + printf '%s\n' "$out" + return 0 + fi + out="$(docker-compose version --short 2>/dev/null || true)" + if [ -n "$out" ]; then + printf '%s\n' "$out" + return 0 + fi + return 1 +} + +# Fail with an actionable message instead of the parser error an older client +# emits for the optional env_file syntax. +require_compose_version() { + local raw major minor min_major min_minor + + min_major="${COMPOSE_MIN_VERSION%%.*}" + min_minor="${COMPOSE_MIN_VERSION#*.}" + min_minor="${min_minor%%.*}" + + raw="$(_compose_version_short || true)" + raw="${raw#v}" + major="${raw%%.*}" + minor="${raw#*.}" + minor="${minor%%.*}" + major="${major//[!0-9]/}" + minor="${minor//[!0-9]/}" + + if [ -z "$major" ] || [ -z "$minor" ]; then + echo -e "${YELLOW}⚠ Could not determine the Docker Compose version; ${COMPOSE_MIN_VERSION} or newer is required.${NC}" + return 0 + fi + if [ "$major" -gt "$min_major" ] || { [ "$major" -eq "$min_major" ] && [ "$minor" -ge "$min_minor" ]; }; then + return 0 fi + + echo -e "${YELLOW}✗ Docker Compose ${raw} is too old — ${COMPOSE_MIN_VERSION} or newer is required.${NC}" + echo "${COMPOSE_FILE} marks its env_file entries optional using the long-form" + echo "'- path: ... / required: false' syntax, which your client cannot parse." + echo "Update Docker Desktop, or install a current Compose v2 plugin:" + echo " https://docs.docker.com/compose/install/" + exit 1 +} + +# Compose interpolates ${DEER_FLOW_ROOT} into host-side paths +# (DEER_FLOW_HOST_BASE_DIR, THREADS_HOST_PATH) that AIO/provisioner sandbox +# modes bind-mount. Unset, those render as /backend/.deer-flow — a plausible +# looking absolute path on the wrong root, so mounts silently miss the checkout. +ensure_deer_flow_root() { if [ -z "$DEER_FLOW_ROOT" ]; then export DEER_FLOW_ROOT="$PROJECT_ROOT" fi +} + +# Read-only with respect to configuration; safe for logs/stop/restart. +compose_preflight() { + require_compose_file + require_compose_version + ensure_deer_flow_root +} + +# Only `start` may create files. Compose env_file entries fail closed on Windows +# when .env is missing ("The specified file cannot be found" / +# "Le fichier spécifique est introuvable"). +ensure_env_files() { ensure_from_example "$PROJECT_ROOT/.env" "$PROJECT_ROOT/.env.example" ".env" ensure_from_example "$PROJECT_ROOT/frontend/.env" "$PROJECT_ROOT/frontend/.env.example" "frontend/.env" } @@ -227,6 +302,9 @@ start() { echo "==========================================" echo "" + # Validate the toolchain before creating any config files below. + compose_preflight + sandbox_mode="$(detect_sandbox_mode)" services="redis frontend gateway nginx" @@ -257,13 +335,11 @@ start() { fi echo "" - # Set DEER_FLOW_ROOT for provisioner if not already set - if [ -z "$DEER_FLOW_ROOT" ]; then - export DEER_FLOW_ROOT="$PROJECT_ROOT" - echo -e "${BLUE}Setting DEER_FLOW_ROOT=$DEER_FLOW_ROOT${NC}" - echo "" - fi - + # Set by compose_preflight above; shown because the provisioner turns it into + # host-side bind-mount paths. + echo -e "${BLUE}Using DEER_FLOW_ROOT=$DEER_FLOW_ROOT${NC}" + echo "" + # Ensure config.yaml exists before starting. if [ ! -f "$PROJECT_ROOT/config.yaml" ]; then if [ -f "$PROJECT_ROOT/config.example.yaml" ]; then @@ -298,7 +374,7 @@ start() { fi fi - prepare_compose_env + ensure_env_files load_proxy_env_from_dotenv echo "Building and starting containers..." @@ -322,7 +398,7 @@ start() { logs() { local service="" - prepare_compose_env + compose_preflight case "$1" in --frontend) @@ -360,7 +436,7 @@ logs() { # Stop Docker development environment stop() { - prepare_compose_env + compose_preflight echo "Stopping Docker development services..." cd "$DOCKER_DIR" && $COMPOSE_CMD down echo "Cleaning up sandbox containers..." @@ -370,7 +446,7 @@ stop() { # Restart Docker development environment restart() { - prepare_compose_env + compose_preflight echo "========================================" echo " Restarting DeerFlow Docker Services" echo "========================================" From abf0c6ba63ad6a95539baed56236183db2cf6301 Mon Sep 17 00:00:00 2001 From: yh M Date: Mon, 24 Aug 2026 17:43:18 +0800 Subject: [PATCH 4/4] fix(docker): reuse the probed Compose binary for wrapper operations The version probe could accept a standalone docker-compose install while COMPOSE_CMD stayed hardcoded to `docker compose`, so preflight passed and start/logs/stop/restart then failed. Keep the selected executable in COMPOSE_BIN (array), refresh COMPOSE_CMD from it in the current shell, and extend the fallback test through an actual stop invocation. Co-authored-by: Cursor --- .../test_docker_sandbox_mode_detection.py | 43 ++++++++++++++----- scripts/docker.sh | 32 ++++++++++---- 2 files changed, 56 insertions(+), 19 deletions(-) diff --git a/backend/tests/test_docker_sandbox_mode_detection.py b/backend/tests/test_docker_sandbox_mode_detection.py index 18bb84952c7..e244992f526 100644 --- a/backend/tests/test_docker_sandbox_mode_detection.py +++ b/backend/tests/test_docker_sandbox_mode_detection.py @@ -250,9 +250,20 @@ def test_require_compose_version_enforces_minimum(reported_version, expected_ret def test_require_compose_version_falls_back_to_hyphenated_binary(): - """When the Compose plugin is missing, probe docker-compose (hyphenated).""" - command = f""" + """Plugin missing + docker-compose 2.24: version check passes and stop uses that binary. + + Regression for the half-fallback where require_compose_version accepted + docker-compose but COMPOSE_CMD stayed hardcoded to `docker compose`. + """ + with tempfile.TemporaryDirectory() as tmpdir: + tmp_root = Path(tmpdir) + _seed_compose_file(tmp_root) + marker = tmp_root / "hyphenated_invoke.txt" + + command = f""" source '{SCRIPT_PATH}' +PROJECT_ROOT='{tmp_root}' +DOCKER_DIR='{tmp_root}' docker() {{ if [ "$1" = compose ]; then echo "docker: unknown command" >&2 @@ -260,17 +271,27 @@ def test_require_compose_version_falls_back_to_hyphenated_binary(): fi command docker "$@" }} -docker-compose() {{ echo '2.24.0'; }} -require_compose_version +docker-compose() {{ + if [ "$1" = version ]; then + echo '2.24.0' + return 0 + fi + # Real wrapper ops (down/logs/...) must hit this binary, not `docker compose`. + printf '%s\n' "$*" > '{marker}' +}} +unset DEER_FLOW_ROOT +stop """ - result = subprocess.run( - [BASH_EXECUTABLE, "-lc", command], - capture_output=True, - text=True, - encoding="utf-8", - ) + result = subprocess.run( + [BASH_EXECUTABLE, "-lc", command], + capture_output=True, + text=True, + encoding="utf-8", + ) - assert result.returncode == 0, result.stdout + result.stderr + assert result.returncode == 0, result.stdout + result.stderr + assert marker.is_file(), "stop never invoked docker-compose for the compose operation" + assert "down" in marker.read_text(encoding="utf-8") def test_require_compose_version_rejects_old_hyphenated_binary(): diff --git a/scripts/docker.sh b/scripts/docker.sh index 229f03b361b..8b84a370735 100755 --- a/scripts/docker.sh +++ b/scripts/docker.sh @@ -16,7 +16,14 @@ DOCKER_DIR="$PROJECT_ROOT/docker" # Use a filename relative to DOCKER_DIR (we always `cd` there) so Windows # Docker Desktop does not receive a Git Bash `/c/...` path it cannot open. COMPOSE_FILE="docker-compose-dev.yaml" -COMPOSE_CMD="docker compose -p deer-flow-dev -f $COMPOSE_FILE" +# Selected by require_compose_version: prefer the V2 plugin, else hyphenated binary. +# Kept as an array so "docker compose" stays two words under set -u / quoting. +COMPOSE_BIN=(docker compose) + +_refresh_compose_cmd() { + COMPOSE_CMD="${COMPOSE_BIN[*]} -p deer-flow-dev -f ${COMPOSE_FILE}" +} +_refresh_compose_cmd # docker-compose-dev.yaml marks its env_file entries optional with the long-form # `- path: ... / required: false` syntax, understood by Compose v2.24.0 and up. @@ -52,21 +59,29 @@ require_compose_file() { } # Prefer the Compose V2 plugin (`docker compose`); fall back to the legacy -# hyphenated binary (`docker-compose`) when the plugin is missing. Direct -# callers of either binary get no such check: see CONTRIBUTING.md. -_compose_version_short() { +# hyphenated binary (`docker-compose`) when the plugin is missing. Whatever +# binary answers is retained in COMPOSE_BIN / COMPOSE_CMD so start/logs/stop/ +# restart use the same executable. Must run in the current shell (not $(...)) +# so the COMPOSE_BIN assignment survives. Direct callers get no such check: +# see CONTRIBUTING.md. +_probe_compose() { local out out="$(docker compose version --short 2>/dev/null || true)" if [ -n "$out" ]; then - printf '%s\n' "$out" + COMPOSE_BIN=(docker compose) + _refresh_compose_cmd + COMPOSE_VERSION_RAW="$out" return 0 fi out="$(docker-compose version --short 2>/dev/null || true)" if [ -n "$out" ]; then - printf '%s\n' "$out" + COMPOSE_BIN=(docker-compose) + _refresh_compose_cmd + COMPOSE_VERSION_RAW="$out" return 0 fi + COMPOSE_VERSION_RAW="" return 1 } @@ -79,8 +94,9 @@ require_compose_version() { min_minor="${COMPOSE_MIN_VERSION#*.}" min_minor="${min_minor%%.*}" - raw="$(_compose_version_short || true)" - raw="${raw#v}" + COMPOSE_VERSION_RAW="" + _probe_compose || true + raw="${COMPOSE_VERSION_RAW#v}" major="${raw%%.*}" minor="${raw#*.}" minor="${minor%%.*}"